// SPDX-License-Identifier: GPL-2.0-only
/*
 * Freescale Memory Controller kernel module
 *
 * Support Power-based SoCs including MPC85xx, MPC86xx, MPC83xx and
 * ARM-based Layerscape SoCs including LS2xxx and LS1021A. Originally
 * split out from mpc85xx_edac EDAC driver.
 *
 * Parts Copyrighted (c) 2013 by Freescale Semiconductor, Inc.
 *
 * Author: Dave Jiang <djiang@mvista.com>
 *
 * 2006-2007 (c) MontaVista Software, Inc.
 */
#include <linux/module.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/ctype.h>
#include <linux/io.h>
#include <linux/mod_devicetable.h>
#include <linux/edac.h>
#include <linux/smp.h>
#include <linux/gfp.h>

#include <linux/of.h>
#include <linux/of_address.h>
#include "edac_module.h"
#include "fsl_ddr_edac.h"

#define EDAC_MOD_STR    "fsl_ddr_edac"

static int edac_mc_idx;

static inline void __iomem *ddr_reg_addr(struct fsl_mc_pdata *pdata, unsigned int off)
{
        if (pdata->flag == TYPE_IMX9 && off >= FSL_MC_DATA_ERR_INJECT_HI && off <= FSL_MC_ERR_SBE)
                return pdata->inject_vbase + off - FSL_MC_DATA_ERR_INJECT_HI
                       + IMX9_MC_DATA_ERR_INJECT_OFF;

        if (pdata->flag == TYPE_IMX9 && off >= IMX9_MC_ERR_EN)
                return pdata->inject_vbase + off - IMX9_MC_ERR_EN;

        return pdata->mc_vbase + off;
}

static inline u32 ddr_in32(struct fsl_mc_pdata *pdata, unsigned int off)
{
        void __iomem *addr = ddr_reg_addr(pdata, off);

        return pdata->little_endian ? ioread32(addr) : ioread32be(addr);
}

static inline void ddr_out32(struct fsl_mc_pdata *pdata, unsigned int off, u32 value)
{
        void __iomem *addr = ddr_reg_addr(pdata, off);

        if (pdata->little_endian)
                iowrite32(value, addr);
        else
                iowrite32be(value, addr);
}

#ifdef CONFIG_EDAC_DEBUG
/************************ MC SYSFS parts ***********************************/

#define to_mci(k) container_of(k, struct mem_ctl_info, dev)

static ssize_t fsl_mc_inject_data_hi_show(struct device *dev,
                                          struct device_attribute *mattr,
                                          char *data)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        return sprintf(data, "0x%08x",
                       ddr_in32(pdata, FSL_MC_DATA_ERR_INJECT_HI));
}

static ssize_t fsl_mc_inject_data_lo_show(struct device *dev,
                                          struct device_attribute *mattr,
                                              char *data)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        return sprintf(data, "0x%08x",
                       ddr_in32(pdata, FSL_MC_DATA_ERR_INJECT_LO));
}

static ssize_t fsl_mc_inject_ctrl_show(struct device *dev,
                                       struct device_attribute *mattr,
                                           char *data)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        return sprintf(data, "0x%08x",
                       ddr_in32(pdata, FSL_MC_ECC_ERR_INJECT));
}

static ssize_t fsl_mc_inject_data_hi_store(struct device *dev,
                                           struct device_attribute *mattr,
                                               const char *data, size_t count)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        unsigned long val;
        int rc;

        if (isdigit(*data)) {
                rc = kstrtoul(data, 0, &val);
                if (rc)
                        return rc;

                ddr_out32(pdata, FSL_MC_DATA_ERR_INJECT_HI, val);
                return count;
        }
        return 0;
}

static ssize_t fsl_mc_inject_data_lo_store(struct device *dev,
                                           struct device_attribute *mattr,
                                               const char *data, size_t count)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        unsigned long val;
        int rc;

        if (isdigit(*data)) {
                rc = kstrtoul(data, 0, &val);
                if (rc)
                        return rc;

                ddr_out32(pdata, FSL_MC_DATA_ERR_INJECT_LO, val);
                return count;
        }
        return 0;
}

static ssize_t fsl_mc_inject_ctrl_store(struct device *dev,
                                        struct device_attribute *mattr,
                                               const char *data, size_t count)
{
        struct mem_ctl_info *mci = to_mci(dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        unsigned long val;
        int rc;

        if (isdigit(*data)) {
                rc = kstrtoul(data, 0, &val);
                if (rc)
                        return rc;

                ddr_out32(pdata, FSL_MC_ECC_ERR_INJECT, val);
                return count;
        }
        return 0;
}

static DEVICE_ATTR(inject_data_hi, S_IRUGO | S_IWUSR,
                   fsl_mc_inject_data_hi_show, fsl_mc_inject_data_hi_store);
static DEVICE_ATTR(inject_data_lo, S_IRUGO | S_IWUSR,
                   fsl_mc_inject_data_lo_show, fsl_mc_inject_data_lo_store);
static DEVICE_ATTR(inject_ctrl, S_IRUGO | S_IWUSR,
                   fsl_mc_inject_ctrl_show, fsl_mc_inject_ctrl_store);
#endif /* CONFIG_EDAC_DEBUG */

static struct attribute *fsl_ddr_dev_attrs[] = {
#ifdef CONFIG_EDAC_DEBUG
        &dev_attr_inject_data_hi.attr,
        &dev_attr_inject_data_lo.attr,
        &dev_attr_inject_ctrl.attr,
#endif
        NULL
};

ATTRIBUTE_GROUPS(fsl_ddr_dev);

/**************************** MC Err device ***************************/

/*
 * Taken from table 8-55 in the MPC8641 User's Manual and/or 9-61 in the
 * MPC8572 User's Manual.  Each line represents a syndrome bit column as a
 * 64-bit value, but split into an upper and lower 32-bit chunk.  The labels
 * below correspond to Freescale's manuals.
 */
static unsigned int ecc_table[16] = {
        /* MSB           LSB */
        /* [0:31]    [32:63] */
        0xf00fe11e, 0xc33c0ff7, /* Syndrome bit 7 */
        0x00ff00ff, 0x00fff0ff,
        0x0f0f0f0f, 0x0f0fff00,
        0x11113333, 0x7777000f,
        0x22224444, 0x8888222f,
        0x44448888, 0xffff4441,
        0x8888ffff, 0x11118882,
        0xffff1111, 0x22221114, /* Syndrome bit 0 */
};

/*
 * Calculate the correct ECC value for a 64-bit value specified by high:low
 */
static u8 calculate_ecc(u32 high, u32 low)
{
        u32 mask_low;
        u32 mask_high;
        int bit_cnt;
        u8 ecc = 0;
        int i;
        int j;

        for (i = 0; i < 8; i++) {
                mask_high = ecc_table[i * 2];
                mask_low = ecc_table[i * 2 + 1];
                bit_cnt = 0;

                for (j = 0; j < 32; j++) {
                        if ((mask_high >> j) & 1)
                                bit_cnt ^= (high >> j) & 1;
                        if ((mask_low >> j) & 1)
                                bit_cnt ^= (low >> j) & 1;
                }

                ecc |= bit_cnt << i;
        }

        return ecc;
}

/*
 * Create the syndrome code which is generated if the data line specified by
 * 'bit' failed.  Eg generate an 8-bit codes seen in Table 8-55 in the MPC8641
 * User's Manual and 9-61 in the MPC8572 User's Manual.
 */
static u8 syndrome_from_bit(unsigned int bit) {
        int i;
        u8 syndrome = 0;

        /*
         * Cycle through the upper or lower 32-bit portion of each value in
         * ecc_table depending on if 'bit' is in the upper or lower half of
         * 64-bit data.
         */
        for (i = bit < 32; i < 16; i += 2)
                syndrome |= ((ecc_table[i] >> (bit % 32)) & 1) << (i / 2);

        return syndrome;
}

/*
 * Decode data and ecc syndrome to determine what went wrong
 * Note: This can only decode single-bit errors
 */
static void sbe_ecc_decode(u32 cap_high, u32 cap_low, u32 cap_ecc,
                       int *bad_data_bit, int *bad_ecc_bit)
{
        int i;
        u8 syndrome;

        *bad_data_bit = -1;
        *bad_ecc_bit = -1;

        /*
         * Calculate the ECC of the captured data and XOR it with the captured
         * ECC to find an ECC syndrome value we can search for
         */
        syndrome = calculate_ecc(cap_high, cap_low) ^ cap_ecc;

        /* Check if a data line is stuck... */
        for (i = 0; i < 64; i++) {
                if (syndrome == syndrome_from_bit(i)) {
                        *bad_data_bit = i;
                        return;
                }
        }

        /* If data is correct, check ECC bits for errors... */
        for (i = 0; i < 8; i++) {
                if ((syndrome >> i) & 0x1) {
                        *bad_ecc_bit = i;
                        return;
                }
        }
}

#define make64(high, low) (((u64)(high) << 32) | (low))

static void fsl_mc_check(struct mem_ctl_info *mci)
{
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        struct csrow_info *csrow;
        u32 bus_width;
        u32 err_detect;
        u32 syndrome;
        u64 err_addr;
        u32 pfn;
        int row_index;
        u32 cap_high;
        u32 cap_low;
        int bad_data_bit;
        int bad_ecc_bit;

        err_detect = ddr_in32(pdata, FSL_MC_ERR_DETECT);
        if (!err_detect)
                return;

        fsl_mc_printk(mci, KERN_ERR, "Err Detect Register: %#8.8x\n",
                      err_detect);

        /* no more processing if not ECC bit errors */
        if (!(err_detect & (DDR_EDE_SBE | DDR_EDE_MBE))) {
                ddr_out32(pdata, FSL_MC_ERR_DETECT, err_detect);
                return;
        }

        syndrome = ddr_in32(pdata, FSL_MC_CAPTURE_ECC);

        /* Mask off appropriate bits of syndrome based on bus width */
        bus_width = (ddr_in32(pdata, FSL_MC_DDR_SDRAM_CFG) &
                     DSC_DBW_MASK) ? 32 : 64;
        if (bus_width == 64)
                syndrome &= 0xff;
        else
                syndrome &= 0xffff;

        err_addr = make64(
                ddr_in32(pdata, FSL_MC_CAPTURE_EXT_ADDRESS),
                ddr_in32(pdata, FSL_MC_CAPTURE_ADDRESS));
        pfn = err_addr >> PAGE_SHIFT;

        for (row_index = 0; row_index < mci->nr_csrows; row_index++) {
                csrow = mci->csrows[row_index];
                if ((pfn >= csrow->first_page) && (pfn <= csrow->last_page))
                        break;
        }

        cap_high = ddr_in32(pdata, FSL_MC_CAPTURE_DATA_HI);
        cap_low = ddr_in32(pdata, FSL_MC_CAPTURE_DATA_LO);

        /*
         * Analyze single-bit errors on 64-bit wide buses
         * TODO: Add support for 32-bit wide buses
         */
        if ((err_detect & DDR_EDE_SBE) && (bus_width == 64)) {
                u64 cap = (u64)cap_high << 32 | cap_low;
                u32 s = syndrome;

                sbe_ecc_decode(cap_high, cap_low, syndrome,
                                &bad_data_bit, &bad_ecc_bit);

                if (bad_data_bit >= 0) {
                        fsl_mc_printk(mci, KERN_ERR, "Faulty Data bit: %d\n", bad_data_bit);
                        cap ^= 1ULL << bad_data_bit;
                }

                if (bad_ecc_bit >= 0) {
                        fsl_mc_printk(mci, KERN_ERR, "Faulty ECC bit: %d\n", bad_ecc_bit);
                        s ^= 1 << bad_ecc_bit;
                }

                fsl_mc_printk(mci, KERN_ERR,
                        "Expected Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
                        upper_32_bits(cap), lower_32_bits(cap), s);
        }

        fsl_mc_printk(mci, KERN_ERR,
                        "Captured Data / ECC:\t%#8.8x_%08x / %#2.2x\n",
                        cap_high, cap_low, syndrome);
        fsl_mc_printk(mci, KERN_ERR, "Err addr: %#8.8llx\n", err_addr);
        fsl_mc_printk(mci, KERN_ERR, "PFN: %#8.8x\n", pfn);

        /* we are out of range */
        if (row_index == mci->nr_csrows)
                fsl_mc_printk(mci, KERN_ERR, "PFN out of range!\n");

        if (err_detect & DDR_EDE_SBE)
                edac_mc_handle_error(HW_EVENT_ERR_CORRECTED, mci, 1,
                                     pfn, err_addr & ~PAGE_MASK, syndrome,
                                     row_index, 0, -1,
                                     mci->ctl_name, "");

        if (err_detect & DDR_EDE_MBE)
                edac_mc_handle_error(HW_EVENT_ERR_UNCORRECTED, mci, 1,
                                     pfn, err_addr & ~PAGE_MASK, syndrome,
                                     row_index, 0, -1,
                                     mci->ctl_name, "");

        ddr_out32(pdata, FSL_MC_ERR_DETECT, err_detect);
}

static irqreturn_t fsl_mc_isr(int irq, void *dev_id)
{
        struct mem_ctl_info *mci = dev_id;
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        u32 err_detect;

        err_detect = ddr_in32(pdata, FSL_MC_ERR_DETECT);
        if (!err_detect)
                return IRQ_NONE;

        fsl_mc_check(mci);

        return IRQ_HANDLED;
}

static void fsl_ddr_init_csrows(struct mem_ctl_info *mci)
{
        struct fsl_mc_pdata *pdata = mci->pvt_info;
        struct csrow_info *csrow;
        struct dimm_info *dimm;
        u32 sdram_ctl;
        u32 sdtype;
        enum mem_type mtype;
        u32 cs_bnds;
        int index;

        sdram_ctl = ddr_in32(pdata, FSL_MC_DDR_SDRAM_CFG);

        sdtype = sdram_ctl & DSC_SDTYPE_MASK;
        if (sdram_ctl & DSC_RD_EN) {
                switch (sdtype) {
                case 0x02000000:
                        mtype = MEM_RDDR;
                        break;
                case 0x03000000:
                        mtype = MEM_RDDR2;
                        break;
                case 0x07000000:
                        mtype = MEM_RDDR3;
                        break;
                case 0x05000000:
                        mtype = MEM_RDDR4;
                        break;
                default:
                        mtype = MEM_UNKNOWN;
                        break;
                }
        } else {
                switch (sdtype) {
                case 0x02000000:
                        mtype = MEM_DDR;
                        break;
                case 0x03000000:
                        mtype = MEM_DDR2;
                        break;
                case 0x07000000:
                        mtype = MEM_DDR3;
                        break;
                case 0x05000000:
                        mtype = MEM_DDR4;
                        break;
                case 0x04000000:
                        mtype = MEM_LPDDR4;
                        break;
                default:
                        mtype = MEM_UNKNOWN;
                        break;
                }
        }

        for (index = 0; index < mci->nr_csrows; index++) {
                u32 start;
                u32 end;

                csrow = mci->csrows[index];
                dimm = csrow->channels[0]->dimm;

                cs_bnds = ddr_in32(pdata, FSL_MC_CS_BNDS_0 +
                                   (index * FSL_MC_CS_BNDS_OFS));

                start = (cs_bnds & 0xffff0000) >> 16;
                end   = (cs_bnds & 0x0000ffff);

                if (start == end)
                        continue;       /* not populated */

                start <<= (24 - PAGE_SHIFT);
                end   <<= (24 - PAGE_SHIFT);
                end    |= (1 << (24 - PAGE_SHIFT)) - 1;

                csrow->first_page = start;
                csrow->last_page = end;

                dimm->nr_pages = end + 1 - start;
                dimm->grain = 8;
                dimm->mtype = mtype;
                dimm->dtype = DEV_UNKNOWN;
                if (pdata->flag == TYPE_IMX9)
                        dimm->dtype = DEV_X16;
                else if (sdram_ctl & DSC_X32_EN)
                        dimm->dtype = DEV_X32;
                dimm->edac_mode = EDAC_SECDED;
        }
}

int fsl_mc_err_probe(struct platform_device *op)
{
        struct mem_ctl_info *mci;
        struct edac_mc_layer layers[2];
        struct fsl_mc_pdata *pdata;
        struct resource r;
        u32 ecc_en_mask;
        u32 sdram_ctl;
        int res;

        if (!devres_open_group(&op->dev, fsl_mc_err_probe, GFP_KERNEL))
                return -ENOMEM;

        layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
        layers[0].size = 4;
        layers[0].is_virt_csrow = true;
        layers[1].type = EDAC_MC_LAYER_CHANNEL;
        layers[1].size = 1;
        layers[1].is_virt_csrow = false;
        mci = edac_mc_alloc(edac_mc_idx, ARRAY_SIZE(layers), layers,
                            sizeof(*pdata));
        if (!mci) {
                devres_release_group(&op->dev, fsl_mc_err_probe);
                return -ENOMEM;
        }

        pdata = mci->pvt_info;
        pdata->name = "fsl_mc_err";
        mci->pdev = &op->dev;
        pdata->edac_idx = edac_mc_idx++;
        dev_set_drvdata(mci->pdev, mci);
        mci->ctl_name = pdata->name;
        mci->dev_name = pdata->name;

        pdata->flag = (unsigned long)device_get_match_data(&op->dev);

        /*
         * Get the endianness of DDR controller registers.
         * Default is big endian.
         */
        pdata->little_endian = of_property_read_bool(op->dev.of_node, "little-endian");

        res = of_address_to_resource(op->dev.of_node, 0, &r);
        if (res) {
                pr_err("%s: Unable to get resource for MC err regs\n",
                       __func__);
                goto err;
        }

        if (!devm_request_mem_region(&op->dev, r.start, resource_size(&r),
                                     pdata->name)) {
                pr_err("%s: Error while requesting mem region\n",
                       __func__);
                res = -EBUSY;
                goto err;
        }

        pdata->mc_vbase = devm_ioremap(&op->dev, r.start, resource_size(&r));
        if (!pdata->mc_vbase) {
                pr_err("%s: Unable to setup MC err regs\n", __func__);
                res = -ENOMEM;
                goto err;
        }

        if (pdata->flag == TYPE_IMX9) {
                pdata->inject_vbase = devm_platform_ioremap_resource_byname(op, "inject");
                if (IS_ERR(pdata->inject_vbase)) {
                        res = -ENOMEM;
                        goto err;
                }
        }

        if (pdata->flag == TYPE_IMX9) {
                sdram_ctl = ddr_in32(pdata, IMX9_MC_ERR_EN);
                ecc_en_mask = ERR_ECC_EN | ERR_INLINE_ECC;
        } else {
                sdram_ctl = ddr_in32(pdata, FSL_MC_DDR_SDRAM_CFG);
                ecc_en_mask = DSC_ECC_EN;
        }

        if ((sdram_ctl & ecc_en_mask) != ecc_en_mask) {
                /* no ECC */
                pr_warn("%s: No ECC DIMMs discovered\n", __func__);
                res = -ENODEV;
                goto err;
        }

        edac_dbg(3, "init mci\n");
        mci->mtype_cap = MEM_FLAG_DDR | MEM_FLAG_RDDR |
                         MEM_FLAG_DDR2 | MEM_FLAG_RDDR2 |
                         MEM_FLAG_DDR3 | MEM_FLAG_RDDR3 |
                         MEM_FLAG_DDR4 | MEM_FLAG_RDDR4 |
                         MEM_FLAG_LPDDR4;
        mci->edac_ctl_cap = EDAC_FLAG_NONE | EDAC_FLAG_SECDED;
        mci->edac_cap = EDAC_FLAG_SECDED;
        mci->mod_name = EDAC_MOD_STR;

        if (edac_op_state == EDAC_OPSTATE_POLL)
                mci->edac_check = fsl_mc_check;

        mci->ctl_page_to_phys = NULL;

        mci->scrub_mode = SCRUB_SW_SRC;

        fsl_ddr_init_csrows(mci);

        /* store the original error disable bits */
        pdata->orig_ddr_err_disable = ddr_in32(pdata, FSL_MC_ERR_DISABLE);
        ddr_out32(pdata, FSL_MC_ERR_DISABLE, 0);

        /* clear all error bits */
        ddr_out32(pdata, FSL_MC_ERR_DETECT, ~0);

        res = edac_mc_add_mc_with_groups(mci, fsl_ddr_dev_groups);
        if (res) {
                edac_dbg(3, "failed edac_mc_add_mc()\n");
                goto err;
        }

        if (edac_op_state == EDAC_OPSTATE_INT) {
                ddr_out32(pdata, FSL_MC_ERR_INT_EN,
                          DDR_EIE_MBEE | DDR_EIE_SBEE);

                /* store the original error management threshold */
                pdata->orig_ddr_err_sbe = ddr_in32(pdata,
                                                   FSL_MC_ERR_SBE) & 0xff0000;

                /* set threshold to 1 error per interrupt */
                ddr_out32(pdata, FSL_MC_ERR_SBE, 0x10000);

                /* register interrupts */
                pdata->irq = platform_get_irq(op, 0);
                res = devm_request_irq(&op->dev, pdata->irq,
                                       fsl_mc_isr,
                                       IRQF_SHARED,
                                       "[EDAC] MC err", mci);
                if (res < 0) {
                        pr_err("%s: Unable to request irq %d for FSL DDR DRAM ERR\n",
                               __func__, pdata->irq);
                        res = -ENODEV;
                        goto err2;
                }

                pr_info(EDAC_MOD_STR " acquired irq %d for MC\n",
                       pdata->irq);
        }

        devres_remove_group(&op->dev, fsl_mc_err_probe);
        edac_dbg(3, "success\n");
        pr_info(EDAC_MOD_STR " MC err registered\n");

        return 0;

err2:
        edac_mc_del_mc(&op->dev);
err:
        devres_release_group(&op->dev, fsl_mc_err_probe);
        edac_mc_free(mci);
        return res;
}

void fsl_mc_err_remove(struct platform_device *op)
{
        struct mem_ctl_info *mci = dev_get_drvdata(&op->dev);
        struct fsl_mc_pdata *pdata = mci->pvt_info;

        edac_dbg(0, "\n");

        if (edac_op_state == EDAC_OPSTATE_INT) {
                ddr_out32(pdata, FSL_MC_ERR_INT_EN, 0);
        }

        ddr_out32(pdata, FSL_MC_ERR_DISABLE,
                  pdata->orig_ddr_err_disable);
        ddr_out32(pdata, FSL_MC_ERR_SBE, pdata->orig_ddr_err_sbe);


        edac_mc_del_mc(&op->dev);
        edac_mc_free(mci);
}