// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2023 Advanced Micro Devices, Inc */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/pci.h>

#include <linux/pds/pds_common.h>

#include "core.h"

MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
MODULE_AUTHOR("Advanced Micro Devices, Inc");
MODULE_LICENSE("GPL");

/* Supported devices */
static const struct pci_device_id pdsc_id_table[] = {
        { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
        { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
        { 0, }  /* end of table */
};
MODULE_DEVICE_TABLE(pci, pdsc_id_table);

static void pdsc_wdtimer_cb(struct timer_list *t)
{
        struct pdsc *pdsc = timer_container_of(pdsc, t, wdtimer);

        dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
        mod_timer(&pdsc->wdtimer,
                  round_jiffies(jiffies + pdsc->wdtimer_period));

        queue_work(pdsc->wq, &pdsc->health_work);
}

static void pdsc_unmap_bars(struct pdsc *pdsc)
{
        struct pdsc_dev_bar *bars = pdsc->bars;
        unsigned int i;

        pdsc->info_regs = NULL;
        pdsc->cmd_regs = NULL;
        pdsc->intr_status = NULL;
        pdsc->intr_ctrl = NULL;

        for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
                if (bars[i].vaddr)
                        pci_iounmap(pdsc->pdev, bars[i].vaddr);
                bars[i].vaddr = NULL;
        }
}

static int pdsc_map_bars(struct pdsc *pdsc)
{
        struct pdsc_dev_bar *bar = pdsc->bars;
        struct pci_dev *pdev = pdsc->pdev;
        struct device *dev = pdsc->dev;
        struct pdsc_dev_bar *bars;
        unsigned int i, j;
        int num_bars = 0;
        int err;
        u32 sig;

        bars = pdsc->bars;

        /* Since the PCI interface in the hardware is configurable,
         * we need to poke into all the bars to find the set we're
         * expecting.
         */
        for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
                if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
                        continue;

                bars[j].len = pci_resource_len(pdev, i);
                bars[j].bus_addr = pci_resource_start(pdev, i);
                bars[j].res_index = i;

                /* only map the whole bar 0 */
                if (j > 0) {
                        bars[j].vaddr = NULL;
                } else {
                        bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
                        if (!bars[j].vaddr) {
                                dev_err(dev, "Cannot map BAR %d, aborting\n", i);
                                return -ENODEV;
                        }
                }

                j++;
        }
        num_bars = j;

        /* BAR0: dev_cmd and interrupts */
        if (num_bars < 1) {
                dev_err(dev, "No bars found\n");
                err = -EFAULT;
                goto err_out;
        }

        if (bar->len < PDS_CORE_BAR0_SIZE) {
                dev_err(dev, "Resource bar size %lu too small\n", bar->len);
                err = -EFAULT;
                goto err_out;
        }

        pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
        pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
        pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
        pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;

        sig = ioread32(&pdsc->info_regs->signature);
        if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
                dev_err(dev, "Incompatible firmware signature %x", sig);
                err = -EFAULT;
                goto err_out;
        }

        /* BAR1: doorbells */
        bar++;
        if (num_bars < 2) {
                dev_err(dev, "Doorbell bar missing\n");
                err = -EFAULT;
                goto err_out;
        }

        pdsc->db_pages = bar->vaddr;
        pdsc->phy_db_pages = bar->bus_addr;

        return 0;

err_out:
        pdsc_unmap_bars(pdsc);
        return err;
}

void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
{
        return pci_iomap_range(pdsc->pdev,
                               pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
                               (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
}

static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
{
        struct pdsc *pdsc = pci_get_drvdata(pdev);
        struct device *dev = pdsc->dev;
        int ret = 0;

        if (num_vfs > 0) {
                pdsc->vfs = kzalloc_objs(struct pdsc_vf, num_vfs);
                if (!pdsc->vfs)
                        return -ENOMEM;
                pdsc->num_vfs = num_vfs;

                ret = pci_enable_sriov(pdev, num_vfs);
                if (ret) {
                        dev_err(dev, "Cannot enable SRIOV: %pe\n",
                                ERR_PTR(ret));
                        goto no_vfs;
                }

                return num_vfs;
        }

no_vfs:
        pci_disable_sriov(pdev);

        kfree(pdsc->vfs);
        pdsc->vfs = NULL;
        pdsc->num_vfs = 0;

        return ret;
}

static int pdsc_init_vf(struct pdsc *vf)
{
        struct devlink *dl;
        struct pdsc *pf;
        int err;

        pf = pdsc_get_pf_struct(vf->pdev);
        if (IS_ERR_OR_NULL(pf))
                return PTR_ERR(pf) ?: -1;

        vf->vf_id = pci_iov_vf_id(vf->pdev);

        dl = priv_to_devlink(vf);
        devl_lock(dl);
        devl_register(dl);
        devl_unlock(dl);

        pf->vfs[vf->vf_id].vf = vf;
        err = pdsc_auxbus_dev_add(vf, pf, PDS_DEV_TYPE_VDPA,
                                  &pf->vfs[vf->vf_id].padev);
        if (err) {
                devl_lock(dl);
                devl_unregister(dl);
                devl_unlock(dl);
        }

        return err;
}

static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
        .name = "fw",
        .diagnose = pdsc_fw_reporter_diagnose,
};

static const struct devlink_param pdsc_dl_params[] = {
        DEVLINK_PARAM_GENERIC(ENABLE_VNET,
                              BIT(DEVLINK_PARAM_CMODE_RUNTIME),
                              pdsc_dl_enable_get,
                              pdsc_dl_enable_set,
                              pdsc_dl_enable_validate),
};

#define PDSC_WQ_NAME_LEN 24

static int pdsc_init_pf(struct pdsc *pdsc)
{
        struct devlink_health_reporter *hr;
        char wq_name[PDSC_WQ_NAME_LEN];
        struct devlink *dl;
        int err;

        pcie_print_link_status(pdsc->pdev);

        err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
        if (err) {
                dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
                        ERR_PTR(err));
                return err;
        }

        err = pdsc_map_bars(pdsc);
        if (err)
                goto err_out_release_regions;

        /* General workqueue and timer, but don't start timer yet */
        snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
        pdsc->wq = create_singlethread_workqueue(wq_name);
        if (!pdsc->wq) {
                err = -ENOMEM;
                goto err_out_unmap_bars;
        }
        INIT_WORK(&pdsc->health_work, pdsc_health_thread);
        INIT_WORK(&pdsc->pci_reset_work, pdsc_pci_reset_thread);
        timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
        pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;

        mutex_init(&pdsc->devcmd_lock);
        mutex_init(&pdsc->config_lock);
        spin_lock_init(&pdsc->adminq_lock);

        mutex_lock(&pdsc->config_lock);
        set_bit(PDSC_S_FW_DEAD, &pdsc->state);

        err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
        if (err) {
                mutex_unlock(&pdsc->config_lock);
                goto err_out_shutdown_timer;
        }

        err = pdsc_start(pdsc);
        if (err) {
                mutex_unlock(&pdsc->config_lock);
                goto err_out_teardown;
        }

        mutex_unlock(&pdsc->config_lock);

        err = pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL, &pdsc->padev);
        if (err)
                goto err_out_stop;

        dl = priv_to_devlink(pdsc);
        devl_lock(dl);
        err = devl_params_register(dl, pdsc_dl_params,
                                   ARRAY_SIZE(pdsc_dl_params));
        if (err) {
                devl_unlock(dl);
                dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
                         ERR_PTR(err));
                goto err_out_del_dev;
        }

        hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, pdsc);
        if (IS_ERR(hr)) {
                devl_unlock(dl);
                dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
                err = PTR_ERR(hr);
                goto err_out_unreg_params;
        }
        pdsc->fw_reporter = hr;

        devl_register(dl);
        devl_unlock(dl);

        /* Lastly, start the health check timer */
        mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));

        return 0;

err_out_unreg_params:
        devlink_params_unregister(dl, pdsc_dl_params,
                                  ARRAY_SIZE(pdsc_dl_params));
err_out_del_dev:
        pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
err_out_stop:
        pdsc_stop(pdsc);
err_out_teardown:
        pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
err_out_shutdown_timer:
        timer_shutdown_sync(&pdsc->wdtimer);
        if (pdsc->wq)
                destroy_workqueue(pdsc->wq);
        mutex_destroy(&pdsc->config_lock);
        mutex_destroy(&pdsc->devcmd_lock);
        pci_free_irq_vectors(pdsc->pdev);
err_out_unmap_bars:
        pdsc_unmap_bars(pdsc);
err_out_release_regions:
        pci_release_regions(pdsc->pdev);

        return err;
}

static const struct devlink_ops pdsc_dl_ops = {
        .info_get       = pdsc_dl_info_get,
        .flash_update   = pdsc_dl_flash_update,
};

static const struct devlink_ops pdsc_dl_vf_ops = {
};

static DEFINE_IDA(pdsc_ida);

static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
{
        struct device *dev = &pdev->dev;
        const struct devlink_ops *ops;
        struct devlink *dl;
        struct pdsc *pdsc;
        bool is_pf;
        int err;

        is_pf = !pdev->is_virtfn;
        ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
        dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
        if (!dl)
                return -ENOMEM;
        pdsc = devlink_priv(dl);

        pdsc->pdev = pdev;
        pdsc->dev = &pdev->dev;
        set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
        pci_set_drvdata(pdev, pdsc);
        pdsc_debugfs_add_dev(pdsc);

        err = ida_alloc(&pdsc_ida, GFP_KERNEL);
        if (err < 0) {
                dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
                        __func__, ERR_PTR(err));
                goto err_out_free_devlink;
        }
        pdsc->uid = err;

        /* Query system for DMA addressing limitation for the device. */
        err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
        if (err) {
                dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
                        ERR_PTR(err));
                goto err_out_free_ida;
        }

        err = pci_enable_device(pdev);
        if (err) {
                dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
                goto err_out_free_ida;
        }
        pci_set_master(pdev);

        if (is_pf)
                err = pdsc_init_pf(pdsc);
        else
                err = pdsc_init_vf(pdsc);
        if (err) {
                dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
                goto err_out_disable_device;
        }

        clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
        return 0;

err_out_disable_device:
        pci_disable_device(pdev);
err_out_free_ida:
        ida_free(&pdsc_ida, pdsc->uid);
err_out_free_devlink:
        pdsc_debugfs_del_dev(pdsc);
        devlink_free(dl);

        return err;
}

static void pdsc_remove(struct pci_dev *pdev)
{
        struct pdsc *pdsc = pci_get_drvdata(pdev);
        struct devlink *dl;

        /* Unhook the registrations first to be sure there
         * are no requests while we're stopping.
         */
        dl = priv_to_devlink(pdsc);
        devl_lock(dl);
        devl_unregister(dl);
        if (!pdev->is_virtfn) {
                if (pdsc->fw_reporter) {
                        devl_health_reporter_destroy(pdsc->fw_reporter);
                        pdsc->fw_reporter = NULL;
                }
                devl_params_unregister(dl, pdsc_dl_params,
                                       ARRAY_SIZE(pdsc_dl_params));
        }
        devl_unlock(dl);

        if (pdev->is_virtfn) {
                struct pdsc *pf;

                pf = pdsc_get_pf_struct(pdsc->pdev);
                if (!IS_ERR(pf)) {
                        pdsc_auxbus_dev_del(pdsc, pf, &pf->vfs[pdsc->vf_id].padev);
                        pf->vfs[pdsc->vf_id].vf = NULL;
                }
        } else {
                /* Remove the VFs and their aux_bus connections before other
                 * cleanup so that the clients can use the AdminQ to cleanly
                 * shut themselves down.
                 */
                pdsc_sriov_configure(pdev, 0);
                pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);

                timer_shutdown_sync(&pdsc->wdtimer);

                mutex_lock(&pdsc->config_lock);
                set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);

                pdsc_stop(pdsc);
                pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
                mutex_unlock(&pdsc->config_lock);

                if (pdsc->wq)
                        destroy_workqueue(pdsc->wq);
                mutex_destroy(&pdsc->config_lock);
                mutex_destroy(&pdsc->devcmd_lock);

                pdsc_unmap_bars(pdsc);
                pci_release_regions(pdev);
        }

        pci_disable_device(pdev);

        ida_free(&pdsc_ida, pdsc->uid);
        pdsc_debugfs_del_dev(pdsc);
        devlink_free(dl);
}

static void pdsc_stop_health_thread(struct pdsc *pdsc)
{
        if (pdsc->pdev->is_virtfn)
                return;

        timer_shutdown_sync(&pdsc->wdtimer);
        if (pdsc->health_work.func)
                cancel_work_sync(&pdsc->health_work);
}

static void pdsc_restart_health_thread(struct pdsc *pdsc)
{
        if (pdsc->pdev->is_virtfn)
                return;

        timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
        mod_timer(&pdsc->wdtimer, jiffies + 1);
}

static void pdsc_reset_prepare(struct pci_dev *pdev)
{
        struct pdsc *pdsc = pci_get_drvdata(pdev);

        pdsc_stop_health_thread(pdsc);
        pdsc_fw_down(pdsc);

        if (pdev->is_virtfn) {
                struct pdsc *pf;

                pf = pdsc_get_pf_struct(pdsc->pdev);
                if (!IS_ERR(pf))
                        pdsc_auxbus_dev_del(pdsc, pf,
                                            &pf->vfs[pdsc->vf_id].padev);
        } else {
                pdsc_auxbus_dev_del(pdsc, pdsc, &pdsc->padev);
        }

        pdsc_unmap_bars(pdsc);
        pci_release_regions(pdev);
        if (pci_is_enabled(pdev))
                pci_disable_device(pdev);
}

static void pdsc_reset_done(struct pci_dev *pdev)
{
        struct pdsc *pdsc = pci_get_drvdata(pdev);
        struct device *dev = pdsc->dev;
        int err;

        err = pci_enable_device(pdev);
        if (err) {
                dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
                return;
        }
        pci_set_master(pdev);

        if (!pdev->is_virtfn) {
                pcie_print_link_status(pdsc->pdev);

                err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
                if (err) {
                        dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
                                ERR_PTR(err));
                        return;
                }

                err = pdsc_map_bars(pdsc);
                if (err)
                        return;
        }

        pdsc_fw_up(pdsc);
        pdsc_restart_health_thread(pdsc);

        if (pdev->is_virtfn) {
                struct pdsc *pf;

                pf = pdsc_get_pf_struct(pdsc->pdev);
                if (!IS_ERR(pf))
                        pdsc_auxbus_dev_add(pdsc, pf, PDS_DEV_TYPE_VDPA,
                                            &pf->vfs[pdsc->vf_id].padev);
        } else {
                pdsc_auxbus_dev_add(pdsc, pdsc, PDS_DEV_TYPE_FWCTL,
                                    &pdsc->padev);
        }
}

static pci_ers_result_t pdsc_pci_error_detected(struct pci_dev *pdev,
                                                pci_channel_state_t error)
{
        if (error == pci_channel_io_frozen) {
                pdsc_reset_prepare(pdev);
                return PCI_ERS_RESULT_NEED_RESET;
        }

        return PCI_ERS_RESULT_NONE;
}

static void pdsc_pci_error_resume(struct pci_dev *pdev)
{
        struct pdsc *pdsc = pci_get_drvdata(pdev);

        if (test_bit(PDSC_S_FW_DEAD, &pdsc->state))
                pci_reset_function_locked(pdev);
}

static const struct pci_error_handlers pdsc_err_handler = {
        /* FLR handling */
        .reset_prepare      = pdsc_reset_prepare,
        .reset_done         = pdsc_reset_done,

        /* AER handling */
        .error_detected     = pdsc_pci_error_detected,
        .resume             = pdsc_pci_error_resume,
};

static struct pci_driver pdsc_driver = {
        .name = PDS_CORE_DRV_NAME,
        .id_table = pdsc_id_table,
        .probe = pdsc_probe,
        .remove = pdsc_remove,
        .sriov_configure = pdsc_sriov_configure,
        .err_handler = &pdsc_err_handler,
};

void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
{
        return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
}
EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);

static int __init pdsc_init_module(void)
{
        if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
                return -EINVAL;

        pdsc_debugfs_create();
        return pci_register_driver(&pdsc_driver);
}

static void __exit pdsc_cleanup_module(void)
{
        pci_unregister_driver(&pdsc_driver);
        pdsc_debugfs_destroy();
}

module_init(pdsc_init_module);
module_exit(pdsc_cleanup_module);