// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2023 Advanced Micro Devices, Inc */

#include <linux/dynamic_debug.h>

#include "core.h"

static int pdsc_process_notifyq(struct pdsc_qcq *qcq)
{
        union pds_core_notifyq_comp *comp;
        struct pdsc *pdsc = qcq->pdsc;
        struct pdsc_cq *cq = &qcq->cq;
        struct pdsc_cq_info *cq_info;
        int nq_work = 0;
        u64 eid;

        cq_info = &cq->info[cq->tail_idx];
        comp = cq_info->comp;
        eid = le64_to_cpu(comp->event.eid);
        while (eid > pdsc->last_eid) {
                u16 ecode = le16_to_cpu(comp->event.ecode);

                switch (ecode) {
                case PDS_EVENT_LINK_CHANGE:
                        dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n",
                                 ecode, eid);
                        pdsc_notify(PDS_EVENT_LINK_CHANGE, comp);
                        break;

                case PDS_EVENT_RESET:
                        dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n",
                                 ecode, eid);
                        pdsc_notify(PDS_EVENT_RESET, comp);
                        break;

                case PDS_EVENT_XCVR:
                        dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n",
                                 ecode, eid);
                        break;

                default:
                        dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n",
                                 ecode, eid);
                        break;
                }

                pdsc->last_eid = eid;
                cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
                cq_info = &cq->info[cq->tail_idx];
                comp = cq_info->comp;
                eid = le64_to_cpu(comp->event.eid);

                nq_work++;
        }

        qcq->accum_work += nq_work;

        return nq_work;
}

static bool pdsc_adminq_inc_if_up(struct pdsc *pdsc)
{
        if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER) ||
            pdsc->state & BIT_ULL(PDSC_S_FW_DEAD))
                return false;

        return refcount_inc_not_zero(&pdsc->adminq_refcnt);
}

void pdsc_process_adminq(struct pdsc_qcq *qcq)
{
        union pds_core_adminq_comp *comp;
        struct pdsc_queue *q = &qcq->q;
        struct pdsc *pdsc = qcq->pdsc;
        struct pdsc_cq *cq = &qcq->cq;
        struct pdsc_q_info *q_info;
        unsigned long irqflags;
        int nq_work = 0;
        int aq_work = 0;

        /* Don't process AdminQ when it's not up */
        if (!pdsc_adminq_inc_if_up(pdsc)) {
                dev_err(pdsc->dev, "%s: called while adminq is unavailable\n",
                        __func__);
                return;
        }

        /* Check for NotifyQ event */
        nq_work = pdsc_process_notifyq(&pdsc->notifyqcq);

        /* Check for empty queue, which can happen if the interrupt was
         * for a NotifyQ event and there are no new AdminQ completions.
         */
        if (q->tail_idx == q->head_idx)
                goto credits;

        /* Find the first completion to clean,
         * run the callback in the related q_info,
         * and continue while we still match done color
         */
        spin_lock_irqsave(&pdsc->adminq_lock, irqflags);
        comp = cq->info[cq->tail_idx].comp;
        while (pdsc_color_match(comp->color, cq->done_color)) {
                q_info = &q->info[q->tail_idx];
                q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);

                if (!completion_done(&q_info->completion)) {
                        memcpy(q_info->dest, comp, sizeof(*comp));
                        complete(&q_info->completion);
                }

                if (cq->tail_idx == cq->num_descs - 1)
                        cq->done_color = !cq->done_color;
                cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
                comp = cq->info[cq->tail_idx].comp;

                aq_work++;
        }
        spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);

        qcq->accum_work += aq_work;

credits:
        /* Return the interrupt credits, one for each completion */
        pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx],
                              nq_work + aq_work,
                              PDS_CORE_INTR_CRED_REARM);
        refcount_dec(&pdsc->adminq_refcnt);
}

void pdsc_work_thread(struct work_struct *work)
{
        struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work);

        pdsc_process_adminq(qcq);
}

irqreturn_t pdsc_adminq_isr(int irq, void *data)
{
        struct pdsc *pdsc = data;
        struct pdsc_qcq *qcq;

        /* Don't process AdminQ when it's not up */
        if (!pdsc_adminq_inc_if_up(pdsc)) {
                dev_err(pdsc->dev, "%s: called while adminq is unavailable\n",
                        __func__);
                return IRQ_HANDLED;
        }

        qcq = &pdsc->adminqcq;
        queue_work(pdsc->wq, &qcq->work);
        refcount_dec(&pdsc->adminq_refcnt);

        return IRQ_HANDLED;
}

static int __pdsc_adminq_post(struct pdsc *pdsc,
                              struct pdsc_qcq *qcq,
                              union pds_core_adminq_cmd *cmd,
                              union pds_core_adminq_comp *comp)
{
        struct pdsc_queue *q = &qcq->q;
        struct pdsc_q_info *q_info;
        unsigned long irqflags;
        unsigned int avail;
        int index;
        int ret;

        spin_lock_irqsave(&pdsc->adminq_lock, irqflags);

        /* Check for space in the queue */
        avail = q->tail_idx;
        if (q->head_idx >= avail)
                avail += q->num_descs - q->head_idx - 1;
        else
                avail -= q->head_idx + 1;
        if (!avail) {
                ret = -ENOSPC;
                goto err_out_unlock;
        }

        /* Check that the FW is running */
        if (!pdsc_is_fw_running(pdsc)) {
                if (pdsc->info_regs) {
                        u8 fw_status =
                                ioread8(&pdsc->info_regs->fw_status);

                        dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n",
                                 __func__, fw_status);
                } else {
                        dev_info(pdsc->dev, "%s: post failed - BARs not setup\n",
                                 __func__);
                }
                ret = -ENXIO;

                goto err_out_unlock;
        }

        /* Post the request */
        index = q->head_idx;
        q_info = &q->info[index];
        q_info->dest = comp;
        memcpy(q_info->desc, cmd, sizeof(*cmd));
        reinit_completion(&q_info->completion);

        dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n",
                q->head_idx, q->tail_idx);
        dev_dbg(pdsc->dev, "post admin queue command:\n");
        dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1,
                         cmd, sizeof(*cmd), true);

        q->head_idx = (q->head_idx + 1) & (q->num_descs - 1);

        pds_core_dbell_ring(pdsc->kern_dbpage,
                            q->hw_type, q->dbval | q->head_idx);
        ret = index;

err_out_unlock:
        spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags);
        return ret;
}

int pdsc_adminq_post(struct pdsc *pdsc,
                     union pds_core_adminq_cmd *cmd,
                     union pds_core_adminq_comp *comp,
                     bool fast_poll)
{
        unsigned long poll_interval = 200;
        unsigned long poll_jiffies;
        unsigned long time_limit;
        unsigned long time_start;
        unsigned long time_done;
        unsigned long remaining;
        struct completion *wc;
        int err = 0;
        int index;

        if (!pdsc_adminq_inc_if_up(pdsc)) {
                dev_dbg(pdsc->dev, "%s: preventing adminq cmd %u\n",
                        __func__, cmd->opcode);
                return -ENXIO;
        }

        index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp);
        if (index < 0) {
                err = index;
                goto err_out;
        }

        wc = &pdsc->adminqcq.q.info[index].completion;
        time_start = jiffies;
        time_limit = time_start + HZ * pdsc->devcmd_timeout;
        do {
                /* Timeslice the actual wait to catch IO errors etc early */
                poll_jiffies = usecs_to_jiffies(poll_interval);
                remaining = wait_for_completion_timeout(wc, poll_jiffies);
                if (remaining)
                        break;

                if (!pdsc_is_fw_running(pdsc)) {
                        if (pdsc->info_regs) {
                                u8 fw_status =
                                        ioread8(&pdsc->info_regs->fw_status);

                                dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n",
                                        __func__, fw_status);
                        } else {
                                dev_dbg(pdsc->dev, "%s: post wait failed - BARs not setup\n",
                                        __func__);
                        }
                        err = -ENXIO;
                        break;
                }

                /* When fast_poll is not requested, prevent aggressive polling
                 * on failures due to timeouts by doing exponential back off.
                 */
                if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL)
                        poll_interval <<= 1;
        } while (time_before(jiffies, time_limit));
        time_done = jiffies;
        dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n",
                __func__, jiffies_to_msecs(time_done - time_start));

        /* Check the results and clear an un-completed timeout */
        if (time_after_eq(time_done, time_limit) && !completion_done(wc)) {
                err = -ETIMEDOUT;
                complete(wc);
        }

        dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index);
        dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1,
                         comp, sizeof(*comp), true);

        if (remaining && comp->status)
                err = pdsc_err_to_errno(comp->status);

err_out:
        if (err) {
                dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n",
                        __func__, cmd->opcode, comp->status, ERR_PTR(err));
                if (err == -ENXIO || err == -ETIMEDOUT)
                        queue_work(pdsc->wq, &pdsc->health_work);
        }

        refcount_dec(&pdsc->adminq_refcnt);

        return err;
}
EXPORT_SYMBOL_GPL(pdsc_adminq_post);