// SPDX-License-Identifier: GPL-2.0-only
/*
 * SolidRun DPU driver for control plane
 *
 * Copyright (C) 2022-2023 SolidRun
 *
 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
 *
 */

#include <linux/iopoll.h>

#include "snet_vdpa.h"

enum snet_ctrl_opcodes {
        SNET_CTRL_OP_DESTROY = 1,
        SNET_CTRL_OP_READ_VQ_STATE,
        SNET_CTRL_OP_SUSPEND,
        SNET_CTRL_OP_RESUME,
};

#define SNET_CTRL_TIMEOUT               2000000

#define SNET_CTRL_DATA_SIZE_MASK        0x0000FFFF
#define SNET_CTRL_IN_PROCESS_MASK       0x00010000
#define SNET_CTRL_CHUNK_RDY_MASK        0x00020000
#define SNET_CTRL_ERROR_MASK            0x0FFC0000

#define SNET_VAL_TO_ERR(val)            (-(((val) & SNET_CTRL_ERROR_MASK) >> 18))
#define SNET_EMPTY_CTRL(val)            (((val) & SNET_CTRL_ERROR_MASK) || \
                                                !((val) & SNET_CTRL_IN_PROCESS_MASK))
#define SNET_DATA_READY(val)            ((val) & (SNET_CTRL_ERROR_MASK | SNET_CTRL_CHUNK_RDY_MASK))

/* Control register used to read data from the DPU */
struct snet_ctrl_reg_ctrl {
        /* Chunk size in 4B words */
        u16 data_size;
        /* We are in the middle of a command */
        u16 in_process:1;
        /* A data chunk is ready and can be consumed */
        u16 chunk_ready:1;
        /* Error code */
        u16 error:10;
        /* Saved for future usage */
        u16 rsvd:4;
};

/* Opcode register */
struct snet_ctrl_reg_op {
        u16 opcode;
        /* Only if VQ index is relevant for the command */
        u16 vq_idx;
};

struct snet_ctrl_regs {
        struct snet_ctrl_reg_op op;
        struct snet_ctrl_reg_ctrl ctrl;
        u32 rsvd;
        u32 data[];
};

static struct snet_ctrl_regs __iomem *snet_get_ctrl(struct snet *snet)
{
        return snet->bar + snet->psnet->cfg.ctrl_off;
}

static int snet_wait_for_empty_ctrl(struct snet_ctrl_regs __iomem *regs)
{
        u32 val;

        return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_EMPTY_CTRL(val), 10,
                                  SNET_CTRL_TIMEOUT);
}

static int snet_wait_for_empty_op(struct snet_ctrl_regs __iomem *regs)
{
        u32 val;

        return readx_poll_timeout(ioread32, &regs->op, val, !val, 10, SNET_CTRL_TIMEOUT);
}

static int snet_wait_for_data(struct snet_ctrl_regs __iomem *regs)
{
        u32 val;

        return readx_poll_timeout(ioread32, &regs->ctrl, val, SNET_DATA_READY(val), 10,
                                  SNET_CTRL_TIMEOUT);
}

static u32 snet_read32_word(struct snet_ctrl_regs __iomem *ctrl_regs, u16 word_idx)
{
        return ioread32(&ctrl_regs->data[word_idx]);
}

static u32 snet_read_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs)
{
        return ioread32(&ctrl_regs->ctrl);
}

static void snet_write_ctrl(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
        iowrite32(val, &ctrl_regs->ctrl);
}

static void snet_write_op(struct snet_ctrl_regs __iomem *ctrl_regs, u32 val)
{
        iowrite32(val, &ctrl_regs->op);
}

static int snet_wait_for_dpu_completion(struct snet_ctrl_regs __iomem *ctrl_regs)
{
        /* Wait until the DPU finishes completely.
         * It will clear the opcode register.
         */
        return snet_wait_for_empty_op(ctrl_regs);
}

/* Reading ctrl from the DPU:
 * buf_size must be 4B aligned
 *
 * Steps:
 *
 * (1) Verify that the DPU is not in the middle of another operation by
 *     reading the in_process and error bits in the control register.
 * (2) Write the request opcode and the VQ idx in the opcode register
 *     and write the buffer size in the control register.
 * (3) Start readind chunks of data, chunk_ready bit indicates that a
 *     data chunk is available, we signal that we read the data by clearing the bit.
 * (4) Detect that the transfer is completed when the in_process bit
 *     in the control register is cleared or when the an error appears.
 */
static int snet_ctrl_read_from_dpu(struct snet *snet, u16 opcode, u16 vq_idx, void *buffer,
                                   u32 buf_size)
{
        struct pci_dev *pdev = snet->pdev;
        struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
        u32 *bfr_ptr = (u32 *)buffer;
        u32 val;
        u16 buf_words;
        int ret;
        u16 words, i, tot_words = 0;

        /* Supported for config 2+ */
        if (!SNET_CFG_VER(snet, 2))
                return -EOPNOTSUPP;

        if (!IS_ALIGNED(buf_size, 4))
                return -EINVAL;

        mutex_lock(&snet->ctrl_lock);

        buf_words = buf_size / 4;

        /* Make sure control register is empty */
        ret = snet_wait_for_empty_ctrl(regs);
        if (ret) {
                SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
                goto exit;
        }

        /* We need to write the buffer size in the control register, and the opcode + vq index in
         * the opcode register.
         * We use a spinlock to serialize the writes.
         */
        spin_lock(&snet->ctrl_spinlock);

        snet_write_ctrl(regs, buf_words);
        snet_write_op(regs, opcode | (vq_idx << 16));

        spin_unlock(&snet->ctrl_spinlock);

        while (buf_words != tot_words) {
                ret = snet_wait_for_data(regs);
                if (ret) {
                        SNET_WARN(pdev, "Timeout waiting for control data\n");
                        goto exit;
                }

                val = snet_read_ctrl(regs);

                /* Error? */
                if (val & SNET_CTRL_ERROR_MASK) {
                        ret = SNET_VAL_TO_ERR(val);
                        SNET_WARN(pdev, "Error while reading control data from DPU, err %d\n", ret);
                        goto exit;
                }

                words = min_t(u16, val & SNET_CTRL_DATA_SIZE_MASK, buf_words - tot_words);

                for (i = 0; i < words; i++) {
                        *bfr_ptr = snet_read32_word(regs, i);
                        bfr_ptr++;
                }

                tot_words += words;

                /* Is the job completed? */
                if (!(val & SNET_CTRL_IN_PROCESS_MASK))
                        break;

                /* Clear the chunk ready bit and continue */
                val &= ~SNET_CTRL_CHUNK_RDY_MASK;
                snet_write_ctrl(regs, val);
        }

        ret = snet_wait_for_dpu_completion(regs);
        if (ret)
                SNET_WARN(pdev, "Timeout waiting for the DPU to complete a control command\n");

exit:
        mutex_unlock(&snet->ctrl_lock);
        return ret;
}

/* Send a control message to the DPU using the old mechanism
 * used with config version 1.
 */
static int snet_send_ctrl_msg_old(struct snet *snet, u32 opcode)
{
        struct pci_dev *pdev = snet->pdev;
        struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
        int ret;

        mutex_lock(&snet->ctrl_lock);

        /* Old mechanism uses just 1 register, the opcode register.
         * Make sure that the opcode register is empty, and that the DPU isn't
         * processing an old message.
         */
        ret = snet_wait_for_empty_op(regs);
        if (ret) {
                SNET_WARN(pdev, "Timeout waiting for previous control message to be ACKed\n");
                goto exit;
        }

        /* Write the message */
        snet_write_op(regs, opcode);

        /* DPU ACKs the message by clearing the opcode register */
        ret = snet_wait_for_empty_op(regs);
        if (ret)
                SNET_WARN(pdev, "Timeout waiting for a control message to be ACKed\n");

exit:
        mutex_unlock(&snet->ctrl_lock);
        return ret;
}

/* Send a control message to the DPU.
 * A control message is a message without payload.
 */
static int snet_send_ctrl_msg(struct snet *snet, u16 opcode, u16 vq_idx)
{
        struct pci_dev *pdev = snet->pdev;
        struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);
        u32 val;
        int ret;

        /* If config version is not 2+, use the old mechanism */
        if (!SNET_CFG_VER(snet, 2))
                return snet_send_ctrl_msg_old(snet, opcode);

        mutex_lock(&snet->ctrl_lock);

        /* Make sure control register is empty */
        ret = snet_wait_for_empty_ctrl(regs);
        if (ret) {
                SNET_WARN(pdev, "Timeout waiting for previous control data to be consumed\n");
                goto exit;
        }

        /* We need to clear the control register and write the opcode + vq index in the opcode
         * register.
         * We use a spinlock to serialize the writes.
         */
        spin_lock(&snet->ctrl_spinlock);

        snet_write_ctrl(regs, 0);
        snet_write_op(regs, opcode | (vq_idx << 16));

        spin_unlock(&snet->ctrl_spinlock);

        /* The DPU ACKs control messages by setting the chunk ready bit
         * without data.
         */
        ret = snet_wait_for_data(regs);
        if (ret) {
                SNET_WARN(pdev, "Timeout waiting for control message to be ACKed\n");
                goto exit;
        }

        /* Check for errors */
        val = snet_read_ctrl(regs);
        ret = SNET_VAL_TO_ERR(val);

        /* Clear the chunk ready bit */
        val &= ~SNET_CTRL_CHUNK_RDY_MASK;
        snet_write_ctrl(regs, val);

        ret = snet_wait_for_dpu_completion(regs);
        if (ret)
                SNET_WARN(pdev, "Timeout waiting for DPU to complete a control command, err %d\n",
                          ret);

exit:
        mutex_unlock(&snet->ctrl_lock);
        return ret;
}

void snet_ctrl_clear(struct snet *snet)
{
        struct snet_ctrl_regs __iomem *regs = snet_get_ctrl(snet);

        snet_write_op(regs, 0);
}

int snet_destroy_dev(struct snet *snet)
{
        return snet_send_ctrl_msg(snet, SNET_CTRL_OP_DESTROY, 0);
}

int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state)
{
        return snet_ctrl_read_from_dpu(snet, SNET_CTRL_OP_READ_VQ_STATE, idx, state,
                                       sizeof(*state));
}

int snet_suspend_dev(struct snet *snet)
{
        return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0);
}

int snet_resume_dev(struct snet *snet)
{
        return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0);
}