// SPDX-License-Identifier: GPL-2.0
/* Marvell RVU Admin Function driver
 *
 * Copyright (C) 2018 Marvell.
 *
 */
#include <linux/bitfield.h>
#include <linux/module.h>
#include <linux/pci.h>

#include "rvu_struct.h"
#include "rvu_reg.h"
#include "rvu.h"

static int npa_aq_enqueue_wait(struct rvu *rvu, struct rvu_block *block,
                               struct npa_aq_inst_s *inst)
{
        struct admin_queue *aq = block->aq;
        struct npa_aq_res_s *result;
        int timeout = 1000;
        u64 reg, head;

        result = (struct npa_aq_res_s *)aq->res->base;

        /* Get current head pointer where to append this instruction */
        reg = rvu_read64(rvu, block->addr, NPA_AF_AQ_STATUS);
        head = (reg >> 4) & AQ_PTR_MASK;

        memcpy((void *)(aq->inst->base + (head * aq->inst->entry_sz)),
               (void *)inst, aq->inst->entry_sz);
        memset(result, 0, sizeof(*result));
        /* sync into memory */
        wmb();

        /* Ring the doorbell and wait for result */
        rvu_write64(rvu, block->addr, NPA_AF_AQ_DOOR, 1);
        while (result->compcode == NPA_AQ_COMP_NOTDONE) {
                cpu_relax();
                udelay(1);
                timeout--;
                if (!timeout)
                        return -EBUSY;
        }

        if (result->compcode != NPA_AQ_COMP_GOOD) {
                /* TODO: Replace this with some error code */
                if (result->compcode == NPA_AQ_COMP_CTX_FAULT ||
                    result->compcode == NPA_AQ_COMP_LOCKERR ||
                    result->compcode == NPA_AQ_COMP_CTX_POISON) {
                        if (rvu_ndc_fix_locked_cacheline(rvu, BLKADDR_NDC_NPA0))
                                dev_err(rvu->dev,
                                        "%s: Not able to unlock cachelines\n", __func__);
                }

                return -EBUSY;
        }

        return 0;
}

int rvu_npa_aq_enq_inst(struct rvu *rvu, struct npa_aq_enq_req *req,
                        struct npa_aq_enq_rsp *rsp)
{
        struct rvu_hwinfo *hw = rvu->hw;
        u16 pcifunc = req->hdr.pcifunc;
        int blkaddr, npalf, rc = 0;
        struct npa_aq_inst_s inst;
        struct rvu_block *block;
        struct admin_queue *aq;
        struct rvu_pfvf *pfvf;
        void *ctx, *mask;
        bool ena;

        pfvf = rvu_get_pfvf(rvu, pcifunc);
        if (!pfvf->aura_ctx || req->aura_id >= pfvf->aura_ctx->qsize)
                return NPA_AF_ERR_AQ_ENQUEUE;

        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
        if (!pfvf->npalf || blkaddr < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        block = &hw->block[blkaddr];
        aq = block->aq;
        if (!aq) {
                dev_warn(rvu->dev, "%s: NPA AQ not initialized\n", __func__);
                return NPA_AF_ERR_AQ_ENQUEUE;
        }

        npalf = rvu_get_lf(rvu, block, pcifunc, 0);
        if (npalf < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        memset(&inst, 0, sizeof(struct npa_aq_inst_s));
        inst.cindex = req->aura_id;
        inst.lf = npalf;
        inst.ctype = req->ctype;
        inst.op = req->op;
        /* Currently we are not supporting enqueuing multiple instructions,
         * so always choose first entry in result memory.
         */
        inst.res_addr = (u64)aq->res->iova;

        /* Hardware uses same aq->res->base for updating result of
         * previous instruction hence wait here till it is done.
         */
        spin_lock(&aq->lock);

        /* Clean result + context memory */
        memset(aq->res->base, 0, aq->res->entry_sz);
        /* Context needs to be written at RES_ADDR + 128 */
        ctx = aq->res->base + 128;
        /* Mask needs to be written at RES_ADDR + 256 */
        mask = aq->res->base + 256;

        switch (req->op) {
        case NPA_AQ_INSTOP_WRITE:
                /* Copy context and write mask */
                if (req->ctype == NPA_AQ_CTYPE_AURA) {
                        memcpy(mask, &req->aura_mask,
                               sizeof(struct npa_aura_s));
                        memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
                } else {
                        memcpy(mask, &req->pool_mask,
                               sizeof(struct npa_pool_s));
                        memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
                }
                break;
        case NPA_AQ_INSTOP_INIT:
                if (req->ctype == NPA_AQ_CTYPE_AURA) {
                        if (req->aura.pool_addr >= pfvf->pool_ctx->qsize) {
                                rc = NPA_AF_ERR_AQ_FULL;
                                break;
                        }
                        /* Set pool's context address */
                        req->aura.pool_addr = pfvf->pool_ctx->iova +
                        (req->aura.pool_addr * pfvf->pool_ctx->entry_sz);
                        memcpy(ctx, &req->aura, sizeof(struct npa_aura_s));
                } else { /* POOL's context */
                        memcpy(ctx, &req->pool, sizeof(struct npa_pool_s));
                }
                break;
        case NPA_AQ_INSTOP_NOP:
        case NPA_AQ_INSTOP_READ:
        case NPA_AQ_INSTOP_LOCK:
        case NPA_AQ_INSTOP_UNLOCK:
                break;
        default:
                rc = NPA_AF_ERR_AQ_FULL;
                break;
        }

        if (rc) {
                spin_unlock(&aq->lock);
                return rc;
        }

        /* Submit the instruction to AQ */
        rc = npa_aq_enqueue_wait(rvu, block, &inst);
        if (rc) {
                spin_unlock(&aq->lock);
                return rc;
        }

        /* Set aura bitmap if aura hw context is enabled */
        if (req->ctype == NPA_AQ_CTYPE_AURA) {
                if (req->op == NPA_AQ_INSTOP_INIT && req->aura.ena)
                        __set_bit(req->aura_id, pfvf->aura_bmap);
                if (req->op == NPA_AQ_INSTOP_WRITE) {
                        ena = (req->aura.ena & req->aura_mask.ena) |
                                (test_bit(req->aura_id, pfvf->aura_bmap) &
                                ~req->aura_mask.ena);
                        if (ena)
                                __set_bit(req->aura_id, pfvf->aura_bmap);
                        else
                                __clear_bit(req->aura_id, pfvf->aura_bmap);
                }
        }

        /* Set pool bitmap if pool hw context is enabled */
        if (req->ctype == NPA_AQ_CTYPE_POOL) {
                if (req->op == NPA_AQ_INSTOP_INIT && req->pool.ena)
                        __set_bit(req->aura_id, pfvf->pool_bmap);
                if (req->op == NPA_AQ_INSTOP_WRITE) {
                        ena = (req->pool.ena & req->pool_mask.ena) |
                                (test_bit(req->aura_id, pfvf->pool_bmap) &
                                ~req->pool_mask.ena);
                        if (ena)
                                __set_bit(req->aura_id, pfvf->pool_bmap);
                        else
                                __clear_bit(req->aura_id, pfvf->pool_bmap);
                }
        }
        spin_unlock(&aq->lock);

        if (rsp) {
                /* Copy read context into mailbox */
                if (req->op == NPA_AQ_INSTOP_READ) {
                        if (req->ctype == NPA_AQ_CTYPE_AURA)
                                memcpy(&rsp->aura, ctx,
                                       sizeof(struct npa_aura_s));
                        else
                                memcpy(&rsp->pool, ctx,
                                       sizeof(struct npa_pool_s));
                }
        }

        return 0;
}

static int npa_lf_hwctx_disable(struct rvu *rvu, struct hwctx_disable_req *req)
{
        struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
        struct npa_aq_enq_req aq_req;
        unsigned long *bmap;
        int id, cnt = 0;
        int err = 0, rc;

        if (!pfvf->pool_ctx || !pfvf->aura_ctx)
                return NPA_AF_ERR_AQ_ENQUEUE;

        memset(&aq_req, 0, sizeof(struct npa_aq_enq_req));
        aq_req.hdr.pcifunc = req->hdr.pcifunc;

        if (req->ctype == NPA_AQ_CTYPE_POOL) {
                aq_req.pool.ena = 0;
                aq_req.pool_mask.ena = 1;
                cnt = pfvf->pool_ctx->qsize;
                bmap = pfvf->pool_bmap;
        } else if (req->ctype == NPA_AQ_CTYPE_AURA) {
                aq_req.aura.ena = 0;
                aq_req.aura_mask.ena = 1;
                aq_req.aura.bp_ena = 0;
                aq_req.aura_mask.bp_ena = 1;
                cnt = pfvf->aura_ctx->qsize;
                bmap = pfvf->aura_bmap;
        }

        aq_req.ctype = req->ctype;
        aq_req.op = NPA_AQ_INSTOP_WRITE;

        for (id = 0; id < cnt; id++) {
                if (!test_bit(id, bmap))
                        continue;
                aq_req.aura_id = id;
                rc = rvu_npa_aq_enq_inst(rvu, &aq_req, NULL);
                if (rc) {
                        err = rc;
                        dev_err(rvu->dev, "Failed to disable %s:%d context\n",
                                (req->ctype == NPA_AQ_CTYPE_AURA) ?
                                "Aura" : "Pool", id);
                }
        }

        return err;
}

#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
static int npa_lf_hwctx_lockdown(struct rvu *rvu, struct npa_aq_enq_req *req)
{
        struct npa_aq_enq_req lock_ctx_req;
        int err;

        if (req->op != NPA_AQ_INSTOP_INIT)
                return 0;

        memset(&lock_ctx_req, 0, sizeof(struct npa_aq_enq_req));
        lock_ctx_req.hdr.pcifunc = req->hdr.pcifunc;
        lock_ctx_req.ctype = req->ctype;
        lock_ctx_req.op = NPA_AQ_INSTOP_LOCK;
        lock_ctx_req.aura_id = req->aura_id;
        err = rvu_npa_aq_enq_inst(rvu, &lock_ctx_req, NULL);
        if (err)
                dev_err(rvu->dev,
                        "PFUNC 0x%x: Failed to lock NPA context %s:%d\n",
                        req->hdr.pcifunc,
                        (req->ctype == NPA_AQ_CTYPE_AURA) ?
                        "Aura" : "Pool", req->aura_id);
        return err;
}

int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
                                struct npa_aq_enq_req *req,
                                struct npa_aq_enq_rsp *rsp)
{
        int err;

        err = rvu_npa_aq_enq_inst(rvu, req, rsp);
        if (!err)
                err = npa_lf_hwctx_lockdown(rvu, req);
        return err;
}
#else

int rvu_mbox_handler_npa_aq_enq(struct rvu *rvu,
                                struct npa_aq_enq_req *req,
                                struct npa_aq_enq_rsp *rsp)
{
        return rvu_npa_aq_enq_inst(rvu, req, rsp);
}
#endif

int rvu_mbox_handler_npa_hwctx_disable(struct rvu *rvu,
                                       struct hwctx_disable_req *req,
                                       struct msg_rsp *rsp)
{
        return npa_lf_hwctx_disable(rvu, req);
}

static void npa_ctx_free(struct rvu *rvu, struct rvu_pfvf *pfvf)
{
        kfree(pfvf->aura_bmap);
        pfvf->aura_bmap = NULL;

        qmem_free(rvu->dev, pfvf->aura_ctx);
        pfvf->aura_ctx = NULL;

        kfree(pfvf->pool_bmap);
        pfvf->pool_bmap = NULL;

        qmem_free(rvu->dev, pfvf->pool_ctx);
        pfvf->pool_ctx = NULL;

        qmem_free(rvu->dev, pfvf->npa_qints_ctx);
        pfvf->npa_qints_ctx = NULL;
}

int rvu_mbox_handler_npa_lf_alloc(struct rvu *rvu,
                                  struct npa_lf_alloc_req *req,
                                  struct npa_lf_alloc_rsp *rsp)
{
        int npalf, qints, hwctx_size, err, rc = 0;
        struct rvu_hwinfo *hw = rvu->hw;
        u16 pcifunc = req->hdr.pcifunc;
        struct rvu_block *block;
        struct rvu_pfvf *pfvf;
        u64 cfg, ctx_cfg;
        int blkaddr;

        if (req->aura_sz > NPA_AURA_SZ_MAX ||
            req->aura_sz == NPA_AURA_SZ_0 || !req->nr_pools)
                return NPA_AF_ERR_PARAM;

        if (req->way_mask)
                req->way_mask &= 0xFFFF;

        pfvf = rvu_get_pfvf(rvu, pcifunc);
        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
        if (!pfvf->npalf || blkaddr < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        block = &hw->block[blkaddr];
        npalf = rvu_get_lf(rvu, block, pcifunc, 0);
        if (npalf < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        /* Reset this NPA LF */
        err = rvu_lf_reset(rvu, block, npalf);
        if (err) {
                dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
                return NPA_AF_ERR_LF_RESET;
        }

        ctx_cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST1);

        /* Alloc memory for aura HW contexts */
        hwctx_size = 1UL << (ctx_cfg & 0xF);
        err = qmem_alloc(rvu->dev, &pfvf->aura_ctx,
                         NPA_AURA_COUNT(req->aura_sz), hwctx_size);
        if (err)
                goto free_mem;

        pfvf->aura_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
                                  GFP_KERNEL);
        if (!pfvf->aura_bmap)
                goto free_mem;

        /* Alloc memory for pool HW contexts */
        hwctx_size = 1UL << ((ctx_cfg >> 4) & 0xF);
        err = qmem_alloc(rvu->dev, &pfvf->pool_ctx, req->nr_pools, hwctx_size);
        if (err)
                goto free_mem;

        pfvf->pool_bmap = kcalloc(NPA_AURA_COUNT(req->aura_sz), sizeof(long),
                                  GFP_KERNEL);
        if (!pfvf->pool_bmap)
                goto free_mem;

        /* Get no of queue interrupts supported */
        cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
        qints = (cfg >> 28) & 0xFFF;

        /* Alloc memory for Qints HW contexts */
        hwctx_size = 1UL << ((ctx_cfg >> 8) & 0xF);
        err = qmem_alloc(rvu->dev, &pfvf->npa_qints_ctx, qints, hwctx_size);
        if (err)
                goto free_mem;

        cfg = rvu_read64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf));
        /* Clear way partition mask and set aura offset to '0' */
        cfg &= ~(BIT_ULL(34) - 1);
        /* Set aura size & enable caching of contexts */
        cfg |= (req->aura_sz << 16) | BIT_ULL(34) | req->way_mask;

        rvu_write64(rvu, blkaddr, NPA_AF_LFX_AURAS_CFG(npalf), cfg);

        /* Configure aura HW context's base */
        rvu_write64(rvu, blkaddr, NPA_AF_LFX_LOC_AURAS_BASE(npalf),
                    (u64)pfvf->aura_ctx->iova);

        /* Enable caching of qints hw context */
        rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_CFG(npalf),
                    BIT_ULL(36) | req->way_mask << 20);
        rvu_write64(rvu, blkaddr, NPA_AF_LFX_QINTS_BASE(npalf),
                    (u64)pfvf->npa_qints_ctx->iova);

        goto exit;

free_mem:
        npa_ctx_free(rvu, pfvf);
        rc = -ENOMEM;

exit:
        /* set stack page info */
        cfg = rvu_read64(rvu, blkaddr, NPA_AF_CONST);
        rsp->stack_pg_ptrs = (cfg >> 8) & 0xFF;
        rsp->stack_pg_bytes = cfg & 0xFF;
        rsp->qints = (cfg >> 28) & 0xFFF;
        if (!is_rvu_otx2(rvu)) {
                cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
                rsp->cache_lines = (cfg >> 1) & 0x3F;
        }
        return rc;
}

int rvu_mbox_handler_npa_lf_free(struct rvu *rvu, struct msg_req *req,
                                 struct msg_rsp *rsp)
{
        struct rvu_hwinfo *hw = rvu->hw;
        u16 pcifunc = req->hdr.pcifunc;
        struct rvu_block *block;
        struct rvu_pfvf *pfvf;
        int npalf, err;
        int blkaddr;

        pfvf = rvu_get_pfvf(rvu, pcifunc);
        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, pcifunc);
        if (!pfvf->npalf || blkaddr < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        block = &hw->block[blkaddr];
        npalf = rvu_get_lf(rvu, block, pcifunc, 0);
        if (npalf < 0)
                return NPA_AF_ERR_AF_LF_INVALID;

        /* Reset this NPA LF */
        err = rvu_lf_reset(rvu, block, npalf);
        if (err) {
                dev_err(rvu->dev, "Failed to reset NPALF%d\n", npalf);
                return NPA_AF_ERR_LF_RESET;
        }

        npa_ctx_free(rvu, pfvf);

        return 0;
}

static void npa_aq_ndc_config(struct rvu *rvu, struct rvu_block *block)
{
        u64 cfg;

        if (is_cn20k(rvu->pdev)) /* NDC not applicable to cn20k */
                return;

        /* Do not bypass NDC cache */
        cfg = rvu_read64(rvu, block->addr, NPA_AF_NDC_CFG);
        cfg &= ~0x03DULL;
#ifdef CONFIG_NDC_DIS_DYNAMIC_CACHING
        /* Disable caching of stack pages */
        cfg |= 0x10ULL;
#endif
        rvu_write64(rvu, block->addr, NPA_AF_NDC_CFG, cfg);
}

static int npa_aq_init(struct rvu *rvu, struct rvu_block *block)
{
        u64 cfg;
        int err;

        /* Set admin queue endianness */
        cfg = rvu_read64(rvu, block->addr, NPA_AF_GEN_CFG);
#ifdef __BIG_ENDIAN
        cfg |= BIT_ULL(1);
        rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
#else
        cfg &= ~BIT_ULL(1);
        rvu_write64(rvu, block->addr, NPA_AF_GEN_CFG, cfg);
#endif

        npa_aq_ndc_config(rvu, block);

        /* For CN10K NPA BATCH DMA set 35 cache lines */
        if (!is_rvu_otx2(rvu)) {
                cfg = rvu_read64(rvu, block->addr, NPA_AF_BATCH_CTL);
                cfg &= ~0x7EULL;
                cfg |= BIT_ULL(6) | BIT_ULL(2) | BIT_ULL(1);
                rvu_write64(rvu, block->addr, NPA_AF_BATCH_CTL, cfg);
        }
        /* Result structure can be followed by Aura/Pool context at
         * RES + 128bytes and a write mask at RES + 256 bytes, depending on
         * operation type. Alloc sufficient result memory for all operations.
         */
        err = rvu_aq_alloc(rvu, &block->aq,
                           Q_COUNT(AQ_SIZE), sizeof(struct npa_aq_inst_s),
                           ALIGN(sizeof(struct npa_aq_res_s), 128) + 256);
        if (err)
                return err;

        rvu_write64(rvu, block->addr, NPA_AF_AQ_CFG, AQ_SIZE);
        rvu_write64(rvu, block->addr,
                    NPA_AF_AQ_BASE, (u64)block->aq->inst->iova);
        return 0;
}

int rvu_npa_init(struct rvu *rvu)
{
        struct rvu_hwinfo *hw = rvu->hw;
        int blkaddr;

        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
        if (blkaddr < 0)
                return 0;

        /* Initialize admin queue */
        return npa_aq_init(rvu, &hw->block[blkaddr]);
}

void rvu_npa_freemem(struct rvu *rvu)
{
        struct rvu_hwinfo *hw = rvu->hw;
        struct rvu_block *block;
        int blkaddr;

        blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPA, 0);
        if (blkaddr < 0)
                return;

        block = &hw->block[blkaddr];
        rvu_aq_free(rvu, block->aq);
}

void rvu_npa_lf_teardown(struct rvu *rvu, u16 pcifunc, int npalf)
{
        struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, pcifunc);
        struct hwctx_disable_req ctx_req;

        /* Disable all pools */
        ctx_req.hdr.pcifunc = pcifunc;
        ctx_req.ctype = NPA_AQ_CTYPE_POOL;
        npa_lf_hwctx_disable(rvu, &ctx_req);

        /* Disable all auras */
        ctx_req.ctype = NPA_AQ_CTYPE_AURA;
        npa_lf_hwctx_disable(rvu, &ctx_req);

        npa_ctx_free(rvu, pfvf);
}

/* Due to an Hardware errata, in some corner cases, AQ context lock
 * operations can result in a NDC way getting into an illegal state
 * of not valid but locked.
 *
 * This API solves the problem by clearing the lock bit of the NDC block.
 * The operation needs to be done for each line of all the NDC banks.
 */
int rvu_ndc_fix_locked_cacheline(struct rvu *rvu, int blkaddr)
{
        int bank, max_bank, line, max_line, err;
        u64 reg, ndc_af_const;

        if (is_cn20k(rvu->pdev)) /* NDC not applicable to cn20k */
                return 0;

        /* Set the ENABLE bit(63) to '0' */
        reg = rvu_read64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL);
        rvu_write64(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, reg & GENMASK_ULL(62, 0));

        /* Poll until the BUSY bits(47:32) are set to '0' */
        err = rvu_poll_reg(rvu, blkaddr, NDC_AF_CAMS_RD_INTERVAL, GENMASK_ULL(47, 32), true);
        if (err) {
                dev_err(rvu->dev, "Timed out while polling for NDC CAM busy bits.\n");
                return err;
        }

        ndc_af_const = rvu_read64(rvu, blkaddr, NDC_AF_CONST);
        max_bank = FIELD_GET(NDC_AF_BANK_MASK, ndc_af_const);
        max_line = FIELD_GET(NDC_AF_BANK_LINE_MASK, ndc_af_const);
        for (bank = 0; bank < max_bank; bank++) {
                for (line = 0; line < max_line; line++) {
                        /* Check if 'cache line valid bit(63)' is not set
                         * but 'cache line lock bit(60)' is set and on
                         * success, reset the lock bit(60).
                         */
                        reg = rvu_read64(rvu, blkaddr,
                                         NDC_AF_BANKX_LINEX_METADATA(bank, line));
                        if (!(reg & BIT_ULL(63)) && (reg & BIT_ULL(60))) {
                                rvu_write64(rvu, blkaddr,
                                            NDC_AF_BANKX_LINEX_METADATA(bank, line),
                                            reg & ~BIT_ULL(60));
                        }
                }
        }

        return 0;
}