// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#include <linux/mlx5/device.h>
#include <linux/mlx5/transobj.h>
#include "clock.h"
#include "aso.h"
#include "wq.h"

struct mlx5_aso_cq {
        /* data path - accessed per cqe */
        struct mlx5_cqwq           wq;

        /* data path - accessed per napi poll */
        struct mlx5_core_cq        mcq;

        /* control */
        struct mlx5_core_dev      *mdev;
        struct mlx5_wq_ctrl        wq_ctrl;
} ____cacheline_aligned_in_smp;

struct mlx5_aso {
        /* data path */
        u16                        cc;
        u16                        pc;

        struct mlx5_wqe_ctrl_seg  *doorbell_cseg;
        struct mlx5_aso_cq         cq;

        /* read only */
        struct mlx5_wq_cyc         wq;
        void __iomem              *uar_map;
        u32                        sqn;

        /* control path */
        struct mlx5_wq_ctrl        wq_ctrl;

} ____cacheline_aligned_in_smp;

static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
{
        mlx5_wq_destroy(&cq->wq_ctrl);
}

static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
                             void *cqc_data, struct mlx5_aso_cq *cq)
{
        struct mlx5_core_cq *mcq = &cq->mcq;
        struct mlx5_wq_param param;
        int err;
        u32 i;

        param.buf_numa_node = numa_node;
        param.db_numa_node = numa_node;

        err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
        if (err)
                return err;

        mcq->cqe_sz     = 64;
        mcq->set_ci_db  = cq->wq_ctrl.db.db;
        mcq->arm_db     = cq->wq_ctrl.db.db + 1;

        for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
                struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);

                cqe->op_own = 0xf1;
        }

        cq->mdev = mdev;

        return 0;
}

static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
{
        u32 out[MLX5_ST_SZ_DW(create_cq_out)];
        struct mlx5_core_dev *mdev = cq->mdev;
        struct mlx5_core_cq *mcq = &cq->mcq;
        void *in, *cqc;
        int inlen, eqn;
        int err;

        err = mlx5_comp_eqn_get(mdev, 0, &eqn);
        if (err)
                return err;

        inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
                sizeof(u64) * cq->wq_ctrl.buf.npages;
        in = kvzalloc(inlen, GFP_KERNEL);
        if (!in)
                return -ENOMEM;

        cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);

        memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));

        mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
                                  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));

        MLX5_SET(cqc,   cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
        MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
        MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.bfreg.up->index);
        MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
                                            MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);

        err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));

        kvfree(in);

        return err;
}

static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
{
        mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
        mlx5_wq_destroy(&cq->wq_ctrl);
}

static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
                              struct mlx5_aso_cq *cq)
{
        void *cqc_data;
        int err;

        cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
        if (!cqc_data)
                return -ENOMEM;

        MLX5_SET(cqc, cqc_data, log_cq_size, 1);
        MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.bfreg.up->index);
        if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
                MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);

        err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
        if (err) {
                mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
                goto err_out;
        }

        err = create_aso_cq(cq, cqc_data);
        if (err) {
                mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
                goto err_free_cq;
        }

        kvfree(cqc_data);
        return 0;

err_free_cq:
        mlx5_aso_free_cq(cq);
err_out:
        kvfree(cqc_data);
        return err;
}

static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
                             void *sqc_data, struct mlx5_aso *sq)
{
        void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
        struct mlx5_wq_cyc *wq = &sq->wq;
        struct mlx5_wq_param param;
        int err;

        sq->uar_map = mdev->priv.bfreg.map;

        param.db_numa_node = numa_node;
        param.buf_numa_node = numa_node;
        err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
        if (err)
                return err;
        wq->db = &wq->db[MLX5_SND_DBR];

        return 0;
}

static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
                         void *sqc_data, struct mlx5_aso *sq)
{
        void *in, *sqc, *wq;
        int inlen, err;
        u8 ts_format;

        inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
                sizeof(u64) * sq->wq_ctrl.buf.npages;
        in = kvzalloc(inlen, GFP_KERNEL);
        if (!in)
                return -ENOMEM;

        sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
        wq = MLX5_ADDR_OF(sqc, sqc, wq);

        memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
        MLX5_SET(sqc,  sqc, cqn, sq->cq.mcq.cqn);

        MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
        MLX5_SET(sqc,  sqc, flush_in_error_en, 1);

        ts_format = mlx5_is_real_time_sq(mdev) ?
                        MLX5_TIMESTAMP_FORMAT_REAL_TIME :
                        MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
        MLX5_SET(sqc, sqc, ts_format, ts_format);

        MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
        MLX5_SET(wq,   wq, uar_page,      mdev->priv.bfreg.index);
        MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
                                          MLX5_ADAPTER_PAGE_SHIFT);
        MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);

        mlx5_fill_page_frag_array(&sq->wq_ctrl.buf,
                                  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));

        err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);

        kvfree(in);

        return err;
}

static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
{
        void *in, *sqc;
        int inlen, err;

        inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
        in = kvzalloc(inlen, GFP_KERNEL);
        if (!in)
                return -ENOMEM;

        MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
        sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
        MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);

        err = mlx5_core_modify_sq(mdev, sqn, in);

        kvfree(in);

        return err;
}

static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
                                  void *sqc_data, struct mlx5_aso *sq)
{
        int err;

        err = create_aso_sq(mdev, pdn, sqc_data, sq);
        if (err)
                return err;

        err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
        if (err)
                mlx5_core_destroy_sq(mdev, sq->sqn);

        return err;
}

static void mlx5_aso_free_sq(struct mlx5_aso *sq)
{
        mlx5_wq_destroy(&sq->wq_ctrl);
}

static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
{
        mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
        mlx5_aso_free_sq(sq);
}

static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
                              u32 pdn, struct mlx5_aso *sq)
{
        void *sqc_data, *wq;
        int err;

        sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
        if (!sqc_data)
                return -ENOMEM;

        wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
        MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
        MLX5_SET(wq, wq, pd, pdn);
        MLX5_SET(wq, wq, log_wq_sz, 1);

        err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
        if (err) {
                mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
                goto err_out;
        }

        err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
        if (err) {
                mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
                goto err_free_asosq;
        }

        mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);

        kvfree(sqc_data);
        return 0;

err_free_asosq:
        mlx5_aso_free_sq(sq);
err_out:
        kvfree(sqc_data);
        return err;
}

struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn)
{
        int numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
        struct mlx5_aso *aso;
        int err;

        aso = kzalloc_obj(*aso);
        if (!aso)
                return ERR_PTR(-ENOMEM);

        err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
        if (err)
                goto err_cq;

        err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
        if (err)
                goto err_sq;

        return aso;

err_sq:
        mlx5_aso_destroy_cq(&aso->cq);
err_cq:
        kfree(aso);
        return ERR_PTR(err);
}

void mlx5_aso_destroy(struct mlx5_aso *aso)
{
        mlx5_aso_destroy_sq(aso);
        mlx5_aso_destroy_cq(&aso->cq);
        kfree(aso);
}

void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
                        struct mlx5_aso_wqe *aso_wqe,
                        u32 obj_id, u32 opc_mode)
{
        struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;

        cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) |
                                             (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
                                             MLX5_OPCODE_ACCESS_ASO);
        cseg->qpn_ds     = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
        cseg->fm_ce_se   = MLX5_WQE_CTRL_CQ_UPDATE;
        cseg->general_id = cpu_to_be32(obj_id);
}

struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso)
{
        struct mlx5_aso_wqe *wqe;
        u16 pi;

        pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
        wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
        memset(wqe, 0, sizeof(*wqe));
        return wqe;
}

void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
                       struct mlx5_wqe_ctrl_seg *doorbell_cseg)
{
        doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
        /* ensure wqe is visible to device before updating doorbell record */
        dma_wmb();

        if (with_data)
                aso->pc += MLX5_ASO_WQEBBS_DATA;
        else
                aso->pc += MLX5_ASO_WQEBBS;
        *aso->wq.db = cpu_to_be32(aso->pc);

        /* ensure doorbell record is visible to device before ringing the
         * doorbell
         */
        wmb();

        mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map);

        /* Ensure doorbell is written on uar_page before poll_cq */
        WRITE_ONCE(doorbell_cseg, NULL);
}

int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
{
        struct mlx5_aso_cq *cq = &aso->cq;
        struct mlx5_cqe64 *cqe;

        cqe = mlx5_cqwq_get_cqe(&cq->wq);
        if (!cqe)
                return -ETIMEDOUT;

        /* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
         * otherwise a cq overrun may occur
         */
        mlx5_cqwq_pop(&cq->wq);

        if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
                struct mlx5_err_cqe *err_cqe;

                mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
                              get_cqe_opcode(cqe));

                err_cqe = (struct mlx5_err_cqe *)cqe;
                mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
                              err_cqe->vendor_err_synd);
                mlx5_core_err(cq->mdev, "syndrome=%x\n",
                              err_cqe->syndrome);
                print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
                               16, 1, err_cqe,
                               sizeof(*err_cqe), false);
        }

        mlx5_cqwq_update_db_record(&cq->wq);

        /* ensure cq space is freed before enabling more cqes */
        wmb();

        if (with_data)
                aso->cc += MLX5_ASO_WQEBBS_DATA;
        else
                aso->cc += MLX5_ASO_WQEBBS;

        return 0;
}