// SPDX-License-Identifier: GPL-2.0-only
/*
 * Huawei HiNIC PCI Express Linux driver
 * Copyright(c) 2017 Huawei Technologies Co., Ltd
 */

#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/errno.h>
#include <linux/pci.h>
#include <linux/device.h>
#include <linux/workqueue.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/dma-mapping.h>
#include <linux/log2.h>
#include <asm/byteorder.h>
#include <asm/barrier.h>

#include "hinic_hw_dev.h"
#include "hinic_hw_csr.h"
#include "hinic_hw_if.h"
#include "hinic_hw_eqs.h"

#define HINIC_EQS_WQ_NAME                       "hinic_eqs"

#define GET_EQ_NUM_PAGES(eq, pg_size)           \
                (ALIGN((eq)->q_len * (eq)->elem_size, pg_size) / (pg_size))

#define GET_EQ_NUM_ELEMS_IN_PG(eq, pg_size)     ((pg_size) / (eq)->elem_size)

#define EQ_CONS_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
                        HINIC_CSR_AEQ_CONS_IDX_ADDR((eq)->q_id) : \
                        HINIC_CSR_CEQ_CONS_IDX_ADDR((eq)->q_id))

#define EQ_PROD_IDX_REG_ADDR(eq)        (((eq)->type == HINIC_AEQ) ? \
                        HINIC_CSR_AEQ_PROD_IDX_ADDR((eq)->q_id) : \
                        HINIC_CSR_CEQ_PROD_IDX_ADDR((eq)->q_id))

#define EQ_HI_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
                        HINIC_CSR_AEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
                        HINIC_CSR_CEQ_HI_PHYS_ADDR_REG((eq)->q_id, pg_num))

#define EQ_LO_PHYS_ADDR_REG(eq, pg_num) (((eq)->type == HINIC_AEQ) ? \
                        HINIC_CSR_AEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num) : \
                        HINIC_CSR_CEQ_LO_PHYS_ADDR_REG((eq)->q_id, pg_num))

#define GET_EQ_ELEMENT(eq, idx)         \
                ((eq)->virt_addr[(idx) / (eq)->num_elem_in_pg] + \
                 (((idx) & ((eq)->num_elem_in_pg - 1)) * (eq)->elem_size))

#define GET_AEQ_ELEM(eq, idx)           ((struct hinic_aeq_elem *) \
                                        GET_EQ_ELEMENT(eq, idx))

#define GET_CEQ_ELEM(eq, idx)           ((u32 *) \
                                         GET_EQ_ELEMENT(eq, idx))

#define GET_CURR_AEQ_ELEM(eq)           GET_AEQ_ELEM(eq, (eq)->cons_idx)

#define GET_CURR_CEQ_ELEM(eq)           GET_CEQ_ELEM(eq, (eq)->cons_idx)

#define PAGE_IN_4K(page_size)           ((page_size) >> 12)
#define EQ_SET_HW_PAGE_SIZE_VAL(eq)     (ilog2(PAGE_IN_4K((eq)->page_size)))

#define ELEMENT_SIZE_IN_32B(eq)         (((eq)->elem_size) >> 5)
#define EQ_SET_HW_ELEM_SIZE_VAL(eq)     (ilog2(ELEMENT_SIZE_IN_32B(eq)))

#define EQ_MAX_PAGES                    8

#define CEQE_TYPE_SHIFT                 23
#define CEQE_TYPE_MASK                  0x7

#define CEQE_TYPE(ceqe)                 (((ceqe) >> CEQE_TYPE_SHIFT) &  \
                                         CEQE_TYPE_MASK)

#define CEQE_DATA_MASK                  0x3FFFFFF
#define CEQE_DATA(ceqe)                 ((ceqe) & CEQE_DATA_MASK)

#define aeq_to_aeqs(eq)                 \
                container_of((eq) - (eq)->q_id, struct hinic_aeqs, aeq[0])

#define ceq_to_ceqs(eq)                 \
                container_of((eq) - (eq)->q_id, struct hinic_ceqs, ceq[0])

#define work_to_aeq_work(work)          \
                container_of(work, struct hinic_eq_work, work)

#define DMA_ATTR_AEQ_DEFAULT            0
#define DMA_ATTR_CEQ_DEFAULT            0

/* No coalescence */
#define THRESH_CEQ_DEFAULT              0

enum eq_int_mode {
        EQ_INT_MODE_ARMED,
        EQ_INT_MODE_ALWAYS
};

enum eq_arm_state {
        EQ_NOT_ARMED,
        EQ_ARMED
};

/**
 * hinic_aeq_register_hw_cb - register AEQ callback for specific event
 * @aeqs: pointer to Async eqs of the chip
 * @event: aeq event to register callback for it
 * @handle: private data will be used by the callback
 * @hwe_handler: callback function
 **/
void hinic_aeq_register_hw_cb(struct hinic_aeqs *aeqs,
                              enum hinic_aeq_type event, void *handle,
                              void (*hwe_handler)(void *handle, void *data,
                                                  u8 size))
{
        struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];

        hwe_cb->hwe_handler = hwe_handler;
        hwe_cb->handle = handle;
        hwe_cb->hwe_state = HINIC_EQE_ENABLED;
}

/**
 * hinic_aeq_unregister_hw_cb - unregister the AEQ callback for specific event
 * @aeqs: pointer to Async eqs of the chip
 * @event: aeq event to unregister callback for it
 **/
void hinic_aeq_unregister_hw_cb(struct hinic_aeqs *aeqs,
                                enum hinic_aeq_type event)
{
        struct hinic_hw_event_cb *hwe_cb = &aeqs->hwe_cb[event];

        hwe_cb->hwe_state &= ~HINIC_EQE_ENABLED;

        while (hwe_cb->hwe_state & HINIC_EQE_RUNNING)
                schedule();

        hwe_cb->hwe_handler = NULL;
}

/**
 * hinic_ceq_register_cb - register CEQ callback for specific event
 * @ceqs: pointer to Completion eqs part of the chip
 * @event: ceq event to register callback for it
 * @handle: private data will be used by the callback
 * @handler: callback function
 **/
void hinic_ceq_register_cb(struct hinic_ceqs *ceqs,
                           enum hinic_ceq_type event, void *handle,
                           void (*handler)(void *handle, u32 ceqe_data))
{
        struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];

        ceq_cb->handler = handler;
        ceq_cb->handle = handle;
        ceq_cb->ceqe_state = HINIC_EQE_ENABLED;
}

/**
 * hinic_ceq_unregister_cb - unregister the CEQ callback for specific event
 * @ceqs: pointer to Completion eqs part of the chip
 * @event: ceq event to unregister callback for it
 **/
void hinic_ceq_unregister_cb(struct hinic_ceqs *ceqs,
                             enum hinic_ceq_type event)
{
        struct hinic_ceq_cb *ceq_cb = &ceqs->ceq_cb[event];

        ceq_cb->ceqe_state &= ~HINIC_EQE_ENABLED;

        while (ceq_cb->ceqe_state & HINIC_EQE_RUNNING)
                schedule();

        ceq_cb->handler = NULL;
}

static u8 eq_cons_idx_checksum_set(u32 val)
{
        u8 checksum = 0;
        int idx;

        for (idx = 0; idx < 32; idx += 4)
                checksum ^= ((val >> idx) & 0xF);

        return (checksum & 0xF);
}

/**
 * eq_update_ci - update the HW cons idx of event queue
 * @eq: the event queue to update the cons idx for
 * @arm_state: the arm bit value of eq's interrupt
 **/
static void eq_update_ci(struct hinic_eq *eq, u32 arm_state)
{
        u32 val, addr = EQ_CONS_IDX_REG_ADDR(eq);

        /* Read Modify Write */
        val = hinic_hwif_read_reg(eq->hwif, addr);

        val = HINIC_EQ_CI_CLEAR(val, IDX)       &
              HINIC_EQ_CI_CLEAR(val, WRAPPED)   &
              HINIC_EQ_CI_CLEAR(val, INT_ARMED) &
              HINIC_EQ_CI_CLEAR(val, XOR_CHKSUM);

        val |= HINIC_EQ_CI_SET(eq->cons_idx, IDX)    |
               HINIC_EQ_CI_SET(eq->wrapped, WRAPPED) |
               HINIC_EQ_CI_SET(arm_state, INT_ARMED);

        val |= HINIC_EQ_CI_SET(eq_cons_idx_checksum_set(val), XOR_CHKSUM);

        hinic_hwif_write_reg(eq->hwif, addr, val);
}

/**
 * aeq_irq_handler - handler for the AEQ event
 * @eq: the Async Event Queue that received the event
 **/
static void aeq_irq_handler(struct hinic_eq *eq)
{
        struct hinic_aeqs *aeqs = aeq_to_aeqs(eq);
        struct hinic_hwif *hwif = aeqs->hwif;
        struct pci_dev *pdev = hwif->pdev;
        struct hinic_aeq_elem *aeqe_curr;
        struct hinic_hw_event_cb *hwe_cb;
        enum hinic_aeq_type event;
        unsigned long eqe_state;
        u32 aeqe_desc;
        int i, size;

        for (i = 0; i < eq->q_len; i++) {
                aeqe_curr = GET_CURR_AEQ_ELEM(eq);

                /* Data in HW is in Big endian Format */
                aeqe_desc = be32_to_cpu(aeqe_curr->desc);

                /* HW toggles the wrapped bit, when it adds eq element */
                if (HINIC_EQ_ELEM_DESC_GET(aeqe_desc, WRAPPED) == eq->wrapped)
                        break;

                dma_rmb();

                event = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, TYPE);
                if (event >= HINIC_MAX_AEQ_EVENTS) {
                        dev_err(&pdev->dev, "Unknown AEQ Event %d\n", event);
                        return;
                }

                if (!HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SRC)) {
                        hwe_cb = &aeqs->hwe_cb[event];

                        size = HINIC_EQ_ELEM_DESC_GET(aeqe_desc, SIZE);

                        eqe_state = cmpxchg(&hwe_cb->hwe_state,
                                            HINIC_EQE_ENABLED,
                                            HINIC_EQE_ENABLED |
                                            HINIC_EQE_RUNNING);
                        if (eqe_state == HINIC_EQE_ENABLED &&
                            hwe_cb->hwe_handler)
                                hwe_cb->hwe_handler(hwe_cb->handle,
                                                    aeqe_curr->data, size);
                        else
                                dev_err(&pdev->dev, "Unhandled AEQ Event %d\n",
                                        event);

                        hwe_cb->hwe_state &= ~HINIC_EQE_RUNNING;
                }

                eq->cons_idx++;

                if (eq->cons_idx == eq->q_len) {
                        eq->cons_idx = 0;
                        eq->wrapped = !eq->wrapped;
                }
        }
}

/**
 * ceq_event_handler - handler for the ceq events
 * @ceqs: ceqs part of the chip
 * @ceqe: ceq element that describes the event
 **/
static void ceq_event_handler(struct hinic_ceqs *ceqs, u32 ceqe)
{
        struct hinic_hwif *hwif = ceqs->hwif;
        struct pci_dev *pdev = hwif->pdev;
        struct hinic_ceq_cb *ceq_cb;
        enum hinic_ceq_type event;
        unsigned long eqe_state;

        event = CEQE_TYPE(ceqe);
        if (event >= HINIC_MAX_CEQ_EVENTS) {
                dev_err(&pdev->dev, "Unknown CEQ event, event = %d\n", event);
                return;
        }

        ceq_cb = &ceqs->ceq_cb[event];

        eqe_state = cmpxchg(&ceq_cb->ceqe_state,
                            HINIC_EQE_ENABLED,
                            HINIC_EQE_ENABLED | HINIC_EQE_RUNNING);

        if (eqe_state == HINIC_EQE_ENABLED && ceq_cb->handler)
                ceq_cb->handler(ceq_cb->handle, CEQE_DATA(ceqe));
        else
                dev_err(&pdev->dev, "Unhandled CEQ Event %d\n", event);

        ceq_cb->ceqe_state &= ~HINIC_EQE_RUNNING;
}

/**
 * ceq_irq_handler - handler for the CEQ event
 * @eq: the Completion Event Queue that received the event
 **/
static void ceq_irq_handler(struct hinic_eq *eq)
{
        struct hinic_ceqs *ceqs = ceq_to_ceqs(eq);
        u32 ceqe;
        int i;

        for (i = 0; i < eq->q_len; i++) {
                ceqe = *(GET_CURR_CEQ_ELEM(eq));

                /* Data in HW is in Big endian Format */
                ceqe = be32_to_cpu(ceqe);

                /* HW toggles the wrapped bit, when it adds eq element event */
                if (HINIC_EQ_ELEM_DESC_GET(ceqe, WRAPPED) == eq->wrapped)
                        break;

                ceq_event_handler(ceqs, ceqe);

                eq->cons_idx++;

                if (eq->cons_idx == eq->q_len) {
                        eq->cons_idx = 0;
                        eq->wrapped = !eq->wrapped;
                }
        }
}

/**
 * eq_irq_handler - handler for the EQ event
 * @data: the Event Queue that received the event
 **/
static void eq_irq_handler(void *data)
{
        struct hinic_eq *eq = data;

        if (eq->type == HINIC_AEQ)
                aeq_irq_handler(eq);
        else if (eq->type == HINIC_CEQ)
                ceq_irq_handler(eq);

        eq_update_ci(eq, EQ_ARMED);
}

/**
 * eq_irq_work - the work of the EQ that received the event
 * @work: the work struct that is associated with the EQ
 **/
static void eq_irq_work(struct work_struct *work)
{
        struct hinic_eq_work *aeq_work = work_to_aeq_work(work);
        struct hinic_eq *aeq;

        aeq = aeq_work->data;
        eq_irq_handler(aeq);
}

/**
 * ceq_tasklet - the tasklet of the EQ that received the event
 * @t: the tasklet struct pointer
 **/
static void ceq_tasklet(struct tasklet_struct *t)
{
        struct hinic_eq *ceq = from_tasklet(ceq, t, ceq_tasklet);

        eq_irq_handler(ceq);
}

/**
 * aeq_interrupt - aeq interrupt handler
 * @irq: irq number
 * @data: the Async Event Queue that collected the event
 **/
static irqreturn_t aeq_interrupt(int irq, void *data)
{
        struct hinic_eq_work *aeq_work;
        struct hinic_eq *aeq = data;
        struct hinic_aeqs *aeqs;

        /* clear resend timer cnt register */
        hinic_msix_attr_cnt_clear(aeq->hwif, aeq->msix_entry.entry);

        aeq_work = &aeq->aeq_work;
        aeq_work->data = aeq;

        aeqs = aeq_to_aeqs(aeq);
        queue_work(aeqs->workq, &aeq_work->work);

        return IRQ_HANDLED;
}

/**
 * ceq_interrupt - ceq interrupt handler
 * @irq: irq number
 * @data: the Completion Event Queue that collected the event
 **/
static irqreturn_t ceq_interrupt(int irq, void *data)
{
        struct hinic_eq *ceq = data;

        /* clear resend timer cnt register */
        hinic_msix_attr_cnt_clear(ceq->hwif, ceq->msix_entry.entry);

        tasklet_schedule(&ceq->ceq_tasklet);

        return IRQ_HANDLED;
}

static u32 get_ctrl0_val(struct hinic_eq *eq, u32 addr)
{
        struct msix_entry *msix_entry = &eq->msix_entry;
        enum hinic_eq_type type = eq->type;
        u32 val, ctrl0;

        if (type == HINIC_AEQ) {
                /* RMW Ctrl0 */
                addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);

                val = hinic_hwif_read_reg(eq->hwif, addr);

                val = HINIC_AEQ_CTRL_0_CLEAR(val, INT_IDX)      &
                      HINIC_AEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
                      HINIC_AEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
                      HINIC_AEQ_CTRL_0_CLEAR(val, INT_MODE);

                ctrl0 = HINIC_AEQ_CTRL_0_SET(msix_entry->entry, INT_IDX)     |
                        HINIC_AEQ_CTRL_0_SET(DMA_ATTR_AEQ_DEFAULT, DMA_ATTR) |
                        HINIC_AEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
                                             PCI_INTF_IDX)                   |
                        HINIC_AEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INT_MODE);

                val |= ctrl0;
        } else {
                /* RMW Ctrl0 */
                addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);

                val = hinic_hwif_read_reg(eq->hwif, addr);

                val = HINIC_CEQ_CTRL_0_CLEAR(val, INTR_IDX)     &
                      HINIC_CEQ_CTRL_0_CLEAR(val, DMA_ATTR)     &
                      HINIC_CEQ_CTRL_0_CLEAR(val, KICK_THRESH)  &
                      HINIC_CEQ_CTRL_0_CLEAR(val, PCI_INTF_IDX) &
                      HINIC_CEQ_CTRL_0_CLEAR(val, INTR_MODE);

                ctrl0 = HINIC_CEQ_CTRL_0_SET(msix_entry->entry, INTR_IDX)     |
                        HINIC_CEQ_CTRL_0_SET(DMA_ATTR_CEQ_DEFAULT, DMA_ATTR)  |
                        HINIC_CEQ_CTRL_0_SET(THRESH_CEQ_DEFAULT, KICK_THRESH) |
                        HINIC_CEQ_CTRL_0_SET(HINIC_HWIF_PCI_INTF(eq->hwif),
                                             PCI_INTF_IDX)                    |
                        HINIC_CEQ_CTRL_0_SET(EQ_INT_MODE_ARMED, INTR_MODE);

                val |= ctrl0;
        }
        return val;
}

static void set_ctrl0(struct hinic_eq *eq)
{
        u32 val, addr;

        if (eq->type == HINIC_AEQ)
                addr = HINIC_CSR_AEQ_CTRL_0_ADDR(eq->q_id);
        else
                addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);

        val = get_ctrl0_val(eq, addr);

        hinic_hwif_write_reg(eq->hwif, addr, val);
}

static u32 get_ctrl1_val(struct hinic_eq *eq, u32 addr)
{
        u32 page_size_val, elem_size, val, ctrl1;
        enum hinic_eq_type type = eq->type;

        if (type == HINIC_AEQ) {
                /* RMW Ctrl1 */
                addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);

                page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);
                elem_size = EQ_SET_HW_ELEM_SIZE_VAL(eq);

                val = hinic_hwif_read_reg(eq->hwif, addr);

                val = HINIC_AEQ_CTRL_1_CLEAR(val, LEN)          &
                      HINIC_AEQ_CTRL_1_CLEAR(val, ELEM_SIZE)    &
                      HINIC_AEQ_CTRL_1_CLEAR(val, PAGE_SIZE);

                ctrl1 = HINIC_AEQ_CTRL_1_SET(eq->q_len, LEN)            |
                        HINIC_AEQ_CTRL_1_SET(elem_size, ELEM_SIZE)      |
                        HINIC_AEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);

                val |= ctrl1;
        } else {
                /* RMW Ctrl1 */
                addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);

                page_size_val = EQ_SET_HW_PAGE_SIZE_VAL(eq);

                val = hinic_hwif_read_reg(eq->hwif, addr);

                val = HINIC_CEQ_CTRL_1_CLEAR(val, LEN) &
                      HINIC_CEQ_CTRL_1_CLEAR(val, PAGE_SIZE);

                ctrl1 = HINIC_CEQ_CTRL_1_SET(eq->q_len, LEN) |
                        HINIC_CEQ_CTRL_1_SET(page_size_val, PAGE_SIZE);

                val |= ctrl1;
        }
        return val;
}

static void set_ctrl1(struct hinic_eq *eq)
{
        u32 addr, val;

        if (eq->type == HINIC_AEQ)
                addr = HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id);
        else
                addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);

        val = get_ctrl1_val(eq, addr);

        hinic_hwif_write_reg(eq->hwif, addr, val);
}

static int set_ceq_ctrl_reg(struct hinic_eq *eq)
{
        struct hinic_ceq_ctrl_reg ceq_ctrl = {0};
        struct hinic_hwdev *hwdev = eq->hwdev;
        u16 out_size = sizeof(ceq_ctrl);
        u16 in_size = sizeof(ceq_ctrl);
        struct hinic_pfhwdev *pfhwdev;
        u32 addr;
        int err;

        pfhwdev = container_of(hwdev, struct hinic_pfhwdev, hwdev);

        addr = HINIC_CSR_CEQ_CTRL_0_ADDR(eq->q_id);
        ceq_ctrl.ctrl0 = get_ctrl0_val(eq, addr);
        addr = HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id);
        ceq_ctrl.ctrl1 = get_ctrl1_val(eq, addr);

        ceq_ctrl.func_id = HINIC_HWIF_FUNC_IDX(hwdev->hwif);
        ceq_ctrl.q_id = eq->q_id;

        err = hinic_msg_to_mgmt(&pfhwdev->pf_to_mgmt, HINIC_MOD_COMM,
                                HINIC_COMM_CMD_CEQ_CTRL_REG_WR_BY_UP,
                                &ceq_ctrl, in_size,
                                &ceq_ctrl, &out_size, HINIC_MGMT_MSG_SYNC);
        if (err || !out_size || ceq_ctrl.status) {
                dev_err(&hwdev->hwif->pdev->dev,
                        "Failed to set ceq %d ctrl reg, err: %d status: 0x%x, out_size: 0x%x\n",
                        eq->q_id, err, ceq_ctrl.status, out_size);
                return -EFAULT;
        }

        return 0;
}

/**
 * set_eq_ctrls - setting eq's ctrl registers
 * @eq: the Event Queue for setting
 **/
static int set_eq_ctrls(struct hinic_eq *eq)
{
        if (HINIC_IS_VF(eq->hwif) && eq->type == HINIC_CEQ)
                return set_ceq_ctrl_reg(eq);

        set_ctrl0(eq);
        set_ctrl1(eq);
        return 0;
}

/**
 * aeq_elements_init - initialize all the elements in the aeq
 * @eq: the Async Event Queue
 * @init_val: value to initialize the elements with it
 **/
static void aeq_elements_init(struct hinic_eq *eq, u32 init_val)
{
        struct hinic_aeq_elem *aeqe;
        int i;

        for (i = 0; i < eq->q_len; i++) {
                aeqe = GET_AEQ_ELEM(eq, i);
                aeqe->desc = cpu_to_be32(init_val);
        }

        wmb();  /* Write the initilzation values */
}

/**
 * ceq_elements_init - Initialize all the elements in the ceq
 * @eq: the event queue
 * @init_val: value to init the elements with
 **/
static void ceq_elements_init(struct hinic_eq *eq, u32 init_val)
{
        u32 *ceqe;
        int i;

        for (i = 0; i < eq->q_len; i++) {
                ceqe = GET_CEQ_ELEM(eq, i);
                *(ceqe) = cpu_to_be32(init_val);
        }

        wmb();  /* Write the initilzation values */
}

/**
 * alloc_eq_pages - allocate the pages for the queue
 * @eq: the event queue
 *
 * Return 0 - Success, Negative - Failure
 **/
static int alloc_eq_pages(struct hinic_eq *eq)
{
        struct hinic_hwif *hwif = eq->hwif;
        struct pci_dev *pdev = hwif->pdev;
        u32 init_val, addr, val;
        int err, pg;

        eq->dma_addr = devm_kcalloc(&pdev->dev, eq->num_pages,
                                    sizeof(*eq->dma_addr), GFP_KERNEL);
        if (!eq->dma_addr)
                return -ENOMEM;

        eq->virt_addr = devm_kcalloc(&pdev->dev, eq->num_pages,
                                     sizeof(*eq->virt_addr), GFP_KERNEL);
        if (!eq->virt_addr) {
                err = -ENOMEM;
                goto err_virt_addr_alloc;
        }

        for (pg = 0; pg < eq->num_pages; pg++) {
                eq->virt_addr[pg] = dma_alloc_coherent(&pdev->dev,
                                                       eq->page_size,
                                                       &eq->dma_addr[pg],
                                                       GFP_KERNEL);
                if (!eq->virt_addr[pg]) {
                        err = -ENOMEM;
                        goto err_dma_alloc;
                }

                addr = EQ_HI_PHYS_ADDR_REG(eq, pg);
                val = upper_32_bits(eq->dma_addr[pg]);

                hinic_hwif_write_reg(hwif, addr, val);

                addr = EQ_LO_PHYS_ADDR_REG(eq, pg);
                val = lower_32_bits(eq->dma_addr[pg]);

                hinic_hwif_write_reg(hwif, addr, val);
        }

        init_val = HINIC_EQ_ELEM_DESC_SET(eq->wrapped, WRAPPED);

        if (eq->type == HINIC_AEQ)
                aeq_elements_init(eq, init_val);
        else if (eq->type == HINIC_CEQ)
                ceq_elements_init(eq, init_val);

        return 0;

err_dma_alloc:
        while (--pg >= 0)
                dma_free_coherent(&pdev->dev, eq->page_size,
                                  eq->virt_addr[pg],
                                  eq->dma_addr[pg]);

        devm_kfree(&pdev->dev, eq->virt_addr);

err_virt_addr_alloc:
        devm_kfree(&pdev->dev, eq->dma_addr);
        return err;
}

/**
 * free_eq_pages - free the pages of the queue
 * @eq: the Event Queue
 **/
static void free_eq_pages(struct hinic_eq *eq)
{
        struct hinic_hwif *hwif = eq->hwif;
        struct pci_dev *pdev = hwif->pdev;
        int pg;

        for (pg = 0; pg < eq->num_pages; pg++)
                dma_free_coherent(&pdev->dev, eq->page_size,
                                  eq->virt_addr[pg],
                                  eq->dma_addr[pg]);

        devm_kfree(&pdev->dev, eq->virt_addr);
        devm_kfree(&pdev->dev, eq->dma_addr);
}

/**
 * init_eq - initialize Event Queue
 * @eq: the event queue
 * @hwif: the HW interface of a PCI function device
 * @type: the type of the event queue, aeq or ceq
 * @q_id: Queue id number
 * @q_len: the number of EQ elements
 * @page_size: the page size of the pages in the event queue
 * @entry: msix entry associated with the event queue
 *
 * Return 0 - Success, Negative - Failure
 **/
static int init_eq(struct hinic_eq *eq, struct hinic_hwif *hwif,
                   enum hinic_eq_type type, int q_id, u32 q_len, u32 page_size,
                   struct msix_entry entry)
{
        struct pci_dev *pdev = hwif->pdev;
        int err;

        eq->hwif = hwif;
        eq->type = type;
        eq->q_id = q_id;
        eq->q_len = q_len;
        eq->page_size = page_size;

        /* Clear PI and CI, also clear the ARM bit */
        hinic_hwif_write_reg(eq->hwif, EQ_CONS_IDX_REG_ADDR(eq), 0);
        hinic_hwif_write_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq), 0);

        eq->cons_idx = 0;
        eq->wrapped = 0;

        if (type == HINIC_AEQ) {
                eq->elem_size = HINIC_AEQE_SIZE;
        } else if (type == HINIC_CEQ) {
                eq->elem_size = HINIC_CEQE_SIZE;
        } else {
                dev_err(&pdev->dev, "Invalid EQ type\n");
                return -EINVAL;
        }

        eq->num_pages = GET_EQ_NUM_PAGES(eq, page_size);
        eq->num_elem_in_pg = GET_EQ_NUM_ELEMS_IN_PG(eq, page_size);

        eq->msix_entry = entry;

        if (eq->num_elem_in_pg & (eq->num_elem_in_pg - 1)) {
                dev_err(&pdev->dev, "num elements in eq page != power of 2\n");
                return -EINVAL;
        }

        if (eq->num_pages > EQ_MAX_PAGES) {
                dev_err(&pdev->dev, "too many pages for eq\n");
                return -EINVAL;
        }

        err = set_eq_ctrls(eq);
        if (err) {
                dev_err(&pdev->dev, "Failed to set eq ctrls\n");
                return err;
        }

        eq_update_ci(eq, EQ_ARMED);

        err = alloc_eq_pages(eq);
        if (err) {
                dev_err(&pdev->dev, "Failed to allocate pages for eq\n");
                return err;
        }

        if (type == HINIC_AEQ) {
                struct hinic_eq_work *aeq_work = &eq->aeq_work;

                INIT_WORK(&aeq_work->work, eq_irq_work);
        } else if (type == HINIC_CEQ) {
                tasklet_setup(&eq->ceq_tasklet, ceq_tasklet);
        }

        /* set the attributes of the msix entry */
        hinic_msix_attr_set(eq->hwif, eq->msix_entry.entry,
                            HINIC_EQ_MSIX_PENDING_LIMIT_DEFAULT,
                            HINIC_EQ_MSIX_COALESC_TIMER_DEFAULT,
                            HINIC_EQ_MSIX_LLI_TIMER_DEFAULT,
                            HINIC_EQ_MSIX_LLI_CREDIT_LIMIT_DEFAULT,
                            HINIC_EQ_MSIX_RESEND_TIMER_DEFAULT);

        if (type == HINIC_AEQ) {
                snprintf(eq->irq_name, sizeof(eq->irq_name), "hinic_aeq%d@pci:%s", eq->q_id,
                         pci_name(pdev));
                err = request_irq(entry.vector, aeq_interrupt, 0, eq->irq_name, eq);
        } else if (type == HINIC_CEQ) {
                snprintf(eq->irq_name, sizeof(eq->irq_name), "hinic_ceq%d@pci:%s", eq->q_id,
                         pci_name(pdev));
                err = request_irq(entry.vector, ceq_interrupt, 0, eq->irq_name, eq);
        }

        if (err) {
                dev_err(&pdev->dev, "Failed to request irq for the EQ\n");
                goto err_req_irq;
        }

        return 0;

err_req_irq:
        free_eq_pages(eq);
        return err;
}

/**
 * remove_eq - remove Event Queue
 * @eq: the event queue
 **/
static void remove_eq(struct hinic_eq *eq)
{
        hinic_set_msix_state(eq->hwif, eq->msix_entry.entry,
                             HINIC_MSIX_DISABLE);
        free_irq(eq->msix_entry.vector, eq);

        if (eq->type == HINIC_AEQ) {
                struct hinic_eq_work *aeq_work = &eq->aeq_work;

                cancel_work_sync(&aeq_work->work);
                /* clear aeq_len to avoid hw access host memory */
                hinic_hwif_write_reg(eq->hwif,
                                     HINIC_CSR_AEQ_CTRL_1_ADDR(eq->q_id), 0);
        } else if (eq->type == HINIC_CEQ) {
                tasklet_kill(&eq->ceq_tasklet);
                /* clear ceq_len to avoid hw access host memory */
                hinic_hwif_write_reg(eq->hwif,
                                     HINIC_CSR_CEQ_CTRL_1_ADDR(eq->q_id), 0);
        }

        /* update cons_idx to avoid invalid interrupt */
        eq->cons_idx = hinic_hwif_read_reg(eq->hwif, EQ_PROD_IDX_REG_ADDR(eq));
        eq_update_ci(eq, EQ_NOT_ARMED);

        free_eq_pages(eq);
}

/**
 * hinic_aeqs_init - initialize all the aeqs
 * @aeqs: pointer to Async eqs of the chip
 * @hwif: the HW interface of a PCI function device
 * @num_aeqs: number of AEQs
 * @q_len: number of EQ elements
 * @page_size: the page size of the pages in the event queue
 * @msix_entries: msix entries associated with the event queues
 *
 * Return 0 - Success, negative - Failure
 **/
int hinic_aeqs_init(struct hinic_aeqs *aeqs, struct hinic_hwif *hwif,
                    int num_aeqs, u32 q_len, u32 page_size,
                    struct msix_entry *msix_entries)
{
        struct pci_dev *pdev = hwif->pdev;
        int err, i, q_id;

        aeqs->workq = create_singlethread_workqueue(HINIC_EQS_WQ_NAME);
        if (!aeqs->workq)
                return -ENOMEM;

        aeqs->hwif = hwif;
        aeqs->num_aeqs = num_aeqs;

        for (q_id = 0; q_id < num_aeqs; q_id++) {
                err = init_eq(&aeqs->aeq[q_id], hwif, HINIC_AEQ, q_id, q_len,
                              page_size, msix_entries[q_id]);
                if (err) {
                        dev_err(&pdev->dev, "Failed to init aeq %d\n", q_id);
                        goto err_init_aeq;
                }
        }

        return 0;

err_init_aeq:
        for (i = 0; i < q_id; i++)
                remove_eq(&aeqs->aeq[i]);

        destroy_workqueue(aeqs->workq);
        return err;
}

/**
 * hinic_aeqs_free - free all the aeqs
 * @aeqs: pointer to Async eqs of the chip
 **/
void hinic_aeqs_free(struct hinic_aeqs *aeqs)
{
        int q_id;

        for (q_id = 0; q_id < aeqs->num_aeqs ; q_id++)
                remove_eq(&aeqs->aeq[q_id]);

        destroy_workqueue(aeqs->workq);
}

/**
 * hinic_ceqs_init - init all the ceqs
 * @ceqs: ceqs part of the chip
 * @hwif: the hardware interface of a pci function device
 * @num_ceqs: number of CEQs
 * @q_len: number of EQ elements
 * @page_size: the page size of the event queue
 * @msix_entries: msix entries associated with the event queues
 *
 * Return 0 - Success, Negative - Failure
 **/
int hinic_ceqs_init(struct hinic_ceqs *ceqs, struct hinic_hwif *hwif,
                    int num_ceqs, u32 q_len, u32 page_size,
                    struct msix_entry *msix_entries)
{
        struct pci_dev *pdev = hwif->pdev;
        int i, q_id, err;

        ceqs->hwif = hwif;
        ceqs->num_ceqs = num_ceqs;

        for (q_id = 0; q_id < num_ceqs; q_id++) {
                ceqs->ceq[q_id].hwdev = ceqs->hwdev;
                err = init_eq(&ceqs->ceq[q_id], hwif, HINIC_CEQ, q_id, q_len,
                              page_size, msix_entries[q_id]);
                if (err) {
                        dev_err(&pdev->dev, "Failed to init ceq %d\n", q_id);
                        goto err_init_ceq;
                }
        }

        return 0;

err_init_ceq:
        for (i = 0; i < q_id; i++)
                remove_eq(&ceqs->ceq[i]);

        return err;
}

/**
 * hinic_ceqs_free - free all the ceqs
 * @ceqs: ceqs part of the chip
 **/
void hinic_ceqs_free(struct hinic_ceqs *ceqs)
{
        int q_id;

        for (q_id = 0; q_id < ceqs->num_ceqs; q_id++)
                remove_eq(&ceqs->ceq[q_id]);
}

void hinic_dump_ceq_info(struct hinic_hwdev *hwdev)
{
        struct hinic_eq *eq = NULL;
        u32 addr, ci, pi;
        int q_id;

        for (q_id = 0; q_id < hwdev->func_to_io.ceqs.num_ceqs; q_id++) {
                eq = &hwdev->func_to_io.ceqs.ceq[q_id];
                addr = EQ_CONS_IDX_REG_ADDR(eq);
                ci = hinic_hwif_read_reg(hwdev->hwif, addr);
                addr = EQ_PROD_IDX_REG_ADDR(eq);
                pi = hinic_hwif_read_reg(hwdev->hwif, addr);
                dev_err(&hwdev->hwif->pdev->dev, "Ceq id: %d, ci: 0x%08x, sw_ci: 0x%08x, pi: 0x%x, tasklet_state: 0x%lx, wrap: %d, ceqe: 0x%x\n",
                        q_id, ci, eq->cons_idx, pi,
                        eq->ceq_tasklet.state,
                        eq->wrapped, be32_to_cpu(*(__be32 *)(GET_CURR_CEQ_ELEM(eq))));
        }
}

void hinic_dump_aeq_info(struct hinic_hwdev *hwdev)
{
        struct hinic_aeq_elem *aeqe_pos = NULL;
        struct hinic_eq *eq = NULL;
        u32 addr, ci, pi;
        int q_id;

        for (q_id = 0; q_id < hwdev->aeqs.num_aeqs; q_id++) {
                eq = &hwdev->aeqs.aeq[q_id];
                addr = EQ_CONS_IDX_REG_ADDR(eq);
                ci = hinic_hwif_read_reg(hwdev->hwif, addr);
                addr = EQ_PROD_IDX_REG_ADDR(eq);
                pi = hinic_hwif_read_reg(hwdev->hwif, addr);
                aeqe_pos = GET_CURR_AEQ_ELEM(eq);
                dev_err(&hwdev->hwif->pdev->dev, "Aeq id: %d, ci: 0x%08x, pi: 0x%x, work_state: 0x%x, wrap: %d, desc: 0x%x\n",
                        q_id, ci, pi, work_busy(&eq->aeq_work.work),
                        eq->wrapped, be32_to_cpu(aeqe_pos->desc));
        }
}