/*
 * Copyright (c) 2015-2024, Broadcom. All rights reserved.  The term
 * Broadcom refers to Broadcom Limited and/or its subsidiaries.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * Description: main (header)
 */

#ifndef __BNXT_RE_H__
#define __BNXT_RE_H__

#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/mutex.h>
#include <linux/list.h>
#include <linux/rculist.h>
#include <linux/spinlock.h>
#include <net/ipv6.h>
#include <linux/if_ether.h>
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/interrupt.h>
#include <linux/vmalloc.h>
#include <linux/delay.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_mad.h>
#include <rdma/ib_cache.h>
#include <linux/pci.h>

#include "bnxt.h"
#include "bnxt_ulp.h"
#include "hsi_struct_def.h"
#include "qplib_res.h"
#include "qplib_sp.h"
#include "qplib_fp.h"
#include "qplib_rcfw.h"
#include "ib_verbs.h"
#include "stats.h"

#define ROCE_DRV_MODULE_NAME            "bnxt_re"
#define ROCE_DRV_MODULE_VERSION         "230.0.133.0"
#define ROCE_DRV_MODULE_RELDATE         "April 22, 2024"

#define BNXT_RE_REF_WAIT_COUNT          20
#define BNXT_RE_ROCE_V1_ETH_TYPE        0x8915
#define BNXT_RE_ROCE_V2_PORT_NO         4791
#define BNXT_RE_RES_FREE_WAIT_COUNT     1000

#define BNXT_RE_PAGE_SHIFT_4K           (12)
#define BNXT_RE_PAGE_SHIFT_8K           (13)
#define BNXT_RE_PAGE_SHIFT_64K          (16)
#define BNXT_RE_PAGE_SHIFT_2M           (21)
#define BNXT_RE_PAGE_SHIFT_8M           (23)
#define BNXT_RE_PAGE_SHIFT_1G           (30)

#define BNXT_RE_PAGE_SIZE_4K            BIT(BNXT_RE_PAGE_SHIFT_4K)
#define BNXT_RE_PAGE_SIZE_8K            BIT(BNXT_RE_PAGE_SHIFT_8K)
#define BNXT_RE_PAGE_SIZE_64K           BIT(BNXT_RE_PAGE_SHIFT_64K)
#define BNXT_RE_PAGE_SIZE_2M            BIT(BNXT_RE_PAGE_SHIFT_2M)
#define BNXT_RE_PAGE_SIZE_8M            BIT(BNXT_RE_PAGE_SHIFT_8M)
#define BNXT_RE_PAGE_SIZE_1G            BIT(BNXT_RE_PAGE_SHIFT_1G)

#define BNXT_RE_MAX_MR_SIZE_LOW         BIT(BNXT_RE_PAGE_SHIFT_1G)
#define BNXT_RE_MAX_MR_SIZE_HIGH        BIT(39)
#define BNXT_RE_MAX_MR_SIZE             BNXT_RE_MAX_MR_SIZE_HIGH

/* Number of MRs to reserve for PF, leaving remainder for VFs */
#define BNXT_RE_RESVD_MR_FOR_PF         (32 * 1024)
#define BNXT_RE_MAX_GID_PER_VF          128

#define BNXT_RE_MAX_VF_QPS_PER_PF       (6 * 1024)

/**
 * min_not_zero - return the minimum that is _not_ zero, unless both are zero
 * @x: value1
 * @y: value2
 */
#ifndef min_not_zero
#define min_not_zero(x, y) ({                   \
        typeof(x) __x = (x);                    \
        typeof(y) __y = (y);                    \
        __x == 0 ? __y : ((__y == 0) ? __x : min(__x, __y)); })
#endif

struct ib_mr_init_attr {
        int             max_reg_descriptors;
        u32             flags;
};

struct bnxt_re_dev;

int bnxt_re_register_netdevice_notifier(struct notifier_block *nb);
int bnxt_re_unregister_netdevice_notifier(struct notifier_block *nb);
int ib_register_device_compat(struct bnxt_re_dev *rdev);

#ifndef __struct_group
#define __struct_group(TAG, NAME, ATTRS, MEMBERS...) \
        union { \
                struct { MEMBERS } ATTRS; \
                struct TAG { MEMBERS } ATTRS NAME; \
        }
#endif /* __struct_group */
#ifndef struct_group_attr
#define struct_group_attr(NAME, ATTRS, MEMBERS...) \
        __struct_group(/* no tag */, NAME, ATTRS, MEMBERS)
#endif /* struct_group_attr */
/*
 * Percentage of resources of each type reserved for PF.
 * Remaining resources are divided equally among VFs.
 * [0, 100]
 */

#define BNXT_RE_RQ_WQE_THRESHOLD        32
#define BNXT_RE_UD_QP_HW_STALL          0x400000

/*
 * Setting the default ack delay value to 16, which means
 * the default timeout is approx. 260ms(4 usec * 2 ^(timeout))
 */

#define BNXT_RE_DEFAULT_ACK_DELAY       16
#define BNXT_RE_BOND_PF_MAX             2

#define BNXT_RE_STATS_CTX_UPDATE_TIMER  250
#define BNXT_RE_30SEC_MSEC              (30 * 1000)

#define BNXT_RE_BOND_RESCHED_CNT        10

#define BNXT_RE_CHIP_NUM_57454         0xC454
#define BNXT_RE_CHIP_NUM_57452         0xC452

#define BNXT_RE_CHIP_NUM_5745X(chip_num)          \
        ((chip_num) == BNXT_RE_CHIP_NUM_57454 ||       \
         (chip_num) == BNXT_RE_CHIP_NUM_57452)

#define BNXT_RE_MIN_KERNEL_QP_TX_DEPTH  4096
#define BNXT_RE_STOP_QPS_BUDGET         200

#define BNXT_RE_HWRM_CMD_TIMEOUT(rdev) \
                ((rdev)->chip_ctx->hwrm_cmd_max_timeout * 1000)

extern unsigned int min_tx_depth;
extern struct mutex bnxt_re_dev_lock;
extern struct mutex bnxt_re_mutex;
extern struct list_head bnxt_re_dev_list;

struct bnxt_re_ring_attr {
        dma_addr_t      *dma_arr;
        int             pages;
        int             type;
        u32             depth;
        u32             lrid; /* Logical ring id */
        u16             flags;
        u8              mode;
        u8              rsvd;
};

#define BNXT_RE_MAX_DEVICES             256
#define BNXT_RE_MSIX_FROM_MOD_PARAM     -1
#define BNXT_RE_MIN_MSIX                2
#define BNXT_RE_MAX_MSIX_VF             2
#define BNXT_RE_MAX_MSIX_PF             9
#define BNXT_RE_MAX_MSIX_NPAR_PF        5
#define BNXT_RE_MAX_MSIX                64
#define BNXT_RE_MAX_MSIX_GEN_P5_PF      BNXT_RE_MAX_MSIX
#define BNXT_RE_GEN_P5_MAX_VF           64

struct bnxt_re_nq_record {
        struct bnxt_msix_entry  msix_entries[BNXT_RE_MAX_MSIX];
        /* FP Notification Queue (CQ & SRQ) */
        struct bnxt_qplib_nq    nq[BNXT_RE_MAX_MSIX];
        int                     num_msix;
        int                     max_init;
        struct mutex            load_lock;
};

struct bnxt_re_work {
        struct work_struct      work;
        unsigned long           event;
        struct bnxt_re_dev      *rdev;
        struct ifnet            *vlan_dev;
        bool do_lag;

        /* netdev where we received the event */
        struct ifnet *netdev;
        struct auxiliary_device *adev;
};

/*
 * Data structure and defines to handle
 * recovery
 */
#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_RETRY   20
#define BNXT_RE_RECOVERY_IB_UNINIT_WAIT_TIME_MS 30000 /* 30sec timeout */
#define BNXT_RE_PRE_RECOVERY_REMOVE 0x1
#define BNXT_RE_COMPLETE_REMOVE 0x2
#define BNXT_RE_POST_RECOVERY_INIT 0x4
#define BNXT_RE_COMPLETE_INIT 0x8
#define BNXT_RE_COMPLETE_SHUTDOWN 0x10

/* QP1 SQ entry data strucutre */
struct bnxt_re_sqp_entries {
        u64 wrid;
        struct bnxt_qplib_sge sge;
        /* For storing the actual qp1 cqe */
        struct bnxt_qplib_cqe cqe;
        struct bnxt_re_qp *qp1_qp;
};

/* GSI QP mode enum */
enum bnxt_re_gsi_mode {
        BNXT_RE_GSI_MODE_INVALID = 0,
        BNXT_RE_GSI_MODE_ALL = 1,
        BNXT_RE_GSI_MODE_ROCE_V1,
        BNXT_RE_GSI_MODE_ROCE_V2_IPV4,
        BNXT_RE_GSI_MODE_ROCE_V2_IPV6,
        BNXT_RE_GSI_MODE_UD
};

enum bnxt_re_roce_cap {
        BNXT_RE_FLAG_ROCEV1_CAP = 1,
        BNXT_RE_FLAG_ROCEV2_CAP,
        BNXT_RE_FLAG_ROCEV1_V2_CAP,
};

#define BNXT_RE_MAX_GSI_SQP_ENTRIES     1024
struct bnxt_re_gsi_context {
        u8      gsi_qp_mode;
        bool    first_cq_created;
        /* Start: used only in gsi_mode_all */
        struct  bnxt_re_qp *gsi_qp;
        struct  bnxt_re_qp *gsi_sqp;
        struct  bnxt_re_ah *gsi_sah;
        struct  bnxt_re_sqp_entries *sqp_tbl;
        /* End: used only in gsi_mode_all */
};

struct bnxt_re_tc_rec {
        u8 cos_id_roce;
        u8 tc_roce;
        u8 cos_id_cnp;
        u8 tc_cnp;
        u8 tc_def;
        u8 cos_id_def;
        u8 max_tc;
        u8 roce_prio;
        u8 cnp_prio;
        u8 roce_dscp;
        u8 cnp_dscp;
        u8 prio_valid;
        u8 dscp_valid;
        bool ecn_enabled;
        bool serv_type_enabled;
        u64 cnp_dscp_bv;
        u64 roce_dscp_bv;
};

struct bnxt_re_dscp2pri {
        u8 dscp;
        u8 mask;
        u8 pri;
};

struct bnxt_re_cos2bw_cfg {
        u8      pad[3];
        struct_group_attr(cfg, __packed,
                u8      queue_id;
                __le32  min_bw;
                __le32  max_bw;
                u8      tsa;
                u8      pri_lvl;
                u8      bw_weight;
        );
        u8      unused;
};

#define BNXT_RE_AEQ_IDX                 0
#define BNXT_RE_MAX_SGID_ENTRIES        256

#define BNXT_RE_DBGFS_FILE_MEM          65536
enum {
        BNXT_RE_STATS_QUERY = 1,
        BNXT_RE_QP_QUERY = 2,
        BNXT_RE_SERVICE_FN_QUERY = 3,
};

struct bnxt_re_dbg_file {
        struct bnxt_re_dev *rdev;
        u32 type;
        union {
                struct bnxt_qplib_query_stats_info sinfo;
                struct bnxt_qplib_query_fn_info fninfo;
        }params;
        char dbg_buf[BNXT_RE_DBGFS_FILE_MEM];
};

struct bnxt_re_debug_entries {
        /* Dir entries */
        struct dentry *qpinfo_dir;
        struct dentry *service_fn_dir;
        /* file entries */
        struct dentry *stat_query;
        struct bnxt_re_dbg_file stat_file;
        struct dentry *qplist_query;
        struct bnxt_re_dbg_file qp_file;
        struct dentry *service_fn_query;
        struct bnxt_re_dbg_file service_fn_file;
};

struct bnxt_re_en_dev_info {
        struct list_head en_list;
        struct bnxt_en_dev *en_dev;
        struct bnxt_re_dev *rdev;
        unsigned long flags;
#define BNXT_RE_FLAG_EN_DEV_NETDEV_REG          0
#define BNXT_RE_FLAG_EN_DEV_PRIMARY_DEV         1
#define BNXT_RE_FLAG_EN_DEV_SECONDARY_DEV       2
        u8 wqe_mode;
        u8 gsi_mode;
        bool te_bypass;
        bool ib_uninit_done;
        u32 num_msix_requested;
        wait_queue_head_t waitq;
};

#define BNXT_RE_DB_FIFO_ROOM_MASK_P5    0x1FFF8000
#define BNXT_RE_MAX_FIFO_DEPTH_P5       0x2c00
#define BNXT_RE_DB_FIFO_ROOM_SHIFT      15

#define BNXT_RE_DB_FIFO_ROOM_MASK_P7    0x3FFF8000
#define BNXT_RE_MAX_FIFO_DEPTH_P7       0x8000

#define BNXT_RE_DB_FIFO_ROOM_MASK(ctx)  \
        (_is_chip_p7((ctx)) ? \
         BNXT_RE_DB_FIFO_ROOM_MASK_P7 :\
         BNXT_RE_DB_FIFO_ROOM_MASK_P5)
#define BNXT_RE_MAX_FIFO_DEPTH(ctx)     \
        (_is_chip_p7((ctx)) ? \
         BNXT_RE_MAX_FIFO_DEPTH_P7 :\
         BNXT_RE_MAX_FIFO_DEPTH_P5)

struct bnxt_dbq_nq_list {
        int num_nql_entries;
        u16 nq_id[16];
};

#define BNXT_RE_ASYNC_ERR_REP_BASE(_type)                               \
         (ASYNC_EVENT_CMPL_ERROR_REPORT_BASE_EVENT_DATA1_ERROR_##_type)

#define BNXT_RE_ASYNC_ERR_DBR_TRESH(_type)                              \
         (ASYNC_EVENT_CMPL_ERROR_REPORT_DOORBELL_DROP_THRESHOLD_##_type)

#define BNXT_RE_EVENT_DBR_EPOCH(data)                                   \
        (((data) &                                                      \
          BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_MASK)) >>       \
         BNXT_RE_ASYNC_ERR_DBR_TRESH(EVENT_DATA1_EPOCH_SFT))

#define BNXT_RE_EVENT_ERROR_REPORT_TYPE(data1)                          \
        (((data1) &                                                     \
          BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_MASK))  >>                    \
         BNXT_RE_ASYNC_ERR_REP_BASE(TYPE_SFT))

#define BNXT_RE_DBR_LIST_ADD(_rdev, _res, _type)                        \
{                                                                       \
        spin_lock(&(_rdev)->res_list[_type].lock);                      \
        list_add_tail(&(_res)->dbr_list,                                \
                      &(_rdev)->res_list[_type].head);                  \
        spin_unlock(&(_rdev)->res_list[_type].lock);                    \
}

#define BNXT_RE_DBR_LIST_DEL(_rdev, _res, _type)                        \
{                                                                       \
        spin_lock(&(_rdev)->res_list[_type].lock);                      \
        list_del(&(_res)->dbr_list);                                    \
        spin_unlock(&(_rdev)->res_list[_type].lock);                    \
}

#define BNXT_RE_CQ_PAGE_LIST_ADD(_uctx, _cq)                            \
{                                                                       \
        mutex_lock(&(_uctx)->cq_lock);                                  \
        list_add_tail(&(_cq)->cq_list, &(_uctx)->cq_list);              \
        mutex_unlock(&(_uctx)->cq_lock);                                \
}

#define BNXT_RE_CQ_PAGE_LIST_DEL(_uctx, _cq)                            \
{                                                                       \
        mutex_lock(&(_uctx)->cq_lock);                                  \
        list_del(&(_cq)->cq_list);                                      \
        mutex_unlock(&(_uctx)->cq_lock);                                \
}

#define BNXT_RE_NETDEV_EVENT(event, x)                                  \
        do {                                                            \
                if ((event) == (x))                                     \
                        return #x;                                      \
        } while (0)

/* Do not change the seq of this enum which is followed by dbr recov */
enum {
        BNXT_RE_RES_TYPE_CQ = 0,
        BNXT_RE_RES_TYPE_UCTX,
        BNXT_RE_RES_TYPE_QP,
        BNXT_RE_RES_TYPE_SRQ,
        BNXT_RE_RES_TYPE_MAX
};

struct bnxt_re_dbr_res_list {
        struct list_head head;
        spinlock_t lock;
};

struct bnxt_re_dbr_drop_recov_work {
        struct work_struct work;
        struct bnxt_re_dev *rdev;
        u32 curr_epoch;
};

struct bnxt_re_aer_work {
        struct work_struct work;
        struct bnxt_re_dev *rdev;
};

struct bnxt_re_dbq_stats {
        u64 fifo_occup_slab_1;
        u64 fifo_occup_slab_2;
        u64 fifo_occup_slab_3;
        u64 fifo_occup_slab_4;
        u64 fifo_occup_water_mark;
        u64 do_pacing_slab_1;
        u64 do_pacing_slab_2;
        u64 do_pacing_slab_3;
        u64 do_pacing_slab_4;
        u64 do_pacing_slab_5;
        u64 do_pacing_water_mark;
};

/* Device debug statistics */
struct bnxt_re_drv_dbg_stats {
        struct bnxt_re_dbq_stats dbq;
};

/* DB pacing counters */
struct bnxt_re_dbr_sw_stats {
        u64 dbq_int_recv;
        u64 dbq_int_en;
        u64 dbq_pacing_resched;
        u64 dbq_pacing_complete;
        u64 dbq_pacing_alerts;
        u64 dbr_drop_recov_events;
        u64 dbr_drop_recov_timeouts;
        u64 dbr_drop_recov_timeout_users;
        u64 dbr_drop_recov_event_skips;
};

struct bnxt_re_dev {
        struct ib_device                ibdev;
        struct list_head                list;
        atomic_t                        ref_count;
        atomic_t                        sched_count;
        unsigned long                   flags;
#define BNXT_RE_FLAG_NETDEV_REGISTERED          0
#define BNXT_RE_FLAG_IBDEV_REGISTERED           1
#define BNXT_RE_FLAG_GOT_MSIX                   2
#define BNXT_RE_FLAG_HAVE_L2_REF                3
#define BNXT_RE_FLAG_ALLOC_RCFW                 4
#define BNXT_RE_FLAG_NET_RING_ALLOC             5
#define BNXT_RE_FLAG_RCFW_CHANNEL_EN            6
#define BNXT_RE_FLAG_ALLOC_CTX                  7
#define BNXT_RE_FLAG_STATS_CTX_ALLOC            8
#define BNXT_RE_FLAG_STATS_CTX2_ALLOC           9
#define BNXT_RE_FLAG_RCFW_CHANNEL_INIT          10
#define BNXT_RE_FLAG_WORKER_REG                 11
#define BNXT_RE_FLAG_TBLS_ALLOCINIT             12
#define BNXT_RE_FLAG_SETUP_NQ                   13
#define BNXT_RE_FLAG_BOND_DEV_REGISTERED        14
#define BNXT_RE_FLAG_PER_PORT_DEBUG_INFO        15
#define BNXT_RE_FLAG_DEV_LIST_INITIALIZED       16
#define BNXT_RE_FLAG_ERR_DEVICE_DETACHED        17
#define BNXT_RE_FLAG_INIT_DCBX_CC_PARAM         18
#define BNXT_RE_FLAG_STOP_IN_PROGRESS           20
#define BNXT_RE_FLAG_ISSUE_ROCE_STATS           29
#define BNXT_RE_FLAG_ISSUE_CFA_FLOW_STATS       30
        struct ifnet                    *netdev;
        struct auxiliary_device         *adev;
        struct bnxt_qplib_chip_ctx      *chip_ctx;
        struct bnxt_en_dev              *en_dev;
        struct bnxt_re_nq_record        nqr;
        int                             id;
        struct delayed_work             worker;
        u16                             worker_30s;
        struct bnxt_re_tc_rec           tc_rec[2];
        u8                              cur_prio_map;
        /* RCFW Channel */
        struct bnxt_qplib_rcfw          rcfw;
        /* Device Resources */
        struct bnxt_qplib_dev_attr      *dev_attr;
        struct bnxt_qplib_res           qplib_res;
        struct bnxt_qplib_dpi           dpi_privileged;
        struct bnxt_qplib_cc_param      cc_param;
        struct mutex                    cc_lock;
        struct mutex                    qp_lock;
        struct list_head                qp_list;
        u8                              roce_mode;

        /* Max of 2 lossless traffic class supported per port */
        u16                             cosq[2];
        /* Start: QP for handling QP1 packets */
        struct bnxt_re_gsi_context      gsi_ctx;
        /* End: QP for handling QP1 packets */
        bool                            is_virtfn;
        u32                             num_vfs;
        u32                             espeed;
        u8                              lanes;
        /*
         * For storing the speed of slave interfaces.
         * Same as espeed when bond is not configured
         */
        u32                             sl_espeed;
        /* To be used for a workaround for ISER stack */
        u32                             min_tx_depth;
        /* To enable qp debug info. Disabled during driver load */
        u32                             en_qp_dbg;
        /* Array to handle gid mapping */
        char                            *gid_map;

        struct bnxt_re_device_stats     stats;
        struct bnxt_re_drv_dbg_stats    *dbg_stats;
        /* debugfs to expose per port information*/
        struct dentry                   *port_debug_dir;
        struct dentry                   *info;
        struct dentry                   *drv_dbg_stats;
        struct dentry                   *sp_perf_stats;
        struct dentry                   *pdev_debug_dir;
        struct dentry                   *pdev_qpinfo_dir;
        struct bnxt_re_debug_entries    *dbg_ent;
        struct workqueue_struct         *dcb_wq;
        struct workqueue_struct         *aer_wq;
        u32                             event_bitmap[3];
        bool unreg_sched;
        u64     dbr_throttling_reg_off;
        u64     dbr_aeq_arm_reg_off;
        u64     dbr_db_fifo_reg_off;
        void *dbr_page;
        u64 dbr_bar_addr;
        u32 pacing_algo_th;
        u32 pacing_en_int_th;
        u32 do_pacing_save;
        struct workqueue_struct         *dbq_wq;
        struct workqueue_struct         *dbr_drop_recov_wq;
        struct work_struct              dbq_fifo_check_work;
        struct delayed_work             dbq_pacing_work;
        /* protect DB pacing */
        struct mutex dbq_lock;
        /* Control DBR pacing feature. Set if enabled */
        bool dbr_pacing;
        /* Control DBR recovery feature. Set if enabled */
        bool dbr_drop_recov;
        bool user_dbr_drop_recov;
        /* DBR recovery feature. Set if running */
        bool dbr_recovery_on;
        u32 user_dbr_drop_recov_timeout;
         /*
          * Value used for pacing algo when pacing is active
          */
#define BNXT_RE_MAX_DBR_DO_PACING 0xFFFF
        u32 dbr_do_pacing;
        u32 dbq_watermark; /* Current watermark set in HW registers */
        u32 dbq_nq_id; /* Current NQ ID for DBQ events */
        u32 dbq_pacing_time; /* ms */
        u32 dbr_def_do_pacing; /* do_pacing when no congestion */
        u32 dbr_evt_curr_epoch;
        bool dbq_int_disable;

        bool mod_exit;
        struct bnxt_re_dbr_sw_stats *dbr_sw_stats;
        struct bnxt_re_dbr_res_list res_list[BNXT_RE_RES_TYPE_MAX];
        struct bnxt_dbq_nq_list nq_list;
        char dev_name[IB_DEVICE_NAME_MAX];
        atomic_t dbq_intr_running;
        u32 num_msix_requested;
        unsigned char   *dev_addr; /* For netdev->dev_addr */
};

static inline u8 bnxt_re_get_prio(u8 prio_map)
{
        u8 prio = 0xFF;

        for (prio = 0; prio < 8; prio++)
                if (prio_map & (1UL << prio))
                        break;
        return prio;
}

/* This should be called with bnxt_re_dev_lock mutex held */
static inline bool __bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
{
        struct bnxt_re_dev *tmp_rdev;

        list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
                if (rdev == tmp_rdev)
                        return true;
        }
        return false;
}

static inline bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev)
{
        struct bnxt_re_dev *tmp_rdev;

        mutex_lock(&bnxt_re_dev_lock);
        list_for_each_entry(tmp_rdev, &bnxt_re_dev_list, list) {
                if (rdev == tmp_rdev) {
                        mutex_unlock(&bnxt_re_dev_lock);
                        return true;
                }
        }
        mutex_unlock(&bnxt_re_dev_lock);

        pr_debug("bnxt_re: %s : Invalid rdev received rdev = %p\n",
                 __func__, rdev);
        return false;
}

int bnxt_re_send_hwrm_cmd(struct bnxt_re_dev *rdev, void *cmd,
                          int cmdlen);
void bnxt_re_stopqps_and_ib_uninit(struct bnxt_re_dev *rdev);
int bnxt_re_set_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
                              struct bnxt_re_dscp2pri *d2p, u16 count,
                              u16 target_id);
int bnxt_re_query_hwrm_dscp2pri(struct bnxt_re_dev *rdev,
                                struct bnxt_re_dscp2pri *d2p, u16 *count,
                                u16 target_id);
int bnxt_re_query_hwrm_qportcfg(struct bnxt_re_dev *rdev,
                                struct bnxt_re_tc_rec *cnprec, u16 tid);
int bnxt_re_hwrm_cos2bw_qcfg(struct bnxt_re_dev *rdev, u16 target_id,
                             struct bnxt_re_cos2bw_cfg *cfg);
int bnxt_re_hwrm_cos2bw_cfg(struct bnxt_re_dev *rdev, u16 target_id,
                            struct bnxt_re_cos2bw_cfg *cfg);
int bnxt_re_hwrm_pri2cos_cfg(struct bnxt_re_dev *rdev,
                             u16 target_id, u16 port_id,
                             u8 *cos_id_map, u8 pri_map);
int bnxt_re_prio_vlan_tx_update(struct bnxt_re_dev *rdev);
int bnxt_re_get_slot_pf_count(struct bnxt_re_dev *rdev);
struct bnxt_re_dev *bnxt_re_get_peer_pf(struct bnxt_re_dev *rdev);
struct bnxt_re_dev *bnxt_re_from_netdev(struct ifnet *netdev);
u8 bnxt_re_get_priority_mask(struct bnxt_re_dev *rdev, u8 selector);
struct bnxt_qplib_nq * bnxt_re_get_nq(struct bnxt_re_dev *rdev);
void bnxt_re_put_nq(struct bnxt_re_dev *rdev, struct bnxt_qplib_nq *nq);

#define to_bnxt_re(ptr, type, member)   \
        container_of(ptr, type, member)

#define to_bnxt_re_dev(ptr, member)     \
        container_of((ptr), struct bnxt_re_dev, member)

/* Even number functions from port 0 and odd number from port 1 */
#define BNXT_RE_IS_PORT0(rdev) (!(rdev->en_dev->pdev->devfn & 1))

#define BNXT_RE_ROCE_V1_PACKET          0
#define BNXT_RE_ROCEV2_IPV4_PACKET      2
#define BNXT_RE_ROCEV2_IPV6_PACKET      3
#define BNXT_RE_ACTIVE_MAP_PORT1    0x1  /*port-1 active */
#define BNXT_RE_ACTIVE_MAP_PORT2    0x2  /*port-2 active */

#define BNXT_RE_MEMBER_PORT_MAP         (BNXT_RE_ACTIVE_MAP_PORT1 | \
                                        BNXT_RE_ACTIVE_MAP_PORT2)

#define rdev_to_dev(rdev)       ((rdev) ? (&(rdev)->ibdev.dev) : NULL)

void bnxt_re_set_dma_device(struct ib_device *ibdev, struct bnxt_re_dev *rdev);
bool bnxt_re_is_rdev_valid(struct bnxt_re_dev *rdev);

#define bnxt_re_rdev_ready(rdev)        (bnxt_re_is_rdev_valid(rdev) && \
                                         (test_bit(BNXT_RE_FLAG_IBDEV_REGISTERED, &rdev->flags)))
#define BNXT_RE_SRIOV_CFG_TIMEOUT 6

int bnxt_re_get_device_stats(struct bnxt_re_dev *rdev);
void bnxt_re_remove_device(struct bnxt_re_dev *rdev, u8 removal_type,
                           struct auxiliary_device *aux_dev);
void bnxt_re_destroy_lag(struct bnxt_re_dev **rdev);
int bnxt_re_add_device(struct bnxt_re_dev **rdev,
                       struct ifnet *netdev,
                       u8 qp_mode, u8 op_type, u32 num_msix_requested,
                       struct auxiliary_device *aux_dev);
void bnxt_re_create_base_interface(bool primary);
int bnxt_re_schedule_work(struct bnxt_re_dev *rdev, unsigned long event,
                          struct ifnet *vlan_dev,
                          struct ifnet *netdev,
                          struct auxiliary_device *aux_dev);
void bnxt_re_get_link_speed(struct bnxt_re_dev *rdev);
int _bnxt_re_ib_init(struct bnxt_re_dev *rdev);
int _bnxt_re_ib_init2(struct bnxt_re_dev *rdev);

/* The rdev ref_count is to protect immature removal of the device */
static inline void bnxt_re_hold(struct bnxt_re_dev *rdev)
{
        atomic_inc(&rdev->ref_count);
        dev_dbg(rdev_to_dev(rdev),
                "Hold ref_count = 0x%x", atomic_read(&rdev->ref_count));
}

static inline void bnxt_re_put(struct bnxt_re_dev *rdev)
{
        atomic_dec(&rdev->ref_count);
        dev_dbg(rdev_to_dev(rdev),
                "Put ref_count = 0x%x", atomic_read(&rdev->ref_count));
}

/*
* Responder Error reason codes
* FIXME: Remove these when the defs
* are properly included in hsi header
*/
enum res_err_state_reason {
        /* No error. */
        CFCQ_RES_ERR_STATE_REASON_NO_ERROR = 0,
        /*
         * Incoming Send, RDMA write, or RDMA read exceeds the maximum
         * transfer length. Detected on RX first and only packets for
         * write. Detected on RX request for read. This is an RX
         * Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_EXCEED_MAX,
        /*
         * RDMA write payload size does not match write length. Detected
         * when total write payload is not equal to the RDMA write
         * length that was given in the first or only packet of the
         * request. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_PAYLOAD_LENGTH_MISMATCH,
        /*
         * Send payload exceeds RQ/SRQ WQE buffer capacity. The total
         * send payload that arrived is more than the size of the WQE
         * buffer that was fetched from the RQ/SRQ. This is an RX
         * Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_EXCEEDS_WQE,
        /*
         * Responder detected opcode error. * First, only, middle, last
         * for incoming requests are improperly ordered with respect to
         * previous (PSN) packet. * First or middle packet is not full
         * MTU size. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_OPCODE_ERROR,
        /*
         * PSN sequence error retry limit exceeded. The responder
         * encountered a PSN sequence error for the same PSN too many
         * times. This can occur via implicit or explicit NAK. This is
         * an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_PSN_SEQ_ERROR_RETRY_LIMIT,
        /*
         * Invalid R_Key. An incoming request contained an R_Key that
         * did not reference a valid MR/MW. This error may be detected
         * by the RX engine for RDMA write or by the TX engine for RDMA
         * read (detected while servicing IRRQ). This is an RX Detected
         * Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_RX_INVALID_R_KEY,
        /*
         * Domain error. An incoming request specified an R_Key which
         * referenced a MR/MW that was not in the same PD as the QP on
         * which the request arrived. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_RX_DOMAIN_ERROR,
        /*
         * No permission. An incoming request contained an R_Key that
         * referenced a MR/MW which did not have the access permission
         * needed for the operation. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_RX_NO_PERMISSION,
        /*
         * Range error. An incoming request had a combination of R_Key,
         * VA, and length that was out of bounds of the associated
         * MR/MW. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_RX_RANGE_ERROR,
        /*
         * Invalid R_Key. An incoming request contained an R_Key that
         * did not reference a valid MR/MW. This error may be detected
         * by the RX engine for RDMA write or by the TX engine for RDMA
         * read (detected while servicing IRRQ). This is a TX Detected
         * Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_TX_INVALID_R_KEY,
        /*
         * Domain error. An incoming request specified an R_Key which
         * referenced a MR/MW that was not in the same PD as the QP on
         * which the request arrived. This is a TX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_TX_DOMAIN_ERROR,
        /*
         * No permission. An incoming request contained an R_Key that
         * referenced a MR/MW which did not have the access permission
         * needed for the operation. This is a TX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_TX_NO_PERMISSION,
        /*
         * Range error. An incoming request had a combination of R_Key,
         * VA, and length that was out of bounds of the associated
         * MR/MW. This is a TX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_TX_RANGE_ERROR,
        /*
         * IRRQ overflow. The peer sent us more RDMA read or atomic
         * requests than the negotiated maximum. This is an RX Detected
         * Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_IRRQ_OFLOW,
        /*
         * Unsupported opcode. The peer sent us a request with an opcode
         * for a request type that is not supported on this QP. This is
         * an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_UNSUPPORTED_OPCODE,
        /*
         * Unaligned atomic operation. The VA of an atomic request is on
         * a memory boundary that prevents atomic execution. This is an
         * RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_UNALIGN_ATOMIC,
        /*
         * Remote invalidate error. A send with invalidate request
         * arrived in which the R_Key to invalidate did not describe a
         * MR/MW which could be invalidated. RQ WQE completes with error
         * status. This error is only reported if the send operation did
         * not fail. If the send operation failed then the remote
         * invalidate error is not reported. This is an RX Detected
         * Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_REM_INVALIDATE,
        /*
         * Local memory error. An RQ/SRQ SGE described an inaccessible
         * memory. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_MEMORY_ERROR,
        /*
         * SRQ in error. The QP is moving to error state because it
         * found SRQ it uses in error. This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_SRQ_ERROR,
        /*
         * Completion error. No CQE space available on queue or CQ not
         * in VALID state. This is a Completion Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_CMP_ERROR,
        /*
         * Invalid R_Key while resending responses to duplicate request.
         * This is a TX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_IVALID_DUP_RKEY,
        /*
         * Problem was found in the format of a WQE in the RQ/SRQ. This
         * is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_WQE_FORMAT_ERROR,
        /*
         * A load error occurred on an attempt to load the CQ Context.
         * This is a Completion Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_CQ_LOAD_ERROR = 0x18,
        /*
         * A load error occurred on an attempt to load the SRQ Context.
         * This is an RX Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_SRQ_LOAD_ERROR,
        /*
         * A fatal error was detected on an attempt to read from or
         * write to PCIe on the transmit side. This error is detected by
         * the TX side, but has the priority of a Completion Detected
         * Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_TX_PCI_ERROR = 0x1b,
        /*
         * A fatal error was detected on an attempt to read from or
         * write to PCIe on the receive side. This error is detected by
         * the RX side (or CAGR), but has the priority of a Completion
         * Detected Error.
         */
        CFCQ_RES_ERR_STATE_REASON_RES_RX_PCI_ERROR = 0x1c
};

int bnxt_re_host_pf_id_query(struct bnxt_re_dev *rdev,
                             struct bnxt_qplib_query_fn_info *fn_info,
                             u32 *pf_mask, u32 *first_pf);

/* Default DCBx and CC values */
#define BNXT_RE_DEFAULT_CNP_DSCP        48
#define BNXT_RE_DEFAULT_CNP_PRI         7
#define BNXT_RE_DEFAULT_ROCE_DSCP       26
#define BNXT_RE_DEFAULT_ROCE_PRI        3

#define BNXT_RE_DEFAULT_L2_BW           50
#define BNXT_RE_DEFAULT_ROCE_BW         50

#define ROCE_PRIO_VALID 0x0
#define CNP_PRIO_VALID  0x1
#define ROCE_DSCP_VALID 0x0
#define CNP_DSCP_VALID  0x1

int bnxt_re_get_pri_dscp_settings(struct bnxt_re_dev *rdev,
                                  u16 target_id,
                                  struct bnxt_re_tc_rec *tc_rec);

int bnxt_re_setup_dscp(struct bnxt_re_dev *rdev);
int bnxt_re_clear_dscp(struct bnxt_re_dev *rdev);
int bnxt_re_setup_cnp_cos(struct bnxt_re_dev *rdev, bool reset);

static inline enum ib_port_state bnxt_re_get_link_state(struct bnxt_re_dev *rdev)
{
        if (if_getdrvflags(rdev->netdev) & IFF_DRV_RUNNING &&
            if_getlinkstate(rdev->netdev) == LINK_STATE_UP)
                return IB_PORT_ACTIVE;
        return IB_PORT_DOWN;
}

static inline int bnxt_re_link_state(struct bnxt_re_dev *rdev)
{
        return bnxt_re_get_link_state(rdev) == IB_PORT_ACTIVE ? 1:0;
}

static inline int is_cc_enabled(struct bnxt_re_dev *rdev)
{
        return rdev->cc_param.enable;
}

static inline void bnxt_re_init_hwrm_hdr(struct bnxt_re_dev *rdev,
                                         struct input *hdr, u16 opcd,
                                         u16 crid, u16 trid)
{
        hdr->req_type = cpu_to_le16(opcd);
        hdr->cmpl_ring = cpu_to_le16(crid);
        hdr->target_id = cpu_to_le16(trid);
}

static inline void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg,
                                       void *msg, int msg_len, void *resp,
                                       int resp_max_len, int timeout)
{
        fw_msg->msg = msg;
        fw_msg->msg_len = msg_len;
        fw_msg->resp = resp;
        fw_msg->resp_max_len = resp_max_len;
        fw_msg->timeout = timeout;
}

static inline bool is_qport_service_type_supported(struct bnxt_re_dev *rdev)
{
        return rdev->tc_rec[0].serv_type_enabled;
}

static inline bool is_bnxt_roce_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
{
        if (is_qport_service_type_supported(rdev))
                return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_ROCE);
        else
                return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSLESS_ROCE);
}

static inline bool is_bnxt_cnp_queue(struct bnxt_re_dev *rdev, u8 ser_prof, u8 prof_type)
{
        if (is_qport_service_type_supported(rdev))
                return (prof_type & HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID1_SERVICE_PROFILE_TYPE_CNP);
        else
                return (ser_prof == HWRM_QUEUE_QPORTCFG_OUTPUT_QUEUE_ID0_SERVICE_PROFILE_LOSSY_ROCE_CNP);
}

#define BNXT_RE_MAP_SH_PAGE             0x0
#define BNXT_RE_MAP_WC                  0x1
#define BNXT_RE_DBR_PAGE                0x2
#define BNXT_RE_MAP_DB_RECOVERY_PAGE    0x3

#define BNXT_RE_DBR_RECOV_USERLAND_TIMEOUT (20)  /*  20 ms */
#define BNXT_RE_DBR_INT_TIME 5 /* ms */
#define BNXT_RE_PACING_EN_INT_THRESHOLD 50 /* Entries in DB FIFO */
#define BNXT_RE_PACING_ALGO_THRESHOLD 250 /* Entries in DB FIFO */
/* Percentage of DB FIFO depth */
#define BNXT_RE_PACING_DBQ_THRESHOLD BNXT_RE_PACING_DBQ_HIGH_WATERMARK

#define BNXT_RE_PACING_ALARM_TH_MULTIPLE(ctx) (_is_chip_p7(ctx) ? 0 : 2)

/*
 * Maximum Percentage of configurable DB FIFO depth.
 * The Doorbell FIFO depth is 0x2c00. But the DBR_REG_DB_THROTTLING register has only 12 bits
 * to program the high watermark. This means user can configure maximum 36% only(4095/11264).
 */
#define BNXT_RE_PACING_DBQ_HIGH_WATERMARK 36

/* Default do_pacing value when there is no congestion */
#define BNXT_RE_DBR_DO_PACING_NO_CONGESTION 0x7F /* 1 in 512 probability */

enum {
        BNXT_RE_DBQ_EVENT_SCHED = 0,
        BNXT_RE_DBR_PACING_EVENT = 1,
        BNXT_RE_DBR_NQ_PACING_NOTIFICATION = 2,
};

struct bnxt_re_dbq_work {
        struct work_struct work;
        struct bnxt_re_dev *rdev;
        struct hwrm_async_event_cmpl cmpl;
        u32 event;
};

int bnxt_re_hwrm_qcaps(struct bnxt_re_dev *rdev);
int bnxt_re_enable_dbr_pacing(struct bnxt_re_dev *rdev);
int bnxt_re_disable_dbr_pacing(struct bnxt_re_dev *rdev);
int bnxt_re_set_dbq_throttling_reg(struct bnxt_re_dev *rdev,
                                   u16 nq_id, u32 throttle);
void bnxt_re_pacing_alert(struct bnxt_re_dev *rdev);
int bnxt_re_hwrm_pri2cos_qcfg(struct bnxt_re_dev *rdev, struct bnxt_re_tc_rec *tc_rec,
                              u16 target_id);
void writel_fbsd(struct bnxt_softc *bp, u32, u8, u32);
u32 readl_fbsd(struct bnxt_softc *bp, u32, u8);

static inline unsigned int bnxt_re_get_total_mr_mw_count(struct bnxt_re_dev *rdev)
{
        return (atomic_read(&rdev->stats.rsors.mr_count) +
                atomic_read(&rdev->stats.rsors.mw_count));
}

static inline void bnxt_re_set_def_pacing_threshold(struct bnxt_re_dev *rdev)
{
        rdev->qplib_res.pacing_data->pacing_th = rdev->pacing_algo_th;
        rdev->qplib_res.pacing_data->alarm_th =
                rdev->pacing_algo_th * BNXT_RE_PACING_ALARM_TH_MULTIPLE(rdev->chip_ctx);
}

static inline void bnxt_re_set_def_do_pacing(struct bnxt_re_dev *rdev)
{
        rdev->qplib_res.pacing_data->do_pacing = rdev->dbr_def_do_pacing;
}

static inline bool bnxt_re_is_var_size_supported(struct bnxt_re_dev *rdev,
                                                struct bnxt_re_ucontext *uctx)
{
        if (uctx)
                return uctx->cmask & BNXT_RE_UCNTX_CAP_VAR_WQE_ENABLED;
        else
                return rdev->chip_ctx->modes.wqe_mode;
}

static inline void bnxt_re_set_pacing_dev_state(struct bnxt_re_dev *rdev)
{
        rdev->qplib_res.pacing_data->dev_err_state =
                test_bit(BNXT_RE_FLAG_ERR_DEVICE_DETACHED, &rdev->flags);
}
#endif