#include <linux/xarray.h>
#include <linux/bitfield.h>
#include "rvu.h"
#include "npc.h"
#include "npc_profile.h"
#include "rvu_npc_hash.h"
#include "rvu_npc.h"
#include "cn20k/npc.h"
#include "cn20k/reg.h"
#include "rvu_npc_fs.h"
static struct npc_priv_t *npc_priv;
static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = {
[NPC_MCAM_KEY_DYN] = "DYNAMIC",
[NPC_MCAM_KEY_X2] = "X2",
[NPC_MCAM_KEY_X4] = "X4",
};
static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = {
[NPC_DFT_RULE_PROMISC_ID] = "Promisc",
[NPC_DFT_RULE_MCAST_ID] = "Mcast",
[NPC_DFT_RULE_BCAST_ID] = "Bcast",
[NPC_DFT_RULE_UCAST_ID] = "Ucast",
};
#define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \
(((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \
((key_ofs) & 0x3F))
#define NPC_DFT_RULE_ID_MK(pcifunc, id) \
((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id))
#define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \
FIELD_GET(GENMASK_ULL(15, 0), rid)
#define NPC_DFT_RULE_ID_2_ID(rid) \
FIELD_GET(GENMASK_ULL(31, 16), rid)
#define NPC_DFT_RULE_PRIO 127
static const char cn20k_def_pfl_name[] = "default";
static struct npc_mcam_kex_extr npc_mkex_extr_default = {
.mkex_sign = MKEX_CN20K_SIGN,
.name = "default",
.kpu_version = NPC_KPU_PROFILE_VER,
.keyx_cfg = {
[NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) |
NPC_CN20K_PARSE_NIBBLE_INTF_RX |
NPC_CN20K_PARSE_NIBBLE_ERRCODE,
[NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) |
NPC_CN20K_PARSE_NIBBLE_INTF_TX,
},
.intf_extr_lid = {
[NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
NPC_LID_LC, NPC_LID_LC, NPC_LID_LD },
[NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB,
NPC_LID_LC, NPC_LID_LD },
},
.intf_extr_lt = {
[NIX_INTF_RX] = {
[0] = {
[NPC_LT_LA_ETHER] =
KEX_EXTR_CFG(0x05, 0x0, 0x1,
NPC_KEXOF_DMAC + 1),
[NPC_LT_LA_CPT_HDR] =
KEX_EXTR_CFG(0x05, 0x0, 0x1,
NPC_KEXOF_DMAC + 1),
},
[1] = {
[NPC_LT_LA_ETHER] =
KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
[NPC_LT_LA_CPT_HDR] =
KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6),
},
[2] = {
[NPC_LT_LB_CTAG] =
KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
[NPC_LT_LB_STAG_QINQ] =
KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8),
[NPC_LT_LB_FDSA] =
KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8),
},
[3] = {
[NPC_LT_LB_CTAG] =
KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
[NPC_LT_LB_STAG_QINQ] =
KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6),
[NPC_LT_LB_FDSA] =
KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6),
},
[4] = {
[NPC_LT_LC_IP] =
KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11),
[NPC_LT_LC_IP6] =
KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11),
},
[5] = {
[NPC_LT_LC_IP] =
KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10),
},
[6] = {
[NPC_LT_LD_UDP] =
KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
[NPC_LT_LD_TCP] =
KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19),
},
},
[NIX_INTF_TX] = {
[0] = {
[NPC_LT_LA_IH_NIX_ETHER] =
KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
[NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4),
},
[1] = {
[NPC_LT_LA_IH_NIX_ETHER] =
KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa),
[NPC_LT_LA_IH_NIX_HIGIG2_ETHER] =
KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa),
},
[2] = {
[NPC_LT_LB_CTAG] =
KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
[NPC_LT_LB_STAG_QINQ] =
KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6),
},
[3] = {
[NPC_LT_LB_CTAG] =
KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8),
[NPC_LT_LB_STAG_QINQ] =
KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8),
},
[4] = {
[NPC_LT_LC_IP] =
KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10),
[NPC_LT_LC_IP6] =
KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10),
},
[5] = {
[NPC_LT_LD_UDP] =
KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
[NPC_LT_LD_TCP] =
KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18),
},
},
},
};
struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void)
{
return &npc_mkex_extr_default;
}
static u16 npc_idx2vidx(u16 idx)
{
unsigned long index;
void *map;
u16 vidx;
int val;
vidx = idx;
index = idx;
map = xa_load(&npc_priv->xa_idx2vidx_map, index);
if (!map)
goto done;
val = xa_to_value(map);
if (val == -1)
goto done;
vidx = val;
done:
return vidx;
}
static bool npc_is_vidx(u16 vidx)
{
return vidx >= npc_priv->bank_depth * 2;
}
static u16 npc_vidx2idx(u16 vidx)
{
unsigned long sentinel = (unsigned long)-1;
unsigned long index;
void *map;
int val;
u16 idx;
idx = vidx;
index = vidx;
map = xa_load(&npc_priv->xa_vidx2idx_map, index);
if (!map)
goto done;
val = xa_to_value(map);
if (val == sentinel)
goto done;
idx = val;
done:
return idx;
}
u16 npc_cn20k_vidx2idx(u16 idx)
{
if (!npc_priv)
return idx;
if (!npc_is_vidx(idx))
return idx;
return npc_vidx2idx(idx);
}
u16 npc_cn20k_idx2vidx(u16 idx)
{
if (!npc_priv)
return idx;
if (npc_is_vidx(idx))
return idx;
return npc_idx2vidx(idx);
}
static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx)
{
u16 mcam_idx;
void *map;
if (!npc_is_vidx(vidx)) {
dev_err(rvu->dev,
"%s: vidx(%u) does not map to proper mcam idx\n",
__func__, vidx);
return -ESRCH;
}
mcam_idx = npc_vidx2idx(vidx);
map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx);
if (!map) {
dev_err(rvu->dev,
"%s: vidx(%u) does not map to proper mcam idx\n",
__func__, vidx);
return -ESRCH;
}
map = xa_erase(&npc_priv->xa_idx2vidx_map, mcam_idx);
if (!map) {
dev_err(rvu->dev,
"%s: vidx(%u) is not valid\n",
__func__, vidx);
return -ESRCH;
}
if (old_midx)
*old_midx = mcam_idx;
return 0;
}
static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx)
{
u16 old_midx;
void *map;
int rc;
if (!npc_is_vidx(vidx)) {
dev_err(rvu->dev,
"%s: vidx(%u) does not map to proper mcam idx\n",
__func__, vidx);
return -ESRCH;
}
map = xa_erase(&npc_priv->xa_vidx2idx_map, vidx);
if (!map) {
dev_err(rvu->dev,
"%s: vidx(%u) could not be deleted from vidx2idx map\n",
__func__, vidx);
return -ESRCH;
}
old_midx = xa_to_value(map);
rc = xa_insert(&npc_priv->xa_vidx2idx_map, vidx,
xa_mk_value(new_midx), GFP_KERNEL);
if (rc) {
dev_err(rvu->dev,
"%s: vidx(%u) cannot be added to vidx2idx map\n",
__func__, vidx);
goto fail1;
}
map = xa_erase(&npc_priv->xa_idx2vidx_map, old_midx);
if (!map) {
dev_err(rvu->dev,
"%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
__func__, old_midx, vidx);
rc = -ESRCH;
goto fail2;
}
rc = xa_insert(&npc_priv->xa_idx2vidx_map, new_midx,
xa_mk_value(vidx), GFP_KERNEL);
if (rc) {
dev_err(rvu->dev,
"%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n",
__func__, new_midx, vidx);
goto fail3;
}
return 0;
fail3:
if (xa_insert(&npc_priv->xa_idx2vidx_map, old_midx,
xa_mk_value(vidx), GFP_KERNEL))
dev_err(rvu->dev,
"%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n",
__func__, old_midx, vidx);
fail2:
if (!xa_erase(&npc_priv->xa_vidx2idx_map, vidx))
dev_err(rvu->dev,
"%s: Failed to roll back vidx2idx vidx=%u\n",
__func__, vidx);
fail1:
if (xa_insert(&npc_priv->xa_vidx2idx_map, vidx,
xa_mk_value(old_midx), GFP_KERNEL))
dev_err(rvu->dev,
"%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n",
__func__, old_midx, vidx);
return rc;
}
static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc,
u16 *vidx)
{
int rc, max, min;
u32 id;
max = npc_priv->bank_depth * 2 * 2 - 1;
min = npc_priv->bank_depth * 2;
rc = xa_alloc(&npc_priv->xa_vidx2idx_map, &id,
xa_mk_value(mcam_idx),
XA_LIMIT(min, max), GFP_KERNEL);
if (rc) {
dev_err(rvu->dev,
"%s: Failed to add to vidx2idx map (%u)\n",
__func__, mcam_idx);
goto fail1;
}
rc = xa_insert(&npc_priv->xa_idx2vidx_map, mcam_idx,
xa_mk_value(id), GFP_KERNEL);
if (rc) {
dev_err(rvu->dev,
"%s: Failed to add to idx2vidx map (%u)\n",
__func__, mcam_idx);
goto fail2;
}
if (vidx)
*vidx = id;
return 0;
fail2:
xa_erase(&npc_priv->xa_vidx2idx_map, id);
fail1:
return rc;
}
static void npc_config_kpmcam(struct rvu *rvu, int blkaddr,
const struct npc_kpu_profile_cam *kpucam,
int kpm, int entry)
{
struct npc_kpu_cam cam0 = {0};
struct npc_kpu_cam cam1 = {0};
cam1.state = kpucam->state & kpucam->state_mask;
cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask;
cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask;
cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask;
cam0.state = ~kpucam->state & kpucam->state_mask;
cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask;
cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask;
cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask;
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1);
}
static void
npc_config_kpmaction(struct rvu *rvu, int blkaddr,
const struct npc_kpu_profile_action *kpuaction,
int kpm, int entry, bool pkind)
{
struct npc_kpm_action0 action0 = {0};
struct npc_kpu_action1 action1 = {0};
u64 reg;
action1.errlev = kpuaction->errlev;
action1.errcode = kpuaction->errcode;
action1.dp0_offset = kpuaction->dp0_offset;
action1.dp1_offset = kpuaction->dp1_offset;
action1.dp2_offset = kpuaction->dp2_offset;
if (pkind)
reg = NPC_AF_PKINDX_ACTION1(entry);
else
reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry);
rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1);
action0.byp_count = kpuaction->bypass_count & 0x7;
action0.capture_ena = kpuaction->cap_ena & 1;
action0.parse_done = kpuaction->parse_done & 1;
action0.next_state = kpuaction->next_state & 0xf;
action0.capture_lid = kpuaction->lid & 0x7;
action0.capture_ltype = kpuaction->ltype & 0xf;
action0.capture_flags = kpuaction->flags & 0xf;
action0.ptr_advance = kpuaction->ptr_advance;
action0.var_len_offset = kpuaction->offset;
action0.var_len_mask = kpuaction->mask;
action0.var_len_right = kpuaction->right & 1;
action0.var_len_shift = kpuaction->shift & 1;
if (pkind)
reg = NPC_AF_PKINDX_ACTION0(entry);
else
reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry);
rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0);
}
static void
npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr,
int kpm, int start_entry,
const struct npc_kpu_profile *profile)
{
int num_cam_entries, num_action_entries;
int entry, num_entries, max_entries;
u64 idx;
num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile);
num_action_entries = npc_get_num_kpu_action_entries(rvu, profile);
if (num_cam_entries != num_action_entries) {
dev_err(rvu->dev,
"kpm%d: CAM and action entries [%d != %d] not equal\n",
kpm, num_cam_entries, num_action_entries);
WARN(1, "Fatal error\n");
return;
}
max_entries = rvu->hw->npc_kpu_entries / 2;
entry = start_entry;
num_entries = min_t(int, num_cam_entries, max_entries);
for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
npc_config_kpmcam(rvu, blkaddr,
npc_get_kpu_cam_nth_entry(rvu, profile, idx),
kpm, entry);
entry = start_entry;
num_entries = min_t(int, num_action_entries, max_entries);
for (idx = 0; entry < num_entries + start_entry; entry++, idx++)
npc_config_kpmaction(rvu, blkaddr,
npc_get_kpu_action_nth_entry(rvu, profile, idx),
kpm, entry, false);
}
static void
npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries)
{
u64 entry_mask;
entry_mask = npc_enable_mask(num_entries);
if (!rvu->kpu.custom)
entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask);
if (num_entries <= 64) {
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 1),
npc_enable_mask(0));
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
npc_enable_mask(0));
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
npc_enable_mask(0));
return;
}
num_entries = num_entries - 64;
entry_mask = npc_enable_mask(num_entries);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask);
if (num_entries <= 64) {
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 2),
npc_enable_mask(0));
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
npc_enable_mask(0));
return;
}
num_entries = num_entries - 64;
entry_mask = npc_enable_mask(num_entries);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask);
if (num_entries <= 64) {
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 3),
npc_enable_mask(0));
return;
}
num_entries = num_entries - 64;
entry_mask = npc_enable_mask(num_entries);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask);
}
#define KPU_OFFSET 8
static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms)
{
const struct npc_kpu_profile *profile1, *profile2;
int pfl1_num_cam_entries, pfl2_num_cam_entries;
int idx, total_cam_entries;
for (idx = 0; idx < num_kpms; idx++) {
profile1 = &rvu->kpu.kpu[idx];
pfl1_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile1);
npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1);
profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET];
pfl2_num_cam_entries = npc_get_num_kpu_cam_entries(rvu, profile2);
npc_program_single_kpm_profile(rvu, blkaddr, idx,
pfl1_num_cam_entries,
profile2);
total_cam_entries = pfl1_num_cam_entries + pfl2_num_cam_entries;
npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries);
rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx),
pfl1_num_cam_entries);
rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01);
rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET),
0x01);
}
}
void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr)
{
struct npc_kpu_profile_action *act;
struct rvu_hwinfo *hw = rvu->hw;
int num_pkinds, idx;
for (idx = 0; idx < hw->npc_kpms; idx++) {
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL);
rvu_write64(rvu, blkaddr,
NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL);
}
for (idx = 0; idx < hw->npc_kpus; idx++)
rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00);
npc_load_kpu_profile(rvu);
rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210);
num_pkinds = rvu->kpu.pkinds;
num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds);
for (idx = 0; idx < num_pkinds; idx++) {
act = npc_get_ikpu_nth_entry(rvu, idx);
if (!act)
continue;
npc_config_kpmaction(rvu, blkaddr, act,
0, idx, true);
}
npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms);
}
struct npc_priv_t *npc_priv_get(void)
{
return npc_priv;
}
static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr,
const struct npc_mcam_kex_extr *mkex_extr,
u8 intf)
{
u8 num_extr = rvu->hw->npc_kex_extr;
int extr, lt;
u64 val;
if (is_npc_intf_tx(intf))
return;
rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
mkex_extr->keyx_cfg[NIX_INTF_RX]);
for (extr = 0; extr < num_extr; extr++)
rvu_write64(rvu, blkaddr,
NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
mkex_extr->intf_extr_lid[intf][extr]);
for (extr = 0; extr < num_extr; extr++) {
for (lt = 0; lt < NPC_MAX_LT; lt++) {
val = mkex_extr->intf_extr_lt[intf][extr][lt];
CN20K_SET_EXTR_LT(intf, extr, lt, val);
}
}
}
static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr,
const struct npc_mcam_kex_extr *mkex_extr,
u8 intf)
{
u8 num_extr = rvu->hw->npc_kex_extr;
int extr, lt;
u64 val;
if (is_npc_intf_rx(intf))
return;
rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf),
mkex_extr->keyx_cfg[NIX_INTF_TX]);
for (extr = 0; extr < num_extr; extr++)
rvu_write64(rvu, blkaddr,
NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr),
mkex_extr->intf_extr_lid[intf][extr]);
for (extr = 0; extr < num_extr; extr++) {
for (lt = 0; lt < NPC_MAX_LT; lt++) {
val = mkex_extr->intf_extr_lt[intf][extr][lt];
CN20K_SET_EXTR_LT(intf, extr, lt, val);
}
}
}
static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr,
const struct npc_mcam_kex_extr *mkex_extr)
{
struct rvu_hwinfo *hw = rvu->hw;
u8 intf;
for (intf = 0; intf < hw->npc_intfs; intf++) {
npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf);
npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf);
}
npc_program_mkex_hash(rvu, blkaddr);
}
void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr,
const char *mkex_profile)
{
struct npc_mcam_kex_extr *mcam_kex_extr;
struct device *dev = &rvu->pdev->dev;
void __iomem *mkex_prfl_addr = NULL;
u64 prfl_sz;
int ret;
if (rvu->kpu_fwdata_sz ||
!strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN))
goto program_mkex_extr;
ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz);
if (ret < 0)
goto program_mkex_extr;
mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr;
while (((s64)prfl_sz > 0) &&
(mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) {
if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN &&
!strncmp(mcam_kex_extr->name, mkex_profile,
MKEX_NAME_LEN)) {
rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr;
goto program_mkex_extr;
}
mcam_kex_extr++;
prfl_sz -= sizeof(struct npc_mcam_kex_extr);
}
dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile);
rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get();
program_mkex_extr:
dev_info(rvu->dev, "Using %s mkex profile\n",
rvu->kpu.mcam_kex_prfl.mkex_extr->name);
npc_program_mkex_profile(rvu, blkaddr,
rvu->kpu.mcam_kex_prfl.mkex_extr);
if (mkex_prfl_addr)
iounmap(mkex_prfl_addr);
}
int
npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr,
int index, bool enable)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int mcam_idx = index % mcam->banksize;
int bank = index / mcam->banksize;
u64 cfg, hw_prio;
u8 kw_type;
if (index < 0 || index >= mcam->total_entries)
return -EINVAL;
if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
return -EINVAL;
if (kw_type == NPC_MCAM_KEY_X2) {
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
bank));
hw_prio = cfg & GENMASK_ULL(30, 24);
cfg = enable ? 1 : 0;
cfg |= hw_prio;
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
cfg);
goto update_en_map;
}
for (bank = 0; bank < mcam->banks_per_entry; bank++) {
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx,
bank));
hw_prio = cfg & GENMASK_ULL(30, 24);
cfg = enable ? 1 : 0;
cfg |= hw_prio;
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
cfg);
}
update_en_map:
if (enable)
set_bit(index, npc_priv->en_map);
else
clear_bit(index, npc_priv->en_map);
return 0;
}
static void
npc_clear_x2_entry(struct rvu *rvu, int blkaddr, int bank, int index)
{
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0);
}
int
npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int mcam_idx)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int bank = npc_get_bank(mcam, mcam_idx);
u8 kw_type;
int index;
if (npc_mcam_idx_2_key_type(rvu, mcam_idx, &kw_type))
return -EINVAL;
index = mcam_idx & (mcam->banksize - 1);
if (kw_type == NPC_MCAM_KEY_X2) {
npc_clear_x2_entry(rvu, blkaddr, bank, index);
return 0;
}
for (bank = 0; bank < mcam->banks_per_entry; bank++)
npc_clear_x2_entry(rvu, blkaddr, bank, index);
return 0;
}
static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx,
u64 *cam0, u64 *cam1)
{
u64 kw_mask;
*cam1 = entry->kw[idx];
kw_mask = entry->kw_mask[idx];
*cam1 &= kw_mask;
*cam0 = ~*cam1 & kw_mask;
}
static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam,
int blkaddr, int index, u8 intf,
struct cn20k_mcam_entry *entry,
int bank, u8 kw_type, int kw,
u8 req_kw_type)
{
u64 intf_ext = 0, intf_ext_mask = 0;
u8 tx_intf_mask = ~intf & 0x3;
u8 tx_intf = intf, kex_type;
u8 kw_type_mask = ~kw_type;
u64 cam0, cam1, kex_cfg;
if (is_npc_intf_tx(intf)) {
tx_intf_mask = 0x1;
tx_intf = intf & tx_intf_mask;
tx_intf_mask = ~tx_intf & tx_intf_mask;
}
kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf));
kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32;
if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) &&
kw_type == NPC_MCAM_KEY_X2) {
kw_type = 0;
kw_type_mask = 0;
}
if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) {
kw_type = 0;
kw_type_mask = 0;
}
intf_ext = ((u64)kw_type << 16) | tx_intf;
intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) |
tx_intf_mask;
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1),
intf_ext);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0),
intf_ext_mask);
npc_cn20k_get_keyword(entry, kw, &cam0, &cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1),
cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0),
cam0);
npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1),
cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0),
cam0);
npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1),
cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0),
cam0);
npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1),
cam1);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0),
cam0);
}
static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam,
int blkaddr, int index, u8 intf,
struct cn20k_mcam_entry *entry,
u8 kw_type, u8 req_kw_type)
{
int kw = 0, bank;
for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4)
npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
index, intf,
entry, bank, kw_type,
kw, req_kw_type);
}
int npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index,
u8 intf, struct cn20k_mcam_entry *entry,
bool enable, u8 hw_prio, u8 req_kw_type)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int mcam_idx = index % mcam->banksize;
int bank = index / mcam->banksize;
u64 bank_cfg = (u64)hw_prio << 24;
int kw = 0;
u8 kw_type;
if (index < 0 || index >= mcam->total_entries)
return -EINVAL;
if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
return -EINVAL;
if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false))
return -EINVAL;
if (kw_type == NPC_MCAM_KEY_X2) {
npc_clear_x2_entry(rvu, blkaddr, bank, mcam_idx);
npc_cn20k_config_kw_x2(rvu, mcam, blkaddr,
mcam_idx, intf, entry,
bank, kw_type, kw, req_kw_type);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 0),
entry->action);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 2),
entry->action2);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 1),
entry->vtag_action);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
bank_cfg);
} else {
npc_clear_x2_entry(rvu, blkaddr, 0, mcam_idx);
npc_clear_x2_entry(rvu, blkaddr, 1, mcam_idx);
npc_cn20k_config_kw_x4(rvu, mcam, blkaddr,
mcam_idx, intf, entry,
kw_type, req_kw_type);
for (bank = 0; bank < mcam->banks_per_entry; bank++) {
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 0),
entry->action);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 1),
entry->vtag_action);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx,
bank, 2),
entry->action2);
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank),
bank_cfg);
}
}
if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable))
return -EINVAL;
return 0;
}
int npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u64 cfg, sreg, dreg, soff, doff;
u8 src_kwtype, dest_kwtype;
int bank, i, sb, db;
int dbank, sbank;
if (src >= mcam->total_entries || dest >= mcam->total_entries)
return -EINVAL;
dbank = npc_get_bank(mcam, dest);
sbank = npc_get_bank(mcam, src);
if (npc_mcam_idx_2_key_type(rvu, src, &src_kwtype))
return -EINVAL;
if (npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype))
return -EINVAL;
if (src_kwtype != dest_kwtype)
return -EINVAL;
src &= (mcam->banksize - 1);
dest &= (mcam->banksize - 1);
for (bank = 0; bank < mcam->banks_per_entry; bank++) {
sb = sbank + bank;
sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0);
db = dbank + bank;
dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0);
for (i = 0; i < 10; i++) {
cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8));
rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg);
}
for (i = 0; i < 3; i++) {
soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src,
sb, i);
cfg = rvu_read64(rvu, blkaddr, soff);
doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db,
i);
rvu_write64(rvu, blkaddr, doff, cfg);
}
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb));
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg);
if (src_kwtype == NPC_MCAM_KEY_X2)
break;
}
return 0;
}
static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx,
u64 cam0, u64 cam1)
{
entry->kw[idx] = cam1;
entry->kw_mask[idx] = cam1 ^ cam0;
}
int npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index,
struct cn20k_mcam_entry *entry,
u8 *intf, u8 *ena, u8 *hw_prio)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u64 cam0, cam1, bank_cfg, cfg;
int kw = 0, bank;
u8 kw_type;
if (index >= mcam->total_entries)
return -EINVAL;
if (npc_mcam_idx_2_key_type(rvu, index, &kw_type))
return -EINVAL;
bank = npc_get_bank(mcam, index);
index &= (mcam->banksize - 1);
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0));
entry->action = cfg;
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 1));
entry->vtag_action = cfg;
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 2));
entry->action2 = cfg;
cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index,
bank, 1)) & 3;
*intf = cfg;
bank_cfg = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank));
*ena = bank_cfg & 0x1;
*hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24;
if (kw_type == NPC_MCAM_KEY_X2) {
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
return 0;
}
for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) {
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1);
cam1 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
bank,
1));
cam0 = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index,
bank,
0));
npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1);
}
return 0;
}
int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu,
struct npc_cn20k_mcam_write_entry_req *req,
struct msg_rsp *rsp)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
struct npc_mcam *mcam = &rvu->hw->mcam;
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, rc;
u8 nix_intf;
req->entry = npc_cn20k_vidx2idx(req->entry);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return NPC_MCAM_INVALID_REQ;
mutex_lock(&mcam->lock);
rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
if (rc)
goto exit;
if (!is_npc_interface_valid(rvu, req->intf)) {
rc = NPC_MCAM_INVALID_REQ;
goto exit;
}
if (is_npc_intf_tx(req->intf))
nix_intf = pfvf->nix_tx_intf;
else
nix_intf = pfvf->nix_rx_intf;
if (is_pffunc_af(req->hdr.pcifunc))
nix_intf = req->intf;
rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf,
&req->entry_data, req->enable_entry,
req->hw_prio, req->req_kw_type);
exit:
mutex_unlock(&mcam->lock);
return rc;
}
int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu,
struct npc_mcam_read_entry_req *req,
struct npc_cn20k_mcam_read_entry_rsp *rsp)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u16 pcifunc = req->hdr.pcifunc;
int blkaddr, rc;
req->entry = npc_cn20k_vidx2idx(req->entry);
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return NPC_MCAM_INVALID_REQ;
mutex_lock(&mcam->lock);
rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry);
if (rc)
goto fail;
rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry,
&rsp->entry_data, &rsp->intf,
&rsp->enable, &rsp->hw_prio);
fail:
mutex_unlock(&mcam->lock);
return rc;
}
int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu,
struct npc_cn20k_mcam_alloc_and_write_entry_req *req,
struct npc_mcam_alloc_and_write_entry_rsp *rsp)
{
struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
struct npc_mcam_free_entry_req free_req = { 0 };
struct npc_mcam_alloc_entry_req entry_req;
struct npc_mcam_alloc_entry_rsp entry_rsp;
struct npc_mcam *mcam = &rvu->hw->mcam;
u16 entry = NPC_MCAM_ENTRY_INVALID;
struct msg_rsp free_rsp;
int blkaddr, rc, err;
u8 nix_intf;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return NPC_MCAM_INVALID_REQ;
if (!is_npc_interface_valid(rvu, req->intf))
return NPC_MCAM_INVALID_REQ;
entry_req.hdr.pcifunc = req->hdr.pcifunc;
entry_req.contig = true;
entry_req.ref_prio = req->ref_prio;
entry_req.ref_entry = req->ref_entry;
entry_req.count = 1;
entry_req.virt = req->virt;
rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu,
&entry_req, &entry_rsp);
if (rc)
return rc;
if (!entry_rsp.count)
return NPC_MCAM_ALLOC_FAILED;
entry = npc_cn20k_vidx2idx(entry_rsp.entry);
mutex_lock(&mcam->lock);
if (is_npc_intf_tx(req->intf))
nix_intf = pfvf->nix_tx_intf;
else
nix_intf = pfvf->nix_rx_intf;
rc = npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf,
&req->entry_data, req->enable_entry,
req->hw_prio, req->req_kw_type);
mutex_unlock(&mcam->lock);
if (rc) {
free_req.hdr.pcifunc = req->hdr.pcifunc;
free_req.entry = entry_rsp.entry;
err = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
if (err)
dev_err(rvu->dev,
"%s: Error to free mcam idx %u\n",
__func__, entry_rsp.entry);
return rc;
}
rsp->entry = entry_rsp.entry;
return 0;
}
static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc)
{
if (is_lbk_vf(rvu, pcifunc))
return NIXLF_PROMISC_ENTRY;
return NIXLF_UCAST_ENTRY;
}
int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu,
struct msg_req *req,
struct npc_cn20k_mcam_read_base_rule_rsp *rsp)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int index, blkaddr, nixlf, rc = 0;
u16 pcifunc = req->hdr.pcifunc;
u8 intf, enable, hw_prio;
struct rvu_pfvf *pfvf;
int rl_type;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return NPC_MCAM_INVALID_REQ;
if (!(pcifunc & RVU_PFVF_FUNC_MASK)) {
pfvf = rvu_get_pfvf(rvu, pcifunc);
rsp->entry.kw[0] = pfvf->rx_chan_base;
rsp->entry.kw_mask[0] = 0xFFFULL;
goto out;
}
mutex_lock(&mcam->lock);
for (index = 0; index < mcam->bmap_entries; index++) {
if (mcam->entry2target_pffunc[index] == pcifunc)
goto read_entry;
}
rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL);
if (rc < 0) {
mutex_unlock(&mcam->lock);
goto out;
}
rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc);
index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type);
if (index < 0) {
mutex_unlock(&mcam->lock);
goto out;
}
read_entry:
rc = npc_cn20k_read_mcam_entry(rvu, blkaddr, index,
&rsp->entry, &intf,
&enable, &hw_prio);
mutex_unlock(&mcam->lock);
out:
return rc;
}
static u8 npc_map2cn20k_flag(u8 flag)
{
switch (flag) {
case NPC_F_LC_U_IP_FRAG:
return NPC_CN20K_F_LC_L_IP_FRAG;
case NPC_F_LC_U_IP6_FRAG:
return NPC_CN20K_F_LC_L_IP6_FRAG;
case NPC_F_LC_L_6TO4:
return NPC_CN20K_F_LC_L_6TO4;
case NPC_F_LC_L_MPLS_IN_IP:
return NPC_CN20K_F_LC_U_MPLS_IN_IP;
case NPC_F_LC_L_IP6_TUN_IP6:
return NPC_CN20K_F_LC_U_IP6_TUN_IP6;
case NPC_F_LC_L_IP6_MPLS_IN_IP:
return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP;
default:
break;
}
WARN(1, "%s: Invalid flag=%u\n", __func__, flag);
return 0xff;
}
static void npc_cn20k_translate_action_flags(struct npc_kpu_profile_action *act)
{
u8 ltype, val;
if (act->lid != NPC_LID_LC)
return;
ltype = act->ltype;
if (ltype != NPC_LT_LC_IP &&
ltype != NPC_LT_LC_IP6 &&
ltype != NPC_LT_LC_IP_OPT &&
ltype != NPC_LT_LC_IP6_EXT)
return;
switch (act->flags) {
case NPC_F_LC_U_IP_FRAG:
case NPC_F_LC_U_IP6_FRAG:
case NPC_F_LC_L_6TO4:
case NPC_F_LC_L_MPLS_IN_IP:
case NPC_F_LC_L_IP6_TUN_IP6:
case NPC_F_LC_L_IP6_MPLS_IN_IP:
val = npc_map2cn20k_flag(act->flags);
if (val != 0xFF)
act->flags = val;
break;
default:
break;
}
}
void
npc_cn20k_update_action_entries_n_flags(struct rvu *rvu,
struct npc_kpu_profile_adapter *pfl)
{
struct npc_kpu_profile_action *action;
int entries;
for (int i = 0; i < pfl->kpus; i++) {
action = pfl->kpu[i].action;
entries = pfl->kpu[i].action_entries;
for (int j = 0; j < entries; j++)
npc_cn20k_translate_action_flags(&action[j]);
}
}
int npc_cn20k_apply_custom_kpu(struct rvu *rvu,
struct npc_kpu_profile_adapter *profile)
{
const struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata;
size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
struct npc_kpu_profile_action *action;
struct npc_kpu_profile_cam *cam;
struct npc_kpu_fwdata *fw_kpu;
size_t offset = 0;
u16 kpu, entry;
int entries;
hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata);
if (rvu->kpu_fwdata_sz < hdr_sz) {
dev_warn(rvu->dev, "Invalid KPU profile size\n");
return -EINVAL;
}
if (le64_to_cpu(fw->signature) != KPU_SIGN) {
dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n",
fw->signature);
return -EINVAL;
}
if (NPC_KPU_VER_MAJ(profile->version) >
NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) {
dev_warn(rvu->dev, "Not supported Major version: %d > %d\n",
NPC_KPU_VER_MAJ(profile->version),
NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER));
return -EINVAL;
}
if (NPC_KPU_VER_MIN(profile->version) <
NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) {
dev_warn(rvu->dev,
"Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n",
NPC_KPU_VER_MAJ(profile->version),
NPC_KPU_VER_MIN(profile->version),
NPC_KPU_VER_PATCH(profile->version),
NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER),
NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER),
NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER));
return -EINVAL;
}
if (fw->kpus > rvu->hw->npc_kpus) {
dev_warn(rvu->dev, "Not enough KPUs: %d > %d\n", fw->kpus,
rvu->hw->npc_kpus);
return -EINVAL;
}
if (fw->kpus > profile->kpus) {
dev_warn(rvu->dev, "Not enough KPUs: %d > %zu\n", fw->kpus,
profile->kpus);
return -EINVAL;
}
profile->mcam_kex_prfl.mkex_extr = &fw->mkex;
if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) {
dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n",
profile->mcam_kex_prfl.mkex_extr->mkex_sign);
return -EINVAL;
}
profile->custom = 1;
profile->name = fw->name;
profile->version = le64_to_cpu(fw->version);
profile->lt_def = &fw->lt_def;
for (kpu = 0; kpu < fw->kpus; kpu++) {
fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset);
if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT)
dev_warn(rvu->dev,
"Too many custom entries on KPU%d: %d > %d\n",
kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT);
cam = (struct npc_kpu_profile_cam *)fw_kpu->data;
offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam);
action = (struct npc_kpu_profile_action *)(fw->data + offset);
offset += fw_kpu->entries * sizeof(*action);
if (rvu->kpu_fwdata_sz < hdr_sz + offset) {
dev_warn(rvu->dev,
"Profile size mismatch on KPU%i parsing.\n",
kpu + 1);
return -EINVAL;
}
for (entry = 0; entry < entries; entry++) {
profile->kpu[kpu].cam[entry] = cam[entry];
profile->kpu[kpu].action[entry] = action[entry];
npc_cn20k_translate_action_flags(&profile->kpu[kpu].action[entry]);
}
}
return 0;
}
int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type)
{
struct npc_subbank *sb;
int bank_off, sb_id;
if (mcam_idx >= npc_priv->bank_depth * 2) {
dev_err(rvu->dev, "%s: bad params\n",
__func__);
return -EINVAL;
}
bank_off = mcam_idx & (npc_priv->bank_depth - 1);
sb_id = bank_off / npc_priv->subbank_depth;
if (sb_id >= npc_priv->num_subbanks) {
dev_err(rvu->dev, "%s: invalid subbank %d\n",
__func__, sb_id);
return -EINVAL;
}
sb = &npc_priv->sb[sb_id];
*key_type = sb->key_type;
return 0;
}
static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb,
u16 sub_off, u16 *mcam_idx)
{
int off, bot;
if (sb->key_type == NPC_MCAM_KEY_X4 &&
sub_off >= npc_priv->subbank_depth) {
dev_err(rvu->dev,
"%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u",
__func__, sb->idx, sub_off);
return -EINVAL;
}
if (sb->key_type == NPC_MCAM_KEY_X2 &&
sub_off >= npc_priv->subbank_depth * 2) {
dev_err(rvu->dev,
"%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u",
__func__, sb->idx, sub_off);
return -EINVAL;
}
off = sub_off & (npc_priv->subbank_depth - 1);
bot = sub_off >= npc_priv->subbank_depth ? sb->b1b : sb->b0b;
*mcam_idx = bot + off;
return 0;
}
int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx,
struct npc_subbank **sb,
int *sb_off)
{
int bank_off, sb_id;
if (mcam_idx >= npc_priv->bank_depth * 2) {
dev_err(rvu->dev, "%s: Invalid mcam idx %u\n",
__func__, mcam_idx);
return -EINVAL;
}
bank_off = mcam_idx & (npc_priv->bank_depth - 1);
sb_id = bank_off / npc_priv->subbank_depth;
if (sb_id >= npc_priv->num_subbanks) {
dev_err(rvu->dev, "%s: invalid subbank %d\n",
__func__, sb_id);
return -EINVAL;
}
*sb = &npc_priv->sb[sb_id];
*sb_off = bank_off % npc_priv->subbank_depth;
if (mcam_idx >= npc_priv->bank_depth)
*sb_off += npc_priv->subbank_depth;
return 0;
}
static int __npc_subbank_contig_alloc(struct rvu *rvu,
struct npc_subbank *sb,
int key_type, int sidx,
int eidx, int prio,
int count, int t, int b,
unsigned long *bmap,
u16 *save)
{
int k, offset, delta = 0;
int cnt = 0, sbd;
sbd = npc_priv->subbank_depth;
if (sidx >= npc_priv->bank_depth)
delta = sbd;
switch (prio) {
case NPC_MCAM_LOWER_PRIO:
case NPC_MCAM_ANY_PRIO:
offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b,
count, 0);
if (offset >= sbd) {
dev_err(rvu->dev,
"%s: Could not find contiguous(%d) entries\n",
__func__, count);
return -EFAULT;
}
dev_dbg(rvu->dev,
"%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n",
__func__, sidx, eidx, t, b, offset,
count, delta);
for (cnt = 0; cnt < count; cnt++)
save[cnt] = offset + cnt + delta;
break;
case NPC_MCAM_HIGHER_PRIO:
for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
if (test_bit(k, bmap)) {
cnt = 0;
continue;
}
save[cnt++] = k + delta;
}
break;
}
if (cnt == count)
return 0;
dev_dbg(rvu->dev,
"%s: Could not find contiguous(%d) entries in subbank=%u\n",
__func__, count, sb->idx);
return -EFAULT;
}
static int __npc_subbank_non_contig_alloc(struct rvu *rvu,
struct npc_subbank *sb,
int key_type, int sidx,
int eidx, int prio,
int t, int b,
unsigned long *bmap,
int count, u16 *save,
bool max_alloc, int *alloc_cnt)
{
unsigned long index;
int cnt = 0, delta;
int k, sbd;
sbd = npc_priv->subbank_depth;
delta = sidx >= npc_priv->bank_depth ? sbd : 0;
switch (prio) {
case NPC_MCAM_LOWER_PRIO:
case NPC_MCAM_ANY_PRIO:
index = find_next_zero_bit(bmap, sbd, sidx - b);
if (index >= sbd) {
dev_err(rvu->dev,
"%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n",
__func__, count,
bitmap_weight(bmap, sbd),
sb->idx);
break;
}
for (k = index; cnt < count && k <= (eidx - b); k++) {
if (test_bit(k, bmap))
continue;
save[cnt++] = k + delta;
}
break;
case NPC_MCAM_HIGHER_PRIO:
for (k = eidx - b; cnt < count && k >= (sidx - b); k--) {
if (test_bit(k, bmap))
continue;
save[cnt++] = k + delta;
}
break;
}
*alloc_cnt = cnt;
if (cnt == count)
return 0;
if (max_alloc && cnt > 0)
return 0;
dev_dbg(rvu->dev,
"%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n",
__func__, count, sb->idx, cnt, max_alloc);
return -EFAULT;
}
static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb,
int sb_off, unsigned long **bmap,
int *off)
{
int sbd;
sbd = npc_priv->subbank_depth;
*off = sb_off & (sbd - 1);
*bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map;
}
static bool __npc_subbank_mark_slot(struct rvu *rvu,
struct npc_subbank *sb,
int sb_off, bool set)
{
unsigned long *bmap;
int off;
__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
dev_dbg(rvu->dev,
"%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n",
__func__, set, sb_off, sb->idx, off);
if (set) {
if (test_bit(off, bmap))
return false;
sb->free_cnt--;
set_bit(off, bmap);
return true;
}
if (!test_bit(off, bmap))
return false;
sb->free_cnt++;
clear_bit(off, bmap);
return true;
}
static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb)
{
int rc, blkaddr;
sb->flags = NPC_SUBBANK_FLAG_FREE;
sb->key_type = 0;
bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth);
bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth);
if (!xa_erase(&npc_priv->xa_sb_used, sb->arr_idx)) {
dev_err(rvu->dev,
"%s: Error to delete from xa_sb_used array\n",
__func__);
return -EFAULT;
}
rc = xa_insert(&npc_priv->xa_sb_free, sb->arr_idx,
xa_mk_value(sb->idx), GFP_KERNEL);
if (rc) {
rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx,
xa_mk_value(sb->idx), GFP_KERNEL);
if (rc)
dev_err(rvu->dev,
"%s: Failed to roll back sb(%u) arr_idx=%d\n",
__func__, sb->idx, sb->arr_idx);
dev_err(rvu->dev,
"%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n",
__func__, sb->idx, sb->arr_idx);
return rc;
}
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
rvu_write64(rvu, blkaddr,
NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
NPC_MCAM_KEY_X2);
return rc;
}
static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb,
int key_type)
{
int rc;
sb->flags = NPC_SUBBANK_FLAG_USED;
sb->key_type = key_type;
if (key_type == NPC_MCAM_KEY_X4)
sb->free_cnt = npc_priv->subbank_depth;
else
sb->free_cnt = 2 * npc_priv->subbank_depth;
bitmap_clear(sb->b0map, 0, npc_priv->subbank_depth);
bitmap_clear(sb->b1map, 0, npc_priv->subbank_depth);
if (!xa_erase(&npc_priv->xa_sb_free, sb->arr_idx)) {
dev_err(rvu->dev,
"%s: Error to delete from xa_sb_free array\n",
__func__);
return -EFAULT;
}
rc = xa_insert(&npc_priv->xa_sb_used, sb->arr_idx,
xa_mk_value(sb->idx), GFP_KERNEL);
if (rc)
dev_err(rvu->dev,
"%s: Error to add to xa_sb_used array\n", __func__);
return rc;
}
static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb,
u16 sb_off)
{
bool deleted = false;
unsigned long *bmap;
int rc, off;
deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false);
if (!deleted)
goto done;
__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
if (sb->key_type == NPC_MCAM_KEY_X4) {
if (sb->free_cnt < npc_priv->subbank_depth)
goto done;
} else {
if (sb->free_cnt < 2 * npc_priv->subbank_depth)
goto done;
}
rc = __npc_subbank_mark_free(rvu, sb);
if (rc)
dev_err(rvu->dev, "%s: Error to free subbank\n", __func__);
done:
return deleted;
}
static int
npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off)
{
bool deleted;
mutex_lock(&sb->lock);
deleted = __npc_subbank_free(rvu, sb, sb_off);
mutex_unlock(&sb->lock);
return deleted ? 0 : -EFAULT;
}
static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
int key_type, int ref, int limit, int prio,
bool contig, int count, u16 *mcam_idx,
int idx_sz, bool max_alloc, int *alloc_cnt)
{
int cnt, t, b, i, blkaddr;
bool new_sub_bank = false;
unsigned long *bmap;
u16 *save = NULL;
int sidx, eidx;
bool diffbank;
int bw, bfree;
int rc = 0;
bool ret;
if (!max_alloc && count > idx_sz) {
dev_err(rvu->dev,
"%s: Less space, count=%d idx_sz=%d sb_id=%d\n",
__func__, count, idx_sz, sb->idx);
return -ENOSPC;
}
if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
(prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
__func__, ref, limit);
return -EINVAL;
}
if (key_type == NPC_MCAM_KEY_X4 &&
(ref >= npc_priv->bank_depth || limit >= npc_priv->bank_depth)) {
dev_err(rvu->dev,
"%s: Wrong ref_enty(%d) or limit(%d) for x4\n",
__func__, ref, limit);
return -EINVAL;
}
diffbank = !!((ref & npc_priv->bank_depth) ^
(limit & npc_priv->bank_depth));
if (diffbank) {
dev_err(rvu->dev,
"%s: request ref and limit should be from same bank\n",
__func__);
return -EINVAL;
}
sidx = min_t(int, limit, ref);
eidx = max_t(int, limit, ref);
cnt = eidx - sidx + 1;
if (contig && cnt < count) {
dev_err(rvu->dev,
"%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n",
__func__, ref, limit, count);
return -EINVAL;
}
if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
if (contig && count > npc_priv->subbank_depth) {
dev_err(rvu->dev, "%s: Less number of entries\n",
__func__);
return -ENOSPC;
}
new_sub_bank = true;
goto process;
}
WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED),
"Used flag is not set(%#x)\n", sb->flags);
if (sb->key_type != key_type) {
dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__);
return -EINVAL;
}
process:
if (ref >= npc_priv->bank_depth) {
bmap = sb->b1map;
t = sb->b1t;
b = sb->b1b;
} else {
bmap = sb->b0map;
t = sb->b0t;
b = sb->b0b;
}
bw = bitmap_weight(bmap, npc_priv->subbank_depth);
bfree = npc_priv->subbank_depth - bw;
if (!bfree) {
dev_dbg(rvu->dev, "%s: subbank is full\n", __func__);
return -ENOSPC;
}
if (contig && bfree < count) {
dev_dbg(rvu->dev, "%s: no space for entry\n", __func__);
return -ENOSPC;
}
save = kcalloc(count, sizeof(u16), GFP_KERNEL);
if (!save) {
rc = -ENOMEM;
goto err1;
}
if (contig) {
rc = __npc_subbank_contig_alloc(rvu, sb, key_type,
sidx, eidx, prio,
count, t, b,
bmap, save);
if (!rc)
*alloc_cnt = count;
} else {
rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type,
sidx, eidx, prio,
t, b, bmap,
count, save,
max_alloc, alloc_cnt);
}
if (rc)
goto err1;
if (new_sub_bank) {
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0) {
dev_err(rvu->dev,
"%s: NPC block not implemented\n", __func__);
rc = -EFAULT;
goto err1;
}
rc = __npc_subbank_mark_used(rvu, sb, key_type);
if (rc) {
dev_err(rvu->dev,
"%s: Error to mark subbank as used\n",
__func__);
goto err2;
}
rvu_write64(rvu, blkaddr,
NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx),
key_type);
}
for (i = 0; i < *alloc_cnt; i++) {
rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i],
&mcam_idx[i]);
if (rc) {
dev_err(rvu->dev,
"%s: Error to find mcam idx for %u\n",
__func__, save[i]);
goto err3;
}
ret = __npc_subbank_mark_slot(rvu, sb, save[i], true);
if (!ret) {
dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n",
__func__, mcam_idx[i]);
rc = -EFAULT;
goto err3;
}
}
kfree(save);
return 0;
err3:
for (int j = 0; j < i; j++)
__npc_subbank_mark_slot(rvu, sb, save[j], false);
err2:
if (new_sub_bank)
__npc_subbank_mark_free(rvu, sb);
err1:
kfree(save);
*alloc_cnt = 0;
return rc;
}
static int
npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb,
int key_type, int ref, int limit, int prio,
bool contig, int count, u16 *mcam_idx,
int idx_sz, bool max_alloc, int *alloc_cnt)
{
int rc;
mutex_lock(&sb->lock);
rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio,
contig, count, mcam_idx, idx_sz,
max_alloc, alloc_cnt);
mutex_unlock(&sb->lock);
return rc;
}
static int
npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx)
{
int pcifunc, idx;
void *map;
map = xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx);
if (!map) {
dev_err(rvu->dev,
"%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n",
__func__, mcam_idx);
return -EFAULT;
}
pcifunc = xa_to_value(map);
map = xa_load(&npc_priv->xa_pf_map, pcifunc);
if (!map) {
dev_err(rvu->dev,
"%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n",
__func__, pcifunc, mcam_idx);
return -ESRCH;
}
idx = xa_to_value(map);
map = xa_erase(&npc_priv->xa_pf2idx_map[idx], mcam_idx);
if (!map) {
dev_err(rvu->dev,
"%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n",
__func__, mcam_idx);
return -EFAULT;
}
return 0;
}
static int
npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc)
{
int rc, idx;
void *map;
dev_dbg(rvu->dev,
"%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n",
__func__, mcam_idx, pcifunc);
rc = xa_insert(&npc_priv->xa_idx2pf_map, mcam_idx,
xa_mk_value(pcifunc), GFP_KERNEL);
if (rc) {
map = xa_load(&npc_priv->xa_idx2pf_map, mcam_idx);
dev_err(rvu->dev,
"%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n",
__func__, mcam_idx, xa_to_value(map));
return -EFAULT;
}
map = xa_load(&npc_priv->xa_pf_map, pcifunc);
if (!map) {
dev_err(rvu->dev,
"%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n",
__func__, pcifunc, mcam_idx);
return -ESRCH;
}
idx = xa_to_value(map);
rc = xa_insert(&npc_priv->xa_pf2idx_map[idx], mcam_idx,
xa_mk_value(pcifunc), GFP_KERNEL);
if (rc) {
map = xa_load(&npc_priv->xa_pf2idx_map[idx], mcam_idx);
xa_erase(&npc_priv->xa_idx2pf_map, mcam_idx);
dev_err(rvu->dev,
"%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n",
__func__, mcam_idx, xa_to_value(map), idx);
return -EFAULT;
}
return 0;
}
static bool
npc_subbank_suits(struct npc_subbank *sb, int key_type)
{
mutex_lock(&sb->lock);
if (!sb->key_type) {
mutex_unlock(&sb->lock);
return true;
}
if (sb->key_type == key_type) {
mutex_unlock(&sb->lock);
return true;
}
mutex_unlock(&sb->lock);
return false;
}
#define SB_ALIGN_UP(val) (((val) + npc_priv->subbank_depth) & \
~((npc_priv->subbank_depth) - 1))
#define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv->subbank_depth)
static void npc_subbank_iter_down(struct rvu *rvu,
int ref, int limit,
int *cur_ref, int *cur_limit,
bool *start, bool *stop)
{
int align;
*stop = false;
if (*start) {
*start = false;
*cur_ref = ref;
align = SB_ALIGN_DOWN(ref);
if (align < limit) {
*stop = true;
*cur_limit = limit;
return;
}
*cur_limit = align;
return;
}
*cur_ref = *cur_limit - 1;
align = *cur_ref - npc_priv->subbank_depth + 1;
if (align <= limit) {
*stop = true;
*cur_limit = limit;
return;
}
*cur_limit = align;
}
static void npc_subbank_iter_up(struct rvu *rvu,
int ref, int limit,
int *cur_ref, int *cur_limit,
bool *start, bool *stop)
{
int align;
*stop = false;
if (*start) {
*start = false;
*cur_ref = ref;
align = SB_ALIGN_UP(ref);
if (align - 1 > limit) {
*stop = true;
*cur_limit = limit;
return;
}
*cur_limit = align - 1;
return;
}
*cur_ref = *cur_limit + 1;
align = *cur_ref + npc_priv->subbank_depth - 1;
if (align >= limit) {
*stop = true;
*cur_limit = limit;
return;
}
*cur_limit = align;
}
static int
npc_subbank_iter(struct rvu *rvu, int key_type,
int ref, int limit, int prio,
int *cur_ref, int *cur_limit,
bool *start, bool *stop)
{
if (prio != NPC_MCAM_HIGHER_PRIO)
npc_subbank_iter_up(rvu, ref, limit,
cur_ref, cur_limit,
start, stop);
else
npc_subbank_iter_down(rvu, ref, limit,
cur_ref, cur_limit,
start, stop);
if (key_type == NPC_MCAM_KEY_X4) {
if (*cur_ref >= npc_priv->bank_depth)
return -EINVAL;
if (*cur_limit >= npc_priv->bank_depth)
return -EINVAL;
}
if (*cur_ref >= 2 * npc_priv->bank_depth)
return -EINVAL;
if (*cur_limit >= 2 * npc_priv->bank_depth)
return -EINVAL;
return 0;
}
static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count,
bool maps_del)
{
struct npc_subbank *sb;
u16 vidx, midx;
int sb_off, i;
bool ret;
int rc;
for (i = 0; i < count; i++) {
rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]),
&sb, &sb_off);
if (rc) {
dev_err(rvu->dev,
"Failed to free mcam idx=%u\n", mcam_idx[i]);
return rc;
}
}
for (i = 0; i < count; i++) {
if (npc_is_vidx(mcam_idx[i])) {
vidx = mcam_idx[i];
midx = npc_vidx2idx(vidx);
} else {
midx = mcam_idx[i];
vidx = npc_idx2vidx(midx);
}
if (midx >= npc_priv->bank_depth * npc_priv->num_banks) {
dev_err(rvu->dev,
"%s: Invalid mcam_idx=%u cannot be deleted\n",
__func__, mcam_idx[i]);
return -EINVAL;
}
rc = npc_mcam_idx_2_subbank_idx(rvu, midx,
&sb, &sb_off);
if (rc) {
dev_err(rvu->dev,
"%s: Failed to find subbank info for vidx=%u\n",
__func__, vidx);
return rc;
}
ret = npc_subbank_free(rvu, sb, sb_off);
if (ret) {
dev_err(rvu->dev,
"%s: Failed to find subbank info for vidx=%u\n",
__func__, vidx);
return -EINVAL;
}
if (!maps_del)
continue;
rc = npc_del_from_pf_maps(rvu, midx);
if (rc)
return rc;
if (vidx == midx)
continue;
rc = npc_vidx_maps_del_entry(rvu, vidx, NULL);
if (rc)
return rc;
}
return 0;
}
static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type,
int ref, int limit, int prio,
bool contig, int count,
u16 *mcam_idx)
{
struct npc_subbank *sb;
unsigned long *bmap;
int sb_off, off, rc;
int cnt = 0;
bool bitset;
if (prio != NPC_MCAM_HIGHER_PRIO) {
while (ref <= limit) {
rc = npc_mcam_idx_2_subbank_idx(rvu, ref,
&sb, &sb_off);
if (rc)
goto err;
if (!npc_subbank_suits(sb, key_type)) {
ref = SB_ALIGN_UP(ref);
if (contig) {
rc = npc_idx_free(rvu, mcam_idx,
cnt, false);
if (rc)
return rc;
cnt = 0;
}
continue;
}
mutex_lock(&sb->lock);
if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
rc = __npc_subbank_mark_used(rvu, sb,
key_type);
if (rc) {
mutex_unlock(&sb->lock);
dev_err(rvu->dev,
"%s:Error to add to use array\n",
__func__);
goto err;
}
}
__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
bitset = test_bit(off, bmap);
if (bitset) {
mutex_unlock(&sb->lock);
if (contig) {
rc = npc_idx_free(rvu, mcam_idx,
cnt, false);
if (rc)
return rc;
cnt = 0;
}
ref++;
continue;
}
set_bit(off, bmap);
sb->free_cnt--;
mcam_idx[cnt++] = ref;
mutex_unlock(&sb->lock);
if (cnt == count)
return 0;
ref++;
}
goto err;
}
while (ref >= limit) {
rc = npc_mcam_idx_2_subbank_idx(rvu, ref,
&sb, &sb_off);
if (rc)
goto err;
if (!npc_subbank_suits(sb, key_type)) {
ref = SB_ALIGN_DOWN(ref) - 1;
if (contig) {
rc = npc_idx_free(rvu, mcam_idx, cnt, false);
if (rc)
return rc;
cnt = 0;
}
continue;
}
mutex_lock(&sb->lock);
if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
rc = __npc_subbank_mark_used(rvu, sb, key_type);
if (rc) {
mutex_unlock(&sb->lock);
dev_err(rvu->dev,
"%s:Error to add to use array\n",
__func__);
goto err;
}
}
__npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off);
bitset = test_bit(off, bmap);
if (bitset) {
mutex_unlock(&sb->lock);
if (contig) {
rc = npc_idx_free(rvu, mcam_idx, cnt, false);
if (rc)
return rc;
cnt = 0;
}
ref--;
continue;
}
mcam_idx[cnt++] = ref;
sb->free_cnt--;
set_bit(off, bmap);
mutex_unlock(&sb->lock);
if (cnt == count)
return 0;
ref--;
}
err:
rc = npc_idx_free(rvu, mcam_idx, cnt, false);
if (rc)
dev_err(rvu->dev,
"%s: Error happened while freeing cnt=%u indexes\n",
__func__, cnt);
return -ENOSPC;
}
static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb,
int key_type)
{
int cnt, spd;
spd = npc_priv->subbank_depth;
mutex_lock(&sb->lock);
if (sb->flags & NPC_SUBBANK_FLAG_FREE)
cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd;
else
cnt = sb->free_cnt;
mutex_unlock(&sb->lock);
return cnt;
}
static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type,
int ref, int limit, int prio,
bool contig, int count,
u16 *mcam_idx)
{
struct npc_subbank *sb1, *sb2;
bool max_alloc, start, stop;
int r, l, sb_idx1, sb_idx2;
int tot = 0, rc;
int alloc_cnt;
max_alloc = !contig;
start = true;
stop = false;
while (!stop) {
rc = npc_subbank_iter(rvu, key_type, ref, limit, prio,
&r, &l, &start, &stop);
dev_dbg(rvu->dev,
"%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n",
__func__, ref, limit, r, l,
start, stop, tot, count, rc);
if (rc)
goto err;
rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1,
&sb_idx1);
if (rc)
goto err;
dev_dbg(rvu->dev,
"%s: ref subbank=%d off=%d\n",
__func__, sb1->idx, sb_idx1);
if (!npc_subbank_suits(sb1, key_type)) {
dev_dbg(rvu->dev,
"%s: not suitable sb=%d key_type=%d\n",
__func__, sb1->idx, key_type);
continue;
}
rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2,
&sb_idx2);
if (rc)
goto err;
dev_dbg(rvu->dev,
"%s: limit subbank=%d off=%d\n",
__func__, sb_idx1, sb_idx2);
if (sb1 != sb2) {
dev_err(rvu->dev,
"%s: l(%d) and r(%d) are not in same subbank\n",
__func__, r, l);
goto err;
}
if (contig &&
npc_subbank_free_cnt(rvu, sb1, key_type) < count) {
dev_dbg(rvu->dev, "%s: less count =%d\n",
__func__,
npc_subbank_free_cnt(rvu, sb1, key_type));
continue;
}
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb1, key_type,
r, l, prio, contig,
count - tot, mcam_idx + tot,
count - tot, max_alloc,
&alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n",
__func__, tot, alloc_cnt);
if (!rc && count == tot)
return 0;
}
err:
dev_dbg(rvu->dev, "%s: Error to allocate\n",
__func__);
if (max_alloc) {
rc = npc_idx_free(rvu, mcam_idx, tot, false);
if (rc)
dev_err(rvu->dev,
"%s: failed to free %u indexes\n",
__func__, tot);
}
return -EFAULT;
}
static int npc_subbank_restricted_idxs[2];
static bool restrict_valid = true;
static bool npc_subbank_restrict_usage(struct rvu *rvu, int index)
{
int i;
if (!restrict_valid)
return false;
for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) {
if (index == npc_subbank_restricted_idxs[i])
return true;
}
return false;
}
static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig,
int count, u16 *mcam_idx)
{
struct npc_subbank *sb;
unsigned long index;
int tot = 0, rc;
bool max_alloc;
int alloc_cnt;
int idx, i;
void *val;
max_alloc = !contig;
xa_for_each(&npc_priv->xa_sb_used, index, val) {
idx = xa_to_value(val);
if (npc_subbank_restrict_usage(rvu, idx))
continue;
sb = &npc_priv->sb[idx];
if (!npc_subbank_suits(sb, key_type))
continue;
if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb, key_type,
sb->b0b, sb->b0t, 0,
contig, count - tot,
mcam_idx + tot,
count - tot,
max_alloc, &alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
if (key_type == NPC_MCAM_KEY_X4)
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb, key_type,
sb->b1b, sb->b1t, 0,
contig, count - tot,
mcam_idx + tot,
count - tot, max_alloc,
&alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
}
xa_for_each(&npc_priv->xa_sb_free, index, val) {
idx = xa_to_value(val);
sb = &npc_priv->sb[idx];
if (npc_subbank_restrict_usage(rvu, idx))
continue;
if (!npc_subbank_suits(sb, key_type))
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb, key_type,
sb->b0b, sb->b0t, 0,
contig, count - tot,
mcam_idx + tot,
count - tot,
max_alloc, &alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
if (key_type == NPC_MCAM_KEY_X4)
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb,
key_type, sb->b1b, sb->b1t, 0,
contig, count - tot,
mcam_idx + tot, count - tot,
max_alloc, &alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
}
for (i = 0; restrict_valid &&
(i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) {
idx = npc_subbank_restricted_idxs[i];
sb = &npc_priv->sb[idx];
if (!npc_subbank_suits(sb, key_type))
continue;
if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count)
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb, key_type,
sb->b0b, sb->b0t, 0,
contig, count - tot,
mcam_idx + tot,
count - tot,
max_alloc, &alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
if (key_type == NPC_MCAM_KEY_X4)
continue;
alloc_cnt = 0;
rc = npc_subbank_alloc(rvu, sb, key_type,
sb->b1b, sb->b1t, 0,
contig, count - tot,
mcam_idx + tot,
count - tot, max_alloc,
&alloc_cnt);
tot += alloc_cnt;
dev_dbg(rvu->dev,
"%s: Allocated %d from subbank %d, tot=%d count=%d\n",
__func__, alloc_cnt, sb->idx, tot, count);
if (!rc && count == tot)
return 0;
}
if (max_alloc)
npc_idx_free(rvu, mcam_idx, tot, false);
dev_dbg(rvu->dev, "%s: non-contig allocation fails\n",
__func__);
return -EFAULT;
}
int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count)
{
return npc_idx_free(rvu, mcam_idx, count, true);
}
int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type,
int prio, u16 *mcam_idx, int ref, int limit,
bool contig, int count, bool virt)
{
bool defrag_candidate = false;
int i, eidx, rc, bd;
bool ref_valid;
u16 vidx;
bd = npc_priv->bank_depth;
if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO &&
count == 1) {
rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
prio, contig, count, mcam_idx);
if (rc)
return rc;
goto add2map;
}
ref_valid = !!(limit || ref);
defrag_candidate = !ref_valid && !contig && virt;
if (!ref_valid) {
if (contig && count > npc_priv->subbank_depth)
goto try_noref_multi_subbank;
rc = npc_subbank_noref_alloc(rvu, key_type, contig,
count, mcam_idx);
if (!rc)
goto add2map;
try_noref_multi_subbank:
eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1;
if (prio == NPC_MCAM_HIGHER_PRIO)
rc = npc_multi_subbank_ref_alloc(rvu, key_type,
eidx, 0,
NPC_MCAM_HIGHER_PRIO,
contig, count,
mcam_idx);
else
rc = npc_multi_subbank_ref_alloc(rvu, key_type,
0, eidx,
NPC_MCAM_LOWER_PRIO,
contig, count,
mcam_idx);
if (!rc)
goto add2map;
return rc;
}
if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) ||
(prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) {
dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
__func__, ref, limit);
return -EINVAL;
}
if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) ||
(key_type == NPC_MCAM_KEY_X2 &&
(ref >= 2 * bd || limit >= 2 * bd))) {
dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n",
__func__, ref, limit);
return -EINVAL;
}
if (contig && count > npc_priv->subbank_depth)
goto try_ref_multi_subbank;
rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit,
prio, contig, count, mcam_idx);
if (!rc)
goto add2map;
try_ref_multi_subbank:
rc = npc_multi_subbank_ref_alloc(rvu, key_type,
ref, limit, prio,
contig, count, mcam_idx);
if (!rc)
goto add2map;
return rc;
add2map:
for (i = 0; i < count; i++) {
rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc);
if (rc)
goto err;
if (!defrag_candidate)
continue;
rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx);
if (rc) {
npc_del_from_pf_maps(rvu, mcam_idx[i]);
goto err;
}
mcam_idx[i] = vidx;
}
return 0;
err:
for (int j = 0; j < i; j++) {
npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j]));
if (!defrag_candidate)
continue;
npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL);
}
return rc;
}
void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free,
int *x4_free, int *sb_free)
{
struct npc_subbank *sb;
int i;
*x2_free = 0;
*x4_free = 0;
*sb_free = 0;
for (i = 0; i < npc_priv->num_subbanks; i++) {
sb = &npc_priv->sb[i];
mutex_lock(&sb->lock);
if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
(*sb_free)++;
goto next;
}
if (sb->key_type == NPC_MCAM_KEY_X4) {
(*x4_free) += sb->free_cnt;
goto next;
}
(*x2_free) += sb->free_cnt;
next:
mutex_unlock(&sb->lock);
}
}
int
rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu,
struct msg_req *req,
struct npc_cn20k_get_fcnt_rsp *rsp)
{
npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2,
&rsp->free_x4, &rsp->free_subbanks);
return 0;
}
int
rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu,
struct msg_req *req,
struct npc_cn20k_get_kex_cfg_rsp *rsp)
{
int extr, lt;
rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX);
rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX);
for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
rsp->intf_extr_lid[NIX_INTF_RX][extr] =
CN20K_GET_EXTR_LID(NIX_INTF_RX, extr);
rsp->intf_extr_lid[NIX_INTF_TX][extr] =
CN20K_GET_EXTR_LID(NIX_INTF_TX, extr);
}
for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) {
for (lt = 0; lt < NPC_MAX_LT; lt++) {
rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] =
CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt);
rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] =
CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt);
}
}
memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN);
return 0;
}
static u32 *subbank_srch_order;
static void npc_populate_restricted_idxs(int num_subbanks)
{
npc_subbank_restricted_idxs[0] = num_subbanks - 1;
npc_subbank_restricted_idxs[1] = 0;
}
static int npc_create_srch_order(int cnt)
{
int val = 0;
subbank_srch_order = kcalloc(cnt, sizeof(u32),
GFP_KERNEL);
if (!subbank_srch_order)
return -ENOMEM;
for (int i = 0; i < cnt; i += 2) {
subbank_srch_order[i] = cnt / 2 - val - 1;
subbank_srch_order[i + 1] = cnt / 2 + 1 + val;
val++;
}
subbank_srch_order[cnt - 1] = cnt / 2;
return 0;
}
static int npc_subbanks_srch_order_init(struct rvu *rvu)
{
struct npc_subbank *sb;
int sb_idx;
int i, j;
int rc;
for (i = 0; i < npc_priv->num_subbanks; i++) {
sb_idx = subbank_srch_order[i];
sb = &npc_priv->sb[sb_idx];
sb->arr_idx = i;
dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n",
__func__, sb->idx, sb->arr_idx);
rc = xa_err(xa_store(&npc_priv->xa_sb_free, sb->arr_idx,
xa_mk_value(sb->idx), GFP_KERNEL));
if (rc) {
dev_err(rvu->dev,
"%s: xa_store(xa_sb_free) failed at slot %d (sb=%d): %d\n",
__func__, i, sb_idx, rc);
for (j = 0; j < i; j++)
xa_erase(&npc_priv->xa_sb_free, j);
return rc;
}
}
return 0;
}
static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx)
{
mutex_init(&sb->lock);
sb->b0b = idx * npc_priv->subbank_depth;
sb->b0t = sb->b0b + npc_priv->subbank_depth - 1;
sb->b1b = npc_priv->bank_depth + idx * npc_priv->subbank_depth;
sb->b1t = sb->b1b + npc_priv->subbank_depth - 1;
sb->flags = NPC_SUBBANK_FLAG_FREE;
sb->idx = idx;
}
static int npc_pcifunc_map_create(struct rvu *rvu)
{
int pf, vf, numvfs;
int cnt = 0;
u16 pcifunc;
u64 cfg;
for (pf = 0; pf < rvu->hw->total_pfs; pf++) {
cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf));
numvfs = (cfg >> 12) & 0xFF;
if (!(cfg & BIT_ULL(20)))
goto chk_vfs;
if (cfg & BIT_ULL(21))
goto chk_vfs;
pcifunc = pf << 9;
xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc,
xa_mk_value(cnt), GFP_KERNEL);
cnt++;
chk_vfs:
for (vf = 0; vf < numvfs; vf++) {
pcifunc = (pf << 9) | (vf + 1);
xa_store(&npc_priv->xa_pf_map, (unsigned long)pcifunc,
xa_mk_value(cnt), GFP_KERNEL);
cnt++;
}
}
return cnt;
}
struct npc_defrag_node {
u8 idx;
u8 key_type;
bool valid;
bool refs;
u16 free_cnt;
u16 vidx_cnt;
u16 *vidx;
struct list_head list;
};
static bool npc_defrag_skip_restricted_sb(int sb_id)
{
int i;
if (!restrict_valid)
return false;
for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++)
if (sb_id == npc_subbank_restricted_idxs[i])
return true;
return false;
}
static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh)
{
struct npc_defrag_node *node, *tnode = NULL;
int min = INT_MAX;
list_for_each_entry(node, lh, list) {
if (!node->valid)
continue;
if (node->refs)
continue;
if (min > node->vidx_cnt) {
min = node->vidx_cnt;
tnode = node;
}
}
return tnode;
}
static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh)
{
struct npc_defrag_node *node, *tnode = NULL;
int max = INT_MIN;
list_for_each_entry(node, lh, list) {
if (!node->valid)
continue;
if (max < node->free_cnt) {
max = node->free_cnt;
tnode = node;
}
}
return tnode;
}
static int npc_defrag_alloc_free_slots(struct rvu *rvu,
struct npc_defrag_node *f,
int cnt, u16 *save)
{
int alloc_cnt1, alloc_cnt2;
struct npc_subbank *sb;
int rc, sb_off, i, err;
bool deleted;
sb = &npc_priv->sb[f->idx];
alloc_cnt1 = 0;
alloc_cnt2 = 0;
rc = __npc_subbank_alloc(rvu, sb,
f->key_type, sb->b0b,
sb->b0t,
NPC_MCAM_LOWER_PRIO,
false, cnt, save, cnt, true,
&alloc_cnt1);
if (alloc_cnt1 < cnt && f->key_type == NPC_MCAM_KEY_X2) {
rc = __npc_subbank_alloc(rvu, sb,
f->key_type, sb->b1b,
sb->b1t,
NPC_MCAM_LOWER_PRIO,
false, cnt - alloc_cnt1,
save + alloc_cnt1,
cnt - alloc_cnt1,
true, &alloc_cnt2);
}
if (alloc_cnt1 + alloc_cnt2 != cnt) {
dev_err(rvu->dev,
"%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n",
__func__, cnt, alloc_cnt1, alloc_cnt2);
rc = -ENOSPC;
goto fail_free_alloc;
}
return 0;
fail_free_alloc:
for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) {
err = npc_mcam_idx_2_subbank_idx(rvu, save[i],
&sb, &sb_off);
if (err) {
dev_err(rvu->dev,
"%s: Error to find subbank for mcam idx=%u\n",
__func__, save[i]);
break;
}
deleted = __npc_subbank_free(rvu, sb, sb_off);
if (!deleted) {
dev_err(rvu->dev,
"%s: Error to free mcam idx=%u\n",
__func__, save[i]);
break;
}
}
return rc;
}
static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx,
u16 new_midx, u16 vidx)
{
struct npc_defrag_show_node *node;
node = kcalloc(1, sizeof(*node), GFP_KERNEL);
if (!node)
return -ENOMEM;
node->old_midx = old_midx;
node->new_midx = new_midx;
node->vidx = vidx;
INIT_LIST_HEAD(&node->list);
mutex_lock(&npc_priv->lock);
list_add_tail(&node->list, &npc_priv->defrag_lh);
mutex_unlock(&npc_priv->lock);
return 0;
}
static
int npc_defrag_move_vdx_to_free(struct rvu *rvu,
struct npc_defrag_node *f,
struct npc_defrag_node *v,
int cnt, u16 *save)
{
u16 new_midx, old_midx, vidx, target_pf;
struct npc_mcam *mcam = &rvu->hw->mcam;
struct rvu_npc_mcam_rule *rule, *tmp;
int i, vidx_cnt, rc, sb_off;
struct npc_subbank *sb;
bool deleted;
u16 pcifunc;
int blkaddr;
void *map;
u8 bank;
u16 midx;
u64 stats;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
vidx_cnt = v->vidx_cnt;
for (i = 0; i < cnt; i++) {
vidx = v->vidx[vidx_cnt - i - 1];
old_midx = npc_vidx2idx(vidx);
new_midx = save[cnt - i - 1];
dev_dbg(rvu->dev,
"%s: Moving %u ---> %u (vidx=%u)\n",
__func__,
old_midx, new_midx, vidx);
rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx);
if (rc)
dev_err(rvu->dev,
"%s: Error happened to add to show list vidx=%u\n",
__func__, vidx);
rc = npc_vidx_maps_modify(rvu, vidx, new_midx);
if (rc)
return rc;
midx = old_midx % mcam->banksize;
bank = old_midx / mcam->banksize;
stats = rvu_read64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx,
bank));
if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false)) {
dev_err(rvu->dev,
"%s: Error happened while disabling old_mid=%u\n",
__func__, old_midx);
return -EFAULT;
}
if (npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx)) {
dev_err(rvu->dev,
"%s: Error happened while copying old_midx=%u new_midx=%u\n",
__func__, old_midx, new_midx);
return -EFAULT;
}
if (npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true)) {
dev_err(rvu->dev,
"%s: Error happened while enabling new_mid=%u\n",
__func__, new_midx);
return -EFAULT;
}
midx = new_midx % mcam->banksize;
bank = new_midx / mcam->banksize;
rvu_write64(rvu, blkaddr,
NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank),
stats);
rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx,
&sb, &sb_off);
if (rc) {
dev_err(rvu->dev,
"%s: Unable to calculate subbank off for mcamidx=%u\n",
__func__, old_midx);
return rc;
}
deleted = __npc_subbank_free(rvu, sb, sb_off);
if (!deleted) {
dev_err(rvu->dev,
"%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n",
__func__, old_midx, sb->idx, sb_off);
return -EFAULT;
}
map = xa_load(&npc_priv->xa_idx2pf_map, old_midx);
pcifunc = xa_to_value(map);
rc = npc_del_from_pf_maps(rvu, old_midx);
if (rc) {
dev_err(rvu->dev,
"%s: Failed to delete pf maps for mcamidx=%u\n",
__func__, old_midx);
return rc;
}
rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc);
if (rc) {
dev_err(rvu->dev,
"%s: Failed to add pf maps for mcamidx=%u\n",
__func__, new_midx);
return rc;
}
mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP;
mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP;
npc_mcam_clear_bit(mcam, old_midx);
mcam->entry2pfvf_map[new_midx] = pcifunc;
mcam->entry2cntr_map[new_midx] = new_midx;
target_pf = mcam->entry2target_pffunc[old_midx];
mcam->entry2target_pffunc[new_midx] = target_pf;
mcam->entry2target_pffunc[old_midx] = NPC_MCAM_INVALID_MAP;
npc_mcam_set_bit(mcam, new_midx);
list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
if (rule->entry != old_midx)
continue;
rule->entry = new_midx;
break;
}
v->vidx[vidx_cnt - i - 1] = -1;
save[cnt - i - 1] = -1;
f->free_cnt--;
v->vidx_cnt--;
}
return 0;
}
static int npc_defrag_process(struct rvu *rvu, struct list_head *lh)
{
struct npc_defrag_node *v = NULL;
struct npc_defrag_node *f = NULL;
int rc = 0, cnt;
u16 *save;
while (1) {
if (!v) {
v = npc_subbank_min_vidx(lh);
if (!v)
break;
}
if (!f) {
f = npc_subbank_max_free(lh);
if (!f)
break;
}
if (!v->vidx_cnt) {
list_del_init(&v->list);
v = NULL;
continue;
}
if (!f->free_cnt) {
list_del_init(&f->list);
f = NULL;
continue;
}
if (f == v) {
list_del_init(&f->list);
f = NULL;
continue;
}
cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt :
f->free_cnt;
dev_dbg(rvu->dev,
"%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n",
__func__, cnt, f->free_cnt, f->idx,
v->vidx_cnt, v->idx);
save = kcalloc(cnt, sizeof(*save), GFP_KERNEL);
if (!save) {
rc = -ENOMEM;
goto err;
}
rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save);
if (rc) {
kfree(save);
goto err;
}
rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save);
if (rc) {
kfree(save);
goto err;
}
kfree(save);
if (!f->free_cnt) {
list_del_init(&f->list);
f = NULL;
}
if (!v->vidx_cnt) {
list_del_init(&v->list);
v = NULL;
}
}
err:
return rc;
}
static void npc_defrag_list_clear(void)
{
struct npc_defrag_show_node *node, *next;
mutex_lock(&npc_priv->lock);
list_for_each_entry_safe(node, next, &npc_priv->defrag_lh, list) {
list_del_init(&node->list);
kfree(node);
}
mutex_unlock(&npc_priv->lock);
}
static void npc_lock_all_subbank(void)
{
int i;
for (i = 0; i < npc_priv->num_subbanks; i++)
mutex_lock(&npc_priv->sb[i].lock);
}
static void npc_unlock_all_subbank(void)
{
int i;
for (i = npc_priv->num_subbanks - 1; i >= 0; i--)
mutex_unlock(&npc_priv->sb[i].lock);
}
int npc_cn20k_search_order_set(struct rvu *rvu,
u64 narr[MAX_NUM_SUB_BANKS], int cnt)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int rsrc[2][MAX_NUM_SUB_BANKS] = { };
u8 save[MAX_NUM_SUB_BANKS] = { };
struct npc_subbank *sb;
struct xarray *xa;
int prio, rc, err;
int sb_idx;
enum {
FREE = 0,
USED = 1,
};
if (cnt != npc_priv->num_subbanks) {
dev_err(rvu->dev, "Number of entries(%u) != %u\n",
cnt, npc_priv->num_subbanks);
return -EINVAL;
}
mutex_lock(&mcam->lock);
npc_lock_all_subbank();
for (sb_idx = 0; sb_idx < cnt; sb_idx++) {
sb = &npc_priv->sb[sb_idx];
save[sb->idx] = sb->arr_idx;
}
for (prio = 0; prio < cnt; prio++) {
sb_idx = narr[prio];
sb = &npc_priv->sb[sb_idx];
if (sb->flags & NPC_SUBBANK_FLAG_USED)
xa = &npc_priv->xa_sb_used;
else
xa = &npc_priv->xa_sb_free;
rc = xa_err(xa_store(xa, prio,
xa_mk_value(sb_idx), GFP_KERNEL));
if (rc) {
dev_err(rvu->dev,
"Setting arr_idx=%d for sb=%d failed\n",
sb->arr_idx, sb_idx);
goto fail;
}
if (sb->flags & NPC_SUBBANK_FLAG_USED) {
rsrc[USED][sb->arr_idx] -= 1;
rsrc[USED][prio] += 1;
} else {
rsrc[FREE][sb->arr_idx] -= 1;
rsrc[FREE][prio] += 1;
}
sb->arr_idx = prio;
}
for (prio = 0; prio < cnt; prio++) {
if (rsrc[FREE][prio] == -1)
xa_erase(&npc_priv->xa_sb_free, prio);
if (rsrc[USED][prio] == -1)
xa_erase(&npc_priv->xa_sb_used, prio);
}
for (int i = 0; i < cnt; i++)
subbank_srch_order[i] = (u32)narr[i];
restrict_valid = false;
npc_unlock_all_subbank();
mutex_unlock(&mcam->lock);
return 0;
fail:
for (prio = 0; prio < cnt; prio++) {
if (rsrc[FREE][prio] == 1)
xa_erase(&npc_priv->xa_sb_free, prio);
if (rsrc[USED][prio] == 1)
xa_erase(&npc_priv->xa_sb_used, prio);
}
for (sb_idx = 0; sb_idx < cnt; sb_idx++) {
sb = &npc_priv->sb[sb_idx];
sb->arr_idx = save[sb_idx];
if (sb->flags & NPC_SUBBANK_FLAG_USED)
xa = &npc_priv->xa_sb_used;
else
xa = &npc_priv->xa_sb_free;
err = xa_err(xa_store(xa, sb->arr_idx,
xa_mk_value(sb->idx), GFP_KERNEL));
WARN(!!err, "Failed to rollback sb=%u idx=%u\n",
sb->idx, sb->arr_idx);
}
npc_unlock_all_subbank();
mutex_unlock(&mcam->lock);
return rc;
}
const u32 *npc_cn20k_search_order_get(bool *restricted_order, u32 *sz)
{
*restricted_order = restrict_valid;
*sz = npc_priv->num_subbanks;
return subbank_srch_order;
}
int npc_cn20k_defrag(struct rvu *rvu)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
struct npc_defrag_node *node, *tnode;
struct list_head x4lh, x2lh, *lh;
int rc = 0, i, sb_off, tot;
struct npc_subbank *sb;
unsigned long index;
void *map;
u16 midx;
npc_defrag_list_clear();
INIT_LIST_HEAD(&x4lh);
INIT_LIST_HEAD(&x2lh);
node = kcalloc(npc_priv->num_subbanks, sizeof(*node), GFP_KERNEL);
if (!node)
return -ENOMEM;
mutex_lock(&mcam->lock);
npc_lock_all_subbank();
for (i = 0; i < npc_priv->num_subbanks; i++) {
sb = &npc_priv->sb[i];
node[i].idx = i;
node[i].key_type = sb->key_type;
node[i].free_cnt = sb->free_cnt;
node[i].vidx = kcalloc(npc_priv->subbank_depth * 2,
sizeof(*node[i].vidx),
GFP_KERNEL);
if (!node[i].vidx) {
rc = -ENOMEM;
goto free_vidx;
}
if (sb->flags & NPC_SUBBANK_FLAG_FREE) {
node[i].valid = false;
continue;
}
if (npc_defrag_skip_restricted_sb(i)) {
node[i].valid = false;
continue;
}
node[i].valid = true;
INIT_LIST_HEAD(&node[i].list);
lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh;
list_add_tail(&node[i].list, lh);
}
xa_for_each_start(&npc_priv->xa_vidx2idx_map, index, map,
npc_priv->bank_depth * 2) {
midx = xa_to_value(map);
rc = npc_mcam_idx_2_subbank_idx(rvu, midx,
&sb, &sb_off);
if (rc) {
dev_err(rvu->dev,
"%s: Error to get mcam_idx for vidx=%lu\n",
__func__, index);
goto free_vidx;
}
tnode = &node[sb->idx];
tnode->vidx[tnode->vidx_cnt] = index;
tnode->vidx_cnt++;
}
for (i = 0; i < npc_priv->num_subbanks; i++) {
tnode = &node[i];
if (!tnode->valid)
continue;
tot = (tnode->key_type == NPC_MCAM_KEY_X2) ?
npc_priv->subbank_depth * 2 : npc_priv->subbank_depth;
if (node[i].vidx_cnt != tot - tnode->free_cnt)
tnode->refs = true;
}
rc = npc_defrag_process(rvu, &x2lh);
if (rc)
goto free_vidx;
rc = npc_defrag_process(rvu, &x4lh);
if (rc)
goto free_vidx;
free_vidx:
npc_unlock_all_subbank();
mutex_unlock(&mcam->lock);
for (i = 0; i < npc_priv->num_subbanks; i++)
kfree(node[i].vidx);
kfree(node);
return rc;
}
int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req,
struct msg_rsp *rsp)
{
return npc_cn20k_defrag(rvu);
}
int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast,
u16 *mcast, u16 *promisc, u16 *ucast)
{
u16 *ptr[4] = {promisc, mcast, bcast, ucast};
unsigned long idx;
bool set = false;
void *val;
int i, j;
for (i = 0; i < ARRAY_SIZE(ptr); i++) {
if (!ptr[i])
continue;
*ptr[i] = USHRT_MAX;
}
if (!npc_priv)
return 0;
if (is_lbk_vf(rvu, pcifunc)) {
if (!ptr[0])
return -EINVAL;
idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
if (!val) {
pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
pcifunc);
return -ESRCH;
}
*ptr[0] = xa_to_value(val);
return 0;
}
if (is_vf(pcifunc)) {
if (!ptr[3])
return -EINVAL;
idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
if (!val) {
pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
pcifunc);
return -ESRCH;
}
*ptr[3] = xa_to_value(val);
return 0;
}
for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++,
j++) {
if (!ptr[j])
continue;
idx = NPC_DFT_RULE_ID_MK(pcifunc, i);
val = xa_load(&npc_priv->xa_pf2dfl_rmap, idx);
if (!val) {
pr_debug("%s: Failed to find %s index for pcifunc=%#x\n",
__func__,
npc_dft_rule_name[i], pcifunc);
continue;
}
*ptr[j] = xa_to_value(val);
set = true;
}
return set ? 0 : -ESRCH;
}
int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req,
struct npc_get_pfl_info_rsp *rsp)
{
if (!is_cn20k(rvu->pdev)) {
dev_err(rvu->dev, "Mbox support is only for cn20k\n");
return -EOPNOTSUPP;
}
rsp->kw_type = npc_priv->kw;
rsp->x4_slots = npc_priv->bank_depth;
return 0;
}
int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu,
struct npc_get_num_kws_req *req,
struct npc_get_num_kws_rsp *rsp)
{
u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 };
struct rvu_npc_mcam_rule dummy = { 0 };
struct mcam_entry_mdata mdata = { };
struct npc_install_flow_req *fl;
int i, cnt = 0, blkaddr;
if (!is_cn20k(rvu->pdev)) {
dev_err(rvu->dev, "Mbox support is only for cn20k\n");
return -EOPNOTSUPP;
}
fl = &req->fl;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0) {
dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
return NPC_MCAM_INVALID_REQ;
}
mdata.kw = kw;
mdata.kw_mask = kw_mask;
npc_update_flow(rvu, &mdata, fl->features, &fl->packet,
&fl->mask, &dummy, fl->intf, blkaddr);
for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) {
if (kw[i] || kw_mask[i]) {
cnt = i + 1;
break;
}
}
rsp->kws = cnt;
return 0;
}
int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req,
struct npc_get_dft_rl_idxs_rsp *rsp)
{
u16 bcast, mcast, promisc, ucast;
u16 pcifunc;
int rc;
if (!is_cn20k(rvu->pdev)) {
dev_err(rvu->dev, "Mbox support is only for cn20k\n");
return -EOPNOTSUPP;
}
pcifunc = req->hdr.pcifunc;
rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast,
&promisc, &ucast);
if (rc)
return rc;
rsp->bcast = bcast;
rsp->mcast = mcast;
rsp->promisc = promisc;
rsp->ucast = ucast;
return 0;
}
bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc)
{
return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) ||
is_lbk_vf(rvu, pcifunc);
}
void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
u16 ptr[4] = {[0 ... 3] = USHRT_MAX};
struct rvu_npc_mcam_rule *rule, *tmp;
unsigned long index;
int blkaddr, rc, i;
void *map;
if (!npc_priv)
return;
if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
dev_dbg(rvu->dev,
"%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
__func__, pcifunc);
return;
}
rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1],
&ptr[2], &ptr[3]);
if (rc)
return;
if (is_lbk_vf(rvu, pcifunc)) {
index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
if (!map)
dev_dbg(rvu->dev,
"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
pcifunc);
goto free_rules;
}
if (is_vf(pcifunc)) {
index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
if (!map)
dev_dbg(rvu->dev,
"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
pcifunc);
goto free_rules;
}
for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) {
index = NPC_DFT_RULE_ID_MK(pcifunc, i);
map = xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
if (!map)
dev_dbg(rvu->dev,
"%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n",
__func__, npc_dft_rule_name[i],
pcifunc);
}
free_rules:
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0)
return;
for (int i = 0; i < 4; i++) {
if (ptr[i] == USHRT_MAX)
continue;
mutex_lock(&mcam->lock);
npc_mcam_clear_bit(mcam, ptr[i]);
mcam->entry2pfvf_map[ptr[i]] = NPC_MCAM_INVALID_MAP;
npc_cn20k_enable_mcam_entry(rvu, blkaddr, ptr[i], false);
mcam->entry2target_pffunc[ptr[i]] = 0x0;
mutex_unlock(&mcam->lock);
rc = npc_cn20k_idx_free(rvu, &ptr[i], 1);
if (rc) {
WARN(1, "%s Error deleting default entries (pcifunc=%#x) mcam_idx=%u\n",
__func__, pcifunc, ptr[i]);
return;
}
}
mutex_lock(&mcam->lock);
list_for_each_entry_safe(rule, tmp, &mcam->mcam_rules, list) {
for (int i = 0; i < 4; i++) {
if (ptr[i] != rule->entry)
continue;
list_del(&rule->list);
kfree(rule);
break;
}
}
mutex_unlock(&mcam->lock);
}
int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc)
{
struct npc_mcam_free_entry_req free_req = { 0 };
u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc;
struct npc_mcam_alloc_entry_req req = { 0 };
struct npc_mcam_alloc_entry_rsp rsp = { 0 };
int ret, eidx, i, k, pf, cnt;
struct rvu_pfvf *pfvf;
unsigned long index;
struct msg_rsp free_rsp;
u16 b, m, p, u;
if (!npc_priv)
return 0;
if (!npc_is_cgx_or_lbk(rvu, pcifunc)) {
dev_dbg(rvu->dev,
"%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n",
__func__, pcifunc);
return 0;
}
ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u);
if (!ret) {
dev_dbg(rvu->dev,
"%s: default rules are already installed (pcifunc=%#x)\n",
__func__, pcifunc);
dev_dbg(rvu->dev,
"%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n",
__func__, b, m, p, u);
return 0;
}
eidx = 2 * npc_priv->bank_depth - 1;
cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx);
if (is_vf(pcifunc)) {
pf = rvu_get_pf(rvu->pdev, pcifunc);
pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT;
ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL,
NULL, NULL, &pf_ucast);
if (!ret)
eidx = pf_ucast;
}
pfvf = rvu_get_pfvf(rvu, pcifunc);
pfvf->hw_prio = NPC_DFT_RULE_PRIO;
if (npc_priv->kw == NPC_MCAM_KEY_X4) {
req.kw_type = NPC_MCAM_KEY_X4;
req.ref_entry = eidx & (npc_priv->bank_depth - 1);
} else {
req.kw_type = NPC_MCAM_KEY_X2;
req.ref_entry = eidx;
}
req.contig = false;
req.ref_prio = NPC_MCAM_HIGHER_PRIO;
req.count = cnt;
req.hdr.pcifunc = pcifunc;
ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
if (!ret) {
for (i = 0; i < cnt; i++)
mcam_idx[i] = rsp.entry_list[i];
goto chk_sanity;
}
if (!is_vf(pcifunc)) {
dev_err(rvu->dev,
"%s: Default index allocation failed for pcifunc=%#x\n",
__func__, pcifunc);
return ret;
}
req.contig = false;
req.count = cnt;
req.hdr.pcifunc = pcifunc;
req.ref_prio = NPC_MCAM_LOWER_PRIO;
if (npc_priv->kw == NPC_MCAM_KEY_X4) {
req.kw_type = NPC_MCAM_KEY_X4;
req.ref_entry = eidx & (npc_priv->bank_depth - 1);
} else {
req.kw_type = NPC_MCAM_KEY_X2;
req.ref_entry = eidx;
}
ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp);
if (ret) {
dev_err(rvu->dev,
"%s: Default index allocation failed for pcifunc=%#x\n",
__func__, pcifunc);
return ret;
}
for (i = 0; i < cnt; i++)
mcam_idx[i] = rsp.entry_list[i];
pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1;
chk_sanity:
if (is_lbk_vf(rvu, pcifunc)) {
index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID);
ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
xa_mk_value(mcam_idx[0]), GFP_KERNEL);
if (ret) {
dev_err(rvu->dev,
"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID],
pcifunc);
goto err;
}
goto done;
}
if (is_vf(pcifunc)) {
index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID);
ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
xa_mk_value(mcam_idx[0]), GFP_KERNEL);
if (ret) {
dev_err(rvu->dev,
"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
__func__,
npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID],
pcifunc);
goto err;
}
goto done;
}
for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID &&
k < cnt; i++, k++) {
index = NPC_DFT_RULE_ID_MK(pcifunc, i);
ret = xa_insert(&npc_priv->xa_pf2dfl_rmap, index,
xa_mk_value(mcam_idx[k]), GFP_KERNEL);
if (ret) {
dev_err(rvu->dev,
"%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n",
__func__, npc_dft_rule_name[i],
pcifunc);
for (int p = NPC_DFT_RULE_START_ID; p < i; p++) {
index = NPC_DFT_RULE_ID_MK(pcifunc, p);
xa_erase(&npc_priv->xa_pf2dfl_rmap, index);
}
goto err;
}
}
done:
return 0;
err:
free_req.hdr.pcifunc = pcifunc;
free_req.all = 1;
ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp);
if (ret)
dev_err(rvu->dev,
"%s: Error deleting default entries (pcifunc=%#x\n",
__func__, pcifunc);
return -EFAULT;
}
static int npc_priv_init(struct rvu *rvu)
{
struct npc_mcam *mcam = &rvu->hw->mcam;
int blkaddr, num_banks, bank_depth;
int num_subbanks, subbank_depth;
u64 npc_const1, npc_const2 = 0;
struct npc_subbank *sb;
int ret = -ENOMEM;
u64 cfg;
int i;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0) {
dev_err(rvu->dev, "%s: NPC block not implemented\n",
__func__);
return -ENODEV;
}
npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1);
if (npc_const1 & BIT_ULL(63))
npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2);
num_banks = mcam->banks;
if (!num_banks || num_banks > MAX_NUM_BANKS) {
dev_err(rvu->dev,
"Number of banks(%u) is invalid\n", num_banks);
return -EINVAL;
}
num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2);
if (!num_subbanks || num_subbanks > MAX_NUM_SUB_BANKS) {
dev_err(rvu->dev,
"Number of subbanks is invalid %u\n",
num_subbanks);
return -EFAULT;
}
if (num_subbanks & (num_subbanks - 1)) {
dev_err(rvu->dev,
"subbanks cnt(%u) should be a power of 2\n",
num_subbanks);
return -EINVAL;
}
bank_depth = mcam->banksize;
if (!bank_depth || bank_depth % num_subbanks) {
dev_err(rvu->dev,
"Bank depth(%u) should be a multiple of num_subbanks(%u)\n",
bank_depth, num_subbanks);
return -EINVAL;
}
subbank_depth = bank_depth / num_subbanks;
if (!subbank_depth || subbank_depth > MAX_SUBBANK_DEPTH) {
dev_err(rvu->dev,
"Invalid subbank depth %u\n",
subbank_depth);
return -EINVAL;
}
npc_priv = kcalloc(1, sizeof(*npc_priv), GFP_KERNEL);
if (!npc_priv)
return -ENOMEM;
npc_priv->num_banks = num_banks;
npc_priv->num_subbanks = num_subbanks;
npc_priv->bank_depth = bank_depth;
npc_priv->subbank_depth = subbank_depth;
cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0));
npc_priv->kw = FIELD_GET(GENMASK_ULL(34, 32), cfg);
dev_info(rvu->dev,
"banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n",
num_banks, bank_depth, num_subbanks, subbank_depth,
npc_kw_name[npc_priv->kw]);
npc_priv->sb = kcalloc(num_subbanks, sizeof(struct npc_subbank),
GFP_KERNEL);
if (!npc_priv->sb)
goto fail1;
xa_init_flags(&npc_priv->xa_sb_used, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_sb_free, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_idx2pf_map, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_pf_map, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_pf2dfl_rmap, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_idx2vidx_map, XA_FLAGS_ALLOC);
xa_init_flags(&npc_priv->xa_vidx2idx_map, XA_FLAGS_ALLOC);
if (npc_create_srch_order(num_subbanks))
goto fail2;
npc_populate_restricted_idxs(num_subbanks);
for (i = 0, sb = npc_priv->sb; i < num_subbanks; i++, sb++)
npc_subbank_init(rvu, sb, i);
ret = npc_subbanks_srch_order_init(rvu);
if (ret)
goto fail3;
npc_priv->pf_cnt = npc_pcifunc_map_create(rvu);
npc_priv->xa_pf2idx_map = kcalloc(npc_priv->pf_cnt,
sizeof(struct xarray),
GFP_KERNEL);
if (!npc_priv->xa_pf2idx_map) {
ret = -ENOMEM;
goto fail3;
}
for (i = 0; i < npc_priv->pf_cnt; i++)
xa_init_flags(&npc_priv->xa_pf2idx_map[i], XA_FLAGS_ALLOC);
INIT_LIST_HEAD(&npc_priv->defrag_lh);
mutex_init(&npc_priv->lock);
return 0;
fail3:
kfree(subbank_srch_order);
subbank_srch_order = NULL;
fail2:
xa_destroy(&npc_priv->xa_sb_used);
xa_destroy(&npc_priv->xa_sb_free);
xa_destroy(&npc_priv->xa_idx2pf_map);
xa_destroy(&npc_priv->xa_pf_map);
xa_destroy(&npc_priv->xa_pf2dfl_rmap);
xa_destroy(&npc_priv->xa_idx2vidx_map);
xa_destroy(&npc_priv->xa_vidx2idx_map);
kfree(npc_priv->sb);
npc_priv->sb = NULL;
fail1:
kfree(npc_priv);
npc_priv = NULL;
return ret;
}
void npc_cn20k_deinit(struct rvu *rvu)
{
int i;
if (!npc_priv)
return;
xa_destroy(&npc_priv->xa_sb_used);
xa_destroy(&npc_priv->xa_sb_free);
xa_destroy(&npc_priv->xa_idx2pf_map);
xa_destroy(&npc_priv->xa_pf_map);
xa_destroy(&npc_priv->xa_pf2dfl_rmap);
xa_destroy(&npc_priv->xa_idx2vidx_map);
xa_destroy(&npc_priv->xa_vidx2idx_map);
for (i = 0; i < npc_priv->pf_cnt; i++)
xa_destroy(&npc_priv->xa_pf2idx_map[i]);
kfree(npc_priv->xa_pf2idx_map);
kfree(npc_priv->sb);
kfree(subbank_srch_order);
bitmap_clear(npc_priv->en_map, 0, MAX_NUM_BANKS * MAX_NUM_SUB_BANKS *
MAX_SUBBANK_DEPTH);
npc_defrag_list_clear();
kfree(npc_priv);
npc_priv = NULL;
}
static int npc_setup_mcam_section(struct rvu *rvu, int key_type)
{
int blkaddr, sec;
blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0);
if (blkaddr < 0) {
dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__);
return -ENODEV;
}
for (sec = 0; sec < npc_priv->num_subbanks; sec++)
rvu_write64(rvu, blkaddr,
NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type);
return 0;
}
int npc_cn20k_init(struct rvu *rvu)
{
int err;
err = npc_priv_init(rvu);
if (err) {
dev_err(rvu->dev, "%s: Error to init\n",
__func__);
return err;
}
err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2);
if (err) {
dev_err(rvu->dev, "%s: mcam section configuration failure\n",
__func__);
goto fail;
}
return 0;
fail:
npc_cn20k_deinit(rvu);
return err;
}
