#include <linux/ethtool.h>
#include <net/flow_offload.h>
#include <net/vxlan.h>
#include "hclge_fd.h"
#include "hclge_main.h"
static const struct key_info meta_data_key_info[] = {
{ PACKET_TYPE_ID, 6 },
{ IP_FRAGEMENT, 1 },
{ ROCE_TYPE, 1 },
{ NEXT_KEY, 5 },
{ VLAN_NUMBER, 2 },
{ SRC_VPORT, 12 },
{ DST_VPORT, 12 },
{ TUNNEL_PACKET, 1 },
};
static const struct key_info tuple_key_info[] = {
{ OUTER_DST_MAC, 48, KEY_OPT_MAC, -1, -1 },
{ OUTER_SRC_MAC, 48, KEY_OPT_MAC, -1, -1 },
{ OUTER_VLAN_TAG_FST, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_ETH_TYPE, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_L2_RSV, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_IP_TOS, 8, KEY_OPT_U8, -1, -1 },
{ OUTER_IP_PROTO, 8, KEY_OPT_U8, -1, -1 },
{ OUTER_SRC_IP, 32, KEY_OPT_IP, -1, -1 },
{ OUTER_DST_IP, 32, KEY_OPT_IP, -1, -1 },
{ OUTER_L3_RSV, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_SRC_PORT, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_DST_PORT, 16, KEY_OPT_LE16, -1, -1 },
{ OUTER_L4_RSV, 32, KEY_OPT_LE32, -1, -1 },
{ OUTER_TUN_VNI, 24, KEY_OPT_VNI,
offsetof(struct hclge_fd_rule, tuples.outer_tun_vni),
offsetof(struct hclge_fd_rule, tuples_mask.outer_tun_vni) },
{ OUTER_TUN_FLOW_ID, 8, KEY_OPT_U8, -1, -1 },
{ INNER_DST_MAC, 48, KEY_OPT_MAC,
offsetof(struct hclge_fd_rule, tuples.dst_mac),
offsetof(struct hclge_fd_rule, tuples_mask.dst_mac) },
{ INNER_SRC_MAC, 48, KEY_OPT_MAC,
offsetof(struct hclge_fd_rule, tuples.src_mac),
offsetof(struct hclge_fd_rule, tuples_mask.src_mac) },
{ INNER_VLAN_TAG_FST, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.vlan_tag1),
offsetof(struct hclge_fd_rule, tuples_mask.vlan_tag1) },
{ INNER_VLAN_TAG_SEC, 16, KEY_OPT_LE16, -1, -1 },
{ INNER_ETH_TYPE, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.ether_proto),
offsetof(struct hclge_fd_rule, tuples_mask.ether_proto) },
{ INNER_L2_RSV, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.l2_user_def),
offsetof(struct hclge_fd_rule, tuples_mask.l2_user_def) },
{ INNER_IP_TOS, 8, KEY_OPT_U8,
offsetof(struct hclge_fd_rule, tuples.ip_tos),
offsetof(struct hclge_fd_rule, tuples_mask.ip_tos) },
{ INNER_IP_PROTO, 8, KEY_OPT_U8,
offsetof(struct hclge_fd_rule, tuples.ip_proto),
offsetof(struct hclge_fd_rule, tuples_mask.ip_proto) },
{ INNER_SRC_IP, 32, KEY_OPT_IP,
offsetof(struct hclge_fd_rule, tuples.src_ip),
offsetof(struct hclge_fd_rule, tuples_mask.src_ip) },
{ INNER_DST_IP, 32, KEY_OPT_IP,
offsetof(struct hclge_fd_rule, tuples.dst_ip),
offsetof(struct hclge_fd_rule, tuples_mask.dst_ip) },
{ INNER_L3_RSV, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.l3_user_def),
offsetof(struct hclge_fd_rule, tuples_mask.l3_user_def) },
{ INNER_SRC_PORT, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.src_port),
offsetof(struct hclge_fd_rule, tuples_mask.src_port) },
{ INNER_DST_PORT, 16, KEY_OPT_LE16,
offsetof(struct hclge_fd_rule, tuples.dst_port),
offsetof(struct hclge_fd_rule, tuples_mask.dst_port) },
{ INNER_L4_RSV, 32, KEY_OPT_LE32,
offsetof(struct hclge_fd_rule, tuples.l4_user_def),
offsetof(struct hclge_fd_rule, tuples_mask.l4_user_def) },
};
static void hclge_sync_fd_state(struct hclge_dev *hdev)
{
if (hlist_empty(&hdev->fd_rule_list))
hdev->fd_active_type = HCLGE_FD_RULE_NONE;
}
static void hclge_fd_inc_rule_cnt(struct hclge_dev *hdev, u16 location)
{
if (!test_bit(location, hdev->fd_bmap)) {
set_bit(location, hdev->fd_bmap);
hdev->hclge_fd_rule_num++;
}
}
static void hclge_fd_dec_rule_cnt(struct hclge_dev *hdev, u16 location)
{
if (test_bit(location, hdev->fd_bmap)) {
clear_bit(location, hdev->fd_bmap);
hdev->hclge_fd_rule_num--;
}
}
static void hclge_fd_free_node(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
hlist_del(&rule->rule_node);
kfree(rule);
hclge_sync_fd_state(hdev);
}
static void hclge_update_fd_rule_node(struct hclge_dev *hdev,
struct hclge_fd_rule *old_rule,
struct hclge_fd_rule *new_rule,
enum HCLGE_FD_NODE_STATE state)
{
switch (state) {
case HCLGE_FD_TO_ADD:
case HCLGE_FD_ACTIVE:
new_rule->rule_node.next = old_rule->rule_node.next;
new_rule->rule_node.pprev = old_rule->rule_node.pprev;
memcpy(old_rule, new_rule, sizeof(*old_rule));
kfree(new_rule);
break;
case HCLGE_FD_DELETED:
hclge_fd_dec_rule_cnt(hdev, old_rule->location);
hclge_fd_free_node(hdev, old_rule);
break;
case HCLGE_FD_TO_DEL:
if (old_rule->state == HCLGE_FD_TO_ADD) {
hclge_fd_dec_rule_cnt(hdev, old_rule->location);
hclge_fd_free_node(hdev, old_rule);
return;
}
old_rule->state = HCLGE_FD_TO_DEL;
break;
}
}
static struct hclge_fd_rule *hclge_find_fd_rule(struct hlist_head *hlist,
u16 location,
struct hclge_fd_rule **parent)
{
struct hclge_fd_rule *rule;
struct hlist_node *node;
hlist_for_each_entry_safe(rule, node, hlist, rule_node) {
if (rule->location == location)
return rule;
else if (rule->location > location)
return NULL;
*parent = rule;
}
return NULL;
}
static void hclge_fd_insert_rule_node(struct hlist_head *hlist,
struct hclge_fd_rule *rule,
struct hclge_fd_rule *parent)
{
INIT_HLIST_NODE(&rule->rule_node);
if (parent)
hlist_add_behind(&rule->rule_node, &parent->rule_node);
else
hlist_add_head(&rule->rule_node, hlist);
}
static int hclge_fd_set_user_def_cmd(struct hclge_dev *hdev,
struct hclge_fd_user_def_cfg *cfg)
{
struct hclge_fd_user_def_cfg_cmd *req;
struct hclge_desc desc;
u16 data = 0;
int ret;
hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_USER_DEF_OP, false);
req = (struct hclge_fd_user_def_cfg_cmd *)desc.data;
hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[0].ref_cnt > 0);
hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M,
HCLGE_FD_USER_DEF_OFT_S, cfg[0].offset);
req->ol2_cfg = cpu_to_le16(data);
data = 0;
hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[1].ref_cnt > 0);
hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M,
HCLGE_FD_USER_DEF_OFT_S, cfg[1].offset);
req->ol3_cfg = cpu_to_le16(data);
data = 0;
hnae3_set_bit(data, HCLGE_FD_USER_DEF_EN_B, cfg[2].ref_cnt > 0);
hnae3_set_field(data, HCLGE_FD_USER_DEF_OFT_M,
HCLGE_FD_USER_DEF_OFT_S, cfg[2].offset);
req->ol4_cfg = cpu_to_le16(data);
ret = hclge_cmd_send(&hdev->hw, &desc, 1);
if (ret)
dev_err(&hdev->pdev->dev,
"failed to set fd user def data, ret= %d\n", ret);
return ret;
}
static void hclge_sync_fd_user_def_cfg(struct hclge_dev *hdev, bool locked)
{
int ret;
if (!test_and_clear_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state))
return;
if (!locked)
spin_lock_bh(&hdev->fd_rule_lock);
ret = hclge_fd_set_user_def_cmd(hdev, hdev->fd_cfg.user_def_cfg);
if (ret)
set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state);
if (!locked)
spin_unlock_bh(&hdev->fd_rule_lock);
}
static int hclge_fd_check_user_def_refcnt(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
struct hlist_head *hlist = &hdev->fd_rule_list;
struct hclge_fd_rule *fd_rule, *parent = NULL;
struct hclge_fd_user_def_info *info, *old_info;
struct hclge_fd_user_def_cfg *cfg;
if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE ||
rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE)
return 0;
cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1];
info = &rule->ep.user_def;
if (!cfg->ref_cnt || cfg->offset == info->offset)
return 0;
if (cfg->ref_cnt > 1)
goto error;
fd_rule = hclge_find_fd_rule(hlist, rule->location, &parent);
if (fd_rule) {
old_info = &fd_rule->ep.user_def;
if (info->layer == old_info->layer)
return 0;
}
error:
dev_err(&hdev->pdev->dev,
"No available offset for layer%d fd rule, each layer only support one user def offset.\n",
info->layer + 1);
return -ENOSPC;
}
static void hclge_fd_inc_user_def_refcnt(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
struct hclge_fd_user_def_cfg *cfg;
if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE ||
rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE)
return;
cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1];
if (!cfg->ref_cnt) {
cfg->offset = rule->ep.user_def.offset;
set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state);
}
cfg->ref_cnt++;
}
static void hclge_fd_dec_user_def_refcnt(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
struct hclge_fd_user_def_cfg *cfg;
if (!rule || rule->rule_type != HCLGE_FD_EP_ACTIVE ||
rule->ep.user_def.layer == HCLGE_FD_USER_DEF_NONE)
return;
cfg = &hdev->fd_cfg.user_def_cfg[rule->ep.user_def.layer - 1];
if (!cfg->ref_cnt)
return;
cfg->ref_cnt--;
if (!cfg->ref_cnt) {
cfg->offset = 0;
set_bit(HCLGE_STATE_FD_USER_DEF_CHANGED, &hdev->state);
}
}
static void hclge_update_fd_list(struct hclge_dev *hdev,
enum HCLGE_FD_NODE_STATE state, u16 location,
struct hclge_fd_rule *new_rule)
{
struct hlist_head *hlist = &hdev->fd_rule_list;
struct hclge_fd_rule *fd_rule, *parent = NULL;
fd_rule = hclge_find_fd_rule(hlist, location, &parent);
if (fd_rule) {
hclge_fd_dec_user_def_refcnt(hdev, fd_rule);
if (state == HCLGE_FD_ACTIVE)
hclge_fd_inc_user_def_refcnt(hdev, new_rule);
hclge_sync_fd_user_def_cfg(hdev, true);
hclge_update_fd_rule_node(hdev, fd_rule, new_rule, state);
return;
}
if (unlikely(state == HCLGE_FD_TO_DEL || state == HCLGE_FD_DELETED)) {
dev_warn(&hdev->pdev->dev,
"failed to delete fd rule %u, it's inexistent\n",
location);
return;
}
hclge_fd_inc_user_def_refcnt(hdev, new_rule);
hclge_sync_fd_user_def_cfg(hdev, true);
hclge_fd_insert_rule_node(hlist, new_rule, parent);
hclge_fd_inc_rule_cnt(hdev, new_rule->location);
if (state == HCLGE_FD_TO_ADD) {
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
hclge_task_schedule(hdev, 0);
}
}
static int hclge_get_fd_mode(struct hclge_dev *hdev, u8 *fd_mode)
{
struct hclge_get_fd_mode_cmd *req;
struct hclge_desc desc;
int ret;
hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_MODE_CTRL, true);
req = (struct hclge_get_fd_mode_cmd *)desc.data;
ret = hclge_cmd_send(&hdev->hw, &desc, 1);
if (ret) {
dev_err(&hdev->pdev->dev, "get fd mode fail, ret=%d\n", ret);
return ret;
}
*fd_mode = req->mode;
return ret;
}
static int hclge_get_fd_allocation(struct hclge_dev *hdev,
u32 *stage1_entry_num,
u32 *stage2_entry_num,
u16 *stage1_counter_num,
u16 *stage2_counter_num)
{
struct hclge_get_fd_allocation_cmd *req;
struct hclge_desc desc;
int ret;
hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_GET_ALLOCATION, true);
req = (struct hclge_get_fd_allocation_cmd *)desc.data;
ret = hclge_cmd_send(&hdev->hw, &desc, 1);
if (ret) {
dev_err(&hdev->pdev->dev, "query fd allocation fail, ret=%d\n",
ret);
return ret;
}
*stage1_entry_num = le32_to_cpu(req->stage1_entry_num);
*stage2_entry_num = le32_to_cpu(req->stage2_entry_num);
*stage1_counter_num = le16_to_cpu(req->stage1_counter_num);
*stage2_counter_num = le16_to_cpu(req->stage2_counter_num);
return ret;
}
static int hclge_set_fd_key_config(struct hclge_dev *hdev,
enum HCLGE_FD_STAGE stage_num)
{
struct hclge_set_fd_key_config_cmd *req;
struct hclge_fd_key_cfg *stage;
struct hclge_desc desc;
int ret;
hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_KEY_CONFIG, false);
req = (struct hclge_set_fd_key_config_cmd *)desc.data;
stage = &hdev->fd_cfg.key_cfg[stage_num];
req->stage = stage_num;
req->key_select = stage->key_sel;
req->inner_sipv6_word_en = stage->inner_sipv6_word_en;
req->inner_dipv6_word_en = stage->inner_dipv6_word_en;
req->outer_sipv6_word_en = stage->outer_sipv6_word_en;
req->outer_dipv6_word_en = stage->outer_dipv6_word_en;
req->tuple_mask = cpu_to_le32(~stage->tuple_active);
req->meta_data_mask = cpu_to_le32(~stage->meta_data_active);
ret = hclge_cmd_send(&hdev->hw, &desc, 1);
if (ret)
dev_err(&hdev->pdev->dev, "set fd key fail, ret=%d\n", ret);
return ret;
}
static void hclge_fd_disable_user_def(struct hclge_dev *hdev)
{
struct hclge_fd_user_def_cfg *cfg = hdev->fd_cfg.user_def_cfg;
spin_lock_bh(&hdev->fd_rule_lock);
memset(cfg, 0, sizeof(hdev->fd_cfg.user_def_cfg));
spin_unlock_bh(&hdev->fd_rule_lock);
hclge_fd_set_user_def_cmd(hdev, cfg);
}
int hclge_init_fd_config(struct hclge_dev *hdev)
{
#define LOW_2_WORDS 0x03
struct hclge_fd_key_cfg *key_cfg;
int ret;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return 0;
ret = hclge_get_fd_mode(hdev, &hdev->fd_cfg.fd_mode);
if (ret)
return ret;
switch (hdev->fd_cfg.fd_mode) {
case HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1:
hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH;
break;
case HCLGE_FD_MODE_DEPTH_4K_WIDTH_200B_STAGE_1:
hdev->fd_cfg.max_key_length = MAX_KEY_LENGTH / 2;
break;
default:
dev_err(&hdev->pdev->dev,
"Unsupported flow director mode %u\n",
hdev->fd_cfg.fd_mode);
return -EOPNOTSUPP;
}
key_cfg = &hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1];
key_cfg->key_sel = HCLGE_FD_KEY_BASE_ON_TUPLE;
key_cfg->inner_sipv6_word_en = LOW_2_WORDS;
key_cfg->inner_dipv6_word_en = LOW_2_WORDS;
key_cfg->outer_sipv6_word_en = 0;
key_cfg->outer_dipv6_word_en = 0;
key_cfg->tuple_active = BIT(INNER_VLAN_TAG_FST) | BIT(INNER_ETH_TYPE) |
BIT(INNER_IP_PROTO) | BIT(INNER_IP_TOS) |
BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) |
BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT);
if (hdev->fd_cfg.fd_mode == HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) {
key_cfg->tuple_active |= BIT(INNER_DST_MAC) |
BIT(INNER_SRC_MAC) |
BIT(OUTER_TUN_VNI);
if (hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V3)
key_cfg->tuple_active |= HCLGE_FD_TUPLE_USER_DEF_TUPLES;
}
key_cfg->meta_data_active = BIT(ROCE_TYPE) | BIT(DST_VPORT);
ret = hclge_get_fd_allocation(hdev,
&hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1],
&hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_2],
&hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1],
&hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_2]);
if (ret)
return ret;
return hclge_set_fd_key_config(hdev, HCLGE_FD_STAGE_1);
}
static int hclge_fd_tcam_config(struct hclge_dev *hdev, u8 stage, bool sel_x,
int loc, u8 *key, bool is_add)
{
struct hclge_fd_tcam_config_1_cmd *req1;
struct hclge_fd_tcam_config_2_cmd *req2;
struct hclge_fd_tcam_config_3_cmd *req3;
struct hclge_desc desc[3];
int ret;
hclge_cmd_setup_basic_desc(&desc[0], HCLGE_OPC_FD_TCAM_OP, false);
desc[0].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT);
hclge_cmd_setup_basic_desc(&desc[1], HCLGE_OPC_FD_TCAM_OP, false);
desc[1].flag |= cpu_to_le16(HCLGE_COMM_CMD_FLAG_NEXT);
hclge_cmd_setup_basic_desc(&desc[2], HCLGE_OPC_FD_TCAM_OP, false);
req1 = (struct hclge_fd_tcam_config_1_cmd *)desc[0].data;
req2 = (struct hclge_fd_tcam_config_2_cmd *)desc[1].data;
req3 = (struct hclge_fd_tcam_config_3_cmd *)desc[2].data;
req1->stage = stage;
req1->xy_sel = sel_x ? 1 : 0;
hnae3_set_bit(req1->port_info, HCLGE_FD_EPORT_SW_EN_B, 0);
req1->index = cpu_to_le32(loc);
req1->entry_vld = sel_x ? is_add : 0;
if (key) {
memcpy(req1->tcam_data, &key[0], sizeof(req1->tcam_data));
memcpy(req2->tcam_data, &key[sizeof(req1->tcam_data)],
sizeof(req2->tcam_data));
memcpy(req3->tcam_data, &key[sizeof(req1->tcam_data) +
sizeof(req2->tcam_data)], sizeof(req3->tcam_data));
}
ret = hclge_cmd_send(&hdev->hw, desc, 3);
if (ret)
dev_err(&hdev->pdev->dev,
"config tcam key fail, ret=%d\n",
ret);
return ret;
}
static int hclge_fd_ad_config(struct hclge_dev *hdev, u8 stage, int loc,
struct hclge_fd_ad_data *action)
{
struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev);
struct hclge_fd_ad_config_cmd *req;
struct hclge_desc desc;
u64 ad_data = 0;
int ret;
hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_FD_AD_OP, false);
req = (struct hclge_fd_ad_config_cmd *)desc.data;
req->index = cpu_to_le32(loc);
req->stage = stage;
hnae3_set_bit(ad_data, HCLGE_FD_AD_WR_RULE_ID_B,
action->write_rule_id_to_bd);
hnae3_set_field(ad_data, HCLGE_FD_AD_RULE_ID_M, HCLGE_FD_AD_RULE_ID_S,
action->rule_id);
if (test_bit(HNAE3_DEV_SUPPORT_FD_FORWARD_TC_B, ae_dev->caps)) {
hnae3_set_bit(ad_data, HCLGE_FD_AD_TC_OVRD_B,
action->override_tc);
hnae3_set_field(ad_data, HCLGE_FD_AD_TC_SIZE_M,
HCLGE_FD_AD_TC_SIZE_S, (u32)action->tc_size);
}
hnae3_set_bit(ad_data, HCLGE_FD_AD_QID_H_B,
action->queue_id >= HCLGE_TQP_MAX_SIZE_DEV_V2 ? 1 : 0);
hnae3_set_bit(ad_data, HCLGE_FD_AD_COUNTER_NUM_H_B,
action->counter_id >= HCLGE_FD_COUNTER_MAX_SIZE_DEV_V2 ?
1 : 0);
ad_data <<= 32;
hnae3_set_bit(ad_data, HCLGE_FD_AD_DROP_B, action->drop_packet);
hnae3_set_bit(ad_data, HCLGE_FD_AD_DIRECT_QID_B,
action->forward_to_direct_queue);
hnae3_set_field(ad_data, HCLGE_FD_AD_QID_L_M, HCLGE_FD_AD_QID_L_S,
action->queue_id);
hnae3_set_bit(ad_data, HCLGE_FD_AD_USE_COUNTER_B, action->use_counter);
hnae3_set_field(ad_data, HCLGE_FD_AD_COUNTER_NUM_L_M,
HCLGE_FD_AD_COUNTER_NUM_L_S, action->counter_id);
hnae3_set_bit(ad_data, HCLGE_FD_AD_NXT_STEP_B, action->use_next_stage);
hnae3_set_field(ad_data, HCLGE_FD_AD_NXT_KEY_M, HCLGE_FD_AD_NXT_KEY_S,
action->next_input_key);
req->ad_data = cpu_to_le64(ad_data);
ret = hclge_cmd_send(&hdev->hw, &desc, 1);
if (ret)
dev_err(&hdev->pdev->dev, "fd ad config fail, ret=%d\n", ret);
return ret;
}
static bool hclge_fd_convert_tuple(u32 tuple_bit, u8 *key_x, u8 *key_y,
struct hclge_fd_rule *rule)
{
int offset, moffset, ip_offset;
enum HCLGE_FD_KEY_OPT key_opt;
u16 tmp_x_s, tmp_y_s;
u32 tmp_x_l, tmp_y_l;
u8 *p = (u8 *)rule;
__le32 le_x, le_y;
int i;
if (rule->unused_tuple & BIT(tuple_bit))
return true;
key_opt = tuple_key_info[tuple_bit].key_opt;
offset = tuple_key_info[tuple_bit].offset;
moffset = tuple_key_info[tuple_bit].moffset;
switch (key_opt) {
case KEY_OPT_U8:
calc_x(*key_x, p[offset], p[moffset]);
calc_y(*key_y, p[offset], p[moffset]);
return true;
case KEY_OPT_LE16:
calc_x(tmp_x_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset]));
calc_y(tmp_y_s, *(u16 *)(&p[offset]), *(u16 *)(&p[moffset]));
*(__le16 *)key_x = cpu_to_le16(tmp_x_s);
*(__le16 *)key_y = cpu_to_le16(tmp_y_s);
return true;
case KEY_OPT_LE32:
calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset]));
calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset]));
*(__le32 *)key_x = cpu_to_le32(tmp_x_l);
*(__le32 *)key_y = cpu_to_le32(tmp_y_l);
return true;
case KEY_OPT_MAC:
for (i = 0; i < ETH_ALEN; i++) {
calc_x(key_x[ETH_ALEN - 1 - i], p[offset + i],
p[moffset + i]);
calc_y(key_y[ETH_ALEN - 1 - i], p[offset + i],
p[moffset + i]);
}
return true;
case KEY_OPT_IP:
ip_offset = IPV4_INDEX * sizeof(u32);
calc_x(tmp_x_l, *(u32 *)(&p[offset + ip_offset]),
*(u32 *)(&p[moffset + ip_offset]));
calc_y(tmp_y_l, *(u32 *)(&p[offset + ip_offset]),
*(u32 *)(&p[moffset + ip_offset]));
*(__le32 *)key_x = cpu_to_le32(tmp_x_l);
*(__le32 *)key_y = cpu_to_le32(tmp_y_l);
return true;
case KEY_OPT_VNI:
calc_x(tmp_x_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset]));
calc_y(tmp_y_l, *(u32 *)(&p[offset]), *(u32 *)(&p[moffset]));
le_x = cpu_to_le32(tmp_x_l);
le_y = cpu_to_le32(tmp_y_l);
memcpy(key_x, &le_x, HCLGE_VNI_LENGTH);
memcpy(key_y, &le_y, HCLGE_VNI_LENGTH);
return true;
default:
return false;
}
}
static void hclge_fd_convert_meta_data(struct hclge_fd_key_cfg *key_cfg,
__le32 *key_x, __le32 *key_y,
struct hclge_fd_rule *rule)
{
u32 tuple_bit, meta_data = 0, tmp_x, tmp_y, port_number;
u8 cur_pos = 0, tuple_size, shift_bits;
unsigned int i;
for (i = 0; i < MAX_META_DATA; i++) {
tuple_size = meta_data_key_info[i].key_length;
tuple_bit = key_cfg->meta_data_active & BIT(i);
switch (tuple_bit) {
case BIT(ROCE_TYPE):
hnae3_set_bit(meta_data, cur_pos, NIC_PACKET);
cur_pos += tuple_size;
break;
case BIT(DST_VPORT):
port_number = hclge_get_port_number(HOST_PORT, 0,
rule->vf_id, 0);
hnae3_set_field(meta_data,
GENMASK(cur_pos + tuple_size, cur_pos),
cur_pos, port_number);
cur_pos += tuple_size;
break;
default:
break;
}
}
calc_x(tmp_x, meta_data, 0xFFFFFFFF);
calc_y(tmp_y, meta_data, 0xFFFFFFFF);
shift_bits = sizeof(meta_data) * 8 - cur_pos;
*key_x = cpu_to_le32(tmp_x << shift_bits);
*key_y = cpu_to_le32(tmp_y << shift_bits);
}
static int hclge_config_key(struct hclge_dev *hdev, u8 stage,
struct hclge_fd_rule *rule)
{
struct hclge_fd_key_cfg *key_cfg = &hdev->fd_cfg.key_cfg[stage];
u8 key_x[MAX_KEY_BYTES], key_y[MAX_KEY_BYTES];
u8 *cur_key_x, *cur_key_y;
u8 meta_data_region;
u8 tuple_size;
int ret;
u32 i;
memset(key_x, 0, sizeof(key_x));
memset(key_y, 0, sizeof(key_y));
cur_key_x = key_x;
cur_key_y = key_y;
for (i = 0; i < MAX_TUPLE; i++) {
bool tuple_valid;
tuple_size = tuple_key_info[i].key_length / 8;
if (!(key_cfg->tuple_active & BIT(i)))
continue;
tuple_valid = hclge_fd_convert_tuple(i, cur_key_x,
cur_key_y, rule);
if (tuple_valid) {
cur_key_x += tuple_size;
cur_key_y += tuple_size;
}
}
meta_data_region = hdev->fd_cfg.max_key_length / 8 -
MAX_META_DATA_LENGTH / 8;
hclge_fd_convert_meta_data(key_cfg,
(__le32 *)(key_x + meta_data_region),
(__le32 *)(key_y + meta_data_region),
rule);
ret = hclge_fd_tcam_config(hdev, stage, false, rule->location, key_y,
true);
if (ret) {
dev_err(&hdev->pdev->dev,
"fd key_y config fail, loc=%u, ret=%d\n",
rule->queue_id, ret);
return ret;
}
ret = hclge_fd_tcam_config(hdev, stage, true, rule->location, key_x,
true);
if (ret)
dev_err(&hdev->pdev->dev,
"fd key_x config fail, loc=%u, ret=%d\n",
rule->queue_id, ret);
return ret;
}
static int hclge_config_action(struct hclge_dev *hdev, u8 stage,
struct hclge_fd_rule *rule)
{
struct hclge_vport *vport = hdev->vport;
struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo;
struct hclge_fd_ad_data ad_data;
memset(&ad_data, 0, sizeof(struct hclge_fd_ad_data));
ad_data.ad_id = rule->location;
if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) {
ad_data.drop_packet = true;
} else if (rule->action == HCLGE_FD_ACTION_SELECT_TC) {
ad_data.override_tc = true;
ad_data.queue_id =
kinfo->tc_info.tqp_offset[rule->cls_flower.tc];
ad_data.tc_size =
ilog2(kinfo->tc_info.tqp_count[rule->cls_flower.tc]);
} else {
ad_data.forward_to_direct_queue = true;
ad_data.queue_id = rule->queue_id;
}
if (hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1]) {
ad_data.use_counter = true;
ad_data.counter_id = rule->vf_id %
hdev->fd_cfg.cnt_num[HCLGE_FD_STAGE_1];
} else {
ad_data.use_counter = false;
ad_data.counter_id = 0;
}
ad_data.use_next_stage = false;
ad_data.next_input_key = 0;
ad_data.write_rule_id_to_bd = true;
ad_data.rule_id = rule->location;
return hclge_fd_ad_config(hdev, stage, ad_data.ad_id, &ad_data);
}
static int hclge_fd_check_tcpip4_tuple(struct ethtool_tcpip4_spec *spec,
u32 *unused_tuple)
{
if (!spec || !unused_tuple)
return -EINVAL;
*unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC);
if (!spec->ip4src)
*unused_tuple |= BIT(INNER_SRC_IP);
if (!spec->ip4dst)
*unused_tuple |= BIT(INNER_DST_IP);
if (!spec->psrc)
*unused_tuple |= BIT(INNER_SRC_PORT);
if (!spec->pdst)
*unused_tuple |= BIT(INNER_DST_PORT);
if (!spec->tos)
*unused_tuple |= BIT(INNER_IP_TOS);
return 0;
}
static int hclge_fd_check_ip4_tuple(struct ethtool_usrip4_spec *spec,
u32 *unused_tuple)
{
if (!spec || !unused_tuple)
return -EINVAL;
*unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) |
BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT);
if (!spec->ip4src)
*unused_tuple |= BIT(INNER_SRC_IP);
if (!spec->ip4dst)
*unused_tuple |= BIT(INNER_DST_IP);
if (!spec->tos)
*unused_tuple |= BIT(INNER_IP_TOS);
if (!spec->proto)
*unused_tuple |= BIT(INNER_IP_PROTO);
if (spec->l4_4_bytes)
return -EOPNOTSUPP;
if (spec->ip_ver != ETH_RX_NFC_IP4)
return -EOPNOTSUPP;
return 0;
}
static int hclge_fd_check_tcpip6_tuple(struct ethtool_tcpip6_spec *spec,
u32 *unused_tuple)
{
if (!spec || !unused_tuple)
return -EINVAL;
*unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC);
if (ipv6_addr_any((struct in6_addr *)spec->ip6src))
*unused_tuple |= BIT(INNER_SRC_IP);
if (ipv6_addr_any((struct in6_addr *)spec->ip6dst))
*unused_tuple |= BIT(INNER_DST_IP);
if (!spec->psrc)
*unused_tuple |= BIT(INNER_SRC_PORT);
if (!spec->pdst)
*unused_tuple |= BIT(INNER_DST_PORT);
if (!spec->tclass)
*unused_tuple |= BIT(INNER_IP_TOS);
return 0;
}
static int hclge_fd_check_ip6_tuple(struct ethtool_usrip6_spec *spec,
u32 *unused_tuple)
{
if (!spec || !unused_tuple)
return -EINVAL;
*unused_tuple |= BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) |
BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT);
if (ipv6_addr_any((struct in6_addr *)spec->ip6src))
*unused_tuple |= BIT(INNER_SRC_IP);
if (ipv6_addr_any((struct in6_addr *)spec->ip6dst))
*unused_tuple |= BIT(INNER_DST_IP);
if (!spec->l4_proto)
*unused_tuple |= BIT(INNER_IP_PROTO);
if (!spec->tclass)
*unused_tuple |= BIT(INNER_IP_TOS);
if (spec->l4_4_bytes)
return -EOPNOTSUPP;
return 0;
}
static int hclge_fd_check_ether_tuple(struct ethhdr *spec, u32 *unused_tuple)
{
if (!spec || !unused_tuple)
return -EINVAL;
*unused_tuple |= BIT(INNER_SRC_IP) | BIT(INNER_DST_IP) |
BIT(INNER_SRC_PORT) | BIT(INNER_DST_PORT) |
BIT(INNER_IP_TOS) | BIT(INNER_IP_PROTO);
if (is_zero_ether_addr(spec->h_source))
*unused_tuple |= BIT(INNER_SRC_MAC);
if (is_zero_ether_addr(spec->h_dest))
*unused_tuple |= BIT(INNER_DST_MAC);
if (!spec->h_proto)
*unused_tuple |= BIT(INNER_ETH_TYPE);
return 0;
}
static int hclge_fd_check_ext_tuple(struct hclge_dev *hdev,
struct ethtool_rx_flow_spec *fs,
u32 *unused_tuple)
{
if (fs->flow_type & FLOW_EXT) {
if (fs->h_ext.vlan_etype) {
dev_err(&hdev->pdev->dev, "vlan-etype is not supported!\n");
return -EOPNOTSUPP;
}
if (!fs->h_ext.vlan_tci)
*unused_tuple |= BIT(INNER_VLAN_TAG_FST);
if (fs->m_ext.vlan_tci &&
be16_to_cpu(fs->h_ext.vlan_tci) >= VLAN_N_VID) {
dev_err(&hdev->pdev->dev,
"failed to config vlan_tci, invalid vlan_tci: %u, max is %d.\n",
ntohs(fs->h_ext.vlan_tci), VLAN_N_VID - 1);
return -EINVAL;
}
} else {
*unused_tuple |= BIT(INNER_VLAN_TAG_FST);
}
if (fs->flow_type & FLOW_MAC_EXT) {
if (hdev->fd_cfg.fd_mode !=
HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) {
dev_err(&hdev->pdev->dev,
"FLOW_MAC_EXT is not supported in current fd mode!\n");
return -EOPNOTSUPP;
}
if (is_zero_ether_addr(fs->h_ext.h_dest))
*unused_tuple |= BIT(INNER_DST_MAC);
else
*unused_tuple &= ~BIT(INNER_DST_MAC);
}
return 0;
}
static int hclge_fd_get_user_def_layer(u32 flow_type, u32 *unused_tuple,
struct hclge_fd_user_def_info *info)
{
switch (flow_type) {
case ETHER_FLOW:
info->layer = HCLGE_FD_USER_DEF_L2;
*unused_tuple &= ~BIT(INNER_L2_RSV);
break;
case IP_USER_FLOW:
case IPV6_USER_FLOW:
info->layer = HCLGE_FD_USER_DEF_L3;
*unused_tuple &= ~BIT(INNER_L3_RSV);
break;
case TCP_V4_FLOW:
case UDP_V4_FLOW:
case TCP_V6_FLOW:
case UDP_V6_FLOW:
info->layer = HCLGE_FD_USER_DEF_L4;
*unused_tuple &= ~BIT(INNER_L4_RSV);
break;
default:
return -EOPNOTSUPP;
}
return 0;
}
static bool hclge_fd_is_user_def_all_masked(struct ethtool_rx_flow_spec *fs)
{
return be32_to_cpu(fs->m_ext.data[1] | fs->m_ext.data[0]) == 0;
}
static int hclge_fd_parse_user_def_field(struct hclge_dev *hdev,
struct ethtool_rx_flow_spec *fs,
u32 *unused_tuple,
struct hclge_fd_user_def_info *info)
{
u32 tuple_active = hdev->fd_cfg.key_cfg[HCLGE_FD_STAGE_1].tuple_active;
u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT);
u16 data, offset, data_mask, offset_mask;
int ret;
info->layer = HCLGE_FD_USER_DEF_NONE;
*unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES;
if (!(fs->flow_type & FLOW_EXT) || hclge_fd_is_user_def_all_masked(fs))
return 0;
data = be32_to_cpu(fs->h_ext.data[1]) & HCLGE_FD_USER_DEF_DATA;
data_mask = be32_to_cpu(fs->m_ext.data[1]) & HCLGE_FD_USER_DEF_DATA;
offset = be32_to_cpu(fs->h_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET;
offset_mask = be32_to_cpu(fs->m_ext.data[0]) & HCLGE_FD_USER_DEF_OFFSET;
if (!(tuple_active & HCLGE_FD_TUPLE_USER_DEF_TUPLES)) {
dev_err(&hdev->pdev->dev, "user-def bytes are not supported\n");
return -EOPNOTSUPP;
}
if (offset > HCLGE_FD_MAX_USER_DEF_OFFSET) {
dev_err(&hdev->pdev->dev,
"user-def offset[%u] should be no more than %u\n",
offset, HCLGE_FD_MAX_USER_DEF_OFFSET);
return -EINVAL;
}
if (offset_mask != HCLGE_FD_USER_DEF_OFFSET_UNMASK) {
dev_err(&hdev->pdev->dev, "user-def offset can't be masked\n");
return -EINVAL;
}
ret = hclge_fd_get_user_def_layer(flow_type, unused_tuple, info);
if (ret) {
dev_err(&hdev->pdev->dev,
"unsupported flow type for user-def bytes, ret = %d\n",
ret);
return ret;
}
info->data = data;
info->data_mask = data_mask;
info->offset = offset;
return 0;
}
static int hclge_fd_check_spec(struct hclge_dev *hdev,
struct ethtool_rx_flow_spec *fs,
u32 *unused_tuple,
struct hclge_fd_user_def_info *info)
{
u32 flow_type;
int ret;
if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) {
dev_err(&hdev->pdev->dev,
"failed to config fd rules, invalid rule location: %u, max is %u\n.",
fs->location,
hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1] - 1);
return -EINVAL;
}
ret = hclge_fd_parse_user_def_field(hdev, fs, unused_tuple, info);
if (ret)
return ret;
flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT);
switch (flow_type) {
case SCTP_V4_FLOW:
case TCP_V4_FLOW:
case UDP_V4_FLOW:
ret = hclge_fd_check_tcpip4_tuple(&fs->h_u.tcp_ip4_spec,
unused_tuple);
break;
case IP_USER_FLOW:
ret = hclge_fd_check_ip4_tuple(&fs->h_u.usr_ip4_spec,
unused_tuple);
break;
case SCTP_V6_FLOW:
case TCP_V6_FLOW:
case UDP_V6_FLOW:
ret = hclge_fd_check_tcpip6_tuple(&fs->h_u.tcp_ip6_spec,
unused_tuple);
break;
case IPV6_USER_FLOW:
ret = hclge_fd_check_ip6_tuple(&fs->h_u.usr_ip6_spec,
unused_tuple);
break;
case ETHER_FLOW:
if (hdev->fd_cfg.fd_mode !=
HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1) {
dev_err(&hdev->pdev->dev,
"ETHER_FLOW is not supported in current fd mode!\n");
return -EOPNOTSUPP;
}
ret = hclge_fd_check_ether_tuple(&fs->h_u.ether_spec,
unused_tuple);
break;
default:
dev_err(&hdev->pdev->dev,
"unsupported protocol type, protocol type = %#x\n",
flow_type);
return -EOPNOTSUPP;
}
if (ret) {
dev_err(&hdev->pdev->dev,
"failed to check flow union tuple, ret = %d\n",
ret);
return ret;
}
return hclge_fd_check_ext_tuple(hdev, fs, unused_tuple);
}
static void hclge_fd_get_tcpip4_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule, u8 ip_proto)
{
rule->tuples.src_ip[IPV4_INDEX] =
be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4src);
rule->tuples_mask.src_ip[IPV4_INDEX] =
be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4src);
rule->tuples.dst_ip[IPV4_INDEX] =
be32_to_cpu(fs->h_u.tcp_ip4_spec.ip4dst);
rule->tuples_mask.dst_ip[IPV4_INDEX] =
be32_to_cpu(fs->m_u.tcp_ip4_spec.ip4dst);
rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc);
rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.psrc);
rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst);
rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip4_spec.pdst);
rule->tuples.ip_tos = fs->h_u.tcp_ip4_spec.tos;
rule->tuples_mask.ip_tos = fs->m_u.tcp_ip4_spec.tos;
rule->tuples.ether_proto = ETH_P_IP;
rule->tuples_mask.ether_proto = 0xFFFF;
rule->tuples.ip_proto = ip_proto;
rule->tuples_mask.ip_proto = 0xFF;
}
static void hclge_fd_get_ip4_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
rule->tuples.src_ip[IPV4_INDEX] =
be32_to_cpu(fs->h_u.usr_ip4_spec.ip4src);
rule->tuples_mask.src_ip[IPV4_INDEX] =
be32_to_cpu(fs->m_u.usr_ip4_spec.ip4src);
rule->tuples.dst_ip[IPV4_INDEX] =
be32_to_cpu(fs->h_u.usr_ip4_spec.ip4dst);
rule->tuples_mask.dst_ip[IPV4_INDEX] =
be32_to_cpu(fs->m_u.usr_ip4_spec.ip4dst);
rule->tuples.ip_tos = fs->h_u.usr_ip4_spec.tos;
rule->tuples_mask.ip_tos = fs->m_u.usr_ip4_spec.tos;
rule->tuples.ip_proto = fs->h_u.usr_ip4_spec.proto;
rule->tuples_mask.ip_proto = fs->m_u.usr_ip4_spec.proto;
rule->tuples.ether_proto = ETH_P_IP;
rule->tuples_mask.ether_proto = 0xFFFF;
}
static void hclge_fd_get_tcpip6_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule, u8 ip_proto)
{
ipv6_addr_be32_to_cpu(rule->tuples.src_ip,
fs->h_u.tcp_ip6_spec.ip6src);
ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip,
fs->m_u.tcp_ip6_spec.ip6src);
ipv6_addr_be32_to_cpu(rule->tuples.dst_ip,
fs->h_u.tcp_ip6_spec.ip6dst);
ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip,
fs->m_u.tcp_ip6_spec.ip6dst);
rule->tuples.src_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.psrc);
rule->tuples_mask.src_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.psrc);
rule->tuples.dst_port = be16_to_cpu(fs->h_u.tcp_ip6_spec.pdst);
rule->tuples_mask.dst_port = be16_to_cpu(fs->m_u.tcp_ip6_spec.pdst);
rule->tuples.ether_proto = ETH_P_IPV6;
rule->tuples_mask.ether_proto = 0xFFFF;
rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass;
rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass;
rule->tuples.ip_proto = ip_proto;
rule->tuples_mask.ip_proto = 0xFF;
}
static void hclge_fd_get_ip6_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
ipv6_addr_be32_to_cpu(rule->tuples.src_ip,
fs->h_u.usr_ip6_spec.ip6src);
ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip,
fs->m_u.usr_ip6_spec.ip6src);
ipv6_addr_be32_to_cpu(rule->tuples.dst_ip,
fs->h_u.usr_ip6_spec.ip6dst);
ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip,
fs->m_u.usr_ip6_spec.ip6dst);
rule->tuples.ip_proto = fs->h_u.usr_ip6_spec.l4_proto;
rule->tuples_mask.ip_proto = fs->m_u.usr_ip6_spec.l4_proto;
rule->tuples.ip_tos = fs->h_u.tcp_ip6_spec.tclass;
rule->tuples_mask.ip_tos = fs->m_u.tcp_ip6_spec.tclass;
rule->tuples.ether_proto = ETH_P_IPV6;
rule->tuples_mask.ether_proto = 0xFFFF;
}
static void hclge_fd_get_ether_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
ether_addr_copy(rule->tuples.src_mac, fs->h_u.ether_spec.h_source);
ether_addr_copy(rule->tuples_mask.src_mac, fs->m_u.ether_spec.h_source);
ether_addr_copy(rule->tuples.dst_mac, fs->h_u.ether_spec.h_dest);
ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_u.ether_spec.h_dest);
rule->tuples.ether_proto = be16_to_cpu(fs->h_u.ether_spec.h_proto);
rule->tuples_mask.ether_proto = be16_to_cpu(fs->m_u.ether_spec.h_proto);
}
static void hclge_fd_get_user_def_tuple(struct hclge_fd_user_def_info *info,
struct hclge_fd_rule *rule)
{
switch (info->layer) {
case HCLGE_FD_USER_DEF_L2:
rule->tuples.l2_user_def = info->data;
rule->tuples_mask.l2_user_def = info->data_mask;
break;
case HCLGE_FD_USER_DEF_L3:
rule->tuples.l3_user_def = info->data;
rule->tuples_mask.l3_user_def = info->data_mask;
break;
case HCLGE_FD_USER_DEF_L4:
rule->tuples.l4_user_def = (u32)info->data << 16;
rule->tuples_mask.l4_user_def = (u32)info->data_mask << 16;
break;
default:
break;
}
rule->ep.user_def = *info;
}
static int hclge_fd_get_tuple(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule,
struct hclge_fd_user_def_info *info)
{
u32 flow_type = fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT);
switch (flow_type) {
case SCTP_V4_FLOW:
hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_SCTP);
break;
case TCP_V4_FLOW:
hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_TCP);
break;
case UDP_V4_FLOW:
hclge_fd_get_tcpip4_tuple(fs, rule, IPPROTO_UDP);
break;
case IP_USER_FLOW:
hclge_fd_get_ip4_tuple(fs, rule);
break;
case SCTP_V6_FLOW:
hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_SCTP);
break;
case TCP_V6_FLOW:
hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_TCP);
break;
case UDP_V6_FLOW:
hclge_fd_get_tcpip6_tuple(fs, rule, IPPROTO_UDP);
break;
case IPV6_USER_FLOW:
hclge_fd_get_ip6_tuple(fs, rule);
break;
case ETHER_FLOW:
hclge_fd_get_ether_tuple(fs, rule);
break;
default:
return -EOPNOTSUPP;
}
if (fs->flow_type & FLOW_EXT) {
rule->tuples.vlan_tag1 = be16_to_cpu(fs->h_ext.vlan_tci);
rule->tuples_mask.vlan_tag1 = be16_to_cpu(fs->m_ext.vlan_tci);
hclge_fd_get_user_def_tuple(info, rule);
}
if (fs->flow_type & FLOW_MAC_EXT) {
ether_addr_copy(rule->tuples.dst_mac, fs->h_ext.h_dest);
ether_addr_copy(rule->tuples_mask.dst_mac, fs->m_ext.h_dest);
}
return 0;
}
static int hclge_fd_config_rule(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
int ret;
ret = hclge_config_action(hdev, HCLGE_FD_STAGE_1, rule);
if (ret)
return ret;
return hclge_config_key(hdev, HCLGE_FD_STAGE_1, rule);
}
static int hclge_add_fd_entry_common(struct hclge_dev *hdev,
struct hclge_fd_rule *rule)
{
int ret;
spin_lock_bh(&hdev->fd_rule_lock);
if (hdev->fd_active_type != rule->rule_type &&
(hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE ||
hdev->fd_active_type == HCLGE_FD_EP_ACTIVE)) {
dev_err(&hdev->pdev->dev,
"mode conflict(new type %d, active type %d), please delete existent rules first\n",
rule->rule_type, hdev->fd_active_type);
spin_unlock_bh(&hdev->fd_rule_lock);
return -EINVAL;
}
ret = hclge_fd_check_user_def_refcnt(hdev, rule);
if (ret)
goto out;
ret = hclge_clear_arfs_rules(hdev);
if (ret)
goto out;
ret = hclge_fd_config_rule(hdev, rule);
if (ret)
goto out;
rule->state = HCLGE_FD_ACTIVE;
hdev->fd_active_type = rule->rule_type;
hclge_update_fd_list(hdev, rule->state, rule->location, rule);
out:
spin_unlock_bh(&hdev->fd_rule_lock);
return ret;
}
bool hclge_is_cls_flower_active(struct hnae3_handle *handle)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
return hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE;
}
static int hclge_fd_parse_ring_cookie(struct hclge_dev *hdev, u64 ring_cookie,
u16 *vport_id, u8 *action, u16 *queue_id)
{
struct hclge_vport *vport = hdev->vport;
if (ring_cookie == RX_CLS_FLOW_DISC) {
*action = HCLGE_FD_ACTION_DROP_PACKET;
} else {
u32 ring = ethtool_get_flow_spec_ring(ring_cookie);
u8 vf = ethtool_get_flow_spec_ring_vf(ring_cookie);
u16 tqps;
if (vf > hdev->num_req_vfs) {
dev_err(&hdev->pdev->dev,
"Error: vf id (%u) should be less than %u\n",
vf - 1U, hdev->num_req_vfs);
return -EINVAL;
}
*vport_id = vf ? hdev->vport[vf].vport_id : vport->vport_id;
tqps = hdev->vport[vf].nic.kinfo.num_tqps;
if (ring >= tqps) {
dev_err(&hdev->pdev->dev,
"Error: queue id (%u) > max tqp num (%u)\n",
ring, tqps - 1U);
return -EINVAL;
}
*action = HCLGE_FD_ACTION_SELECT_QUEUE;
*queue_id = ring;
}
return 0;
}
int hclge_add_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct hclge_fd_user_def_info info;
u16 dst_vport_id = 0, q_index = 0;
struct ethtool_rx_flow_spec *fs;
struct hclge_fd_rule *rule;
u32 unused = 0;
u8 action;
int ret;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
dev_err(&hdev->pdev->dev,
"flow table director is not supported\n");
return -EOPNOTSUPP;
}
if (!hdev->fd_en) {
dev_err(&hdev->pdev->dev,
"please enable flow director first\n");
return -EOPNOTSUPP;
}
fs = (struct ethtool_rx_flow_spec *)&cmd->fs;
ret = hclge_fd_check_spec(hdev, fs, &unused, &info);
if (ret)
return ret;
ret = hclge_fd_parse_ring_cookie(hdev, fs->ring_cookie, &dst_vport_id,
&action, &q_index);
if (ret)
return ret;
rule = kzalloc_obj(*rule);
if (!rule)
return -ENOMEM;
ret = hclge_fd_get_tuple(fs, rule, &info);
if (ret) {
kfree(rule);
return ret;
}
rule->flow_type = fs->flow_type;
rule->location = fs->location;
rule->unused_tuple = unused;
rule->vf_id = dst_vport_id;
rule->queue_id = q_index;
rule->action = action;
rule->rule_type = HCLGE_FD_EP_ACTIVE;
ret = hclge_add_fd_entry_common(hdev, rule);
if (ret)
kfree(rule);
return ret;
}
int hclge_del_fd_entry(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct ethtool_rx_flow_spec *fs;
int ret;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return -EOPNOTSUPP;
fs = (struct ethtool_rx_flow_spec *)&cmd->fs;
if (fs->location >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1])
return -EINVAL;
spin_lock_bh(&hdev->fd_rule_lock);
if (hdev->fd_active_type == HCLGE_FD_TC_FLOWER_ACTIVE ||
!test_bit(fs->location, hdev->fd_bmap)) {
dev_err(&hdev->pdev->dev,
"Delete fail, rule %u is inexistent\n", fs->location);
spin_unlock_bh(&hdev->fd_rule_lock);
return -ENOENT;
}
ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, fs->location,
NULL, false);
if (ret)
goto out;
hclge_update_fd_list(hdev, HCLGE_FD_DELETED, fs->location, NULL);
out:
spin_unlock_bh(&hdev->fd_rule_lock);
return ret;
}
static void hclge_clear_fd_rules_in_list(struct hclge_dev *hdev,
bool clear_list)
{
struct hclge_fd_rule *rule;
struct hlist_node *node;
u16 location;
spin_lock_bh(&hdev->fd_rule_lock);
for_each_set_bit(location, hdev->fd_bmap,
hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1])
hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, location,
NULL, false);
if (clear_list) {
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list,
rule_node) {
hlist_del(&rule->rule_node);
kfree(rule);
}
hdev->fd_active_type = HCLGE_FD_RULE_NONE;
hdev->hclge_fd_rule_num = 0;
bitmap_zero(hdev->fd_bmap,
hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]);
}
spin_unlock_bh(&hdev->fd_rule_lock);
}
void hclge_del_all_fd_entries(struct hclge_dev *hdev)
{
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return;
hclge_clear_fd_rules_in_list(hdev, true);
hclge_fd_disable_user_def(hdev);
}
int hclge_restore_fd_entries(struct hnae3_handle *handle)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct hclge_fd_rule *rule;
struct hlist_node *node;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return 0;
if (!hdev->fd_en)
return 0;
spin_lock_bh(&hdev->fd_rule_lock);
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) {
if (rule->state == HCLGE_FD_ACTIVE)
rule->state = HCLGE_FD_TO_ADD;
}
spin_unlock_bh(&hdev->fd_rule_lock);
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
return 0;
}
int hclge_get_fd_rule_cnt(struct hnae3_handle *handle,
struct ethtool_rxnfc *cmd)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev) ||
hclge_is_cls_flower_active(handle))
return -EOPNOTSUPP;
cmd->rule_cnt = hdev->hclge_fd_rule_num;
cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1];
return 0;
}
static void hclge_fd_get_tcpip4_info(struct hclge_fd_rule *rule,
struct ethtool_tcpip4_spec *spec,
struct ethtool_tcpip4_spec *spec_mask)
{
spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]);
spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ?
0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]);
spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]);
spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ?
0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]);
spec->psrc = cpu_to_be16(rule->tuples.src_port);
spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ?
0 : cpu_to_be16(rule->tuples_mask.src_port);
spec->pdst = cpu_to_be16(rule->tuples.dst_port);
spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ?
0 : cpu_to_be16(rule->tuples_mask.dst_port);
spec->tos = rule->tuples.ip_tos;
spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ?
0 : rule->tuples_mask.ip_tos;
}
static void hclge_fd_get_ip4_info(struct hclge_fd_rule *rule,
struct ethtool_usrip4_spec *spec,
struct ethtool_usrip4_spec *spec_mask)
{
spec->ip4src = cpu_to_be32(rule->tuples.src_ip[IPV4_INDEX]);
spec_mask->ip4src = rule->unused_tuple & BIT(INNER_SRC_IP) ?
0 : cpu_to_be32(rule->tuples_mask.src_ip[IPV4_INDEX]);
spec->ip4dst = cpu_to_be32(rule->tuples.dst_ip[IPV4_INDEX]);
spec_mask->ip4dst = rule->unused_tuple & BIT(INNER_DST_IP) ?
0 : cpu_to_be32(rule->tuples_mask.dst_ip[IPV4_INDEX]);
spec->tos = rule->tuples.ip_tos;
spec_mask->tos = rule->unused_tuple & BIT(INNER_IP_TOS) ?
0 : rule->tuples_mask.ip_tos;
spec->proto = rule->tuples.ip_proto;
spec_mask->proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ?
0 : rule->tuples_mask.ip_proto;
spec->ip_ver = ETH_RX_NFC_IP4;
}
static void hclge_fd_get_tcpip6_info(struct hclge_fd_rule *rule,
struct ethtool_tcpip6_spec *spec,
struct ethtool_tcpip6_spec *spec_mask)
{
ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip);
ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip);
if (rule->unused_tuple & BIT(INNER_SRC_IP))
memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src));
else
ipv6_addr_cpu_to_be32(spec_mask->ip6src,
rule->tuples_mask.src_ip);
if (rule->unused_tuple & BIT(INNER_DST_IP))
memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst));
else
ipv6_addr_cpu_to_be32(spec_mask->ip6dst,
rule->tuples_mask.dst_ip);
spec->tclass = rule->tuples.ip_tos;
spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ?
0 : rule->tuples_mask.ip_tos;
spec->psrc = cpu_to_be16(rule->tuples.src_port);
spec_mask->psrc = rule->unused_tuple & BIT(INNER_SRC_PORT) ?
0 : cpu_to_be16(rule->tuples_mask.src_port);
spec->pdst = cpu_to_be16(rule->tuples.dst_port);
spec_mask->pdst = rule->unused_tuple & BIT(INNER_DST_PORT) ?
0 : cpu_to_be16(rule->tuples_mask.dst_port);
}
static void hclge_fd_get_ip6_info(struct hclge_fd_rule *rule,
struct ethtool_usrip6_spec *spec,
struct ethtool_usrip6_spec *spec_mask)
{
ipv6_addr_cpu_to_be32(spec->ip6src, rule->tuples.src_ip);
ipv6_addr_cpu_to_be32(spec->ip6dst, rule->tuples.dst_ip);
if (rule->unused_tuple & BIT(INNER_SRC_IP))
memset(spec_mask->ip6src, 0, sizeof(spec_mask->ip6src));
else
ipv6_addr_cpu_to_be32(spec_mask->ip6src,
rule->tuples_mask.src_ip);
if (rule->unused_tuple & BIT(INNER_DST_IP))
memset(spec_mask->ip6dst, 0, sizeof(spec_mask->ip6dst));
else
ipv6_addr_cpu_to_be32(spec_mask->ip6dst,
rule->tuples_mask.dst_ip);
spec->tclass = rule->tuples.ip_tos;
spec_mask->tclass = rule->unused_tuple & BIT(INNER_IP_TOS) ?
0 : rule->tuples_mask.ip_tos;
spec->l4_proto = rule->tuples.ip_proto;
spec_mask->l4_proto = rule->unused_tuple & BIT(INNER_IP_PROTO) ?
0 : rule->tuples_mask.ip_proto;
}
static void hclge_fd_get_ether_info(struct hclge_fd_rule *rule,
struct ethhdr *spec,
struct ethhdr *spec_mask)
{
ether_addr_copy(spec->h_source, rule->tuples.src_mac);
ether_addr_copy(spec->h_dest, rule->tuples.dst_mac);
if (rule->unused_tuple & BIT(INNER_SRC_MAC))
eth_zero_addr(spec_mask->h_source);
else
ether_addr_copy(spec_mask->h_source, rule->tuples_mask.src_mac);
if (rule->unused_tuple & BIT(INNER_DST_MAC))
eth_zero_addr(spec_mask->h_dest);
else
ether_addr_copy(spec_mask->h_dest, rule->tuples_mask.dst_mac);
spec->h_proto = cpu_to_be16(rule->tuples.ether_proto);
spec_mask->h_proto = rule->unused_tuple & BIT(INNER_ETH_TYPE) ?
0 : cpu_to_be16(rule->tuples_mask.ether_proto);
}
static void hclge_fd_get_user_def_info(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
if ((rule->unused_tuple & HCLGE_FD_TUPLE_USER_DEF_TUPLES) ==
HCLGE_FD_TUPLE_USER_DEF_TUPLES) {
fs->h_ext.data[0] = 0;
fs->h_ext.data[1] = 0;
fs->m_ext.data[0] = 0;
fs->m_ext.data[1] = 0;
} else {
fs->h_ext.data[0] = cpu_to_be32(rule->ep.user_def.offset);
fs->h_ext.data[1] = cpu_to_be32(rule->ep.user_def.data);
fs->m_ext.data[0] =
cpu_to_be32(HCLGE_FD_USER_DEF_OFFSET_UNMASK);
fs->m_ext.data[1] = cpu_to_be32(rule->ep.user_def.data_mask);
}
}
static void hclge_fd_get_ext_info(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
if (fs->flow_type & FLOW_EXT) {
fs->h_ext.vlan_tci = cpu_to_be16(rule->tuples.vlan_tag1);
fs->m_ext.vlan_tci =
rule->unused_tuple & BIT(INNER_VLAN_TAG_FST) ?
0 : cpu_to_be16(rule->tuples_mask.vlan_tag1);
hclge_fd_get_user_def_info(fs, rule);
}
if (fs->flow_type & FLOW_MAC_EXT) {
ether_addr_copy(fs->h_ext.h_dest, rule->tuples.dst_mac);
if (rule->unused_tuple & BIT(INNER_DST_MAC))
eth_zero_addr(fs->m_u.ether_spec.h_dest);
else
ether_addr_copy(fs->m_u.ether_spec.h_dest,
rule->tuples_mask.dst_mac);
}
}
static struct hclge_fd_rule *hclge_get_fd_rule(struct hclge_dev *hdev,
u16 location)
{
struct hclge_fd_rule *rule = NULL;
struct hlist_node *node2;
hlist_for_each_entry_safe(rule, node2, &hdev->fd_rule_list, rule_node) {
if (rule->location == location)
return rule;
else if (rule->location > location)
return NULL;
}
return NULL;
}
static void hclge_fd_get_ring_cookie(struct ethtool_rx_flow_spec *fs,
struct hclge_fd_rule *rule)
{
if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) {
fs->ring_cookie = RX_CLS_FLOW_DISC;
} else {
u64 vf_id;
fs->ring_cookie = rule->queue_id;
vf_id = rule->vf_id;
vf_id <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF;
fs->ring_cookie |= vf_id;
}
}
int hclge_get_fd_rule_info(struct hnae3_handle *handle,
struct ethtool_rxnfc *cmd)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_fd_rule *rule = NULL;
struct hclge_dev *hdev = vport->back;
struct ethtool_rx_flow_spec *fs;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return -EOPNOTSUPP;
fs = (struct ethtool_rx_flow_spec *)&cmd->fs;
spin_lock_bh(&hdev->fd_rule_lock);
rule = hclge_get_fd_rule(hdev, fs->location);
if (!rule) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -ENOENT;
}
fs->flow_type = rule->flow_type;
switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT)) {
case SCTP_V4_FLOW:
case TCP_V4_FLOW:
case UDP_V4_FLOW:
hclge_fd_get_tcpip4_info(rule, &fs->h_u.tcp_ip4_spec,
&fs->m_u.tcp_ip4_spec);
break;
case IP_USER_FLOW:
hclge_fd_get_ip4_info(rule, &fs->h_u.usr_ip4_spec,
&fs->m_u.usr_ip4_spec);
break;
case SCTP_V6_FLOW:
case TCP_V6_FLOW:
case UDP_V6_FLOW:
hclge_fd_get_tcpip6_info(rule, &fs->h_u.tcp_ip6_spec,
&fs->m_u.tcp_ip6_spec);
break;
case IPV6_USER_FLOW:
hclge_fd_get_ip6_info(rule, &fs->h_u.usr_ip6_spec,
&fs->m_u.usr_ip6_spec);
break;
default:
hclge_fd_get_ether_info(rule, &fs->h_u.ether_spec,
&fs->m_u.ether_spec);
break;
}
hclge_fd_get_ext_info(fs, rule);
hclge_fd_get_ring_cookie(fs, rule);
spin_unlock_bh(&hdev->fd_rule_lock);
return 0;
}
int hclge_get_all_rules(struct hnae3_handle *handle,
struct ethtool_rxnfc *cmd, u32 *rule_locs)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct hclge_fd_rule *rule;
struct hlist_node *node2;
u32 cnt = 0;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return -EOPNOTSUPP;
cmd->data = hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1];
spin_lock_bh(&hdev->fd_rule_lock);
hlist_for_each_entry_safe(rule, node2,
&hdev->fd_rule_list, rule_node) {
if (cnt == cmd->rule_cnt) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -EMSGSIZE;
}
if (rule->state == HCLGE_FD_TO_DEL)
continue;
rule_locs[cnt] = rule->location;
cnt++;
}
spin_unlock_bh(&hdev->fd_rule_lock);
cmd->rule_cnt = cnt;
return 0;
}
static void hclge_fd_get_flow_tuples(const struct flow_keys *fkeys,
struct hclge_fd_rule_tuples *tuples)
{
#define flow_ip6_src fkeys->addrs.v6addrs.src.in6_u.u6_addr32
#define flow_ip6_dst fkeys->addrs.v6addrs.dst.in6_u.u6_addr32
tuples->ether_proto = be16_to_cpu(fkeys->basic.n_proto);
tuples->ip_proto = fkeys->basic.ip_proto;
tuples->dst_port = be16_to_cpu(fkeys->ports.dst);
if (fkeys->basic.n_proto == htons(ETH_P_IP)) {
tuples->src_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.src);
tuples->dst_ip[3] = be32_to_cpu(fkeys->addrs.v4addrs.dst);
} else {
int i;
for (i = 0; i < IPV6_ADDR_WORDS; i++) {
tuples->src_ip[i] = be32_to_cpu(flow_ip6_src[i]);
tuples->dst_ip[i] = be32_to_cpu(flow_ip6_dst[i]);
}
}
}
static struct hclge_fd_rule *
hclge_fd_search_flow_keys(struct hclge_dev *hdev,
const struct hclge_fd_rule_tuples *tuples)
{
struct hclge_fd_rule *rule = NULL;
struct hlist_node *node;
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) {
if (!memcmp(tuples, &rule->tuples, sizeof(*tuples)))
return rule;
}
return NULL;
}
static void hclge_fd_build_arfs_rule(const struct hclge_fd_rule_tuples *tuples,
struct hclge_fd_rule *rule)
{
rule->unused_tuple = BIT(INNER_SRC_MAC) | BIT(INNER_DST_MAC) |
BIT(INNER_VLAN_TAG_FST) | BIT(INNER_IP_TOS) |
BIT(INNER_SRC_PORT);
rule->action = 0;
rule->vf_id = 0;
rule->rule_type = HCLGE_FD_ARFS_ACTIVE;
rule->state = HCLGE_FD_TO_ADD;
if (tuples->ether_proto == ETH_P_IP) {
if (tuples->ip_proto == IPPROTO_TCP)
rule->flow_type = TCP_V4_FLOW;
else
rule->flow_type = UDP_V4_FLOW;
} else {
if (tuples->ip_proto == IPPROTO_TCP)
rule->flow_type = TCP_V6_FLOW;
else
rule->flow_type = UDP_V6_FLOW;
}
memcpy(&rule->tuples, tuples, sizeof(rule->tuples));
memset(&rule->tuples_mask, 0xFF, sizeof(rule->tuples_mask));
}
int hclge_add_fd_entry_by_arfs(struct hnae3_handle *handle, u16 queue_id,
u16 flow_id, struct flow_keys *fkeys)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_fd_rule_tuples new_tuples = {};
struct hclge_dev *hdev = vport->back;
struct hclge_fd_rule *rule;
u16 bit_id;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return -EOPNOTSUPP;
spin_lock_bh(&hdev->fd_rule_lock);
if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE &&
hdev->fd_active_type != HCLGE_FD_RULE_NONE) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -EOPNOTSUPP;
}
hclge_fd_get_flow_tuples(fkeys, &new_tuples);
rule = hclge_fd_search_flow_keys(hdev, &new_tuples);
if (!rule) {
bit_id = find_first_zero_bit(hdev->fd_bmap, MAX_FD_FILTER_NUM);
if (bit_id >= hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -ENOSPC;
}
rule = kzalloc_obj(*rule, GFP_ATOMIC);
if (!rule) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -ENOMEM;
}
rule->location = bit_id;
rule->arfs.flow_id = flow_id;
rule->queue_id = queue_id;
hclge_fd_build_arfs_rule(&new_tuples, rule);
hclge_update_fd_list(hdev, rule->state, rule->location, rule);
hdev->fd_active_type = HCLGE_FD_ARFS_ACTIVE;
} else if (rule->queue_id != queue_id) {
rule->queue_id = queue_id;
rule->state = HCLGE_FD_TO_ADD;
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
hclge_task_schedule(hdev, 0);
}
spin_unlock_bh(&hdev->fd_rule_lock);
return rule->location;
}
void hclge_rfs_filter_expire(struct hclge_dev *hdev)
{
#ifdef CONFIG_RFS_ACCEL
struct hnae3_handle *handle = &hdev->vport[0].nic;
struct hclge_fd_rule *rule;
struct hlist_node *node;
spin_lock_bh(&hdev->fd_rule_lock);
if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE) {
spin_unlock_bh(&hdev->fd_rule_lock);
return;
}
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) {
if (rule->state != HCLGE_FD_ACTIVE)
continue;
if (rps_may_expire_flow(handle->netdev, rule->queue_id,
rule->arfs.flow_id, rule->location)) {
rule->state = HCLGE_FD_TO_DEL;
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
}
}
spin_unlock_bh(&hdev->fd_rule_lock);
#endif
}
int hclge_clear_arfs_rules(struct hclge_dev *hdev)
{
#ifdef CONFIG_RFS_ACCEL
struct hclge_fd_rule *rule;
struct hlist_node *node;
int ret;
if (hdev->fd_active_type != HCLGE_FD_ARFS_ACTIVE)
return 0;
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) {
switch (rule->state) {
case HCLGE_FD_TO_DEL:
case HCLGE_FD_ACTIVE:
ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true,
rule->location, NULL, false);
if (ret)
return ret;
fallthrough;
case HCLGE_FD_TO_ADD:
hclge_fd_dec_rule_cnt(hdev, rule->location);
hlist_del(&rule->rule_node);
kfree(rule);
break;
default:
break;
}
}
hclge_sync_fd_state(hdev);
#endif
return 0;
}
static void hclge_get_cls_key_basic(const struct flow_rule *flow,
struct hclge_fd_rule *rule)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_BASIC)) {
struct flow_match_basic match;
u16 ethtype_key, ethtype_mask;
flow_rule_match_basic(flow, &match);
ethtype_key = ntohs(match.key->n_proto);
ethtype_mask = ntohs(match.mask->n_proto);
if (ethtype_key == ETH_P_ALL) {
ethtype_key = 0;
ethtype_mask = 0;
}
rule->tuples.ether_proto = ethtype_key;
rule->tuples_mask.ether_proto = ethtype_mask;
rule->tuples.ip_proto = match.key->ip_proto;
rule->tuples_mask.ip_proto = match.mask->ip_proto;
} else {
rule->unused_tuple |= BIT(INNER_IP_PROTO);
rule->unused_tuple |= BIT(INNER_ETH_TYPE);
}
}
static void hclge_get_cls_key_mac(const struct flow_rule *flow,
struct hclge_fd_rule *rule)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
struct flow_match_eth_addrs match;
flow_rule_match_eth_addrs(flow, &match);
ether_addr_copy(rule->tuples.dst_mac, match.key->dst);
ether_addr_copy(rule->tuples_mask.dst_mac, match.mask->dst);
ether_addr_copy(rule->tuples.src_mac, match.key->src);
ether_addr_copy(rule->tuples_mask.src_mac, match.mask->src);
if (is_zero_ether_addr(match.mask->dst))
rule->unused_tuple |= BIT(INNER_DST_MAC);
if (is_zero_ether_addr(match.mask->src))
rule->unused_tuple |= BIT(INNER_SRC_MAC);
} else {
rule->unused_tuple |= BIT(INNER_DST_MAC);
rule->unused_tuple |= BIT(INNER_SRC_MAC);
}
}
static void hclge_get_cls_key_vlan(const struct flow_rule *flow,
struct hclge_fd_rule *rule)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_VLAN)) {
struct flow_match_vlan match;
flow_rule_match_vlan(flow, &match);
rule->tuples.vlan_tag1 = match.key->vlan_id |
(match.key->vlan_priority << VLAN_PRIO_SHIFT);
rule->tuples_mask.vlan_tag1 = match.mask->vlan_id |
(match.mask->vlan_priority << VLAN_PRIO_SHIFT);
} else {
rule->unused_tuple |= BIT(INNER_VLAN_TAG_FST);
}
}
static int hclge_get_cls_key_ip(const struct flow_rule *flow,
struct hclge_fd_rule *rule,
struct netlink_ext_ack *extack)
{
u16 addr_type = 0;
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_CONTROL)) {
struct flow_match_control match;
flow_rule_match_control(flow, &match);
addr_type = match.key->addr_type;
if (flow_rule_has_control_flags(match.mask->flags, extack))
return -EOPNOTSUPP;
}
if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
struct flow_match_ipv4_addrs match;
flow_rule_match_ipv4_addrs(flow, &match);
rule->tuples.src_ip[IPV4_INDEX] = be32_to_cpu(match.key->src);
rule->tuples_mask.src_ip[IPV4_INDEX] =
be32_to_cpu(match.mask->src);
rule->tuples.dst_ip[IPV4_INDEX] = be32_to_cpu(match.key->dst);
rule->tuples_mask.dst_ip[IPV4_INDEX] =
be32_to_cpu(match.mask->dst);
if (!match.mask->src)
rule->unused_tuple |= BIT(INNER_SRC_IP);
if (!match.mask->dst)
rule->unused_tuple |= BIT(INNER_DST_IP);
} else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
struct flow_match_ipv6_addrs match;
flow_rule_match_ipv6_addrs(flow, &match);
ipv6_addr_be32_to_cpu(rule->tuples.src_ip,
match.key->src.s6_addr32);
ipv6_addr_be32_to_cpu(rule->tuples_mask.src_ip,
match.mask->src.s6_addr32);
ipv6_addr_be32_to_cpu(rule->tuples.dst_ip,
match.key->dst.s6_addr32);
ipv6_addr_be32_to_cpu(rule->tuples_mask.dst_ip,
match.mask->dst.s6_addr32);
if (ipv6_addr_any(&match.mask->src))
rule->unused_tuple |= BIT(INNER_SRC_IP);
if (ipv6_addr_any(&match.mask->dst))
rule->unused_tuple |= BIT(INNER_DST_IP);
} else {
rule->unused_tuple |= BIT(INNER_SRC_IP);
rule->unused_tuple |= BIT(INNER_DST_IP);
}
return 0;
}
static void hclge_get_cls_key_port(const struct flow_rule *flow,
struct hclge_fd_rule *rule)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_PORTS)) {
struct flow_match_ports match;
flow_rule_match_ports(flow, &match);
rule->tuples.src_port = be16_to_cpu(match.key->src);
rule->tuples_mask.src_port = be16_to_cpu(match.mask->src);
rule->tuples.dst_port = be16_to_cpu(match.key->dst);
rule->tuples_mask.dst_port = be16_to_cpu(match.mask->dst);
} else {
rule->unused_tuple |= BIT(INNER_SRC_PORT);
rule->unused_tuple |= BIT(INNER_DST_PORT);
}
}
static int hclge_get_cls_enc_keyid(struct hclge_dev *hdev,
const struct flow_rule *flow,
struct hclge_fd_rule *rule,
struct netlink_ext_ack *extack)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
struct flow_match_enc_keyid match;
flow_rule_match_enc_keyid(flow, &match);
if (be32_to_cpu(match.mask->keyid) !=
HCLGE_FD_VXLAN_VNI_UNMASK ||
be32_to_cpu(match.key->keyid) >= VXLAN_N_VID ||
!match.key->keyid) {
NL_SET_ERR_MSG_MOD(extack, "invalid enc_keyid");
return -EINVAL;
}
rule->tuples.outer_tun_vni = be32_to_cpu(match.key->keyid);
rule->tuples_mask.outer_tun_vni =
be32_to_cpu(match.mask->keyid);
} else {
rule->unused_tuple |= BIT(OUTER_TUN_VNI);
}
return 0;
}
static int hclge_get_cls_key_ip_tos(const struct flow_rule *flow,
struct hclge_fd_rule *rule,
struct netlink_ext_ack *extack)
{
if (flow_rule_match_key(flow, FLOW_DISSECTOR_KEY_IP)) {
struct flow_match_ip match;
flow_rule_match_ip(flow, &match);
if (match.mask->ttl) {
NL_SET_ERR_MSG_MOD(extack, "unsupported TTL");
return -EOPNOTSUPP;
}
rule->tuples.ip_tos = match.key->tos;
rule->tuples_mask.ip_tos = match.mask->tos;
if (!rule->tuples_mask.ip_tos)
rule->unused_tuple |= BIT(INNER_IP_TOS);
} else {
rule->unused_tuple |= BIT(INNER_IP_TOS);
}
return 0;
}
static int hclge_get_tc_flower_action(struct hclge_dev *hdev,
struct flow_cls_offload *cls_flower,
struct hclge_fd_rule *rule)
{
struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower);
struct netlink_ext_ack *extack = cls_flower->common.extack;
struct hnae3_handle *handle = &hdev->vport[0].nic;
struct flow_action *action = &flow->action;
struct flow_action_entry *act;
int tc;
if (!flow_action_has_entries(&flow->action)) {
tc = tc_classid_to_hwtc(handle->netdev, cls_flower->classid);
if (tc < 0 || tc > hdev->tc_max) {
NL_SET_ERR_MSG_FMT_MOD(extack,
"invalid traffic class: %d", tc);
return -EINVAL;
}
rule->action = HCLGE_FD_ACTION_SELECT_TC;
rule->cls_flower.tc = tc;
return 0;
}
act = &action->entries[0];
switch (act->id) {
case FLOW_ACTION_RX_QUEUE_MAPPING:
if (act->rx_queue >= handle->kinfo.num_tqps) {
NL_SET_ERR_MSG_FMT_MOD(extack,
"queue id (%u) should be less than %u",
act->rx_queue,
handle->kinfo.num_tqps);
return -EINVAL;
}
rule->queue_id = act->rx_queue;
rule->action = HCLGE_FD_ACTION_SELECT_QUEUE;
return 0;
case FLOW_ACTION_DROP:
rule->action = HCLGE_FD_ACTION_DROP_PACKET;
return 0;
default:
NL_SET_ERR_MSG_FMT_MOD(extack,
"unsupported action(%d)", act->id);
return -EOPNOTSUPP;
}
}
static int hclge_parse_cls_flower(struct hclge_dev *hdev,
struct flow_cls_offload *cls_flower,
struct hclge_fd_rule *rule)
{
struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower);
struct netlink_ext_ack *extack = cls_flower->common.extack;
int ret;
rule->unused_tuple |= HCLGE_FD_TUPLE_USER_DEF_TUPLES;
hclge_get_cls_key_basic(flow, rule);
hclge_get_cls_key_mac(flow, rule);
hclge_get_cls_key_vlan(flow, rule);
ret = hclge_get_cls_key_ip(flow, rule, extack);
if (ret)
return ret;
hclge_get_cls_key_port(flow, rule);
ret = hclge_get_cls_key_ip_tos(flow, rule, extack);
if (ret)
return ret;
return hclge_get_cls_enc_keyid(hdev, flow, rule, extack);
}
static int hclge_check_cls_flower(struct hclge_dev *hdev,
struct flow_cls_offload *cls_flower)
{
struct flow_rule *flow = flow_cls_offload_flow_rule(cls_flower);
struct netlink_ext_ack *extack = cls_flower->common.extack;
struct flow_dissector *dissector = flow->match.dissector;
u32 prio = cls_flower->common.prio;
u64 support_keys;
if (prio == 0 ||
prio > hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]) {
NL_SET_ERR_MSG_FMT_MOD(extack,
"prio %u should be in range[1, %u]",
prio,
hdev->fd_cfg.rule_num[HCLGE_FD_STAGE_1]);
return -EINVAL;
}
if (test_bit(prio - 1, hdev->fd_bmap)) {
NL_SET_ERR_MSG_FMT_MOD(extack,
"prio %u is already used", prio);
return -EINVAL;
}
support_keys = BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
BIT_ULL(FLOW_DISSECTOR_KEY_IP);
if (hdev->fd_cfg.fd_mode == HCLGE_FD_MODE_DEPTH_2K_WIDTH_400B_STAGE_1)
support_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID);
if (dissector->used_keys & ~support_keys) {
NL_SET_ERR_MSG_FMT_MOD(extack, "unsupported key set: %#llx",
dissector->used_keys);
return -EOPNOTSUPP;
}
if (cls_flower->classid && flow_action_has_entries(&flow->action)) {
NL_SET_ERR_MSG_MOD(extack,
"cannot specify both classid and action");
return -EOPNOTSUPP;
}
if (!flow_action_has_entries(&flow->action) && !cls_flower->classid) {
NL_SET_ERR_MSG_MOD(extack,
"must specify either classid or action");
return -EINVAL;
}
if (flow_action_has_entries(&flow->action) &&
!flow_offload_has_one_action(&flow->action)) {
NL_SET_ERR_MSG_MOD(extack, "unsupported multiple actions");
return -EOPNOTSUPP;
}
return 0;
}
int hclge_add_cls_flower(struct hnae3_handle *handle,
struct flow_cls_offload *cls_flower)
{
struct netlink_ext_ack *extack = cls_flower->common.extack;
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct hclge_fd_rule *rule;
int ret;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev)) {
NL_SET_ERR_MSG_MOD(extack, "cls flower is not supported");
return -EOPNOTSUPP;
}
ret = hclge_check_cls_flower(hdev, cls_flower);
if (ret)
return ret;
rule = kzalloc_obj(*rule);
if (!rule)
return -ENOMEM;
ret = hclge_parse_cls_flower(hdev, cls_flower, rule);
if (ret) {
kfree(rule);
return ret;
}
ret = hclge_get_tc_flower_action(hdev, cls_flower, rule);
if (ret) {
kfree(rule);
return ret;
}
rule->location = cls_flower->common.prio - 1;
rule->vf_id = 0;
rule->cls_flower.cookie = cls_flower->cookie;
rule->rule_type = HCLGE_FD_TC_FLOWER_ACTIVE;
ret = hclge_add_fd_entry_common(hdev, rule);
if (ret)
kfree(rule);
return ret;
}
static struct hclge_fd_rule *hclge_find_cls_flower(struct hclge_dev *hdev,
unsigned long cookie)
{
struct hclge_fd_rule *rule;
struct hlist_node *node;
hlist_for_each_entry_safe(rule, node, &hdev->fd_rule_list, rule_node) {
if (rule->cls_flower.cookie == cookie)
return rule;
}
return NULL;
}
int hclge_del_cls_flower(struct hnae3_handle *handle,
struct flow_cls_offload *cls_flower)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
struct hclge_fd_rule *rule;
int ret;
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return -EOPNOTSUPP;
spin_lock_bh(&hdev->fd_rule_lock);
rule = hclge_find_cls_flower(hdev, cls_flower->cookie);
if (!rule) {
spin_unlock_bh(&hdev->fd_rule_lock);
return -EINVAL;
}
ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true, rule->location,
NULL, false);
if (ret) {
hclge_update_fd_list(hdev, HCLGE_FD_TO_DEL,
rule->location, NULL);
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
spin_unlock_bh(&hdev->fd_rule_lock);
return ret;
}
hclge_update_fd_list(hdev, HCLGE_FD_DELETED, rule->location, NULL);
spin_unlock_bh(&hdev->fd_rule_lock);
return 0;
}
static void hclge_sync_fd_list(struct hclge_dev *hdev, struct hlist_head *hlist)
{
struct hclge_fd_rule *rule;
struct hlist_node *node;
int ret = 0;
if (!test_and_clear_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state))
return;
spin_lock_bh(&hdev->fd_rule_lock);
hlist_for_each_entry_safe(rule, node, hlist, rule_node) {
switch (rule->state) {
case HCLGE_FD_TO_ADD:
ret = hclge_fd_config_rule(hdev, rule);
if (ret)
goto out;
rule->state = HCLGE_FD_ACTIVE;
break;
case HCLGE_FD_TO_DEL:
ret = hclge_fd_tcam_config(hdev, HCLGE_FD_STAGE_1, true,
rule->location, NULL, false);
if (ret)
goto out;
hclge_fd_dec_rule_cnt(hdev, rule->location);
hclge_fd_free_node(hdev, rule);
break;
default:
break;
}
}
out:
if (ret)
set_bit(HCLGE_STATE_FD_TBL_CHANGED, &hdev->state);
spin_unlock_bh(&hdev->fd_rule_lock);
}
void hclge_sync_fd_table(struct hclge_dev *hdev)
{
if (!hnae3_ae_dev_fd_supported(hdev->ae_dev))
return;
if (test_and_clear_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state)) {
bool clear_list = hdev->fd_active_type == HCLGE_FD_ARFS_ACTIVE;
hclge_clear_fd_rules_in_list(hdev, clear_list);
}
hclge_sync_fd_user_def_cfg(hdev, false);
hclge_sync_fd_list(hdev, &hdev->fd_rule_list);
}
void hclge_enable_fd(struct hnae3_handle *handle, bool enable)
{
struct hclge_vport *vport = hclge_get_vport(handle);
struct hclge_dev *hdev = vport->back;
hdev->fd_en = enable;
if (!enable)
set_bit(HCLGE_STATE_FD_CLEAR_ALL, &hdev->state);
else
hclge_restore_fd_entries(handle);
hclge_task_schedule(hdev, 0);
}
