#include <linux/mlx5/device.h>
#include <linux/mlx5/mlx5_ifc.h>
#include <linux/xarray.h>
#include <linux/if_vlan.h>
#include <linux/iopoll.h>
#include "en.h"
#include "lib/aso.h"
#include "lib/crypto.h"
#include "en_accel/macsec.h"
#define MLX5_MACSEC_EPN_SCOPE_MID 0x80000000L
#define MLX5E_MACSEC_ASO_CTX_SZ MLX5_ST_SZ_BYTES(macsec_aso)
enum mlx5_macsec_aso_event_arm {
MLX5E_ASO_EPN_ARM = BIT(0),
};
enum {
MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
};
struct mlx5e_macsec_handle {
struct mlx5e_macsec *macsec;
u32 obj_id;
u8 idx;
};
enum {
MLX5_MACSEC_EPN,
};
struct mlx5e_macsec_aso_out {
u8 event_arm;
u32 mode_param;
};
struct mlx5e_macsec_aso_in {
u8 mode;
u32 obj_id;
};
struct mlx5e_macsec_epn_state {
u32 epn_msb;
u8 epn_enabled;
u8 overlap;
};
struct mlx5e_macsec_async_work {
struct mlx5e_macsec *macsec;
struct mlx5_core_dev *mdev;
struct work_struct work;
u32 obj_id;
};
struct mlx5e_macsec_sa {
bool active;
u8 assoc_num;
u32 macsec_obj_id;
u32 enc_key_id;
u32 next_pn;
sci_t sci;
ssci_t ssci;
salt_t salt;
union mlx5_macsec_rule *macsec_rule;
struct rcu_head rcu_head;
struct mlx5e_macsec_epn_state epn_state;
};
struct mlx5e_macsec_rx_sc;
struct mlx5e_macsec_rx_sc_xarray_element {
u32 fs_id;
struct mlx5e_macsec_rx_sc *rx_sc;
};
struct mlx5e_macsec_rx_sc {
bool active;
sci_t sci;
struct mlx5e_macsec_sa *rx_sa[MACSEC_NUM_AN];
struct list_head rx_sc_list_element;
struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
struct metadata_dst *md_dst;
struct rcu_head rcu_head;
};
struct mlx5e_macsec_umr {
u8 __aligned(64) ctx[MLX5_ST_SZ_BYTES(macsec_aso)];
dma_addr_t dma_addr;
u32 mkey;
};
struct mlx5e_macsec_aso {
struct mlx5_aso *maso;
struct mutex aso_lock;
struct mlx5e_macsec_umr *umr;
u32 pdn;
};
struct mlx5e_macsec_device {
const struct net_device *netdev;
struct mlx5e_macsec_sa *tx_sa[MACSEC_NUM_AN];
struct list_head macsec_rx_sc_list_head;
unsigned char *dev_addr;
struct list_head macsec_device_list_element;
};
struct mlx5e_macsec {
struct list_head macsec_device_list_head;
int num_of_devices;
struct mutex lock;
struct xarray sc_xarray;
struct mlx5_core_dev *mdev;
struct mlx5e_macsec_aso aso;
struct notifier_block nb;
struct workqueue_struct *wq;
};
struct mlx5_macsec_obj_attrs {
u32 aso_pdn;
u32 next_pn;
__be64 sci;
u32 enc_key_id;
bool encrypt;
struct mlx5e_macsec_epn_state epn_state;
salt_t salt;
__be32 ssci;
bool replay_protect;
u32 replay_window;
};
struct mlx5_aso_ctrl_param {
u8 data_mask_mode;
u8 condition_0_operand;
u8 condition_1_operand;
u8 condition_0_offset;
u8 condition_1_offset;
u8 data_offset;
u8 condition_operand;
u32 condition_0_data;
u32 condition_0_mask;
u32 condition_1_data;
u32 condition_1_mask;
u64 bitwise_data;
u64 data_mask;
};
static int mlx5e_macsec_aso_reg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso)
{
struct mlx5e_macsec_umr *umr;
struct device *dma_device;
dma_addr_t dma_addr;
int err;
umr = kzalloc_obj(*umr);
if (!umr) {
err = -ENOMEM;
return err;
}
dma_device = mlx5_core_dma_dev(mdev);
dma_addr = dma_map_single(dma_device, umr->ctx, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
err = dma_mapping_error(dma_device, dma_addr);
if (err) {
mlx5_core_err(mdev, "Can't map dma device, err=%d\n", err);
goto out_dma;
}
err = mlx5e_create_mkey(mdev, aso->pdn, &umr->mkey);
if (err) {
mlx5_core_err(mdev, "Can't create mkey, err=%d\n", err);
goto out_mkey;
}
umr->dma_addr = dma_addr;
aso->umr = umr;
return 0;
out_mkey:
dma_unmap_single(dma_device, dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
out_dma:
kfree(umr);
return err;
}
static void mlx5e_macsec_aso_dereg_mr(struct mlx5_core_dev *mdev, struct mlx5e_macsec_aso *aso)
{
struct mlx5e_macsec_umr *umr = aso->umr;
mlx5_core_destroy_mkey(mdev, umr->mkey);
dma_unmap_single(&mdev->pdev->dev, umr->dma_addr, sizeof(umr->ctx), DMA_BIDIRECTIONAL);
kfree(umr);
}
static int macsec_set_replay_protection(struct mlx5_macsec_obj_attrs *attrs, void *aso_ctx)
{
u8 window_sz;
if (!attrs->replay_protect)
return 0;
switch (attrs->replay_window) {
case 256:
window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_256BIT;
break;
case 128:
window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_128BIT;
break;
case 64:
window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_64BIT;
break;
case 32:
window_sz = MLX5_MACSEC_ASO_REPLAY_WIN_32BIT;
break;
default:
return -EINVAL;
}
MLX5_SET(macsec_aso, aso_ctx, window_size, window_sz);
MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_REPLAY_PROTECTION);
return 0;
}
static int mlx5e_macsec_create_object(struct mlx5_core_dev *mdev,
struct mlx5_macsec_obj_attrs *attrs,
bool is_tx,
u32 *macsec_obj_id)
{
u32 in[MLX5_ST_SZ_DW(create_macsec_obj_in)] = {};
u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
void *aso_ctx;
void *obj;
int err;
obj = MLX5_ADDR_OF(create_macsec_obj_in, in, macsec_object);
aso_ctx = MLX5_ADDR_OF(macsec_offload_obj, obj, macsec_aso);
MLX5_SET(macsec_offload_obj, obj, confidentiality_en, attrs->encrypt);
MLX5_SET(macsec_offload_obj, obj, dekn, attrs->enc_key_id);
MLX5_SET(macsec_offload_obj, obj, aso_return_reg, MLX5_MACSEC_ASO_REG_C_4_5);
MLX5_SET(macsec_offload_obj, obj, macsec_aso_access_pd, attrs->aso_pdn);
MLX5_SET(macsec_aso, aso_ctx, mode_parameter, attrs->next_pn);
if (attrs->epn_state.epn_enabled) {
void *salt_p;
int i;
MLX5_SET(macsec_aso, aso_ctx, epn_event_arm, 1);
MLX5_SET(macsec_offload_obj, obj, epn_en, 1);
MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb);
MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap);
MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)attrs->ssci);
salt_p = MLX5_ADDR_OF(macsec_offload_obj, obj, salt);
for (i = 0; i < 3 ; i++)
memcpy((u32 *)salt_p + i, &attrs->salt.bytes[4 * (2 - i)], 4);
} else {
MLX5_SET64(macsec_offload_obj, obj, sci, (__force u64)(attrs->sci));
}
MLX5_SET(macsec_aso, aso_ctx, valid, 0x1);
if (is_tx) {
MLX5_SET(macsec_aso, aso_ctx, mode, MLX5_MACSEC_ASO_INC_SN);
} else {
err = macsec_set_replay_protection(attrs, aso_ctx);
if (err)
return err;
}
MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
if (err) {
mlx5_core_err(mdev,
"MACsec offload: Failed to create MACsec object (err = %d)\n",
err);
return err;
}
*macsec_obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
return err;
}
static void mlx5e_macsec_destroy_object(struct mlx5_core_dev *mdev, u32 macsec_obj_id)
{
u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_obj_id);
mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}
static void mlx5e_macsec_cleanup_sa_fs(struct mlx5e_macsec *macsec,
struct mlx5e_macsec_sa *sa, bool is_tx,
struct net_device *netdev, u32 fs_id)
{
int action = (is_tx) ? MLX5_ACCEL_MACSEC_ACTION_ENCRYPT :
MLX5_ACCEL_MACSEC_ACTION_DECRYPT;
if (!sa->macsec_rule)
return;
mlx5_macsec_fs_del_rule(macsec->mdev->macsec_fs, sa->macsec_rule, action, netdev,
fs_id);
sa->macsec_rule = NULL;
}
static void mlx5e_macsec_cleanup_sa(struct mlx5e_macsec *macsec,
struct mlx5e_macsec_sa *sa, bool is_tx,
struct net_device *netdev, u32 fs_id)
{
mlx5e_macsec_cleanup_sa_fs(macsec, sa, is_tx, netdev, fs_id);
mlx5e_macsec_destroy_object(macsec->mdev, sa->macsec_obj_id);
}
static int mlx5e_macsec_init_sa_fs(struct macsec_context *ctx,
struct mlx5e_macsec_sa *sa, bool encrypt,
bool is_tx, u32 *fs_id)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5_macsec_fs *macsec_fs = priv->mdev->macsec_fs;
const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
struct mlx5_macsec_rule_attrs rule_attrs;
union mlx5_macsec_rule *macsec_rule;
if (is_tx && tx_sc->encoding_sa != sa->assoc_num)
return 0;
rule_attrs.macsec_obj_id = sa->macsec_obj_id;
rule_attrs.sci = sa->sci;
rule_attrs.assoc_num = sa->assoc_num;
rule_attrs.action = (is_tx) ? MLX5_ACCEL_MACSEC_ACTION_ENCRYPT :
MLX5_ACCEL_MACSEC_ACTION_DECRYPT;
macsec_rule = mlx5_macsec_fs_add_rule(macsec_fs, ctx, &rule_attrs, fs_id);
if (!macsec_rule)
return -ENOMEM;
sa->macsec_rule = macsec_rule;
return 0;
}
static int mlx5e_macsec_init_sa(struct macsec_context *ctx,
struct mlx5e_macsec_sa *sa,
bool encrypt, bool is_tx, u32 *fs_id)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec *macsec = priv->macsec;
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5_macsec_obj_attrs obj_attrs;
int err;
obj_attrs.next_pn = sa->next_pn;
obj_attrs.sci = cpu_to_be64((__force u64)sa->sci);
obj_attrs.enc_key_id = sa->enc_key_id;
obj_attrs.encrypt = encrypt;
obj_attrs.aso_pdn = macsec->aso.pdn;
obj_attrs.epn_state = sa->epn_state;
if (sa->epn_state.epn_enabled) {
obj_attrs.ssci = cpu_to_be32((__force u32)sa->ssci);
memcpy(&obj_attrs.salt, &sa->salt, sizeof(sa->salt));
}
obj_attrs.replay_window = ctx->secy->replay_window;
obj_attrs.replay_protect = ctx->secy->replay_protect;
err = mlx5e_macsec_create_object(mdev, &obj_attrs, is_tx, &sa->macsec_obj_id);
if (err)
return err;
if (sa->active) {
err = mlx5e_macsec_init_sa_fs(ctx, sa, encrypt, is_tx, fs_id);
if (err)
goto destroy_macsec_object;
}
return 0;
destroy_macsec_object:
mlx5e_macsec_destroy_object(mdev, sa->macsec_obj_id);
return err;
}
static struct mlx5e_macsec_rx_sc *
mlx5e_macsec_get_rx_sc_from_sc_list(const struct list_head *list, sci_t sci)
{
struct mlx5e_macsec_rx_sc *iter;
list_for_each_entry_rcu(iter, list, rx_sc_list_element) {
if (iter->sci == sci)
return iter;
}
return NULL;
}
static int macsec_rx_sa_active_update(struct macsec_context *ctx,
struct mlx5e_macsec_sa *rx_sa,
bool active, u32 *fs_id)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec *macsec = priv->macsec;
int err = 0;
if (rx_sa->active == active)
return 0;
rx_sa->active = active;
if (!active) {
mlx5e_macsec_cleanup_sa(macsec, rx_sa, false, ctx->secy->netdev, *fs_id);
return 0;
}
err = mlx5e_macsec_init_sa(ctx, rx_sa, true, false, fs_id);
if (err)
rx_sa->active = false;
return err;
}
static bool mlx5e_macsec_secy_features_validate(struct macsec_context *ctx)
{
const struct net_device *netdev = ctx->netdev;
const struct macsec_secy *secy = ctx->secy;
if (secy->validate_frames != MACSEC_VALIDATE_STRICT) {
netdev_err(netdev,
"MACsec offload is supported only when validate_frame is in strict mode\n");
return false;
}
if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) {
netdev_err(netdev, "MACsec offload is supported only when icv_len is %d\n",
MACSEC_DEFAULT_ICV_LEN);
return false;
}
if (!secy->protect_frames) {
netdev_err(netdev,
"MACsec offload is supported only when protect_frames is set\n");
return false;
}
if (!ctx->secy->tx_sc.encrypt) {
netdev_err(netdev, "MACsec offload: encrypt off isn't supported\n");
return false;
}
return true;
}
static struct mlx5e_macsec_device *
mlx5e_macsec_get_macsec_device_context(const struct mlx5e_macsec *macsec,
const struct macsec_context *ctx)
{
struct mlx5e_macsec_device *iter;
const struct list_head *list;
list = &macsec->macsec_device_list_head;
list_for_each_entry_rcu(iter, list, macsec_device_list_element) {
if (iter->netdev == ctx->secy->netdev)
return iter;
}
return NULL;
}
static void update_macsec_epn(struct mlx5e_macsec_sa *sa, const struct macsec_key *key,
const pn_t *next_pn_halves, ssci_t ssci)
{
struct mlx5e_macsec_epn_state *epn_state = &sa->epn_state;
sa->ssci = ssci;
sa->salt = key->salt;
epn_state->epn_enabled = 1;
epn_state->epn_msb = next_pn_halves->upper;
epn_state->overlap = next_pn_halves->lower < MLX5_MACSEC_EPN_SCOPE_MID ? 0 : 1;
}
static int mlx5e_macsec_add_txsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
const struct macsec_secy *secy = ctx->secy;
struct mlx5e_macsec_device *macsec_device;
struct mlx5_core_dev *mdev = priv->mdev;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_sa *tx_sa;
struct mlx5e_macsec *macsec;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EEXIST;
goto out;
}
if (macsec_device->tx_sa[assoc_num]) {
netdev_err(ctx->netdev, "MACsec offload tx_sa: %d already exist\n", assoc_num);
err = -EEXIST;
goto out;
}
tx_sa = kzalloc_obj(*tx_sa);
if (!tx_sa) {
err = -ENOMEM;
goto out;
}
tx_sa->active = ctx_tx_sa->active;
tx_sa->next_pn = ctx_tx_sa->next_pn_halves.lower;
tx_sa->sci = secy->sci;
tx_sa->assoc_num = assoc_num;
if (secy->xpn)
update_macsec_epn(tx_sa, &ctx_tx_sa->key, &ctx_tx_sa->next_pn_halves,
ctx_tx_sa->ssci);
err = mlx5_create_encryption_key(mdev, ctx->sa.key, secy->key_len,
MLX5_ACCEL_OBJ_MACSEC_KEY,
&tx_sa->enc_key_id);
if (err)
goto destroy_sa;
macsec_device->tx_sa[assoc_num] = tx_sa;
if (!secy->operational)
goto out;
err = mlx5e_macsec_init_sa(ctx, tx_sa, tx_sc->encrypt, true, NULL);
if (err)
goto destroy_encryption_key;
mutex_unlock(&macsec->lock);
return 0;
destroy_encryption_key:
macsec_device->tx_sa[assoc_num] = NULL;
mlx5_destroy_encryption_key(mdev, tx_sa->enc_key_id);
destroy_sa:
kfree(tx_sa);
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_upd_txsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
const struct macsec_tx_sa *ctx_tx_sa = ctx->sa.tx_sa;
struct mlx5e_macsec_device *macsec_device;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_sa *tx_sa;
struct mlx5e_macsec *macsec;
struct net_device *netdev;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
netdev = ctx->netdev;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
tx_sa = macsec_device->tx_sa[assoc_num];
if (!tx_sa) {
netdev_err(netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num);
err = -EEXIST;
goto out;
}
if (ctx->sa.update_pn) {
netdev_err(netdev, "MACsec offload: update TX sa %d PN isn't supported\n",
assoc_num);
err = -EINVAL;
goto out;
}
if (tx_sa->active == ctx_tx_sa->active)
goto out;
tx_sa->active = ctx_tx_sa->active;
if (tx_sa->assoc_num != tx_sc->encoding_sa)
goto out;
if (ctx_tx_sa->active) {
err = mlx5e_macsec_init_sa_fs(ctx, tx_sa, tx_sc->encrypt, true, NULL);
if (err)
goto out;
} else {
if (!tx_sa->macsec_rule) {
err = -EINVAL;
goto out;
}
mlx5e_macsec_cleanup_sa_fs(macsec, tx_sa, true, ctx->secy->netdev, 0);
}
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_del_txsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec_device *macsec_device;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_sa *tx_sa;
struct mlx5e_macsec *macsec;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
tx_sa = macsec_device->tx_sa[assoc_num];
if (!tx_sa) {
netdev_err(ctx->netdev, "MACsec offload: TX sa 0x%x doesn't exist\n", assoc_num);
err = -EEXIST;
goto out;
}
mlx5e_macsec_cleanup_sa(macsec, tx_sa, true, ctx->secy->netdev, 0);
mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id);
kfree_rcu_mightsleep(tx_sa);
macsec_device->tx_sa[assoc_num] = NULL;
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_add_rxsc(struct macsec_context *ctx)
{
struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec_rx_sc *rx_sc;
struct list_head *rx_sc_list;
struct mlx5e_macsec *macsec;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
rx_sc_list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(rx_sc_list, ctx_rx_sc->sci);
if (rx_sc) {
netdev_err(ctx->netdev, "MACsec offload: rx_sc (sci %lld) already exists\n",
ctx_rx_sc->sci);
err = -EEXIST;
goto out;
}
rx_sc = kzalloc_obj(*rx_sc);
if (!rx_sc) {
err = -ENOMEM;
goto out;
}
sc_xarray_element = kzalloc_obj(*sc_xarray_element);
if (!sc_xarray_element) {
err = -ENOMEM;
goto destroy_rx_sc;
}
sc_xarray_element->rx_sc = rx_sc;
err = xa_alloc(&macsec->sc_xarray, &sc_xarray_element->fs_id, sc_xarray_element,
XA_LIMIT(1, MLX5_MACEC_RX_FS_ID_MAX), GFP_KERNEL);
if (err) {
if (err == -EBUSY)
netdev_err(ctx->netdev,
"MACsec offload: unable to create entry for RX SC (%d Rx SCs already allocated)\n",
MLX5_MACEC_RX_FS_ID_MAX);
goto destroy_sc_xarray_elemenet;
}
rx_sc->md_dst = metadata_dst_alloc(0, METADATA_MACSEC, GFP_KERNEL);
if (!rx_sc->md_dst) {
err = -ENOMEM;
goto erase_xa_alloc;
}
rx_sc->sci = ctx_rx_sc->sci;
rx_sc->active = ctx_rx_sc->active;
list_add_rcu(&rx_sc->rx_sc_list_element, rx_sc_list);
rx_sc->sc_xarray_element = sc_xarray_element;
rx_sc->md_dst->u.macsec_info.sci = rx_sc->sci;
mutex_unlock(&macsec->lock);
return 0;
erase_xa_alloc:
xa_erase(&macsec->sc_xarray, sc_xarray_element->fs_id);
destroy_sc_xarray_elemenet:
kfree(sc_xarray_element);
destroy_rx_sc:
kfree(rx_sc);
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_upd_rxsc(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_rx_sc *ctx_rx_sc = ctx->rx_sc;
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec_rx_sc *rx_sc;
struct mlx5e_macsec_sa *rx_sa;
struct mlx5e_macsec *macsec;
struct list_head *list;
int i;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx_rx_sc->sci);
if (!rx_sc) {
err = -EINVAL;
goto out;
}
if (rx_sc->active == ctx_rx_sc->active)
goto out;
rx_sc->active = ctx_rx_sc->active;
for (i = 0; i < MACSEC_NUM_AN; ++i) {
rx_sa = rx_sc->rx_sa[i];
if (!rx_sa)
continue;
err = macsec_rx_sa_active_update(ctx, rx_sa, rx_sa->active && ctx_rx_sc->active,
&rx_sc->sc_xarray_element->fs_id);
if (err)
goto out;
}
out:
mutex_unlock(&macsec->lock);
return err;
}
static void macsec_del_rxsc_ctx(struct mlx5e_macsec *macsec, struct mlx5e_macsec_rx_sc *rx_sc,
struct net_device *netdev)
{
struct mlx5e_macsec_sa *rx_sa;
int i;
for (i = 0; i < MACSEC_NUM_AN; ++i) {
rx_sa = rx_sc->rx_sa[i];
if (!rx_sa)
continue;
mlx5e_macsec_cleanup_sa(macsec, rx_sa, false, netdev,
rx_sc->sc_xarray_element->fs_id);
mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id);
kfree(rx_sa);
rx_sc->rx_sa[i] = NULL;
}
list_del_rcu(&rx_sc->rx_sc_list_element);
xa_erase(&macsec->sc_xarray, rx_sc->sc_xarray_element->fs_id);
metadata_dst_free(rx_sc->md_dst);
kfree(rx_sc->sc_xarray_element);
kfree_rcu_mightsleep(rx_sc);
}
static int mlx5e_macsec_del_rxsc(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec_rx_sc *rx_sc;
struct mlx5e_macsec *macsec;
struct list_head *list;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, ctx->rx_sc->sci);
if (!rx_sc) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld doesn't exist\n",
ctx->sa.rx_sa->sc->sci);
err = -EINVAL;
goto out;
}
macsec_del_rxsc_ctx(macsec, rx_sc, ctx->secy->netdev);
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_add_rxsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
struct mlx5e_macsec_device *macsec_device;
struct mlx5_core_dev *mdev = priv->mdev;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_rx_sc *rx_sc;
sci_t sci = ctx_rx_sa->sc->sci;
struct mlx5e_macsec_sa *rx_sa;
struct mlx5e_macsec *macsec;
struct list_head *list;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
if (!rx_sc) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld doesn't exist\n",
ctx->sa.rx_sa->sc->sci);
err = -EINVAL;
goto out;
}
if (rx_sc->rx_sa[assoc_num]) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld rx_sa %d already exist\n",
sci, assoc_num);
err = -EEXIST;
goto out;
}
rx_sa = kzalloc_obj(*rx_sa);
if (!rx_sa) {
err = -ENOMEM;
goto out;
}
rx_sa->active = ctx_rx_sa->active;
rx_sa->next_pn = ctx_rx_sa->next_pn;
rx_sa->sci = sci;
rx_sa->assoc_num = assoc_num;
if (ctx->secy->xpn)
update_macsec_epn(rx_sa, &ctx_rx_sa->key, &ctx_rx_sa->next_pn_halves,
ctx_rx_sa->ssci);
err = mlx5_create_encryption_key(mdev, ctx->sa.key, ctx->secy->key_len,
MLX5_ACCEL_OBJ_MACSEC_KEY,
&rx_sa->enc_key_id);
if (err)
goto destroy_sa;
rx_sc->rx_sa[assoc_num] = rx_sa;
if (!rx_sa->active)
goto out;
err = mlx5e_macsec_init_sa(ctx, rx_sa, true, false, &rx_sc->sc_xarray_element->fs_id);
if (err)
goto destroy_encryption_key;
goto out;
destroy_encryption_key:
rx_sc->rx_sa[assoc_num] = NULL;
mlx5_destroy_encryption_key(mdev, rx_sa->enc_key_id);
destroy_sa:
kfree(rx_sa);
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_upd_rxsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_rx_sa *ctx_rx_sa = ctx->sa.rx_sa;
struct mlx5e_macsec_device *macsec_device;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_rx_sc *rx_sc;
sci_t sci = ctx_rx_sa->sc->sci;
struct mlx5e_macsec_sa *rx_sa;
struct mlx5e_macsec *macsec;
struct list_head *list;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
if (!rx_sc) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld doesn't exist\n",
ctx->sa.rx_sa->sc->sci);
err = -EINVAL;
goto out;
}
rx_sa = rx_sc->rx_sa[assoc_num];
if (!rx_sa) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld rx_sa %d doesn't exist\n",
sci, assoc_num);
err = -EINVAL;
goto out;
}
if (ctx->sa.update_pn) {
netdev_err(ctx->netdev,
"MACsec offload update RX sa %d PN isn't supported\n",
assoc_num);
err = -EINVAL;
goto out;
}
err = macsec_rx_sa_active_update(ctx, rx_sa, ctx_rx_sa->active,
&rx_sc->sc_xarray_element->fs_id);
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_del_rxsa(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec_device *macsec_device;
sci_t sci = ctx->sa.rx_sa->sc->sci;
struct mlx5e_macsec_rx_sc *rx_sc;
u8 assoc_num = ctx->sa.assoc_num;
struct mlx5e_macsec_sa *rx_sa;
struct mlx5e_macsec *macsec;
struct list_head *list;
int err = 0;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
list = &macsec_device->macsec_rx_sc_list_head;
rx_sc = mlx5e_macsec_get_rx_sc_from_sc_list(list, sci);
if (!rx_sc) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld doesn't exist\n",
ctx->sa.rx_sa->sc->sci);
err = -EINVAL;
goto out;
}
rx_sa = rx_sc->rx_sa[assoc_num];
if (!rx_sa) {
netdev_err(ctx->netdev,
"MACsec offload rx_sc sci %lld rx_sa %d doesn't exist\n",
sci, assoc_num);
err = -EINVAL;
goto out;
}
if (rx_sa->active)
mlx5e_macsec_cleanup_sa(macsec, rx_sa, false, ctx->secy->netdev,
rx_sc->sc_xarray_element->fs_id);
mlx5_destroy_encryption_key(macsec->mdev, rx_sa->enc_key_id);
kfree(rx_sa);
rx_sc->rx_sa[assoc_num] = NULL;
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_add_secy(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct net_device *dev = ctx->secy->netdev;
const struct net_device *netdev = ctx->netdev;
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec *macsec;
int err = 0;
if (!mlx5e_macsec_secy_features_validate(ctx))
return -EINVAL;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
if (mlx5e_macsec_get_macsec_device_context(macsec, ctx)) {
netdev_err(netdev, "MACsec offload: MACsec net_device already exist\n");
goto out;
}
if (macsec->num_of_devices >= MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES) {
netdev_err(netdev, "Currently, only %d MACsec offload devices can be set\n",
MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES);
err = -EBUSY;
goto out;
}
macsec_device = kzalloc_obj(*macsec_device);
if (!macsec_device) {
err = -ENOMEM;
goto out;
}
macsec_device->dev_addr = kmemdup(dev->dev_addr, dev->addr_len, GFP_KERNEL);
if (!macsec_device->dev_addr) {
kfree(macsec_device);
err = -ENOMEM;
goto out;
}
macsec_device->netdev = dev;
INIT_LIST_HEAD_RCU(&macsec_device->macsec_rx_sc_list_head);
list_add_rcu(&macsec_device->macsec_device_list_element, &macsec->macsec_device_list_head);
++macsec->num_of_devices;
out:
mutex_unlock(&macsec->lock);
return err;
}
static int macsec_upd_secy_hw_address(struct macsec_context *ctx,
struct mlx5e_macsec_device *macsec_device)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct net_device *dev = ctx->secy->netdev;
struct mlx5e_macsec *macsec = priv->macsec;
struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
struct mlx5e_macsec_sa *rx_sa;
struct list_head *list;
int i, err = 0;
list = &macsec_device->macsec_rx_sc_list_head;
list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) {
for (i = 0; i < MACSEC_NUM_AN; ++i) {
rx_sa = rx_sc->rx_sa[i];
if (!rx_sa || !rx_sa->macsec_rule)
continue;
mlx5e_macsec_cleanup_sa_fs(macsec, rx_sa, false, ctx->secy->netdev,
rx_sc->sc_xarray_element->fs_id);
}
}
list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element) {
for (i = 0; i < MACSEC_NUM_AN; ++i) {
rx_sa = rx_sc->rx_sa[i];
if (!rx_sa)
continue;
if (rx_sa->active) {
err = mlx5e_macsec_init_sa_fs(ctx, rx_sa, true, false,
&rx_sc->sc_xarray_element->fs_id);
if (err)
goto out;
}
}
}
memcpy(macsec_device->dev_addr, dev->dev_addr, dev->addr_len);
out:
return err;
}
static int mlx5e_macsec_upd_secy(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
const struct macsec_tx_sc *tx_sc = &ctx->secy->tx_sc;
const struct net_device *dev = ctx->secy->netdev;
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec_sa *tx_sa;
struct mlx5e_macsec *macsec;
int i, err = 0;
if (!mlx5e_macsec_secy_features_validate(ctx))
return -EINVAL;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
if (!memcmp(macsec_device->dev_addr, dev->dev_addr, dev->addr_len)) {
err = macsec_upd_secy_hw_address(ctx, macsec_device);
if (err)
goto out;
}
for (i = 0; i < MACSEC_NUM_AN; ++i) {
tx_sa = macsec_device->tx_sa[i];
if (!tx_sa)
continue;
mlx5e_macsec_cleanup_sa_fs(macsec, tx_sa, true, ctx->secy->netdev, 0);
}
for (i = 0; i < MACSEC_NUM_AN; ++i) {
tx_sa = macsec_device->tx_sa[i];
if (!tx_sa)
continue;
if (tx_sa->assoc_num == tx_sc->encoding_sa && tx_sa->active) {
err = mlx5e_macsec_init_sa_fs(ctx, tx_sa, tx_sc->encrypt, true, NULL);
if (err)
goto out;
}
}
out:
mutex_unlock(&macsec->lock);
return err;
}
static int mlx5e_macsec_del_secy(struct macsec_context *ctx)
{
struct mlx5e_priv *priv = macsec_netdev_priv(ctx->netdev);
struct mlx5e_macsec_device *macsec_device;
struct mlx5e_macsec_rx_sc *rx_sc, *tmp;
struct mlx5e_macsec_sa *tx_sa;
struct mlx5e_macsec *macsec;
struct list_head *list;
int err = 0;
int i;
mutex_lock(&priv->macsec->lock);
macsec = priv->macsec;
macsec_device = mlx5e_macsec_get_macsec_device_context(macsec, ctx);
if (!macsec_device) {
netdev_err(ctx->netdev, "MACsec offload: Failed to find device context\n");
err = -EINVAL;
goto out;
}
for (i = 0; i < MACSEC_NUM_AN; ++i) {
tx_sa = macsec_device->tx_sa[i];
if (!tx_sa)
continue;
mlx5e_macsec_cleanup_sa(macsec, tx_sa, true, ctx->secy->netdev, 0);
mlx5_destroy_encryption_key(macsec->mdev, tx_sa->enc_key_id);
kfree(tx_sa);
macsec_device->tx_sa[i] = NULL;
}
list = &macsec_device->macsec_rx_sc_list_head;
list_for_each_entry_safe(rx_sc, tmp, list, rx_sc_list_element)
macsec_del_rxsc_ctx(macsec, rx_sc, ctx->secy->netdev);
kfree(macsec_device->dev_addr);
macsec_device->dev_addr = NULL;
list_del_rcu(&macsec_device->macsec_device_list_element);
--macsec->num_of_devices;
kfree(macsec_device);
out:
mutex_unlock(&macsec->lock);
return err;
}
static void macsec_build_accel_attrs(struct mlx5e_macsec_sa *sa,
struct mlx5_macsec_obj_attrs *attrs)
{
attrs->epn_state.epn_msb = sa->epn_state.epn_msb;
attrs->epn_state.overlap = sa->epn_state.overlap;
}
static void macsec_aso_build_wqe_ctrl_seg(struct mlx5e_macsec_aso *macsec_aso,
struct mlx5_wqe_aso_ctrl_seg *aso_ctrl,
struct mlx5_aso_ctrl_param *param)
{
struct mlx5e_macsec_umr *umr = macsec_aso->umr;
memset(aso_ctrl, 0, sizeof(*aso_ctrl));
aso_ctrl->va_l = cpu_to_be32(umr->dma_addr | ASO_CTRL_READ_EN);
aso_ctrl->va_h = cpu_to_be32((u64)umr->dma_addr >> 32);
aso_ctrl->l_key = cpu_to_be32(umr->mkey);
if (!param)
return;
aso_ctrl->data_mask_mode = param->data_mask_mode << 6;
aso_ctrl->condition_1_0_operand = param->condition_1_operand |
param->condition_0_operand << 4;
aso_ctrl->condition_1_0_offset = param->condition_1_offset |
param->condition_0_offset << 4;
aso_ctrl->data_offset_condition_operand = param->data_offset |
param->condition_operand << 6;
aso_ctrl->condition_0_data = cpu_to_be32(param->condition_0_data);
aso_ctrl->condition_0_mask = cpu_to_be32(param->condition_0_mask);
aso_ctrl->condition_1_data = cpu_to_be32(param->condition_1_data);
aso_ctrl->condition_1_mask = cpu_to_be32(param->condition_1_mask);
aso_ctrl->bitwise_data = cpu_to_be64(param->bitwise_data);
aso_ctrl->data_mask = cpu_to_be64(param->data_mask);
}
static int mlx5e_macsec_modify_obj(struct mlx5_core_dev *mdev, struct mlx5_macsec_obj_attrs *attrs,
u32 macsec_id)
{
u32 in[MLX5_ST_SZ_DW(modify_macsec_obj_in)] = {};
u32 out[MLX5_ST_SZ_DW(query_macsec_obj_out)];
u64 modify_field_select = 0;
void *obj;
int err;
MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_MACSEC);
MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, macsec_id);
err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
if (err) {
mlx5_core_err(mdev, "Query MACsec object failed (Object id %d), err = %d\n",
macsec_id, err);
return err;
}
obj = MLX5_ADDR_OF(query_macsec_obj_out, out, macsec_object);
modify_field_select = MLX5_GET64(macsec_offload_obj, obj, modify_field_select);
if (!(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP) ||
!(modify_field_select & MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB)) {
mlx5_core_dbg(mdev, "MACsec object field is not modifiable (Object id %d)\n",
macsec_id);
return -EOPNOTSUPP;
}
obj = MLX5_ADDR_OF(modify_macsec_obj_in, in, macsec_object);
MLX5_SET64(macsec_offload_obj, obj, modify_field_select,
MLX5_MODIFY_MACSEC_BITMASK_EPN_OVERLAP | MLX5_MODIFY_MACSEC_BITMASK_EPN_MSB);
MLX5_SET(macsec_offload_obj, obj, epn_msb, attrs->epn_state.epn_msb);
MLX5_SET(macsec_offload_obj, obj, epn_overlap, attrs->epn_state.overlap);
MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}
static void macsec_aso_build_ctrl(struct mlx5e_macsec_aso *aso,
struct mlx5_wqe_aso_ctrl_seg *aso_ctrl,
struct mlx5e_macsec_aso_in *in)
{
struct mlx5_aso_ctrl_param param = {};
param.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT;
param.condition_0_operand = MLX5_ASO_ALWAYS_TRUE;
param.condition_1_operand = MLX5_ASO_ALWAYS_TRUE;
if (in->mode == MLX5_MACSEC_EPN) {
param.data_offset = MLX5_MACSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
param.bitwise_data = BIT_ULL(54);
param.data_mask = param.bitwise_data;
}
macsec_aso_build_wqe_ctrl_seg(aso, aso_ctrl, ¶m);
}
static int macsec_aso_set_arm_event(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec,
struct mlx5e_macsec_aso_in *in)
{
struct mlx5e_macsec_aso *aso;
struct mlx5_aso_wqe *aso_wqe;
struct mlx5_aso *maso;
int err;
aso = &macsec->aso;
maso = aso->maso;
mutex_lock(&aso->aso_lock);
aso_wqe = mlx5_aso_get_wqe(maso);
mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id,
MLX5_ACCESS_ASO_OPC_MOD_MACSEC);
macsec_aso_build_ctrl(aso, &aso_wqe->aso_ctrl, in);
mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC,
false, maso, false);
mutex_unlock(&aso->aso_lock);
return err;
}
static int macsec_aso_query(struct mlx5_core_dev *mdev, struct mlx5e_macsec *macsec,
struct mlx5e_macsec_aso_in *in, struct mlx5e_macsec_aso_out *out)
{
struct mlx5e_macsec_aso *aso;
struct mlx5_aso_wqe *aso_wqe;
struct mlx5_aso *maso;
int err;
aso = &macsec->aso;
maso = aso->maso;
mutex_lock(&aso->aso_lock);
aso_wqe = mlx5_aso_get_wqe(maso);
mlx5_aso_build_wqe(maso, MLX5_MACSEC_ASO_DS_CNT, aso_wqe, in->obj_id,
MLX5_ACCESS_ASO_OPC_MOD_MACSEC);
macsec_aso_build_wqe_ctrl_seg(aso, &aso_wqe->aso_ctrl, NULL);
mlx5_aso_post_wqe(maso, false, &aso_wqe->ctrl);
read_poll_timeout(mlx5_aso_poll_cq, err, !err, 10, 10 * USEC_PER_MSEC,
false, maso, false);
if (err)
goto err_out;
if (MLX5_GET(macsec_aso, aso->umr->ctx, epn_event_arm))
out->event_arm |= MLX5E_ASO_EPN_ARM;
out->mode_param = MLX5_GET(macsec_aso, aso->umr->ctx, mode_parameter);
err_out:
mutex_unlock(&aso->aso_lock);
return err;
}
static struct mlx5e_macsec_sa *get_macsec_tx_sa_from_obj_id(const struct mlx5e_macsec *macsec,
const u32 obj_id)
{
const struct list_head *device_list;
struct mlx5e_macsec_sa *macsec_sa;
struct mlx5e_macsec_device *iter;
int i;
device_list = &macsec->macsec_device_list_head;
list_for_each_entry(iter, device_list, macsec_device_list_element) {
for (i = 0; i < MACSEC_NUM_AN; ++i) {
macsec_sa = iter->tx_sa[i];
if (!macsec_sa || !macsec_sa->active)
continue;
if (macsec_sa->macsec_obj_id == obj_id)
return macsec_sa;
}
}
return NULL;
}
static struct mlx5e_macsec_sa *get_macsec_rx_sa_from_obj_id(const struct mlx5e_macsec *macsec,
const u32 obj_id)
{
const struct list_head *device_list, *sc_list;
struct mlx5e_macsec_rx_sc *mlx5e_rx_sc;
struct mlx5e_macsec_sa *macsec_sa;
struct mlx5e_macsec_device *iter;
int i;
device_list = &macsec->macsec_device_list_head;
list_for_each_entry(iter, device_list, macsec_device_list_element) {
sc_list = &iter->macsec_rx_sc_list_head;
list_for_each_entry(mlx5e_rx_sc, sc_list, rx_sc_list_element) {
for (i = 0; i < MACSEC_NUM_AN; ++i) {
macsec_sa = mlx5e_rx_sc->rx_sa[i];
if (!macsec_sa || !macsec_sa->active)
continue;
if (macsec_sa->macsec_obj_id == obj_id)
return macsec_sa;
}
}
}
return NULL;
}
static void macsec_epn_update(struct mlx5e_macsec *macsec, struct mlx5_core_dev *mdev,
struct mlx5e_macsec_sa *sa, u32 obj_id, u32 mode_param)
{
struct mlx5_macsec_obj_attrs attrs = {};
struct mlx5e_macsec_aso_in in = {};
if (mode_param < MLX5_MACSEC_EPN_SCOPE_MID) {
sa->epn_state.epn_msb++;
sa->epn_state.overlap = 0;
} else {
sa->epn_state.overlap = 1;
}
macsec_build_accel_attrs(sa, &attrs);
mlx5e_macsec_modify_obj(mdev, &attrs, obj_id);
in.obj_id = obj_id;
in.mode = MLX5_MACSEC_EPN;
macsec_aso_set_arm_event(mdev, macsec, &in);
}
static void macsec_async_event(struct work_struct *work)
{
struct mlx5e_macsec_async_work *async_work;
struct mlx5e_macsec_aso_out out = {};
struct mlx5e_macsec_aso_in in = {};
struct mlx5e_macsec_sa *macsec_sa;
struct mlx5e_macsec *macsec;
struct mlx5_core_dev *mdev;
u32 obj_id;
async_work = container_of(work, struct mlx5e_macsec_async_work, work);
macsec = async_work->macsec;
mutex_lock(&macsec->lock);
mdev = async_work->mdev;
obj_id = async_work->obj_id;
macsec_sa = get_macsec_tx_sa_from_obj_id(macsec, obj_id);
if (!macsec_sa) {
macsec_sa = get_macsec_rx_sa_from_obj_id(macsec, obj_id);
if (!macsec_sa) {
mlx5_core_dbg(mdev, "MACsec SA is not found (SA object id %d)\n", obj_id);
goto out_async_work;
}
}
in.obj_id = obj_id;
macsec_aso_query(mdev, macsec, &in, &out);
if (macsec_sa->epn_state.epn_enabled && !(out.event_arm & MLX5E_ASO_EPN_ARM))
macsec_epn_update(macsec, mdev, macsec_sa, obj_id, out.mode_param);
out_async_work:
kfree(async_work);
mutex_unlock(&macsec->lock);
}
static int macsec_obj_change_event(struct notifier_block *nb, unsigned long event, void *data)
{
struct mlx5e_macsec *macsec = container_of(nb, struct mlx5e_macsec, nb);
struct mlx5e_macsec_async_work *async_work;
struct mlx5_eqe_obj_change *obj_change;
struct mlx5_eqe *eqe = data;
u16 obj_type;
u32 obj_id;
if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE)
return NOTIFY_DONE;
obj_change = &eqe->data.obj_change;
obj_type = be16_to_cpu(obj_change->obj_type);
obj_id = be32_to_cpu(obj_change->obj_id);
if (obj_type != MLX5_GENERAL_OBJECT_TYPES_MACSEC)
return NOTIFY_DONE;
async_work = kzalloc_obj(*async_work, GFP_ATOMIC);
if (!async_work)
return NOTIFY_DONE;
async_work->macsec = macsec;
async_work->mdev = macsec->mdev;
async_work->obj_id = obj_id;
INIT_WORK(&async_work->work, macsec_async_event);
WARN_ON(!queue_work(macsec->wq, &async_work->work));
return NOTIFY_OK;
}
static int mlx5e_macsec_aso_init(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev)
{
struct mlx5_aso *maso;
int err;
err = mlx5_core_alloc_pd(mdev, &aso->pdn);
if (err) {
mlx5_core_err(mdev,
"MACsec offload: Failed to alloc pd for MACsec ASO, err=%d\n",
err);
return err;
}
maso = mlx5_aso_create(mdev, aso->pdn);
if (IS_ERR(maso)) {
err = PTR_ERR(maso);
goto err_aso;
}
err = mlx5e_macsec_aso_reg_mr(mdev, aso);
if (err)
goto err_aso_reg;
mutex_init(&aso->aso_lock);
aso->maso = maso;
return 0;
err_aso_reg:
mlx5_aso_destroy(maso);
err_aso:
mlx5_core_dealloc_pd(mdev, aso->pdn);
return err;
}
static void mlx5e_macsec_aso_cleanup(struct mlx5e_macsec_aso *aso, struct mlx5_core_dev *mdev)
{
if (!aso)
return;
mlx5e_macsec_aso_dereg_mr(mdev, aso);
mlx5_aso_destroy(aso->maso);
mlx5_core_dealloc_pd(mdev, aso->pdn);
}
static const struct macsec_ops macsec_offload_ops = {
.mdo_add_txsa = mlx5e_macsec_add_txsa,
.mdo_upd_txsa = mlx5e_macsec_upd_txsa,
.mdo_del_txsa = mlx5e_macsec_del_txsa,
.mdo_add_rxsc = mlx5e_macsec_add_rxsc,
.mdo_upd_rxsc = mlx5e_macsec_upd_rxsc,
.mdo_del_rxsc = mlx5e_macsec_del_rxsc,
.mdo_add_rxsa = mlx5e_macsec_add_rxsa,
.mdo_upd_rxsa = mlx5e_macsec_upd_rxsa,
.mdo_del_rxsa = mlx5e_macsec_del_rxsa,
.mdo_add_secy = mlx5e_macsec_add_secy,
.mdo_upd_secy = mlx5e_macsec_upd_secy,
.mdo_del_secy = mlx5e_macsec_del_secy,
.rx_uses_md_dst = true,
};
bool mlx5e_macsec_handle_tx_skb(struct mlx5e_macsec *macsec, struct sk_buff *skb)
{
struct metadata_dst *md_dst = skb_metadata_dst(skb);
u32 fs_id;
fs_id = mlx5_macsec_fs_get_fs_id_from_hashtable(macsec->mdev->macsec_fs,
&md_dst->u.macsec_info.sci);
if (!fs_id)
goto err_out;
return true;
err_out:
dev_kfree_skb_any(skb);
return false;
}
void mlx5e_macsec_tx_build_eseg(struct mlx5e_macsec *macsec,
struct sk_buff *skb,
struct mlx5_wqe_eth_seg *eseg)
{
struct metadata_dst *md_dst = skb_metadata_dst(skb);
u32 fs_id;
fs_id = mlx5_macsec_fs_get_fs_id_from_hashtable(macsec->mdev->macsec_fs,
&md_dst->u.macsec_info.sci);
if (!fs_id)
return;
eseg->flow_table_metadata = cpu_to_be32(MLX5_MACSEC_TX_METADATA(fs_id));
}
void mlx5e_macsec_offload_handle_rx_skb(struct net_device *netdev,
struct sk_buff *skb,
struct mlx5_cqe64 *cqe)
{
struct mlx5e_macsec_rx_sc_xarray_element *sc_xarray_element;
u32 macsec_meta_data = be32_to_cpu(cqe->ft_metadata);
struct mlx5e_priv *priv = macsec_netdev_priv(netdev);
struct mlx5e_macsec_rx_sc *rx_sc;
struct mlx5e_macsec *macsec;
u32 fs_id;
macsec = priv->macsec;
if (!macsec)
return;
fs_id = MLX5_MACSEC_RX_METADAT_HANDLE(macsec_meta_data);
rcu_read_lock();
sc_xarray_element = xa_load(&macsec->sc_xarray, fs_id);
rx_sc = sc_xarray_element->rx_sc;
if (rx_sc) {
dst_hold(&rx_sc->md_dst->dst);
skb_dst_set(skb, &rx_sc->md_dst->dst);
}
rcu_read_unlock();
}
void mlx5e_macsec_build_netdev(struct mlx5e_priv *priv)
{
struct net_device *netdev = priv->netdev;
if (!mlx5e_is_macsec_device(priv->mdev))
return;
mlx5_core_dbg(priv->mdev, "mlx5e: MACsec acceleration enabled\n");
netdev->macsec_ops = &macsec_offload_ops;
netdev->features |= NETIF_F_HW_MACSEC;
netif_keep_dst(netdev);
}
int mlx5e_macsec_init(struct mlx5e_priv *priv)
{
struct mlx5_core_dev *mdev = priv->mdev;
struct mlx5e_macsec *macsec = NULL;
struct mlx5_macsec_fs *macsec_fs;
int err;
if (!mlx5e_is_macsec_device(priv->mdev)) {
mlx5_core_dbg(mdev, "Not a MACsec offload device\n");
return 0;
}
macsec = kzalloc_obj(*macsec);
if (!macsec)
return -ENOMEM;
INIT_LIST_HEAD(&macsec->macsec_device_list_head);
mutex_init(&macsec->lock);
err = mlx5e_macsec_aso_init(&macsec->aso, priv->mdev);
if (err) {
mlx5_core_err(mdev, "MACsec offload: Failed to init aso, err=%d\n", err);
goto err_aso;
}
macsec->wq = alloc_ordered_workqueue("mlx5e_macsec_%s", 0, priv->netdev->name);
if (!macsec->wq) {
err = -ENOMEM;
goto err_wq;
}
xa_init_flags(&macsec->sc_xarray, XA_FLAGS_ALLOC1);
priv->macsec = macsec;
macsec->mdev = mdev;
macsec_fs = mlx5_macsec_fs_init(mdev);
if (!macsec_fs) {
err = -ENOMEM;
goto err_out;
}
mdev->macsec_fs = macsec_fs;
macsec->nb.notifier_call = macsec_obj_change_event;
mlx5_notifier_register(mdev, &macsec->nb);
mlx5_core_dbg(mdev, "MACsec attached to netdevice\n");
return 0;
err_out:
destroy_workqueue(macsec->wq);
err_wq:
mlx5e_macsec_aso_cleanup(&macsec->aso, priv->mdev);
err_aso:
kfree(macsec);
priv->macsec = NULL;
return err;
}
void mlx5e_macsec_cleanup(struct mlx5e_priv *priv)
{
struct mlx5e_macsec *macsec = priv->macsec;
struct mlx5_core_dev *mdev = priv->mdev;
if (!macsec)
return;
mlx5_notifier_unregister(mdev, &macsec->nb);
mlx5_macsec_fs_cleanup(mdev->macsec_fs);
destroy_workqueue(macsec->wq);
mlx5e_macsec_aso_cleanup(&macsec->aso, mdev);
mutex_destroy(&macsec->lock);
kfree(macsec);
}
