// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2019 Mellanox Technologies.

#include <linux/debugfs.h>
#include "en_accel/ktls.h"
#include "en_accel/ktls_txrx.h"
#include "en_accel/ktls_utils.h"

struct mlx5e_dump_wqe {
        struct mlx5_wqe_ctrl_seg ctrl;
        struct mlx5_wqe_data_seg data;
};

#define MLX5E_KTLS_DUMP_WQEBBS \
        (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))

static u8
mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags,
                          unsigned int sync_len)
{
        /* Given the MTU and sync_len, calculates an upper bound for the
         * number of DUMP WQEs needed for the TX resync of a record.
         */
        return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu));
}

u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
{
        u16 num_dumps, stop_room = 0;

        if (!mlx5e_is_ktls_tx(mdev))
                return 0;

        num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);

        stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS);
        stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS);
        stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS);
        stop_room += 1; /* fence nop */

        return stop_room;
}

static void mlx5e_ktls_set_tisc(struct mlx5_core_dev *mdev, void *tisc)
{
        MLX5_SET(tisc, tisc, tls_en, 1);
        MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn);
        MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn);
}

static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
{
        u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};

        mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx));

        return mlx5_core_create_tis(mdev, in, tisn);
}

static int mlx5e_ktls_create_tis_cb(struct mlx5_core_dev *mdev,
                                    struct mlx5_async_ctx *async_ctx,
                                    u32 *out, int outlen,
                                    mlx5_async_cbk_t callback,
                                    struct mlx5_async_work *context)
{
        u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};

        mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx));
        MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);

        return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in),
                                out, outlen, callback, context);
}

static int mlx5e_ktls_destroy_tis_cb(struct mlx5_core_dev *mdev, u32 tisn,
                                     struct mlx5_async_ctx *async_ctx,
                                     u32 *out, int outlen,
                                     mlx5_async_cbk_t callback,
                                     struct mlx5_async_work *context)
{
        u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {};

        MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
        MLX5_SET(destroy_tis_in, in, tisn, tisn);

        return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in),
                                out, outlen, callback, context);
}

struct mlx5e_ktls_offload_context_tx {
        /* fast path */
        u32 expected_seq;
        u32 tisn;
        bool ctx_post_pending;
        /* control / resync */
        struct list_head list_node; /* member of the pool */
        union mlx5e_crypto_info crypto_info;
        struct tls_offload_context_tx *tx_ctx;
        struct mlx5_core_dev *mdev;
        struct mlx5e_tls_sw_stats *sw_stats;
        struct mlx5_crypto_dek *dek;
        u8 create_err : 1;
};

static void
mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
                           struct mlx5e_ktls_offload_context_tx *priv_tx)
{
        struct mlx5e_ktls_offload_context_tx **ctx =
                __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);

        BUILD_BUG_ON(sizeof(priv_tx) > TLS_DRIVER_STATE_SIZE_TX);

        *ctx = priv_tx;
}

static struct mlx5e_ktls_offload_context_tx *
mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
{
        struct mlx5e_ktls_offload_context_tx **ctx =
                __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);

        return *ctx;
}

/* struct for callback API management */
struct mlx5e_async_ctx {
        struct mlx5_async_work context;
        struct mlx5_async_ctx *async_ctx;
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        int err;
        union {
                u32 out_create[MLX5_ST_SZ_DW(create_tis_out)];
                u32 out_destroy[MLX5_ST_SZ_DW(destroy_tis_out)];
        };
};

struct mlx5e_bulk_async_ctx {
        struct mlx5_async_ctx async_ctx;
        DECLARE_FLEX_ARRAY(struct mlx5e_async_ctx, arr);
};

static struct mlx5e_bulk_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n)
{
        struct mlx5e_bulk_async_ctx *bulk_async;
        int sz;
        int i;

        sz = struct_size(bulk_async, arr, n);
        bulk_async = kvzalloc(sz, GFP_KERNEL);
        if (!bulk_async)
                return NULL;

        mlx5_cmd_init_async_ctx(mdev, &bulk_async->async_ctx);

        for (i = 0; i < n; i++)
                bulk_async->arr[i].async_ctx = &bulk_async->async_ctx;

        return bulk_async;
}

static void mlx5e_bulk_async_cleanup(struct mlx5e_bulk_async_ctx *bulk_async)
{
        mlx5_cmd_cleanup_async_ctx(&bulk_async->async_ctx);
        kvfree(bulk_async);
}

static void create_tis_callback(int status, struct mlx5_async_work *context)
{
        struct mlx5e_async_ctx *async =
                container_of(context, struct mlx5e_async_ctx, context);
        struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx;

        if (status) {
                async->err = status;
                priv_tx->create_err = 1;
                return;
        }

        priv_tx->tisn = MLX5_GET(create_tis_out, async->out_create, tisn);
}

static void destroy_tis_callback(int status, struct mlx5_async_work *context)
{
        struct mlx5e_async_ctx *async =
                container_of(context, struct mlx5e_async_ctx, context);
        struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx;

        kfree(priv_tx);
}

static struct mlx5e_ktls_offload_context_tx *
mlx5e_tls_priv_tx_init(struct mlx5_core_dev *mdev, struct mlx5e_tls_sw_stats *sw_stats,
                       struct mlx5e_async_ctx *async)
{
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        int err;

        priv_tx = kzalloc_obj(*priv_tx);
        if (!priv_tx)
                return ERR_PTR(-ENOMEM);

        priv_tx->mdev = mdev;
        priv_tx->sw_stats = sw_stats;

        if (!async) {
                err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn);
                if (err)
                        goto err_out;
        } else {
                async->priv_tx = priv_tx;
                err = mlx5e_ktls_create_tis_cb(mdev, async->async_ctx,
                                               async->out_create, sizeof(async->out_create),
                                               create_tis_callback, &async->context);
                if (err)
                        goto err_out;
        }

        return priv_tx;

err_out:
        kfree(priv_tx);
        return ERR_PTR(err);
}

static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv_tx,
                                      struct mlx5e_async_ctx *async)
{
        if (priv_tx->create_err) {
                kfree(priv_tx);
                return;
        }
        async->priv_tx = priv_tx;
        mlx5e_ktls_destroy_tis_cb(priv_tx->mdev, priv_tx->tisn,
                                  async->async_ctx,
                                  async->out_destroy, sizeof(async->out_destroy),
                                  destroy_tis_callback, &async->context);
}

static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev,
                                           struct list_head *list, int size)
{
        struct mlx5e_ktls_offload_context_tx *obj, *n;
        struct mlx5e_bulk_async_ctx *bulk_async;
        int i;

        bulk_async = mlx5e_bulk_async_init(mdev, size);
        if (!bulk_async)
                return;

        i = 0;
        list_for_each_entry_safe(obj, n, list, list_node) {
                mlx5e_tls_priv_tx_cleanup(obj, &bulk_async->arr[i]);
                i++;
        }

        mlx5e_bulk_async_cleanup(bulk_async);
}

/* Recycling pool API */

#define MLX5E_TLS_TX_POOL_BULK (16)
#define MLX5E_TLS_TX_POOL_HIGH (4 * 1024)
#define MLX5E_TLS_TX_POOL_LOW (MLX5E_TLS_TX_POOL_HIGH / 4)

struct mlx5e_tls_tx_pool {
        struct mlx5_core_dev *mdev;
        struct mlx5e_tls_sw_stats *sw_stats;
        struct mutex lock; /* Protects access to the pool */
        struct list_head list;
        size_t size;

        struct workqueue_struct *wq;
        struct work_struct create_work;
        struct work_struct destroy_work;
};

static void create_work(struct work_struct *work)
{
        struct mlx5e_tls_tx_pool *pool =
                container_of(work, struct mlx5e_tls_tx_pool, create_work);
        struct mlx5e_ktls_offload_context_tx *obj;
        struct mlx5e_bulk_async_ctx *bulk_async;
        LIST_HEAD(local_list);
        int i, j, err = 0;

        bulk_async = mlx5e_bulk_async_init(pool->mdev, MLX5E_TLS_TX_POOL_BULK);
        if (!bulk_async)
                return;

        for (i = 0; i < MLX5E_TLS_TX_POOL_BULK; i++) {
                obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async->arr[i]);
                if (IS_ERR(obj)) {
                        err = PTR_ERR(obj);
                        break;
                }
                list_add(&obj->list_node, &local_list);
        }

        for (j = 0; j < i; j++) {
                struct mlx5e_async_ctx *async = &bulk_async->arr[j];

                if (!err && async->err)
                        err = async->err;
        }
        atomic64_add(i, &pool->sw_stats->tx_tls_pool_alloc);
        mlx5e_bulk_async_cleanup(bulk_async);
        if (err)
                goto err_out;

        mutex_lock(&pool->lock);
        if (pool->size + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH) {
                mutex_unlock(&pool->lock);
                goto err_out;
        }
        list_splice(&local_list, &pool->list);
        pool->size += MLX5E_TLS_TX_POOL_BULK;
        if (pool->size <= MLX5E_TLS_TX_POOL_LOW)
                queue_work(pool->wq, work);
        mutex_unlock(&pool->lock);
        return;

err_out:
        mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, i);
        atomic64_add(i, &pool->sw_stats->tx_tls_pool_free);
}

static void destroy_work(struct work_struct *work)
{
        struct mlx5e_tls_tx_pool *pool =
                container_of(work, struct mlx5e_tls_tx_pool, destroy_work);
        struct mlx5e_ktls_offload_context_tx *obj;
        LIST_HEAD(local_list);
        int i = 0;

        mutex_lock(&pool->lock);
        if (pool->size < MLX5E_TLS_TX_POOL_HIGH) {
                mutex_unlock(&pool->lock);
                return;
        }

        list_for_each_entry(obj, &pool->list, list_node)
                if (++i == MLX5E_TLS_TX_POOL_BULK)
                        break;

        list_cut_position(&local_list, &pool->list, &obj->list_node);
        pool->size -= MLX5E_TLS_TX_POOL_BULK;
        if (pool->size >= MLX5E_TLS_TX_POOL_HIGH)
                queue_work(pool->wq, work);
        mutex_unlock(&pool->lock);

        mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK);
        atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free);
}

static struct mlx5e_tls_tx_pool *mlx5e_tls_tx_pool_init(struct mlx5_core_dev *mdev,
                                                        struct mlx5e_tls_sw_stats *sw_stats)
{
        struct mlx5e_tls_tx_pool *pool;

        BUILD_BUG_ON(MLX5E_TLS_TX_POOL_LOW + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH);

        pool = kvzalloc_obj(*pool);
        if (!pool)
                return NULL;

        pool->wq = create_singlethread_workqueue("mlx5e_tls_tx_pool");
        if (!pool->wq)
                goto err_free;

        INIT_LIST_HEAD(&pool->list);
        mutex_init(&pool->lock);

        INIT_WORK(&pool->create_work, create_work);
        INIT_WORK(&pool->destroy_work, destroy_work);

        pool->mdev = mdev;
        pool->sw_stats = sw_stats;

        return pool;

err_free:
        kvfree(pool);
        return NULL;
}

static void mlx5e_tls_tx_pool_list_cleanup(struct mlx5e_tls_tx_pool *pool)
{
        while (pool->size > MLX5E_TLS_TX_POOL_BULK) {
                struct mlx5e_ktls_offload_context_tx *obj;
                LIST_HEAD(local_list);
                int i = 0;

                list_for_each_entry(obj, &pool->list, list_node)
                        if (++i == MLX5E_TLS_TX_POOL_BULK)
                                break;

                list_cut_position(&local_list, &pool->list, &obj->list_node);
                mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK);
                atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free);
                pool->size -= MLX5E_TLS_TX_POOL_BULK;
        }
        if (pool->size) {
                mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &pool->list, pool->size);
                atomic64_add(pool->size, &pool->sw_stats->tx_tls_pool_free);
        }
}

static void mlx5e_tls_tx_pool_cleanup(struct mlx5e_tls_tx_pool *pool)
{
        mlx5e_tls_tx_pool_list_cleanup(pool);
        destroy_workqueue(pool->wq);
        kvfree(pool);
}

static void pool_push(struct mlx5e_tls_tx_pool *pool, struct mlx5e_ktls_offload_context_tx *obj)
{
        mutex_lock(&pool->lock);
        list_add(&obj->list_node, &pool->list);
        if (++pool->size == MLX5E_TLS_TX_POOL_HIGH)
                queue_work(pool->wq, &pool->destroy_work);
        mutex_unlock(&pool->lock);
}

static struct mlx5e_ktls_offload_context_tx *pool_pop(struct mlx5e_tls_tx_pool *pool)
{
        struct mlx5e_ktls_offload_context_tx *obj;

        mutex_lock(&pool->lock);
        if (unlikely(pool->size == 0)) {
                /* pool is empty:
                 * - trigger the populating work, and
                 * - serve the current context via the regular blocking api.
                 */
                queue_work(pool->wq, &pool->create_work);
                mutex_unlock(&pool->lock);
                obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, NULL);
                if (!IS_ERR(obj))
                        atomic64_inc(&pool->sw_stats->tx_tls_pool_alloc);
                return obj;
        }

        obj = list_first_entry(&pool->list, struct mlx5e_ktls_offload_context_tx,
                               list_node);
        list_del(&obj->list_node);
        if (--pool->size == MLX5E_TLS_TX_POOL_LOW)
                queue_work(pool->wq, &pool->create_work);
        mutex_unlock(&pool->lock);
        return obj;
}

/* End of pool API */

int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
                      struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn)
{
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        struct mlx5e_tls_tx_pool *pool;
        struct tls_context *tls_ctx;
        struct mlx5_crypto_dek *dek;
        struct mlx5e_priv *priv;
        int err;

        tls_ctx = tls_get_ctx(sk);
        priv = netdev_priv(netdev);
        pool = priv->tls->tx_pool;

        priv_tx = pool_pop(pool);
        if (IS_ERR(priv_tx))
                return PTR_ERR(priv_tx);

        switch (crypto_info->cipher_type) {
        case TLS_CIPHER_AES_GCM_128:
                priv_tx->crypto_info.crypto_info_128 =
                        *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
                break;
        case TLS_CIPHER_AES_GCM_256:
                priv_tx->crypto_info.crypto_info_256 =
                        *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
                break;
        default:
                WARN_ONCE(1, "Unsupported cipher type %u\n",
                          crypto_info->cipher_type);
                err = -EOPNOTSUPP;
                goto err_pool_push;
        }

        dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info);
        if (IS_ERR(dek)) {
                err = PTR_ERR(dek);
                goto err_pool_push;
        }

        priv_tx->dek = dek;
        priv_tx->expected_seq = start_offload_tcp_sn;
        priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx);

        mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);

        priv_tx->ctx_post_pending = true;
        atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx);

        return 0;

err_pool_push:
        pool_push(pool, priv_tx);
        return err;
}

void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
{
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        struct mlx5e_tls_tx_pool *pool;
        struct mlx5e_priv *priv;

        priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
        priv = netdev_priv(netdev);
        pool = priv->tls->tx_pool;

        atomic64_inc(&priv_tx->sw_stats->tx_tls_del);
        mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_tx->dek);
        pool_push(pool, priv_tx);
}

static void tx_fill_wi(struct mlx5e_txqsq *sq,
                       u16 pi, u8 num_wqebbs, u32 num_bytes,
                       struct page *page)
{
        struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi];

        *wi = (struct mlx5e_tx_wqe_info) {
                .num_wqebbs = num_wqebbs,
                .num_bytes  = num_bytes,
                .resync_dump_frag_page = page,
        };
}

static bool
mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
{
        bool ret = priv_tx->ctx_post_pending;

        priv_tx->ctx_post_pending = false;

        return ret;
}

static void
post_static_params(struct mlx5e_txqsq *sq,
                   struct mlx5e_ktls_offload_context_tx *priv_tx,
                   bool fence)
{
        struct mlx5e_set_tls_static_params_wqe *wqe;
        u16 pi, num_wqebbs;

        num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
        pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
        wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
        mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
                                       priv_tx->tisn,
                                       mlx5_crypto_dek_get_id(priv_tx->dek),
                                       0, fence, TLS_OFFLOAD_CTX_DIR_TX);
        tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
        sq->pc += num_wqebbs;
}

static void
post_progress_params(struct mlx5e_txqsq *sq,
                     struct mlx5e_ktls_offload_context_tx *priv_tx,
                     bool fence)
{
        struct mlx5e_set_tls_progress_params_wqe *wqe;
        u16 pi, num_wqebbs;

        num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
        pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
        wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
        mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0,
                                         TLS_OFFLOAD_CTX_DIR_TX);
        tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
        sq->pc += num_wqebbs;
}

static void tx_post_fence_nop(struct mlx5e_txqsq *sq)
{
        struct mlx5_wq_cyc *wq = &sq->wq;
        u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);

        tx_fill_wi(sq, pi, 1, 0, NULL);

        mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc);
}

static void
mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq,
                              struct mlx5e_ktls_offload_context_tx *priv_tx,
                              bool skip_static_post, bool fence_first_post)
{
        bool progress_fence = skip_static_post || !fence_first_post;

        if (!skip_static_post)
                post_static_params(sq, priv_tx, fence_first_post);

        post_progress_params(sq, priv_tx, progress_fence);
        tx_post_fence_nop(sq);
}

struct tx_sync_info {
        u64 rcd_sn;
        u32 sync_len;
        int nr_frags;
        skb_frag_t frags[MAX_SKB_FRAGS];
};

enum mlx5e_ktls_sync_retval {
        MLX5E_KTLS_SYNC_DONE,
        MLX5E_KTLS_SYNC_FAIL,
        MLX5E_KTLS_SYNC_SKIP_NO_DATA,
};

static enum mlx5e_ktls_sync_retval
tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
                 u32 tcp_seq, int datalen, struct tx_sync_info *info)
{
        struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx;
        enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE;
        struct tls_record_info *record;
        int remaining, i = 0;
        unsigned long flags;
        bool ends_before;

        spin_lock_irqsave(&tx_ctx->lock, flags);
        record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn);

        if (unlikely(!record)) {
                ret = MLX5E_KTLS_SYNC_FAIL;
                goto out;
        }

        /* There are the following cases:
         * 1. packet ends before start marker: bypass offload.
         * 2. packet starts before start marker and ends after it: drop,
         *    not supported, breaks contract with kernel.
         * 3. packet ends before tls record info starts: drop,
         *    this packet was already acknowledged and its record info
         *    was released.
         */
        ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record));

        if (unlikely(tls_record_is_start_marker(record))) {
                ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL;
                goto out;
        } else if (ends_before) {
                ret = MLX5E_KTLS_SYNC_FAIL;
                goto out;
        }

        info->sync_len = tcp_seq - tls_record_start_seq(record);
        remaining = info->sync_len;
        while (remaining > 0) {
                skb_frag_t *frag = &record->frags[i];

                page_ref_inc(skb_frag_page(frag));
                remaining -= skb_frag_size(frag);
                info->frags[i++] = *frag;
        }
        /* reduce the part which will be sent with the original SKB */
        if (remaining < 0)
                skb_frag_size_add(&info->frags[i - 1], remaining);
        info->nr_frags = i;
out:
        spin_unlock_irqrestore(&tx_ctx->lock, flags);
        return ret;
}

static void
tx_post_resync_params(struct mlx5e_txqsq *sq,
                      struct mlx5e_ktls_offload_context_tx *priv_tx,
                      u64 rcd_sn)
{
        __be64 rn_be = cpu_to_be64(rcd_sn);
        bool skip_static_post;
        u16 rec_seq_sz;
        char *rec_seq;

        switch (priv_tx->crypto_info.crypto_info.cipher_type) {
        case TLS_CIPHER_AES_GCM_128: {
                struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info.crypto_info_128;

                rec_seq = info->rec_seq;
                rec_seq_sz = sizeof(info->rec_seq);
                break;
        }
        case TLS_CIPHER_AES_GCM_256: {
                struct tls12_crypto_info_aes_gcm_256 *info = &priv_tx->crypto_info.crypto_info_256;

                rec_seq = info->rec_seq;
                rec_seq_sz = sizeof(info->rec_seq);
                break;
        }
        default:
                WARN_ONCE(1, "Unsupported cipher type %u\n",
                          priv_tx->crypto_info.crypto_info.cipher_type);
                return;
        }

        skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz);
        if (!skip_static_post)
                memcpy(rec_seq, &rn_be, rec_seq_sz);

        mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true);
}

static int
tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn)
{
        struct mlx5_wqe_ctrl_seg *cseg;
        struct mlx5_wqe_data_seg *dseg;
        struct mlx5e_dump_wqe *wqe;
        dma_addr_t dma_addr = 0;
        u16 ds_cnt;
        int fsz;
        u16 pi;

        BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1);
        pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
        wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi);

        ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;

        cseg = &wqe->ctrl;
        dseg = &wqe->data;

        cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8)  | MLX5_OPCODE_DUMP);
        cseg->qpn_ds           = cpu_to_be32((sq->sqn << 8) | ds_cnt);
        cseg->tis_tir_num      = cpu_to_be32(tisn << 8);

        fsz = skb_frag_size(frag);
        dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
                                    DMA_TO_DEVICE);
        if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
                return -ENOMEM;

        dseg->addr       = cpu_to_be64(dma_addr);
        dseg->lkey       = sq->mkey_be;
        dseg->byte_count = cpu_to_be32(fsz);
        mlx5e_dma_push_netmem(sq, skb_frag_netmem(frag), dma_addr, fsz);

        tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag));
        sq->pc += MLX5E_KTLS_DUMP_WQEBBS;

        return 0;
}

void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
                                           struct mlx5e_tx_wqe_info *wi,
                                           u32 *dma_fifo_cc)
{
        struct mlx5e_sq_stats *stats;
        struct mlx5e_sq_dma *dma;

        dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++);
        stats = sq->stats;

        mlx5e_tx_dma_unmap(sq->pdev, dma);
        page_ref_dec(wi->resync_dump_frag_page);
        stats->tls_dump_packets++;
        stats->tls_dump_bytes += wi->num_bytes;
}

static enum mlx5e_ktls_sync_retval
mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
                         struct mlx5e_txqsq *sq,
                         int datalen,
                         u32 seq)
{
        enum mlx5e_ktls_sync_retval ret;
        struct tx_sync_info info = {};
        int i;

        ret = tx_sync_info_get(priv_tx, seq, datalen, &info);
        if (unlikely(ret != MLX5E_KTLS_SYNC_DONE))
                /* We might get here with ret == FAIL if a retransmission
                 * reaches the driver after the relevant record is acked.
                 * It should be safe to drop the packet in this case
                 */
                return ret;

        tx_post_resync_params(sq, priv_tx, info.rcd_sn);

        for (i = 0; i < info.nr_frags; i++) {
                unsigned int orig_fsz, frag_offset = 0, n = 0;
                skb_frag_t *f = &info.frags[i];

                orig_fsz = skb_frag_size(f);

                do {
                        unsigned int fsz;

                        n++;
                        fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset);
                        skb_frag_size_set(f, fsz);
                        if (tx_post_resync_dump(sq, f, priv_tx->tisn)) {
                                page_ref_add(skb_frag_page(f), n - 1);
                                goto err_out;
                        }

                        skb_frag_off_add(f, fsz);
                        frag_offset += fsz;
                } while (frag_offset < orig_fsz);

                page_ref_add(skb_frag_page(f), n - 1);
        }

        return MLX5E_KTLS_SYNC_DONE;

err_out:
        for (; i < info.nr_frags; i++)
                /* The page_ref_dec() here undoes the page ref obtained in tx_sync_info_get().
                 * Page refs obtained for the DUMP WQEs above (by page_ref_add) will be
                 * released only upon their completions (or in mlx5e_free_txqsq_descs,
                 * if channel closes).
                 */
                page_ref_dec(skb_frag_page(&info.frags[i]));

        return MLX5E_KTLS_SYNC_FAIL;
}

bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq,
                              struct sk_buff *skb,
                              struct mlx5e_accel_tx_tls_state *state)
{
        struct mlx5e_ktls_offload_context_tx *priv_tx;
        struct mlx5e_sq_stats *stats = sq->stats;
        struct net_device *tls_netdev;
        struct tls_context *tls_ctx;
        int datalen;
        u32 seq;

        datalen = skb->len - skb_tcp_all_headers(skb);
        if (!datalen)
                return true;

        mlx5e_tx_mpwqe_ensure_complete(sq);

        tls_ctx = tls_get_ctx(skb->sk);
        tls_netdev = rcu_dereference_bh(tls_ctx->netdev);
        /* Don't WARN on NULL: if tls_device_down is running in parallel,
         * netdev might become NULL, even if tls_is_skb_tx_device_offloaded was
         * true. Rather continue processing this packet.
         */
        if (WARN_ON_ONCE(tls_netdev && tls_netdev != netdev))
                goto err_out;

        priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);

        if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx)))
                mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false);

        seq = ntohl(tcp_hdr(skb)->seq);
        if (unlikely(priv_tx->expected_seq != seq)) {
                enum mlx5e_ktls_sync_retval ret =
                        mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq);

                stats->tls_ooo++;

                switch (ret) {
                case MLX5E_KTLS_SYNC_DONE:
                        break;
                case MLX5E_KTLS_SYNC_SKIP_NO_DATA:
                        stats->tls_skip_no_sync_data++;
                        if (likely(!skb->decrypted))
                                goto out;
                        WARN_ON_ONCE(1);
                        goto err_out;
                case MLX5E_KTLS_SYNC_FAIL:
                        stats->tls_drop_no_sync_data++;
                        goto err_out;
                }
        }

        priv_tx->expected_seq = seq + datalen;

        state->tls_tisn = priv_tx->tisn;

        stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
        stats->tls_encrypted_bytes   += datalen;

out:
        return true;

err_out:
        dev_kfree_skb_any(skb);
        return false;
}

static void mlx5e_tls_tx_debugfs_init(struct mlx5e_tls *tls,
                                      struct dentry *dfs_root)
{
        if (IS_ERR_OR_NULL(dfs_root))
                return;

        tls->debugfs.dfs_tx = debugfs_create_dir("tx", dfs_root);

        debugfs_create_size_t("pool_size", 0400, tls->debugfs.dfs_tx,
                              &tls->tx_pool->size);
}

int mlx5e_ktls_init_tx(struct mlx5e_priv *priv)
{
        struct mlx5_crypto_dek_pool *dek_pool;
        struct mlx5e_tls *tls = priv->tls;
        int err;

        if (!mlx5e_is_ktls_device(priv->mdev))
                return 0;

        /* DEK pool could be used by either or both of TX and RX. But we have to
         * put the creation here to avoid syndrome when doing devlink reload.
         */
        dek_pool = mlx5_crypto_dek_pool_create(priv->mdev, MLX5_ACCEL_OBJ_TLS_KEY);
        if (IS_ERR(dek_pool))
                return PTR_ERR(dek_pool);
        tls->dek_pool = dek_pool;

        if (!mlx5e_is_ktls_tx(priv->mdev))
                return 0;

        priv->tls->tx_pool = mlx5e_tls_tx_pool_init(priv->mdev, &priv->tls->sw_stats);
        if (!priv->tls->tx_pool) {
                err = -ENOMEM;
                goto err_tx_pool_init;
        }

        mlx5e_tls_tx_debugfs_init(tls, tls->debugfs.dfs);

        return 0;

err_tx_pool_init:
        mlx5_crypto_dek_pool_destroy(dek_pool);
        return err;
}

void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv)
{
        if (!mlx5e_is_ktls_tx(priv->mdev))
                goto dek_pool_destroy;

        debugfs_remove_recursive(priv->tls->debugfs.dfs_tx);
        priv->tls->debugfs.dfs_tx = NULL;

        mlx5e_tls_tx_pool_cleanup(priv->tls->tx_pool);
        priv->tls->tx_pool = NULL;

dek_pool_destroy:
        if (mlx5e_is_ktls_device(priv->mdev))
                mlx5_crypto_dek_pool_destroy(priv->tls->dek_pool);
}