// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2024 NVIDIA Corporation & Affiliates */

#include "internal.h"

static u16 hws_bwc_gen_queue_idx(struct mlx5hws_context *ctx)
{
        /* assign random queue */
        return get_random_u8() % mlx5hws_bwc_queues(ctx);
}

static u16
hws_bwc_get_burst_th(struct mlx5hws_context *ctx, u16 queue_id)
{
        return min(ctx->send_queue[queue_id].num_entries / 2,
                   MLX5HWS_BWC_MATCHER_REHASH_BURST_TH);
}

static struct mutex *
hws_bwc_get_queue_lock(struct mlx5hws_context *ctx, u16 idx)
{
        return &ctx->bwc_send_queue_locks[idx];
}

static void hws_bwc_lock_all_queues(struct mlx5hws_context *ctx)
{
        u16 bwc_queues = mlx5hws_bwc_queues(ctx);
        struct mutex *queue_lock; /* Protect the queue */
        int i;

        for (i = 0; i < bwc_queues; i++) {
                queue_lock = hws_bwc_get_queue_lock(ctx, i);
                mutex_lock(queue_lock);
        }
}

static void hws_bwc_unlock_all_queues(struct mlx5hws_context *ctx)
{
        u16 bwc_queues = mlx5hws_bwc_queues(ctx);
        struct mutex *queue_lock; /* Protect the queue */
        int i = bwc_queues;

        while (i--) {
                queue_lock = hws_bwc_get_queue_lock(ctx, i);
                mutex_unlock(queue_lock);
        }
}

static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher,
                                      u32 priority,
                                      u8 size_log_rx, u8 size_log_tx,
                                      struct mlx5hws_matcher_attr *attr)
{
        memset(attr, 0, sizeof(*attr));

        attr->priority = priority;
        attr->optimize_using_rule_idx = 0;
        attr->mode = MLX5HWS_MATCHER_RESOURCE_MODE_RULE;
        attr->optimize_flow_src = MLX5HWS_MATCHER_FLOW_SRC_ANY;
        attr->insert_mode = MLX5HWS_MATCHER_INSERT_BY_HASH;
        attr->distribute_mode = MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH;
        attr->size[MLX5HWS_MATCHER_SIZE_TYPE_RX].rule.num_log = size_log_rx;
        attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX].rule.num_log = size_log_tx;
        attr->resizable = true;
        attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;
}

static int
hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_matcher *matcher = bwc_matcher->matcher;
        int drain_error = 0, move_error = 0, poll_error = 0;
        u16 bwc_queues = mlx5hws_bwc_queues(ctx);
        struct mlx5hws_rule_attr rule_attr;
        struct mlx5hws_bwc_rule *bwc_rule;
        struct mlx5hws_send_engine *queue;
        struct list_head *rules_list;
        u32 pending_rules;
        int i, ret = 0;
        bool drain;

        mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);

        for (i = 0; i < bwc_queues; i++) {
                if (list_empty(&bwc_matcher->rules[i]))
                        continue;

                pending_rules = 0;
                rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i);
                rules_list = &bwc_matcher->rules[i];

                list_for_each_entry(bwc_rule, rules_list, list_node) {
                        ret = mlx5hws_matcher_resize_rule_move(matcher,
                                                               bwc_rule->rule,
                                                               &rule_attr);
                        if (unlikely(ret)) {
                                if (!move_error) {
                                        mlx5hws_err(ctx,
                                                    "Moving BWC rule: move failed (%d), attempting to move rest of the rules\n",
                                                    ret);
                                        move_error = ret;
                                }
                                /* Rule wasn't queued, no need to poll */
                                continue;
                        }

                        pending_rules++;
                        drain = pending_rules >=
                                hws_bwc_get_burst_th(ctx, rule_attr.queue_id);
                        ret = mlx5hws_bwc_queue_poll(ctx,
                                                     rule_attr.queue_id,
                                                     &pending_rules,
                                                     drain);
                        if (unlikely(ret)) {
                                if (ret == -ETIMEDOUT) {
                                        mlx5hws_err(ctx,
                                                    "Moving BWC rule: timeout polling for completions (%d), aborting rehash\n",
                                                    ret);
                                        return ret;
                                }
                                if (!poll_error) {
                                        mlx5hws_err(ctx,
                                                    "Moving BWC rule: polling for completions failed (%d), attempting to move rest of the rules\n",
                                                    ret);
                                        poll_error = ret;
                                }
                        }
                }

                if (pending_rules) {
                        queue = &ctx->send_queue[rule_attr.queue_id];
                        mlx5hws_send_engine_flush_queue(queue);
                        ret = mlx5hws_bwc_queue_poll(ctx,
                                                     rule_attr.queue_id,
                                                     &pending_rules,
                                                     true);
                        if (unlikely(ret)) {
                                if (ret == -ETIMEDOUT) {
                                        mlx5hws_err(ctx,
                                                    "Moving bwc rule: timeout draining completions (%d), aborting rehash\n",
                                                    ret);
                                        return ret;
                                }
                                if (!drain_error) {
                                        mlx5hws_err(ctx,
                                                    "Moving bwc rule: drain failed (%d), attempting to move rest of the rules\n",
                                                    ret);
                                        drain_error = ret;
                                }
                        }
                }
        }

        /* Return the first error that happened */
        if (unlikely(move_error))
                return move_error;
        if (unlikely(poll_error))
                return poll_error;
        if (unlikely(drain_error))
                return drain_error;

        return ret;
}

static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        switch (bwc_matcher->matcher_type) {
        case MLX5HWS_BWC_MATCHER_SIMPLE:
                return hws_bwc_matcher_move_all_simple(bwc_matcher);
        case MLX5HWS_BWC_MATCHER_COMPLEX_FIRST:
                return mlx5hws_bwc_matcher_complex_move_first(bwc_matcher);
        case MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER:
                return mlx5hws_bwc_matcher_complex_move(bwc_matcher);
        default:
                return -EINVAL;
        }
}

static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_matcher_attr matcher_attr = {0};
        struct mlx5hws_matcher *old_matcher;
        struct mlx5hws_matcher *new_matcher;
        int ret;

        hws_bwc_matcher_init_attr(bwc_matcher,
                                  bwc_matcher->priority,
                                  bwc_matcher->rx_size.size_log,
                                  bwc_matcher->tx_size.size_log,
                                  &matcher_attr);

        old_matcher = bwc_matcher->matcher;
        new_matcher = mlx5hws_matcher_create(old_matcher->tbl,
                                             &bwc_matcher->mt, 1,
                                             bwc_matcher->at,
                                             bwc_matcher->num_of_at,
                                             &matcher_attr);
        if (!new_matcher) {
                mlx5hws_err(ctx, "Rehash error: matcher creation failed\n");
                return -ENOMEM;
        }

        ret = mlx5hws_matcher_resize_set_target(old_matcher, new_matcher);
        if (ret) {
                mlx5hws_err(ctx, "Rehash error: failed setting resize target\n");
                return ret;
        }

        ret = hws_bwc_matcher_move_all(bwc_matcher);
        if (ret)
                mlx5hws_err(ctx, "Rehash error: moving rules failed, attempting to remove the old matcher\n");

        /* Error during rehash can't be rolled back.
         * The best option here is to allow the rehash to complete and remove
         * the old matcher - can't leave the matcher in the 'in_resize' state.
         */

        bwc_matcher->matcher = new_matcher;
        mlx5hws_matcher_destroy(old_matcher);

        return ret;
}

int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher,
                                      struct mlx5hws_table *table,
                                      u32 priority,
                                      u8 match_criteria_enable,
                                      struct mlx5hws_match_parameters *mask,
                                      enum mlx5hws_action_type action_types[])
{
        enum mlx5hws_action_type init_action_types[1] = { MLX5HWS_ACTION_TYP_LAST };
        struct mlx5hws_context *ctx = table->ctx;
        u16 bwc_queues = mlx5hws_bwc_queues(ctx);
        struct mlx5hws_matcher_attr attr = {0};
        int i;

        bwc_matcher->rules = kzalloc_objs(*bwc_matcher->rules, bwc_queues);
        if (!bwc_matcher->rules)
                goto err;

        for (i = 0; i < bwc_queues; i++)
                INIT_LIST_HEAD(&bwc_matcher->rules[i]);

        hws_bwc_matcher_init_attr(bwc_matcher,
                                  priority,
                                  bwc_matcher->rx_size.size_log,
                                  bwc_matcher->tx_size.size_log,
                                  &attr);

        bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_SIMPLE;
        bwc_matcher->priority = priority;

        bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM;
        bwc_matcher->at = kzalloc_objs(*bwc_matcher->at,
                                       bwc_matcher->size_of_at_array);
        if (!bwc_matcher->at)
                goto free_bwc_matcher_rules;

        /* create dummy action template */
        bwc_matcher->at[0] =
                mlx5hws_action_template_create(action_types ?
                                               action_types : init_action_types);
        if (!bwc_matcher->at[0]) {
                mlx5hws_err(table->ctx, "BWC matcher: failed creating action template\n");
                goto free_bwc_matcher_at_array;
        }

        bwc_matcher->num_of_at = 1;

        bwc_matcher->mt = mlx5hws_match_template_create(ctx,
                                                        mask->match_buf,
                                                        mask->match_sz,
                                                        match_criteria_enable);
        if (!bwc_matcher->mt) {
                mlx5hws_err(table->ctx, "BWC matcher: failed creating match template\n");
                goto free_at;
        }

        bwc_matcher->matcher = mlx5hws_matcher_create(table,
                                                      &bwc_matcher->mt, 1,
                                                      &bwc_matcher->at[0],
                                                      bwc_matcher->num_of_at,
                                                      &attr);
        if (!bwc_matcher->matcher) {
                mlx5hws_err(table->ctx, "BWC matcher: failed creating HWS matcher\n");
                goto free_mt;
        }

        return 0;

free_mt:
        mlx5hws_match_template_destroy(bwc_matcher->mt);
free_at:
        mlx5hws_action_template_destroy(bwc_matcher->at[0]);
free_bwc_matcher_at_array:
        kfree(bwc_matcher->at);
free_bwc_matcher_rules:
        kfree(bwc_matcher->rules);
err:
        return -EINVAL;
}

static void
hws_bwc_matcher_init_size_rxtx(struct mlx5hws_bwc_matcher_size *size)
{
        size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG;
        atomic_set(&size->num_of_rules, 0);
        atomic_set(&size->rehash_required, false);
}

static void hws_bwc_matcher_init_size(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        hws_bwc_matcher_init_size_rxtx(&bwc_matcher->rx_size);
        hws_bwc_matcher_init_size_rxtx(&bwc_matcher->tx_size);
}

struct mlx5hws_bwc_matcher *
mlx5hws_bwc_matcher_create(struct mlx5hws_table *table,
                           u32 priority,
                           u8 match_criteria_enable,
                           struct mlx5hws_match_parameters *mask)
{
        struct mlx5hws_bwc_matcher *bwc_matcher;
        bool is_complex;
        int ret;

        if (!mlx5hws_context_bwc_supported(table->ctx)) {
                mlx5hws_err(table->ctx,
                            "BWC matcher: context created w/o BWC API compatibility\n");
                return NULL;
        }

        bwc_matcher = kzalloc_obj(*bwc_matcher);
        if (!bwc_matcher)
                return NULL;

        hws_bwc_matcher_init_size(bwc_matcher);

        /* Check if the required match params can be all matched
         * in single STE, otherwise complex matcher is needed.
         */

        is_complex = mlx5hws_bwc_match_params_is_complex(table->ctx, match_criteria_enable, mask);
        if (is_complex)
                ret = mlx5hws_bwc_matcher_create_complex(bwc_matcher,
                                                         table,
                                                         priority,
                                                         match_criteria_enable,
                                                         mask);
        else
                ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher,
                                                        table,
                                                        priority,
                                                        match_criteria_enable,
                                                        mask,
                                                        NULL);
        if (ret)
                goto free_bwc_matcher;

        return bwc_matcher;

free_bwc_matcher:
        kfree(bwc_matcher);

        return NULL;
}

int mlx5hws_bwc_matcher_destroy_simple(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        int i;

        mlx5hws_matcher_destroy(bwc_matcher->matcher);
        bwc_matcher->matcher = NULL;

        for (i = 0; i < bwc_matcher->num_of_at; i++)
                mlx5hws_action_template_destroy(bwc_matcher->at[i]);
        kfree(bwc_matcher->at);

        mlx5hws_match_template_destroy(bwc_matcher->mt);
        kfree(bwc_matcher->rules);

        return 0;
}

int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        u32 rx_rules = atomic_read(&bwc_matcher->rx_size.num_of_rules);
        u32 tx_rules = atomic_read(&bwc_matcher->tx_size.num_of_rules);

        if (rx_rules || tx_rules)
                mlx5hws_err(bwc_matcher->matcher->tbl->ctx,
                            "BWC matcher destroy: matcher still has %u RX and %u TX rules\n",
                            rx_rules, tx_rules);

        if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST)
                mlx5hws_bwc_matcher_destroy_complex(bwc_matcher);
        else
                mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);

        kfree(bwc_matcher);
        return 0;
}

int mlx5hws_bwc_queue_poll(struct mlx5hws_context *ctx,
                           u16 queue_id,
                           u32 *pending_rules,
                           bool drain)
{
        unsigned long timeout = jiffies +
                                secs_to_jiffies(MLX5HWS_BWC_POLLING_TIMEOUT);
        struct mlx5hws_flow_op_result comp[MLX5HWS_BWC_MATCHER_REHASH_BURST_TH];
        u16 burst_th = hws_bwc_get_burst_th(ctx, queue_id);
        bool got_comp = *pending_rules >= burst_th;
        bool queue_full;
        int err = 0;
        int ret;
        int i;

        /* Check if there are any completions at all */
        if (!got_comp && !drain)
                return 0;

        queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]);
        while (queue_full || ((got_comp || drain) && *pending_rules)) {
                ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th);
                if (unlikely(ret < 0)) {
                        mlx5hws_err(ctx, "BWC poll error: polling queue %d returned %d\n",
                                    queue_id, ret);
                        return -EINVAL;
                }

                if (ret) {
                        (*pending_rules) -= ret;
                        for (i = 0; i < ret; i++) {
                                if (unlikely(comp[i].status != MLX5HWS_FLOW_OP_SUCCESS)) {
                                        mlx5hws_err(ctx,
                                                    "BWC poll error: polling queue %d returned completion with error\n",
                                                    queue_id);
                                        err = -EINVAL;
                                }
                        }
                        queue_full = false;
                }

                got_comp = !!ret;

                if (unlikely(!got_comp && time_after(jiffies, timeout))) {
                        mlx5hws_err(ctx, "BWC poll error: polling queue %d - TIMEOUT\n", queue_id);
                        return -ETIMEDOUT;
                }
        }

        return err;
}

void
mlx5hws_bwc_rule_fill_attr(struct mlx5hws_bwc_matcher *bwc_matcher,
                           u16 bwc_queue_idx,
                           u32 flow_source,
                           struct mlx5hws_rule_attr *rule_attr)
{
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;

        /* no use of INSERT_BY_INDEX in bwc rule */
        rule_attr->rule_idx = 0;

        /* notify HW at each rule insertion/deletion */
        rule_attr->burst = 0;

        /* We don't need user data, but the API requires it to exist */
        rule_attr->user_data = (void *)0xFACADE;

        rule_attr->queue_id = mlx5hws_bwc_get_queue_id(ctx, bwc_queue_idx);
        rule_attr->flow_source = flow_source;
}

struct mlx5hws_bwc_rule *
mlx5hws_bwc_rule_alloc(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        struct mlx5hws_bwc_rule *bwc_rule;

        bwc_rule = kzalloc_obj(*bwc_rule);
        if (unlikely(!bwc_rule))
                goto out_err;

        bwc_rule->rule = kzalloc_obj(*bwc_rule->rule);
        if (unlikely(!bwc_rule->rule))
                goto free_rule;

        bwc_rule->bwc_matcher = bwc_matcher;
        return bwc_rule;

free_rule:
        kfree(bwc_rule);
out_err:
        return NULL;
}

void mlx5hws_bwc_rule_free(struct mlx5hws_bwc_rule *bwc_rule)
{
        if (likely(bwc_rule->rule))
                kfree(bwc_rule->rule);
        kfree(bwc_rule);
}

static void hws_bwc_rule_list_add(struct mlx5hws_bwc_rule *bwc_rule, u16 idx)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;

        bwc_rule->bwc_queue_idx = idx;
        list_add(&bwc_rule->list_node, &bwc_matcher->rules[idx]);
}

static void hws_bwc_rule_list_remove(struct mlx5hws_bwc_rule *bwc_rule)
{
        list_del_init(&bwc_rule->list_node);
}

static int
hws_bwc_rule_destroy_hws_async(struct mlx5hws_bwc_rule *bwc_rule,
                               struct mlx5hws_rule_attr *attr)
{
        return mlx5hws_rule_destroy(bwc_rule->rule, attr);
}

static int
hws_bwc_rule_destroy_hws_sync(struct mlx5hws_bwc_rule *bwc_rule,
                              struct mlx5hws_rule_attr *rule_attr)
{
        struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
        u32 expected_completions = 1;
        int ret;

        ret = hws_bwc_rule_destroy_hws_async(bwc_rule, rule_attr);
        if (unlikely(ret))
                return ret;

        ret = mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id,
                                     &expected_completions, true);
        if (unlikely(ret))
                return ret;

        if (unlikely(bwc_rule->rule->status != MLX5HWS_RULE_STATUS_DELETED &&
                     bwc_rule->rule->status != MLX5HWS_RULE_STATUS_DELETING)) {
                mlx5hws_err(ctx, "Failed destroying BWC rule: rule status %d\n",
                            bwc_rule->rule->status);
                return -EINVAL;
        }

        return 0;
}

static void hws_bwc_rule_cnt_dec(struct mlx5hws_bwc_rule *bwc_rule)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;

        if (!bwc_rule->skip_rx)
                atomic_dec(&bwc_matcher->rx_size.num_of_rules);
        if (!bwc_rule->skip_tx)
                atomic_dec(&bwc_matcher->tx_size.num_of_rules);
}

static int
hws_bwc_matcher_rehash_shrink(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        struct mlx5hws_bwc_matcher_size *rx_size = &bwc_matcher->rx_size;
        struct mlx5hws_bwc_matcher_size *tx_size = &bwc_matcher->tx_size;

        /* It is possible that another thread has added a rule.
         * Need to check again if we really need rehash/shrink.
         */
        if (atomic_read(&rx_size->num_of_rules) ||
            atomic_read(&tx_size->num_of_rules))
                return 0;

        /* If the current matcher RX/TX size is already at its initial size. */
        if (rx_size->size_log == MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG &&
            tx_size->size_log == MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG)
                return 0;

        /* Now we've done all the checking - do the shrinking:
         *  - reset match RTC size to the initial size
         *  - create new matcher
         *  - move the rules, which will not do anything as the matcher is empty
         *  - destroy the old matcher
         */

        rx_size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG;
        tx_size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG;

        return hws_bwc_matcher_move(bwc_matcher);
}

static int hws_bwc_rule_cnt_dec_with_shrink(struct mlx5hws_bwc_rule *bwc_rule,
                                            u16 bwc_queue_idx)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mutex *queue_lock; /* Protect the queue */
        int ret;

        hws_bwc_rule_cnt_dec(bwc_rule);

        if (atomic_read(&bwc_matcher->rx_size.num_of_rules) ||
            atomic_read(&bwc_matcher->tx_size.num_of_rules))
                return 0;

        /* Matcher has no more rules - shrink it to save ICM. */

        queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx);
        mutex_unlock(queue_lock);

        hws_bwc_lock_all_queues(ctx);
        ret = hws_bwc_matcher_rehash_shrink(bwc_matcher);
        hws_bwc_unlock_all_queues(ctx);

        mutex_lock(queue_lock);

        if (unlikely(ret))
                mlx5hws_err(ctx,
                            "BWC rule deletion: shrinking empty matcher failed (%d)\n",
                            ret);

        return ret;
}

int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        u16 idx = bwc_rule->bwc_queue_idx;
        struct mlx5hws_rule_attr attr;
        struct mutex *queue_lock; /* Protect the queue */
        int ret;

        mlx5hws_bwc_rule_fill_attr(bwc_matcher, idx, 0, &attr);

        queue_lock = hws_bwc_get_queue_lock(ctx, idx);

        mutex_lock(queue_lock);

        ret = hws_bwc_rule_destroy_hws_sync(bwc_rule, &attr);
        hws_bwc_rule_list_remove(bwc_rule);
        hws_bwc_rule_cnt_dec_with_shrink(bwc_rule, idx);

        mutex_unlock(queue_lock);

        return ret;
}

int mlx5hws_bwc_rule_destroy(struct mlx5hws_bwc_rule *bwc_rule)
{
        bool is_complex = bwc_rule->bwc_matcher->matcher_type ==
                          MLX5HWS_BWC_MATCHER_COMPLEX_FIRST;
        int ret = 0;

        if (is_complex)
                ret = mlx5hws_bwc_rule_destroy_complex(bwc_rule);
        else
                ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);

        mlx5hws_bwc_rule_free(bwc_rule);
        return ret;
}

static int
hws_bwc_rule_create_async(struct mlx5hws_bwc_rule *bwc_rule,
                          u32 *match_param,
                          u8 at_idx,
                          struct mlx5hws_rule_action rule_actions[],
                          struct mlx5hws_rule_attr *rule_attr)
{
        return mlx5hws_rule_create(bwc_rule->bwc_matcher->matcher,
                                   0, /* only one match template supported */
                                   match_param,
                                   at_idx,
                                   rule_actions,
                                   rule_attr,
                                   bwc_rule->rule);
}

static int
hws_bwc_rule_create_sync(struct mlx5hws_bwc_rule *bwc_rule,
                         u32 *match_param,
                         u8 at_idx,
                         struct mlx5hws_rule_action rule_actions[],
                         struct mlx5hws_rule_attr *rule_attr)

{
        struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
        u32 expected_completions = 1;
        int ret;

        ret = hws_bwc_rule_create_async(bwc_rule, match_param,
                                        at_idx, rule_actions,
                                        rule_attr);
        if (unlikely(ret))
                return ret;

        return mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id,
                                      &expected_completions, true);
}

static int
hws_bwc_rule_update_sync(struct mlx5hws_bwc_rule *bwc_rule,
                         u8 at_idx,
                         struct mlx5hws_rule_action rule_actions[],
                         struct mlx5hws_rule_attr *rule_attr)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        u32 expected_completions = 1;
        int ret;

        ret = mlx5hws_rule_action_update(bwc_rule->rule,
                                         at_idx,
                                         rule_actions,
                                         rule_attr);
        if (unlikely(ret))
                return ret;

        ret = mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id,
                                     &expected_completions, true);
        if (unlikely(ret))
                mlx5hws_err(ctx, "Failed updating BWC rule (%d)\n", ret);

        return ret;
}

static bool
hws_bwc_matcher_size_maxed_out(struct mlx5hws_bwc_matcher *bwc_matcher,
                               struct mlx5hws_bwc_matcher_size *size)
{
        struct mlx5hws_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps;

        /* check the match RTC size */
        return (size->size_log + MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH +
                MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP) >
               (caps->ste_alloc_log_max - 1);
}

static bool
hws_bwc_matcher_rehash_size_needed(struct mlx5hws_bwc_matcher *bwc_matcher,
                                   struct mlx5hws_bwc_matcher_size *size,
                                   u32 num_of_rules)
{
        if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size)))
                return false;

        if (unlikely((num_of_rules * 100 / MLX5HWS_BWC_MATCHER_REHASH_PERCENT_TH) >=
                     (1UL << size->size_log)))
                return true;

        return false;
}

static void
hws_bwc_rule_actions_to_action_types(struct mlx5hws_rule_action rule_actions[],
                                     enum mlx5hws_action_type action_types[])
{
        int i = 0;

        for (i = 0;
             rule_actions[i].action && (rule_actions[i].action->type != MLX5HWS_ACTION_TYP_LAST);
             i++) {
                action_types[i] = (enum mlx5hws_action_type)rule_actions[i].action->type;
        }

        action_types[i] = MLX5HWS_ACTION_TYP_LAST;
}

static int
hws_bwc_matcher_extend_at(struct mlx5hws_bwc_matcher *bwc_matcher,
                          struct mlx5hws_rule_action rule_actions[])
{
        enum mlx5hws_action_type action_types[MLX5HWS_BWC_MAX_ACTS];
        void *p;

        if (unlikely(bwc_matcher->num_of_at >= bwc_matcher->size_of_at_array)) {
                if (bwc_matcher->size_of_at_array >= MLX5HWS_MATCHER_MAX_AT)
                        return -ENOMEM;
                bwc_matcher->size_of_at_array *= 2;
                p = krealloc(bwc_matcher->at,
                             bwc_matcher->size_of_at_array *
                                     sizeof(*bwc_matcher->at),
                             __GFP_ZERO | GFP_KERNEL);
                if (!p) {
                        bwc_matcher->size_of_at_array /= 2;
                        return -ENOMEM;
                }

                bwc_matcher->at = p;
        }

        hws_bwc_rule_actions_to_action_types(rule_actions, action_types);

        bwc_matcher->at[bwc_matcher->num_of_at] =
                mlx5hws_action_template_create(action_types);

        if (unlikely(!bwc_matcher->at[bwc_matcher->num_of_at]))
                return -ENOMEM;

        bwc_matcher->num_of_at++;
        return 0;
}

static int
hws_bwc_matcher_extend_size(struct mlx5hws_bwc_matcher *bwc_matcher,
                            struct mlx5hws_bwc_matcher_size *size)
{
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_cmd_query_caps *caps = ctx->caps;

        if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size))) {
                mlx5hws_err(ctx, "Can't resize matcher: depth exceeds limit %d\n",
                            caps->rtc_log_depth_max);
                return -ENOMEM;
        }

        size->size_log = min(size->size_log + MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP,
                             caps->ste_alloc_log_max -
                                     MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH);

        return 0;
}

static int
hws_bwc_matcher_find_at(struct mlx5hws_bwc_matcher *bwc_matcher,
                        struct mlx5hws_rule_action rule_actions[])
{
        enum mlx5hws_action_type *action_type_arr;
        int i, j;

        /* start from index 1 - first action template is a dummy */
        for (i = 1; i < bwc_matcher->num_of_at; i++) {
                j = 0;
                action_type_arr = bwc_matcher->at[i]->action_type_arr;

                while (rule_actions[j].action &&
                       rule_actions[j].action->type != MLX5HWS_ACTION_TYP_LAST) {
                        if (action_type_arr[j] != rule_actions[j].action->type)
                                break;
                        j++;
                }

                if (action_type_arr[j] == MLX5HWS_ACTION_TYP_LAST &&
                    (!rule_actions[j].action ||
                     rule_actions[j].action->type == MLX5HWS_ACTION_TYP_LAST))
                        return i;
        }

        return -1;
}

static int
hws_bwc_matcher_rehash_size(struct mlx5hws_bwc_matcher *bwc_matcher)
{
        bool need_rx_rehash, need_tx_rehash;
        int ret;

        need_rx_rehash = atomic_read(&bwc_matcher->rx_size.rehash_required);
        need_tx_rehash = atomic_read(&bwc_matcher->tx_size.rehash_required);

        /* It is possible that another rule has already performed rehash.
         * Need to check again if we really need rehash.
         */
        if (!need_rx_rehash && !need_tx_rehash)
                return 0;

        /* If the current matcher RX/TX size is already at its max size,
         * it can't be rehashed.
         */
        if (need_rx_rehash &&
            hws_bwc_matcher_size_maxed_out(bwc_matcher,
                                           &bwc_matcher->rx_size)) {
                atomic_set(&bwc_matcher->rx_size.rehash_required, false);
                need_rx_rehash = false;
        }
        if (need_tx_rehash &&
            hws_bwc_matcher_size_maxed_out(bwc_matcher,
                                           &bwc_matcher->tx_size)) {
                atomic_set(&bwc_matcher->tx_size.rehash_required, false);
                need_tx_rehash = false;
        }

        /* If both RX and TX rehash flags are now off, it means that whatever
         * we wanted to rehash is now at its max size - no rehash can be done.
         * Return and try adding the rule again - perhaps there was some change.
         */
        if (!need_rx_rehash && !need_tx_rehash)
                return 0;

        /* Now we're done all the checking - do the rehash:
         *  - extend match RTC size
         *  - create new matcher
         *  - move all the rules to the new matcher
         *  - destroy the old matcher
         */
        atomic_set(&bwc_matcher->rx_size.rehash_required, false);
        atomic_set(&bwc_matcher->tx_size.rehash_required, false);

        if (need_rx_rehash) {
                ret = hws_bwc_matcher_extend_size(bwc_matcher,
                                                  &bwc_matcher->rx_size);
                if (ret)
                        return ret;
        }

        if (need_tx_rehash) {
                ret = hws_bwc_matcher_extend_size(bwc_matcher,
                                                  &bwc_matcher->tx_size);
                if (ret)
                        return ret;
        }

        return hws_bwc_matcher_move(bwc_matcher);
}

static int hws_bwc_rule_get_at_idx(struct mlx5hws_bwc_rule *bwc_rule,
                                   struct mlx5hws_rule_action rule_actions[],
                                   u16 bwc_queue_idx)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mutex *queue_lock; /* Protect the queue */
        int at_idx, ret;

        /* check if rehash needed due to missing action template */
        at_idx = hws_bwc_matcher_find_at(bwc_matcher, rule_actions);
        if (likely(at_idx >= 0))
                return at_idx;

        /* we need to extend BWC matcher action templates array */
        queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx);
        mutex_unlock(queue_lock);
        hws_bwc_lock_all_queues(ctx);

        /* check again - perhaps other thread already did extend_at */
        at_idx = hws_bwc_matcher_find_at(bwc_matcher, rule_actions);
        if (at_idx >= 0)
                goto out;

        ret = hws_bwc_matcher_extend_at(bwc_matcher, rule_actions);
        if (unlikely(ret)) {
                mlx5hws_err(ctx, "BWC rule: failed extending AT (%d)", ret);
                at_idx = -EINVAL;
                goto out;
        }

        /* action templates array was extended, we need the last idx */
        at_idx = bwc_matcher->num_of_at - 1;
        ret = mlx5hws_matcher_attach_at(bwc_matcher->matcher,
                                        bwc_matcher->at[at_idx]);
        if (unlikely(ret)) {
                mlx5hws_err(ctx, "BWC rule: failed attaching new AT (%d)", ret);
                at_idx = -EINVAL;
                goto out;
        }

out:
        hws_bwc_unlock_all_queues(ctx);
        mutex_lock(queue_lock);
        return at_idx;
}

static void hws_bwc_rule_cnt_inc_rxtx(struct mlx5hws_bwc_rule *bwc_rule,
                                      struct mlx5hws_bwc_matcher_size *size)
{
        u32 num_of_rules = atomic_inc_return(&size->num_of_rules);

        if (unlikely(hws_bwc_matcher_rehash_size_needed(bwc_rule->bwc_matcher,
                                                        size, num_of_rules)))
                atomic_set(&size->rehash_required, true);
}

static void hws_bwc_rule_cnt_inc(struct mlx5hws_bwc_rule *bwc_rule)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;

        if (!bwc_rule->skip_rx)
                hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->rx_size);
        if (!bwc_rule->skip_tx)
                hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->tx_size);
}

static int hws_bwc_rule_cnt_inc_with_rehash(struct mlx5hws_bwc_rule *bwc_rule,
                                            u16 bwc_queue_idx)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mutex *queue_lock; /* Protect the queue */
        int ret;

        hws_bwc_rule_cnt_inc(bwc_rule);

        if (!atomic_read(&bwc_matcher->rx_size.rehash_required) &&
            !atomic_read(&bwc_matcher->tx_size.rehash_required))
                return 0;

        queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx);
        mutex_unlock(queue_lock);

        hws_bwc_lock_all_queues(ctx);
        ret = hws_bwc_matcher_rehash_size(bwc_matcher);
        hws_bwc_unlock_all_queues(ctx);

        mutex_lock(queue_lock);

        if (likely(!ret))
                return 0;

        /* Failed to rehash. Print a diagnostic and rollback the counters. */
        mlx5hws_err(ctx,
                    "BWC rule insertion: rehash to sizes [%d, %d] failed (%d)\n",
                    bwc_matcher->rx_size.size_log,
                    bwc_matcher->tx_size.size_log, ret);
        hws_bwc_rule_cnt_dec(bwc_rule);

        return ret;
}

int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule,
                                   u32 *match_param,
                                   struct mlx5hws_rule_action rule_actions[],
                                   u32 flow_source,
                                   u16 bwc_queue_idx)
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_rule_attr rule_attr;
        struct mutex *queue_lock; /* Protect the queue */
        int ret = 0;
        int at_idx;

        mlx5hws_bwc_rule_fill_attr(bwc_matcher, bwc_queue_idx, flow_source, &rule_attr);

        queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx);

        mutex_lock(queue_lock);

        at_idx = hws_bwc_rule_get_at_idx(bwc_rule, rule_actions, bwc_queue_idx);
        if (unlikely(at_idx < 0)) {
                mutex_unlock(queue_lock);
                mlx5hws_err(ctx, "BWC rule create: failed getting AT (%d)",
                            ret);
                return -EINVAL;
        }

        ret = hws_bwc_rule_cnt_inc_with_rehash(bwc_rule, bwc_queue_idx);
        if (unlikely(ret)) {
                mutex_unlock(queue_lock);
                return ret;
        }

        ret = hws_bwc_rule_create_sync(bwc_rule,
                                       match_param,
                                       at_idx,
                                       rule_actions,
                                       &rule_attr);
        if (likely(!ret)) {
                hws_bwc_rule_list_add(bwc_rule, bwc_queue_idx);
                mutex_unlock(queue_lock);
                return 0; /* rule inserted successfully */
        }

        /* Rule insertion could fail due to queue being full, timeout, or
         * matcher in resize. In such cases, no point in trying to rehash.
         */
        if (ret == -EBUSY || ret == -ETIMEDOUT || ret == -EAGAIN) {
                mutex_unlock(queue_lock);
                mlx5hws_err(ctx,
                            "BWC rule insertion failed - %s (%d)\n",
                            ret == -EBUSY ? "queue is full" :
                            ret == -ETIMEDOUT ? "timeout" :
                            ret == -EAGAIN ? "matcher in resize" : "N/A",
                            ret);
                hws_bwc_rule_cnt_dec(bwc_rule);
                return ret;
        }

        /* At this point the rule wasn't added.
         * It could be because there was collision, or some other problem.
         * Try rehash by size and insert rule again - last chance.
         */
        if (!bwc_rule->skip_rx)
                atomic_set(&bwc_matcher->rx_size.rehash_required, true);
        if (!bwc_rule->skip_tx)
                atomic_set(&bwc_matcher->tx_size.rehash_required, true);

        mutex_unlock(queue_lock);

        hws_bwc_lock_all_queues(ctx);
        ret = hws_bwc_matcher_rehash_size(bwc_matcher);
        hws_bwc_unlock_all_queues(ctx);

        if (ret) {
                mlx5hws_err(ctx, "BWC rule insertion: rehash failed (%d)\n", ret);
                hws_bwc_rule_cnt_dec(bwc_rule);
                return ret;
        }

        /* Rehash done, but we still have that pesky rule to add */
        mutex_lock(queue_lock);

        ret = hws_bwc_rule_create_sync(bwc_rule,
                                       match_param,
                                       at_idx,
                                       rule_actions,
                                       &rule_attr);

        if (unlikely(ret)) {
                mutex_unlock(queue_lock);
                mlx5hws_err(ctx, "BWC rule insertion failed (%d)\n", ret);
                hws_bwc_rule_cnt_dec(bwc_rule);
                return ret;
        }

        hws_bwc_rule_list_add(bwc_rule, bwc_queue_idx);
        mutex_unlock(queue_lock);

        return 0;
}

struct mlx5hws_bwc_rule *
mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher,
                        struct mlx5hws_match_parameters *params,
                        u32 flow_source,
                        struct mlx5hws_rule_action rule_actions[])
{
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_bwc_rule *bwc_rule;
        u16 bwc_queue_idx;
        int ret;

        if (unlikely(!mlx5hws_context_bwc_supported(ctx))) {
                mlx5hws_err(ctx, "BWC rule: Context created w/o BWC API compatibility\n");
                return NULL;
        }

        bwc_rule = mlx5hws_bwc_rule_alloc(bwc_matcher);
        if (unlikely(!bwc_rule))
                return NULL;

        bwc_rule->flow_source = flow_source;
        mlx5hws_rule_skip(bwc_matcher->matcher, flow_source,
                          &bwc_rule->skip_rx, &bwc_rule->skip_tx);

        bwc_queue_idx = hws_bwc_gen_queue_idx(ctx);

        if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST)
                ret = mlx5hws_bwc_rule_create_complex(bwc_rule,
                                                      params,
                                                      flow_source,
                                                      rule_actions,
                                                      bwc_queue_idx);
        else
                ret = mlx5hws_bwc_rule_create_simple(bwc_rule,
                                                     params->match_buf,
                                                     rule_actions,
                                                     flow_source,
                                                     bwc_queue_idx);
        if (unlikely(ret)) {
                mlx5hws_bwc_rule_free(bwc_rule);
                return NULL;
        }

        return bwc_rule;
}

static int
hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule,
                           struct mlx5hws_rule_action rule_actions[])
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
        struct mlx5hws_rule_attr rule_attr;
        struct mutex *queue_lock; /* Protect the queue */
        int at_idx, ret;
        u16 idx;

        idx = bwc_rule->bwc_queue_idx;

        mlx5hws_bwc_rule_fill_attr(bwc_matcher, idx, bwc_rule->flow_source,
                                   &rule_attr);
        queue_lock = hws_bwc_get_queue_lock(ctx, idx);

        mutex_lock(queue_lock);

        at_idx = hws_bwc_rule_get_at_idx(bwc_rule, rule_actions, idx);
        if (unlikely(at_idx < 0)) {
                mutex_unlock(queue_lock);
                mlx5hws_err(ctx, "BWC rule update: failed getting AT\n");
                return -EINVAL;
        }

        ret = hws_bwc_rule_update_sync(bwc_rule,
                                       at_idx,
                                       rule_actions,
                                       &rule_attr);
        mutex_unlock(queue_lock);

        if (unlikely(ret))
                mlx5hws_err(ctx, "BWC rule: update failed (%d)\n", ret);

        return ret;
}

int mlx5hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule,
                                   struct mlx5hws_rule_action rule_actions[])
{
        struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
        struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;

        if (unlikely(!mlx5hws_context_bwc_supported(ctx))) {
                mlx5hws_err(ctx, "BWC rule: Context created w/o BWC API compatibility\n");
                return -EINVAL;
        }

        /* For complex rules, the update should happen on the last subrule. */
        while (bwc_rule->next_subrule)
                bwc_rule = bwc_rule->next_subrule;

        return hws_bwc_rule_action_update(bwc_rule, rule_actions);
}