// SPDX-License-Identifier: GPL-2.0-only
/*
 * RDMA resource limiting controller for cgroups.
 *
 * Used to allow a cgroup hierarchy to stop processes from consuming
 * additional RDMA resources after a certain limit is reached.
 *
 * Copyright (C) 2016 Parav Pandit <pandit.parav@gmail.com>
 */

#include <linux/bitops.h>
#include <linux/limits.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/cgroup.h>
#include <linux/parser.h>
#include <linux/cgroup_rdma.h>

#define RDMACG_MAX_STR "max"

enum rdmacg_limit_tokens {
        RDMACG_HCA_HANDLE_VAL,
        RDMACG_HCA_HANDLE_MAX,
        RDMACG_HCA_OBJECT_VAL,
        RDMACG_HCA_OBJECT_MAX,
        NR_RDMACG_LIMIT_TOKENS,
};

static const match_table_t rdmacg_limit_tokens = {
        { RDMACG_HCA_HANDLE_VAL,        "hca_handle=%d" },
        { RDMACG_HCA_HANDLE_MAX,        "hca_handle=max"        },
        { RDMACG_HCA_OBJECT_VAL,        "hca_object=%d" },
        { RDMACG_HCA_OBJECT_MAX,        "hca_object=max"        },
        { NR_RDMACG_LIMIT_TOKENS,       NULL                    },
};

/*
 * Protects list of resource pools maintained on per cgroup basis
 * and rdma device list.
 */
static DEFINE_MUTEX(rdmacg_mutex);
static LIST_HEAD(rdmacg_devices);

enum rdmacg_file_type {
        RDMACG_RESOURCE_TYPE_MAX,
        RDMACG_RESOURCE_TYPE_STAT,
        RDMACG_RESOURCE_TYPE_PEAK,
};

/*
 * resource table definition as to be seen by the user.
 * Need to add entries to it when more resources are
 * added/defined at IB verb/core layer.
 */
static char const *rdmacg_resource_names[] = {
        [RDMACG_RESOURCE_HCA_HANDLE]    = "hca_handle",
        [RDMACG_RESOURCE_HCA_OBJECT]    = "hca_object",
};

/* resource tracker for each resource of rdma cgroup */
struct rdmacg_resource {
        int max;
        int usage;
        int peak;
};

/*
 * resource pool object which represents per cgroup, per device
 * resources. There are multiple instances of this object per cgroup,
 * therefore it cannot be embedded within rdma_cgroup structure. It
 * is maintained as list.
 */
struct rdmacg_resource_pool {
        struct rdmacg_device    *device;
        struct rdmacg_resource  resources[RDMACG_RESOURCE_MAX];

        struct list_head        cg_node;
        struct list_head        dev_node;

        /* count active user tasks of this pool */
        u64                     usage_sum;
        /* total number counts which are set to max */
        int                     num_max_cnt;

        /* per-resource event counters */
        u64                     events_max[RDMACG_RESOURCE_MAX];
        u64                     events_alloc_fail[RDMACG_RESOURCE_MAX];
        u64                     events_local_max[RDMACG_RESOURCE_MAX];
        u64                     events_local_alloc_fail[RDMACG_RESOURCE_MAX];
};

static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css)
{
        return container_of(css, struct rdma_cgroup, css);
}

static struct rdma_cgroup *parent_rdmacg(struct rdma_cgroup *cg)
{
        return css_rdmacg(cg->css.parent);
}

static inline struct rdma_cgroup *get_current_rdmacg(void)
{
        return css_rdmacg(task_get_css(current, rdma_cgrp_id));
}

static void set_resource_limit(struct rdmacg_resource_pool *rpool,
                               int index, int new_max)
{
        if (new_max == S32_MAX) {
                if (rpool->resources[index].max != S32_MAX)
                        rpool->num_max_cnt++;
        } else {
                if (rpool->resources[index].max == S32_MAX)
                        rpool->num_max_cnt--;
        }
        rpool->resources[index].max = new_max;
}

static void set_all_resource_max_limit(struct rdmacg_resource_pool *rpool)
{
        int i;

        for (i = 0; i < RDMACG_RESOURCE_MAX; i++)
                set_resource_limit(rpool, i, S32_MAX);
}

static void free_cg_rpool_locked(struct rdmacg_resource_pool *rpool)
{
        lockdep_assert_held(&rdmacg_mutex);

        list_del(&rpool->cg_node);
        list_del(&rpool->dev_node);
        kfree(rpool);
}

static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool)
{
        int i;

        /*
         * Keep the rpool alive if any peak value is non-zero,
         * so that rdma.peak persists as a historical high-
         * watermark even after all resources are freed.
         */
        for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
                if (rpool->resources[i].peak ||
                    rpool->events_max[i] ||
                    rpool->events_local_max[i] ||
                    rpool->events_alloc_fail[i] ||
                    rpool->events_local_alloc_fail[i])
                        return true;
        }
        return false;
}

static struct rdmacg_resource_pool *
find_cg_rpool_locked(struct rdma_cgroup *cg,
                     struct rdmacg_device *device)

{
        struct rdmacg_resource_pool *pool;

        lockdep_assert_held(&rdmacg_mutex);

        list_for_each_entry(pool, &cg->rpools, cg_node)
                if (pool->device == device)
                        return pool;

        return NULL;
}

static struct rdmacg_resource_pool *
get_cg_rpool_locked(struct rdma_cgroup *cg, struct rdmacg_device *device)
{
        struct rdmacg_resource_pool *rpool;

        rpool = find_cg_rpool_locked(cg, device);
        if (rpool)
                return rpool;

        rpool = kzalloc_obj(*rpool);
        if (!rpool)
                return ERR_PTR(-ENOMEM);

        rpool->device = device;
        set_all_resource_max_limit(rpool);

        INIT_LIST_HEAD(&rpool->cg_node);
        INIT_LIST_HEAD(&rpool->dev_node);
        list_add_tail(&rpool->cg_node, &cg->rpools);
        list_add_tail(&rpool->dev_node, &device->rpools);
        return rpool;
}

/**
 * uncharge_cg_locked - uncharge resource for rdma cgroup
 * @cg: pointer to cg to uncharge and all parents in hierarchy
 * @device: pointer to rdmacg device
 * @index: index of the resource to uncharge in cg (resource pool)
 *
 * It also frees the resource pool which was created as part of
 * charging operation when there are no resources attached to
 * resource pool.
 */
static void
uncharge_cg_locked(struct rdma_cgroup *cg,
                   struct rdmacg_device *device,
                   enum rdmacg_resource_type index)
{
        struct rdmacg_resource_pool *rpool;

        rpool = find_cg_rpool_locked(cg, device);

        /*
         * rpool cannot be null at this stage. Let kernel operate in case
         * if there a bug in IB stack or rdma controller, instead of crashing
         * the system.
         */
        if (unlikely(!rpool)) {
                pr_warn("Invalid device %p or rdma cgroup %p\n", device, cg);
                return;
        }

        rpool->resources[index].usage--;

        /*
         * A negative count (or overflow) is invalid,
         * it indicates a bug in the rdma controller.
         */
        WARN_ON_ONCE(rpool->resources[index].usage < 0);
        rpool->usage_sum--;
        if (rpool->usage_sum == 0 &&
            rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
                if (!rpool_has_persistent_state(rpool)) {
                        /*
                         * No user of the rpool and all entries are set to max, so
                         * safe to delete this rpool.
                         */
                        free_cg_rpool_locked(rpool);
                }
        }
}

/**
 * rdmacg_event_locked - fire event when resource allocation exceeds limit
 * @cg: requesting cgroup
 * @over_cg: cgroup whose limit was exceeded
 * @device: rdma device
 * @index: resource type index
 *
 * Must be called under rdmacg_mutex. Updates event counters in the
 * resource pools of @cg and @over_cg, propagates hierarchical max
 * events from @over_cg (including itself) upward, and notifies
 * userspace via cgroup_file_notify().
 */
static void rdmacg_event_locked(struct rdma_cgroup *cg,
                                struct rdma_cgroup *over_cg,
                                struct rdmacg_device *device,
                                enum rdmacg_resource_type index)
{
        struct rdmacg_resource_pool *rpool;
        struct rdma_cgroup *p;

        lockdep_assert_held(&rdmacg_mutex);

        /* Increment local alloc_fail in requesting cgroup */
        rpool = find_cg_rpool_locked(cg, device);
        if (rpool) {
                rpool->events_local_alloc_fail[index]++;
                cgroup_file_notify(&cg->events_local_file);
        }

        /* Increment local max in the over-limit cgroup */
        rpool = find_cg_rpool_locked(over_cg, device);
        if (rpool) {
                rpool->events_local_max[index]++;
                cgroup_file_notify(&over_cg->events_local_file);
        }

        /* Propagate hierarchical max events upward */
        for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) {
                rpool = get_cg_rpool_locked(p, device);
                if (!IS_ERR(rpool)) {
                        rpool->events_max[index]++;
                        cgroup_file_notify(&p->events_file);
                }
        }
        /* Propagate hierarchical alloc_fail from requesting cgroup upward */
        for (p = cg; parent_rdmacg(p); p = parent_rdmacg(p)) {
                rpool = get_cg_rpool_locked(p, device);
                if (!IS_ERR(rpool)) {
                        rpool->events_alloc_fail[index]++;
                        cgroup_file_notify(&p->events_file);
                }
        }
}

/**
 * rdmacg_uncharge_hierarchy - hierarchically uncharge rdma resource count
 * @cg: pointer to cg to uncharge and all parents in hierarchy
 * @device: pointer to rdmacg device
 * @stop_cg: while traversing hirerchy, when meet with stop_cg cgroup
 *           stop uncharging
 * @index: index of the resource to uncharge in cg in given resource pool
 */
static void rdmacg_uncharge_hierarchy(struct rdma_cgroup *cg,
                                     struct rdmacg_device *device,
                                     struct rdma_cgroup *stop_cg,
                                     enum rdmacg_resource_type index)
{
        struct rdma_cgroup *p;

        mutex_lock(&rdmacg_mutex);

        for (p = cg; p != stop_cg; p = parent_rdmacg(p))
                uncharge_cg_locked(p, device, index);

        mutex_unlock(&rdmacg_mutex);

        css_put(&cg->css);
}

/**
 * rdmacg_uncharge - hierarchically uncharge rdma resource count
 * @cg: pointer to cg to uncharge and all parents in hierarchy
 * @device: pointer to rdmacg device
 * @index: index of the resource to uncharge in cgroup in given resource pool
 */
void rdmacg_uncharge(struct rdma_cgroup *cg,
                     struct rdmacg_device *device,
                     enum rdmacg_resource_type index)
{
        if (index >= RDMACG_RESOURCE_MAX)
                return;

        rdmacg_uncharge_hierarchy(cg, device, NULL, index);
}
EXPORT_SYMBOL(rdmacg_uncharge);

/**
 * rdmacg_try_charge - hierarchically try to charge the rdma resource
 * @rdmacg: pointer to rdma cgroup which will own this resource
 * @device: pointer to rdmacg device
 * @index: index of the resource to charge in cgroup (resource pool)
 *
 * This function follows charging resource in hierarchical way.
 * It will fail if the charge would cause the new value to exceed the
 * hierarchical limit.
 * Returns 0 if the charge succeeded, otherwise -EAGAIN, -ENOMEM or -EINVAL.
 * Returns pointer to rdmacg for this resource when charging is successful.
 *
 * Charger needs to account resources on two criteria.
 * (a) per cgroup & (b) per device resource usage.
 * Per cgroup resource usage ensures that tasks of cgroup doesn't cross
 * the configured limits. Per device provides granular configuration
 * in multi device usage. It allocates resource pool in the hierarchy
 * for each parent it come across for first resource. Later on resource
 * pool will be available. Therefore it will be much faster thereon
 * to charge/uncharge.
 */
int rdmacg_try_charge(struct rdma_cgroup **rdmacg,
                      struct rdmacg_device *device,
                      enum rdmacg_resource_type index)
{
        struct rdma_cgroup *cg, *p;
        struct rdmacg_resource_pool *rpool;
        s64 new;
        int ret = 0;

        if (index >= RDMACG_RESOURCE_MAX)
                return -EINVAL;

        /*
         * hold on to css, as cgroup can be removed but resource
         * accounting happens on css.
         */
        cg = get_current_rdmacg();

        mutex_lock(&rdmacg_mutex);
        for (p = cg; p; p = parent_rdmacg(p)) {
                rpool = get_cg_rpool_locked(p, device);
                if (IS_ERR(rpool)) {
                        ret = PTR_ERR(rpool);
                        goto err;
                } else {
                        new = (s64)rpool->resources[index].usage + 1;
                        if (new > rpool->resources[index].max) {
                                ret = -EAGAIN;
                                goto err;
                        } else {
                                rpool->resources[index].usage = new;
                                rpool->usage_sum++;
                        }
                }
        }
        /* Update peak only after all charges succeed */
        for (p = cg; p; p = parent_rdmacg(p)) {
                rpool = find_cg_rpool_locked(p, device);
                if (rpool && rpool->resources[index].usage > rpool->resources[index].peak)
                        rpool->resources[index].peak = rpool->resources[index].usage;
        }
        mutex_unlock(&rdmacg_mutex);

        *rdmacg = cg;
        return 0;

err:
        if (ret == -EAGAIN)
                rdmacg_event_locked(cg, p, device, index);
        mutex_unlock(&rdmacg_mutex);
        rdmacg_uncharge_hierarchy(cg, device, p, index);
        return ret;
}
EXPORT_SYMBOL(rdmacg_try_charge);

/**
 * rdmacg_register_device - register rdmacg device to rdma controller.
 * @device: pointer to rdmacg device whose resources need to be accounted.
 *
 * If IB stack wish a device to participate in rdma cgroup resource
 * tracking, it must invoke this API to register with rdma cgroup before
 * any user space application can start using the RDMA resources.
 */
void rdmacg_register_device(struct rdmacg_device *device)
{
        INIT_LIST_HEAD(&device->dev_node);
        INIT_LIST_HEAD(&device->rpools);

        mutex_lock(&rdmacg_mutex);
        list_add_tail(&device->dev_node, &rdmacg_devices);
        mutex_unlock(&rdmacg_mutex);
}
EXPORT_SYMBOL(rdmacg_register_device);

/**
 * rdmacg_unregister_device - unregister rdmacg device from rdma controller.
 * @device: pointer to rdmacg device which was previously registered with rdma
 *          controller using rdmacg_register_device().
 *
 * IB stack must invoke this after all the resources of the IB device
 * are destroyed and after ensuring that no more resources will be created
 * when this API is invoked.
 */
void rdmacg_unregister_device(struct rdmacg_device *device)
{
        struct rdmacg_resource_pool *rpool, *tmp;

        /*
         * Synchronize with any active resource settings,
         * usage query happening via configfs.
         */
        mutex_lock(&rdmacg_mutex);
        list_del_init(&device->dev_node);

        /*
         * Now that this device is off the cgroup list, its safe to free
         * all the rpool resources.
         */
        list_for_each_entry_safe(rpool, tmp, &device->rpools, dev_node)
                free_cg_rpool_locked(rpool);

        mutex_unlock(&rdmacg_mutex);
}
EXPORT_SYMBOL(rdmacg_unregister_device);

static struct rdmacg_device *rdmacg_get_device_locked(const char *name)
{
        struct rdmacg_device *device;

        lockdep_assert_held(&rdmacg_mutex);

        list_for_each_entry(device, &rdmacg_devices, dev_node)
                if (!strcmp(name, device->name))
                        return device;

        return NULL;
}

static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of,
                                       char *buf, size_t nbytes, loff_t off)
{
        struct rdma_cgroup *cg = css_rdmacg(of_css(of));
        const char *dev_name;
        struct rdmacg_resource_pool *rpool;
        struct rdmacg_device *device;
        char *options = strstrip(buf);
        char *p;
        int *new_limits;
        unsigned long enables = 0;
        int i = 0, ret = 0;

        /* extract the device name first */
        dev_name = strsep(&options, " ");
        if (!dev_name) {
                ret = -EINVAL;
                goto err;
        }

        new_limits = kzalloc_objs(int, RDMACG_RESOURCE_MAX);
        if (!new_limits) {
                ret = -ENOMEM;
                goto err;
        }

        /* parse resource limit tokens */
        while ((p = strsep(&options, " \t\n"))) {
                substring_t args[MAX_OPT_ARGS];
                int tok, intval;

                if (!*p)
                        continue;

                tok = match_token(p, rdmacg_limit_tokens, args);
                switch (tok) {
                case RDMACG_HCA_HANDLE_VAL:
                        if (match_int(&args[0], &intval) || intval < 0) {
                                ret = -EINVAL;
                                goto parse_err;
                        }
                        new_limits[RDMACG_RESOURCE_HCA_HANDLE] = intval;
                        enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE);
                        break;
                case RDMACG_HCA_HANDLE_MAX:
                        new_limits[RDMACG_RESOURCE_HCA_HANDLE] = S32_MAX;
                        enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE);
                        break;
                case RDMACG_HCA_OBJECT_VAL:
                        if (match_int(&args[0], &intval) || intval < 0) {
                                ret = -EINVAL;
                                goto parse_err;
                        }
                        new_limits[RDMACG_RESOURCE_HCA_OBJECT] = intval;
                        enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT);
                        break;
                case RDMACG_HCA_OBJECT_MAX:
                        new_limits[RDMACG_RESOURCE_HCA_OBJECT] = S32_MAX;
                        enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT);
                        break;
                default:
                        ret = -EINVAL;
                        goto parse_err;
                }
        }

        /* acquire lock to synchronize with hot plug devices */
        mutex_lock(&rdmacg_mutex);

        device = rdmacg_get_device_locked(dev_name);
        if (!device) {
                ret = -ENODEV;
                goto dev_err;
        }

        rpool = get_cg_rpool_locked(cg, device);
        if (IS_ERR(rpool)) {
                ret = PTR_ERR(rpool);
                goto dev_err;
        }

        /* now set the new limits of the rpool */
        for_each_set_bit(i, &enables, RDMACG_RESOURCE_MAX)
                set_resource_limit(rpool, i, new_limits[i]);

        if (rpool->usage_sum == 0 &&
            rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
                if (!rpool_has_persistent_state(rpool)) {
                        /*
                         * No user of the rpool and all entries are set to max, so
                         * safe to delete this rpool.
                         */
                        free_cg_rpool_locked(rpool);
                }
        }

dev_err:
        mutex_unlock(&rdmacg_mutex);

parse_err:
        kfree(new_limits);

err:
        return ret ?: nbytes;
}

static void print_rpool_values(struct seq_file *sf,
                               struct rdmacg_resource_pool *rpool)
{
        enum rdmacg_file_type sf_type;
        int i;
        u32 value;

        sf_type = seq_cft(sf)->private;

        for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
                seq_puts(sf, rdmacg_resource_names[i]);
                seq_putc(sf, '=');
                if (sf_type == RDMACG_RESOURCE_TYPE_MAX) {
                        if (rpool)
                                value = rpool->resources[i].max;
                        else
                                value = S32_MAX;
                } else if (sf_type == RDMACG_RESOURCE_TYPE_PEAK) {
                        value = rpool ? rpool->resources[i].peak : 0;
                } else {
                        if (rpool)
                                value = rpool->resources[i].usage;
                        else
                                value = 0;
                }

                if (value == S32_MAX)
                        seq_puts(sf, RDMACG_MAX_STR);
                else
                        seq_printf(sf, "%d", value);
                seq_putc(sf, ' ');
        }
}

static int rdmacg_resource_read(struct seq_file *sf, void *v)
{
        struct rdmacg_device *device;
        struct rdmacg_resource_pool *rpool;
        struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));

        mutex_lock(&rdmacg_mutex);

        list_for_each_entry(device, &rdmacg_devices, dev_node) {
                seq_printf(sf, "%s ", device->name);

                rpool = find_cg_rpool_locked(cg, device);
                print_rpool_values(sf, rpool);

                seq_putc(sf, '\n');
        }

        mutex_unlock(&rdmacg_mutex);
        return 0;
}

static int rdmacg_events_show(struct seq_file *sf, void *v)
{
        struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));
        struct rdmacg_resource_pool *rpool;
        struct rdmacg_device *device;
        int i;

        mutex_lock(&rdmacg_mutex);

        list_for_each_entry(device, &rdmacg_devices, dev_node) {
                rpool = find_cg_rpool_locked(cg, device);

                seq_printf(sf, "%s ", device->name);
                for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
                        seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu",
                                   rdmacg_resource_names[i],
                                   rpool ? rpool->events_max[i] : 0ULL,
                                   rdmacg_resource_names[i],
                                   rpool ? rpool->events_alloc_fail[i] : 0ULL);
                        if (i < RDMACG_RESOURCE_MAX - 1)
                                seq_putc(sf, ' ');
                }
                seq_putc(sf, '\n');
        }

        mutex_unlock(&rdmacg_mutex);
        return 0;
}

static int rdmacg_events_local_show(struct seq_file *sf, void *v)
{
        struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));
        struct rdmacg_resource_pool *rpool;
        struct rdmacg_device *device;
        int i;

        mutex_lock(&rdmacg_mutex);

        list_for_each_entry(device, &rdmacg_devices, dev_node) {
                rpool = find_cg_rpool_locked(cg, device);

                seq_printf(sf, "%s ", device->name);
                for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
                        seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu",
                                   rdmacg_resource_names[i],
                                   rpool ? rpool->events_local_max[i] : 0ULL,
                                   rdmacg_resource_names[i],
                                   rpool ? rpool->events_local_alloc_fail[i] : 0ULL);
                        if (i < RDMACG_RESOURCE_MAX - 1)
                                seq_putc(sf, ' ');
                }
                seq_putc(sf, '\n');
        }

        mutex_unlock(&rdmacg_mutex);
        return 0;
}

static struct cftype rdmacg_files[] = {
        {
                .name = "max",
                .write = rdmacg_resource_set_max,
                .seq_show = rdmacg_resource_read,
                .private = RDMACG_RESOURCE_TYPE_MAX,
                .flags = CFTYPE_NOT_ON_ROOT,
        },
        {
                .name = "current",
                .seq_show = rdmacg_resource_read,
                .private = RDMACG_RESOURCE_TYPE_STAT,
                .flags = CFTYPE_NOT_ON_ROOT,
        },
        {
                .name = "peak",
                .seq_show = rdmacg_resource_read,
                .private = RDMACG_RESOURCE_TYPE_PEAK,
                .flags = CFTYPE_NOT_ON_ROOT,
        },
        {
                .name = "events",
                .seq_show = rdmacg_events_show,
                .file_offset = offsetof(struct rdma_cgroup, events_file),
                .flags = CFTYPE_NOT_ON_ROOT,
        },
        {
                .name = "events.local",
                .seq_show = rdmacg_events_local_show,
                .file_offset = offsetof(struct rdma_cgroup, events_local_file),
                .flags = CFTYPE_NOT_ON_ROOT,
        },
        { }     /* terminate */
};

static struct cgroup_subsys_state *
rdmacg_css_alloc(struct cgroup_subsys_state *parent)
{
        struct rdma_cgroup *cg;

        cg = kzalloc_obj(*cg);
        if (!cg)
                return ERR_PTR(-ENOMEM);

        INIT_LIST_HEAD(&cg->rpools);
        return &cg->css;
}

static void rdmacg_css_free(struct cgroup_subsys_state *css)
{
        struct rdma_cgroup *cg = css_rdmacg(css);
        struct rdmacg_resource_pool *rpool, *tmp;

        /* Clean up rpools kept alive by non-zero peak values */
        mutex_lock(&rdmacg_mutex);
        list_for_each_entry_safe(rpool, tmp, &cg->rpools, cg_node)
                free_cg_rpool_locked(rpool);
        mutex_unlock(&rdmacg_mutex);

        kfree(cg);
}

/**
 * rdmacg_css_offline - cgroup css_offline callback
 * @css: css of interest
 *
 * This function is called when @css is about to go away and responsible
 * for shooting down all rdmacg associated with @css. As part of that it
 * marks all the resource pool entries to max value, so that when resources are
 * uncharged, associated resource pool can be freed as well.
 */
static void rdmacg_css_offline(struct cgroup_subsys_state *css)
{
        struct rdma_cgroup *cg = css_rdmacg(css);
        struct rdmacg_resource_pool *rpool;

        mutex_lock(&rdmacg_mutex);

        list_for_each_entry(rpool, &cg->rpools, cg_node)
                set_all_resource_max_limit(rpool);

        mutex_unlock(&rdmacg_mutex);
}

struct cgroup_subsys rdma_cgrp_subsys = {
        .css_alloc      = rdmacg_css_alloc,
        .css_free       = rdmacg_css_free,
        .css_offline    = rdmacg_css_offline,
        .legacy_cftypes = rdmacg_files,
        .dfl_cftypes    = rdmacg_files,
};