blkcg
struct blkcg *blkcg = css_to_blkcg(css);
struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
spin_lock_irq(&blkcg->lock);
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
spin_unlock_irq(&blkcg->lock);
struct blkcg *blkcg = css_to_blkcg(of_css(of));
ret = blkg_conf_prep(blkcg, &blkcg_policy_bfq, &ctx);
static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
struct blkcg *blkcg, gfp_t gfp)
struct bfq_group_data *d = blkcg_to_bfqgd(blkg->blkcg);
bio_associate_blkg_from_css(bio, &blkg->blkcg->css);
&bfqg_to_blkg(bfqd->root_group)->blkcg->css);
serial_nr = bfqg_to_blkg(bfqg)->blkcg->css.serial_nr;
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
blkcg_print_blkgs(sf, blkcg, bfqg_prfill_weight_device,
depth = bfqg_to_blkg(bfqq_group(bfqq))->blkcg->css.cgroup->level + 1;
&bfqg_to_blkg(bfqq_group(bfqq))->blkcg->css, \
struct blkcg *blkcg;
blkcg = css_to_blkcg(css);
strscpy(blkcg->fc_app_id, app_id, app_id_len);
if (!bio->bi_blkg || bio->bi_blkg->blkcg->fc_app_id[0] == '\0')
return bio->bi_blkg->blkcg->fc_app_id;
plhead = per_cpu_ptr(parent->blkcg->lhead, cpu);
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
css_rstat_flush(&blkcg->css);
guard(spinlock_irq)(&blkcg->lock);
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node)
static struct blkcg_gq *blkcg_get_first_blkg(struct blkcg *blkcg)
spin_lock_irq(&blkcg->lock);
if (!hlist_empty(&blkcg->blkg_list)) {
blkg = hlist_entry(blkcg->blkg_list.first, struct blkcg_gq,
spin_unlock_irq(&blkcg->lock);
static void blkcg_destroy_blkgs(struct blkcg *blkcg)
while ((blkg = blkcg_get_first_blkg(blkcg))) {
spin_lock(&blkcg->lock);
spin_unlock(&blkcg->lock);
struct blkcg *blkcg = css_to_blkcg(blkcg_css);
struct blkcg *parent;
if (!refcount_dec_and_test(&blkcg->online_pin))
parent = blkcg_parent(blkcg);
blkcg_destroy_blkgs(blkcg);
blkcg = parent;
} while (blkcg);
struct blkcg *blkcg = css_to_blkcg(css);
list_del(&blkcg->all_blkcgs_node);
if (blkcg->cpd[i])
blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
free_percpu(blkcg->lhead);
kfree(blkcg);
struct blkcg *blkcg;
blkcg = &blkcg_root;
blkcg = kzalloc_obj(*blkcg);
if (!blkcg)
if (init_blkcg_llists(blkcg))
blkcg->cpd[i] = cpd;
cpd->blkcg = blkcg;
spin_lock_init(&blkcg->lock);
refcount_set(&blkcg->online_pin, 1);
INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT);
INIT_HLIST_HEAD(&blkcg->blkg_list);
INIT_LIST_HEAD(&blkcg->cgwb_list);
list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
return &blkcg->css;
if (blkcg->cpd[i])
blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
free_percpu(blkcg->lhead);
if (blkcg != &blkcg_root)
kfree(blkcg);
css_put(&blkg->blkcg->css);
struct blkcg *parent = blkcg_parent(css_to_blkcg(css));
pd = pol->pd_alloc_fn(disk, blkg->blkcg,
pd_prealloc = pol->pd_alloc_fn(disk, blkg->blkcg,
spin_lock(&blkg->blkcg->lock);
spin_unlock(&blkg->blkcg->lock);
struct blkcg *blkcg = blkg->blkcg;
spin_lock(&blkcg->lock);
spin_unlock(&blkcg->lock);
struct blkcg *blkcg = blkg->blkcg;
spin_lock(&blkcg->lock);
spin_unlock(&blkcg->lock);
struct blkcg *blkcg;
list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
if (blkcg->cpd[pol->plid]) {
pol->cpd_free_fn(blkcg->cpd[pol->plid]);
blkcg->cpd[pol->plid] = NULL;
struct blkcg *blkcg;
list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
blkcg->cpd[pol->plid] = cpd;
cpd->blkcg = blkcg;
struct blkcg *blkcg = blkg->blkcg;
struct blkcg *blkcg;
blkcg = css_to_blkcg(blkcg_css());
if (!blkcg)
blkg = blkg_lookup(blkcg, disk->queue);
__blkcg_rstat_flush(blkcg, cpu);
struct blkcg *blkcg = css_to_blkcg(css);
blkg = blkg_lookup(blkcg, q);
blkg = blkg_lookup_create(blkcg, bio->bi_bdev->bd_disk);
struct blkcg *blkcg = bio->bi_blkg->blkcg;
if (!cgroup_parent(blkcg->css.cgroup))
struct llist_head *lhead = this_cpu_ptr(blkcg->lhead);
__css_rstat_updated(&blkcg->css, cpu);
struct blkcg *blkcg;
for (blkcg = css_to_blkcg(blkcg_css()); blkcg;
blkcg = blkcg_parent(blkcg)) {
if (atomic_read(&blkcg->congestion_count)) {
return &bio->bi_blkg->blkcg->css;
static inline struct blkcg *blkcg_parent(struct blkcg *blkcg)
return css_to_blkcg(blkcg->css.parent);
static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
if (!css_tryget_online(&blkcg->css))
blkg->blkcg = blkcg;
pd = pol->pd_alloc_fn(disk, blkcg, gfp_mask);
css_put(&blkcg->css);
static struct blkcg_gq *blkg_create(struct blkcg *blkcg, struct gendisk *disk,
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu);
new_blkg = blkg_alloc(blkcg, disk, GFP_NOWAIT);
if (blkcg_parent(blkcg)) {
blkg->parent = blkg_lookup(blkcg_parent(blkcg), disk->queue);
spin_lock(&blkcg->lock);
ret = radix_tree_insert(&blkcg->blkg_tree, disk->queue->id, blkg);
hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
spin_unlock(&blkcg->lock);
static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
blkg = blkg_lookup(blkcg, q);
if (blkcg != &blkcg_root &&
blkg != rcu_dereference(blkcg->blkg_hint))
rcu_assign_pointer(blkcg->blkg_hint, blkg);
struct blkcg *pos = blkcg;
struct blkcg *parent = blkcg_parent(blkcg);
struct blkcg blkcg_root;
if (pos == blkcg)
struct blkcg *blkcg = blkg->blkcg;
lockdep_assert_held(&blkcg->lock);
radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
rcu_assign_pointer(blkcg->blkg_hint, NULL);
struct blkcg *blkcg = blkg->blkcg;
spin_lock(&blkcg->lock);
spin_unlock(&blkcg->lock);
struct blkcg *blkcg = css_to_blkcg(css);
spin_lock_irq(&blkcg->lock);
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
spin_unlock_irq(&blkcg->lock);
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
static int init_blkcg_llists(struct blkcg *blkcg)
blkg = blkg_lookup(blkcg, q);
struct blkcg *pos = blkcg;
struct blkcg *parent;
parent = blkcg_parent(blkcg);
blkcg->lhead = alloc_percpu_gfp(struct llist_head, GFP_KERNEL);
if (!blkcg->lhead)
if (pos == blkcg)
init_llist_head(per_cpu_ptr(blkcg->lhead, cpu));
static void __blkcg_rstat_flush(struct blkcg *blkcg, int cpu)
struct llist_head *lhead = per_cpu_ptr(blkcg->lhead, cpu);
static inline struct blkcg *css_to_blkcg(struct cgroup_subsys_state *css)
return css ? container_of(css, struct blkcg, css) : NULL;
struct blkcg *blkcg;
extern struct blkcg blkcg_root;
void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg,
if (blkcg == &blkcg_root)
blkg = rcu_dereference_check(blkcg->blkg_hint,
blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
static inline struct blkcg_policy_data *blkcg_to_cpd(struct blkcg *blkcg,
return blkcg ? blkcg->cpd[pol->plid] : NULL;
static inline struct blkcg *cpd_to_blkcg(struct blkcg_policy_data *cpd)
return cpd ? cpd->blkcg : NULL;
css_for_each_descendant_pre((pos_css), &(p_blkg)->blkcg->css) \
css_for_each_descendant_post((pos_css), &(p_blkg)->blkcg->css) \
atomic_inc(&blkg->blkcg->congestion_count);
atomic_dec(&blkg->blkcg->congestion_count);
atomic_inc(&blkg->blkcg->congestion_count);
atomic_dec(&blkg->blkcg->congestion_count);
static inline struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, void *key) { return NULL; }
struct blkcg *blkcg;
struct ioc_cgrp *iocc = blkcg_to_iocc(blkg->blkcg);
pr_cont_cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup);
cgroup_path(iocg_to_blkg(iocg)->blkcg->css.cgroup, \
struct blkcg *blkcg, gfp_t gfp)
int levels = blkcg->css.cgroup->level + 1;
iocg->level = blkg->blkcg->css.cgroup->level;
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
blkcg_print_blkgs(sf, blkcg, ioc_weight_prfill,
struct blkcg *blkcg = css_to_blkcg(of_css(of));
struct ioc_cgrp *iocc = blkcg_to_iocc(blkcg);
spin_lock_irq(&blkcg->lock);
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
spin_unlock_irq(&blkcg->lock);
ret = blkg_conf_prep(blkcg, &blkcg_policy_iocost, &ctx);
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
blkcg_print_blkgs(sf, blkcg, ioc_qos_prfill,
struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
blkcg_print_blkgs(sf, blkcg, ioc_cost_model_prfill,
static struct ioc_cgrp *blkcg_to_iocc(struct blkcg *blkcg)
return container_of(blkcg_to_cpd(blkcg, &blkcg_policy_iocost),
struct blkcg *blkcg = css_to_blkcg(of_css(of));
ret = blkg_conf_prep(blkcg, &blkcg_policy_iolatency, &ctx);
struct blkcg *blkcg, gfp_t gfp)
struct ioprio_blkcg *blkcg;
blkcg = kzalloc_obj(*blkcg, gfp);
if (!blkcg)
blkcg->prio_policy = POLICY_NO_CHANGE;
return &blkcg->cpd;
struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
kfree(blkcg);
struct ioprio_blkcg *blkcg = blkcg_to_ioprio_blkcg(bio->bi_blkg->blkcg);
if (!blkcg || blkcg->prio_policy == POLICY_NO_CHANGE)
if (blkcg->prio_policy == POLICY_PROMOTE_TO_RT ||
blkcg->prio_policy == POLICY_NONE_TO_RT) {
IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
blkcg->prio_policy = ret;
&tg_to_blkg(__tg)->blkcg->css, "throtl " fmt, ##args);\
struct blkcg *blkcg = css_to_blkcg(of_css(of));
ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, &ctx);
struct blkcg *blkcg = css_to_blkcg(of_css(of));
ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, &ctx);
struct blkcg *blkcg, gfp_t gfp)