block/kyber-iosched.c

205

unsigned int sched_domain, unsigned int type)

block/kyber-iosched.c

207

unsigned int *buckets = kqd->latency_buckets[sched_domain][type];

block/kyber-iosched.c

208

atomic_t *cpu_buckets = cpu_latency->buckets[sched_domain][type];

block/kyber-iosched.c

220

unsigned int sched_domain, unsigned int type,

block/kyber-iosched.c

223

unsigned int *buckets = kqd->latency_buckets[sched_domain][type];

block/kyber-iosched.c

236

if (!kqd->latency_timeout[sched_domain])

block/kyber-iosched.c

237

kqd->latency_timeout[sched_domain] = max(jiffies + HZ, 1UL);

block/kyber-iosched.c

239

time_is_after_jiffies(kqd->latency_timeout[sched_domain])) {

block/kyber-iosched.c

242

kqd->latency_timeout[sched_domain] = 0;

block/kyber-iosched.c

250

memset(buckets, 0, sizeof(kqd->latency_buckets[sched_domain][type]));

block/kyber-iosched.c

252

trace_kyber_latency(kqd->dev, kyber_domain_names[sched_domain],

block/kyber-iosched.c

260

unsigned int sched_domain, unsigned int depth)

block/kyber-iosched.c

262

depth = clamp(depth, 1U, kyber_depth[sched_domain]);

block/kyber-iosched.c

263

if (depth != kqd->domain_tokens[sched_domain].sb.depth) {

block/kyber-iosched.c

264

sbitmap_queue_resize(&kqd->domain_tokens[sched_domain], depth);

block/kyber-iosched.c

265

trace_kyber_adjust(kqd->dev, kyber_domain_names[sched_domain],

block/kyber-iosched.c

273

unsigned int sched_domain;

block/kyber-iosched.c

282

for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {

block/kyber-iosched.c

283

flush_latency_buckets(kqd, cpu_latency, sched_domain,

block/kyber-iosched.c

285

flush_latency_buckets(kqd, cpu_latency, sched_domain,

block/kyber-iosched.c

295

for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {

block/kyber-iosched.c

298

p90 = calculate_percentile(kqd, sched_domain, KYBER_IO_LATENCY,

block/kyber-iosched.c

309

for (sched_domain = 0; sched_domain < KYBER_OTHER; sched_domain++) {

block/kyber-iosched.c

313

p99 = calculate_percentile(kqd, sched_domain,

block/kyber-iosched.c

325

p99 = kqd->domain_p99[sched_domain];

block/kyber-iosched.c

326

kqd->domain_p99[sched_domain] = -1;

block/kyber-iosched.c

328

kqd->domain_p99[sched_domain] = p99;

block/kyber-iosched.c

343

orig_depth = kqd->domain_tokens[sched_domain].sb.depth;

block/kyber-iosched.c

345

kyber_resize_domain(kqd, sched_domain, depth);

block/kyber-iosched.c

536

unsigned int sched_domain;

block/kyber-iosched.c

541

sched_domain = kyber_sched_domain(rq->cmd_flags);

block/kyber-iosched.c

542

sbitmap_queue_clear(&kqd->domain_tokens[sched_domain], nr,

block/kyber-iosched.c

560

unsigned int sched_domain = kyber_sched_domain(bio->bi_opf);

block/kyber-iosched.c

561

struct list_head *rq_list = &kcq->rq_list[sched_domain];

block/kyber-iosched.c

584

unsigned int sched_domain = kyber_sched_domain(rq->cmd_flags);

block/kyber-iosched.c

586

struct list_head *head = &kcq->rq_list[sched_domain];

block/kyber-iosched.c

594

sbitmap_set_bit(&khd->kcq_map[sched_domain],

block/kyber-iosched.c

608

unsigned int sched_domain, unsigned int type,

block/kyber-iosched.c

622

atomic_inc(&cpu_latency->buckets[sched_domain][type][bucket]);

block/kyber-iosched.c

629

unsigned int sched_domain;

block/kyber-iosched.c

632

sched_domain = kyber_sched_domain(rq->cmd_flags);

block/kyber-iosched.c

633

if (sched_domain == KYBER_OTHER)

block/kyber-iosched.c

637

target = kqd->latency_targets[sched_domain];

block/kyber-iosched.c

638

add_latency_sample(cpu_latency, sched_domain, KYBER_TOTAL_LATENCY,

block/kyber-iosched.c

640

add_latency_sample(cpu_latency, sched_domain, KYBER_IO_LATENCY, target,

block/kyber-iosched.c

649

unsigned int sched_domain;

block/kyber-iosched.c

659

list_splice_tail_init(&kcq->rq_list[flush_data->sched_domain],

block/kyber-iosched.c

668

unsigned int sched_domain,

block/kyber-iosched.c

673

.sched_domain = sched_domain,

block/kyber-iosched.c

677

sbitmap_for_each_set(&khd->kcq_map[sched_domain],

block/kyber-iosched.c

696

unsigned int sched_domain = khd->cur_domain;

block/kyber-iosched.c

697

struct sbitmap_queue *domain_tokens = &kqd->domain_tokens[sched_domain];

block/kyber-iosched.c

698

struct sbq_wait *wait = &khd->domain_wait[sched_domain];

block/kyber-iosched.c

711

&khd->wait_index[sched_domain]);

block/kyber-iosched.c

712

khd->domain_ws[sched_domain] = ws;

block/kyber-iosched.c

730

ws = khd->domain_ws[sched_domain];

include/linux/sched/topology.h

175

static inline struct cpumask *sched_domain_span(struct sched_domain *sd)

include/linux/sched/topology.h

210

struct sched_domain *__percpu *sd;

include/linux/sched/topology.h

93

struct sched_domain __rcu *parent; /* top domain must be null terminated */

include/linux/sched/topology.h

94

struct sched_domain __rcu *child; /* bottom domain must be null terminated */

kernel/sched/core.c

1285

struct sched_domain *sd;

kernel/sched/core.c

3700

struct sched_domain *sd;

kernel/sched/core.c

6507

static bool steal_cookie_task(int cpu, struct sched_domain *sd)

kernel/sched/core.c

6528

struct sched_domain *sd;

kernel/sched/deadline.c

2932

struct sched_domain *sd;

kernel/sched/debug.c

726

static void register_sd(struct sched_domain *sd, struct dentry *parent)

kernel/sched/debug.c

779

struct sched_domain *sd;

kernel/sched/ext/idle.c

233

struct sched_domain *sd;

kernel/sched/ext/idle.c

248

struct sched_domain *sd;

kernel/sched/ext/idle.c

263

struct sched_domain *sd;

kernel/sched/ext/idle.c

282

struct sched_domain *sd;

kernel/sched/fair.c

10259

struct sched_domain *sd;

kernel/sched/fair.c

11341

static void update_cpu_capacity(struct sched_domain *sd, int cpu)

kernel/sched/fair.c

11357

void update_group_capacity(struct sched_domain *sd, int cpu)

kernel/sched/fair.c

11359

struct sched_domain *child = sd->child;

kernel/sched/fair.c

11418

check_cpu_capacity(struct rq *rq, struct sched_domain *sd)

kernel/sched/fair.c

11565

static bool sched_use_asym_prio(struct sched_domain *sd, int cpu)

kernel/sched/fair.c

11576

static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu)

kernel/sched/fair.c

11678

sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)

kernel/sched/fair.c

11827

struct sched_domain *sd_tmp;

kernel/sched/fair.c

12147

static inline void update_sg_wakeup_stats(struct sched_domain *sd,

kernel/sched/fair.c

12263

sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)

kernel/sched/fair.c

12432

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

12931

struct sched_domain __maybe_unused *sd_tmp;

kernel/sched/fair.c

13111

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

13127

struct sched_domain *sd = env->sd;

kernel/sched/fair.c

13226

static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd,

kernel/sched/fair.c

13270

struct sched_domain *sd, enum cpu_idle_type idle,

kernel/sched/fair.c

13274

struct sched_domain *sd_parent = sd->parent;

kernel/sched/fair.c

13564

get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)

kernel/sched/fair.c

13588

update_next_balance(struct sched_domain *sd, unsigned long *next_balance)

kernel/sched/fair.c

13612

struct sched_domain *sd;

kernel/sched/fair.c

13693

static inline void update_newidle_stats(struct sched_domain *sd, unsigned int success)

kernel/sched/fair.c

13729

update_newidle_cost(struct sched_domain *sd, u64 cost, unsigned int success)

kernel/sched/fair.c

13771

struct sched_domain *sd;

kernel/sched/fair.c

13922

struct sched_domain *sd;

kernel/sched/fair.c

14037

struct sched_domain *sd;

kernel/sched/fair.c

14066

struct sched_domain *sd;

kernel/sched/fair.c

14352

struct sched_domain *sd;

kernel/sched/fair.c

1438

struct sched_domain *sd;

kernel/sched/fair.c

1505

struct sched_domain *sd;

kernel/sched/fair.c

1538

struct sched_domain *sd;

kernel/sched/fair.c

1873

struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu));

kernel/sched/fair.c

3264

struct sched_domain *sd;

kernel/sched/fair.c

8232

wake_affine_weight(struct sched_domain *sd, struct task_struct *p,

kernel/sched/fair.c

8274

static int wake_affine(struct sched_domain *sd, struct task_struct *p,

kernel/sched/fair.c

8295

sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int this_cpu);

kernel/sched/fair.c

8357

static inline int sched_balance_find_dst_cpu(struct sched_domain *sd, struct task_struct *p,

kernel/sched/fair.c

8374

struct sched_domain *tmp;

kernel/sched/fair.c

8508

static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)

kernel/sched/fair.c

8533

static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)

kernel/sched/fair.c

8656

select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)

kernel/sched/fair.c

8803

struct sched_domain *sd;

kernel/sched/fair.c

9366

struct sched_domain *sd;

kernel/sched/fair.c

9547

struct sched_domain *tmp, *sd = NULL;

kernel/sched/rt.c

1776

struct sched_domain *sd;

kernel/sched/sched.h

1257

struct sched_domain __rcu *sd;

kernel/sched/sched.h

2161

static inline struct sched_domain *highest_flag_domain(int cpu, int flag)

kernel/sched/sched.h

2163

struct sched_domain *sd, *hsd = NULL;

kernel/sched/sched.h

2182

static inline struct sched_domain *lowest_flag_domain(int cpu, int flag)

kernel/sched/sched.h

2184

struct sched_domain *sd;

kernel/sched/sched.h

2194

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_llc);

kernel/sched/sched.h

2200

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);

kernel/sched/sched.h

2201

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);

kernel/sched/sched.h

2202

DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);

kernel/sched/sched.h

2867

extern void update_group_capacity(struct sched_domain *sd, int cpu);

kernel/sched/stats.c

118

struct sched_domain *sd;

kernel/sched/topology.c

1001

struct sched_domain *sd, *sdp;

kernel/sched/topology.c

1067

static bool sd_in_multi_llcs(struct sched_domain *sd)

kernel/sched/topology.c

1069

struct sched_domain *sdp = sd->parent;

kernel/sched/topology.c

1202

build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask)

kernel/sched/topology.c

1206

struct sched_domain *sibling;

kernel/sched/topology.c

1239

build_group_from_child_sched_domain(struct sched_domain *sd, int cpu)

kernel/sched/topology.c

1262

static void init_overlap_sched_group(struct sched_domain *sd,

kernel/sched/topology.c

1290

static struct sched_domain *

kernel/sched/topology.c

1291

find_descended_sibling(struct sched_domain *sd, struct sched_domain *sibling)

kernel/sched/topology.c

1316

build_overlap_sched_groups(struct sched_domain *sd, int cpu)

kernel/sched/topology.c

1322

struct sched_domain *sibling;

kernel/sched/topology.c

142

static void sched_domain_debug(struct sched_domain *sd, int cpu)

kernel/sched/topology.c

1483

struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);

kernel/sched/topology.c

1484

struct sched_domain *child = sd->child;

kernel/sched/topology.c

1527

build_sched_groups(struct sched_domain *sd, int cpu)

kernel/sched/topology.c

1572

static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)

kernel/sched/topology.c

1616

struct sched_domain *sd;

kernel/sched/topology.c

173

static int sd_degenerate(struct sched_domain *sd)

kernel/sched/topology.c

1800

static void set_domain_attribute(struct sched_domain *sd,

kernel/sched/topology.c

1855

d->sd = alloc_percpu(struct sched_domain *);

kernel/sched/topology.c

1872

struct sched_domain *sd;

kernel/sched/topology.c

191

sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)

kernel/sched/topology.c

1932

static struct sched_domain *

kernel/sched/topology.c

1935

struct sched_domain *child, int cpu)

kernel/sched/topology.c

1938

struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);

kernel/sched/topology.c

1955

*sd = (struct sched_domain){

kernel/sched/topology.c

2671

sdd->sd = alloc_percpu(struct sched_domain *);

kernel/sched/topology.c

2684

struct sched_domain *sd;

kernel/sched/topology.c

2688

sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(),

kernel/sched/topology.c

2727

struct sched_domain *sd;

kernel/sched/topology.c

2786

static struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl,

kernel/sched/topology.c

2788

struct sched_domain *child, int cpu)

kernel/sched/topology.c

2790

struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);

kernel/sched/topology.c

2875

static void adjust_numa_imbalance(struct sched_domain *sd_llc)

kernel/sched/topology.c

2877

struct sched_domain *parent;

kernel/sched/topology.c

2940

init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags)

kernel/sched/topology.c

3004

struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu);

kernel/sched/topology.c

3005

struct sched_domain *sd_asym;

kernel/sched/topology.c

3086

struct sched_domain *sd;

kernel/sched/topology.c

3442

struct sched_domain *sd;

kernel/sched/topology.c

46

static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,

kernel/sched/topology.c

632

static void destroy_sched_domain(struct sched_domain *sd)

kernel/sched/topology.c

651

struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu);

kernel/sched/topology.c

654

struct sched_domain *parent = sd->parent;

kernel/sched/topology.c

660

static void destroy_sched_domains(struct sched_domain *sd)

kernel/sched/topology.c

675

DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);

kernel/sched/topology.c

681

DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);

kernel/sched/topology.c

682

DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);

kernel/sched/topology.c

683

DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);

kernel/sched/topology.c

691

struct sched_domain *sd;

kernel/sched/topology.c

746

cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)

kernel/sched/topology.c

749

struct sched_domain *tmp;

kernel/sched/topology.c

753

struct sched_domain *parent = tmp->parent;

kernel/sched/topology.c

836

struct sched_domain * __percpu *sd;

kernel/sched/topology.c

875

struct sched_domain *sd)

kernel/sched/topology.c

894

struct sched_domain *sd, *top_llc, *parent;