kernel/sched/core.c

10898

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

1264

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

1326

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

2122

init_uclamp_rq(cpu_rq(cpu));

kernel/sched/core.c

2556

rq = cpu_rq(new_cpu);

kernel/sched/core.c

3402

dst_rq = cpu_rq(cpu);

kernel/sched/core.c

3436

src_rq = cpu_rq(arg->src_cpu);

kernel/sched/core.c

3437

dst_rq = cpu_rq(arg->dst_cpu);

kernel/sched/core.c

3645

struct task_struct *old_stop = cpu_rq(cpu)->stop;

kernel/sched/core.c

3675

cpu_rq(cpu)->stop = stop;

kernel/sched/core.c

3936

if (set_nr_if_polling(cpu_rq(cpu)->idle)) {

kernel/sched/core.c

3952

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

3964

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

4050

if (!cpu_rq(cpu)->nr_running)

kernel/sched/core.c

4069

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

415

raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);

kernel/sched/core.c

426

raw_spin_unlock(&cpu_rq(t)->__lock);

kernel/sched/core.c

447

cpu_rq(t)->core_enabled = enabled;

kernel/sched/core.c

449

cpu_rq(cpu)->core->core_forceidle_start = 0;

kernel/sched/core.c

4521

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

460

cpu_rq(cpu)->core_enabled = enabled;

kernel/sched/core.c

470

WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree));

kernel/sched/core.c

5527

sum += cpu_rq(i)->nr_running;

kernel/sched/core.c

5553

return cpu_rq(cpu)->nr_switches;

kernel/sched/core.c

5562

sum += cpu_rq(i)->nr_switches;

kernel/sched/core.c

5576

return atomic_read(&cpu_rq(cpu)->nr_iowait);

kernel/sched/core.c

5765

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

5854

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

6330

rq_i = cpu_rq(i);

kernel/sched/core.c

6358

rq_i = cpu_rq(i);

kernel/sched/core.c

6404

rq_i = cpu_rq(i);

kernel/sched/core.c

6457

struct rq *dst = cpu_rq(this), *src = cpu_rq(that);

kernel/sched/core.c

6569

struct rq *rq = cpu_rq(cpu), *core_rq = NULL;

kernel/sched/core.c

6584

rq = cpu_rq(t);

kernel/sched/core.c

6596

rq = cpu_rq(t);

kernel/sched/core.c

6608

struct rq *rq = cpu_rq(cpu), *core_rq = NULL;

kernel/sched/core.c

6627

core_rq = cpu_rq(t);

kernel/sched/core.c

6651

rq = cpu_rq(t);

kernel/sched/core.c

6658

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

6817

struct rq *target_rq = cpu_rq(target_cpu);

kernel/sched/core.c

7080

rq = cpu_rq(cpu);

kernel/sched/core.c

8257

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8505

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8663

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8701

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8762

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8834

struct rq *rq = cpu_rq(cpu);

kernel/sched/core.c

8971

rq = cpu_rq(i);

kernel/sched/core_sched.c

277

rq_i = cpu_rq(i);

kernel/sched/cpuacct.c

114

raw_spin_rq_lock_irq(cpu_rq(cpu));

kernel/sched/cpuacct.c

131

raw_spin_rq_unlock_irq(cpu_rq(cpu));

kernel/sched/cpuacct.c

150

raw_spin_rq_lock_irq(cpu_rq(cpu));

kernel/sched/cpuacct.c

158

raw_spin_rq_unlock_irq(cpu_rq(cpu));

kernel/sched/cpuacct.c

341

lockdep_assert_rq_held(cpu_rq(cpu));

kernel/sched/cpufreq_schedutil.c

372

if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))

kernel/sched/cpufreq_schedutil.c

395

if (cpu_bw_dl(cpu_rq(sg_cpu->cpu)) > sg_cpu->bw_min)

kernel/sched/cputime.c

1112

struct rq *rq = cpu_rq(cpu);

kernel/sched/cputime.c

1180

rq = cpu_rq(cpu);

kernel/sched/cputime.c

1254

if (atomic_read(&cpu_rq(cpu)->nr_iowait) > 0)

kernel/sched/cputime.c

1278

rq = cpu_rq(cpu);

kernel/sched/deadline.c

125

return &cpu_rq(i)->rd->dl_bw;

kernel/sched/deadline.c

130

struct root_domain *rd = cpu_rq(i)->rd;

kernel/sched/deadline.c

162

return __dl_bw_capacity(cpu_rq(i)->rd->span);

kernel/sched/deadline.c

168

struct root_domain *rd = cpu_rq(cpu)->rd;

kernel/sched/deadline.c

1851

rq = cpu_rq(cpu);

kernel/sched/deadline.c

186

struct rq *rq = cpu_rq(i);

kernel/sched/deadline.c

2615

rq = cpu_rq(cpu);

kernel/sched/deadline.c

2646

dl_task_is_earliest_deadline(p, cpu_rq(target)))

kernel/sched/deadline.c

3063

later_rq = cpu_rq(cpu);

kernel/sched/deadline.c

3236

src_rq = cpu_rq(cpu);

kernel/sched/deadline.c

3439

rq = cpu_rq(cpu);

kernel/sched/deadline.c

3452

dl_se = &cpu_rq(cpu)->fair_server;

kernel/sched/deadline.c

3457

dl_se = &cpu_rq(cpu)->ext_server;

kernel/sched/deadline.c

3467

if (cpu_rq(cpu)->fair_server.dl_server &&

kernel/sched/deadline.c

3468

cpu_rq(cpu)->fair_server.dl_bw_attached)

kernel/sched/deadline.c

3469

dl_bw += cpu_rq(cpu)->fair_server.dl_bw;

kernel/sched/deadline.c

3472

if (cpu_rq(cpu)->ext_server.dl_server &&

kernel/sched/deadline.c

3473

cpu_rq(cpu)->ext_server.dl_bw_attached)

kernel/sched/deadline.c

3474

dl_bw += cpu_rq(cpu)->ext_server.dl_bw;

kernel/sched/deadline.c

3492

cpu_rq(i)->dl.extra_bw = cpu_rq(i)->dl.max_bw;

kernel/sched/deadline.c

3504

dl_clear_root_domain(cpu_rq(cpu)->rd);

kernel/sched/deadline.c

3748

init_dl_rq_bw_ratio(&cpu_rq(cpu)->dl);

kernel/sched/deadline.c

4094

print_dl_rq(m, cpu, &cpu_rq(cpu)->dl);

kernel/sched/deadline.c

676

later_rq = cpu_rq(cpu);

kernel/sched/debug.c

1092

dl_bw = &cpu_rq(cpu)->rd->dl_bw;

kernel/sched/debug.c

1101

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

1459

cpu, latency, cpu_rq(cpu)->ticks_without_resched);

kernel/sched/debug.c

387

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

467

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

476

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

502

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

511

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

534

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

543

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

588

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

597

struct rq *rq = cpu_rq(cpu);

kernel/sched/debug.c

967

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

10201

return cpu_rq(cpu);

kernel/sched/ext/ext.c

10251

return rcu_dereference(cpu_rq(cpu)->curr);

kernel/sched/ext/ext.c

10275

return rcu_dereference(cpu_rq(cpu)->curr);

kernel/sched/ext/ext.c

1745

return &cpu_rq(cpu)->scx.local_dsq;

kernel/sched/ext/ext.c

3158

smp_load_acquire(&cpu_rq(cpu)->scx.kick_sync) != ksyncs[cpu]) {

kernel/sched/ext/ext.c

3164

while (READ_ONCE(cpu_rq(cpu)->scx.kick_sync) == ksyncs[cpu]) {

kernel/sched/ext/ext.c

3504

if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))

kernel/sched/ext/ext.c

4697

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

5260

struct rq *donor_rq = cpu_rq(donor);

kernel/sched/ext/ext.c

5315

if (!task_can_run_on_remote_rq(sch, p, cpu_rq(donee), false))

kernel/sched/ext/ext.c

5330

dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);

kernel/sched/ext/ext.c

5622

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

6156

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

6488

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

7152

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

7242

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

7420

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

8339

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

8384

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

8535

struct rq *rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

8556

cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE;

kernel/sched/ext/ext.c

9223

rq = cpu_rq(smp_processor_id());

kernel/sched/ext/ext.c

9390

struct rq *target_rq = cpu_rq(cpu);

kernel/sched/ext/ext.c

9492

ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);

kernel/sched/ext/ext.c

9996

struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq();

kernel/sched/ext/ext.h

28

return cpu_rq(cpu)->scx.cpuperf_target;

kernel/sched/ext/idle.c

554

cpu_rq(cpu)->scx.local_dsq.nr == 0 &&

kernel/sched/fair.c

10319

if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))

kernel/sched/fair.c

10405

dst_cfs_rq = &cpu_rq(dest_cpu)->cfs;

kernel/sched/fair.c

11241

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

11317

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

11349

cpu_rq(cpu)->cpu_capacity = capacity;

kernel/sched/fair.c

11350

trace_sched_cpu_capacity_tp(cpu_rq(cpu));

kernel/sched/fair.c

11811

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

12123

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

12161

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

12283

if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))

kernel/sched/fair.c

12941

rq = cpu_rq(i);

kernel/sched/fair.c

13397

env.dst_rq = cpu_rq(env.new_dst_cpu);

kernel/sched/fair.c

13611

struct rq *target_rq = cpu_rq(target_cpu);

kernel/sched/fair.c

13911

smp_call_function_single_async(ilb_cpu, &cpu_rq(ilb_cpu)->nohz_csd);

kernel/sched/fair.c

14083

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

14211

rq = cpu_rq(balance_cpu);

kernel/sched/fair.c

14302

_nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK);

kernel/sched/fair.c

1443

sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd);

kernel/sched/fair.c

14835

cfs_rq = &cpu_rq(cpu)->cfs;

kernel/sched/fair.c

15145

rq = cpu_rq(i);

kernel/sched/fair.c

15164

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

15193

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

15238

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

15289

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

15407

for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)

kernel/sched/fair.c

15447

INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i));

kernel/sched/fair.c

15448

INIT_LIST_HEAD(&cpu_rq(i)->cfsb_csd_list);

kernel/sched/fair.c

1583

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

1881

occ = fraction_mm_sched(cpu_rq(i),

kernel/sched/fair.c

1889

cur = rcu_dereference_all(cpu_rq(i)->curr);

kernel/sched/fair.c

2888

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

2920

struct rq *rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2935

rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

2950

rq = cpu_rq(env->best_cpu);

kernel/sched/fair.c

3009

struct rq *dst_rq = cpu_rq(env->dst_cpu);

kernel/sched/fair.c

3366

best_rq = cpu_rq(env.best_cpu);

kernel/sched/fair.c

3824

tsk = READ_ONCE(cpu_rq(cpu)->curr);

kernel/sched/fair.c

5789

capacity -= max(hw_load_avg(cpu_rq(cpu)), cpufreq_get_pressure(cpu));

kernel/sched/fair.c

7111

__cfsb_csd_unthrottle(cpu_rq(this_cpu));

kernel/sched/fair.c

7323

cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));

kernel/sched/fair.c

7466

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

7717

rq_util_max = uclamp_rq_get(cpu_rq(cpu), UCLAMP_MAX);

kernel/sched/fair.c

7766

choose_sched_idle_rq(cpu_rq(cpu), p);

kernel/sched/fair.c

8138

return cpu_rq(cpu)->cpu_capacity;

kernel/sched/fair.c

8219

struct rq *rq = cpu_rq(this_cpu);

kernel/sched/fair.c

8238

this_eff_load = cpu_load(cpu_rq(this_cpu));

kernel/sched/fair.c

8256

prev_eff_load = cpu_load(cpu_rq(prev_cpu));

kernel/sched/fair.c

8316

struct rq *rq = cpu_rq(i);

kernel/sched/fair.c

8346

load = cpu_load(cpu_rq(i));

kernel/sched/fair.c

8412

if (choose_idle_cpu(cpu, p) && sched_cpu_cookie_match(cpu_rq(cpu), p))

kernel/sched/fair.c

8485

if (choose_sched_idle_rq(cpu_rq(cpu), p) &&

kernel/sched/fair.c

8973

struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;

kernel/sched/fair.c

9097

struct rq *rq = cpu_rq(cpu);

kernel/sched/fair.c

9197

unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu));

kernel/sched/fair.c

9280

if (uclamp_rq_is_idle(cpu_rq(cpu)))

kernel/sched/fair.c

9412

struct rq *rq = cpu_rq(cpu);

kernel/sched/membarrier.c

307

if (!(READ_ONCE(cpu_rq(cpu)->membarrier_state) &

kernel/sched/membarrier.c

315

p = rcu_dereference(cpu_rq(cpu)->curr);

kernel/sched/membarrier.c

386

p = rcu_dereference(cpu_rq(cpu_id)->curr);

kernel/sched/membarrier.c

411

p = rcu_dereference(cpu_rq(cpu)->curr);

kernel/sched/membarrier.c

486

struct rq *rq = cpu_rq(cpu);

kernel/sched/psi.c

1235

guard(rq_lock_irq)(cpu_rq(cpu));

kernel/sched/psi.c

804

lockdep_assert_rq_held(cpu_rq(cpu));

kernel/sched/rt.c

1514

rq = cpu_rq(cpu);

kernel/sched/rt.c

1565

p->prio < cpu_rq(target)->rt.highest_prio.curr)

kernel/sched/rt.c

1908

lowest_rq = cpu_rq(cpu);

kernel/sched/rt.c

2293

src_rq = cpu_rq(cpu);

kernel/sched/rt.c

230

struct rq *rq = cpu_rq(cpu);

kernel/sched/rt.c

2594

rt_rq = &cpu_rq(cpu)->rt;

kernel/sched/rt.c

2942

for_each_rt_rq(rt_rq, iter, cpu_rq(cpu))

kernel/sched/rt.c

950

return &cpu_rq(cpu)->rt;

kernel/sched/sched.h

1417

#define task_rq(p) cpu_rq(task_cpu(p))

kernel/sched/sched.h

1418

#define cpu_curr(cpu) (cpu_rq(cpu)->curr)

kernel/sched/sched.h

1434

if (!idle_rq(cpu_rq(cpu)))

kernel/sched/sched.h

1548

if (sched_core_cookie_match(cpu_rq(cpu), p))

kernel/sched/sched.h

2141

for (__sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd); \

kernel/sched/sched.h

3439

#define nohz_flags(cpu) (&cpu_rq(cpu)->nohz_flags)

kernel/sched/stats.c

121

rq = cpu_rq(cpu);

kernel/sched/syscalls.c

183

return idle_rq(cpu_rq(cpu));

kernel/sched/syscalls.c

194

return cpu_rq(cpu)->idle;

kernel/sched/syscalls.c

200

struct rq *rq = cpu_rq(cpu);

kernel/sched/topology.c

1019

sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);

kernel/sched/topology.c

3204

rq = cpu_rq(i);

kernel/sched/topology.c

3446

sd = rcu_dereference(cpu_rq(cpu)->sd);

kernel/sched/topology.c

3469

cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) {

kernel/sched/topology.c

416

struct root_domain *rd = cpu_rq(cpu)->rd;

kernel/sched/topology.c

748

struct rq *rq = cpu_rq(cpu);