kernel/sched/core.c

2616

struct rq *rq = this_rq();

kernel/sched/core.c

2727

struct rq *lowest_rq = NULL, *rq = this_rq();

kernel/sched/core.c

3694

rq = this_rq();

kernel/sched/core.c

3894

struct rq *rq = this_rq();

kernel/sched/core.c

5209

__acquires(__rq_lockp(this_rq()))

kernel/sched/core.c

5227

__acquire(__rq_lockp(this_rq()));

kernel/sched/core.c

5319

__releases(__rq_lockp(this_rq()))

kernel/sched/core.c

5321

struct rq *rq = this_rq();

kernel/sched/core.c

5421

__releases(__rq_lockp(this_rq()))

kernel/sched/core.c

6098

schedstat_inc(this_rq()->sched_count);

kernel/sched/core.c

8406

struct rq *rq = this_rq();

kernel/sched/core.c

8455

if (!cpu_dying(rq->cpu) || rq != this_rq())

kernel/sched/core.c

8526

struct rq *rq = this_rq();

kernel/sched/core.c

984

if (rq == this_rq())

kernel/sched/cputime.c

229

struct rq *rq = this_rq();

kernel/sched/cputime.c

277

steal -= this_rq()->prev_steal_time;

kernel/sched/cputime.c

280

this_rq()->prev_steal_time += steal;

kernel/sched/cputime.c

410

} else if (p == this_rq()->idle) {

kernel/sched/cputime.c

713

else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))

kernel/sched/deadline.c

3215

static void pull_dl_task(struct rq *this_rq)

kernel/sched/deadline.c

3217

int this_cpu = this_rq->cpu, cpu;

kernel/sched/deadline.c

3223

if (likely(!dl_overloaded(this_rq)))

kernel/sched/deadline.c

3232

for_each_cpu(cpu, this_rq->rd->dlo_mask) {

kernel/sched/deadline.c

3242

if (this_rq->dl.dl_nr_running &&

kernel/sched/deadline.c

3243

dl_time_before(this_rq->dl.earliest_dl.curr,

kernel/sched/deadline.c

3249

double_lock_balance(this_rq, src_rq);

kernel/sched/deadline.c

3266

dl_task_is_earliest_deadline(p, this_rq)) {

kernel/sched/deadline.c

3281

move_queued_task_locked(src_rq, this_rq, p);

kernel/sched/deadline.c

3289

double_unlock_balance(this_rq, src_rq);

kernel/sched/deadline.c

3293

raw_spin_rq_unlock(this_rq);

kernel/sched/deadline.c

3297

raw_spin_rq_lock(this_rq);

kernel/sched/deadline.c

3302

resched_curr(this_rq);

kernel/sched/ext/ext.c

10341

rq = this_rq();

kernel/sched/ext/ext.c

1245

rq = this_rq();

kernel/sched/ext/ext.c

2429

static bool consume_remote_task(struct rq *this_rq,

kernel/sched/ext/ext.c

2433

raw_spin_rq_unlock(this_rq);

kernel/sched/ext/ext.c

2436

move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq);

kernel/sched/ext/ext.c

2440

raw_spin_rq_lock(this_rq);

kernel/sched/ext/ext.c

2740

dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p));

kernel/sched/ext/ext.c

3358

this_rq()->scx.in_select_cpu = true;

kernel/sched/ext/ext.c

3362

this_rq()->scx.in_select_cpu = false;

kernel/sched/ext/ext.c

8337

static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *ksyncs)

kernel/sched/ext/ext.c

8340

struct scx_rq *this_scx = &this_rq->scx;

kernel/sched/ext/ext.c

8354

if ((cpu_online(cpu) || cpu == cpu_of(this_rq)) &&

kernel/sched/ext/ext.c

8382

static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq)

kernel/sched/ext/ext.c

8390

(cpu_online(cpu) || cpu == cpu_of(this_rq)))

kernel/sched/ext/ext.c

8398

struct rq *this_rq = this_rq();

kernel/sched/ext/ext.c

8399

struct scx_rq *this_scx = &this_rq->scx;

kernel/sched/ext/ext.c

8412

should_wait |= kick_one_cpu(cpu, this_rq, ksyncs);

kernel/sched/ext/ext.c

8418

kick_one_cpu_if_idle(cpu, this_rq);

kernel/sched/ext/ext.c

8427

raw_spin_rq_lock(this_rq);

kernel/sched/ext/ext.c

8429

resched_curr(this_rq);

kernel/sched/ext/ext.c

8430

raw_spin_rq_unlock(this_rq);

kernel/sched/ext/ext.c

8839

struct rq *this_rq, *src_rq, *locked_rq;

kernel/sched/ext/ext.c

8878

this_rq = this_rq();

kernel/sched/ext/ext.c

8879

in_balance = this_rq->scx.flags & SCX_RQ_IN_BALANCE;

kernel/sched/ext/ext.c

8882

if (this_rq != src_rq)

kernel/sched/ext/ext.c

8883

switch_rq_lock(this_rq, src_rq);

kernel/sched/ext/ext.c

8898

dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p));

kernel/sched/ext/ext.c

8915

if (this_rq != locked_rq)

kernel/sched/ext/ext.c

8916

switch_rq_lock(locked_rq, this_rq);

kernel/sched/ext/ext.c

9156

struct rq *this_rq = this_rq();

kernel/sched/ext/ext.c

9175

return scx_dispatch_sched(child, this_rq, this_rq->scx.sub_dispatch_prev,

kernel/sched/ext/ext.c

9369

struct rq *this_rq;

kernel/sched/ext/ext.c

9374

this_rq = this_rq();

kernel/sched/ext/ext.c

9381

if (scx_bypassing(sch, cpu_of(this_rq)))

kernel/sched/ext/ext.c

9402

cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle);

kernel/sched/ext/ext.c

9404

cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick);

kernel/sched/ext/ext.c

9407

cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt);

kernel/sched/ext/ext.c

9409

cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait);

kernel/sched/ext/ext.c

9412

irq_work_queue(&this_rq->scx.kick_cpus_irq_work);

kernel/sched/ext/ext.c

9486

ret = READ_ONCE(this_rq()->scx.local_dsq.nr);

kernel/sched/ext/ext.c

9687

dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, smp_processor_id());

kernel/sched/ext/idle.c

936

if (this_rq()->scx.in_select_cpu) {

kernel/sched/fair.c

13269

static int sched_balance_rq(int this_cpu, struct rq *this_rq,

kernel/sched/fair.c

13282

.dst_rq = this_rq,

kernel/sched/fair.c

14053

WARN_ON_ONCE(rq != this_rq());

kernel/sched/fair.c

14156

static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)

kernel/sched/fair.c

14163

int this_cpu = this_rq->cpu;

kernel/sched/fair.c

14259

static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)

kernel/sched/fair.c

14261

unsigned int flags = this_rq->nohz_idle_balance;

kernel/sched/fair.c

14266

this_rq->nohz_idle_balance = 0;

kernel/sched/fair.c

14271

_nohz_idle_balance(this_rq, flags);

kernel/sched/fair.c

14305

static void nohz_newidle_balance(struct rq *this_rq)

kernel/sched/fair.c

14307

int this_cpu = this_rq->cpu;

kernel/sched/fair.c

14310

if (this_rq->avg_idle < sysctl_sched_migration_cost)

kernel/sched/fair.c

14328

static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)

kernel/sched/fair.c

14333

static inline void nohz_newidle_balance(struct rq *this_rq) { }

kernel/sched/fair.c

14345

static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)

kernel/sched/fair.c

14346

__must_hold(__rq_lockp(this_rq))

kernel/sched/fair.c

14349

int this_cpu = this_rq->cpu;

kernel/sched/fair.c

14355

update_misfit_status(NULL, this_rq);

kernel/sched/fair.c

14361

if (this_rq->ttwu_pending)

kernel/sched/fair.c

14369

this_rq->idle_stamp = rq_clock(this_rq);

kernel/sched/fair.c

14383

rq_unpin_lock(this_rq, rf);

kernel/sched/fair.c

14385

sd = rcu_dereference_sched_domain(this_rq->sd);

kernel/sched/fair.c

14389

if (!get_rd_overloaded(this_rq->rd) ||

kernel/sched/fair.c

14390

this_rq->avg_idle < sd->max_newidle_lb_cost) {

kernel/sched/fair.c

14402

__sched_balance_update_blocked_averages(this_rq);

kernel/sched/fair.c

14404

rq_modified_begin(this_rq, &fair_sched_class);

kernel/sched/fair.c

14405

raw_spin_rq_unlock(this_rq);

kernel/sched/fair.c

14412

if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)

kernel/sched/fair.c

14433

pulled_task = sched_balance_rq(this_cpu, this_rq,

kernel/sched/fair.c

14457

raw_spin_rq_lock(this_rq);

kernel/sched/fair.c

14459

if (curr_cost > this_rq->max_idle_balance_cost)

kernel/sched/fair.c

14460

this_rq->max_idle_balance_cost = curr_cost;

kernel/sched/fair.c

14467

if (this_rq->cfs.h_nr_queued && !pulled_task)

kernel/sched/fair.c

14471

if (rq_modified_above(this_rq, &fair_sched_class))

kernel/sched/fair.c

14476

if (time_after(this_rq->next_balance, next_balance))

kernel/sched/fair.c

14477

this_rq->next_balance = next_balance;

kernel/sched/fair.c

14480

this_rq->idle_stamp = 0;

kernel/sched/fair.c

14482

nohz_newidle_balance(this_rq);

kernel/sched/fair.c

14484

rq_repin_lock(this_rq, rf);

kernel/sched/fair.c

14499

struct rq *this_rq = this_rq();

kernel/sched/fair.c

14500

enum cpu_idle_type idle = this_rq->idle_balance;

kernel/sched/fair.c

14509

if (nohz_idle_balance(this_rq, idle))

kernel/sched/fair.c

14513

sched_balance_update_blocked_averages(this_rq->cpu);

kernel/sched/fair.c

14514

sched_balance_domains(this_rq, idle);

kernel/sched/fair.c

5735

static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf)

kernel/sched/fair.c

5736

__must_hold(__rq_lockp(this_rq));

kernel/sched/fair.c

7007

if (rq == this_rq()) {

kernel/sched/fair.c

8852

this_rq()->nr_running <= 1 &&

kernel/sched/fair.c

9361

struct root_domain *rd = this_rq()->rd;

kernel/sched/fair.c

9565

if (!is_rd_overutilized(this_rq()->rd)) {

kernel/sched/idle.c

24

idle_set_state(this_rq(), idle_state);

kernel/sched/loadavg.c

253

calc_load_nohz_fold(this_rq());

kernel/sched/loadavg.c

267

struct rq *this_rq = this_rq();

kernel/sched/loadavg.c

272

this_rq->calc_load_update = READ_ONCE(calc_load_update);

kernel/sched/loadavg.c

273

if (time_before(jiffies, this_rq->calc_load_update))

kernel/sched/loadavg.c

281

if (time_before(jiffies, this_rq->calc_load_update + 10))

kernel/sched/loadavg.c

282

this_rq->calc_load_update += LOAD_FREQ;

kernel/sched/loadavg.c

387

void calc_global_load_tick(struct rq *this_rq)

kernel/sched/loadavg.c

391

if (time_before(jiffies, this_rq->calc_load_update))

kernel/sched/loadavg.c

394

delta = calc_load_fold_active(this_rq, 0);

kernel/sched/loadavg.c

398

this_rq->calc_load_update += LOAD_FREQ;

kernel/sched/loadavg.c

80

long calc_load_fold_active(struct rq *this_rq, long adjust)

kernel/sched/loadavg.c

84

nr_active = this_rq->nr_running - adjust;

kernel/sched/loadavg.c

85

nr_active += (long)this_rq->nr_uninterruptible;

kernel/sched/loadavg.c

87

if (nr_active != this_rq->calc_load_active) {

kernel/sched/loadavg.c

88

delta = nr_active - this_rq->calc_load_active;

kernel/sched/loadavg.c

89

this_rq->calc_load_active = nr_active;

kernel/sched/membarrier.c

267

struct rq *rq = this_rq();

kernel/sched/rt.c

2230

rq = this_rq();

kernel/sched/rt.c

2260

static void pull_rt_task(struct rq *this_rq)

kernel/sched/rt.c

2262

int this_cpu = this_rq->cpu, cpu;

kernel/sched/rt.c

2266

int rt_overload_count = rt_overloaded(this_rq);

kernel/sched/rt.c

2279

cpumask_test_cpu(this_rq->cpu, this_rq->rd->rto_mask))

kernel/sched/rt.c

2284

tell_cpu_to_push(this_rq);

kernel/sched/rt.c

2289

for_each_cpu(cpu, this_rq->rd->rto_mask) {

kernel/sched/rt.c

2303

this_rq->rt.highest_prio.curr)

kernel/sched/rt.c

2312

double_lock_balance(this_rq, src_rq);

kernel/sched/rt.c

2324

if (p && (p->prio < this_rq->rt.highest_prio.curr)) {

kernel/sched/rt.c

2342

move_queued_task_locked(src_rq, this_rq, p);

kernel/sched/rt.c

2353

double_unlock_balance(this_rq, src_rq);

kernel/sched/rt.c

2357

raw_spin_rq_unlock(this_rq);

kernel/sched/rt.c

2361

raw_spin_rq_lock(this_rq);

kernel/sched/rt.c

2366

resched_curr(this_rq);

kernel/sched/rt.c

583

return this_rq()->rd->span;

kernel/sched/sched.h

107

extern void calc_global_load_tick(struct rq *this_rq);

kernel/sched/sched.h

108

extern long calc_load_fold_active(struct rq *this_rq, long adjust);

kernel/sched/sched.h

2046

rq = this_rq();

kernel/sched/sched.h

2659

void (*task_woken)(struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2695

void (*switching_from)(struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2696

void (*switched_from) (struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2697

void (*switching_to) (struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2698

void (*switched_to) (struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2699

u64 (*get_prio) (struct rq *this_rq, struct task_struct *task);

kernel/sched/sched.h

2700

void (*prio_changed) (struct rq *this_rq, struct task_struct *task,

kernel/sched/sched.h

2707

void (*reweight_task)(struct rq *this_rq, struct task_struct *task,

kernel/sched/sched.h

3250

static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)

kernel/sched/sched.h

3251

__must_hold(__rq_lockp(this_rq))

kernel/sched/sched.h

3254

raw_spin_rq_unlock(this_rq);

kernel/sched/sched.h

3255

double_rq_lock(this_rq, busiest);

kernel/sched/sched.h

3268

static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest)

kernel/sched/sched.h

3269

__must_hold(__rq_lockp(this_rq))

kernel/sched/sched.h

3272

if (__rq_lockp(this_rq) == __rq_lockp(busiest)) {

kernel/sched/sched.h

3274

double_rq_clock_clear_update(this_rq, busiest);

kernel/sched/sched.h

3279

double_rq_clock_clear_update(this_rq, busiest);

kernel/sched/sched.h

3283

if (rq_order_less(this_rq, busiest)) {

kernel/sched/sched.h

3285

double_rq_clock_clear_update(this_rq, busiest);

kernel/sched/sched.h

3289

raw_spin_rq_unlock(this_rq);

kernel/sched/sched.h

3290

double_rq_lock(this_rq, busiest);

kernel/sched/sched.h

3300

static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest)

kernel/sched/sched.h

3301

__must_hold(__rq_lockp(this_rq))

kernel/sched/sched.h

3306

return _double_lock_balance(this_rq, busiest);

kernel/sched/sched.h

3309

static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)

kernel/sched/sched.h

3312

if (__rq_lockp(this_rq) != __rq_lockp(busiest))

kernel/sched/sched.h

3316

lock_set_subclass(&__rq_lockp(this_rq)->dep_map, 0, _RET_IP_);

kernel/sched/syscalls.c

1417

rq = this_rq();