sys/kern/sched_ule.c

1002

tdq_lock_pair(struct tdq *one, struct tdq *two)

sys/kern/sched_ule.c

1017

tdq_unlock_pair(struct tdq *one, struct tdq *two)

sys/kern/sched_ule.c

1028

sched_balance_pair(struct tdq *high, struct tdq *low)

sys/kern/sched_ule.c

1067

tdq_move(struct tdq *from, struct tdq *to)

sys/kern/sched_ule.c

1097

tdq_idled(struct tdq *tdq)

sys/kern/sched_ule.c

1100

struct tdq *steal;

sys/kern/sched_ule.c

1104

if (smp_started == 0 || steal_idle == 0 || tdq->tdq_cg == NULL)

sys/kern/sched_ule.c

1109

switchcnt = TDQ_SWITCHCNT(tdq);

sys/kern/sched_ule.c

1110

for (cg = tdq->tdq_cg, goup = 0; ; ) {

sys/kern/sched_ule.c

1116

if (TDQ_LOAD(tdq))

sys/kern/sched_ule.c

1161

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

1162

if (tdq->tdq_load > 0) {

sys/kern/sched_ule.c

1167

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

1180

switchcnt != TDQ_SWITCHCNT(tdq)) {

sys/kern/sched_ule.c

1181

tdq_unlock_pair(tdq, steal);

sys/kern/sched_ule.c

1187

if (tdq_move(steal, tdq) != -1)

sys/kern/sched_ule.c

1198

tdq_unlock_pair(tdq, steal);

sys/kern/sched_ule.c

1212

tdq_notify(struct tdq *tdq, int lowpri)

sys/kern/sched_ule.c

1216

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

1217

KASSERT(tdq->tdq_lowpri <= lowpri,

sys/kern/sched_ule.c

1218

("tdq_notify: lowpri %d > tdq_lowpri %d", lowpri, tdq->tdq_lowpri));

sys/kern/sched_ule.c

1220

if (tdq->tdq_owepreempt)

sys/kern/sched_ule.c

1227

if (!sched_shouldpreempt(tdq->tdq_lowpri, lowpri, 1))

sys/kern/sched_ule.c

1242

cpu = TDQ_ID(tdq);

sys/kern/sched_ule.c

1243

if (TD_IS_IDLETHREAD(tdq->tdq_curthread) &&

sys/kern/sched_ule.c

1244

(atomic_load_int(&tdq->tdq_cpu_idle) == 0 || cpu_idle_wakeup(cpu)))

sys/kern/sched_ule.c

1251

tdq->tdq_owepreempt = 1;

sys/kern/sched_ule.c

1337

tdq_steal(struct tdq *tdq, int cpu)

sys/kern/sched_ule.c

1341

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

1342

td = runq_steal_realtime(&tdq->tdq_runq, cpu);

sys/kern/sched_ule.c

1345

td = runq_steal_timeshare(&tdq->tdq_runq, cpu, tdq->tdq_ts_deq_off);

sys/kern/sched_ule.c

1348

return (runq_steal_idle(&tdq->tdq_runq, cpu));

sys/kern/sched_ule.c

1355

static inline struct tdq *

sys/kern/sched_ule.c

1359

struct tdq *tdq;

sys/kern/sched_ule.c

1363

tdq = TDQ_CPU(cpu);

sys/kern/sched_ule.c

1368

if (td->td_lock == TDQ_LOCKPTR(tdq)) {

sys/kern/sched_ule.c

1371

return (tdq);

sys/kern/sched_ule.c

1382

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

1383

thread_lock_unblock(td, TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

1385

return (tdq);

sys/kern/sched_ule.c

1400

struct tdq *tdq;

sys/kern/sched_ule.c

1421

tdq = TDQ_SELF();

sys/kern/sched_ule.c

1422

if (tdq->tdq_lowpri >= PRI_MIN_IDLE) {

sys/kern/sched_ule.c

1428

cg = tdq->tdq_cg;

sys/kern/sched_ule.c

1432

tdq = TDQ_CPU(ts->ts_cpu);

sys/kern/sched_ule.c

1433

cg = tdq->tdq_cg;

sys/kern/sched_ule.c

1440

atomic_load_char(&tdq->tdq_lowpri) >= PRI_MIN_IDLE &&

sys/kern/sched_ule.c

1517

tdq = TDQ_CPU(cpu);

sys/kern/sched_ule.c

1519

atomic_load_char(&tdq->tdq_lowpri) < PRI_MIN_IDLE &&

sys/kern/sched_ule.c

1520

TDQ_LOAD(TDQ_SELF()) <= TDQ_LOAD(tdq) + 1) {

sys/kern/sched_ule.c

1563

tdq_choose(struct tdq *tdq)

sys/kern/sched_ule.c

1567

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

1568

td = runq_choose_realtime(&tdq->tdq_runq);

sys/kern/sched_ule.c

1571

td = runq_choose_timeshare(&tdq->tdq_runq, tdq->tdq_ts_deq_off);

sys/kern/sched_ule.c

1578

td = runq_choose_idle(&tdq->tdq_runq);

sys/kern/sched_ule.c

1593

tdq_setup(struct tdq *tdq, int id)

sys/kern/sched_ule.c

1598

runq_init(&tdq->tdq_runq);

sys/kern/sched_ule.c

1599

tdq->tdq_id = id;

sys/kern/sched_ule.c

1600

snprintf(tdq->tdq_name, sizeof(tdq->tdq_name),

sys/kern/sched_ule.c

1601

"sched lock %d", (int)TDQ_ID(tdq));

sys/kern/sched_ule.c

1602

mtx_init(&tdq->tdq_lock, tdq->tdq_name, "sched lock", MTX_SPIN);

sys/kern/sched_ule.c

1604

snprintf(tdq->tdq_loadname, sizeof(tdq->tdq_loadname),

sys/kern/sched_ule.c

1605

"CPU %d load", (int)TDQ_ID(tdq));

sys/kern/sched_ule.c

1613

struct tdq *tdq;

sys/kern/sched_ule.c

1617

tdq = DPCPU_ID_PTR(i, tdq);

sys/kern/sched_ule.c

1618

tdq_setup(tdq, i);

sys/kern/sched_ule.c

1619

tdq->tdq_cg = smp_topo_find(cpu_top, i);

sys/kern/sched_ule.c

1620

if (tdq->tdq_cg == NULL)

sys/kern/sched_ule.c

1624

PCPU_SET(sched, DPCPU_PTR(tdq));

sys/kern/sched_ule.c

1636

struct tdq *tdq;

sys/kern/sched_ule.c

1643

tdq = TDQ_SELF();

sys/kern/sched_ule.c

1646

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

1647

thread0.td_lock = TDQ_LOCKPTR(tdq);

sys/kern/sched_ule.c

1648

tdq_load_add(tdq, &thread0);

sys/kern/sched_ule.c

1649

tdq->tdq_curthread = &thread0;

sys/kern/sched_ule.c

1650

tdq->tdq_lowpri = thread0.td_priority;

sys/kern/sched_ule.c

1651

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

1913

PCPU_SET(sched, DPCPU_PTR(tdq));

sys/kern/sched_ule.c

1993

struct tdq *tdq;

sys/kern/sched_ule.c

2027

tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);

sys/kern/sched_ule.c

2030

if (prio < tdq->tdq_lowpri)

sys/kern/sched_ule.c

2031

tdq->tdq_lowpri = prio;

sys/kern/sched_ule.c

2032

else if (tdq->tdq_lowpri == oldpri)

sys/kern/sched_ule.c

2033

tdq_setlowpri(tdq, td);

sys/kern/sched_ule.c

2165

tdq_trysteal(struct tdq *tdq)

sys/kern/sched_ule.c

2168

struct tdq *steal;

sys/kern/sched_ule.c

2173

tdq->tdq_cg == NULL)

sys/kern/sched_ule.c

2179

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

2180

for (i = 1, cg = tdq->tdq_cg, goup = 0; ; ) {

sys/kern/sched_ule.c

2186

if (TDQ_LOAD(tdq) > 0) {

sys/kern/sched_ule.c

2187

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2203

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2208

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2237

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2238

if (tdq->tdq_load > 0)

sys/kern/sched_ule.c

2256

if (tdq_move(steal, tdq) == -1) {

sys/kern/sched_ule.c

2272

sched_switch_migrate(struct tdq *tdq, struct thread *td, int flags)

sys/kern/sched_ule.c

2274

struct tdq *tdn;

sys/kern/sched_ule.c

2287

tdq_load_rem(tdq, td);

sys/kern/sched_ule.c

2293

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

2298

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2323

struct tdq *tdq;

sys/kern/sched_ule.c

2335

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2351

atomic_store_char(&tdq->tdq_owepreempt, 0);

sys/kern/sched_ule.c

2353

TDQ_SWITCHCNT_INC(tdq);

sys/kern/sched_ule.c

2361

MPASS(mtx == TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

2364

MPASS(mtx == TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

2373

tdq_runq_add(tdq, td, srqflag);

sys/kern/sched_ule.c

2375

mtx = sched_switch_migrate(tdq, td, srqflag);

sys/kern/sched_ule.c

2378

if (mtx != TDQ_LOCKPTR(tdq)) {

sys/kern/sched_ule.c

2380

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2382

tdq_load_rem(tdq, td);

sys/kern/sched_ule.c

2384

if (tdq->tdq_load == 0)

sys/kern/sched_ule.c

2385

tdq_trysteal(tdq);

sys/kern/sched_ule.c

2404

TDQ_LOCK_ASSERT(tdq, MA_OWNED | MA_NOTRECURSED);

sys/kern/sched_ule.c

2405

MPASS(td == tdq->tdq_curthread);

sys/kern/sched_ule.c

2408

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

2562

struct tdq *tdq;

sys/kern/sched_ule.c

2564

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2573

child->td_lock = TDQ_LOCKPTR(tdq);

sys/kern/sched_ule.c

2594

ts2->ts_slice = tdq_slice(tdq) - sched_slice_min;

sys/kern/sched_ule.c

2655

struct tdq *tdq;

sys/kern/sched_ule.c

2661

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2662

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

2663

if (td->td_priority > tdq->tdq_lowpri) {

sys/kern/sched_ule.c

2674

tdq->tdq_owepreempt = 0;

sys/kern/sched_ule.c

2699

td_slice(struct thread *td, struct tdq *tdq)

sys/kern/sched_ule.c

2703

return (tdq_slice(tdq));

sys/kern/sched_ule.c

2713

struct tdq *tdq;

sys/kern/sched_ule.c

2717

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2722

if (balance_tdq == tdq && smp_started != 0 && rebalance != 0 &&

sys/kern/sched_ule.c

2734

tdq->tdq_oldswitchcnt = tdq->tdq_switchcnt;

sys/kern/sched_ule.c

2735

tdq->tdq_switchcnt = tdq->tdq_load;

sys/kern/sched_ule.c

2748

if (tdq->tdq_ts_off == tdq->tdq_ts_deq_off) {

sys/kern/sched_ule.c

2749

tdq->tdq_ts_ticks += cnt;

sys/kern/sched_ule.c

2750

tdq->tdq_ts_off = (tdq->tdq_ts_off + 2 * cnt -

sys/kern/sched_ule.c

2751

tdq->tdq_ts_ticks / 4) % RQ_TS_POL_MODULO;

sys/kern/sched_ule.c

2752

tdq->tdq_ts_ticks %= 4;

sys/kern/sched_ule.c

2753

tdq_advance_ts_deq_off(tdq, false);

sys/kern/sched_ule.c

2775

if (ts->ts_slice >= td_slice(td, tdq)) {

sys/kern/sched_ule.c

2810

struct tdq *tdq;

sys/kern/sched_ule.c

2812

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2813

return (TDQ_LOAD(tdq) > (TD_IS_IDLETHREAD(curthread) ? 0 : 1));

sys/kern/sched_ule.c

2824

struct tdq *tdq;

sys/kern/sched_ule.c

2826

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2827

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

2828

td = tdq_choose(tdq);

sys/kern/sched_ule.c

2830

tdq_runq_rem(tdq, td);

sys/kern/sched_ule.c

2831

tdq->tdq_lowpri = td->td_priority;

sys/kern/sched_ule.c

2833

tdq->tdq_lowpri = PRI_MAX_IDLE;

sys/kern/sched_ule.c

2836

tdq->tdq_curthread = td;

sys/kern/sched_ule.c

2870

tdq_add(struct tdq *tdq, struct thread *td, int flags)

sys/kern/sched_ule.c

2874

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

2883

lowpri = tdq->tdq_lowpri;

sys/kern/sched_ule.c

2885

tdq->tdq_lowpri = td->td_priority;

sys/kern/sched_ule.c

2886

tdq_runq_add(tdq, td, flags);

sys/kern/sched_ule.c

2887

tdq_load_add(tdq, td);

sys/kern/sched_ule.c

2900

struct tdq *tdq;

sys/kern/sched_ule.c

2925

tdq = sched_setcpu(td, cpu, flags);

sys/kern/sched_ule.c

2926

lowpri = tdq_add(tdq, td, flags);

sys/kern/sched_ule.c

2928

tdq_notify(tdq, lowpri);

sys/kern/sched_ule.c

2932

tdq = TDQ_SELF();

sys/kern/sched_ule.c

2937

if (td->td_lock != TDQ_LOCKPTR(tdq)) {

sys/kern/sched_ule.c

2938

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

2940

td->td_lock = TDQ_LOCKPTR(tdq);

sys/kern/sched_ule.c

2942

thread_lock_set(td, TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

2944

(void)tdq_add(tdq, td, flags);

sys/kern/sched_ule.c

2960

struct tdq *tdq;

sys/kern/sched_ule.c

2965

tdq = TDQ_CPU(td_get_sched(td)->ts_cpu);

sys/kern/sched_ule.c

2966

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

2967

MPASS(td->td_lock == TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

2970

tdq_runq_rem(tdq, td);

sys/kern/sched_ule.c

2971

tdq_load_rem(tdq, td);

sys/kern/sched_ule.c

2973

if (td->td_priority == tdq->tdq_lowpri)

sys/kern/sched_ule.c

2974

tdq_setlowpri(tdq, NULL);

sys/kern/sched_ule.c

299

#define TDQ_LOAD(tdq) atomic_load_int(&(tdq)->tdq_load)

sys/kern/sched_ule.c

300

#define TDQ_TRANSFERABLE(tdq) atomic_load_int(&(tdq)->tdq_transferable)

sys/kern/sched_ule.c

301

#define TDQ_SWITCHCNT(tdq) (atomic_load_short(&(tdq)->tdq_switchcnt) + \

sys/kern/sched_ule.c

302

atomic_load_short(&(tdq)->tdq_oldswitchcnt))

sys/kern/sched_ule.c

303

#define TDQ_SWITCHCNT_INC(tdq) (atomic_store_short(&(tdq)->tdq_switchcnt, \

sys/kern/sched_ule.c

304

atomic_load_short(&(tdq)->tdq_switchcnt) + 1))

sys/kern/sched_ule.c

3120

#define TDQ_IDLESPIN(tdq) \

sys/kern/sched_ule.c

3121

((tdq)->tdq_cg != NULL && ((tdq)->tdq_cg->cg_flags & CG_FLAG_THREAD) == 0)

sys/kern/sched_ule.c

3123

#define TDQ_IDLESPIN(tdq) 1

sys/kern/sched_ule.c

3133

struct tdq *tdq;

sys/kern/sched_ule.c

3139

tdq = TDQ_SELF();

sys/kern/sched_ule.c

3143

if (TDQ_LOAD(tdq)) {

sys/kern/sched_ule.c

3147

switchcnt = TDQ_SWITCHCNT(tdq);

sys/kern/sched_ule.c

3151

if (tdq_idled(tdq) == 0)

sys/kern/sched_ule.c

3154

switchcnt = TDQ_SWITCHCNT(tdq);

sys/kern/sched_ule.c

3165

if (TDQ_IDLESPIN(tdq) && switchcnt > sched_idlespinthresh) {

sys/kern/sched_ule.c

3167

if (TDQ_LOAD(tdq))

sys/kern/sched_ule.c

3174

switchcnt = TDQ_SWITCHCNT(tdq);

sys/kern/sched_ule.c

3175

if (TDQ_LOAD(tdq) != 0 || switchcnt != oldswitchcnt)

sys/kern/sched_ule.c

3179

atomic_store_int(&tdq->tdq_cpu_idle, 1);

sys/kern/sched_ule.c

3191

if (TDQ_LOAD(tdq) != 0) {

sys/kern/sched_ule.c

3192

atomic_store_int(&tdq->tdq_cpu_idle, 0);

sys/kern/sched_ule.c

3196

atomic_store_int(&tdq->tdq_cpu_idle, 0);

sys/kern/sched_ule.c

3202

switchcnt = TDQ_SWITCHCNT(tdq);

sys/kern/sched_ule.c

3205

TDQ_SWITCHCNT_INC(tdq);

sys/kern/sched_ule.c

3217

sched_throw_grab(struct tdq *tdq)

sys/kern/sched_ule.c

3223

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

3236

struct tdq *tdq;

sys/kern/sched_ule.c

3238

tdq = TDQ_SELF();

sys/kern/sched_ule.c

3241

THREAD_LOCKPTR_ASSERT(curthread, TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

3243

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

3249

newtd = sched_throw_grab(tdq);

sys/kern/sched_ule.c

3266

struct tdq *tdq;

sys/kern/sched_ule.c

3268

tdq = TDQ_SELF();

sys/kern/sched_ule.c

3272

THREAD_LOCKPTR_ASSERT(td, TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

3274

tdq_load_rem(tdq, td);

sys/kern/sched_ule.c

3279

newtd = sched_throw_grab(tdq);

sys/kern/sched_ule.c

3286

cpu_switch(td, newtd, TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

329

static struct tdq __read_mostly *balance_tdq;

sys/kern/sched_ule.c

3296

struct tdq *tdq;

sys/kern/sched_ule.c

3306

tdq = TDQ_SELF();

sys/kern/sched_ule.c

3307

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

3309

MPASS(td->td_lock == TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

331

DPCPU_DEFINE_STATIC(struct tdq, tdq);

sys/kern/sched_ule.c

334

#define TDQ_SELF() ((struct tdq *)PCPU_GET(sched))

sys/kern/sched_ule.c

335

#define TDQ_CPU(x) (DPCPU_ID_PTR((x), tdq))

sys/kern/sched_ule.c

338

static struct tdq tdq_cpu;

sys/kern/sched_ule.c

366

static struct thread *tdq_choose(struct tdq *);

sys/kern/sched_ule.c

368

static void tdq_setup(struct tdq *, int i);

sys/kern/sched_ule.c

369

static void tdq_load_add(struct tdq *, struct thread *);

sys/kern/sched_ule.c

370

static void tdq_load_rem(struct tdq *, struct thread *);

sys/kern/sched_ule.c

371

static inline void tdq_runq_add(struct tdq *, struct thread *, int);

sys/kern/sched_ule.c

372

static inline void tdq_advance_ts_deq_off(struct tdq *, bool);

sys/kern/sched_ule.c

373

static inline void tdq_runq_rem(struct tdq *, struct thread *);

sys/kern/sched_ule.c

377

static int tdq_add(struct tdq *, struct thread *, int);

sys/kern/sched_ule.c

379

static int tdq_move(struct tdq *, struct tdq *);

sys/kern/sched_ule.c

380

static int tdq_idled(struct tdq *);

sys/kern/sched_ule.c

381

static void tdq_notify(struct tdq *, int lowpri);

sys/kern/sched_ule.c

393

static struct thread *tdq_steal(struct tdq *, int);

sys/kern/sched_ule.c

397

static bool sched_balance_pair(struct tdq *, struct tdq *);

sys/kern/sched_ule.c

398

static inline struct tdq *sched_setcpu(struct thread *, int, int);

sys/kern/sched_ule.c

436

struct tdq *tdq;

sys/kern/sched_ule.c

438

tdq = TDQ_CPU(cpu);

sys/kern/sched_ule.c

440

printf("tdq %d:\n", TDQ_ID(tdq));

sys/kern/sched_ule.c

441

printf("\tlock %p\n", TDQ_LOCKPTR(tdq));

sys/kern/sched_ule.c

442

printf("\tLock name: %s\n", tdq->tdq_name);

sys/kern/sched_ule.c

443

printf("\tload: %d\n", tdq->tdq_load);

sys/kern/sched_ule.c

444

printf("\tswitch cnt: %d\n", tdq->tdq_switchcnt);

sys/kern/sched_ule.c

445

printf("\told switch cnt: %d\n", tdq->tdq_oldswitchcnt);

sys/kern/sched_ule.c

446

printf("\tTS insert offset: %d\n", tdq->tdq_ts_off);

sys/kern/sched_ule.c

447

printf("\tTS dequeue offset: %d\n", tdq->tdq_ts_deq_off);

sys/kern/sched_ule.c

448

printf("\tload transferable: %d\n", tdq->tdq_transferable);

sys/kern/sched_ule.c

449

printf("\tlowest priority: %d\n", tdq->tdq_lowpri);

sys/kern/sched_ule.c

451

runq_print(&tdq->tdq_runq);

sys/kern/sched_ule.c

504

tdq_runq_add(struct tdq *tdq, struct thread *td, int flags)

sys/kern/sched_ule.c

509

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

516

tdq->tdq_transferable++;

sys/kern/sched_ule.c

529

idx = tdq->tdq_ts_deq_off;

sys/kern/sched_ule.c

535

tdq->tdq_ts_off);

sys/kern/sched_ule.c

554

if (tdq->tdq_ts_deq_off != tdq->tdq_ts_off &&

sys/kern/sched_ule.c

555

normalize_ts_off(idx - tdq->tdq_ts_deq_off) <

sys/kern/sched_ule.c

556

normalize_ts_off(tdq->tdq_ts_off -

sys/kern/sched_ule.c

557

tdq->tdq_ts_deq_off))

sys/kern/sched_ule.c

558

idx = normalize_ts_off(tdq->tdq_ts_deq_off - 1);

sys/kern/sched_ule.c

562

runq_add_idx(&tdq->tdq_runq, td, idx, flags);

sys/kern/sched_ule.c

564

runq_add(&tdq->tdq_runq, td, flags);

sys/kern/sched_ule.c

576

tdq_advance_ts_deq_off(struct tdq *tdq, bool deq_queue_known_empty)

sys/kern/sched_ule.c

582

while (tdq->tdq_ts_deq_off != tdq->tdq_ts_off) {

sys/kern/sched_ule.c

585

else if (!runq_is_queue_empty(&tdq->tdq_runq,

sys/kern/sched_ule.c

586

tdq->tdq_ts_deq_off + RQ_TS_POL_MIN))

sys/kern/sched_ule.c

589

tdq->tdq_ts_deq_off = (tdq->tdq_ts_deq_off + 1) %

sys/kern/sched_ule.c

600

tdq_runq_rem(struct tdq *tdq, struct thread *td)

sys/kern/sched_ule.c

606

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

609

tdq->tdq_transferable--;

sys/kern/sched_ule.c

612

queue_empty = runq_remove(&tdq->tdq_runq, td);

sys/kern/sched_ule.c

621

tdq->tdq_ts_deq_off + RQ_TS_POL_MIN == td->td_rqindex)

sys/kern/sched_ule.c

622

tdq_advance_ts_deq_off(tdq, true);

sys/kern/sched_ule.c

630

tdq_load_add(struct tdq *tdq, struct thread *td)

sys/kern/sched_ule.c

633

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

636

tdq->tdq_load++;

sys/kern/sched_ule.c

638

tdq->tdq_sysload++;

sys/kern/sched_ule.c

639

KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);

sys/kern/sched_ule.c

640

SDT_PROBE2(sched, , , load__change, (int)TDQ_ID(tdq), tdq->tdq_load);

sys/kern/sched_ule.c

648

tdq_load_rem(struct tdq *tdq, struct thread *td)

sys/kern/sched_ule.c

651

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

653

KASSERT(tdq->tdq_load != 0,

sys/kern/sched_ule.c

654

("tdq_load_rem: Removing with 0 load on queue %d", TDQ_ID(tdq)));

sys/kern/sched_ule.c

656

tdq->tdq_load--;

sys/kern/sched_ule.c

658

tdq->tdq_sysload--;

sys/kern/sched_ule.c

659

KTR_COUNTER0(KTR_SCHED, "load", tdq->tdq_loadname, tdq->tdq_load);

sys/kern/sched_ule.c

660

SDT_PROBE2(sched, , , load__change, (int)TDQ_ID(tdq), tdq->tdq_load);

sys/kern/sched_ule.c

670

tdq_slice(struct tdq *tdq)

sys/kern/sched_ule.c

679

load = tdq->tdq_sysload - 1;

sys/kern/sched_ule.c

692

tdq_setlowpri(struct tdq *tdq, struct thread *ctd)

sys/kern/sched_ule.c

696

TDQ_LOCK_ASSERT(tdq, MA_OWNED);

sys/kern/sched_ule.c

698

ctd = tdq->tdq_curthread;

sys/kern/sched_ule.c

699

td = tdq_choose(tdq);

sys/kern/sched_ule.c

701

tdq->tdq_lowpri = ctd->td_priority;

sys/kern/sched_ule.c

703

tdq->tdq_lowpri = td->td_priority;

sys/kern/sched_ule.c

751

struct tdq *tdq;

sys/kern/sched_ule.c

788

tdq = TDQ_CPU(c);

sys/kern/sched_ule.c

789

l = TDQ_LOAD(tdq);

sys/kern/sched_ule.c

805

(atomic_load_char(&tdq->tdq_lowpri) <= s->cs_pri &&

sys/kern/sched_ule.c

833

struct tdq *tdq;

sys/kern/sched_ule.c

859

tdq = TDQ_CPU(c);

sys/kern/sched_ule.c

860

l = TDQ_LOAD(tdq);

sys/kern/sched_ule.c

867

if (l < s->cs_load || TDQ_TRANSFERABLE(tdq) < s->cs_trans ||

sys/kern/sched_ule.c

922

struct tdq *tdq;

sys/kern/sched_ule.c

938

tdq = TDQ_CPU(high);

sys/kern/sched_ule.c

939

if (TDQ_LOAD(tdq) == 1) {

sys/kern/sched_ule.c

944

TDQ_LOCK(tdq);

sys/kern/sched_ule.c

945

td = tdq->tdq_curthread;

sys/kern/sched_ule.c

946

if (td->td_lock == TDQ_LOCKPTR(tdq) &&

sys/kern/sched_ule.c

954

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

959

if (TDQ_TRANSFERABLE(tdq) == 0)

sys/kern/sched_ule.c

961

low = sched_lowest(cg, &lmask, -1, TDQ_LOAD(tdq) - 1, high, 1);

sys/kern/sched_ule.c

969

if (sched_balance_pair(tdq, TDQ_CPU(low))) {

sys/kern/sched_ule.c

988

struct tdq *tdq;

sys/kern/sched_ule.c

992

tdq = TDQ_SELF();

sys/kern/sched_ule.c

993

TDQ_UNLOCK(tdq);

sys/kern/sched_ule.c

995

TDQ_LOCK(tdq);