include/linux/stackdepot.h

73

unsigned long rcu_state; /* RCU cookie */

include/net/sch_generic.h

1433

unsigned long rcu_state;

kernel/rcu/rcu.h

391

#define rcu_first_leaf_node() (rcu_state.level[rcu_num_lvls - 1])

kernel/rcu/rcu.h

397

#define rcu_is_last_leaf_node(rnp) ((rnp) == &rcu_state.node[rcu_num_nodes - 1])

kernel/rcu/rcu.h

408

_rcu_for_each_node_breadth_first(&rcu_state, rnp)

kernel/rcu/rcu.h

420

(rnp) < &rcu_state.node[rcu_num_nodes]; (rnp)++)

kernel/rcu/tree.c

100

.exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),

kernel/rcu/tree.c

101

.exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),

kernel/rcu/tree.c

103

.srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work,

kernel/rcu/tree.c

1068

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

1069

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

107

.nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex),

kernel/rcu/tree.c

1070

if (!READ_ONCE(rcu_state.gp_kthread)) {

kernel/rcu/tree.c

1074

trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));

kernel/rcu/tree.c

1121

struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);

kernel/rcu/tree.c

1124

!READ_ONCE(rcu_state.gp_flags) || !t)

kernel/rcu/tree.c

1126

WRITE_ONCE(rcu_state.gp_wake_time, jiffies);

kernel/rcu/tree.c

1127

WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));

kernel/rcu/tree.c

1128

swake_up_one(&rcu_state.gp_wq);

kernel/rcu/tree.c

1167

gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

1173

trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));

kernel/rcu/tree.c

1175

trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));

kernel/rcu/tree.c

1196

c = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

1290

trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));

kernel/rcu/tree.c

1306

trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));

kernel/rcu/tree.c

1373

!(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))

kernel/rcu/tree.c

1420

if (!rcu_seq_state(rcu_state.gp_seq_polled))

kernel/rcu/tree.c

1421

rcu_seq_start(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

1424

*snap = rcu_state.gp_seq_polled;

kernel/rcu/tree.c

1438

if (*snap && *snap == rcu_state.gp_seq_polled) {

kernel/rcu/tree.c

1439

rcu_seq_end(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

1440

rcu_state.gp_seq_polled_snap = 0;

kernel/rcu/tree.c

1441

rcu_state.gp_seq_polled_exp_snap = 0;

kernel/rcu/tree.c

1609

return &(rcu_state.srs_wait_nodes)[0].node <= node &&

kernel/rcu/tree.c

1610

node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;

kernel/rcu/tree.c

1619

sr_wn = &(rcu_state.srs_wait_nodes)[i];

kernel/rcu/tree.c

1670

done = smp_load_acquire(&rcu_state.srs_done_tail);

kernel/rcu/tree.c

1697

atomic_dec_return_release(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1708

wait_tail = rcu_state.srs_wait_tail;

kernel/rcu/tree.c

1712

rcu_state.srs_wait_tail = NULL;

kernel/rcu/tree.c

1713

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);

kernel/rcu/tree.c

1740

!atomic_read_acquire(&rcu_state.srs_cleanups_pending)) {

kernel/rcu/tree.c

1746

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail);

kernel/rcu/tree.c

1747

smp_store_release(&rcu_state.srs_done_tail, wait_tail);

kernel/rcu/tree.c

1755

atomic_inc(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1756

if (!queue_work(sync_wq, &rcu_state.srs_cleanup_work))

kernel/rcu/tree.c

1757

atomic_dec(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1770

first = READ_ONCE(rcu_state.srs_next.first);

kernel/rcu/tree.c

1782

llist_add(wait_head, &rcu_state.srs_next);

kernel/rcu/tree.c

1789

WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL);

kernel/rcu/tree.c

1790

rcu_state.srs_wait_tail = wait_head;

kernel/rcu/tree.c

1791

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);

kernel/rcu/tree.c

1798

llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);

kernel/rcu/tree.c

1814

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

1816

if (!rcu_state.gp_flags) {

kernel/rcu/tree.c

1821

WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */

kernel/rcu/tree.c

1840

old_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

1853

rcu_seq_start(&rcu_state.gp_seq);

kernel/rcu/tree.c

1856

rcu_seq_done_exact(&old_gp_seq, rcu_seq_snap(&rcu_state.gp_seq)));

kernel/rcu/tree.c

1858

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

kernel/rcu/tree.c

1859

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));

kernel/rcu/tree.c

1860

rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

1885

WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF);

kernel/rcu/tree.c

1893

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1899

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1936

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1953

WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT);

kernel/rcu/tree.c

1960

WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);

kernel/rcu/tree.c

1964

trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,

kernel/rcu/tree.c

1980

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2014

*gfp = READ_ONCE(rcu_state.gp_flags);

kernel/rcu/tree.c

2030

int nr_fqs = READ_ONCE(rcu_state.nr_fqs_jiffies_stall);

kernel/rcu/tree.c

2033

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2034

WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);

kernel/rcu/tree.c

2040

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree.c

2043

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, --nr_fqs);

kernel/rcu/tree.c

2054

if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {

kernel/rcu/tree.c

2056

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & ~RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

2073

if (rcu_state.cbovld)

kernel/rcu/tree.c

2077

if (rcu_state.cbovld) {

kernel/rcu/tree.c

2082

if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) {

kernel/rcu/tree.c

2083

WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j);

kernel/rcu/tree.c

2089

WRITE_ONCE(rcu_state.jiffies_kick_kthreads,

kernel/rcu/tree.c

2092

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2094

WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS);

kernel/rcu/tree.c

2095

(void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq,

kernel/rcu/tree.c

2098

WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS);

kernel/rcu/tree.c

2113

if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||

kernel/rcu/tree.c

2115

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2121

gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;

kernel/rcu/tree.c

2123

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2126

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2132

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2134

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2138

if (time_after(jiffies, rcu_state.jiffies_force_qs))

kernel/rcu/tree.c

2141

j = rcu_state.jiffies_force_qs - j;

kernel/rcu/tree.c

2161

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2163

rcu_state.gp_end = jiffies;

kernel/rcu/tree.c

2164

gp_duration = rcu_state.gp_end - rcu_state.gp_start;

kernel/rcu/tree.c

2165

if (gp_duration > rcu_state.gp_max)

kernel/rcu/tree.c

2166

rcu_state.gp_max = gp_duration;

kernel/rcu/tree.c

2176

rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

2188

new_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

2213

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

222

return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));

kernel/rcu/tree.c

2220

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));

kernel/rcu/tree.c

2221

rcu_seq_end(&rcu_state.gp_seq);

kernel/rcu/tree.c

2222

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

kernel/rcu/tree.c

2223

WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE);

kernel/rcu/tree.c

2245

WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

2246

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

2247

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));

kernel/rcu/tree.c

2256

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

2278

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2280

WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS);

kernel/rcu/tree.c

2281

swait_event_idle_exclusive(rcu_state.gp_wq,

kernel/rcu/tree.c

2282

READ_ONCE(rcu_state.gp_flags) &

kernel/rcu/tree.c

2285

WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS);

kernel/rcu/tree.c

2290

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2292

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2300

WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP);

kernel/rcu/tree.c

2302

WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED);

kernel/rcu/tree.c

2320

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

2363

trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,

kernel/rcu/tree.c

2556

trace_rcu_batch_start(rcu_state.name,

kernel/rcu/tree.c

2558

trace_rcu_batch_end(rcu_state.name, 0,

kernel/rcu/tree.c

2592

trace_rcu_batch_start(rcu_state.name,

kernel/rcu/tree.c

2612

trace_rcu_invoke_callback(rcu_state.name, rhp);

kernel/rcu/tree.c

2653

trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),

kernel/rcu/tree.c

2668

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

2738

rcu_state.cbovld = rcu_state.cbovldnext;

kernel/rcu/tree.c

2739

rcu_state.cbovldnext = false;

kernel/rcu/tree.c

2746

rcu_state.cbovldnext |= !!rnp->cbovldmask;

kernel/rcu/tree.c

2803

ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||

kernel/rcu/tree.c

2816

if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {

kernel/rcu/tree.c

2820

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

3001

trace_rcu_callback(rcu_state.name, head,

kernel/rcu/tree.c

3043

if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&

kernel/rcu/tree.c

3046

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

3281

trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));

kernel/rcu/tree.c

3308

trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));

kernel/rcu/tree.c

3372

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

3373

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

3381

rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);

kernel/rcu/tree.c

3383

rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

3417

return rcu_seq_snap(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

3449

rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

3450

rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);

kernel/rcu/tree.c

3475

needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq));

kernel/rcu/tree.c

3554

rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) {

kernel/rcu/tree.c

3603

rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) {

kernel/rcu/tree.c

3689

time_before(jiffies, READ_ONCE(rcu_state.gp_start) +

kernel/rcu/tree.c

3724

trace_rcu_barrier(rcu_state.name, s, cpu,

kernel/rcu/tree.c

3725

atomic_read(&rcu_state.barrier_cpu_count), done);

kernel/rcu/tree.c

3740

unsigned long __maybe_unused s = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3743

if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {

kernel/rcu/tree.c

3745

complete(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3756

unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence);

kernel/rcu/tree.c

3761

lockdep_assert_held(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3764

rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3777

atomic_inc(&rcu_state.barrier_cpu_count);

kernel/rcu/tree.c

3780

rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3799

raw_spin_lock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3801

raw_spin_unlock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3823

unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3828

mutex_lock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3831

if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {

kernel/rcu/tree.c

3832

rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3834

mutex_unlock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3839

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3840

rcu_seq_start(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3841

gseq = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3842

rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3851

init_completion(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3852

atomic_set(&rcu_state.barrier_cpu_count, 2);

kernel/rcu/tree.c

3853

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3865

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3868

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3869

rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3875

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3876

rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3879

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3885

rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3892

if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))

kernel/rcu/tree.c

3893

complete(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3896

wait_for_completion(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3899

rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3900

rcu_seq_end(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3901

gseq = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3909

mutex_unlock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3938

unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3943

if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {

kernel/rcu/tree.c

4048

if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock))

kernel/rcu/tree.c

4061

return !!READ_ONCE(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4151

rdp->barrier_seq_snap = rcu_state.barrier_sequence;

kernel/rcu/tree.c

4152

rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4154

rdp->rcu_onl_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4250

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

4276

trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));

kernel/rcu/tree.c

4282

ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4283

WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);

kernel/rcu/tree.c

4350

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4352

raw_spin_lock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

4355

raw_spin_unlock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

4359

smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */

kernel/rcu/tree.c

4360

ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);

kernel/rcu/tree.c

4362

rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

4363

rdp->rcu_onl_gp_state = READ_ONCE(rcu_state.gp_state);

kernel/rcu/tree.c

4377

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4424

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4426

rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

4427

rdp->rcu_ofl_gp_state = READ_ONCE(rcu_state.gp_state);

kernel/rcu/tree.c

4437

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4458

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

4460

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

4475

raw_spin_unlock(&rcu_state.barrier_lock); /* irqs remain disabled. */

kernel/rcu/tree.c

4506

ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4507

WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1);

kernel/rcu/tree.c

4524

trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),

kernel/rcu/tree.c

4587

t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);

kernel/rcu/tree.c

4596

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

4597

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

4599

smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */

kernel/rcu/tree.c

4639

rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4672

rcu_state.level[i] =

kernel/rcu/tree.c

4673

rcu_state.level[i - 1] + num_rcu_lvl[i - 1];

kernel/rcu/tree.c

4680

rnp = rcu_state.level[i];

kernel/rcu/tree.c

4688

rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4689

rnp->gp_seq_needed = rcu_state.gp_seq;

kernel/rcu/tree.c

4690

rnp->completedqs = rcu_state.gp_seq;

kernel/rcu/tree.c

4704

rnp->parent = rcu_state.level[i - 1] +

kernel/rcu/tree.c

4722

init_swait_queue_head(&rcu_state.gp_wq);

kernel/rcu/tree.c

4723

init_swait_queue_head(&rcu_state.expedited_wq);

kernel/rcu/tree.c

523

return READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

535

return rcu_state.expedited_sequence;

kernel/rcu/tree.c

544

return &rcu_state.node[0];

kernel/rcu/tree.c

552

*flags = READ_ONCE(rcu_state.gp_flags);

kernel/rcu/tree.c

553

*gp_seq = rcu_seq_current(&rcu_state.gp_seq);

kernel/rcu/tree.c

560

return ((READ_ONCE(rcu_state.gp_seq) & 0xffffULL) << 40) |

kernel/rcu/tree.c

561

((READ_ONCE(rcu_state.expedited_sequence) & 0xffffffULL) << 16) |

kernel/rcu/tree.c

562

(READ_ONCE(rcu_state.gp_seq_polled) & 0xffffULL);

kernel/rcu/tree.c

834

trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));

kernel/rcu/tree.c

870

trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));

kernel/rcu/tree.c

92

static struct rcu_state rcu_state = {

kernel/rcu/tree.c

922

(time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||

kernel/rcu/tree.c

923

time_after(jiffies, rcu_state.jiffies_resched) ||

kernel/rcu/tree.c

924

rcu_state.cbovld)) {

kernel/rcu/tree.c

928

} else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {

kernel/rcu/tree.c

93

.level = { &rcu_state.node[0] },

kernel/rcu/tree.c

942

rcu_state.cbovld)) {

kernel/rcu/tree.c

955

if (time_after(jiffies, rcu_state.jiffies_resched)) {

kernel/rcu/tree.c

96

.barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),

kernel/rcu/tree.c

97

.barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock),

kernel/rcu/tree.c

995

trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),

kernel/rcu/tree_exp.h

1004

pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_polled));

kernel/rcu/tree_exp.h

210

swake_up_one(&rcu_state.expedited_wq);

kernel/rcu/tree_exp.h

22

rcu_seq_start(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

23

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

282

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done"));

kernel/rcu/tree_exp.h

311

mutex_trylock(&rcu_state.exp_mutex))

kernel/rcu/tree_exp.h

32

return rcu_seq_endval(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

331

trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,

kernel/rcu/tree_exp.h

340

trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,

kernel/rcu/tree_exp.h

343

mutex_lock(&rcu_state.exp_mutex);

kernel/rcu/tree_exp.h

346

mutex_unlock(&rcu_state.exp_mutex);

kernel/rcu/tree_exp.h

350

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start"));

kernel/rcu/tree_exp.h

40

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

41

rcu_seq_end(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

432

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("selectofl"));

kernel/rcu/tree_exp.h

509

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("reset"));

kernel/rcu/tree_exp.h

511

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("select"));

kernel/rcu/tree_exp.h

545

t = swait_event_timeout_exclusive(rcu_state.expedited_wq,

kernel/rcu/tree_exp.h

55

s = rcu_seq_snap(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

56

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap"));

kernel/rcu/tree_exp.h

567

pr_err("INFO: %s detected expedited stalls, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);

kernel/rcu/tree_exp.h

570

pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", rcu_state.name);

kernel/rcu/tree_exp.h

590

j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask),

kernel/rcu/tree_exp.h

636

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));

kernel/rcu/tree_exp.h

67

return rcu_seq_done(&rcu_state.expedited_sequence, s);

kernel/rcu/tree_exp.h

672

trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall"));

kernel/rcu/tree_exp.h

697

mutex_lock(&rcu_state.exp_wake_mutex);

kernel/rcu/tree_exp.h

699

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end"));

kernel/rcu/tree_exp.h

712

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake"));

kernel/rcu/tree_exp.h

713

mutex_unlock(&rcu_state.exp_wake_mutex);

kernel/rcu/tree_exp.h

83

int ncpus = smp_load_acquire(&rcu_state.ncpus); /* Order vs. locking. */

kernel/rcu/tree_exp.h

88

if (likely(ncpus == rcu_state.ncpus_snap))

kernel/rcu/tree_exp.h

90

rcu_state.ncpus_snap = ncpus;

kernel/rcu/tree_exp.h

943

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

944

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

948

rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT);

kernel/rcu/tree_exp.h

980

mutex_unlock(&rcu_state.exp_mutex);

kernel/rcu/tree_nocb.h

1090

mutex_lock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1101

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1158

mutex_lock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1169

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1187

if (!mutex_trylock(&rcu_state.nocb_mutex))

kernel/rcu/tree_nocb.h

1197

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1215

if (!mutex_trylock(&rcu_state.nocb_mutex)) {

kernel/rcu/tree_nocb.h

1256

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1275

!rcu_state.nocb_is_setup && !cpumask)

kernel/rcu/tree_nocb.h

1287

rcu_state.nocb_is_setup = true;

kernel/rcu/tree_nocb.h

1290

if (!rcu_state.nocb_is_setup)

kernel/rcu/tree_nocb.h

1354

if (!rcu_scheduler_fully_active || !rcu_state.nocb_is_setup)

kernel/rcu/tree_nocb.h

1380

"rcuo%c/%d", rcu_state.abbr, cpu);

kernel/rcu/tree_nocb.h

1404

mutex_lock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1409

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

211

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

224

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));

kernel/rcu/tree_nocb.h

300

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);

kernel/rcu/tree_nocb.h

449

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

468

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

499

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));

kernel/rcu/tree_nocb.h

514

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

518

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

543

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

561

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

572

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));

kernel/rcu/tree_nocb.h

618

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));

kernel/rcu/tree_nocb.h

621

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));

kernel/rcu/tree_nocb.h

670

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));

kernel/rcu/tree_nocb.h

68

rcu_state.nocb_is_setup = true;

kernel/rcu/tree_nocb.h

697

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

727

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

766

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));

kernel/rcu/tree_nocb.h

879

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));

kernel/rcu/tree_nocb.h

907

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));

kernel/rcu/tree_nocb.h

954

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));

kernel/rcu/tree_nocb.h

966

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));

kernel/rcu/tree_plugin.h

1272

(!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld ||

kernel/rcu/tree_plugin.h

1355

time_before(jiffies, READ_ONCE(rcu_state.gp_start) + HZ)))

kernel/rcu/tree_plugin.h

28

!(lockdep_is_held(&rcu_state.barrier_mutex) ||

kernel/rcu/tree_plugin.h

31

lockdep_is_held(&rcu_state.nocb_mutex) ||

kernel/rcu/tree_plugin.h

349

trace_rcu_preempt_task(rcu_state.name,

kernel/rcu/tree_plugin.h

417

if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && rcu_state.gp_kthread)

kernel/rcu/tree_plugin.h

833

time_after(jiffies, rcu_state.gp_start + HZ))

kernel/rcu/tree_plugin.h

917

if (irqs_disabled() || in_atomic_preempt_off() || !rcu_state.gp_kthread)

kernel/rcu/tree_stall.h

1013

!smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.

kernel/rcu/tree_stall.h

1016

if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1017

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1026

time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1027

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1040

time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1041

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1071

__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start)));

kernel/rcu/tree_stall.h

1075

__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end)));

kernel/rcu/tree_stall.h

192

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 3);

kernel/rcu/tree_stall.h

193

WRITE_ONCE(rcu_state.jiffies_stall, ULONG_MAX);

kernel/rcu/tree_stall.h

206

WRITE_ONCE(rcu_state.gp_start, j);

kernel/rcu/tree_stall.h

209

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 0);

kernel/rcu/tree_stall.h

210

WRITE_ONCE(rcu_state.jiffies_stall, j + j1);

kernel/rcu/tree_stall.h

211

rcu_state.jiffies_resched = j + j1 / 2;

kernel/rcu/tree_stall.h

212

rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree_stall.h

233

j = READ_ONCE(rcu_state.jiffies_kick_kthreads);

kernel/rcu/tree_stall.h

234

if (time_after(jiffies, j) && rcu_state.gp_kthread &&

kernel/rcu/tree_stall.h

235

(rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {

kernel/rcu/tree_stall.h

237

rcu_state.name);

kernel/rcu/tree_stall.h

239

wake_up_process(rcu_state.gp_kthread);

kernel/rcu/tree_stall.h

240

WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);

kernel/rcu/tree_stall.h

405

if (gp_seq != data_race(rcu_state.gp_seq)) {

kernel/rcu/tree_stall.h

450

unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity);

kernel/rcu/tree_stall.h

535

ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);

kernel/rcu/tree_stall.h

561

data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,

kernel/rcu/tree_stall.h

572

struct task_struct *gpk = rcu_state.gp_kthread;

kernel/rcu/tree_stall.h

578

rcu_state.name, j,

kernel/rcu/tree_stall.h

579

(long)rcu_seq_current(&rcu_state.gp_seq),

kernel/rcu/tree_stall.h

580

data_race(READ_ONCE(rcu_state.gp_flags)),

kernel/rcu/tree_stall.h

581

gp_state_getname(rcu_state.gp_state),

kernel/rcu/tree_stall.h

582

data_race(READ_ONCE(rcu_state.gp_state)),

kernel/rcu/tree_stall.h

587

pr_err("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name);

kernel/rcu/tree_stall.h

604

struct task_struct *gpk = rcu_state.gp_kthread;

kernel/rcu/tree_stall.h

613

gp_state = smp_load_acquire(&rcu_state.gp_state);

kernel/rcu/tree_stall.h

614

jiffies_fqs = READ_ONCE(rcu_state.jiffies_force_qs);

kernel/rcu/tree_stall.h

621

rcu_state.name, (jiffies - jiffies_fqs),

kernel/rcu/tree_stall.h

622

(long)rcu_seq_current(&rcu_state.gp_seq),

kernel/rcu/tree_stall.h

623

data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read

kernel/rcu/tree_stall.h

655

trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected"));

kernel/rcu/tree_stall.h

656

pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);

kernel/rcu/tree_stall.h

674

(long)rcu_seq_current(&rcu_state.gp_seq), totqlen,

kernel/rcu/tree_stall.h

675

data_race(rcu_state.n_online_cpus)); // Diagnostic read

kernel/rcu/tree_stall.h

683

if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {

kernel/rcu/tree_stall.h

687

gpa = data_race(READ_ONCE(rcu_state.gp_activity));

kernel/rcu/tree_stall.h

689

rcu_state.name, j - gpa, j, gpa,

kernel/rcu/tree_stall.h

695

if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))

kernel/rcu/tree_stall.h

696

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree_stall.h

731

trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected"));

kernel/rcu/tree_stall.h

732

pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);

kernel/rcu/tree_stall.h

740

(long)rcu_seq_current(&rcu_state.gp_seq), totqlen,

kernel/rcu/tree_stall.h

741

data_race(rcu_state.n_online_cpus)); // Diagnostic read

kernel/rcu/tree_stall.h

750

if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))

kernel/rcu/tree_stall.h

751

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree_stall.h

795

if (READ_ONCE(rcu_state.nr_fqs_jiffies_stall) > 0)

kernel/rcu/tree_stall.h

818

gs1 = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree_stall.h

820

js = READ_ONCE(rcu_state.jiffies_stall);

kernel/rcu/tree_stall.h

822

gps = READ_ONCE(rcu_state.gp_start);

kernel/rcu/tree_stall.h

824

gs2 = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree_stall.h

835

cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {

kernel/rcu/tree_stall.h

850

pr_err("INFO: %s detected stall, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);

kernel/rcu/tree_stall.h

862

if (READ_ONCE(rcu_state.jiffies_stall) == jn) {

kernel/rcu/tree_stall.h

864

WRITE_ONCE(rcu_state.jiffies_stall, jn);

kernel/rcu/tree_stall.h

945

struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);

kernel/rcu/tree_stall.h

948

ja = j - data_race(READ_ONCE(rcu_state.gp_activity));

kernel/rcu/tree_stall.h

949

jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity));

kernel/rcu/tree_stall.h

950

js = j - data_race(READ_ONCE(rcu_state.gp_start));

kernel/rcu/tree_stall.h

951

jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time));

kernel/rcu/tree_stall.h

953

rcu_state.name, gp_state_getname(rcu_state.gp_state),

kernel/rcu/tree_stall.h

954

data_race(READ_ONCE(rcu_state.gp_state)),

kernel/rcu/tree_stall.h

956

js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)),

kernel/rcu/tree_stall.h

957

(long)data_race(READ_ONCE(rcu_state.gp_seq)),

kernel/rcu/tree_stall.h

959

data_race(READ_ONCE(rcu_state.gp_max)),

kernel/rcu/tree_stall.h

960

data_race(READ_ONCE(rcu_state.gp_flags)));

kernel/rcu/tree_stall.h

962

if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) &&

kernel/rcu/tree_stall.h

981

ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),

lib/stackdepot.c

413

if (!poll_state_synchronize_rcu(stack->rcu_state))

lib/stackdepot.c

551

stack->rcu_state = get_state_synchronize_rcu();

net/sched/sch_generic.c

1554

cond_synchronize_rcu(miniq->rcu_state);

net/sched/sch_generic.c

1555

else if (!poll_state_synchronize_rcu(miniq->rcu_state))

net/sched/sch_generic.c

1567

miniq_old->rcu_state = start_poll_synchronize_rcu();

net/sched/sch_generic.c

1586

miniqp->miniq1.rcu_state = get_state_synchronize_rcu();

net/sched/sch_generic.c

1587

miniqp->miniq2.rcu_state = miniqp->miniq1.rcu_state;