include/linux/stackdepot.h

73

unsigned long rcu_state; /* RCU cookie */

include/net/sch_generic.h

1499

unsigned long rcu_state;

kernel/rcu/rcu.h

391

#define rcu_first_leaf_node() (rcu_state.level[rcu_num_lvls - 1])

kernel/rcu/rcu.h

397

#define rcu_is_last_leaf_node(rnp) ((rnp) == &rcu_state.node[rcu_num_nodes - 1])

kernel/rcu/rcu.h

408

_rcu_for_each_node_breadth_first(&rcu_state, rnp)

kernel/rcu/rcu.h

420

(rnp) < &rcu_state.node[rcu_num_nodes]; (rnp)++)

kernel/rcu/tree.c

100

.exp_mutex = __MUTEX_INITIALIZER(rcu_state.exp_mutex),

kernel/rcu/tree.c

101

.exp_wake_mutex = __MUTEX_INITIALIZER(rcu_state.exp_wake_mutex),

kernel/rcu/tree.c

103

.srs_cleanup_work = __WORK_INITIALIZER(rcu_state.srs_cleanup_work,

kernel/rcu/tree.c

1065

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

1066

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

1067

if (!READ_ONCE(rcu_state.gp_kthread)) {

kernel/rcu/tree.c

107

.nocb_mutex = __MUTEX_INITIALIZER(rcu_state.nocb_mutex),

kernel/rcu/tree.c

1071

trace_rcu_grace_period(rcu_state.name, data_race(rcu_state.gp_seq), TPS("newreq"));

kernel/rcu/tree.c

1118

struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);

kernel/rcu/tree.c

1121

!READ_ONCE(rcu_state.gp_flags) || !t)

kernel/rcu/tree.c

1123

WRITE_ONCE(rcu_state.gp_wake_time, jiffies);

kernel/rcu/tree.c

1124

WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));

kernel/rcu/tree.c

1125

swake_up_one(&rcu_state.gp_wq);

kernel/rcu/tree.c

1164

gp_seq_req = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

1170

trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccWaitCB"));

kernel/rcu/tree.c

1172

trace_rcu_grace_period(rcu_state.name, gp_seq_req, TPS("AccReadyCB"));

kernel/rcu/tree.c

1193

c = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

1287

trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));

kernel/rcu/tree.c

1303

trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));

kernel/rcu/tree.c

1370

!(rcu_seq_ctr(rcu_state.gp_seq) % (rcu_num_nodes * PER_RCU_NODE_PERIOD * delay)))

kernel/rcu/tree.c

1417

if (!rcu_seq_state(rcu_state.gp_seq_polled))

kernel/rcu/tree.c

1418

rcu_seq_start(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

1421

*snap = rcu_state.gp_seq_polled;

kernel/rcu/tree.c

1435

if (*snap && *snap == rcu_state.gp_seq_polled) {

kernel/rcu/tree.c

1436

rcu_seq_end(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

1437

rcu_state.gp_seq_polled_snap = 0;

kernel/rcu/tree.c

1438

rcu_state.gp_seq_polled_exp_snap = 0;

kernel/rcu/tree.c

1606

return &(rcu_state.srs_wait_nodes)[0].node <= node &&

kernel/rcu/tree.c

1607

node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;

kernel/rcu/tree.c

1616

sr_wn = &(rcu_state.srs_wait_nodes)[i];

kernel/rcu/tree.c

1680

done = smp_load_acquire(&rcu_state.srs_done_tail);

kernel/rcu/tree.c

1707

atomic_dec_return_release(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1718

wait_tail = rcu_state.srs_wait_tail;

kernel/rcu/tree.c

1722

rcu_state.srs_wait_tail = NULL;

kernel/rcu/tree.c

1723

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);

kernel/rcu/tree.c

1750

!atomic_read_acquire(&rcu_state.srs_cleanups_pending)) {

kernel/rcu/tree.c

1756

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail);

kernel/rcu/tree.c

1757

smp_store_release(&rcu_state.srs_done_tail, wait_tail);

kernel/rcu/tree.c

1765

atomic_inc(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1766

if (!queue_work(sync_wq, &rcu_state.srs_cleanup_work))

kernel/rcu/tree.c

1767

atomic_dec(&rcu_state.srs_cleanups_pending);

kernel/rcu/tree.c

1780

first = READ_ONCE(rcu_state.srs_next.first);

kernel/rcu/tree.c

1792

llist_add(wait_head, &rcu_state.srs_next);

kernel/rcu/tree.c

1799

WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL);

kernel/rcu/tree.c

1800

rcu_state.srs_wait_tail = wait_head;

kernel/rcu/tree.c

1801

ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);

kernel/rcu/tree.c

1826

llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);

kernel/rcu/tree.c

1842

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

1844

if (!rcu_state.gp_flags) {

kernel/rcu/tree.c

1849

WRITE_ONCE(rcu_state.gp_flags, 0); /* Clear all flags: New GP. */

kernel/rcu/tree.c

1868

old_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

1881

rcu_seq_start(&rcu_state.gp_seq);

kernel/rcu/tree.c

1884

rcu_seq_done_exact(&old_gp_seq, rcu_seq_snap(&rcu_state.gp_seq)));

kernel/rcu/tree.c

1886

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

kernel/rcu/tree.c

1887

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("start"));

kernel/rcu/tree.c

1888

rcu_poll_gp_seq_start(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

1913

WRITE_ONCE(rcu_state.gp_state, RCU_GP_ONOFF);

kernel/rcu/tree.c

1921

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1927

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1964

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

1981

WRITE_ONCE(rcu_state.gp_state, RCU_GP_INIT);

kernel/rcu/tree.c

1988

WRITE_ONCE(rnp->gp_seq, rcu_state.gp_seq);

kernel/rcu/tree.c

1992

trace_rcu_grace_period_init(rcu_state.name, rnp->gp_seq,

kernel/rcu/tree.c

2008

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2042

*gfp = READ_ONCE(rcu_state.gp_flags);

kernel/rcu/tree.c

2058

int nr_fqs = READ_ONCE(rcu_state.nr_fqs_jiffies_stall);

kernel/rcu/tree.c

2061

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2062

WRITE_ONCE(rcu_state.n_force_qs, rcu_state.n_force_qs + 1);

kernel/rcu/tree.c

2068

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree.c

2071

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, --nr_fqs);

kernel/rcu/tree.c

2082

if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {

kernel/rcu/tree.c

2084

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & ~RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

2101

if (rcu_state.cbovld)

kernel/rcu/tree.c

2105

if (rcu_state.cbovld) {

kernel/rcu/tree.c

2110

if (!ret || time_before(jiffies + j, rcu_state.jiffies_force_qs)) {

kernel/rcu/tree.c

2111

WRITE_ONCE(rcu_state.jiffies_force_qs, jiffies + j);

kernel/rcu/tree.c

2117

WRITE_ONCE(rcu_state.jiffies_kick_kthreads,

kernel/rcu/tree.c

2120

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2122

WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_FQS);

kernel/rcu/tree.c

2123

(void)swait_event_idle_timeout_exclusive(rcu_state.gp_wq,

kernel/rcu/tree.c

2126

WRITE_ONCE(rcu_state.gp_state, RCU_GP_DOING_FQS);

kernel/rcu/tree.c

2141

if (!time_after(rcu_state.jiffies_force_qs, jiffies) ||

kernel/rcu/tree.c

2143

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2149

gf = rcu_state.cbovld ? RCU_GP_FLAG_OVLD : 0;

kernel/rcu/tree.c

2151

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2154

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2160

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2162

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2166

if (time_after(jiffies, rcu_state.jiffies_force_qs))

kernel/rcu/tree.c

2169

j = rcu_state.jiffies_force_qs - j;

kernel/rcu/tree.c

2189

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2191

rcu_state.gp_end = jiffies;

kernel/rcu/tree.c

2192

gp_duration = rcu_state.gp_end - rcu_state.gp_start;

kernel/rcu/tree.c

2193

if (gp_duration > rcu_state.gp_max)

kernel/rcu/tree.c

2194

rcu_state.gp_max = gp_duration;

kernel/rcu/tree.c

2204

rcu_poll_gp_seq_end(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

2216

new_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

222

return rcu_seq_state(rcu_seq_current(&rcu_state.gp_seq));

kernel/rcu/tree.c

2241

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2248

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("end"));

kernel/rcu/tree.c

2249

rcu_seq_end(&rcu_state.gp_seq);

kernel/rcu/tree.c

2250

ASSERT_EXCLUSIVE_WRITER(rcu_state.gp_seq);

kernel/rcu/tree.c

2251

WRITE_ONCE(rcu_state.gp_state, RCU_GP_IDLE);

kernel/rcu/tree.c

2273

WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

2274

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

2275

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));

kernel/rcu/tree.c

2284

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags & RCU_GP_FLAG_INIT);

kernel/rcu/tree.c

2306

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2308

WRITE_ONCE(rcu_state.gp_state, RCU_GP_WAIT_GPS);

kernel/rcu/tree.c

2309

swait_event_idle_exclusive(rcu_state.gp_wq,

kernel/rcu/tree.c

2310

READ_ONCE(rcu_state.gp_flags) &

kernel/rcu/tree.c

2313

WRITE_ONCE(rcu_state.gp_state, RCU_GP_DONE_GPS);

kernel/rcu/tree.c

2318

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

2320

trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,

kernel/rcu/tree.c

2328

WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANUP);

kernel/rcu/tree.c

2330

WRITE_ONCE(rcu_state.gp_state, RCU_GP_CLEANED);

kernel/rcu/tree.c

2348

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

2391

trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,

kernel/rcu/tree.c

2584

trace_rcu_batch_start(rcu_state.name,

kernel/rcu/tree.c

2586

trace_rcu_batch_end(rcu_state.name, 0,

kernel/rcu/tree.c

2620

trace_rcu_batch_start(rcu_state.name,

kernel/rcu/tree.c

2640

trace_rcu_invoke_callback(rcu_state.name, rhp);

kernel/rcu/tree.c

2681

trace_rcu_batch_end(rcu_state.name, count, !!rcl.head, need_resched(),

kernel/rcu/tree.c

2696

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

2766

rcu_state.cbovld = rcu_state.cbovldnext;

kernel/rcu/tree.c

2767

rcu_state.cbovldnext = false;

kernel/rcu/tree.c

2774

rcu_state.cbovldnext |= !!rnp->cbovldmask;

kernel/rcu/tree.c

2831

ret = (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) ||

kernel/rcu/tree.c

2844

if (READ_ONCE(rcu_state.gp_flags) & RCU_GP_FLAG_FQS) {

kernel/rcu/tree.c

2848

WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags | RCU_GP_FLAG_FQS);

kernel/rcu/tree.c

3029

trace_rcu_callback(rcu_state.name, head,

kernel/rcu/tree.c

3071

if (READ_ONCE(rcu_state.n_force_qs) == rdp->n_force_qs_snap &&

kernel/rcu/tree.c

3074

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

3310

trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));

kernel/rcu/tree.c

3336

trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));

kernel/rcu/tree.c

3401

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

3402

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_snap);

kernel/rcu/tree.c

3410

rcu_state.gp_seq += (1 << RCU_SEQ_CTR_SHIFT);

kernel/rcu/tree.c

3412

rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

3446

return rcu_seq_snap(&rcu_state.gp_seq_polled);

kernel/rcu/tree.c

3478

rgosp->rgos_norm = rcu_seq_snap(&rcu_state.gp_seq);

kernel/rcu/tree.c

3479

rgosp->rgos_exp = rcu_seq_snap(&rcu_state.expedited_sequence);

kernel/rcu/tree.c

3504

needwake = rcu_start_this_gp(rnp, rdp, rcu_seq_snap(&rcu_state.gp_seq));

kernel/rcu/tree.c

3583

rcu_seq_done_exact(&rcu_state.gp_seq_polled, oldstate)) {

kernel/rcu/tree.c

3632

rcu_seq_done_exact(&rcu_state.expedited_sequence, rgosp->rgos_exp)) {

kernel/rcu/tree.c

3718

time_before(jiffies, READ_ONCE(rcu_state.gp_start) +

kernel/rcu/tree.c

3753

trace_rcu_barrier(rcu_state.name, s, cpu,

kernel/rcu/tree.c

3754

atomic_read(&rcu_state.barrier_cpu_count), done);

kernel/rcu/tree.c

3769

unsigned long __maybe_unused s = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3772

if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {

kernel/rcu/tree.c

3774

complete(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3785

unsigned long gseq = READ_ONCE(rcu_state.barrier_sequence);

kernel/rcu/tree.c

3790

lockdep_assert_held(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3793

rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3806

atomic_inc(&rcu_state.barrier_cpu_count);

kernel/rcu/tree.c

3809

rcu_barrier_trace(TPS("IRQNQ"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3828

raw_spin_lock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3830

raw_spin_unlock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

3852

unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3857

mutex_lock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3860

if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {

kernel/rcu/tree.c

3861

rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3863

mutex_unlock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3868

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3869

rcu_seq_start(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3870

gseq = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3871

rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3880

init_completion(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3881

atomic_set(&rcu_state.barrier_cpu_count, 2);

kernel/rcu/tree.c

3882

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3894

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3897

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3898

rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3904

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3905

rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3908

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

3914

rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3921

if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))

kernel/rcu/tree.c

3922

complete(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3925

wait_for_completion(&rcu_state.barrier_completion);

kernel/rcu/tree.c

3928

rcu_barrier_trace(TPS("Inc2"), -1, rcu_state.barrier_sequence);

kernel/rcu/tree.c

3929

rcu_seq_end(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3930

gseq = rcu_state.barrier_sequence;

kernel/rcu/tree.c

3938

mutex_unlock(&rcu_state.barrier_mutex);

kernel/rcu/tree.c

3967

unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);

kernel/rcu/tree.c

3972

if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {

kernel/rcu/tree.c

4077

if (rcu_rdp_cpu_online(rdp) || arch_spin_is_locked(&rcu_state.ofl_lock))

kernel/rcu/tree.c

4090

return !!READ_ONCE(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4180

rdp->barrier_seq_snap = rcu_state.barrier_sequence;

kernel/rcu/tree.c

4181

rdp->rcu_ofl_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4183

rdp->rcu_onl_gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4279

rdp->n_force_qs_snap = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree.c

4305

trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));

kernel/rcu/tree.c

4311

ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4312

WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus + 1);

kernel/rcu/tree.c

4379

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4381

raw_spin_lock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

4384

raw_spin_unlock(&rcu_state.barrier_lock);

kernel/rcu/tree.c

4388

smp_store_release(&rcu_state.ncpus, rcu_state.ncpus + newcpu); /* ^^^ */

kernel/rcu/tree.c

4389

ASSERT_EXCLUSIVE_WRITER(rcu_state.ncpus);

kernel/rcu/tree.c

4391

rdp->rcu_onl_gp_seq = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

4392

rdp->rcu_onl_gp_state = READ_ONCE(rcu_state.gp_state);

kernel/rcu/tree.c

4406

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4453

arch_spin_lock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4455

rdp->rcu_ofl_gp_seq = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

4456

rdp->rcu_ofl_gp_state = READ_ONCE(rcu_state.gp_state);

kernel/rcu/tree.c

4466

arch_spin_unlock(&rcu_state.ofl_lock);

kernel/rcu/tree.c

4487

raw_spin_lock_irqsave(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

4489

raw_spin_unlock_irqrestore(&rcu_state.barrier_lock, flags);

kernel/rcu/tree.c

4504

raw_spin_unlock(&rcu_state.barrier_lock); /* irqs remain disabled. */

kernel/rcu/tree.c

4535

ASSERT_EXCLUSIVE_WRITER(rcu_state.n_online_cpus);

kernel/rcu/tree.c

4536

WRITE_ONCE(rcu_state.n_online_cpus, rcu_state.n_online_cpus - 1);

kernel/rcu/tree.c

4553

trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),

kernel/rcu/tree.c

4616

t = kthread_create(rcu_gp_kthread, NULL, "%s", rcu_state.name);

kernel/rcu/tree.c

4625

WRITE_ONCE(rcu_state.gp_activity, jiffies);

kernel/rcu/tree.c

4626

WRITE_ONCE(rcu_state.gp_req_activity, jiffies);

kernel/rcu/tree.c

4628

smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */

kernel/rcu/tree.c

4668

rnp->gp_seq_needed = rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4701

rcu_state.level[i] =

kernel/rcu/tree.c

4702

rcu_state.level[i - 1] + num_rcu_lvl[i - 1];

kernel/rcu/tree.c

4709

rnp = rcu_state.level[i];

kernel/rcu/tree.c

4717

rnp->gp_seq = rcu_state.gp_seq;

kernel/rcu/tree.c

4718

rnp->gp_seq_needed = rcu_state.gp_seq;

kernel/rcu/tree.c

4719

rnp->completedqs = rcu_state.gp_seq;

kernel/rcu/tree.c

4733

rnp->parent = rcu_state.level[i - 1] +

kernel/rcu/tree.c

4751

init_swait_queue_head(&rcu_state.gp_wq);

kernel/rcu/tree.c

4752

init_swait_queue_head(&rcu_state.expedited_wq);

kernel/rcu/tree.c

523

return READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree.c

535

return rcu_state.expedited_sequence;

kernel/rcu/tree.c

544

return &rcu_state.node[0];

kernel/rcu/tree.c

552

*flags = READ_ONCE(rcu_state.gp_flags);

kernel/rcu/tree.c

553

*gp_seq = rcu_seq_current(&rcu_state.gp_seq);

kernel/rcu/tree.c

560

return ((READ_ONCE(rcu_state.gp_seq) & 0xffffULL) << 40) |

kernel/rcu/tree.c

561

((READ_ONCE(rcu_state.expedited_sequence) & 0xffffffULL) << 16) |

kernel/rcu/tree.c

562

(READ_ONCE(rcu_state.gp_seq_polled) & 0xffffULL);

kernel/rcu/tree.c

834

trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));

kernel/rcu/tree.c

870

trace_rcu_fqs(rcu_state.name, rdp->gp_seq, rdp->cpu, TPS("dti"));

kernel/rcu/tree.c

92

static struct rcu_state rcu_state = {

kernel/rcu/tree.c

922

(time_after(jiffies, rcu_state.gp_start + jtsq * 2) ||

kernel/rcu/tree.c

923

time_after(jiffies, rcu_state.jiffies_resched) ||

kernel/rcu/tree.c

924

rcu_state.cbovld)) {

kernel/rcu/tree.c

928

} else if (time_after(jiffies, rcu_state.gp_start + jtsq)) {

kernel/rcu/tree.c

93

.level = { &rcu_state.node[0] },

kernel/rcu/tree.c

942

rcu_state.cbovld)) {

kernel/rcu/tree.c

955

if (time_after(jiffies, rcu_state.jiffies_resched)) {

kernel/rcu/tree.c

96

.barrier_mutex = __MUTEX_INITIALIZER(rcu_state.barrier_mutex),

kernel/rcu/tree.c

97

.barrier_lock = __RAW_SPIN_LOCK_UNLOCKED(rcu_state.barrier_lock),

kernel/rcu/tree.c

992

trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),

kernel/rcu/tree_exp.h

1004

pr_info("%s: i = %d s = %lx gp_seq_polled = %lx\n", __func__, i, s, READ_ONCE(rcu_state.gp_seq_polled));

kernel/rcu/tree_exp.h

210

swake_up_one(&rcu_state.expedited_wq);

kernel/rcu/tree_exp.h

22

rcu_seq_start(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

23

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

282

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("done"));

kernel/rcu/tree_exp.h

311

mutex_trylock(&rcu_state.exp_mutex))

kernel/rcu/tree_exp.h

32

return rcu_seq_endval(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

331

trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,

kernel/rcu/tree_exp.h

340

trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,

kernel/rcu/tree_exp.h

343

mutex_lock(&rcu_state.exp_mutex);

kernel/rcu/tree_exp.h

346

mutex_unlock(&rcu_state.exp_mutex);

kernel/rcu/tree_exp.h

350

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("start"));

kernel/rcu/tree_exp.h

40

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

41

rcu_seq_end(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

432

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("selectofl"));

kernel/rcu/tree_exp.h

509

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("reset"));

kernel/rcu/tree_exp.h

511

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("select"));

kernel/rcu/tree_exp.h

545

t = swait_event_timeout_exclusive(rcu_state.expedited_wq,

kernel/rcu/tree_exp.h

55

s = rcu_seq_snap(&rcu_state.expedited_sequence);

kernel/rcu/tree_exp.h

56

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("snap"));

kernel/rcu/tree_exp.h

567

pr_err("INFO: %s detected expedited stalls, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);

kernel/rcu/tree_exp.h

570

pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {", rcu_state.name);

kernel/rcu/tree_exp.h

590

j - jiffies_start, rcu_state.expedited_sequence, data_race(rnp_root->expmask),

kernel/rcu/tree_exp.h

636

trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));

kernel/rcu/tree_exp.h

67

return rcu_seq_done(&rcu_state.expedited_sequence, s);

kernel/rcu/tree_exp.h

672

trace_rcu_stall_warning(rcu_state.name, TPS("ExpeditedStall"));

kernel/rcu/tree_exp.h

697

mutex_lock(&rcu_state.exp_wake_mutex);

kernel/rcu/tree_exp.h

699

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("end"));

kernel/rcu/tree_exp.h

712

trace_rcu_exp_grace_period(rcu_state.name, s, TPS("endwake"));

kernel/rcu/tree_exp.h

713

mutex_unlock(&rcu_state.exp_wake_mutex);

kernel/rcu/tree_exp.h

83

int ncpus = smp_load_acquire(&rcu_state.ncpus); /* Order vs. locking. */

kernel/rcu/tree_exp.h

88

if (likely(ncpus == rcu_state.ncpus_snap))

kernel/rcu/tree_exp.h

90

rcu_state.ncpus_snap = ncpus;

kernel/rcu/tree_exp.h

943

rcu_poll_gp_seq_start_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

944

rcu_poll_gp_seq_end_unlocked(&rcu_state.gp_seq_polled_exp_snap);

kernel/rcu/tree_exp.h

948

rcu_state.expedited_sequence += (1 << RCU_SEQ_CTR_SHIFT);

kernel/rcu/tree_exp.h

980

mutex_unlock(&rcu_state.exp_mutex);

kernel/rcu/tree_nocb.h

1158

mutex_lock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1182

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1210

if (!mutex_trylock(&rcu_state.nocb_mutex))

kernel/rcu/tree_nocb.h

1220

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1238

if (!mutex_trylock(&rcu_state.nocb_mutex)) {

kernel/rcu/tree_nocb.h

1279

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1298

!rcu_state.nocb_is_setup && !cpumask)

kernel/rcu/tree_nocb.h

1310

rcu_state.nocb_is_setup = true;

kernel/rcu/tree_nocb.h

1313

if (!rcu_state.nocb_is_setup)

kernel/rcu/tree_nocb.h

1377

if (!rcu_scheduler_fully_active || !rcu_state.nocb_is_setup)

kernel/rcu/tree_nocb.h

1403

"rcuo%c/%d", rcu_state.abbr, cpu);

kernel/rcu/tree_nocb.h

1427

mutex_lock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

1432

mutex_unlock(&rcu_state.nocb_mutex);

kernel/rcu/tree_nocb.h

211

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

224

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DoWake"));

kernel/rcu/tree_nocb.h

300

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, reason);

kernel/rcu/tree_nocb.h

482

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

498

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

529

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("FirstBQ"));

kernel/rcu/tree_nocb.h

544

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

548

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

573

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

591

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

602

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WakeNot"));

kernel/rcu/tree_nocb.h

648

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Sleep"));

kernel/rcu/tree_nocb.h

651

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("EndSleep"));

kernel/rcu/tree_nocb.h

68

rcu_state.nocb_is_setup = true;

kernel/rcu/tree_nocb.h

700

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Check"));

kernel/rcu/tree_nocb.h

720

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

750

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu,

kernel/rcu/tree_nocb.h

789

trace_rcu_nocb_wake(rcu_state.name, cpu, TPS("Poll"));

kernel/rcu/tree_nocb.h

902

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("WokeEmpty"));

kernel/rcu/tree_nocb.h

930

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("CBSleep"));

kernel/rcu/tree_nocb.h

977

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("DeferredWake"));

kernel/rcu/tree_nocb.h

989

trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, TPS("Timer"));

kernel/rcu/tree_plugin.h

1272

(!time_after(rnp->boost_time, jiffies) || rcu_state.cbovld ||

kernel/rcu/tree_plugin.h

1355

time_before(jiffies, READ_ONCE(rcu_state.gp_start) + HZ)))

kernel/rcu/tree_plugin.h

28

!(lockdep_is_held(&rcu_state.barrier_mutex) ||

kernel/rcu/tree_plugin.h

31

lockdep_is_held(&rcu_state.nocb_mutex) ||

kernel/rcu/tree_plugin.h

349

trace_rcu_preempt_task(rcu_state.name,

kernel/rcu/tree_plugin.h

417

if (IS_ENABLED(CONFIG_RCU_STRICT_GRACE_PERIOD) && rcu_state.gp_kthread)

kernel/rcu/tree_plugin.h

833

time_after(jiffies, rcu_state.gp_start + HZ))

kernel/rcu/tree_plugin.h

917

if (irqs_disabled() || in_atomic_preempt_off() || !rcu_state.gp_kthread)

kernel/rcu/tree_stall.h

1012

!smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.

kernel/rcu/tree_stall.h

1015

if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1016

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1025

time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1026

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1039

time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1040

time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||

kernel/rcu/tree_stall.h

1070

__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_start)));

kernel/rcu/tree_stall.h

1074

__func__, jiffies - data_race(READ_ONCE(rcu_state.gp_end)));

kernel/rcu/tree_stall.h

192

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 3);

kernel/rcu/tree_stall.h

193

WRITE_ONCE(rcu_state.jiffies_stall, ULONG_MAX);

kernel/rcu/tree_stall.h

206

WRITE_ONCE(rcu_state.gp_start, j);

kernel/rcu/tree_stall.h

209

WRITE_ONCE(rcu_state.nr_fqs_jiffies_stall, 0);

kernel/rcu/tree_stall.h

210

WRITE_ONCE(rcu_state.jiffies_stall, j + j1);

kernel/rcu/tree_stall.h

211

rcu_state.jiffies_resched = j + j1 / 2;

kernel/rcu/tree_stall.h

212

rcu_state.n_force_qs_gpstart = READ_ONCE(rcu_state.n_force_qs);

kernel/rcu/tree_stall.h

233

j = READ_ONCE(rcu_state.jiffies_kick_kthreads);

kernel/rcu/tree_stall.h

234

if (time_after(jiffies, j) && rcu_state.gp_kthread &&

kernel/rcu/tree_stall.h

235

(rcu_gp_in_progress() || READ_ONCE(rcu_state.gp_flags))) {

kernel/rcu/tree_stall.h

237

rcu_state.name);

kernel/rcu/tree_stall.h

239

wake_up_process(rcu_state.gp_kthread);

kernel/rcu/tree_stall.h

240

WRITE_ONCE(rcu_state.jiffies_kick_kthreads, j + HZ);

kernel/rcu/tree_stall.h

405

if (gp_seq != data_race(rcu_state.gp_seq)) {

kernel/rcu/tree_stall.h

450

unsigned long j = jiffies - READ_ONCE(rcu_state.gp_activity);

kernel/rcu/tree_stall.h

534

ticks_value = rcu_seq_ctr(rcu_state.gp_seq - rdp->gp_seq);

kernel/rcu/tree_stall.h

560

data_race(rcu_state.n_force_qs) - rcu_state.n_force_qs_gpstart,

kernel/rcu/tree_stall.h

571

struct task_struct *gpk = rcu_state.gp_kthread;

kernel/rcu/tree_stall.h

577

rcu_state.name, j,

kernel/rcu/tree_stall.h

578

(long)rcu_seq_current(&rcu_state.gp_seq),

kernel/rcu/tree_stall.h

579

data_race(READ_ONCE(rcu_state.gp_flags)),

kernel/rcu/tree_stall.h

580

gp_state_getname(rcu_state.gp_state),

kernel/rcu/tree_stall.h

581

data_race(READ_ONCE(rcu_state.gp_state)),

kernel/rcu/tree_stall.h

586

pr_err("\tUnless %s kthread gets sufficient CPU time, OOM is now expected behavior.\n", rcu_state.name);

kernel/rcu/tree_stall.h

603

struct task_struct *gpk = rcu_state.gp_kthread;

kernel/rcu/tree_stall.h

612

gp_state = smp_load_acquire(&rcu_state.gp_state);

kernel/rcu/tree_stall.h

613

jiffies_fqs = READ_ONCE(rcu_state.jiffies_force_qs);

kernel/rcu/tree_stall.h

620

rcu_state.name, (jiffies - jiffies_fqs),

kernel/rcu/tree_stall.h

621

(long)rcu_seq_current(&rcu_state.gp_seq),

kernel/rcu/tree_stall.h

622

data_race(READ_ONCE(rcu_state.gp_flags)), // Diagnostic read

kernel/rcu/tree_stall.h

654

trace_rcu_stall_warning(rcu_state.name, TPS("StallDetected"));

kernel/rcu/tree_stall.h

655

pr_err("INFO: %s detected stalls on CPUs/tasks:\n", rcu_state.name);

kernel/rcu/tree_stall.h

673

(long)rcu_seq_current(&rcu_state.gp_seq), totqlen,

kernel/rcu/tree_stall.h

674

data_race(rcu_state.n_online_cpus)); // Diagnostic read

kernel/rcu/tree_stall.h

682

if (rcu_seq_current(&rcu_state.gp_seq) != gp_seq) {

kernel/rcu/tree_stall.h

686

gpa = data_race(READ_ONCE(rcu_state.gp_activity));

kernel/rcu/tree_stall.h

688

rcu_state.name, j - gpa, j, gpa,

kernel/rcu/tree_stall.h

694

if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))

kernel/rcu/tree_stall.h

695

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree_stall.h

730

trace_rcu_stall_warning(rcu_state.name, TPS("SelfDetected"));

kernel/rcu/tree_stall.h

731

pr_err("INFO: %s self-detected stall on CPU\n", rcu_state.name);

kernel/rcu/tree_stall.h

739

(long)rcu_seq_current(&rcu_state.gp_seq), totqlen,

kernel/rcu/tree_stall.h

740

data_race(rcu_state.n_online_cpus)); // Diagnostic read

kernel/rcu/tree_stall.h

749

if (ULONG_CMP_GE(jiffies, READ_ONCE(rcu_state.jiffies_stall)))

kernel/rcu/tree_stall.h

750

WRITE_ONCE(rcu_state.jiffies_stall,

kernel/rcu/tree_stall.h

794

if (READ_ONCE(rcu_state.nr_fqs_jiffies_stall) > 0)

kernel/rcu/tree_stall.h

817

gs1 = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree_stall.h

819

js = READ_ONCE(rcu_state.jiffies_stall);

kernel/rcu/tree_stall.h

821

gps = READ_ONCE(rcu_state.gp_start);

kernel/rcu/tree_stall.h

823

gs2 = READ_ONCE(rcu_state.gp_seq);

kernel/rcu/tree_stall.h

834

cmpxchg(&rcu_state.jiffies_stall, js, jn) == js) {

kernel/rcu/tree_stall.h

849

pr_err("INFO: %s detected stall, but suppressed full report due to a stuck CSD-lock.\n", rcu_state.name);

kernel/rcu/tree_stall.h

861

if (READ_ONCE(rcu_state.jiffies_stall) == jn) {

kernel/rcu/tree_stall.h

863

WRITE_ONCE(rcu_state.jiffies_stall, jn);

kernel/rcu/tree_stall.h

944

struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);

kernel/rcu/tree_stall.h

947

ja = j - data_race(READ_ONCE(rcu_state.gp_activity));

kernel/rcu/tree_stall.h

948

jr = j - data_race(READ_ONCE(rcu_state.gp_req_activity));

kernel/rcu/tree_stall.h

949

js = j - data_race(READ_ONCE(rcu_state.gp_start));

kernel/rcu/tree_stall.h

950

jw = j - data_race(READ_ONCE(rcu_state.gp_wake_time));

kernel/rcu/tree_stall.h

952

rcu_state.name, gp_state_getname(rcu_state.gp_state),

kernel/rcu/tree_stall.h

953

data_race(READ_ONCE(rcu_state.gp_state)),

kernel/rcu/tree_stall.h

955

js, ja, jr, jw, (long)data_race(READ_ONCE(rcu_state.gp_wake_seq)),

kernel/rcu/tree_stall.h

956

(long)data_race(READ_ONCE(rcu_state.gp_seq)),

kernel/rcu/tree_stall.h

958

data_race(READ_ONCE(rcu_state.gp_max)),

kernel/rcu/tree_stall.h

959

data_race(READ_ONCE(rcu_state.gp_flags)));

kernel/rcu/tree_stall.h

961

if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq), READ_ONCE(rnp->gp_seq_needed)) &&

kernel/rcu/tree_stall.h

980

ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),

lib/stackdepot.c

413

if (!poll_state_synchronize_rcu(stack->rcu_state))

lib/stackdepot.c

551

stack->rcu_state = get_state_synchronize_rcu();

net/sched/sch_generic.c

1605

cond_synchronize_rcu(miniq->rcu_state);

net/sched/sch_generic.c

1606

else if (!poll_state_synchronize_rcu(miniq->rcu_state))

net/sched/sch_generic.c

1618

miniq_old->rcu_state = start_poll_synchronize_rcu();

net/sched/sch_generic.c

1637

miniqp->miniq1.rcu_state = get_state_synchronize_rcu();

net/sched/sch_generic.c

1638

miniqp->miniq2.rcu_state = miniqp->miniq1.rcu_state;