Symbol: local_storage
include/linux/bpf_local_storage.h
143
void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,
include/linux/bpf_local_storage.h
148
bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
include/linux/bpf_local_storage.h
156
sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],
include/linux/bpf_local_storage.h
162
hlist_for_each_entry_rcu(selem, &local_storage->list, snode,
include/linux/bpf_local_storage.h
170
__bpf_local_storage_insert_cache(local_storage, smap, selem);
include/linux/bpf_local_storage.h
174
u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage);
include/linux/bpf_local_storage.h
184
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
include/linux/bpf_local_storage.h
190
struct bpf_local_storage *local_storage,
include/linux/bpf_local_storage.h
80
struct bpf_local_storage __rcu *local_storage;
kernel/bpf/bpf_cgrp_storage.c
23
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_cgrp_storage.c
26
local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);
kernel/bpf/bpf_cgrp_storage.c
27
if (!local_storage)
kernel/bpf/bpf_cgrp_storage.c
30
bpf_local_storage_destroy(local_storage);
kernel/bpf/bpf_inode_storage.c
58
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_inode_storage.c
67
local_storage = rcu_dereference(bsb->storage);
kernel/bpf/bpf_inode_storage.c
68
if (!local_storage)
kernel/bpf/bpf_inode_storage.c
71
bpf_local_storage_destroy(local_storage);
kernel/bpf/bpf_local_storage.c
108
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
114
local_storage = container_of(rcu, struct bpf_local_storage, rcu);
kernel/bpf/bpf_local_storage.c
115
kfree(local_storage);
kernel/bpf/bpf_local_storage.c
119
static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
123
kfree_rcu(local_storage, rcu);
kernel/bpf/bpf_local_storage.c
125
call_rcu_tasks_trace(&local_storage->rcu,
kernel/bpf/bpf_local_storage.c
131
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
133
local_storage = container_of(rcu, struct bpf_local_storage, rcu);
kernel/bpf/bpf_local_storage.c
134
kfree_nolock(local_storage);
kernel/bpf/bpf_local_storage.c
146
static void bpf_local_storage_free(struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
149
if (!local_storage)
kernel/bpf/bpf_local_storage.c
152
if (!local_storage->use_kmalloc_nolock) {
kernel/bpf/bpf_local_storage.c
153
__bpf_local_storage_free(local_storage, reuse_now);
kernel/bpf/bpf_local_storage.c
158
call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu);
kernel/bpf/bpf_local_storage.c
162
call_rcu_tasks_trace(&local_storage->rcu,
kernel/bpf/bpf_local_storage.c
22
struct bpf_local_storage *local_storage)
kernel/bpf/bpf_local_storage.c
24
return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)];
kernel/bpf/bpf_local_storage.c
265
struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
268
void *owner = local_storage->owner;
kernel/bpf/bpf_local_storage.c
271
if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==
kernel/bpf/bpf_local_storage.c
273
RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);
kernel/bpf/bpf_local_storage.c
275
if (pin_owner && !refcount_inc_not_zero(&local_storage->owner_refcnt))
kernel/bpf/bpf_local_storage.c
278
uncharge += free_local_storage ? sizeof(*local_storage) : 0;
kernel/bpf/bpf_local_storage.c
279
mem_uncharge(smap, local_storage->owner, uncharge);
kernel/bpf/bpf_local_storage.c
280
local_storage->mem_charge -= uncharge;
kernel/bpf/bpf_local_storage.c
283
local_storage->owner = NULL;
kernel/bpf/bpf_local_storage.c
290
refcount_dec(&local_storage->owner_refcnt);
kernel/bpf/bpf_local_storage.c
297
static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
307
&local_storage->list);
kernel/bpf/bpf_local_storage.c
309
bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage,
kernel/bpf/bpf_local_storage.c
319
void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
325
local_storage->mem_charge += smap->elem_size;
kernel/bpf/bpf_local_storage.c
327
RCU_INIT_POINTER(selem->local_storage, local_storage);
kernel/bpf/bpf_local_storage.c
328
hlist_add_head_rcu(&selem->snode, &local_storage->list);
kernel/bpf/bpf_local_storage.c
333
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
339
local_storage = rcu_dereference_check(selem->local_storage,
kernel/bpf/bpf_local_storage.c
342
b = select_bucket(smap, local_storage);
kernel/bpf/bpf_local_storage.c
359
struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
366
b = select_bucket(smap, local_storage);
kernel/bpf/bpf_local_storage.c
390
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
400
local_storage = rcu_dereference_check(selem->local_storage,
kernel/bpf/bpf_local_storage.c
403
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
417
local_storage, selem, &selem_free_list);
kernel/bpf/bpf_local_storage.c
420
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
425
bpf_local_storage_free(local_storage, false);
kernel/bpf/bpf_local_storage.c
439
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
444
local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held());
kernel/bpf/bpf_local_storage.c
448
b = b ? : select_bucket(smap, local_storage);
kernel/bpf/bpf_local_storage.c
477
if (local_storage) {
kernel/bpf/bpf_local_storage.c
478
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
482
&local_storage->list);
kernel/bpf/bpf_local_storage.c
490
selem, smap, local_storage,
kernel/bpf/bpf_local_storage.c
495
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
498
RCU_INIT_POINTER(selem->local_storage, NULL);
kernel/bpf/bpf_local_storage.c
515
bpf_local_storage_free(local_storage, true);
kernel/bpf/bpf_local_storage.c
518
void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,
kernel/bpf/bpf_local_storage.c
530
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
535
rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));
kernel/bpf/bpf_local_storage.c
536
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
635
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_local_storage.c
651
local_storage = rcu_dereference_check(*owner_storage(smap, owner),
kernel/bpf/bpf_local_storage.c
653
if (!local_storage || hlist_empty(&local_storage->list)) {
kernel/bpf/bpf_local_storage.c
679
bpf_local_storage_lookup(local_storage, smap, false);
kernel/bpf/bpf_local_storage.c
697
err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
702
if (unlikely(hlist_empty(&local_storage->list))) {
kernel/bpf/bpf_local_storage.c
712
old_sdata = bpf_local_storage_lookup(local_storage, smap, false);
kernel/bpf/bpf_local_storage.c
724
b = select_bucket(smap, local_storage);
kernel/bpf/bpf_local_storage.c
735
bpf_selem_link_storage_nolock(local_storage, selem);
kernel/bpf/bpf_local_storage.c
740
bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),
kernel/bpf/bpf_local_storage.c
746
raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);
kernel/bpf/bpf_local_storage.c
819
u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage)
kernel/bpf/bpf_local_storage.c
832
hlist_for_each_entry_rcu(selem, &local_storage->list, snode)
kernel/bpf/bpf_local_storage.c
835
if (!refcount_dec_and_test(&local_storage->owner_refcnt)) {
kernel/bpf/bpf_local_storage.c
836
while (refcount_read(&local_storage->owner_refcnt))
kernel/bpf/bpf_local_storage.c
845
return local_storage->mem_charge;
kernel/bpf/bpf_task_storage.c
48
struct bpf_local_storage *local_storage;
kernel/bpf/bpf_task_storage.c
52
local_storage = rcu_dereference(task->bpf_storage);
kernel/bpf/bpf_task_storage.c
53
if (!local_storage)
kernel/bpf/bpf_task_storage.c
56
bpf_local_storage_destroy(local_storage);
net/core/bpf_sk_storage.c
215
rcu_dereference(copy_selem->local_storage);
net/core/bpf_sk_storage.c
731
sk_storage = rcu_dereference(selem->local_storage);
net/core/bpf_sk_storage.c
744
sk_storage = rcu_dereference(selem->local_storage);
net/core/bpf_sk_storage.c
811
sk_storage = rcu_dereference(selem->local_storage);
tools/testing/selftests/bpf/prog_tests/test_local_storage.c
26
static int run_self_unlink(struct local_storage *skel, const char *rm_path)
tools/testing/selftests/bpf/prog_tests/test_local_storage.c
95
struct local_storage *skel = NULL;
tools/testing/selftests/bpf/progs/local_storage.c
106
struct local_storage *storage;
tools/testing/selftests/bpf/progs/local_storage.c
136
struct local_storage *storage;
tools/testing/selftests/bpf/progs/local_storage.c
173
struct local_storage *storage;
tools/testing/selftests/bpf/progs/local_storage.c
196
struct local_storage *storage;
tools/testing/selftests/bpf/progs/local_storage.c
30
__type(value, struct local_storage);
tools/testing/selftests/bpf/progs/local_storage.c
37
__type(value, struct local_storage);
tools/testing/selftests/bpf/progs/local_storage.c
44
__type(value, struct local_storage);
tools/testing/selftests/bpf/progs/local_storage.c
51
__type(value, struct local_storage);
tools/testing/selftests/bpf/progs/local_storage.c
58
__type(value, struct local_storage);
tools/testing/selftests/bpf/progs/local_storage.c
65
struct local_storage *storage;