include/linux/bpf_local_storage.h

139

void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,

include/linux/bpf_local_storage.h

144

bpf_local_storage_lookup(struct bpf_local_storage *local_storage,

include/linux/bpf_local_storage.h

152

sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],

include/linux/bpf_local_storage.h

158

hlist_for_each_entry_rcu(selem, &local_storage->list, snode,

include/linux/bpf_local_storage.h

166

__bpf_local_storage_insert_cache(local_storage, smap, selem);

include/linux/bpf_local_storage.h

170

u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage);

include/linux/bpf_local_storage.h

180

void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,

include/linux/bpf_local_storage.h

186

struct bpf_local_storage *local_storage,

include/linux/bpf_local_storage.h

79

struct bpf_local_storage __rcu *local_storage;

kernel/bpf/bpf_cgrp_storage.c

23

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_cgrp_storage.c

26

local_storage = rcu_dereference(cgroup->bpf_cgrp_storage);

kernel/bpf/bpf_cgrp_storage.c

27

if (!local_storage)

kernel/bpf/bpf_cgrp_storage.c

30

bpf_local_storage_destroy(local_storage);

kernel/bpf/bpf_inode_storage.c

58

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_inode_storage.c

67

local_storage = rcu_dereference(bsb->storage);

kernel/bpf/bpf_inode_storage.c

68

if (!local_storage)

kernel/bpf/bpf_inode_storage.c

71

bpf_local_storage_destroy(local_storage);

kernel/bpf/bpf_local_storage.c

101

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_local_storage.c

107

local_storage = container_of(rcu, struct bpf_local_storage, rcu);

kernel/bpf/bpf_local_storage.c

108

kfree(local_storage);

kernel/bpf/bpf_local_storage.c

111

static void bpf_local_storage_free(struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

114

if (!local_storage)

kernel/bpf/bpf_local_storage.c

118

kfree_rcu(local_storage, rcu);

kernel/bpf/bpf_local_storage.c

122

call_rcu_tasks_trace(&local_storage->rcu,

kernel/bpf/bpf_local_storage.c

177

struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

180

void *owner = local_storage->owner;

kernel/bpf/bpf_local_storage.c

183

if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==

kernel/bpf/bpf_local_storage.c

185

RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);

kernel/bpf/bpf_local_storage.c

187

if (pin_owner && !refcount_inc_not_zero(&local_storage->owner_refcnt))

kernel/bpf/bpf_local_storage.c

190

uncharge += free_local_storage ? sizeof(*local_storage) : 0;

kernel/bpf/bpf_local_storage.c

191

mem_uncharge(smap, local_storage->owner, uncharge);

kernel/bpf/bpf_local_storage.c

192

local_storage->mem_charge -= uncharge;

kernel/bpf/bpf_local_storage.c

195

local_storage->owner = NULL;

kernel/bpf/bpf_local_storage.c

202

refcount_dec(&local_storage->owner_refcnt);

kernel/bpf/bpf_local_storage.c

209

static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

219

&local_storage->list);

kernel/bpf/bpf_local_storage.c

22

struct bpf_local_storage *local_storage)

kernel/bpf/bpf_local_storage.c

221

bpf_selem_unlink_storage_nolock_misc(selem, smap, local_storage,

kernel/bpf/bpf_local_storage.c

231

void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

237

local_storage->mem_charge += smap->elem_size;

kernel/bpf/bpf_local_storage.c

239

RCU_INIT_POINTER(selem->local_storage, local_storage);

kernel/bpf/bpf_local_storage.c

24

return &smap->buckets[hash_ptr(local_storage, smap->bucket_log)];

kernel/bpf/bpf_local_storage.c

240

hlist_add_head_rcu(&selem->snode, &local_storage->list);

kernel/bpf/bpf_local_storage.c

245

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_local_storage.c

251

local_storage = rcu_dereference_check(selem->local_storage,

kernel/bpf/bpf_local_storage.c

254

b = select_bucket(smap, local_storage);

kernel/bpf/bpf_local_storage.c

271

struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

278

b = select_bucket(smap, local_storage);

kernel/bpf/bpf_local_storage.c

302

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_local_storage.c

315

local_storage = rcu_dereference_check(selem->local_storage,

kernel/bpf/bpf_local_storage.c

318

err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

332

local_storage, selem, &selem_free_list);

kernel/bpf/bpf_local_storage.c

335

raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

340

bpf_local_storage_free(local_storage, false);

kernel/bpf/bpf_local_storage.c

354

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_local_storage.c

359

local_storage = rcu_dereference_check(selem->local_storage, bpf_rcu_lock_held());

kernel/bpf/bpf_local_storage.c

363

b = b ? : select_bucket(smap, local_storage);

kernel/bpf/bpf_local_storage.c

392

if (local_storage) {

kernel/bpf/bpf_local_storage.c

393

err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

397

&local_storage->list);

kernel/bpf/bpf_local_storage.c

405

selem, smap, local_storage,

kernel/bpf/bpf_local_storage.c

410

raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

421

RCU_INIT_POINTER(selem->local_storage, NULL);

kernel/bpf/bpf_local_storage.c

438

bpf_local_storage_free(local_storage, true);

kernel/bpf/bpf_local_storage.c

441

void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage,

kernel/bpf/bpf_local_storage.c

453

err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

458

rcu_assign_pointer(local_storage->cache[smap->cache_idx], SDATA(selem));

kernel/bpf/bpf_local_storage.c

459

raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

552

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_local_storage.c

565

local_storage = rcu_dereference_check(*owner_storage(smap, owner),

kernel/bpf/bpf_local_storage.c

567

if (!local_storage || hlist_empty(&local_storage->list)) {

kernel/bpf/bpf_local_storage.c

593

bpf_local_storage_lookup(local_storage, smap, false);

kernel/bpf/bpf_local_storage.c

611

err = raw_res_spin_lock_irqsave(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

616

if (unlikely(hlist_empty(&local_storage->list))) {

kernel/bpf/bpf_local_storage.c

626

old_sdata = bpf_local_storage_lookup(local_storage, smap, false);

kernel/bpf/bpf_local_storage.c

638

b = select_bucket(smap, local_storage);

kernel/bpf/bpf_local_storage.c

649

bpf_selem_link_storage_nolock(local_storage, selem);

kernel/bpf/bpf_local_storage.c

654

bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),

kernel/bpf/bpf_local_storage.c

660

raw_res_spin_unlock_irqrestore(&local_storage->lock, flags);

kernel/bpf/bpf_local_storage.c

733

u32 bpf_local_storage_destroy(struct bpf_local_storage *local_storage)

kernel/bpf/bpf_local_storage.c

746

hlist_for_each_entry_rcu(selem, &local_storage->list, snode)

kernel/bpf/bpf_local_storage.c

749

if (!refcount_dec_and_test(&local_storage->owner_refcnt)) {

kernel/bpf/bpf_local_storage.c

750

while (refcount_read(&local_storage->owner_refcnt))

kernel/bpf/bpf_local_storage.c

759

return local_storage->mem_charge;

kernel/bpf/bpf_task_storage.c

48

struct bpf_local_storage *local_storage;

kernel/bpf/bpf_task_storage.c

52

local_storage = rcu_dereference(task->bpf_storage);

kernel/bpf/bpf_task_storage.c

53

if (!local_storage)

kernel/bpf/bpf_task_storage.c

56

bpf_local_storage_destroy(local_storage);

net/core/bpf_sk_storage.c

213

rcu_dereference(copy_selem->local_storage);

net/core/bpf_sk_storage.c

728

sk_storage = rcu_dereference(selem->local_storage);

net/core/bpf_sk_storage.c

741

sk_storage = rcu_dereference(selem->local_storage);

net/core/bpf_sk_storage.c

808

sk_storage = rcu_dereference(selem->local_storage);

tools/testing/selftests/bpf/prog_tests/test_local_storage.c

26

static int run_self_unlink(struct local_storage *skel, const char *rm_path)

tools/testing/selftests/bpf/prog_tests/test_local_storage.c

95

struct local_storage *skel = NULL;

tools/testing/selftests/bpf/progs/local_storage.c

106

struct local_storage *storage;

tools/testing/selftests/bpf/progs/local_storage.c

136

struct local_storage *storage;

tools/testing/selftests/bpf/progs/local_storage.c

173

struct local_storage *storage;

tools/testing/selftests/bpf/progs/local_storage.c

196

struct local_storage *storage;

tools/testing/selftests/bpf/progs/local_storage.c

30

__type(value, struct local_storage);

tools/testing/selftests/bpf/progs/local_storage.c

37

__type(value, struct local_storage);

tools/testing/selftests/bpf/progs/local_storage.c

44

__type(value, struct local_storage);

tools/testing/selftests/bpf/progs/local_storage.c

51

__type(value, struct local_storage);

tools/testing/selftests/bpf/progs/local_storage.c

58

__type(value, struct local_storage);

tools/testing/selftests/bpf/progs/local_storage.c

65

struct local_storage *storage;