arch/powerpc/kvm/book3s_32_mmu_host.c

110

ulong pteg = htab;

arch/powerpc/kvm/book3s_32_mmu_host.c

123

htab, hash, htabmask, pteg);

arch/powerpc/kvm/book3s_32_mmu_host.c

379

htab = (ulong)__va(sdr1 & 0xffff0000);

arch/powerpc/kvm/book3s_32_mmu_host.c

46

static ulong htab;

drivers/s390/char/sclp_rw.c

235

} while (buffer->current_length % buffer->htab);

drivers/s390/char/sclp_rw.c

41

sclp_make_buffer(void *page, unsigned short columns, unsigned short htab)

drivers/s390/char/sclp_rw.c

59

buffer->htab = htab;

drivers/s390/char/sclp_rw.h

75

unsigned short htab;

kernel/bpf/hashtab.c

1002

static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l)

kernel/bpf/hashtab.c

1004

htab_put_fd_value(htab, l);

kernel/bpf/hashtab.c

1006

if (htab_is_prealloc(htab)) {

kernel/bpf/hashtab.c

1007

bpf_map_dec_elem_count(&htab->map);

kernel/bpf/hashtab.c

1008

check_and_cancel_fields(htab, l);

kernel/bpf/hashtab.c

1009

pcpu_freelist_push(&htab->freelist, &l->fnode);

kernel/bpf/hashtab.c

1011

dec_elem_count(htab);

kernel/bpf/hashtab.c

1012

htab_elem_free(htab, l);

kernel/bpf/hashtab.c

1016

static void pcpu_copy_value(struct bpf_htab *htab, void __percpu *pptr,

kernel/bpf/hashtab.c

1024

copy_map_value(&htab->map, ptr, value);

kernel/bpf/hashtab.c

1025

bpf_obj_cancel_fields(&htab->map, ptr);

kernel/bpf/hashtab.c

1027

u32 size = round_up(htab->map.value_size, 8);

kernel/bpf/hashtab.c

1034

copy_map_value(&htab->map, ptr, value);

kernel/bpf/hashtab.c

1035

bpf_obj_cancel_fields(&htab->map, ptr);

kernel/bpf/hashtab.c

1042

copy_map_value(&htab->map, ptr, val);

kernel/bpf/hashtab.c

1043

bpf_obj_cancel_fields(&htab->map, ptr);

kernel/bpf/hashtab.c

1048

static void pcpu_init_value(struct bpf_htab *htab, void __percpu *pptr,

kernel/bpf/hashtab.c

1062

copy_map_value(&htab->map, per_cpu_ptr(pptr, cpu), value);

kernel/bpf/hashtab.c

1064

zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu));

kernel/bpf/hashtab.c

1067

pcpu_copy_value(htab, pptr, value, onallcpus, map_flags);

kernel/bpf/hashtab.c

1071

static bool fd_htab_map_needs_adjust(const struct bpf_htab *htab)

kernel/bpf/hashtab.c

1073

return is_fd_htab(htab) && BITS_PER_LONG == 64;

kernel/bpf/hashtab.c

1076

static struct htab_elem *alloc_htab_elem(struct bpf_htab *htab, void *key,

kernel/bpf/hashtab.c

1081

u32 size = htab->map.value_size;

kernel/bpf/hashtab.c

1082

bool prealloc = htab_is_prealloc(htab);

kernel/bpf/hashtab.c

1091

pl_new = this_cpu_ptr(htab->extra_elems);

kernel/bpf/hashtab.c

1097

l = __pcpu_freelist_pop(&htab->freelist);

kernel/bpf/hashtab.c

1101

bpf_map_inc_elem_count(&htab->map);

kernel/bpf/hashtab.c

1104

if (is_map_full(htab))

kernel/bpf/hashtab.c

1112

inc_elem_count(htab);

kernel/bpf/hashtab.c

1113

l_new = bpf_mem_cache_alloc(&htab->ma);

kernel/bpf/hashtab.c

1126

void *ptr = bpf_mem_cache_alloc(&htab->pcpu_ma);

kernel/bpf/hashtab.c

1129

bpf_mem_cache_free(&htab->ma, l_new);

kernel/bpf/hashtab.c

1137

pcpu_init_value(htab, pptr, value, onallcpus, map_flags);

kernel/bpf/hashtab.c

1141

} else if (fd_htab_map_needs_adjust(htab)) {

kernel/bpf/hashtab.c

1145

copy_map_value_locked(&htab->map,

kernel/bpf/hashtab.c

1149

copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value);

kernel/bpf/hashtab.c

1155

dec_elem_count(htab);

kernel/bpf/hashtab.c

1159

static int check_flags(struct bpf_htab *htab, struct htab_elem *l_old,

kernel/bpf/hashtab.c

1177

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1193

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1195

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

1203

htab->n_buckets);

kernel/bpf/hashtab.c

1204

ret = check_flags(htab, l_old, map_flags);

kernel/bpf/hashtab.c

1226

ret = check_flags(htab, l_old, map_flags);

kernel/bpf/hashtab.c

1244

l_new = alloc_htab_elem(htab, key, value, key_size, hash, false, false,

kernel/bpf/hashtab.c

1262

if (htab_is_prealloc(htab))

kernel/bpf/hashtab.c

1263

check_and_cancel_fields(htab, l_old);

kernel/bpf/hashtab.c

1266

if (l_old && !htab_is_prealloc(htab))

kernel/bpf/hashtab.c

1267

free_htab_elem(htab, l_old);

kernel/bpf/hashtab.c

1274

static void htab_lru_push_free(struct bpf_htab *htab, struct htab_elem *elem)

kernel/bpf/hashtab.c

1276

check_and_cancel_fields(htab, elem);

kernel/bpf/hashtab.c

1277

bpf_map_dec_elem_count(&htab->map);

kernel/bpf/hashtab.c

1278

bpf_lru_push_free(&htab->lru, &elem->lru_node);

kernel/bpf/hashtab.c

1284

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1300

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1302

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

1310

l_new = prealloc_lru_pop(htab, key, hash);

kernel/bpf/hashtab.c

1313

copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value);

kernel/bpf/hashtab.c

1321

ret = check_flags(htab, l_old, map_flags);

kernel/bpf/hashtab.c

134

static inline bool htab_is_prealloc(const struct bpf_htab *htab)

kernel/bpf/hashtab.c

1340

htab_lru_push_free(htab, l_new);

kernel/bpf/hashtab.c

1342

htab_lru_push_free(htab, l_old);

kernel/bpf/hashtab.c

136

return !(htab->map.map_flags & BPF_F_NO_PREALLOC);

kernel/bpf/hashtab.c

1360

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1377

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1379

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

1388

ret = check_flags(htab, l_old, map_flags);

kernel/bpf/hashtab.c

139

static void htab_init_buckets(struct bpf_htab *htab)

kernel/bpf/hashtab.c

1395

pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),

kernel/bpf/hashtab.c

1404

l_new = alloc_htab_elem(htab, key, value, key_size,

kernel/bpf/hashtab.c

1423

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

143

for (i = 0; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

1439

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

144

INIT_HLIST_NULLS_HEAD(&htab->buckets[i].head, i);

kernel/bpf/hashtab.c

1441

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

145

raw_res_spin_lock_init(&htab->buckets[i].raw_lock);

kernel/bpf/hashtab.c

1450

l_new = prealloc_lru_pop(htab, key, hash);

kernel/bpf/hashtab.c

1461

ret = check_flags(htab, l_old, map_flags);

kernel/bpf/hashtab.c

1469

pcpu_copy_value(htab, htab_elem_get_ptr(l_old, key_size),

kernel/bpf/hashtab.c

1472

pcpu_init_value(htab, htab_elem_get_ptr(l_new, key_size),

kernel/bpf/hashtab.c

1482

bpf_map_dec_elem_count(&htab->map);

kernel/bpf/hashtab.c

1483

bpf_lru_push_free(&htab->lru, &l_new->lru_node);

kernel/bpf/hashtab.c

1504

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1516

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1517

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

1533

free_htab_elem(htab, l);

kernel/bpf/hashtab.c

1539

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1551

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1552

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

1568

htab_lru_push_free(htab, l);

kernel/bpf/hashtab.c

1572

static void delete_all_elements(struct bpf_htab *htab)

kernel/bpf/hashtab.c

1579

for (i = 0; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

1580

struct hlist_nulls_head *head = select_bucket(htab, i);

kernel/bpf/hashtab.c

1586

htab_elem_free(htab, l);

kernel/bpf/hashtab.c

1592

static void htab_free_malloced_internal_structs(struct bpf_htab *htab)

kernel/bpf/hashtab.c

1597

for (i = 0; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

1598

struct hlist_nulls_head *head = select_bucket(htab, i);

kernel/bpf/hashtab.c

1604

bpf_map_free_internal_structs(&htab->map,

kernel/bpf/hashtab.c

1605

htab_elem_value(l, htab->map.key_size));

kernel/bpf/hashtab.c

1614

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1620

if (htab_is_prealloc(htab))

kernel/bpf/hashtab.c

1621

htab_free_prealloced_internal_structs(htab);

kernel/bpf/hashtab.c

1623

htab_free_malloced_internal_structs(htab);

kernel/bpf/hashtab.c

1629

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

1640

if (!htab_is_prealloc(htab)) {

kernel/bpf/hashtab.c

1641

delete_all_elements(htab);

kernel/bpf/hashtab.c

1643

htab_free_prealloced_fields(htab);

kernel/bpf/hashtab.c

1644

prealloc_destroy(htab);

kernel/bpf/hashtab.c

1648

free_percpu(htab->extra_elems);

kernel/bpf/hashtab.c

1649

bpf_map_area_free(htab->buckets);

kernel/bpf/hashtab.c

1650

bpf_mem_alloc_destroy(&htab->pcpu_ma);

kernel/bpf/hashtab.c

1651

bpf_mem_alloc_destroy(&htab->ma);

kernel/bpf/hashtab.c

1652

if (htab->use_percpu_counter)

kernel/bpf/hashtab.c

1653

percpu_counter_destroy(&htab->pcount);

kernel/bpf/hashtab.c

1654

bpf_map_area_free(htab);

kernel/bpf/hashtab.c

1682

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

169

static bool htab_is_lru(const struct bpf_htab *htab)

kernel/bpf/hashtab.c

1692

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

1693

b = __select_bucket(htab, hash);

kernel/bpf/hashtab.c

171

return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH ||

kernel/bpf/hashtab.c

1713

copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu));

kernel/bpf/hashtab.c

1714

check_and_init_map_value(&htab->map, value + off);

kernel/bpf/hashtab.c

172

htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;

kernel/bpf/hashtab.c

1734

htab_lru_push_free(htab, l);

kernel/bpf/hashtab.c

1736

free_htab_elem(htab, l);

kernel/bpf/hashtab.c

175

static bool htab_is_percpu(const struct bpf_htab *htab)

kernel/bpf/hashtab.c

177

return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH ||

kernel/bpf/hashtab.c

1779

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

178

htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH;

kernel/bpf/hashtab.c

181

static inline bool is_fd_htab(const struct bpf_htab *htab)

kernel/bpf/hashtab.c

1819

if (batch >= htab->n_buckets)

kernel/bpf/hashtab.c

1822

key_size = htab->map.key_size;

kernel/bpf/hashtab.c

1823

value_size = htab->map.value_size;

kernel/bpf/hashtab.c

183

return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS;

kernel/bpf/hashtab.c

1850

b = &htab->buckets[batch];

kernel/bpf/hashtab.c

1910

copy_map_value(&htab->map, dst_val, per_cpu_ptr(pptr, cpu));

kernel/bpf/hashtab.c

1911

check_and_init_map_value(&htab->map, dst_val);

kernel/bpf/hashtab.c

1914

copy_map_value_long(&htab->map, dst_val + off,

kernel/bpf/hashtab.c

1916

check_and_init_map_value(&htab->map, dst_val + off);

kernel/bpf/hashtab.c

1922

if (is_fd_htab(htab)) {

kernel/bpf/hashtab.c

1965

htab_lru_push_free(htab, l);

kernel/bpf/hashtab.c

1967

free_htab_elem(htab, l);

kernel/bpf/hashtab.c

1974

if (!bucket_cnt && (batch + 1 < htab->n_buckets)) {

kernel/bpf/hashtab.c

1991

if (batch >= htab->n_buckets) {

kernel/bpf/hashtab.c

207

static struct htab_elem *get_htab_elem(struct bpf_htab *htab, int i)

kernel/bpf/hashtab.c

2084

struct bpf_htab *htab;

kernel/bpf/hashtab.c

209

return (struct htab_elem *) (htab->elems + i * (u64)htab->elem_size);

kernel/bpf/hashtab.c

2094

const struct bpf_htab *htab = info->htab;

kernel/bpf/hashtab.c

2103

if (bucket_id >= htab->n_buckets)

kernel/bpf/hashtab.c

2117

b = &htab->buckets[bucket_id++];

kernel/bpf/hashtab.c

2122

for (i = bucket_id; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

2123

b = &htab->buckets[i];

kernel/bpf/hashtab.c

216

static bool htab_has_extra_elems(struct bpf_htab *htab)

kernel/bpf/hashtab.c

218

return !htab_is_percpu(htab) && !htab_is_lru(htab) && !is_fd_htab(htab);

kernel/bpf/hashtab.c

221

static void htab_free_prealloced_internal_structs(struct bpf_htab *htab)

kernel/bpf/hashtab.c

223

u32 num_entries = htab->map.max_entries;

kernel/bpf/hashtab.c

2240

seq_info->htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

226

if (htab_has_extra_elems(htab))

kernel/bpf/hashtab.c

2269

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

2285

is_percpu = htab_is_percpu(htab);

kernel/bpf/hashtab.c

2291

for (i = 0; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

2292

b = &htab->buckets[i];

kernel/bpf/hashtab.c

232

elem = get_htab_elem(htab, i);

kernel/bpf/hashtab.c

2321

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

2322

u32 value_size = round_up(htab->map.value_size, 8);

kernel/bpf/hashtab.c

2323

bool prealloc = htab_is_prealloc(htab);

kernel/bpf/hashtab.c

2324

bool percpu = htab_is_percpu(htab);

kernel/bpf/hashtab.c

2325

bool lru = htab_is_lru(htab);

kernel/bpf/hashtab.c

2329

sizeof(struct bucket) * htab->n_buckets;

kernel/bpf/hashtab.c

233

bpf_map_free_internal_structs(&htab->map,

kernel/bpf/hashtab.c

2333

if (htab_has_extra_elems(htab))

kernel/bpf/hashtab.c

2336

usage += htab->elem_size * num_entries;

kernel/bpf/hashtab.c

234

htab_elem_value(elem, htab->map.key_size));

kernel/bpf/hashtab.c

2345

num_entries = htab->use_percpu_counter ?

kernel/bpf/hashtab.c

2346

percpu_counter_sum(&htab->pcount) :

kernel/bpf/hashtab.c

2347

atomic_read(&htab->count);

kernel/bpf/hashtab.c

2348

usage += (htab->elem_size + LLIST_NODE_SZ) * num_entries;

kernel/bpf/hashtab.c

2375

BATCH_OPS(htab),

kernel/bpf/hashtab.c

239

static void htab_free_prealloced_fields(struct bpf_htab *htab)

kernel/bpf/hashtab.c

241

u32 num_entries = htab->map.max_entries;

kernel/bpf/hashtab.c

244

if (IS_ERR_OR_NULL(htab->map.record))

kernel/bpf/hashtab.c

250

if (htab_has_extra_elems(htab))

kernel/bpf/hashtab.c

2517

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

2521

if (htab_is_lru(htab))

kernel/bpf/hashtab.c

255

elem = get_htab_elem(htab, i);

kernel/bpf/hashtab.c

256

if (htab_is_percpu(htab)) {

kernel/bpf/hashtab.c

257

void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);

kernel/bpf/hashtab.c

261

bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu));

kernel/bpf/hashtab.c

2613

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

2619

for (i = 0; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

2620

head = select_bucket(htab, i);

kernel/bpf/hashtab.c

265

bpf_obj_free_fields(htab->map.record,

kernel/bpf/hashtab.c

266

htab_elem_value(elem, htab->map.key_size));

kernel/bpf/hashtab.c

273

static void htab_free_elems(struct bpf_htab *htab)

kernel/bpf/hashtab.c

2742

BATCH_OPS(htab),

kernel/bpf/hashtab.c

277

if (!htab_is_percpu(htab))

kernel/bpf/hashtab.c

280

for (i = 0; i < htab->map.max_entries; i++) {

kernel/bpf/hashtab.c

283

pptr = htab_elem_get_ptr(get_htab_elem(htab, i),

kernel/bpf/hashtab.c

284

htab->map.key_size);

kernel/bpf/hashtab.c

289

bpf_map_area_free(htab->elems);

kernel/bpf/hashtab.c

303

static struct htab_elem *prealloc_lru_pop(struct bpf_htab *htab, void *key,

kernel/bpf/hashtab.c

306

struct bpf_lru_node *node = bpf_lru_pop_free(&htab->lru, hash);

kernel/bpf/hashtab.c

310

bpf_map_inc_elem_count(&htab->map);

kernel/bpf/hashtab.c

312

memcpy(l->key, key, htab->map.key_size);

kernel/bpf/hashtab.c

319

static int prealloc_init(struct bpf_htab *htab)

kernel/bpf/hashtab.c

321

u32 num_entries = htab->map.max_entries;

kernel/bpf/hashtab.c

324

if (htab_has_extra_elems(htab))

kernel/bpf/hashtab.c

327

htab->elems = bpf_map_area_alloc((u64)htab->elem_size * num_entries,

kernel/bpf/hashtab.c

328

htab->map.numa_node);

kernel/bpf/hashtab.c

329

if (!htab->elems)

kernel/bpf/hashtab.c

332

if (!htab_is_percpu(htab))

kernel/bpf/hashtab.c

336

u32 size = round_up(htab->map.value_size, 8);

kernel/bpf/hashtab.c

339

pptr = bpf_map_alloc_percpu(&htab->map, size, 8,

kernel/bpf/hashtab.c

343

htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size,

kernel/bpf/hashtab.c

349

if (htab_is_lru(htab))

kernel/bpf/hashtab.c

350

err = bpf_lru_init(&htab->lru,

kernel/bpf/hashtab.c

351

htab->map.map_flags & BPF_F_NO_COMMON_LRU,

kernel/bpf/hashtab.c

355

htab);

kernel/bpf/hashtab.c

357

err = pcpu_freelist_init(&htab->freelist);

kernel/bpf/hashtab.c

362

if (htab_is_lru(htab))

kernel/bpf/hashtab.c

363

bpf_lru_populate(&htab->lru, htab->elems,

kernel/bpf/hashtab.c

365

htab->elem_size, num_entries);

kernel/bpf/hashtab.c

367

pcpu_freelist_populate(&htab->freelist,

kernel/bpf/hashtab.c

368

htab->elems + offsetof(struct htab_elem, fnode),

kernel/bpf/hashtab.c

369

htab->elem_size, num_entries);

kernel/bpf/hashtab.c

374

htab_free_elems(htab);

kernel/bpf/hashtab.c

378

static void prealloc_destroy(struct bpf_htab *htab)

kernel/bpf/hashtab.c

380

htab_free_elems(htab);

kernel/bpf/hashtab.c

382

if (htab_is_lru(htab))

kernel/bpf/hashtab.c

383

bpf_lru_destroy(&htab->lru);

kernel/bpf/hashtab.c

385

pcpu_freelist_destroy(&htab->freelist);

kernel/bpf/hashtab.c

388

static int alloc_extra_elems(struct bpf_htab *htab)

kernel/bpf/hashtab.c

394

pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8,

kernel/bpf/hashtab.c

400

l = pcpu_freelist_pop(&htab->freelist);

kernel/bpf/hashtab.c

407

htab->extra_elems = pptr;

kernel/bpf/hashtab.c

531

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

533

if (htab_is_prealloc(htab))

kernel/bpf/hashtab.c

539

if (htab_is_percpu(htab))

kernel/bpf/hashtab.c

540

return bpf_ma_set_dtor(map, &htab->pcpu_ma, htab_pcpu_mem_dtor);

kernel/bpf/hashtab.c

542

return bpf_ma_set_dtor(map, &htab->ma, htab_mem_dtor);

kernel/bpf/hashtab.c

556

struct bpf_htab *htab;

kernel/bpf/hashtab.c

559

htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);

kernel/bpf/hashtab.c

560

if (!htab)

kernel/bpf/hashtab.c

563

bpf_map_init_from_attr(&htab->map, attr);

kernel/bpf/hashtab.c

570

htab->map.max_entries = roundup(attr->max_entries,

kernel/bpf/hashtab.c

572

if (htab->map.max_entries < attr->max_entries)

kernel/bpf/hashtab.c

573

htab->map.max_entries = rounddown(attr->max_entries,

kernel/bpf/hashtab.c

581

if (htab->map.max_entries > 1UL << 31)

kernel/bpf/hashtab.c

584

htab->n_buckets = roundup_pow_of_two(htab->map.max_entries);

kernel/bpf/hashtab.c

586

htab->elem_size = sizeof(struct htab_elem) +

kernel/bpf/hashtab.c

587

round_up(htab->map.key_size, 8);

kernel/bpf/hashtab.c

589

htab->elem_size += sizeof(void *);

kernel/bpf/hashtab.c

591

htab->elem_size += round_up(htab->map.value_size, 8);

kernel/bpf/hashtab.c

594

if (htab->n_buckets > U32_MAX / sizeof(struct bucket))

kernel/bpf/hashtab.c

597

err = bpf_map_init_elem_count(&htab->map);

kernel/bpf/hashtab.c

602

htab->buckets = bpf_map_area_alloc(htab->n_buckets *

kernel/bpf/hashtab.c

604

htab->map.numa_node);

kernel/bpf/hashtab.c

605

if (!htab->buckets)

kernel/bpf/hashtab.c

608

if (htab->map.map_flags & BPF_F_ZERO_SEED)

kernel/bpf/hashtab.c

609

htab->hashrnd = 0;

kernel/bpf/hashtab.c

611

htab->hashrnd = get_random_u32();

kernel/bpf/hashtab.c

613

htab_init_buckets(htab);

kernel/bpf/hashtab.c

630

htab->use_percpu_counter = true;

kernel/bpf/hashtab.c

632

if (htab->use_percpu_counter) {

kernel/bpf/hashtab.c

633

err = percpu_counter_init(&htab->pcount, 0, GFP_KERNEL);

kernel/bpf/hashtab.c

639

err = prealloc_init(htab);

kernel/bpf/hashtab.c

643

if (htab_has_extra_elems(htab)) {

kernel/bpf/hashtab.c

644

err = alloc_extra_elems(htab);

kernel/bpf/hashtab.c

649

err = bpf_mem_alloc_init(&htab->ma, htab->elem_size, false);

kernel/bpf/hashtab.c

653

err = bpf_mem_alloc_init(&htab->pcpu_ma,

kernel/bpf/hashtab.c

654

round_up(htab->map.value_size, 8), true);

kernel/bpf/hashtab.c

660

return &htab->map;

kernel/bpf/hashtab.c

663

prealloc_destroy(htab);

kernel/bpf/hashtab.c

665

if (htab->use_percpu_counter)

kernel/bpf/hashtab.c

666

percpu_counter_destroy(&htab->pcount);

kernel/bpf/hashtab.c

667

bpf_map_area_free(htab->buckets);

kernel/bpf/hashtab.c

668

bpf_mem_alloc_destroy(&htab->pcpu_ma);

kernel/bpf/hashtab.c

669

bpf_mem_alloc_destroy(&htab->ma);

kernel/bpf/hashtab.c

671

bpf_map_free_elem_count(&htab->map);

kernel/bpf/hashtab.c

673

bpf_map_area_free(htab);

kernel/bpf/hashtab.c

684

static inline struct bucket *__select_bucket(struct bpf_htab *htab, u32 hash)

kernel/bpf/hashtab.c

686

return &htab->buckets[hash & (htab->n_buckets - 1)];

kernel/bpf/hashtab.c

689

static inline struct hlist_nulls_head *select_bucket(struct bpf_htab *htab, u32 hash)

kernel/bpf/hashtab.c

691

return &__select_bucket(htab, hash)->head;

kernel/bpf/hashtab.c

737

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

746

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

748

head = select_bucket(htab, hash);

kernel/bpf/hashtab.c

750

l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);

kernel/bpf/hashtab.c

840

static void check_and_cancel_fields(struct bpf_htab *htab,

kernel/bpf/hashtab.c

843

if (IS_ERR_OR_NULL(htab->map.record))

kernel/bpf/hashtab.c

846

if (htab_is_percpu(htab)) {

kernel/bpf/hashtab.c

847

void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size);

kernel/bpf/hashtab.c

851

bpf_obj_cancel_fields(&htab->map, per_cpu_ptr(pptr, cpu));

kernel/bpf/hashtab.c

853

void *map_value = htab_elem_value(elem, htab->map.key_size);

kernel/bpf/hashtab.c

855

bpf_obj_cancel_fields(&htab->map, map_value);

kernel/bpf/hashtab.c

864

struct bpf_htab *htab = arg;

kernel/bpf/hashtab.c

873

b = __select_bucket(htab, tgt_l->hash);

kernel/bpf/hashtab.c

883

bpf_map_dec_elem_count(&htab->map);

kernel/bpf/hashtab.c

890

check_and_cancel_fields(htab, l);

kernel/bpf/hashtab.c

897

struct bpf_htab *htab = container_of(map, struct bpf_htab, map);

kernel/bpf/hashtab.c

910

hash = htab_map_hash(key, key_size, htab->hashrnd);

kernel/bpf/hashtab.c

912

head = select_bucket(htab, hash);

kernel/bpf/hashtab.c

915

l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets);

kernel/bpf/hashtab.c

931

i = hash & (htab->n_buckets - 1);

kernel/bpf/hashtab.c

936

for (; i < htab->n_buckets; i++) {

kernel/bpf/hashtab.c

937

head = select_bucket(htab, i);

kernel/bpf/hashtab.c

953

static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l)

kernel/bpf/hashtab.c

955

check_and_cancel_fields(htab, l);

kernel/bpf/hashtab.c

957

if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH)

kernel/bpf/hashtab.c

958

bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr);

kernel/bpf/hashtab.c

959

bpf_mem_cache_free(&htab->ma, l);

kernel/bpf/hashtab.c

962

static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l)

kernel/bpf/hashtab.c

964

struct bpf_map *map = &htab->map;

kernel/bpf/hashtab.c

973

static bool is_map_full(struct bpf_htab *htab)

kernel/bpf/hashtab.c

975

if (htab->use_percpu_counter)

kernel/bpf/hashtab.c

976

return __percpu_counter_compare(&htab->pcount, htab->map.max_entries,

kernel/bpf/hashtab.c

978

return atomic_read(&htab->count) >= htab->map.max_entries;

kernel/bpf/hashtab.c

981

static void inc_elem_count(struct bpf_htab *htab)

kernel/bpf/hashtab.c

983

bpf_map_inc_elem_count(&htab->map);

kernel/bpf/hashtab.c

985

if (htab->use_percpu_counter)

kernel/bpf/hashtab.c

986

percpu_counter_add_batch(&htab->pcount, 1, PERCPU_COUNTER_BATCH);

kernel/bpf/hashtab.c

988

atomic_inc(&htab->count);

kernel/bpf/hashtab.c

991

static void dec_elem_count(struct bpf_htab *htab)

kernel/bpf/hashtab.c

993

bpf_map_dec_elem_count(&htab->map);

kernel/bpf/hashtab.c

995

if (htab->use_percpu_counter)

kernel/bpf/hashtab.c

996

percpu_counter_add_batch(&htab->pcount, -1, PERCPU_COUNTER_BATCH);

kernel/bpf/hashtab.c

998

atomic_dec(&htab->count);

net/core/sock_map.c

1020

bucket = sock_hash_select_bucket(htab, hash);

net/core/sock_map.c

1032

elem_new = sock_hash_alloc_elem(htab, key, key_size, hash, sk, elem);

net/core/sock_map.c

1046

sock_hash_free_elem(htab, elem);

net/core/sock_map.c

1061

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

1070

head = &sock_hash_select_bucket(htab, hash)->head;

net/core/sock_map.c

1082

i = hash & (htab->buckets_num - 1);

net/core/sock_map.c

1085

for (; i < htab->buckets_num; i++) {

net/core/sock_map.c

1086

head = &sock_hash_select_bucket(htab, i)->head;

net/core/sock_map.c

1100

struct bpf_shtab *htab;

net/core/sock_map.c

1112

htab = bpf_map_area_alloc(sizeof(*htab), NUMA_NO_NODE);

net/core/sock_map.c

1113

if (!htab)

net/core/sock_map.c

1116

bpf_map_init_from_attr(&htab->map, attr);

net/core/sock_map.c

1118

htab->buckets_num = roundup_pow_of_two(htab->map.max_entries);

net/core/sock_map.c

1119

htab->elem_size = sizeof(struct bpf_shtab_elem) +

net/core/sock_map.c

1120

round_up(htab->map.key_size, 8);

net/core/sock_map.c

1121

if (htab->buckets_num == 0 ||

net/core/sock_map.c

1122

htab->buckets_num > U32_MAX / sizeof(struct bpf_shtab_bucket)) {

net/core/sock_map.c

1127

htab->buckets = bpf_map_area_alloc(htab->buckets_num *

net/core/sock_map.c

1129

htab->map.numa_node);

net/core/sock_map.c

1130

if (!htab->buckets) {

net/core/sock_map.c

1135

for (i = 0; i < htab->buckets_num; i++) {

net/core/sock_map.c

1136

INIT_HLIST_HEAD(&htab->buckets[i].head);

net/core/sock_map.c

1137

spin_lock_init(&htab->buckets[i].lock);

net/core/sock_map.c

1140

return &htab->map;

net/core/sock_map.c

1142

bpf_map_area_free(htab);

net/core/sock_map.c

1148

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

1160

for (i = 0; i < htab->buckets_num; i++) {

net/core/sock_map.c

1161

bucket = sock_hash_select_bucket(htab, i);

net/core/sock_map.c

1187

sock_hash_free_elem(htab, elem);

net/core/sock_map.c

1195

bpf_map_area_free(htab->buckets);

net/core/sock_map.c

1196

bpf_map_area_free(htab);

net/core/sock_map.c

1314

struct bpf_shtab *htab;

net/core/sock_map.c

1321

const struct bpf_shtab *htab = info->htab;

net/core/sock_map.c

1337

for (; info->bucket_id < htab->buckets_num; info->bucket_id++) {

net/core/sock_map.c

1338

bucket = &htab->buckets[info->bucket_id];

net/core/sock_map.c

1418

info->htab = container_of(aux->map, struct bpf_shtab, map);

net/core/sock_map.c

1431

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

1432

u64 usage = sizeof(*htab);

net/core/sock_map.c

1434

usage += htab->buckets_num * sizeof(struct bpf_shtab_bucket);

net/core/sock_map.c

1435

usage += atomic_read(&htab->count) * (u64)htab->elem_size;

net/core/sock_map.c

874

static struct bpf_shtab_bucket *sock_hash_select_bucket(struct bpf_shtab *htab,

net/core/sock_map.c

877

return &htab->buckets[hash & (htab->buckets_num - 1)];

net/core/sock_map.c

897

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

905

bucket = sock_hash_select_bucket(htab, hash);

net/core/sock_map.c

911

static void sock_hash_free_elem(struct bpf_shtab *htab,

net/core/sock_map.c

914

atomic_dec(&htab->count);

net/core/sock_map.c

921

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

926

bucket = sock_hash_select_bucket(htab, elem->hash);

net/core/sock_map.c

938

sock_hash_free_elem(htab, elem);

net/core/sock_map.c

945

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/core/sock_map.c

952

bucket = sock_hash_select_bucket(htab, hash);

net/core/sock_map.c

959

sock_hash_free_elem(htab, elem);

net/core/sock_map.c

966

static struct bpf_shtab_elem *sock_hash_alloc_elem(struct bpf_shtab *htab,

net/core/sock_map.c

973

if (atomic_inc_return(&htab->count) > htab->map.max_entries) {

net/core/sock_map.c

975

atomic_dec(&htab->count);

net/core/sock_map.c

980

new = bpf_map_kmalloc_node(&htab->map, htab->elem_size,

net/core/sock_map.c

982

htab->map.numa_node);

net/core/sock_map.c

984

atomic_dec(&htab->count);

net/core/sock_map.c

996

struct bpf_shtab *htab = container_of(map, struct bpf_shtab, map);

net/xfrm/xfrm_policy.c

4233

struct xfrm_policy_hash *htab;

net/xfrm/xfrm_policy.c

4238

htab = &net->xfrm.policy_bydst[dir];

net/xfrm/xfrm_policy.c

4239

rcu_assign_pointer(htab->table, xfrm_hash_alloc(sz));

net/xfrm/xfrm_policy.c

4240

if (!htab->table)

net/xfrm/xfrm_policy.c

4242

htab->hmask = hmask;

net/xfrm/xfrm_policy.c

4243

htab->dbits4 = 32;

net/xfrm/xfrm_policy.c

4244

htab->sbits4 = 32;

net/xfrm/xfrm_policy.c

4245

htab->dbits6 = 128;

net/xfrm/xfrm_policy.c

4246

htab->sbits6 = 128;

net/xfrm/xfrm_policy.c

4263

struct xfrm_policy_hash *htab;

net/xfrm/xfrm_policy.c

4265

htab = &net->xfrm.policy_bydst[dir];

net/xfrm/xfrm_policy.c

4266

xfrm_hash_free(rcu_dereference_protected(htab->table, true), sz);

net/xfrm/xfrm_policy.c

4292

struct xfrm_policy_hash *htab;

net/xfrm/xfrm_policy.c

4294

htab = &net->xfrm.policy_bydst[dir];

net/xfrm/xfrm_policy.c

4295

sz = (htab->hmask + 1) * sizeof(struct hlist_head);

net/xfrm/xfrm_policy.c

4296

WARN_ON(!hlist_empty(rcu_dereference_protected(htab->table, true)));

net/xfrm/xfrm_policy.c

4297

xfrm_hash_free(rcu_dereference_protected(htab->table, true), sz);

tools/testing/selftests/bpf/benchs/bench_htab_mem.c

180

map = ctx.skel->maps.htab;

tools/testing/selftests/bpf/bpf_arena_htab.h

18

static inline htab_bucket_t *__select_bucket(struct htab __arena *htab, __u32 hash)

tools/testing/selftests/bpf/bpf_arena_htab.h

20

htab_bucket_t *b = htab->buckets;

tools/testing/selftests/bpf/bpf_arena_htab.h

23

return &b[hash & (htab->n_buckets - 1)];

tools/testing/selftests/bpf/bpf_arena_htab.h

26

static inline arena_list_head_t *select_bucket(struct htab __arena *htab, __u32 hash)

tools/testing/selftests/bpf/bpf_arena_htab.h

28

return &__select_bucket(htab, hash)->head;

tools/testing/selftests/bpf/bpf_arena_htab.h

55

__weak int htab_lookup_elem(struct htab __arena *htab, int key)

tools/testing/selftests/bpf/bpf_arena_htab.h

60

cast_kern(htab);

tools/testing/selftests/bpf/bpf_arena_htab.h

61

head = select_bucket(htab, key);

tools/testing/selftests/bpf/bpf_arena_htab.h

68

__weak int htab_update_elem(struct htab __arena *htab, int key, int value)

tools/testing/selftests/bpf/bpf_arena_htab.h

73

cast_kern(htab);

tools/testing/selftests/bpf/bpf_arena_htab.h

74

head = select_bucket(htab, key);

tools/testing/selftests/bpf/bpf_arena_htab.h

92

void htab_init(struct htab __arena *htab)

tools/testing/selftests/bpf/bpf_arena_htab.h

97

htab->buckets = buckets;

tools/testing/selftests/bpf/bpf_arena_htab.h

98

htab->n_buckets = 2 * PAGE_SIZE / sizeof(struct htab_bucket);

tools/testing/selftests/bpf/prog_tests/arena_htab.c

16

static void test_arena_htab_common(struct htab *htab)

tools/testing/selftests/bpf/prog_tests/arena_htab.c

20

printf("htab %p buckets %p n_buckets %d\n", htab, htab->buckets, htab->n_buckets);

tools/testing/selftests/bpf/prog_tests/arena_htab.c

21

ASSERT_OK_PTR(htab->buckets, "htab->buckets shouldn't be NULL");

tools/testing/selftests/bpf/prog_tests/arena_htab.c

22

for (i = 0; htab->buckets && i < 16; i += 4) {

tools/testing/selftests/bpf/prog_tests/arena_htab.c

27

int val = htab_lookup_elem(htab, i);

tools/testing/selftests/bpf/prog_tests/arena_htab.c

37

struct htab *htab;

tools/testing/selftests/bpf/prog_tests/arena_htab.c

59

htab = skel->bss->htab_for_user;

tools/testing/selftests/bpf/prog_tests/arena_htab.c

60

test_arena_htab_common(htab);

tools/testing/selftests/bpf/prog_tests/arena_htab.c

69

struct htab *htab;

tools/testing/selftests/bpf/prog_tests/arena_htab.c

79

htab = skel->bss->htab_for_user;

tools/testing/selftests/bpf/prog_tests/arena_htab.c

80

test_arena_htab_common(htab);

tools/testing/selftests/bpf/prog_tests/htab_reuse.c

74

ctx.fd = bpf_map__fd(skel->maps.htab);

tools/testing/selftests/bpf/prog_tests/htab_update.c

106

ctx.fd = bpf_map__fd(skel->maps.htab);

tools/testing/selftests/bpf/prog_tests/htab_update.c

36

value_size = bpf_map__value_size(skel->maps.htab);

tools/testing/selftests/bpf/prog_tests/htab_update.c

46

err = bpf_map_update_elem(bpf_map__fd(skel->maps.htab), &key, value, BPF_ANY);

tools/testing/selftests/bpf/prog_tests/htab_update.c

58

err = bpf_map_update_elem(bpf_map__fd(skel->maps.htab), &key, value, BPF_ANY);

tools/testing/selftests/bpf/progs/arena_htab.c

29

struct htab __arena *htab;

tools/testing/selftests/bpf/progs/arena_htab.c

33

htab = bpf_alloc(sizeof(*htab));

tools/testing/selftests/bpf/progs/arena_htab.c

34

cast_kern(htab);

tools/testing/selftests/bpf/progs/arena_htab.c

35

htab_init(htab);

tools/testing/selftests/bpf/progs/arena_htab.c

41

htab_update_elem(htab, i, i);

tools/testing/selftests/bpf/progs/arena_htab.c

47

htab_update_elem(htab, i, i);

tools/testing/selftests/bpf/progs/arena_htab.c

51

cast_user(htab);

tools/testing/selftests/bpf/progs/arena_htab.c

52

htab_for_user = htab;

tools/testing/selftests/bpf/progs/htab_mem_bench.c

21

} htab SEC(".maps");

tools/testing/selftests/bpf/progs/htab_mem_bench.c

31

bpf_map_update_elem(&htab, &ctx->from, zeroed_value, flags);

tools/testing/selftests/bpf/progs/htab_mem_bench.c

49

bpf_map_delete_elem(&htab, &ctx->from);

tools/testing/selftests/bpf/progs/htab_reuse.c

19

} htab SEC(".maps");

tools/testing/selftests/bpf/progs/htab_update.c

20

} htab SEC(".maps");

tools/testing/selftests/bpf/progs/htab_update.c

34

update_err = bpf_map_update_elem(&htab, &key, &value, BPF_ANY);

tools/testing/selftests/net/sample_map_ret0.bpf.c

10

} htab SEC(".maps");

tools/testing/selftests/net/sample_map_ret0.bpf.c

26

value = bpf_map_lookup_elem(&htab, &key);