block/bio.c

108

kmem_cache_destroy(bslab->slab);

block/bio.c

139

return bslab->slab;

block/bio.c

154

WARN_ON_ONCE(bslab->slab != bs->bio_slab);

block/bio.c

163

kmem_cache_destroy(bslab->slab);

block/bio.c

1912

return mempool_init_slab_pool(pool, pool_entries, bp->slab);

block/bio.c

2013

bvs->slab = kmem_cache_create(bvs->name,

block/bio.c

207

kmem_cache_free(biovec_slab(bio->bi_max_vecs)->slab,

block/bio.c

43

struct kmem_cache *slab;

block/bio.c

579

bvecs = kmem_cache_alloc(bvs->slab, gfp);

block/bio.c

80

struct kmem_cache *slab;

block/bio.c

96

bslab->slab = kmem_cache_create(bslab->name, size,

block/bio.c

99

if (!bslab->slab)

drivers/md/dm-vdo/dedupe.c

1282

struct vdo_slab *slab = vdo_get_slab(depot, agent->duplicate.pbn);

drivers/md/dm-vdo/dedupe.c

1283

int result = vdo_acquire_provisional_reference(slab, agent->duplicate.pbn, lock);

drivers/md/dm-vdo/repair.c

810

struct vdo_slab *slab;

drivers/md/dm-vdo/repair.c

830

slab = vdo_get_slab(vdo->depot, pbn);

drivers/md/dm-vdo/repair.c

831

if (slab->allocator != allocator)

drivers/md/dm-vdo/repair.c

834

if (!vdo_attempt_replay_into_slab(slab, pbn, entry.operation, increment,

drivers/md/dm-vdo/slab-depot.c

1017

struct vdo_slab *slab = container_of(waiter, struct vdo_slab, summary_waiter);

drivers/md/dm-vdo/slab-depot.c

1020

slab->active_count--;

drivers/md/dm-vdo/slab-depot.c

1024

vdo_enter_read_only_mode(slab->allocator->depot->vdo, result);

drivers/md/dm-vdo/slab-depot.c

1027

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

1043

struct vdo_slab *slab = context;

drivers/md/dm-vdo/slab-depot.c

1045

if (vdo_is_read_only(slab->allocator->depot->vdo))

drivers/md/dm-vdo/slab-depot.c

1048

slab->active_count++;

drivers/md/dm-vdo/slab-depot.c

1051

acquire_vio_from_pool(slab->allocator->vio_pool, waiter);

drivers/md/dm-vdo/slab-depot.c

1054

static void save_dirty_reference_blocks(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1056

vdo_waitq_notify_all_waiters(&slab->dirty_blocks,

drivers/md/dm-vdo/slab-depot.c

1057

launch_reference_block_write, slab);

drivers/md/dm-vdo/slab-depot.c

1058

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

1071

struct vdo_slab *slab = block->slab;

drivers/md/dm-vdo/slab-depot.c

1074

slab->active_count--;

drivers/md/dm-vdo/slab-depot.c

1077

adjust_slab_journal_block_reference(&slab->journal,

drivers/md/dm-vdo/slab-depot.c

1088

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

1094

vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter);

drivers/md/dm-vdo/slab-depot.c

1095

if (vdo_is_state_draining(&slab->state)) {

drivers/md/dm-vdo/slab-depot.c

1097

save_dirty_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

1107

if ((slab->active_count > 0) || vdo_waitq_has_waiters(&slab->dirty_blocks)) {

drivers/md/dm-vdo/slab-depot.c

1108

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

1112

offset = slab->allocator->summary_entries[slab->slab_number].tail_block_offset;

drivers/md/dm-vdo/slab-depot.c

1113

slab->active_count++;

drivers/md/dm-vdo/slab-depot.c

1114

slab->summary_waiter.callback = finish_summary_update;

drivers/md/dm-vdo/slab-depot.c

1115

update_slab_summary_entry(slab, &slab->summary_waiter, offset,

drivers/md/dm-vdo/slab-depot.c

1116

true, true, slab->free_blocks);

drivers/md/dm-vdo/slab-depot.c

1127

size_t block_index = block - block->slab->reference_blocks;

drivers/md/dm-vdo/slab-depot.c

1129

return &block->slab->counters[block_index * COUNTS_PER_BLOCK];

drivers/md/dm-vdo/slab-depot.c

1144

vdo_pack_journal_point(&block->slab->slab_journal_point, &commit_point);

drivers/md/dm-vdo/slab-depot.c

1157

thread_id_t thread_id = block->slab->allocator->thread_id;

drivers/md/dm-vdo/slab-depot.c

1170

struct vdo_slab *slab = ((struct reference_block *) completion->parent)->slab;

drivers/md/dm-vdo/slab-depot.c

1174

slab->active_count -= vio->io_size / VDO_BLOCK_SIZE;

drivers/md/dm-vdo/slab-depot.c

1175

vdo_enter_read_only_mode(slab->allocator->depot->vdo, result);

drivers/md/dm-vdo/slab-depot.c

1176

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

1195

block_offset = (block - block->slab->reference_blocks);

drivers/md/dm-vdo/slab-depot.c

1196

pbn = (block->slab->ref_counts_origin + block_offset);

drivers/md/dm-vdo/slab-depot.c

1212

WRITE_ONCE(block->slab->allocator->ref_counts_statistics.blocks_written,

drivers/md/dm-vdo/slab-depot.c

1213

block->slab->allocator->ref_counts_statistics.blocks_written + 1);

drivers/md/dm-vdo/slab-depot.c

1223

struct vdo_slab *slab = journal->slab;

drivers/md/dm-vdo/slab-depot.c

1224

block_count_t write_count = vdo_waitq_num_waiters(&slab->dirty_blocks);

drivers/md/dm-vdo/slab-depot.c

1239

vdo_waitq_notify_next_waiter(&slab->dirty_blocks,

drivers/md/dm-vdo/slab-depot.c

1240

launch_reference_block_write, slab);

drivers/md/dm-vdo/slab-depot.c

1274

vdo_waitq_enqueue_waiter(&block->slab->dirty_blocks, &block->waiter);

drivers/md/dm-vdo/slab-depot.c

1282

static struct reference_block * __must_check get_reference_block(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

1285

return &slab->reference_blocks[index / COUNTS_PER_BLOCK];

drivers/md/dm-vdo/slab-depot.c

1297

static int __must_check slab_block_number_from_pbn(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

1303

if (pbn < slab->start)

drivers/md/dm-vdo/slab-depot.c

1306

slab_block_number = pbn - slab->start;

drivers/md/dm-vdo/slab-depot.c

1307

if (slab_block_number >= slab->allocator->depot->slab_config.data_blocks)

drivers/md/dm-vdo/slab-depot.c

1320

static int __must_check get_reference_counter(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

1325

int result = slab_block_number_from_pbn(slab, pbn, &index);

drivers/md/dm-vdo/slab-depot.c

1330

*counter_ptr = &slab->counters[index];

drivers/md/dm-vdo/slab-depot.c

1335

static unsigned int calculate_slab_priority(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1337

block_count_t free_blocks = slab->free_blocks;

drivers/md/dm-vdo/slab-depot.c

1338

unsigned int unopened_slab_priority = slab->allocator->unopened_slab_priority;

drivers/md/dm-vdo/slab-depot.c

136

static bool is_slab_journal_blank(const struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1362

if (is_slab_journal_blank(slab))

drivers/md/dm-vdo/slab-depot.c

1374

static void prioritize_slab(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1376

VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry),

drivers/md/dm-vdo/slab-depot.c

1378

slab->priority = calculate_slab_priority(slab);

drivers/md/dm-vdo/slab-depot.c

1379

vdo_priority_table_enqueue(slab->allocator->prioritized_slabs,

drivers/md/dm-vdo/slab-depot.c

138

return ((slab->journal.tail == 1) &&

drivers/md/dm-vdo/slab-depot.c

1380

slab->priority, &slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

1388

static void adjust_free_block_count(struct vdo_slab *slab, bool incremented)

drivers/md/dm-vdo/slab-depot.c

139

(slab->journal.tail_header.entry_count == 0));

drivers/md/dm-vdo/slab-depot.c

1390

struct block_allocator *allocator = slab->allocator;

drivers/md/dm-vdo/slab-depot.c

1396

if (slab == allocator->open_slab)

drivers/md/dm-vdo/slab-depot.c

1400

if (slab->priority == calculate_slab_priority(slab))

drivers/md/dm-vdo/slab-depot.c

1407

vdo_priority_table_remove(allocator->prioritized_slabs, &slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

1408

prioritize_slab(slab);

drivers/md/dm-vdo/slab-depot.c

1423

static int increment_for_data(struct vdo_slab *slab, struct reference_block *block,

drivers/md/dm-vdo/slab-depot.c

1433

slab->free_blocks--;

drivers/md/dm-vdo/slab-depot.c

1435

adjust_free_block_count(slab, false);

drivers/md/dm-vdo/slab-depot.c

1448

slab->slab_number, block_number);

drivers/md/dm-vdo/slab-depot.c

1470

static int decrement_for_data(struct vdo_slab *slab, struct reference_block *block,

drivers/md/dm-vdo/slab-depot.c

1480

block_number, slab->slab_number);

drivers/md/dm-vdo/slab-depot.c

1501

slab->free_blocks++;

drivers/md/dm-vdo/slab-depot.c

1503

adjust_free_block_count(slab, true);

drivers/md/dm-vdo/slab-depot.c

151

struct list_head *dirty_list = &journal->slab->allocator->dirty_slab_journals;

drivers/md/dm-vdo/slab-depot.c

1533

static int increment_for_block_map(struct vdo_slab *slab, struct reference_block *block,

drivers/md/dm-vdo/slab-depot.c

1544

slab->slab_number, block_number);

drivers/md/dm-vdo/slab-depot.c

1549

slab->free_blocks--;

drivers/md/dm-vdo/slab-depot.c

1551

adjust_free_block_count(slab, false);

drivers/md/dm-vdo/slab-depot.c

1559

slab->slab_number, block_number);

drivers/md/dm-vdo/slab-depot.c

1569

*counter_ptr, slab->slab_number,

drivers/md/dm-vdo/slab-depot.c

1593

static int update_reference_count(struct vdo_slab *slab, struct reference_block *block,

drivers/md/dm-vdo/slab-depot.c

1600

vdo_refcount_t *counter_ptr = &slab->counters[block_number];

drivers/md/dm-vdo/slab-depot.c

1605

result = decrement_for_data(slab, block, block_number, old_status,

drivers/md/dm-vdo/slab-depot.c

1613

result = increment_for_data(slab, block, block_number, old_status,

drivers/md/dm-vdo/slab-depot.c

1616

result = increment_for_block_map(slab, block, block_number, old_status,

drivers/md/dm-vdo/slab-depot.c

1625

slab->slab_journal_point = *slab_journal_point;

drivers/md/dm-vdo/slab-depot.c

1630

static int __must_check adjust_reference_count(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

1639

if (!is_slab_open(slab))

drivers/md/dm-vdo/slab-depot.c

1642

result = slab_block_number_from_pbn(slab, updater->zpbn.pbn, &block_number);

drivers/md/dm-vdo/slab-depot.c

1646

block = get_reference_block(slab, block_number);

drivers/md/dm-vdo/slab-depot.c

1647

result = update_reference_count(slab, block, block_number, slab_journal_point,

drivers/md/dm-vdo/slab-depot.c

1665

adjust_slab_journal_block_reference(&slab->journal, entry_lock, -1);

drivers/md/dm-vdo/slab-depot.c

170

static void check_if_slab_drained(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1712

zone_count_t zone_number = journal->slab->allocator->zone_number;

drivers/md/dm-vdo/slab-depot.c

1728

if (journal->slab->status != VDO_SLAB_REBUILT) {

drivers/md/dm-vdo/slab-depot.c

173

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

1738

result = adjust_reference_count(journal->slab, updater,

drivers/md/dm-vdo/slab-depot.c

176

if (!vdo_is_state_draining(&slab->state) ||

drivers/md/dm-vdo/slab-depot.c

1783

(journal->slab->status == VDO_SLAB_REBUILDING)) {

drivers/md/dm-vdo/slab-depot.c

1817

save_dirty_reference_blocks(journal->slab);

drivers/md/dm-vdo/slab-depot.c

182

(slab->active_count > 0))

drivers/md/dm-vdo/slab-depot.c

1843

save_dirty_reference_blocks(journal->slab);

drivers/md/dm-vdo/slab-depot.c

1855

struct vdo_slab *slab = journal->slab;

drivers/md/dm-vdo/slab-depot.c

186

code = vdo_get_admin_state_code(&slab->state);

drivers/md/dm-vdo/slab-depot.c

1866

for (i = 0; i < slab->reference_block_count; i++) {

drivers/md/dm-vdo/slab-depot.c

1867

slab->reference_blocks[i].slab_journal_lock = 1;

drivers/md/dm-vdo/slab-depot.c

1868

dirty_block(&slab->reference_blocks[i]);

drivers/md/dm-vdo/slab-depot.c

187

read_only = vdo_is_read_only(slab->allocator->depot->vdo);

drivers/md/dm-vdo/slab-depot.c

1872

slab->reference_block_count);

drivers/md/dm-vdo/slab-depot.c

1883

if (vdo_is_state_draining(&journal->slab->state) &&

drivers/md/dm-vdo/slab-depot.c

1884

!vdo_is_state_suspending(&journal->slab->state) &&

drivers/md/dm-vdo/slab-depot.c

189

vdo_waitq_has_waiters(&slab->dirty_blocks) &&

drivers/md/dm-vdo/slab-depot.c

1894

static void reset_search_cursor(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1896

struct search_cursor *cursor = &slab->search_cursor;

drivers/md/dm-vdo/slab-depot.c

1900

cursor->end_index = min_t(u32, COUNTS_PER_BLOCK, slab->block_count);

drivers/md/dm-vdo/slab-depot.c

1912

static bool advance_search_cursor(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

1914

struct search_cursor *cursor = &slab->search_cursor;

drivers/md/dm-vdo/slab-depot.c

1921

reset_search_cursor(slab);

drivers/md/dm-vdo/slab-depot.c

1931

cursor->end_index = slab->block_count;

drivers/md/dm-vdo/slab-depot.c

194

vdo_finish_draining_with_result(&slab->state,

drivers/md/dm-vdo/slab-depot.c

1954

struct vdo_slab *slab = vdo_get_slab(depot, pbn);

drivers/md/dm-vdo/slab-depot.c

1960

result = slab_block_number_from_pbn(slab, pbn, &block_number);

drivers/md/dm-vdo/slab-depot.c

1964

block = get_reference_block(slab, block_number);

drivers/md/dm-vdo/slab-depot.c

1965

result = update_reference_count(slab, block, block_number, NULL,

drivers/md/dm-vdo/slab-depot.c

1985

static int replay_reference_count_change(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

1990

struct reference_block *block = get_reference_block(slab, entry.sbn);

drivers/md/dm-vdo/slab-depot.c

2003

result = update_reference_count(slab, block, entry.sbn, entry_point,

drivers/md/dm-vdo/slab-depot.c

2051

static bool find_free_block(const struct vdo_slab *slab, slab_block_number *index_ptr)

drivers/md/dm-vdo/slab-depot.c

2054

slab_block_number next_index = slab->search_cursor.index;

drivers/md/dm-vdo/slab-depot.c

2055

slab_block_number end_index = slab->search_cursor.end_index;

drivers/md/dm-vdo/slab-depot.c

2056

u8 *next_counter = &slab->counters[next_index];

drivers/md/dm-vdo/slab-depot.c

2057

u8 *end_counter = &slab->counters[end_index];

drivers/md/dm-vdo/slab-depot.c

2108

static bool search_current_reference_block(const struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

2112

return ((slab->search_cursor.block->allocated_count < COUNTS_PER_BLOCK) &&

drivers/md/dm-vdo/slab-depot.c

2113

find_free_block(slab, free_index_ptr));

drivers/md/dm-vdo/slab-depot.c

2127

static bool search_reference_blocks(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

2131

if (search_current_reference_block(slab, free_index_ptr))

drivers/md/dm-vdo/slab-depot.c

2135

while (advance_search_cursor(slab)) {

drivers/md/dm-vdo/slab-depot.c

2136

if (search_current_reference_block(slab, free_index_ptr))

drivers/md/dm-vdo/slab-depot.c

2148

static void make_provisional_reference(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

2151

struct reference_block *block = get_reference_block(slab, block_number);

drivers/md/dm-vdo/slab-depot.c

2157

slab->counters[block_number] = PROVISIONAL_REFERENCE_COUNT;

drivers/md/dm-vdo/slab-depot.c

2161

slab->free_blocks--;

drivers/md/dm-vdo/slab-depot.c

2168

static void dirty_all_reference_blocks(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2172

for (i = 0; i < slab->reference_block_count; i++)

drivers/md/dm-vdo/slab-depot.c

2173

dirty_block(&slab->reference_blocks[i]);

drivers/md/dm-vdo/slab-depot.c

2276

struct vdo_slab *slab = block->slab;

drivers/md/dm-vdo/slab-depot.c

2286

if (vdo_before_journal_point(&slab->slab_journal_point,

drivers/md/dm-vdo/slab-depot.c

2288

slab->slab_journal_point = block->commit_points[i];

drivers/md/dm-vdo/slab-depot.c

2293

size_t block_index = block - block->slab->reference_blocks;

drivers/md/dm-vdo/slab-depot.c

2296

i, block_index, block->slab->slab_number);

drivers/md/dm-vdo/slab-depot.c

2312

struct vdo_slab *slab = block->slab;

drivers/md/dm-vdo/slab-depot.c

2321

slab->free_blocks -= block->allocated_count;

drivers/md/dm-vdo/slab-depot.c

2324

slab->active_count -= block_count;

drivers/md/dm-vdo/slab-depot.c

2326

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

2335

block->slab->allocator->thread_id);

drivers/md/dm-vdo/slab-depot.c

2350

u32 block_offset = block - block->slab->reference_blocks;

drivers/md/dm-vdo/slab-depot.c

2351

u32 max_block_count = block->slab->reference_block_count - block_offset;

drivers/md/dm-vdo/slab-depot.c

2355

vdo_submit_metadata_vio_with_size(vio, block->slab->ref_counts_origin + block_offset,

drivers/md/dm-vdo/slab-depot.c

2365

static void load_reference_blocks(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2368

u64 blocks_per_vio = slab->allocator->refcount_blocks_per_big_vio;

drivers/md/dm-vdo/slab-depot.c

2369

struct vio_pool *pool = slab->allocator->refcount_big_vio_pool;

drivers/md/dm-vdo/slab-depot.c

2372

pool = slab->allocator->vio_pool;

drivers/md/dm-vdo/slab-depot.c

2376

slab->free_blocks = slab->block_count;

drivers/md/dm-vdo/slab-depot.c

2377

slab->active_count = slab->reference_block_count;

drivers/md/dm-vdo/slab-depot.c

2378

for (i = 0; i < slab->reference_block_count; i += blocks_per_vio) {

drivers/md/dm-vdo/slab-depot.c

2379

struct vdo_waiter *waiter = &slab->reference_blocks[i].waiter;

drivers/md/dm-vdo/slab-depot.c

2393

static void drain_slab(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2397

const struct admin_state_code *state = vdo_get_admin_state_code(&slab->state);

drivers/md/dm-vdo/slab-depot.c

2404

commit_tail(&slab->journal);

drivers/md/dm-vdo/slab-depot.c

2406

if ((state == VDO_ADMIN_STATE_RECOVERING) || (slab->counters == NULL))

drivers/md/dm-vdo/slab-depot.c

2410

load = slab->allocator->summary_entries[slab->slab_number].load_ref_counts;

drivers/md/dm-vdo/slab-depot.c

2413

load_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

2419

dirty_all_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

2427

block_count_t data_blocks = slab->allocator->depot->slab_config.data_blocks;

drivers/md/dm-vdo/slab-depot.c

2429

if (load || (slab->free_blocks != data_blocks) ||

drivers/md/dm-vdo/slab-depot.c

2430

!is_slab_journal_blank(slab)) {

drivers/md/dm-vdo/slab-depot.c

2431

dirty_all_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

2435

save = (slab->status == VDO_SLAB_REBUILT);

drivers/md/dm-vdo/slab-depot.c

2437

vdo_finish_draining_with_result(&slab->state, VDO_SUCCESS);

drivers/md/dm-vdo/slab-depot.c

2442

save_dirty_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

2445

static int allocate_slab_counters(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2450

result = VDO_ASSERT(slab->reference_blocks == NULL,

drivers/md/dm-vdo/slab-depot.c

2452

slab->slab_number);

drivers/md/dm-vdo/slab-depot.c

2456

result = vdo_allocate(slab->reference_block_count, __func__, &slab->reference_blocks);

drivers/md/dm-vdo/slab-depot.c

2464

bytes = (slab->reference_block_count * COUNTS_PER_BLOCK) + (2 * BYTES_PER_WORD);

drivers/md/dm-vdo/slab-depot.c

2465

result = vdo_allocate(bytes, "ref counts array", &slab->counters);

drivers/md/dm-vdo/slab-depot.c

2467

vdo_free(vdo_forget(slab->reference_blocks));

drivers/md/dm-vdo/slab-depot.c

2471

slab->search_cursor.first_block = slab->reference_blocks;

drivers/md/dm-vdo/slab-depot.c

2472

slab->search_cursor.last_block = &slab->reference_blocks[slab->reference_block_count - 1];

drivers/md/dm-vdo/slab-depot.c

2473

reset_search_cursor(slab);

drivers/md/dm-vdo/slab-depot.c

2475

for (index = 0; index < slab->reference_block_count; index++) {

drivers/md/dm-vdo/slab-depot.c

2476

slab->reference_blocks[index] = (struct reference_block) {

drivers/md/dm-vdo/slab-depot.c

2477

.slab = slab,

drivers/md/dm-vdo/slab-depot.c

2484

static int allocate_counters_if_clean(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2486

if (vdo_is_state_clean_load(&slab->state))

drivers/md/dm-vdo/slab-depot.c

2487

return allocate_slab_counters(slab);

drivers/md/dm-vdo/slab-depot.c

2496

struct vdo_slab *slab = journal->slab;

drivers/md/dm-vdo/slab-depot.c

2504

(header.nonce == slab->allocator->nonce)) {

drivers/md/dm-vdo/slab-depot.c

2511

journal->head = (slab->allocator->summary_entries[slab->slab_number].is_dirty ?

drivers/md/dm-vdo/slab-depot.c

2518

vdo_finish_loading_with_result(&slab->state, allocate_counters_if_clean(slab));

drivers/md/dm-vdo/slab-depot.c

2527

journal->slab->allocator->thread_id);

drivers/md/dm-vdo/slab-depot.c

2538

vdo_finish_loading_with_result(&journal->slab->state, result);

drivers/md/dm-vdo/slab-depot.c

2553

struct vdo_slab *slab = journal->slab;

drivers/md/dm-vdo/slab-depot.c

2557

slab->allocator->summary_entries[slab->slab_number].tail_block_offset;

drivers/md/dm-vdo/slab-depot.c

2568

vio->completion.callback_thread_id = slab->allocator->thread_id;

drivers/md/dm-vdo/slab-depot.c

2569

vdo_submit_metadata_vio(vio, slab->journal_origin + tail_block,

drivers/md/dm-vdo/slab-depot.c

2578

static void load_slab_journal(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2580

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

2583

last_commit_point = slab->allocator->summary_entries[slab->slab_number].tail_block_offset;

drivers/md/dm-vdo/slab-depot.c

2585

!slab->allocator->summary_entries[slab->slab_number].load_ref_counts) {

drivers/md/dm-vdo/slab-depot.c

2594

vdo_finish_loading_with_result(&slab->state,

drivers/md/dm-vdo/slab-depot.c

2595

allocate_counters_if_clean(slab));

drivers/md/dm-vdo/slab-depot.c

2600

acquire_vio_from_pool(slab->allocator->vio_pool, &journal->resource_waiter);

drivers/md/dm-vdo/slab-depot.c

2603

static void register_slab_for_scrubbing(struct vdo_slab *slab, bool high_priority)

drivers/md/dm-vdo/slab-depot.c

2605

struct slab_scrubber *scrubber = &slab->allocator->scrubber;

drivers/md/dm-vdo/slab-depot.c

2607

VDO_ASSERT_LOG_ONLY((slab->status != VDO_SLAB_REBUILT),

drivers/md/dm-vdo/slab-depot.c

2610

if (slab->status != VDO_SLAB_REQUIRES_SCRUBBING)

drivers/md/dm-vdo/slab-depot.c

2613

list_del_init(&slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

2614

if (!slab->was_queued_for_scrubbing) {

drivers/md/dm-vdo/slab-depot.c

2616

slab->was_queued_for_scrubbing = true;

drivers/md/dm-vdo/slab-depot.c

2620

slab->status = VDO_SLAB_REQUIRES_HIGH_PRIORITY_SCRUBBING;

drivers/md/dm-vdo/slab-depot.c

2621

list_add_tail(&slab->allocq_entry, &scrubber->high_priority_slabs);

drivers/md/dm-vdo/slab-depot.c

2625

list_add_tail(&slab->allocq_entry, &scrubber->slabs);

drivers/md/dm-vdo/slab-depot.c

2629

static void queue_slab(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2631

struct block_allocator *allocator = slab->allocator;

drivers/md/dm-vdo/slab-depot.c

2635

VDO_ASSERT_LOG_ONLY(list_empty(&slab->allocq_entry),

drivers/md/dm-vdo/slab-depot.c

2641

free_blocks = slab->free_blocks;

drivers/md/dm-vdo/slab-depot.c

2644

slab->slab_number, (unsigned long long) free_blocks,

drivers/md/dm-vdo/slab-depot.c

2651

if (slab->status != VDO_SLAB_REBUILT) {

drivers/md/dm-vdo/slab-depot.c

2652

register_slab_for_scrubbing(slab, false);

drivers/md/dm-vdo/slab-depot.c

2656

if (!vdo_is_state_resuming(&slab->state)) {

drivers/md/dm-vdo/slab-depot.c

2664

if (!is_slab_journal_blank(slab)) {

drivers/md/dm-vdo/slab-depot.c

2671

reopen_slab_journal(slab);

drivers/md/dm-vdo/slab-depot.c

2673

prioritize_slab(slab);

drivers/md/dm-vdo/slab-depot.c

2679

struct vdo_slab *slab = container_of(state, struct vdo_slab, state);

drivers/md/dm-vdo/slab-depot.c

2685

slab->status = VDO_SLAB_REBUILDING;

drivers/md/dm-vdo/slab-depot.c

2687

drain_slab(slab);

drivers/md/dm-vdo/slab-depot.c

2688

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

2693

load_slab_journal(slab);

drivers/md/dm-vdo/slab-depot.c

2698

queue_slab(slab);

drivers/md/dm-vdo/slab-depot.c

2714

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.c

2716

slab = list_first_entry_or_null(&scrubber->high_priority_slabs,

drivers/md/dm-vdo/slab-depot.c

2718

if (slab != NULL)

drivers/md/dm-vdo/slab-depot.c

2719

return slab;

drivers/md/dm-vdo/slab-depot.c

2817

struct vdo_slab *slab = scrubber->slab;

drivers/md/dm-vdo/slab-depot.c

2819

slab->status = VDO_SLAB_REBUILT;

drivers/md/dm-vdo/slab-depot.c

2820

queue_slab(slab);

drivers/md/dm-vdo/slab-depot.c

2821

reopen_slab_journal(slab);

drivers/md/dm-vdo/slab-depot.c

2861

sequence_number_t block_number, struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

2868

slab_block_number max_sbn = slab->end - slab->start;

drivers/md/dm-vdo/slab-depot.c

2883

result = replay_reference_count_change(slab, &entry_point, entry);

drivers/md/dm-vdo/slab-depot.c

2890

entry.sbn, slab->slab_number);

drivers/md/dm-vdo/slab-depot.c

2910

struct vdo_slab *slab = scrubber->slab;

drivers/md/dm-vdo/slab-depot.c

2911

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

2924

struct journal_point ref_counts_point = slab->slab_journal_point;

drivers/md/dm-vdo/slab-depot.c

2936

if ((header.nonce != slab->allocator->nonce) ||

drivers/md/dm-vdo/slab-depot.c

2944

slab->slab_number);

drivers/md/dm-vdo/slab-depot.c

2949

result = apply_block_entries(block, header.entry_count, sequence, slab);

drivers/md/dm-vdo/slab-depot.c

2976

slab->allocator->thread_id, completion->parent);

drivers/md/dm-vdo/slab-depot.c

2977

vdo_start_operation_with_waiter(&slab->state,

drivers/md/dm-vdo/slab-depot.c

2988

scrubber->slab->allocator->thread_id);

drivers/md/dm-vdo/slab-depot.c

3001

struct vdo_slab *slab = scrubber->slab;

drivers/md/dm-vdo/slab-depot.c

3003

if (!slab->allocator->summary_entries[slab->slab_number].is_dirty) {

drivers/md/dm-vdo/slab-depot.c

3008

vdo_submit_metadata_vio(&scrubber->vio, slab->journal_origin,

drivers/md/dm-vdo/slab-depot.c

3020

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.c

3033

slab = get_next_slab(scrubber);

drivers/md/dm-vdo/slab-depot.c

3034

if ((slab == NULL) ||

drivers/md/dm-vdo/slab-depot.c

3043

list_del_init(&slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

3044

scrubber->slab = slab;

drivers/md/dm-vdo/slab-depot.c

3046

slab->allocator->thread_id, completion->parent);

drivers/md/dm-vdo/slab-depot.c

3047

vdo_start_operation_with_waiter(&slab->state, VDO_ADMIN_STATE_SCRUBBING,

drivers/md/dm-vdo/slab-depot.c

3084

struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

3087

allocator->last_slab = slab->slab_number;

drivers/md/dm-vdo/slab-depot.c

3130

struct vdo_slab *slab = iterator->next;

drivers/md/dm-vdo/slab-depot.c

3132

if ((slab == NULL) || (slab->slab_number < iterator->end + iterator->stride))

drivers/md/dm-vdo/slab-depot.c

3135

iterator->next = iterator->slabs[slab->slab_number - iterator->stride];

drivers/md/dm-vdo/slab-depot.c

3137

return slab;

drivers/md/dm-vdo/slab-depot.c

3172

struct vdo_slab *slab = next_slab(&iterator);

drivers/md/dm-vdo/slab-depot.c

3174

vdo_waitq_notify_all_waiters(&slab->journal.entry_waiters,

drivers/md/dm-vdo/slab-depot.c

3175

abort_waiter, &slab->journal);

drivers/md/dm-vdo/slab-depot.c

3176

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

3191

int vdo_acquire_provisional_reference(struct vdo_slab *slab, physical_block_number_t pbn,

drivers/md/dm-vdo/slab-depot.c

3200

if (!is_slab_open(slab))

drivers/md/dm-vdo/slab-depot.c

3203

result = slab_block_number_from_pbn(slab, pbn, &block_number);

drivers/md/dm-vdo/slab-depot.c

3207

if (slab->counters[block_number] == EMPTY_REFERENCE_COUNT) {

drivers/md/dm-vdo/slab-depot.c

3208

make_provisional_reference(slab, block_number);

drivers/md/dm-vdo/slab-depot.c

3214

adjust_free_block_count(slab, false);

drivers/md/dm-vdo/slab-depot.c

3219

static int __must_check allocate_slab_block(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.c

3224

if (!is_slab_open(slab))

drivers/md/dm-vdo/slab-depot.c

3227

if (!search_reference_blocks(slab, &free_index))

drivers/md/dm-vdo/slab-depot.c

3230

VDO_ASSERT_LOG_ONLY((slab->counters[free_index] == EMPTY_REFERENCE_COUNT),

drivers/md/dm-vdo/slab-depot.c

3232

make_provisional_reference(slab, free_index);

drivers/md/dm-vdo/slab-depot.c

3233

adjust_free_block_count(slab, false);

drivers/md/dm-vdo/slab-depot.c

3239

slab->search_cursor.index = (free_index + 1);

drivers/md/dm-vdo/slab-depot.c

3241

*block_number_ptr = slab->start + free_index;

drivers/md/dm-vdo/slab-depot.c

3249

static void open_slab(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

3251

reset_search_cursor(slab);

drivers/md/dm-vdo/slab-depot.c

3252

if (is_slab_journal_blank(slab)) {

drivers/md/dm-vdo/slab-depot.c

3253

WRITE_ONCE(slab->allocator->statistics.slabs_opened,

drivers/md/dm-vdo/slab-depot.c

3254

slab->allocator->statistics.slabs_opened + 1);

drivers/md/dm-vdo/slab-depot.c

3255

dirty_all_reference_blocks(slab);

drivers/md/dm-vdo/slab-depot.c

3257

WRITE_ONCE(slab->allocator->statistics.slabs_reopened,

drivers/md/dm-vdo/slab-depot.c

3258

slab->allocator->statistics.slabs_reopened + 1);

drivers/md/dm-vdo/slab-depot.c

3261

slab->allocator->open_slab = slab;

drivers/md/dm-vdo/slab-depot.c

3326

struct vdo_slab *slab = vdo_get_slab(completion->vdo->depot, updater->zpbn.pbn);

drivers/md/dm-vdo/slab-depot.c

3328

if (!is_slab_open(slab)) {

drivers/md/dm-vdo/slab-depot.c

3338

vdo_waitq_enqueue_waiter(&slab->journal.entry_waiters, &updater->waiter);

drivers/md/dm-vdo/slab-depot.c

3339

if ((slab->status != VDO_SLAB_REBUILT) && requires_reaping(&slab->journal))

drivers/md/dm-vdo/slab-depot.c

3340

register_slab_for_scrubbing(slab, true);

drivers/md/dm-vdo/slab-depot.c

3342

add_entries(&slab->journal);

drivers/md/dm-vdo/slab-depot.c

3442

struct vdo_slab *slab = next_slab(&iterator);

drivers/md/dm-vdo/slab-depot.c

3444

list_del_init(&slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

3446

vdo_start_operation_with_waiter(&slab->state, operation,

drivers/md/dm-vdo/slab-depot.c

3492

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.c

3503

slab = next_slab(&allocator->slabs_to_erase);

drivers/md/dm-vdo/slab-depot.c

3504

pbn = slab->journal_origin - depot->vdo->geometry.bio_offset;

drivers/md/dm-vdo/slab-depot.c

3608

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.c

361

static void update_slab_summary_entry(struct vdo_slab *slab, struct vdo_waiter *waiter,

drivers/md/dm-vdo/slab-depot.c

3613

slab = depot->slabs[current_slab_status.slab_number];

drivers/md/dm-vdo/slab-depot.c

3616

(!allocator->summary_entries[slab->slab_number].load_ref_counts &&

drivers/md/dm-vdo/slab-depot.c

3618

queue_slab(slab);

drivers/md/dm-vdo/slab-depot.c

3622

slab->status = VDO_SLAB_REQUIRES_SCRUBBING;

drivers/md/dm-vdo/slab-depot.c

3623

journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

3627

register_slab_for_scrubbing(slab, high_priority);

drivers/md/dm-vdo/slab-depot.c

366

u8 index = slab->slab_number / VDO_SLAB_SUMMARY_ENTRIES_PER_BLOCK;

drivers/md/dm-vdo/slab-depot.c

3660

struct vdo_slab *slab = next_slab(&iterator);

drivers/md/dm-vdo/slab-depot.c

3661

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

3663

if (slab->reference_blocks != NULL) {

drivers/md/dm-vdo/slab-depot.c

3665

vdo_log_info("slab %u: P%u, %llu free", slab->slab_number,

drivers/md/dm-vdo/slab-depot.c

3666

slab->priority,

drivers/md/dm-vdo/slab-depot.c

3667

(unsigned long long) slab->free_blocks);

drivers/md/dm-vdo/slab-depot.c

3669

vdo_log_info("slab %u: status %s", slab->slab_number,

drivers/md/dm-vdo/slab-depot.c

367

struct block_allocator *allocator = slab->allocator;

drivers/md/dm-vdo/slab-depot.c

3670

status_to_string(slab->status));

drivers/md/dm-vdo/slab-depot.c

3690

if (slab->counters != NULL) {

drivers/md/dm-vdo/slab-depot.c

3693

slab->free_blocks, slab->block_count,

drivers/md/dm-vdo/slab-depot.c

3694

slab->reference_block_count,

drivers/md/dm-vdo/slab-depot.c

3695

vdo_waitq_num_waiters(&slab->dirty_blocks),

drivers/md/dm-vdo/slab-depot.c

3696

slab->active_count,

drivers/md/dm-vdo/slab-depot.c

3697

(unsigned long long) slab->slab_journal_point.sequence_number,

drivers/md/dm-vdo/slab-depot.c

3698

slab->slab_journal_point.entry_count);

drivers/md/dm-vdo/slab-depot.c

3720

static void free_slab(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

3722

if (slab == NULL)

drivers/md/dm-vdo/slab-depot.c

3725

list_del(&slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

3726

vdo_free(vdo_forget(slab->journal.block));

drivers/md/dm-vdo/slab-depot.c

3727

vdo_free(vdo_forget(slab->journal.locks));

drivers/md/dm-vdo/slab-depot.c

3728

vdo_free(vdo_forget(slab->counters));

drivers/md/dm-vdo/slab-depot.c

3729

vdo_free(vdo_forget(slab->reference_blocks));

drivers/md/dm-vdo/slab-depot.c

3730

vdo_free(slab);

drivers/md/dm-vdo/slab-depot.c

3733

static int initialize_slab_journal(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

3735

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

3736

const struct slab_config *slab_config = &slab->allocator->depot->slab_config;

drivers/md/dm-vdo/slab-depot.c

3748

journal->slab = slab;

drivers/md/dm-vdo/slab-depot.c

3755

journal->events = &slab->allocator->slab_journal_statistics;

drivers/md/dm-vdo/slab-depot.c

3756

journal->recovery_journal = slab->allocator->depot->vdo->recovery_journal;

drivers/md/dm-vdo/slab-depot.c

3773

journal->tail_header.nonce = slab->allocator->nonce;

drivers/md/dm-vdo/slab-depot.c

3796

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.c

3799

result = vdo_allocate(1, __func__, &slab);

drivers/md/dm-vdo/slab-depot.c

3803

*slab = (struct vdo_slab) {

drivers/md/dm-vdo/slab-depot.c

3816

INIT_LIST_HEAD(&slab->allocq_entry);

drivers/md/dm-vdo/slab-depot.c

3818

result = initialize_slab_journal(slab);

drivers/md/dm-vdo/slab-depot.c

3820

free_slab(slab);

drivers/md/dm-vdo/slab-depot.c

3825

vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NEW);

drivers/md/dm-vdo/slab-depot.c

3826

result = allocate_slab_counters(slab);

drivers/md/dm-vdo/slab-depot.c

3828

free_slab(slab);

drivers/md/dm-vdo/slab-depot.c

3832

vdo_set_admin_state_code(&slab->state, VDO_ADMIN_STATE_NORMAL_OPERATION);

drivers/md/dm-vdo/slab-depot.c

3835

*slab_ptr = slab;

drivers/md/dm-vdo/slab-depot.c

385

entry = &allocator->summary_entries[slab->slab_number];

drivers/md/dm-vdo/slab-depot.c

3928

vdo_is_read_only(journal->slab->allocator->depot->vdo))

drivers/md/dm-vdo/slab-depot.c

405

check_if_slab_drained(journal->slab);

drivers/md/dm-vdo/slab-depot.c

4214

struct vdo_slab *slab = depot->new_slabs[i];

drivers/md/dm-vdo/slab-depot.c

4216

register_slab_with_allocator(slab->allocator, slab);

drivers/md/dm-vdo/slab-depot.c

441

journal->slab->allocator->thread_id);

drivers/md/dm-vdo/slab-depot.c

4454

struct vdo_slab *slab = vdo_get_slab(depot, pbn);

drivers/md/dm-vdo/slab-depot.c

4458

if ((slab == NULL) || (slab->status != VDO_SLAB_REBUILT))

drivers/md/dm-vdo/slab-depot.c

4461

result = get_reference_counter(slab, pbn, &counter_ptr);

drivers/md/dm-vdo/slab-depot.c

474

if ((journal->slab->status != VDO_SLAB_REBUILT) ||

drivers/md/dm-vdo/slab-depot.c

475

!vdo_is_state_normal(&journal->slab->state) ||

drivers/md/dm-vdo/slab-depot.c

476

vdo_is_read_only(journal->slab->allocator->depot->vdo)) {

drivers/md/dm-vdo/slab-depot.c

4817

struct vdo_slab *slab = depot->new_slabs[i];

drivers/md/dm-vdo/slab-depot.c

4819

if (slab->allocator == allocator)

drivers/md/dm-vdo/slab-depot.c

4820

register_slab_with_allocator(allocator, slab);

drivers/md/dm-vdo/slab-depot.c

510

acquire_vio_from_pool(journal->slab->allocator->vio_pool,

drivers/md/dm-vdo/slab-depot.c

531

if (journal->slab->status == VDO_SLAB_REPLAYING) {

drivers/md/dm-vdo/slab-depot.c

54

static bool is_slab_open(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

56

return (!vdo_is_state_quiescing(&slab->state) &&

drivers/md/dm-vdo/slab-depot.c

57

!vdo_is_state_quiescent(&slab->state));

drivers/md/dm-vdo/slab-depot.c

577

vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result);

drivers/md/dm-vdo/slab-depot.c

578

check_if_slab_drained(journal->slab);

drivers/md/dm-vdo/slab-depot.c

595

zone_count_t zone_number = journal->slab->allocator->zone_number;

drivers/md/dm-vdo/slab-depot.c

626

struct vdo_slab *slab = journal->slab;

drivers/md/dm-vdo/slab-depot.c

629

vdo_is_read_only(journal->slab->allocator->depot->vdo) ||

drivers/md/dm-vdo/slab-depot.c

631

check_if_slab_drained(slab);

drivers/md/dm-vdo/slab-depot.c

635

if (slab->status != VDO_SLAB_REBUILT) {

drivers/md/dm-vdo/slab-depot.c

636

u8 hint = slab->allocator->summary_entries[slab->slab_number].fullness_hint;

drivers/md/dm-vdo/slab-depot.c

638

free_block_count = ((block_count_t) hint) << slab->allocator->depot->hint_shift;

drivers/md/dm-vdo/slab-depot.c

640

free_block_count = slab->free_blocks;

drivers/md/dm-vdo/slab-depot.c

652

update_slab_summary_entry(slab, &journal->slab_summary_waiter,

drivers/md/dm-vdo/slab-depot.c

661

static void reopen_slab_journal(struct vdo_slab *slab)

drivers/md/dm-vdo/slab-depot.c

663

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

68

return ((journal->slab->status != VDO_SLAB_REBUILDING) &&

drivers/md/dm-vdo/slab-depot.c

709

vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result);

drivers/md/dm-vdo/slab-depot.c

710

check_if_slab_drained(journal->slab);

drivers/md/dm-vdo/slab-depot.c

734

continue_vio_after_io(vio, complete_write, journal->slab->allocator->thread_id);

drivers/md/dm-vdo/slab-depot.c

773

block_number = journal->slab->journal_origin +

drivers/md/dm-vdo/slab-depot.c

790

operation = vdo_get_admin_state_code(&journal->slab->state);

drivers/md/dm-vdo/slab-depot.c

792

vdo_finish_operation(&journal->slab->state,

drivers/md/dm-vdo/slab-depot.c

793

(vdo_is_read_only(journal->slab->allocator->depot->vdo) ?

drivers/md/dm-vdo/slab-depot.c

815

if (vdo_is_read_only(journal->slab->allocator->depot->vdo) ||

drivers/md/dm-vdo/slab-depot.c

834

acquire_vio_from_pool(journal->slab->allocator->vio_pool,

drivers/md/dm-vdo/slab-depot.c

918

vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo, result);

drivers/md/dm-vdo/slab-depot.c

927

vdo_enter_read_only_mode(journal->slab->allocator->depot->vdo,

drivers/md/dm-vdo/slab-depot.c

934

pbn - journal->slab->start, operation, increment);

drivers/md/dm-vdo/slab-depot.c

957

bool vdo_attempt_replay_into_slab(struct vdo_slab *slab, physical_block_number_t pbn,

drivers/md/dm-vdo/slab-depot.c

962

struct slab_journal *journal = &slab->journal;

drivers/md/dm-vdo/slab-depot.c

980

vdo_start_operation_with_waiter(&journal->slab->state,

drivers/md/dm-vdo/slab-depot.c

996

if (journal->slab->status == VDO_SLAB_REBUILT)

drivers/md/dm-vdo/slab-depot.c

997

journal->slab->status = VDO_SLAB_REPLAYING;

drivers/md/dm-vdo/slab-depot.h

167

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.h

300

struct vdo_slab *slab;

drivers/md/dm-vdo/slab-depot.h

517

bool __must_check vdo_attempt_replay_into_slab(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.h

534

int __must_check vdo_acquire_provisional_reference(struct vdo_slab *slab,

drivers/md/dm-vdo/slab-depot.h

84

struct vdo_slab *slab;

drivers/md/dm-vdo/vdo.c

1834

struct vdo_slab *slab;

drivers/md/dm-vdo/vdo.c

1850

slab = vdo_get_slab(vdo->depot, pbn);

drivers/md/dm-vdo/vdo.c

1851

result = VDO_ASSERT(slab != NULL, "vdo_get_slab must succeed on all valid PBNs");

drivers/md/dm-vdo/vdo.c

1855

*zone_ptr = &vdo->physical_zones->zones[slab->allocator->zone_number];

fs/erofs/zdata.c

123

struct kmem_cache *slab;

fs/erofs/zdata.c

223

if (!pcluster_pool[i].slab)

fs/erofs/zdata.c

225

kmem_cache_destroy(pcluster_pool[i].slab);

fs/erofs/zdata.c

226

pcluster_pool[i].slab = NULL;

fs/erofs/zdata.c

241

pcs->slab = kmem_cache_create(pcs->name, size, 0,

fs/erofs/zdata.c

243

if (pcs->slab)

fs/erofs/zdata.c

263

pcl = kmem_cache_zalloc(pcs->slab, GFP_KERNEL);

fs/erofs/zdata.c

282

kmem_cache_free(pcs->slab, pcl);

fs/nfsd/nfs4state.c

5076

static inline void *alloc_stateowner(struct kmem_cache *slab, struct xdr_netobj *owner, struct nfs4_client *clp)

fs/nfsd/nfs4state.c

5080

sop = kmem_cache_alloc(slab, GFP_KERNEL);

fs/nfsd/nfs4state.c

5086

kmem_cache_free(slab, sop);

fs/nfsd/nfs4state.c

904

struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,

fs/nfsd/nfs4state.c

910

stid = kmem_cache_zalloc(slab, GFP_KERNEL);

fs/nfsd/nfs4state.c

934

kmem_cache_free(slab, stid);

fs/nfsd/state.h

812

struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,

include/linux/kasan.h

130

void __kasan_poison_slab(struct slab *slab);

include/linux/kasan.h

131

static __always_inline void kasan_poison_slab(struct slab *slab)

include/linux/kasan.h

134

__kasan_poison_slab(slab);

include/linux/kasan.h

14

struct slab;

include/linux/kasan.h

416

static inline void kasan_poison_slab(struct slab *slab) {}

include/linux/kfence.h

222

bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);

include/linux/kfence.h

246

static inline bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)

include/linux/page-flags.h

1040

PAGE_TYPE_OPS(Slab, slab, slab)

include/net/request_sock.h

31

struct kmem_cache *slab;

include/net/sock.h

1382

struct kmem_cache *slab;

lib/sg_pool.c

13

struct kmem_cache *slab;

lib/sg_pool.c

150

sgp->slab = kmem_cache_create(sgp->name, size, 0,

lib/sg_pool.c

152

if (!sgp->slab) {

lib/sg_pool.c

159

sgp->slab);

lib/sg_pool.c

174

kmem_cache_destroy(sgp->slab);

mm/debug.c

52

DEF_PAGETYPE_NAME(slab),

mm/kasan/common.c

155

void __kasan_poison_slab(struct slab *slab)

mm/kasan/common.c

157

struct page *page = slab_page(slab);

mm/kasan/common.c

45

struct slab *kasan_addr_to_slab(const void *addr)

mm/kasan/common.c

461

struct slab *slab;

mm/kasan/common.c

479

slab = virt_to_slab(object);

mm/kasan/common.c

482

if (unlikely(!slab))

mm/kasan/common.c

485

poison_kmalloc_redzone(slab->slab_cache, object, size, flags);

mm/kasan/common.c

522

struct slab *slab;

mm/kasan/common.c

534

slab = page_slab(page);

mm/kasan/common.c

536

if (check_slab_allocation(slab->slab_cache, ptr, ip))

mm/kasan/common.c

539

poison_slab_object(slab->slab_cache, ptr, false);

mm/kasan/common.c

545

struct slab *slab;

mm/kasan/common.c

548

slab = virt_to_slab(ptr);

mm/kasan/common.c

554

if (unlikely(!slab)) {

mm/kasan/common.c

564

unpoison_slab_object(slab->slab_cache, ptr, flags, false);

mm/kasan/common.c

567

if (is_kmalloc_cache(slab->slab_cache))

mm/kasan/common.c

568

poison_kmalloc_redzone(slab->slab_cache, ptr, size, flags);

mm/kasan/generic.c

541

struct slab *slab = kasan_addr_to_slab(addr);

mm/kasan/generic.c

546

if (is_kfence_address(addr) || !slab)

mm/kasan/generic.c

549

cache = slab->slab_cache;

mm/kasan/generic.c

550

object = nearest_obj(cache, slab, addr);

mm/kasan/kasan.h

385

struct slab *kasan_addr_to_slab(const void *addr);

mm/kasan/report.c

492

struct slab *slab;

mm/kasan/report.c

500

slab = kasan_addr_to_slab(addr);

mm/kasan/report.c

501

if (slab) {

mm/kasan/report.c

502

info->cache = slab->slab_cache;

mm/kasan/report.c

503

info->object = nearest_obj(info->cache, slab, addr);

mm/kfence/core.c

431

struct slab *slab;

mm/kfence/core.c

499

slab = virt_to_slab(addr);

mm/kfence/core.c

500

slab->slab_cache = cache;

mm/kfence/core.c

501

slab->objects = 1;

mm/kfence/core.c

640

struct slab *slab = page_slab(page);

mm/kfence/core.c

641

slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts |

mm/kfence/core.c

708

struct slab *slab = page_slab(page);

mm/kfence/core.c

709

slab->obj_exts = 0;

mm/kfence/kfence_test.c

287

struct slab *slab = virt_to_slab(alloc);

mm/kfence/kfence_test.c

297

KUNIT_EXPECT_EQ(test, obj_to_index(s, slab, alloc), 0U);

mm/kfence/kfence_test.c

298

KUNIT_EXPECT_EQ(test, objs_per_slab(s, slab), 1);

mm/kfence/report.c

342

bool __kfence_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)

mm/kfence/report.c

362

kpp->kp_slab = slab;

mm/memcontrol.c

2859

struct mem_cgroup *mem_cgroup_from_obj_slab(struct slab *slab, void *p)

mm/memcontrol.c

2870

obj_exts = slab_obj_exts(slab);

mm/memcontrol.c

2875

off = obj_to_index(slab->slab_cache, slab, p);

mm/memcontrol.c

2876

obj_ext = slab_obj_ext(slab, obj_exts, off);

mm/memcontrol.c

2899

struct slab *slab;

mm/memcontrol.c

2904

slab = virt_to_slab(p);

mm/memcontrol.c

2905

if (slab)

mm/memcontrol.c

2906

return mem_cgroup_from_obj_slab(slab, p);

mm/memcontrol.c

3543

struct slab *slab;

mm/memcontrol.c

3584

slab = virt_to_slab(p[i]);

mm/memcontrol.c

3586

if (!slab_obj_exts(slab) &&

mm/memcontrol.c

3587

alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) {

mm/memcontrol.c

3614

slab_pgdat(slab), cache_vmstat_idx(s));

mm/memcontrol.c

3617

obj_exts = slab_obj_exts(slab);

mm/memcontrol.c

3619

off = obj_to_index(s, slab, p[i]);

mm/memcontrol.c

3620

obj_ext = slab_obj_ext(slab, obj_exts, off);

mm/memcontrol.c

3629

void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,

mm/memcontrol.c

3640

off = obj_to_index(s, slab, p[i]);

mm/memcontrol.c

3641

obj_ext = slab_obj_ext(slab, obj_exts, off);

mm/memcontrol.c

3651

slab_pgdat(slab), cache_vmstat_idx(s));

mm/memcontrol.c

4454

int slab = atomic_xchg(&pn->slab_reclaimable, 0);

mm/memcontrol.c

4457

lstats->state[index] += slab;

mm/memcontrol.c

4459

plstats->state_pending[index] += slab;

mm/memcontrol.c

4460

memcg->vmstats->state[index] += slab;

mm/memcontrol.c

4462

parent->vmstats->state_pending[index] += slab;

mm/memcontrol.c

4465

int slab = atomic_xchg(&pn->slab_unreclaimable, 0);

mm/memcontrol.c

4468

lstats->state[index] += slab;

mm/memcontrol.c

4470

plstats->state_pending[index] += slab;

mm/memcontrol.c

4471

memcg->vmstats->state[index] += slab;

mm/memcontrol.c

4473

parent->vmstats->state_pending[index] += slab;

mm/slab.h

120

static_assert(offsetof(struct page, pg) == offsetof(struct slab, sl))

mm/slab.h

130

static_assert(sizeof(struct slab) <= sizeof(struct page));

mm/slab.h

132

static_assert(IS_ALIGNED(offsetof(struct slab, freelist), sizeof(struct freelist_counters)));

mm/slab.h

147

const struct slab *: (const struct folio *)s, \

mm/slab.h

148

struct slab *: (struct folio *)s))

mm/slab.h

157

static inline struct slab *page_slab(const struct page *page)

mm/slab.h

163

return (struct slab *)page;

mm/slab.h

176

static inline void *slab_address(const struct slab *slab)

mm/slab.h

178

return folio_address(slab_folio(slab));

mm/slab.h

181

static inline int slab_nid(const struct slab *slab)

mm/slab.h

183

return memdesc_nid(slab->flags);

mm/slab.h

186

static inline pg_data_t *slab_pgdat(const struct slab *slab)

mm/slab.h

188

return NODE_DATA(slab_nid(slab));

mm/slab.h

191

static inline struct slab *virt_to_slab(const void *addr)

mm/slab.h

196

static inline int slab_order(const struct slab *slab)

mm/slab.h

198

return folio_order(slab_folio(slab));

mm/slab.h

201

static inline size_t slab_size(const struct slab *slab)

mm/slab.h

203

return PAGE_SIZE << slab_order(slab);

mm/slab.h

306

const struct slab *slab, void *x)

mm/slab.h

308

void *object = x - (x - slab_address(slab)) % cache->size;

mm/slab.h

309

void *last_object = slab_address(slab) +

mm/slab.h

310

(slab->objects - 1) * cache->size;

mm/slab.h

326

const struct slab *slab, const void *obj)

mm/slab.h

330

return __obj_to_index(cache, slab_address(slab), obj);

mm/slab.h

334

const struct slab *slab)

mm/slab.h

336

return slab->objects;

mm/slab.h

584

static inline unsigned long slab_obj_exts(struct slab *slab)

mm/slab.h

586

unsigned long obj_exts = READ_ONCE(slab->obj_exts);

mm/slab.h

594

obj_exts != OBJEXTS_ALLOC_FAIL, slab_page(slab));

mm/slab.h

595

VM_BUG_ON_PAGE(obj_exts & MEMCG_DATA_KMEM, slab_page(slab));

mm/slab.h

613

static inline void slab_set_stride(struct slab *slab, unsigned int stride)

mm/slab.h

615

slab->stride = stride;

mm/slab.h

617

static inline unsigned int slab_get_stride(struct slab *slab)

mm/slab.h

619

return slab->stride;

mm/slab.h

622

static inline void slab_set_stride(struct slab *slab, unsigned int stride)

mm/slab.h

626

static inline unsigned int slab_get_stride(struct slab *slab)

mm/slab.h

642

static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,

mm/slab.h

648

VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab));

mm/slab.h

651

slab_get_stride(slab) * index);

mm/slab.h

655

int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,

mm/slab.h

660

static inline unsigned long slab_obj_exts(struct slab *slab)

mm/slab.h

665

static inline struct slabobj_ext *slab_obj_ext(struct slab *slab,

mm/slab.h

672

static inline void slab_set_stride(struct slab *slab, unsigned int stride) { }

mm/slab.h

673

static inline unsigned int slab_get_stride(struct slab *slab) { return 0; }

mm/slab.h

688

void __memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,

mm/slab.h

759

struct slab *kp_slab;

mm/slab.h

767

void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab);

mm/slab.h

771

const struct slab *slab, bool to_user);

mm/slab_common.c

1265

struct slab *slab;

mm/slab_common.c

1270

slab = virt_to_slab((void *)(long)addr);

mm/slab_common.c

1271

return slab ? slab->slab_cache : NULL;

mm/slab_common.c

1614

struct slab *slab;

mm/slab_common.c

1619

slab = virt_to_slab(obj);

mm/slab_common.c

1620

if (unlikely(!slab))

mm/slab_common.c

1623

s = slab->slab_cache;

mm/slab_common.c

1624

if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id()))

mm/slab_common.c

616

static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)

mm/slab_common.c

618

if (__kfence_obj_info(kpp, object, slab))

mm/slab_common.c

620

__kmem_obj_info(kpp, object, slab);

mm/slab_common.c

642

struct slab *slab;

mm/slab_common.c

649

slab = virt_to_slab(object);

mm/slab_common.c

650

if (!slab)

mm/slab_common.c

653

kmem_obj_info(&kp, object, slab);

mm/slub.c

1006

struct slab *slab, void *object)

mm/slub.c

1013

base = slab_address(slab);

mm/slub.c

1016

if (object < base || object >= base + slab->objects * s->size ||

mm/slub.c

1124

static void print_slab_info(const struct slab *slab)

mm/slub.c

1127

slab, slab->objects, slab->inuse, slab->freelist,

mm/slub.c

1128

&slab->flags.f);

mm/slub.c

1175

static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)

mm/slub.c

1178

u8 *addr = slab_address(slab);

mm/slub.c

1182

print_slab_info(slab);

mm/slub.c

1209

if (obj_exts_in_object(s, slab))

mm/slub.c

1218

static void object_err(struct kmem_cache *s, struct slab *slab,

mm/slub.c

1225

if (!object || !check_valid_pointer(s, slab, object)) {

mm/slub.c

1226

print_slab_info(slab);

mm/slub.c

1229

print_trailer(s, slab, object);

mm/slub.c

1236

static void __slab_err(struct slab *slab)

mm/slub.c

1241

print_slab_info(slab);

mm/slub.c

1247

static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,

mm/slub.c

1259

__slab_err(slab);

mm/slub.c

1310

check_bytes_and_report(struct kmem_cache *s, struct slab *slab,

mm/slub.c

1316

u8 *addr = slab_address(slab);

mm/slub.c

1335

object_err(s, slab, object, "Object corrupt");

mm/slub.c

1400

static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)

mm/slub.c

1414

if (obj_exts_in_object(s, slab))

mm/slub.c

1420

return check_bytes_and_report(s, slab, p, "Object padding",

mm/slub.c

1426

slab_pad_check(struct kmem_cache *s, struct slab *slab)

mm/slub.c

1438

start = slab_address(slab);

mm/slub.c

1439

length = slab_size(slab);

mm/slub.c

1442

if (obj_exts_in_slab(s, slab) && !obj_exts_in_object(s, slab)) {

mm/slub.c

1444

remainder -= obj_exts_offset_in_slab(s, slab);

mm/slub.c

1445

remainder -= obj_exts_size_in_slab(slab);

mm/slub.c

1465

__slab_err(slab);

mm/slub.c

1470

static int check_object(struct kmem_cache *s, struct slab *slab,

mm/slub.c

1479

if (!check_bytes_and_report(s, slab, object, "Left Redzone",

mm/slub.c

1483

if (!check_bytes_and_report(s, slab, object, "Right Redzone",

mm/slub.c

1491

!check_bytes_and_report(s, slab, object,

mm/slub.c

1499

if (!check_bytes_and_report(s, slab, p, "Alignment padding",

mm/slub.c

1515

!check_bytes_and_report(s, slab, p, "Poison",

mm/slub.c

1520

!check_bytes_and_report(s, slab, p, "End Poison",

mm/slub.c

1527

if (!check_pad_bytes(s, slab, p))

mm/slub.c

1536

!check_valid_pointer(s, slab, get_freepointer(s, p))) {

mm/slub.c

1537

object_err(s, slab, p, "Freepointer corrupt");

mm/slub.c

1555

static int check_slab(struct kmem_cache *s, struct slab *slab)

mm/slub.c

1559

maxobj = order_objects(slab_order(slab), s->size);

mm/slub.c

1560

if (slab->objects > maxobj) {

mm/slub.c

1561

slab_err(s, slab, "objects %u > max %u",

mm/slub.c

1562

slab->objects, maxobj);

mm/slub.c

1565

if (slab->inuse > slab->objects) {

mm/slub.c

1566

slab_err(s, slab, "inuse %u > max %u",

mm/slub.c

1567

slab->inuse, slab->objects);

mm/slub.c

1570

if (slab->frozen) {

mm/slub.c

1571

slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed");

mm/slub.c

1576

slab_pad_check(s, slab);

mm/slub.c

1584

static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search)

mm/slub.c

1591

fp = slab->freelist;

mm/slub.c

1592

while (fp && nr <= slab->objects) {

mm/slub.c

1595

if (!check_valid_pointer(s, slab, fp)) {

mm/slub.c

1597

object_err(s, slab, object,

mm/slub.c

1602

slab_err(s, slab, "Freepointer corrupt");

mm/slub.c

1603

slab->freelist = NULL;

mm/slub.c

1604

slab->inuse = slab->objects;

mm/slub.c

1614

if (nr > slab->objects) {

mm/slub.c

1615

slab_err(s, slab, "Freelist cycle detected");

mm/slub.c

1616

slab->freelist = NULL;

mm/slub.c

1617

slab->inuse = slab->objects;

mm/slub.c

1622

max_objects = order_objects(slab_order(slab), s->size);

mm/slub.c

1626

if (slab->objects != max_objects) {

mm/slub.c

1627

slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",

mm/slub.c

1628

slab->objects, max_objects);

mm/slub.c

1629

slab->objects = max_objects;

mm/slub.c

1632

if (slab->inuse != slab->objects - nr) {

mm/slub.c

1633

slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",

mm/slub.c

1634

slab->inuse, slab->objects - nr);

mm/slub.c

1635

slab->inuse = slab->objects - nr;

mm/slub.c

1641

static void trace(struct kmem_cache *s, struct slab *slab, void *object,

mm/slub.c

1648

object, slab->inuse,

mm/slub.c

1649

slab->freelist);

mm/slub.c

1663

struct kmem_cache_node *n, struct slab *slab)

mm/slub.c

1669

list_add(&slab->slab_list, &n->full);

mm/slub.c

1672

static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)

mm/slub.c

1678

list_del(&slab->slab_list);

mm/slub.c

1712

void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)

mm/slub.c

1718

memset(kasan_reset_tag(addr), POISON_INUSE, slab_size(slab));

mm/slub.c

1723

struct slab *slab, void *object)

mm/slub.c

1725

if (!check_slab(s, slab))

mm/slub.c

1728

if (!check_valid_pointer(s, slab, object)) {

mm/slub.c

1729

object_err(s, slab, object, "Freelist Pointer check fails");

mm/slub.c

1733

if (!check_object(s, slab, object, SLUB_RED_INACTIVE))

mm/slub.c

1740

struct slab *slab, void *object, int orig_size)

mm/slub.c

1743

if (!alloc_consistency_checks(s, slab, object))

mm/slub.c

1748

trace(s, slab, object, 1);

mm/slub.c

1759

slab->inuse = slab->objects;

mm/slub.c

1760

slab->freelist = NULL;

mm/slub.c

1761

slab->frozen = 1; /* mark consistency-failed slab as frozen */

mm/slub.c

1767

struct slab *slab, void *object, unsigned long addr)

mm/slub.c

1769

if (!check_valid_pointer(s, slab, object)) {

mm/slub.c

1770

slab_err(s, slab, "Invalid object pointer 0x%p", object);

mm/slub.c

1774

if (on_freelist(s, slab, object)) {

mm/slub.c

1775

object_err(s, slab, object, "Object already free");

mm/slub.c

1779

if (!check_object(s, slab, object, SLUB_RED_ACTIVE))

mm/slub.c

1782

if (unlikely(s != slab->slab_cache)) {

mm/slub.c

1783

if (!slab->slab_cache) {

mm/slub.c

1784

slab_err(NULL, slab, "No slab cache for object 0x%p",

mm/slub.c

1787

object_err(s, slab, object,

mm/slub.c

2014

void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}

mm/slub.c

2017

struct slab *slab, void *object, int orig_size) { return true; }

mm/slub.c

2020

struct slab *slab, void *head, void *tail, int *bulk_cnt,

mm/slub.c

2023

static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}

mm/slub.c

2024

static inline int check_object(struct kmem_cache *s, struct slab *slab,

mm/slub.c

2030

struct slab *slab) {}

mm/slub.c

2032

struct slab *slab) {}

mm/slub.c

2067

struct slab *obj_slab;

mm/slub.c

2091

static inline bool mark_failed_objexts_alloc(struct slab *slab)

mm/slub.c

2093

return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0;

mm/slub.c

2115

static inline bool mark_failed_objexts_alloc(struct slab *slab) { return false; }

mm/slub.c

2121

static inline void init_slab_obj_exts(struct slab *slab)

mm/slub.c

2123

slab->obj_exts = 0;

mm/slub.c

2126

int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,

mm/slub.c

2130

unsigned int objects = objs_per_slab(s, slab);

mm/slub.c

2135

size_t sz = sizeof(struct slabobj_ext) * slab->objects;

mm/slub.c

2149

vec = kmalloc_flags(sz, gfp | __GFP_ZERO, alloc_flags, slab_nid(slab));

mm/slub.c

2157

if (!mark_failed_objexts_alloc(slab) &&

mm/slub.c

2158

slab_obj_exts(slab))

mm/slub.c

2166

struct slab *exts_slab;

mm/slub.c

2185

old_exts = READ_ONCE(slab->obj_exts);

mm/slub.c

2194

slab->obj_exts = new_exts;

mm/slub.c

2206

} else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) {

mm/slub.c

2216

static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)

mm/slub.c

2220

obj_exts = (struct slabobj_ext *)slab_obj_exts(slab);

mm/slub.c

2227

slab->obj_exts = 0;

mm/slub.c

2231

if (obj_exts_in_slab(slab->slab_cache, slab)) {

mm/slub.c

2232

slab->obj_exts = 0;

mm/slub.c

2240

slab->obj_exts = 0;

mm/slub.c

2248

static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)

mm/slub.c

2254

slab_set_stride(slab, sizeof(struct slabobj_ext));

mm/slub.c

2259

if (obj_exts_fit_within_slab_leftover(s, slab)) {

mm/slub.c

2260

addr = slab_address(slab) + obj_exts_offset_in_slab(s, slab);

mm/slub.c

2265

memset(addr, 0, obj_exts_size_in_slab(slab));

mm/slub.c

2271

slab->obj_exts = obj_exts;

mm/slub.c

2275

obj_exts = (unsigned long)slab_address(slab);

mm/slub.c

2280

for_each_object(addr, s, slab_address(slab), slab->objects)

mm/slub.c

2288

slab->obj_exts = obj_exts;

mm/slub.c

2289

slab_set_stride(slab, s->size);

mm/slub.c

2299

static inline void init_slab_obj_exts(struct slab *slab)

mm/slub.c

2303

static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,

mm/slub.c

2309

static inline void free_slab_obj_exts(struct slab *slab, bool allow_spin)

mm/slub.c

2314

struct slab *slab)

mm/slub.c

2323

prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab,

mm/slub.c

2326

if (!slab_obj_exts(slab) &&

mm/slub.c

2327

alloc_slab_obj_exts(slab, s, flags, alloc_flags)) {

mm/slub.c

2333

return slab_obj_exts(slab);

mm/slub.c

2344

struct slab *slab;

mm/slub.c

2355

slab = virt_to_slab(object);

mm/slub.c

2356

obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object);

mm/slub.c

2363

unsigned int obj_idx = obj_to_index(s, slab, object);

mm/slub.c

2366

obj_ext = slab_obj_ext(slab, obj_exts, obj_idx);

mm/slub.c

2384

__alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

mm/slub.c

2394

obj_exts = slab_obj_exts(slab);

mm/slub.c

2400

unsigned int off = obj_to_index(s, slab, p[i]);

mm/slub.c

2402

alloc_tag_sub(&slab_obj_ext(slab, obj_exts, off)->ref, s->size);

mm/slub.c

2408

alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

mm/slub.c

2412

__alloc_tagging_slab_free_hook(s, slab, p, objects);

mm/slub.c

2424

alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

mm/slub.c

2462

void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,

mm/slub.c

2470

obj_exts = slab_obj_exts(slab);

mm/slub.c

2475

__memcg_slab_free_hook(s, slab, p, objects, obj_exts);

mm/slub.c

2486

struct slab *slab;

mm/slub.c

2512

slab = page_slab(page);

mm/slub.c

2513

s = slab->slab_cache;

mm/slub.c

2524

obj_exts = slab_obj_exts(slab);

mm/slub.c

2527

off = obj_to_index(s, slab, p);

mm/slub.c

2528

obj_ext = slab_obj_ext(slab, obj_exts, off);

mm/slub.c

2549

static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,

mm/slub.c

2923

struct slab *slab = virt_to_slab(p[i]);

mm/slub.c

2925

memcg_slab_free_hook(s, slab, p + i, 1);

mm/slub.c

2926

alloc_tagging_slab_free_hook(s, slab, p + i, 1);

mm/slub.c

2933

if (slab_test_pfmemalloc(slab))

mm/slub.c

3252

static inline struct slab *alloc_slab_page(gfp_t flags, int node,

mm/slub.c

3257

struct slab *slab;

mm/slub.c

3272

slab = page_slab(page);

mm/slub.c

3274

slab_set_pfmemalloc(slab);

mm/slub.c

3276

return slab;

mm/slub.c

3330

static __always_inline void account_slab(struct slab *slab, int order,

mm/slub.c

3336

!slab_obj_exts(slab))

mm/slub.c

3337

alloc_slab_obj_exts(slab, s, gfp,

mm/slub.c

3340

mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),

mm/slub.c

3344

static __always_inline void unaccount_slab(struct slab *slab, int order,

mm/slub.c

3352

free_slab_obj_exts(slab, allow_spin);

mm/slub.c

3354

mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),

mm/slub.c

3359

static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags,

mm/slub.c

3363

struct slab *slab;

mm/slub.c

3380

slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin);

mm/slub.c

3381

if (unlikely(!slab)) {

mm/slub.c

3388

slab = alloc_slab_page(alloc_gfp, node, oo, allow_spin);

mm/slub.c

3389

if (unlikely(!slab))

mm/slub.c

3394

slab->objects = oo_objects(oo);

mm/slub.c

3395

slab->inuse = 0;

mm/slub.c

3396

slab->frozen = 0;

mm/slub.c

3398

slab->slab_cache = s;

mm/slub.c

3400

kasan_poison_slab(slab);

mm/slub.c

3402

start = slab_address(slab);

mm/slub.c

3404

setup_slab_debug(s, slab, start);

mm/slub.c

3405

init_slab_obj_exts(slab);

mm/slub.c

3410

alloc_slab_obj_exts_early(s, slab);

mm/slub.c

3411

account_slab(slab, oo_order(oo), s, flags, alloc_flags);

mm/slub.c

3413

return slab;

mm/slub.c

3416

static struct slab *new_slab(struct kmem_cache *s, gfp_t flags,

mm/slub.c

3429

static void __free_slab(struct kmem_cache *s, struct slab *slab, bool allow_spin)

mm/slub.c

3431

struct page *page = slab_page(slab);

mm/slub.c

3435

__slab_clear_pfmemalloc(slab);

mm/slub.c

3439

unaccount_slab(slab, order, s, allow_spin);

mm/slub.c

3446

static void free_new_slab_nolock(struct kmem_cache *s, struct slab *slab)

mm/slub.c

3452

__free_slab(s, slab, false);

mm/slub.c

3457

struct slab *slab = container_of(h, struct slab, rcu_head);

mm/slub.c

3459

__free_slab(slab->slab_cache, slab, true);

mm/slub.c

3462

static void free_slab(struct kmem_cache *s, struct slab *slab)

mm/slub.c

3467

slab_pad_check(s, slab);

mm/slub.c

3468

for_each_object(p, s, slab_address(slab), slab->objects)

mm/slub.c

3469

check_object(s, slab, p, SLUB_RED_INACTIVE);

mm/slub.c

3473

call_rcu(&slab->rcu_head, rcu_free_slab);

mm/slub.c

3475

__free_slab(s, slab, true);

mm/slub.c

3478

static void discard_slab(struct kmem_cache *s, struct slab *slab)

mm/slub.c

3480

dec_slabs_node(s, slab_nid(slab), slab->objects);

mm/slub.c

3481

free_slab(s, slab);

mm/slub.c

3484

static inline bool slab_test_node_partial(const struct slab *slab)

mm/slub.c

3486

return test_bit(SL_partial, &slab->flags.f);

mm/slub.c

3489

static inline void slab_set_node_partial(struct slab *slab)

mm/slub.c

3491

set_bit(SL_partial, &slab->flags.f);

mm/slub.c

3494

static inline void slab_clear_node_partial(struct slab *slab)

mm/slub.c

3496

clear_bit(SL_partial, &slab->flags.f);

mm/slub.c

3503

struct slab *slab)

mm/slub.c

3505

slab_set_node_partial(slab);

mm/slub.c

3510

__add_partial(struct kmem_cache_node *n, struct slab *slab, enum add_mode mode)

mm/slub.c

3513

list_add_tail(&slab->slab_list, &n->partial);

mm/slub.c

3515

list_add(&slab->slab_list, &n->partial);

mm/slub.c

3516

set_node_partial_state(n, slab);

mm/slub.c

3520

struct slab *slab, enum add_mode mode)

mm/slub.c

3523

__add_partial(n, slab, mode);

mm/slub.c

3527

struct slab *slab)

mm/slub.c

3529

slab_clear_node_partial(slab);

mm/slub.c

3534

struct slab *slab)

mm/slub.c

3537

list_del(&slab->slab_list);

mm/slub.c

3538

clear_node_partial_state(n, slab);

mm/slub.c

3548

struct kmem_cache_node *n, struct slab *slab, int orig_size)

mm/slub.c

3556

if (!validate_slab_ptr(slab)) {

mm/slub.c

3557

slab_err(s, slab, "Not a valid slab page");

mm/slub.c

3563

object = slab->freelist;

mm/slub.c

3564

slab->freelist = get_freepointer(s, object);

mm/slub.c

3565

slab->inuse++;

mm/slub.c

3567

if (!alloc_debug_processing(s, slab, object, orig_size)) {

mm/slub.c

3568

remove_partial(n, slab);

mm/slub.c

3572

if (slab->inuse == slab->objects) {

mm/slub.c

3573

remove_partial(n, slab);

mm/slub.c

3574

add_full(s, n, slab);

mm/slub.c

3606

static inline void build_slab_freelist(struct kmem_cache *s, struct slab *slab,

mm/slub.c

3609

unsigned int nr = slab->objects - slab->inuse;

mm/slub.c

3614

slab->freelist = NULL;

mm/slub.c

3619

slab->freelist = cur;

mm/slub.c

3631

static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab,

mm/slub.c

3636

iter->start = fixup_red_left(s, slab_address(slab));

mm/slub.c

3639

iter->random = (slab->objects >= 2 && s->random_seq);

mm/slub.c

3644

iter->page_limit = slab->objects * s->size;

mm/slub.c

3667

static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,

mm/slub.c

3677

init_slab_obj_iter(s, slab, &iter, allow_spin);

mm/slub.c

3679

slab->inuse = 1;

mm/slub.c

3681

needs_add_partial = (slab->objects > 1);

mm/slub.c

3682

build_slab_freelist(s, slab, &iter);

mm/slub.c

3685

set_freepointer(s, object, slab->freelist);

mm/slub.c

3687

if (!alloc_debug_processing(s, slab, object, ac->orig_size)) {

mm/slub.c

3697

n = get_node(s, slab_nid(slab));

mm/slub.c

3706

free_new_slab_nolock(s, slab);

mm/slub.c

3711

add_partial(n, slab, ADD_TO_HEAD);

mm/slub.c

3713

add_full(s, n, slab);

mm/slub.c

3719

inc_slabs_node(s, slab_nid(slab), slab->objects);

mm/slub.c

3725

static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);

mm/slub.c

3732

struct slab *slab, *slab2;

mm/slub.c

3733

struct slab *first = NULL, *last = NULL;

mm/slub.c

3748

list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {

mm/slub.c

3752

if (!pfmemalloc_match(slab, pc->flags)) {

mm/slub.c

3769

flc.counters = data_race(READ_ONCE(slab->counters));

mm/slub.c

3778

first = slab;

mm/slub.c

3779

last = slab;

mm/slub.c

3780

clear_node_partial_state(n, slab);

mm/slub.c

3803

struct slab *slab, *slab2;

mm/slub.c

3820

list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) {

mm/slub.c

3824

if (!pfmemalloc_match(slab, gfp_flags))

mm/slub.c

3828

object = alloc_single_from_partial(s, n, slab,

mm/slub.c

3841

old.freelist = slab->freelist;

mm/slub.c

3842

old.counters = slab->counters;

mm/slub.c

3848

} while (!__slab_update_freelist(s, slab, &old, &new, "get_from_partial_node"));

mm/slub.c

3852

remove_partial(n, slab);

mm/slub.c

4158

static int count_free(struct slab *slab)

mm/slub.c

4160

return slab->objects - slab->inuse;

mm/slub.c

4170

struct slab *slab, void *head, void *tail, int *bulk_cnt,

mm/slub.c

4178

if (!check_slab(s, slab))

mm/slub.c

4182

if (slab->inuse < *bulk_cnt) {

mm/slub.c

4183

slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",

mm/slub.c

4184

slab->inuse, *bulk_cnt);

mm/slub.c

4194

if (!free_consistency_checks(s, slab, object, addr))

mm/slub.c

4200

trace(s, slab, object, 0);

mm/slub.c

4213

slab_err(s, slab, "Bulk free expected %d objects but found %d\n",

mm/slub.c

4229

int (*get_count)(struct slab *))

mm/slub.c

4233

struct slab *slab;

mm/slub.c

4236

list_for_each_entry(slab, &n->partial, slab_list)

mm/slub.c

4237

x += get_count(slab);

mm/slub.c

4250

struct slab *slab;

mm/slub.c

4254

list_for_each_entry(slab, &n->partial, slab_list)

mm/slub.c

4255

x += slab->objects - slab->inuse;

mm/slub.c

4264

list_for_each_entry(slab, &n->partial, slab_list) {

mm/slub.c

4265

x += slab->objects - slab->inuse;

mm/slub.c

4269

list_for_each_entry_reverse(slab, &n->partial, slab_list) {

mm/slub.c

4270

x += slab->objects - slab->inuse;

mm/slub.c

4321

static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)

mm/slub.c

4323

if (unlikely(slab_test_pfmemalloc(slab)))

mm/slub.c

4337

static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab,

mm/slub.c

4343

old.freelist = slab->freelist;

mm/slub.c

4344

old.counters = slab->counters;

mm/slub.c

4352

} while (!slab_update_freelist(s, slab, &old, &new, "get_freelist_nofreeze"));

mm/slub.c

4373

static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,

mm/slub.c

4385

if (count >= slab->objects) {

mm/slub.c

4387

count = slab->objects;

mm/slub.c

4390

init_slab_obj_iter(s, slab, &iter, allow_spin);

mm/slub.c

4396

slab->inuse = count;

mm/slub.c

4397

build_slab_freelist(s, slab, &iter);

mm/slub.c

4400

struct kmem_cache_node *n = get_node(s, slab_nid(slab));

mm/slub.c

4410

free_new_slab_nolock(s, slab);

mm/slub.c

4413

add_partial(n, slab, ADD_TO_HEAD);

mm/slub.c

4417

inc_slabs_node(s, slab_nid(slab), slab->objects);

mm/slub.c

4434

struct slab *slab;

mm/slub.c

4463

slab = new_slab(s, trynode_flags, ac->alloc_flags, node);

mm/slub.c

4465

if (unlikely(!slab)) {

mm/slub.c

4478

object = alloc_single_from_new_slab(s, slab, ac);

mm/slub.c

4484

if (alloc_from_new_slab(s, slab, &object, 1, allow_spin))

mm/slub.c

5538

struct kmem_cache *s, struct slab *slab,

mm/slub.c

5542

struct kmem_cache_node *n = get_node(s, slab_nid(slab));

mm/slub.c

5543

struct slab *slab_free = NULL;

mm/slub.c

5557

if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {

mm/slub.c

5558

void *prior = slab->freelist;

mm/slub.c

5561

slab->inuse -= cnt;

mm/slub.c

5563

slab->freelist = head;

mm/slub.c

5570

if (slab->inuse == 0 && n->nr_partial >= s->min_partial)

mm/slub.c

5571

slab_free = slab;

mm/slub.c

5575

remove_full(s, n, slab);

mm/slub.c

5577

add_partial(n, slab, ADD_TO_TAIL);

mm/slub.c

5581

remove_partial(n, slab);

mm/slub.c

5609

static bool __slab_try_return_freelist(struct kmem_cache *s, struct slab *slab,

mm/slub.c

5614

old.freelist = slab->freelist;

mm/slub.c

5615

old.counters = slab->counters;

mm/slub.c

5624

if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist"))

mm/slub.c

5638

static void __slab_free(struct kmem_cache *s, struct slab *slab,

mm/slub.c

5650

free_to_partial_list(s, slab, head, tail, cnt, addr);

mm/slub.c

5660

old.freelist = slab->freelist;

mm/slub.c

5661

old.counters = slab->counters;

mm/slub.c

5678

n = get_node(s, slab_nid(slab));

mm/slub.c

5689

on_node_partial = slab_test_node_partial(slab);

mm/slub.c

5692

} while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));

mm/slub.c

5723

add_partial(n, slab, ADD_TO_TAIL);

mm/slub.c

5735

remove_partial(n, slab);

mm/slub.c

5741

discard_slab(s, slab);

mm/slub.c

6128

static __always_inline bool can_free_to_pcs(struct slab *slab)

mm/slub.c

6136

slab_node = slab_nid(slab);

mm/slub.c

6176

return likely(!slab_test_pfmemalloc(slab));

mm/slub.c

6195

struct slab *slab = virt_to_slab(p[i]);

mm/slub.c

6197

memcg_slab_free_hook(s, slab, p + i, 1);

mm/slub.c

6198

alloc_tagging_slab_free_hook(s, slab, p + i, 1);

mm/slub.c

6205

if (unlikely(!can_free_to_pcs(slab))) {

mm/slub.c

621

static inline bool slab_test_pfmemalloc(const struct slab *slab)

mm/slub.c

623

return test_bit(SL_pfmemalloc, &slab->flags.f);

mm/slub.c

626

static inline void slab_set_pfmemalloc(struct slab *slab)

mm/slub.c

628

set_bit(SL_pfmemalloc, &slab->flags.f);

mm/slub.c

631

static inline void __slab_clear_pfmemalloc(struct slab *slab)

mm/slub.c

6328

struct slab *slab;

mm/slub.c

633

__clear_bit(SL_pfmemalloc, &slab->flags.f);

mm/slub.c

6331

slab = virt_to_slab(x);

mm/slub.c

6332

s = slab->slab_cache;

mm/slub.c

6344

__slab_free(s, slab, x, x, 1, _THIS_IP_);

mm/slub.c

6371

void slab_free(struct kmem_cache *s, struct slab *slab, void *object,

mm/slub.c

6374

memcg_slab_free_hook(s, slab, &object, 1);

mm/slub.c

6375

alloc_tagging_slab_free_hook(s, slab, &object, 1);

mm/slub.c

6380

if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true)))

mm/slub.c

6383

__slab_free(s, slab, object, object, 1, addr);

mm/slub.c

639

static __always_inline void slab_lock(struct slab *slab)

mm/slub.c

6392

struct slab *slab = virt_to_slab(object);

mm/slub.c

6394

alloc_tagging_slab_free_hook(s, slab, &object, 1);

mm/slub.c

6397

__slab_free(s, slab, object, object, 1, _RET_IP_);

mm/slub.c

6402

void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,

mm/slub.c

6405

memcg_slab_free_hook(s, slab, p, cnt);

mm/slub.c

6406

alloc_tagging_slab_free_hook(s, slab, p, cnt);

mm/slub.c

641

bit_spin_lock(SL_locked, &slab->flags.f);

mm/slub.c

6412

__slab_free(s, slab, head, tail, cnt, addr);

mm/slub.c

6423

struct slab *slab = virt_to_slab(object);

mm/slub.c

6432

if (WARN_ON(!slab))

mm/slub.c

6434

s = slab->slab_cache;

mm/slub.c

644

static __always_inline void slab_unlock(struct slab *slab)

mm/slub.c

6440

__slab_free(s, slab, object, object, 1, _THIS_IP_);

mm/slub.c

6457

struct slab *slab;

mm/slub.c

6459

slab = virt_to_slab(obj);

mm/slub.c

646

bit_spin_unlock(SL_locked, &slab->flags.f);

mm/slub.c

6460

if (WARN_ONCE(!slab,

mm/slub.c

6465

cachep = slab->slab_cache;

mm/slub.c

6486

struct slab *slab;

mm/slub.c

6488

slab = virt_to_slab(x);

mm/slub.c

6497

if (unlikely(!slab || (slab->slab_cache != s))) {

mm/slub.c

650

__update_freelist_fast(struct slab *slab, struct freelist_counters *old,

mm/slub.c

6504

slab_free(s, slab, x, _RET_IP_);

mm/slub.c

6508

static inline size_t slab_ksize(struct slab *slab)

mm/slub.c

6510

struct kmem_cache *s = slab->slab_cache;

mm/slub.c

6529

else if (obj_exts_in_object(s, slab))

mm/slub.c

654

return try_cmpxchg_freelist(&slab->freelist_counters,

mm/slub.c

6540

struct slab *slab;

mm/slub.c

6550

slab = page_slab(page);

mm/slub.c

6552

if (WARN_ON(!slab))

mm/slub.c

6556

skip_orig_size_check(slab->slab_cache, object);

mm/slub.c

6559

return slab_ksize(slab);

mm/slub.c

6628

struct slab *slab;

mm/slub.c

663

__update_freelist_slow(struct slab *slab, struct freelist_counters *old,

mm/slub.c

6639

slab = page_slab(page);

mm/slub.c

6640

if (!slab) {

mm/slub.c

6650

s = slab->slab_cache;

mm/slub.c

6651

slab_addr = slab_address(slab);

mm/slub.c

6662

slab_free(s, slab, obj, _RET_IP_);

mm/slub.c

6674

struct slab *slab;

mm/slub.c

668

slab_lock(slab);

mm/slub.c

6684

slab = page_slab(page);

mm/slub.c

6685

if (!slab) {

mm/slub.c

669

if (slab->freelist == old->freelist &&

mm/slub.c

6691

s = slab->slab_cache;

mm/slub.c

6692

slab_free(s, slab, x, _RET_IP_);

mm/slub.c

670

slab->counters == old->counters) {

mm/slub.c

6707

struct slab *slab;

mm/slub.c

671

slab->freelist = new->freelist;

mm/slub.c

6714

slab = virt_to_slab(object);

mm/slub.c

6715

if (unlikely(!slab)) {

mm/slub.c

6720

s = slab->slab_cache;

mm/slub.c

6722

memcg_slab_free_hook(s, slab, &x, 1);

mm/slub.c

6723

alloc_tagging_slab_free_hook(s, slab, &x, 1);

mm/slub.c

673

WRITE_ONCE(slab->counters, new->counters);

mm/slub.c

6752

if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false)))

mm/slub.c

676

slab_unlock(slab);

mm/slub.c

6783

struct slab *slab = page_slab(page);

mm/slub.c

6785

if (!slab) {

mm/slub.c

6791

s = slab->slab_cache;

mm/slub.c

688

static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,

mm/slub.c

697

ret = __update_freelist_fast(slab, old, new);

mm/slub.c

699

ret = __update_freelist_slow(slab, old, new);

mm/slub.c

7022

struct slab *slab;

mm/slub.c

7048

struct slab *slab;

mm/slub.c

7053

slab = page_slab(page);

mm/slub.c

7056

if (!slab) {

mm/slub.c

7058

df->slab = NULL;

mm/slub.c

7062

df->slab = slab;

mm/slub.c

7063

df->s = slab->slab_cache;

mm/slub.c

7065

df->slab = slab;

mm/slub.c

7083

if (df->slab == virt_to_slab(object)) {

mm/slub.c

7115

if (!df.slab)

mm/slub.c

7121

__slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,

mm/slub.c

714

static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,

mm/slub.c

7145

if (!df.slab)

mm/slub.c

7148

slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],

mm/slub.c

7160

struct slab *slab, *slab2;

mm/slub.c

7172

list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {

mm/slub.c

7176

list_del(&slab->slab_list);

mm/slub.c

7178

object = get_freelist_nofreeze(s, slab, &count);

mm/slub.c

7199

if (__slab_try_return_freelist(s, slab, head, count)) {

mm/slub.c

720

ret = __update_freelist_fast(slab, old, new);

mm/slub.c

7200

list_add(&slab->slab_list, &pc.slabs);

mm/slub.c

7208

__slab_free(s, slab, head, tail, count, _RET_IP_);

mm/slub.c

7218

list_for_each_entry(slab, &pc.slabs, slab_list)

mm/slub.c

7219

set_node_partial_state(n, slab);

mm/slub.c

725

ret = __update_freelist_slow(slab, old, new);

mm/slub.c

7295

struct slab *slab;

mm/slub.c

7310

slab = new_slab(s, gfp, SLAB_ALLOC_DEFAULT, local_node);

mm/slub.c

7311

if (!slab)

mm/slub.c

7316

refilled += alloc_from_new_slab(s, slab, p + refilled, max - refilled,

mm/slub.c

7655

struct slab *slab;

mm/slub.c

7661

slab = new_slab(kmem_cache_node, GFP_NOWAIT, SLAB_ALLOC_DEFAULT, node);

mm/slub.c

7663

BUG_ON(!slab);

mm/slub.c

7664

if (slab_nid(slab) != node) {

mm/slub.c

7669

init_slab_obj_iter(kmem_cache_node, slab, &iter, true);

mm/slub.c

7674

slab->inuse = 1;

mm/slub.c

7675

build_slab_freelist(kmem_cache_node, slab, &iter);

mm/slub.c

7683

inc_slabs_node(kmem_cache_node, node, slab->objects);

mm/slub.c

7689

__add_partial(n, slab, ADD_TO_HEAD);

mm/slub.c

7971

static void list_slab_objects(struct kmem_cache *s, struct slab *slab)

mm/slub.c

7974

void *addr = slab_address(slab);

mm/slub.c

7981

__fill_map(object_map, s, slab);

mm/slub.c

7983

for_each_object(p, s, addr, slab->objects) {

mm/slub.c

7994

__slab_err(slab);

mm/slub.c

8006

struct slab *slab, *h;

mm/slub.c

8010

list_for_each_entry_safe(slab, h, &n->partial, slab_list) {

mm/slub.c

8011

if (!slab->inuse) {

mm/slub.c

8012

remove_partial(n, slab);

mm/slub.c

8013

list_add(&slab->slab_list, &discard);

mm/slub.c

8015

list_slab_objects(s, slab);

mm/slub.c

8020

list_for_each_entry_safe(slab, h, &discard, slab_list)

mm/slub.c

8021

discard_slab(s, slab);

mm/slub.c

804

static inline unsigned int obj_exts_size_in_slab(struct slab *slab)

mm/slub.c

806

return sizeof(struct slabobj_ext) * slab->objects;

mm/slub.c

8066

void __kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)

mm/slub.c

8073

struct kmem_cache *s = slab->slab_cache;

mm/slub.c

8077

kpp->kp_slab = slab;

mm/slub.c

8079

base = slab_address(slab);

mm/slub.c

8086

objnr = obj_to_index(s, slab, objp);

mm/slub.c

8090

if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size

mm/slub.c

810

struct slab *slab)

mm/slub.c

814

objext_offset = s->size * slab->objects;

mm/slub.c

820

struct slab *slab)

mm/slub.c

8204

const struct slab *slab, bool to_user)

mm/slub.c

8213

s = slab->slab_cache;

mm/slub.c

8216

if (ptr < slab_address(slab))

mm/slub.c

822

unsigned long objext_offset = obj_exts_offset_in_slab(s, slab);

mm/slub.c

8224

offset = (ptr - slab_address(slab)) % s->size;

mm/slub.c

823

unsigned long objext_size = obj_exts_size_in_slab(slab);

mm/slub.c

825

return objext_offset + objext_size <= slab_size(slab);

mm/slub.c

8260

struct slab *slab;

mm/slub.c

8261

struct slab *t;

mm/slub.c

828

static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)

mm/slub.c

8287

list_for_each_entry_safe(slab, t, &n->partial, slab_list) {

mm/slub.c

8288

int free = slab->objects - slab->inuse;

mm/slub.c

8296

if (free == slab->objects) {

mm/slub.c

8297

list_move(&slab->slab_list, &discard);

mm/slub.c

8298

clear_node_partial_state(n, slab);

mm/slub.c

8299

dec_slabs_node(s, node, slab->objects);

mm/slub.c

8301

list_move(&slab->slab_list, promote + free - 1);

mm/slub.c

8314

list_for_each_entry_safe(slab, t, &discard, slab_list)

mm/slub.c

8315

free_slab(s, slab);

mm/slub.c

834

obj_exts = slab_obj_exts(slab);

mm/slub.c

838

start = (unsigned long)slab_address(slab);

mm/slub.c

839

end = start + slab_size(slab);

mm/slub.c

8448

struct slab *p;

mm/slub.c

848

static inline unsigned int obj_exts_size_in_slab(struct slab *slab)

mm/slub.c

854

struct slab *slab)

mm/slub.c

860

struct slab *slab)

mm/slub.c

865

static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)

mm/slub.c

8708

static int count_inuse(struct slab *slab)

mm/slub.c

8710

return slab->inuse;

mm/slub.c

8713

static int count_total(struct slab *slab)

mm/slub.c

8715

return slab->objects;

mm/slub.c

8720

static void validate_slab(struct kmem_cache *s, struct slab *slab,

mm/slub.c

8724

void *addr = slab_address(slab);

mm/slub.c

8726

if (!validate_slab_ptr(slab)) {

mm/slub.c

8727

slab_err(s, slab, "Not a valid slab page");

mm/slub.c

873

static bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)

mm/slub.c

8731

if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))

mm/slub.c

8735

__fill_map(obj_map, s, slab);

mm/slub.c

8736

for_each_object(p, s, addr, slab->objects) {

mm/slub.c

8740

if (!check_object(s, slab, p, val))

mm/slub.c

8749

struct slab *slab;

mm/slub.c

8754

list_for_each_entry(slab, &n->partial, slab_list) {

mm/slub.c

8755

validate_slab(s, slab, obj_map);

mm/slub.c

8767

list_for_each_entry(slab, &n->full, slab_list) {

mm/slub.c

8768

validate_slab(s, slab, obj_map);

mm/slub.c

881

return obj_exts_in_slab(s, slab) &&

mm/slub.c

882

(slab_get_stride(slab) == s->size);

mm/slub.c

8952

struct slab *slab, enum track_item alloc,

mm/slub.c

8955

void *addr = slab_address(slab);

mm/slub.c

8959

__fill_map(obj_map, s, slab);

mm/slub.c

8961

for_each_object(p, s, addr, slab->objects)

mm/slub.c

900

static inline bool obj_exts_in_object(struct kmem_cache *s, struct slab *slab)

mm/slub.c

917

static inline bool validate_slab_ptr(struct slab *slab)

mm/slub.c

919

return PageSlab(slab_page(slab));

mm/slub.c

926

struct slab *slab)

mm/slub.c

928

void *addr = slab_address(slab);

mm/slub.c

931

bitmap_zero(obj_map, slab->objects);

mm/slub.c

933

for (p = slab->freelist; p; p = get_freepointer(s, p))

mm/slub.c

9540

ATTRIBUTE_GROUPS(slab);

mm/slub.c

9912

struct slab *slab;

mm/slub.c

9918

list_for_each_entry(slab, &n->partial, slab_list)

mm/slub.c

9919

process_slab(t, s, slab, alloc, obj_map);

mm/slub.c

9920

list_for_each_entry(slab, &n->full, slab_list)

mm/slub.c

9921

process_slab(t, s, slab, alloc, obj_map);

mm/usercopy.c

168

struct slab *slab;

mm/usercopy.c

194

slab = page_slab(page);

mm/usercopy.c

195

if (slab) {

mm/usercopy.c

197

__check_heap_object(ptr, n, slab, to_user);

net/core/sock.c

2242

struct kmem_cache *slab;

net/core/sock.c

2244

slab = prot->slab;

net/core/sock.c

2245

if (slab != NULL) {

net/core/sock.c

2246

sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);

net/core/sock.c

2267

if (slab != NULL)

net/core/sock.c

2268

kmem_cache_free(slab, sk);

net/core/sock.c

2276

struct kmem_cache *slab;

net/core/sock.c

2280

slab = prot->slab;

net/core/sock.c

2288

if (slab != NULL)

net/core/sock.c

2289

kmem_cache_free(slab, sk);

net/core/sock.c

4215

kmem_cache_destroy(rsk_prot->slab);

net/core/sock.c

4216

rsk_prot->slab = NULL;

net/core/sock.c

4231

rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,

net/core/sock.c

4236

if (!rsk_prot->slab) {

net/core/sock.c

4264

prot->slab = kmem_cache_create(prot->name, prot->obj_size,

net/core/sock.c

4268

if (prot->slab == NULL) {

net/core/sock.c

4298

kmem_cache_destroy(prot->slab);

net/core/sock.c

4299

prot->slab = NULL;

net/core/sock.c

4313

kmem_cache_destroy(prot->slab);

net/core/sock.c

4314

prot->slab = NULL;

net/core/sock.c

4389

proto->slab == NULL ? "no" : "yes",

net/ipv4/af_inet.c

330

WARN_ON(!answer_prot->slab);

net/ipv4/inet_connection_sock.c

855

req = kmem_cache_alloc_noprof(ops->slab, GFP_ATOMIC | __GFP_NOWARN);

net/ipv4/inet_connection_sock.c

861

kmem_cache_free(ops->slab, req);

net/ipv4/inet_connection_sock.c

910

kmem_cache_free(req->rsk_ops->slab, req);

net/ipv4/inet_connection_sock.c

919

nreq = kmem_cache_alloc(req->rsk_ops->slab, GFP_ATOMIC | __GFP_NOWARN);

net/ipv6/af_inet6.c

175

WARN_ON(!answer_prot->slab);

net/mptcp/protocol.c

4727

mptcp_v6_prot.slab = NULL;

net/mptcp/subflow.c

2130

subflow_ops->slab = kmem_cache_create(subflow_ops->slab_name,

net/mptcp/subflow.c

2135

if (!subflow_ops->slab)

tools/mm/slabinfo.c

1145

a->slab = s;

tools/mm/slabinfo.c

1164

if (!show_single_ref && a->slab->refs == 1)

tools/mm/slabinfo.c

1169

if (strcmp(a->slab->name, active) == 0) {

tools/mm/slabinfo.c

1174

printf("\n%-12s <- %s", a->slab->name, a->name);

tools/mm/slabinfo.c

1175

active = a->slab->name;

tools/mm/slabinfo.c

1178

printf("%-15s -> %s\n", a->name, a->slab->name);

tools/mm/slabinfo.c

1208

static int slab_mismatch(char *slab)

tools/mm/slabinfo.c

1210

return regexec(&pattern, slab, 0, NULL, 0);

tools/mm/slabinfo.c

1217

struct slabinfo *slab = slabinfo;

tools/mm/slabinfo.c

1249

if (slab - slabinfo == MAX_SLABS)

tools/mm/slabinfo.c

1252

fatal("Unable to access slab %s\n", slab->name);

tools/mm/slabinfo.c

1253

slab->name = strdup(de->d_name);

tools/mm/slabinfo.c

1254

slab->alias = 0;

tools/mm/slabinfo.c

1255

slab->refs = 0;

tools/mm/slabinfo.c

1256

slab->aliases = get_obj("aliases");

tools/mm/slabinfo.c

1257

slab->align = get_obj("align");

tools/mm/slabinfo.c

1258

slab->cache_dma = get_obj("cache_dma");

tools/mm/slabinfo.c

1259

slab->cpu_slabs = get_obj("cpu_slabs");

tools/mm/slabinfo.c

1260

slab->destroy_by_rcu = get_obj("destroy_by_rcu");

tools/mm/slabinfo.c

1261

slab->hwcache_align = get_obj("hwcache_align");

tools/mm/slabinfo.c

1262

slab->object_size = get_obj("object_size");

tools/mm/slabinfo.c

1263

slab->objects = get_obj("objects");

tools/mm/slabinfo.c

1264

slab->objects_partial = get_obj("objects_partial");

tools/mm/slabinfo.c

1265

slab->total_objects = get_obj("total_objects");

tools/mm/slabinfo.c

1266

slab->objs_per_slab = get_obj("objs_per_slab");

tools/mm/slabinfo.c

1267

slab->order = get_obj("order");

tools/mm/slabinfo.c

1268

slab->partial = get_obj_and_str("partial", &t);

tools/mm/slabinfo.c

1269

decode_numa_list(slab->numa_partial, t);

tools/mm/slabinfo.c

1271

slab->poison = get_obj("poison");

tools/mm/slabinfo.c

1272

slab->reclaim_account = get_obj("reclaim_account");

tools/mm/slabinfo.c

1273

slab->red_zone = get_obj("red_zone");

tools/mm/slabinfo.c

1274

slab->sanity_checks = get_obj("sanity_checks");

tools/mm/slabinfo.c

1275

slab->slab_size = get_obj("slab_size");

tools/mm/slabinfo.c

1276

slab->slabs = get_obj_and_str("slabs", &t);

tools/mm/slabinfo.c

1277

decode_numa_list(slab->numa, t);

tools/mm/slabinfo.c

1279

slab->store_user = get_obj("store_user");

tools/mm/slabinfo.c

1280

slab->trace = get_obj("trace");

tools/mm/slabinfo.c

1281

slab->alloc_fastpath = get_obj("alloc_fastpath");

tools/mm/slabinfo.c

1282

slab->alloc_slowpath = get_obj("alloc_slowpath");

tools/mm/slabinfo.c

1283

slab->free_fastpath = get_obj("free_fastpath");

tools/mm/slabinfo.c

1284

slab->free_slowpath = get_obj("free_slowpath");

tools/mm/slabinfo.c

1285

slab->free_frozen= get_obj("free_frozen");

tools/mm/slabinfo.c

1286

slab->free_add_partial = get_obj("free_add_partial");

tools/mm/slabinfo.c

1287

slab->free_remove_partial = get_obj("free_remove_partial");

tools/mm/slabinfo.c

1288

slab->alloc_from_partial = get_obj("alloc_from_partial");

tools/mm/slabinfo.c

1289

slab->alloc_slab = get_obj("alloc_slab");

tools/mm/slabinfo.c

1290

slab->alloc_refill = get_obj("alloc_refill");

tools/mm/slabinfo.c

1291

slab->free_slab = get_obj("free_slab");

tools/mm/slabinfo.c

1292

slab->cpuslab_flush = get_obj("cpuslab_flush");

tools/mm/slabinfo.c

1293

slab->deactivate_full = get_obj("deactivate_full");

tools/mm/slabinfo.c

1294

slab->deactivate_empty = get_obj("deactivate_empty");

tools/mm/slabinfo.c

1295

slab->deactivate_to_head = get_obj("deactivate_to_head");

tools/mm/slabinfo.c

1296

slab->deactivate_to_tail = get_obj("deactivate_to_tail");

tools/mm/slabinfo.c

1297

slab->deactivate_remote_frees = get_obj("deactivate_remote_frees");

tools/mm/slabinfo.c

1298

slab->order_fallback = get_obj("order_fallback");

tools/mm/slabinfo.c

1299

slab->cmpxchg_double_cpu_fail = get_obj("cmpxchg_double_cpu_fail");

tools/mm/slabinfo.c

1300

slab->cmpxchg_double_fail = get_obj("cmpxchg_double_fail");

tools/mm/slabinfo.c

1301

slab->cpu_partial_alloc = get_obj("cpu_partial_alloc");

tools/mm/slabinfo.c

1302

slab->cpu_partial_free = get_obj("cpu_partial_free");

tools/mm/slabinfo.c

1303

slab->alloc_node_mismatch = get_obj("alloc_node_mismatch");

tools/mm/slabinfo.c

1304

slab->deactivate_bypass = get_obj("deactivate_bypass");

tools/mm/slabinfo.c

1307

slab->name);

tools/mm/slabinfo.c

1308

if (slab->name[0] == ':')

tools/mm/slabinfo.c

1310

slab++;

tools/mm/slabinfo.c

1317

slabs = slab - slabinfo;

tools/mm/slabinfo.c

1324

struct slabinfo *slab;

tools/mm/slabinfo.c

1327

for (slab = slabinfo; (slab < slabinfo + slabs) &&

tools/mm/slabinfo.c

1328

lines != 0; slab++) {

tools/mm/slabinfo.c

1330

if (slab->alias)

tools/mm/slabinfo.c

1337

slab_numa(slab, 0);

tools/mm/slabinfo.c

1339

show_tracking(slab);

tools/mm/slabinfo.c

1341

slab_validate(slab);

tools/mm/slabinfo.c

1343

slab_shrink(slab);

tools/mm/slabinfo.c

1345

slab_debug(slab);

tools/mm/slabinfo.c

1347

ops(slab);

tools/mm/slabinfo.c

1349

slabcache(slab);

tools/mm/slabinfo.c

1351

report(slab);

tools/mm/slabinfo.c

359

if (a->slab == find &&

tools/mm/slabinfo.c

54

struct slabinfo *slab;

tools/sched_ext/scx_qmap.bpf.c

1072

u8 __arena *slab;

tools/sched_ext/scx_qmap.bpf.c

1114

slab = bpf_arena_alloc_pages(&arena, NULL, nr_pages, NUMA_NO_NODE, 0);

tools/sched_ext/scx_qmap.bpf.c

1115

if (!slab) {

tools/sched_ext/scx_qmap.bpf.c

1119

qa.task_ctxs = (task_ctx_t *)slab;

tools/sched_ext/scx_qmap.bpf.c

1125

task_ctx_t *cur = (task_ctx_t *)(slab + i * TASK_CTX_STRIDE);

tools/sched_ext/scx_qmap.bpf.c

1127

(task_ctx_t *)(slab + (i + 1) * TASK_CTX_STRIDE) : NULL;

tools/sched_ext/scx_qmap.bpf.c

1130

qa.task_free_head = (task_ctx_t *)slab;

tools/sched_ext/scx_sdt.bpf.c

104

void __arena *slab;

tools/sched_ext/scx_sdt.bpf.c

112

slab = bpf_arena_alloc_pages(&arena, NULL,

tools/sched_ext/scx_sdt.bpf.c

114

if (!slab)

tools/sched_ext/scx_sdt.bpf.c

117

pool->slab = slab;

tools/sched_ext/scx_sdt.bpf.c

121

ptr = (void __arena *)((__u64) pool->slab + elem_size * pool->idx);

tools/sched_ext/scx_sdt.h

23

void __arena *slab;

tools/testing/selftests/cgroup/test_kmem.c

356

long current, percpu, slab;

tools/testing/selftests/cgroup/test_kmem.c

386

slab = cg_read_key_long(parent, "memory.stat", "slab ");

tools/testing/selftests/cgroup/test_kmem.c

388

if (current > 0 && percpu > 0 && slab >= 0 &&

tools/testing/selftests/cgroup/test_kmem.c

389

labs(current - (percpu + slab)) < MAX_VMSTAT_ERROR)

tools/testing/selftests/cgroup/test_kmem.c

393

current, percpu, slab, current - (percpu + slab));