mm/zpdesc.h

40

struct zspage *zspage;

mm/zpdesc.h

59

ZPDESC_MATCH(private, zspage);

mm/zsmalloc.c

1000

static bool zspage_full(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

1002

return get_zspage_inuse(zspage) == class->objs_per_zspage;

mm/zsmalloc.c

1005

static bool zspage_empty(struct zspage *zspage)

mm/zsmalloc.c

1007

return get_zspage_inuse(zspage) == 0;

mm/zsmalloc.c

1040

struct zspage *zspage;

mm/zsmalloc.c

1051

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1054

zspage_read_lock(zspage);

mm/zsmalloc.c

1057

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1060

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1090

struct zspage *zspage;

mm/zsmalloc.c

1098

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1099

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1102

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1110

zspage_read_unlock(zspage);

mm/zsmalloc.c

1117

struct zspage *zspage;

mm/zsmalloc.c

1127

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1130

zspage_read_lock(zspage);

mm/zsmalloc.c

1133

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1136

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1163

struct zspage *zspage;

mm/zsmalloc.c

1170

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1172

zspage_read_unlock(zspage);

mm/zsmalloc.c

1179

struct zspage *zspage;

mm/zsmalloc.c

1189

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1192

zspage_read_lock(zspage);

mm/zsmalloc.c

1195

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1198

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1221

zspage_read_unlock(zspage);

mm/zsmalloc.c

1245

struct zspage *zspage, unsigned long handle)

mm/zsmalloc.c

1256

class = pool->size_class[zspage->class];

mm/zsmalloc.c

1257

obj = get_freeobj(zspage);

mm/zsmalloc.c

1262

m_zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1269

set_freeobj(zspage, link->next >> OBJ_TAG_BITS);

mm/zsmalloc.c

1270

if (likely(!ZsHugePage(zspage)))

mm/zsmalloc.c

1274

zspage->first_zpdesc->handle = handle | OBJ_ALLOCATED_TAG;

mm/zsmalloc.c

1277

mod_zspage_inuse(zspage, 1);

mm/zsmalloc.c

1303

struct zspage *zspage;

mm/zsmalloc.c

1321

zspage = find_get_zspage(class);

mm/zsmalloc.c

1322

if (likely(zspage)) {

mm/zsmalloc.c

1323

obj_malloc(pool, zspage, handle);

mm/zsmalloc.c

1325

fix_fullness_group(class, zspage);

mm/zsmalloc.c

1333

zspage = alloc_zspage(pool, class, gfp, nid);

mm/zsmalloc.c

1334

if (!zspage) {

mm/zsmalloc.c

1340

obj_malloc(pool, zspage, handle);

mm/zsmalloc.c

1341

newfg = get_fullness_group(class, zspage);

mm/zsmalloc.c

1342

insert_zspage(class, zspage, newfg);

mm/zsmalloc.c

1348

SetZsPageMovable(pool, zspage);

mm/zsmalloc.c

1359

struct zspage *zspage;

mm/zsmalloc.c

1368

zspage = get_zspage(f_zpdesc);

mm/zsmalloc.c

1374

if (likely(!ZsHugePage(zspage)))

mm/zsmalloc.c

1375

link->next = get_freeobj(zspage) << OBJ_TAG_BITS;

mm/zsmalloc.c

1378

set_freeobj(zspage, f_objidx);

mm/zsmalloc.c

1381

mod_zspage_inuse(zspage, -1);

mm/zsmalloc.c

1386

struct zspage *zspage;

mm/zsmalloc.c

1402

zspage = get_zspage(f_zpdesc);

mm/zsmalloc.c

1403

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1410

fullness = fix_fullness_group(class, zspage);

mm/zsmalloc.c

1412

free_zspage(pool, class, zspage);

mm/zsmalloc.c

1519

static void migrate_zspage(struct zs_pool *pool, struct zspage *src_zspage,

mm/zsmalloc.c

1520

struct zspage *dst_zspage)

mm/zsmalloc.c

1554

static struct zspage *isolate_src_zspage(struct size_class *class)

mm/zsmalloc.c

1556

struct zspage *zspage;

mm/zsmalloc.c

1560

zspage = list_first_entry_or_null(&class->fullness_list[fg],

mm/zsmalloc.c

1561

struct zspage, list);

mm/zsmalloc.c

1562

if (zspage) {

mm/zsmalloc.c

1563

remove_zspage(class, zspage);

mm/zsmalloc.c

1564

return zspage;

mm/zsmalloc.c

1568

return zspage;

mm/zsmalloc.c

1571

static struct zspage *isolate_dst_zspage(struct size_class *class)

mm/zsmalloc.c

1573

struct zspage *zspage;

mm/zsmalloc.c

1577

zspage = list_first_entry_or_null(&class->fullness_list[fg],

mm/zsmalloc.c

1578

struct zspage, list);

mm/zsmalloc.c

1579

if (zspage) {

mm/zsmalloc.c

1580

remove_zspage(class, zspage);

mm/zsmalloc.c

1581

return zspage;

mm/zsmalloc.c

1585

return zspage;

mm/zsmalloc.c

1595

static int putback_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

1599

fullness = get_fullness_group(class, zspage);

mm/zsmalloc.c

1600

insert_zspage(class, zspage, fullness);

mm/zsmalloc.c

1610

static void lock_zspage(struct zspage *zspage)

mm/zsmalloc.c

1623

zspage_read_lock(zspage);

mm/zsmalloc.c

1624

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1628

zspage_read_unlock(zspage);

mm/zsmalloc.c

1639

zspage_read_unlock(zspage);

mm/zsmalloc.c

1642

zspage_read_lock(zspage);

mm/zsmalloc.c

1645

zspage_read_unlock(zspage);

mm/zsmalloc.c

1651

static void replace_sub_page(struct size_class *class, struct zspage *zspage,

mm/zsmalloc.c

1659

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1668

create_page_chain(class, zspage, zpdescs);

mm/zsmalloc.c

1671

if (unlikely(ZsHugePage(zspage)))

mm/zsmalloc.c

1683

return page_zpdesc(page)->zspage;

mm/zsmalloc.c

1691

struct zspage *zspage;

mm/zsmalloc.c

1708

if (!zpdesc->zspage)

mm/zsmalloc.c

1712

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1713

pool = zspage->pool;

mm/zsmalloc.c

1720

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1727

if (!zspage_write_trylock(zspage)) {

mm/zsmalloc.c

1758

replace_sub_page(class, zspage, newzpdesc, zpdesc);

mm/zsmalloc.c

1765

zspage_write_unlock(zspage);

mm/zsmalloc.c

1797

struct zspage *zspage, *tmp;

mm/zsmalloc.c

1813

list_for_each_entry_safe(zspage, tmp, &free_pages, list) {

mm/zsmalloc.c

1814

list_del(&zspage->list);

mm/zsmalloc.c

1815

lock_zspage(zspage);

mm/zsmalloc.c

1817

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1820

__free_zspage(pool, class, zspage);

mm/zsmalloc.c

1840

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)

mm/zsmalloc.c

1842

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1877

struct zspage *src_zspage = NULL;

mm/zsmalloc.c

1878

struct zspage *dst_zspage = NULL;

mm/zsmalloc.c

2215

zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),

mm/zsmalloc.c

276

static void zspage_lock_init(struct zspage *zspage)

mm/zsmalloc.c

279

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

304

static void zspage_read_lock(struct zspage *zspage)

mm/zsmalloc.c

306

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

317

static void zspage_read_unlock(struct zspage *zspage)

mm/zsmalloc.c

319

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

328

static __must_check bool zspage_write_trylock(struct zspage *zspage)

mm/zsmalloc.c

330

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

344

static void zspage_write_unlock(struct zspage *zspage)

mm/zsmalloc.c

346

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

355

static void SetZsHugePage(struct zspage *zspage)

mm/zsmalloc.c

357

zspage->huge = 1;

mm/zsmalloc.c

360

static bool ZsHugePage(struct zspage *zspage)

mm/zsmalloc.c

362

return zspage->huge;

mm/zsmalloc.c

368

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);

mm/zsmalloc.c

372

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}

mm/zsmalloc.c

387

static struct zspage *cache_alloc_zspage(gfp_t gfp)

mm/zsmalloc.c

394

static void cache_free_zspage(struct zspage *zspage)

mm/zsmalloc.c

396

kmem_cache_free(zspage_cachep, zspage);

mm/zsmalloc.c

411

static inline int get_zspage_inuse(struct zspage *zspage)

mm/zsmalloc.c

413

return zspage->inuse;

mm/zsmalloc.c

416

static inline void mod_zspage_inuse(struct zspage *zspage, int val)

mm/zsmalloc.c

418

zspage->inuse += val;

mm/zsmalloc.c

421

static struct zpdesc *get_first_zpdesc(struct zspage *zspage)

mm/zsmalloc.c

423

struct zpdesc *first_zpdesc = zspage->first_zpdesc;

mm/zsmalloc.c

447

static inline unsigned int get_freeobj(struct zspage *zspage)

mm/zsmalloc.c

449

return zspage->freeobj;

mm/zsmalloc.c

452

static inline void set_freeobj(struct zspage *zspage, unsigned int obj)

mm/zsmalloc.c

454

zspage->freeobj = obj;

mm/zsmalloc.c

458

struct zspage *zspage)

mm/zsmalloc.c

460

return pool->size_class[zspage->class];

mm/zsmalloc.c

624

static int get_fullness_group(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

628

inuse = get_zspage_inuse(zspage);

mm/zsmalloc.c

652

struct zspage *zspage,

mm/zsmalloc.c

656

list_add(&zspage->list, &class->fullness_list[fullness]);

mm/zsmalloc.c

657

zspage->fullness = fullness;

mm/zsmalloc.c

664

static void remove_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

666

int fullness = zspage->fullness;

mm/zsmalloc.c

670

list_del_init(&zspage->list);

mm/zsmalloc.c

683

static int fix_fullness_group(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

687

newfg = get_fullness_group(class, zspage);

mm/zsmalloc.c

688

if (newfg == zspage->fullness)

mm/zsmalloc.c

691

remove_zspage(class, zspage);

mm/zsmalloc.c

692

insert_zspage(class, zspage, newfg);

mm/zsmalloc.c

697

static struct zspage *get_zspage(struct zpdesc *zpdesc)

mm/zsmalloc.c

699

struct zspage *zspage = zpdesc->zspage;

mm/zsmalloc.c

701

BUG_ON(zspage->magic != ZSPAGE_MAGIC);

mm/zsmalloc.c

702

return zspage;

mm/zsmalloc.c

707

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

709

if (unlikely(ZsHugePage(zspage)))

mm/zsmalloc.c

757

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

759

if (unlikely(ZsHugePage(zspage))) {

mm/zsmalloc.c

778

zpdesc->zspage = NULL;

mm/zsmalloc.c

783

static int trylock_zspage(struct zspage *zspage)

mm/zsmalloc.c

787

for (cursor = get_first_zpdesc(zspage); cursor != NULL; cursor =

mm/zsmalloc.c

797

for (cursor = get_first_zpdesc(zspage); cursor != fail; cursor =

mm/zsmalloc.c

805

struct zspage *zspage)

mm/zsmalloc.c

811

VM_BUG_ON(get_zspage_inuse(zspage));

mm/zsmalloc.c

812

VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);

mm/zsmalloc.c

814

next = zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

825

cache_free_zspage(zspage);

mm/zsmalloc.c

832

struct zspage *zspage)

mm/zsmalloc.c

834

VM_BUG_ON(get_zspage_inuse(zspage));

mm/zsmalloc.c

835

VM_BUG_ON(list_empty(&zspage->list));

mm/zsmalloc.c

842

if (!trylock_zspage(zspage)) {

mm/zsmalloc.c

847

remove_zspage(class, zspage);

mm/zsmalloc.c

848

__free_zspage(pool, class, zspage);

mm/zsmalloc.c

852

static void init_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

856

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

893

set_freeobj(zspage, 0);

mm/zsmalloc.c

896

static void create_page_chain(struct size_class *class, struct zspage *zspage,

mm/zsmalloc.c

914

zpdesc->zspage = zspage;

mm/zsmalloc.c

917

zspage->first_zpdesc = zpdesc;

mm/zsmalloc.c

921

SetZsHugePage(zspage);

mm/zsmalloc.c

932

static struct zspage *alloc_zspage(struct zs_pool *pool,

mm/zsmalloc.c

938

struct zspage *zspage = cache_alloc_zspage(gfp);

mm/zsmalloc.c

940

if (!zspage)

mm/zsmalloc.c

946

zspage->magic = ZSPAGE_MAGIC;

mm/zsmalloc.c

947

zspage->pool = pool;

mm/zsmalloc.c

948

zspage->class = class->index;

mm/zsmalloc.c

949

zspage_lock_init(zspage);

mm/zsmalloc.c

960

cache_free_zspage(zspage);

mm/zsmalloc.c

969

create_page_chain(class, zspage, zpdescs);

mm/zsmalloc.c

970

init_zspage(class, zspage);

mm/zsmalloc.c

972

return zspage;

mm/zsmalloc.c

975

static struct zspage *find_get_zspage(struct size_class *class)

mm/zsmalloc.c

978

struct zspage *zspage;

mm/zsmalloc.c

981

zspage = list_first_entry_or_null(&class->fullness_list[i],

mm/zsmalloc.c

982

struct zspage, list);

mm/zsmalloc.c

983

if (zspage)

mm/zsmalloc.c

987

return zspage;