mm/zpdesc.h

40

struct zspage *zspage;

mm/zpdesc.h

59

ZPDESC_MATCH(private, zspage);

mm/zsmalloc.c

1001

return get_zspage_inuse(zspage) == class->objs_per_zspage;

mm/zsmalloc.c

1004

static bool zspage_empty(struct zspage *zspage)

mm/zsmalloc.c

1006

return get_zspage_inuse(zspage) == 0;

mm/zsmalloc.c

1039

struct zspage *zspage;

mm/zsmalloc.c

1050

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1053

zspage_read_lock(zspage);

mm/zsmalloc.c

1056

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1059

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1089

struct zspage *zspage;

mm/zsmalloc.c

1097

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1098

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1101

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1109

zspage_read_unlock(zspage);

mm/zsmalloc.c

1116

struct zspage *zspage;

mm/zsmalloc.c

1126

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1129

zspage_read_lock(zspage);

mm/zsmalloc.c

1132

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1135

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1162

struct zspage *zspage;

mm/zsmalloc.c

1169

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1171

zspage_read_unlock(zspage);

mm/zsmalloc.c

1178

struct zspage *zspage;

mm/zsmalloc.c

1188

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1191

zspage_read_lock(zspage);

mm/zsmalloc.c

1194

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1197

if (!ZsHugePage(zspage))

mm/zsmalloc.c

1220

zspage_read_unlock(zspage);

mm/zsmalloc.c

1244

struct zspage *zspage, unsigned long handle)

mm/zsmalloc.c

1255

class = pool->size_class[zspage->class];

mm/zsmalloc.c

1256

obj = get_freeobj(zspage);

mm/zsmalloc.c

1261

m_zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1268

set_freeobj(zspage, link->next >> OBJ_TAG_BITS);

mm/zsmalloc.c

1269

if (likely(!ZsHugePage(zspage)))

mm/zsmalloc.c

1273

zspage->first_zpdesc->handle = handle | OBJ_ALLOCATED_TAG;

mm/zsmalloc.c

1276

mod_zspage_inuse(zspage, 1);

mm/zsmalloc.c

1302

struct zspage *zspage;

mm/zsmalloc.c

1320

zspage = find_get_zspage(class);

mm/zsmalloc.c

1321

if (likely(zspage)) {

mm/zsmalloc.c

1322

obj_malloc(pool, zspage, handle);

mm/zsmalloc.c

1324

fix_fullness_group(class, zspage);

mm/zsmalloc.c

1332

zspage = alloc_zspage(pool, class, gfp, nid);

mm/zsmalloc.c

1333

if (!zspage) {

mm/zsmalloc.c

1339

obj_malloc(pool, zspage, handle);

mm/zsmalloc.c

1340

newfg = get_fullness_group(class, zspage);

mm/zsmalloc.c

1341

insert_zspage(class, zspage, newfg);

mm/zsmalloc.c

1347

SetZsPageMovable(pool, zspage);

mm/zsmalloc.c

1358

struct zspage *zspage;

mm/zsmalloc.c

1367

zspage = get_zspage(f_zpdesc);

mm/zsmalloc.c

1373

if (likely(!ZsHugePage(zspage)))

mm/zsmalloc.c

1374

link->next = get_freeobj(zspage) << OBJ_TAG_BITS;

mm/zsmalloc.c

1377

set_freeobj(zspage, f_objidx);

mm/zsmalloc.c

1380

mod_zspage_inuse(zspage, -1);

mm/zsmalloc.c

1385

struct zspage *zspage;

mm/zsmalloc.c

1401

zspage = get_zspage(f_zpdesc);

mm/zsmalloc.c

1402

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1409

fullness = fix_fullness_group(class, zspage);

mm/zsmalloc.c

1411

free_zspage(pool, class, zspage);

mm/zsmalloc.c

1518

static void migrate_zspage(struct zs_pool *pool, struct zspage *src_zspage,

mm/zsmalloc.c

1519

struct zspage *dst_zspage)

mm/zsmalloc.c

1553

static struct zspage *isolate_src_zspage(struct size_class *class)

mm/zsmalloc.c

1555

struct zspage *zspage;

mm/zsmalloc.c

1559

zspage = list_first_entry_or_null(&class->fullness_list[fg],

mm/zsmalloc.c

1560

struct zspage, list);

mm/zsmalloc.c

1561

if (zspage) {

mm/zsmalloc.c

1562

remove_zspage(class, zspage);

mm/zsmalloc.c

1563

return zspage;

mm/zsmalloc.c

1567

return zspage;

mm/zsmalloc.c

1570

static struct zspage *isolate_dst_zspage(struct size_class *class)

mm/zsmalloc.c

1572

struct zspage *zspage;

mm/zsmalloc.c

1576

zspage = list_first_entry_or_null(&class->fullness_list[fg],

mm/zsmalloc.c

1577

struct zspage, list);

mm/zsmalloc.c

1578

if (zspage) {

mm/zsmalloc.c

1579

remove_zspage(class, zspage);

mm/zsmalloc.c

1580

return zspage;

mm/zsmalloc.c

1584

return zspage;

mm/zsmalloc.c

1594

static int putback_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

1598

fullness = get_fullness_group(class, zspage);

mm/zsmalloc.c

1599

insert_zspage(class, zspage, fullness);

mm/zsmalloc.c

1609

static void lock_zspage(struct zspage *zspage)

mm/zsmalloc.c

1622

zspage_read_lock(zspage);

mm/zsmalloc.c

1623

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1627

zspage_read_unlock(zspage);

mm/zsmalloc.c

1638

zspage_read_unlock(zspage);

mm/zsmalloc.c

1641

zspage_read_lock(zspage);

mm/zsmalloc.c

1644

zspage_read_unlock(zspage);

mm/zsmalloc.c

1650

static void replace_sub_page(struct size_class *class, struct zspage *zspage,

mm/zsmalloc.c

1658

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1667

create_page_chain(class, zspage, zpdescs);

mm/zsmalloc.c

1670

if (unlikely(ZsHugePage(zspage)))

mm/zsmalloc.c

1682

return page_zpdesc(page)->zspage;

mm/zsmalloc.c

1690

struct zspage *zspage;

mm/zsmalloc.c

1707

if (!zpdesc->zspage)

mm/zsmalloc.c

1711

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1712

pool = zspage->pool;

mm/zsmalloc.c

1719

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1726

if (!zspage_write_trylock(zspage)) {

mm/zsmalloc.c

1770

replace_sub_page(class, zspage, newzpdesc, zpdesc);

mm/zsmalloc.c

1777

zspage_write_unlock(zspage);

mm/zsmalloc.c

1809

struct zspage *zspage, *tmp;

mm/zsmalloc.c

1825

list_for_each_entry_safe(zspage, tmp, &free_pages, list) {

mm/zsmalloc.c

1826

list_del(&zspage->list);

mm/zsmalloc.c

1827

lock_zspage(zspage);

mm/zsmalloc.c

1829

class = zspage_class(pool, zspage);

mm/zsmalloc.c

1832

__free_zspage(pool, class, zspage);

mm/zsmalloc.c

1852

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)

mm/zsmalloc.c

1854

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1889

struct zspage *src_zspage = NULL;

mm/zsmalloc.c

1890

struct zspage *dst_zspage = NULL;

mm/zsmalloc.c

2227

zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),

mm/zsmalloc.c

276

static void zspage_lock_init(struct zspage *zspage)

mm/zsmalloc.c

279

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

304

static void zspage_read_lock(struct zspage *zspage)

mm/zsmalloc.c

306

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

317

static void zspage_read_unlock(struct zspage *zspage)

mm/zsmalloc.c

319

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

328

static __must_check bool zspage_write_trylock(struct zspage *zspage)

mm/zsmalloc.c

330

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

344

static void zspage_write_unlock(struct zspage *zspage)

mm/zsmalloc.c

346

struct zspage_lock *zsl = &zspage->zsl;

mm/zsmalloc.c

355

static void SetZsHugePage(struct zspage *zspage)

mm/zsmalloc.c

357

zspage->huge = 1;

mm/zsmalloc.c

360

static bool ZsHugePage(struct zspage *zspage)

mm/zsmalloc.c

362

return zspage->huge;

mm/zsmalloc.c

368

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);

mm/zsmalloc.c

372

static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}

mm/zsmalloc.c

387

static struct zspage *cache_alloc_zspage(gfp_t gfp)

mm/zsmalloc.c

394

static void cache_free_zspage(struct zspage *zspage)

mm/zsmalloc.c

396

kmem_cache_free(zspage_cachep, zspage);

mm/zsmalloc.c

411

static inline int get_zspage_inuse(struct zspage *zspage)

mm/zsmalloc.c

413

return zspage->inuse;

mm/zsmalloc.c

416

static inline void mod_zspage_inuse(struct zspage *zspage, int val)

mm/zsmalloc.c

418

zspage->inuse += val;

mm/zsmalloc.c

421

static struct zpdesc *get_first_zpdesc(struct zspage *zspage)

mm/zsmalloc.c

423

struct zpdesc *first_zpdesc = zspage->first_zpdesc;

mm/zsmalloc.c

447

static inline unsigned int get_freeobj(struct zspage *zspage)

mm/zsmalloc.c

449

return zspage->freeobj;

mm/zsmalloc.c

452

static inline void set_freeobj(struct zspage *zspage, unsigned int obj)

mm/zsmalloc.c

454

zspage->freeobj = obj;

mm/zsmalloc.c

458

struct zspage *zspage)

mm/zsmalloc.c

460

return pool->size_class[zspage->class];

mm/zsmalloc.c

623

static int get_fullness_group(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

627

inuse = get_zspage_inuse(zspage);

mm/zsmalloc.c

651

struct zspage *zspage,

mm/zsmalloc.c

655

list_add(&zspage->list, &class->fullness_list[fullness]);

mm/zsmalloc.c

656

zspage->fullness = fullness;

mm/zsmalloc.c

663

static void remove_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

665

int fullness = zspage->fullness;

mm/zsmalloc.c

669

list_del_init(&zspage->list);

mm/zsmalloc.c

682

static int fix_fullness_group(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

686

newfg = get_fullness_group(class, zspage);

mm/zsmalloc.c

687

if (newfg == zspage->fullness)

mm/zsmalloc.c

690

remove_zspage(class, zspage);

mm/zsmalloc.c

691

insert_zspage(class, zspage, newfg);

mm/zsmalloc.c

696

static struct zspage *get_zspage(struct zpdesc *zpdesc)

mm/zsmalloc.c

698

struct zspage *zspage = zpdesc->zspage;

mm/zsmalloc.c

700

BUG_ON(zspage->magic != ZSPAGE_MAGIC);

mm/zsmalloc.c

701

return zspage;

mm/zsmalloc.c

706

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

708

if (unlikely(ZsHugePage(zspage)))

mm/zsmalloc.c

756

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

758

if (unlikely(ZsHugePage(zspage))) {

mm/zsmalloc.c

777

zpdesc->zspage = NULL;

mm/zsmalloc.c

782

static int trylock_zspage(struct zspage *zspage)

mm/zsmalloc.c

786

for (cursor = get_first_zpdesc(zspage); cursor != NULL; cursor =

mm/zsmalloc.c

796

for (cursor = get_first_zpdesc(zspage); cursor != fail; cursor =

mm/zsmalloc.c

804

struct zspage *zspage)

mm/zsmalloc.c

810

VM_BUG_ON(get_zspage_inuse(zspage));

mm/zsmalloc.c

811

VM_BUG_ON(zspage->fullness != ZS_INUSE_RATIO_0);

mm/zsmalloc.c

813

next = zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

824

cache_free_zspage(zspage);

mm/zsmalloc.c

831

struct zspage *zspage)

mm/zsmalloc.c

833

VM_BUG_ON(get_zspage_inuse(zspage));

mm/zsmalloc.c

834

VM_BUG_ON(list_empty(&zspage->list));

mm/zsmalloc.c

841

if (!trylock_zspage(zspage)) {

mm/zsmalloc.c

846

remove_zspage(class, zspage);

mm/zsmalloc.c

847

__free_zspage(pool, class, zspage);

mm/zsmalloc.c

851

static void init_zspage(struct size_class *class, struct zspage *zspage)

mm/zsmalloc.c

855

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

892

set_freeobj(zspage, 0);

mm/zsmalloc.c

895

static void create_page_chain(struct size_class *class, struct zspage *zspage,

mm/zsmalloc.c

913

zpdesc->zspage = zspage;

mm/zsmalloc.c

916

zspage->first_zpdesc = zpdesc;

mm/zsmalloc.c

920

SetZsHugePage(zspage);

mm/zsmalloc.c

931

static struct zspage *alloc_zspage(struct zs_pool *pool,

mm/zsmalloc.c

937

struct zspage *zspage = cache_alloc_zspage(gfp);

mm/zsmalloc.c

939

if (!zspage)

mm/zsmalloc.c

945

zspage->magic = ZSPAGE_MAGIC;

mm/zsmalloc.c

946

zspage->pool = pool;

mm/zsmalloc.c

947

zspage->class = class->index;

mm/zsmalloc.c

948

zspage_lock_init(zspage);

mm/zsmalloc.c

959

cache_free_zspage(zspage);

mm/zsmalloc.c

968

create_page_chain(class, zspage, zpdescs);

mm/zsmalloc.c

969

init_zspage(class, zspage);

mm/zsmalloc.c

971

return zspage;

mm/zsmalloc.c

974

static struct zspage *find_get_zspage(struct size_class *class)

mm/zsmalloc.c

977

struct zspage *zspage;

mm/zsmalloc.c

980

zspage = list_first_entry_or_null(&class->fullness_list[i],

mm/zsmalloc.c

981

struct zspage, list);

mm/zsmalloc.c

982

if (zspage)

mm/zsmalloc.c

986

return zspage;

mm/zsmalloc.c

999

static bool zspage_full(struct size_class *class, struct zspage *zspage)