mm/zpdesc.h

107

const struct page *: (const struct zpdesc *)(p), \

mm/zpdesc.h

108

struct page *: (struct zpdesc *)(p)))

mm/zpdesc.h

110

static inline void zpdesc_lock(struct zpdesc *zpdesc)

mm/zpdesc.h

112

folio_lock(zpdesc_folio(zpdesc));

mm/zpdesc.h

115

static inline bool zpdesc_trylock(struct zpdesc *zpdesc)

mm/zpdesc.h

117

return folio_trylock(zpdesc_folio(zpdesc));

mm/zpdesc.h

120

static inline void zpdesc_unlock(struct zpdesc *zpdesc)

mm/zpdesc.h

122

folio_unlock(zpdesc_folio(zpdesc));

mm/zpdesc.h

125

static inline void zpdesc_wait_locked(struct zpdesc *zpdesc)

mm/zpdesc.h

127

folio_wait_locked(zpdesc_folio(zpdesc));

mm/zpdesc.h

130

static inline void zpdesc_get(struct zpdesc *zpdesc)

mm/zpdesc.h

132

folio_get(zpdesc_folio(zpdesc));

mm/zpdesc.h

135

static inline void zpdesc_put(struct zpdesc *zpdesc)

mm/zpdesc.h

137

folio_put(zpdesc_folio(zpdesc));

mm/zpdesc.h

140

static inline void *kmap_local_zpdesc(struct zpdesc *zpdesc)

mm/zpdesc.h

142

return kmap_local_page(zpdesc_page(zpdesc));

mm/zpdesc.h

145

static inline unsigned long zpdesc_pfn(struct zpdesc *zpdesc)

mm/zpdesc.h

147

return page_to_pfn(zpdesc_page(zpdesc));

mm/zpdesc.h

150

static inline struct zpdesc *pfn_zpdesc(unsigned long pfn)

mm/zpdesc.h

155

static inline void __zpdesc_set_movable(struct zpdesc *zpdesc)

mm/zpdesc.h

157

SetPageMovableOps(zpdesc_page(zpdesc));

mm/zpdesc.h

160

static inline void __zpdesc_set_zsmalloc(struct zpdesc *zpdesc)

mm/zpdesc.h

162

__SetPageZsmalloc(zpdesc_page(zpdesc));

mm/zpdesc.h

165

static inline struct zone *zpdesc_zone(struct zpdesc *zpdesc)

mm/zpdesc.h

167

return page_zone(zpdesc_page(zpdesc));

mm/zpdesc.h

170

static inline bool zpdesc_is_locked(struct zpdesc *zpdesc)

mm/zpdesc.h

172

return folio_test_locked(zpdesc_folio(zpdesc));

mm/zpdesc.h

37

struct zpdesc *next;

mm/zpdesc.h

52

static_assert(offsetof(struct page, pg) == offsetof(struct zpdesc, zp))

mm/zpdesc.h

63

static_assert(sizeof(struct zpdesc) <= sizeof(struct page));

mm/zpdesc.h

75

const struct zpdesc *: (const struct page *)(zp), \

mm/zpdesc.h

76

struct zpdesc *: (struct page *)(zp)))

mm/zpdesc.h

92

const struct zpdesc *: (const struct folio *)(zp), \

mm/zpdesc.h

93

struct zpdesc *: (struct folio *)(zp)))

mm/zsmalloc.c

1041

struct zpdesc *zpdesc;

mm/zsmalloc.c

1050

obj_to_location(obj, &zpdesc, &obj_idx);

mm/zsmalloc.c

1051

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1065

addr = kmap_local_zpdesc(zpdesc);

mm/zsmalloc.c

1075

memcpy_from_page(addr, zpdesc_page(zpdesc),

mm/zsmalloc.c

1077

zpdesc = get_next_zpdesc(zpdesc);

mm/zsmalloc.c

1079

zpdesc_page(zpdesc),

mm/zsmalloc.c

1091

struct zpdesc *zpdesc;

mm/zsmalloc.c

1097

obj_to_location(obj, &zpdesc, &obj_idx);

mm/zsmalloc.c

1098

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1118

struct zpdesc *zpdesc;

mm/zsmalloc.c

1126

obj_to_location(obj, &zpdesc, &obj_idx);

mm/zsmalloc.c

1127

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1142

sg_set_page(sg, zpdesc_page(zpdesc), mem_len, off);

mm/zsmalloc.c

1151

sg_set_page(sg, zpdesc_page(zpdesc), sizes[0], off);

mm/zsmalloc.c

1153

zpdesc = get_next_zpdesc(zpdesc);

mm/zsmalloc.c

1156

sg_set_page(sg, zpdesc_page(zpdesc), sizes[1], 0);

mm/zsmalloc.c

1164

struct zpdesc *zpdesc;

mm/zsmalloc.c

1169

obj_to_location(obj, &zpdesc, &obj_idx);

mm/zsmalloc.c

1170

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1180

struct zpdesc *zpdesc;

mm/zsmalloc.c

1188

obj_to_location(obj, &zpdesc, &obj_idx);

mm/zsmalloc.c

1189

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1203

void *dst = kmap_local_zpdesc(zpdesc);

mm/zsmalloc.c

1214

memcpy_to_page(zpdesc_page(zpdesc), off,

mm/zsmalloc.c

1216

zpdesc = get_next_zpdesc(zpdesc);

mm/zsmalloc.c

1217

memcpy_to_page(zpdesc_page(zpdesc), 0,

mm/zsmalloc.c

1252

struct zpdesc *m_zpdesc;

mm/zsmalloc.c

1360

struct zpdesc *f_zpdesc;

mm/zsmalloc.c

1387

struct zpdesc *f_zpdesc;

mm/zsmalloc.c

1422

struct zpdesc *s_zpdesc, *d_zpdesc;

mm/zsmalloc.c

1494

struct zpdesc *zpdesc, int *obj_idx)

mm/zsmalloc.c

1499

void *addr = kmap_local_zpdesc(zpdesc);

mm/zsmalloc.c

1501

offset = get_first_obj_offset(zpdesc);

mm/zsmalloc.c

1505

if (obj_allocated(zpdesc, addr + offset, &handle))

mm/zsmalloc.c

1525

struct zpdesc *s_zpdesc = get_first_zpdesc(src_zspage);

mm/zsmalloc.c

1612

struct zpdesc *curr_zpdesc, *zpdesc;

mm/zsmalloc.c

1624

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1625

if (zpdesc_trylock(zpdesc))

mm/zsmalloc.c

1627

zpdesc_get(zpdesc);

mm/zsmalloc.c

1629

zpdesc_wait_locked(zpdesc);

mm/zsmalloc.c

1630

zpdesc_put(zpdesc);

mm/zsmalloc.c

1633

curr_zpdesc = zpdesc;

mm/zsmalloc.c

1634

while ((zpdesc = get_next_zpdesc(curr_zpdesc))) {

mm/zsmalloc.c

1635

if (zpdesc_trylock(zpdesc)) {

mm/zsmalloc.c

1636

curr_zpdesc = zpdesc;

mm/zsmalloc.c

1638

zpdesc_get(zpdesc);

mm/zsmalloc.c

1640

zpdesc_wait_locked(zpdesc);

mm/zsmalloc.c

1641

zpdesc_put(zpdesc);

mm/zsmalloc.c

1652

struct zpdesc *newzpdesc, struct zpdesc *oldzpdesc)

mm/zsmalloc.c

1654

struct zpdesc *zpdesc;

mm/zsmalloc.c

1655

struct zpdesc *zpdescs[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };

mm/zsmalloc.c

1659

zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1661

if (zpdesc == oldzpdesc)

mm/zsmalloc.c

1664

zpdescs[idx] = zpdesc;

mm/zsmalloc.c

1666

} while ((zpdesc = get_next_zpdesc(zpdesc)) != NULL);

mm/zsmalloc.c

1692

struct zpdesc *dummy;

mm/zsmalloc.c

1693

struct zpdesc *newzpdesc = page_zpdesc(newpage);

mm/zsmalloc.c

1694

struct zpdesc *zpdesc = page_zpdesc(page);

mm/zsmalloc.c

1708

if (!zpdesc->zspage)

mm/zsmalloc.c

1712

zspage = get_zspage(zpdesc);

mm/zsmalloc.c

1736

offset = get_first_obj_offset(zpdesc);

mm/zsmalloc.c

1737

s_addr = kmap_local_zpdesc(zpdesc);

mm/zsmalloc.c

1748

if (obj_allocated(zpdesc, addr, &handle)) {

mm/zsmalloc.c

1758

replace_sub_page(class, zspage, newzpdesc, zpdesc);

mm/zsmalloc.c

1768

if (zpdesc_zone(newzpdesc) != zpdesc_zone(zpdesc)) {

mm/zsmalloc.c

1769

zpdesc_dec_zone_page_state(zpdesc);

mm/zsmalloc.c

1773

reset_zpdesc(zpdesc);

mm/zsmalloc.c

1774

zpdesc_put(zpdesc);

mm/zsmalloc.c

1842

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

1845

WARN_ON(!zpdesc_trylock(zpdesc));

mm/zsmalloc.c

1846

__zpdesc_set_movable(zpdesc);

mm/zsmalloc.c

1847

zpdesc_unlock(zpdesc);

mm/zsmalloc.c

1848

} while ((zpdesc = get_next_zpdesc(zpdesc)) != NULL);

mm/zsmalloc.c

222

static inline void zpdesc_set_first(struct zpdesc *zpdesc)

mm/zsmalloc.c

224

SetPagePrivate(zpdesc_page(zpdesc));

mm/zsmalloc.c

227

static inline void zpdesc_inc_zone_page_state(struct zpdesc *zpdesc)

mm/zsmalloc.c

229

inc_zone_page_state(zpdesc_page(zpdesc), NR_ZSPAGES);

mm/zsmalloc.c

232

static inline void zpdesc_dec_zone_page_state(struct zpdesc *zpdesc)

mm/zsmalloc.c

234

dec_zone_page_state(zpdesc_page(zpdesc), NR_ZSPAGES);

mm/zsmalloc.c

237

static inline struct zpdesc *alloc_zpdesc(gfp_t gfp, const int nid)

mm/zsmalloc.c

244

static inline void free_zpdesc(struct zpdesc *zpdesc)

mm/zsmalloc.c

246

struct page *page = zpdesc_page(zpdesc);

mm/zsmalloc.c

270

struct zpdesc *first_zpdesc;

mm/zsmalloc.c

405

static inline bool __maybe_unused is_first_zpdesc(struct zpdesc *zpdesc)

mm/zsmalloc.c

407

return PagePrivate(zpdesc_page(zpdesc));

mm/zsmalloc.c

421

static struct zpdesc *get_first_zpdesc(struct zspage *zspage)

mm/zsmalloc.c

423

struct zpdesc *first_zpdesc = zspage->first_zpdesc;

mm/zsmalloc.c

431

static inline unsigned int get_first_obj_offset(struct zpdesc *zpdesc)

mm/zsmalloc.c

433

VM_WARN_ON_ONCE(!PageZsmalloc(zpdesc_page(zpdesc)));

mm/zsmalloc.c

434

return zpdesc->first_obj_offset & FIRST_OBJ_PAGE_TYPE_MASK;

mm/zsmalloc.c

437

static inline void set_first_obj_offset(struct zpdesc *zpdesc, unsigned int offset)

mm/zsmalloc.c

441

VM_WARN_ON_ONCE(!PageZsmalloc(zpdesc_page(zpdesc)));

mm/zsmalloc.c

443

zpdesc->first_obj_offset &= ~FIRST_OBJ_PAGE_TYPE_MASK;

mm/zsmalloc.c

444

zpdesc->first_obj_offset |= offset & FIRST_OBJ_PAGE_TYPE_MASK;

mm/zsmalloc.c

697

static struct zspage *get_zspage(struct zpdesc *zpdesc)

mm/zsmalloc.c

699

struct zspage *zspage = zpdesc->zspage;

mm/zsmalloc.c

705

static struct zpdesc *get_next_zpdesc(struct zpdesc *zpdesc)

mm/zsmalloc.c

707

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

712

return zpdesc->next;

mm/zsmalloc.c

721

static void obj_to_location(unsigned long obj, struct zpdesc **zpdesc,

mm/zsmalloc.c

724

*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);

mm/zsmalloc.c

728

static void obj_to_zpdesc(unsigned long obj, struct zpdesc **zpdesc)

mm/zsmalloc.c

730

*zpdesc = pfn_zpdesc(obj >> OBJ_INDEX_BITS);

mm/zsmalloc.c

738

static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx)

mm/zsmalloc.c

742

obj = zpdesc_pfn(zpdesc) << OBJ_INDEX_BITS;

mm/zsmalloc.c

753

static inline bool obj_allocated(struct zpdesc *zpdesc, void *obj,

mm/zsmalloc.c

757

struct zspage *zspage = get_zspage(zpdesc);

mm/zsmalloc.c

760

VM_BUG_ON_PAGE(!is_first_zpdesc(zpdesc), zpdesc_page(zpdesc));

mm/zsmalloc.c

761

handle = zpdesc->handle;

mm/zsmalloc.c

773

static void reset_zpdesc(struct zpdesc *zpdesc)

mm/zsmalloc.c

775

struct page *page = zpdesc_page(zpdesc);

mm/zsmalloc.c

778

zpdesc->zspage = NULL;

mm/zsmalloc.c

779

zpdesc->next = NULL;

mm/zsmalloc.c

785

struct zpdesc *cursor, *fail;

mm/zsmalloc.c

807

struct zpdesc *zpdesc, *next;

mm/zsmalloc.c

814

next = zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

816

VM_BUG_ON_PAGE(!zpdesc_is_locked(zpdesc), zpdesc_page(zpdesc));

mm/zsmalloc.c

817

next = get_next_zpdesc(zpdesc);

mm/zsmalloc.c

818

reset_zpdesc(zpdesc);

mm/zsmalloc.c

819

zpdesc_unlock(zpdesc);

mm/zsmalloc.c

820

zpdesc_dec_zone_page_state(zpdesc);

mm/zsmalloc.c

821

zpdesc_put(zpdesc);

mm/zsmalloc.c

822

zpdesc = next;

mm/zsmalloc.c

823

} while (zpdesc != NULL);

mm/zsmalloc.c

856

struct zpdesc *zpdesc = get_first_zpdesc(zspage);

mm/zsmalloc.c

858

while (zpdesc) {

mm/zsmalloc.c

859

struct zpdesc *next_zpdesc;

mm/zsmalloc.c

863

set_first_obj_offset(zpdesc, off);

mm/zsmalloc.c

865

vaddr = kmap_local_zpdesc(zpdesc);

mm/zsmalloc.c

878

next_zpdesc = get_next_zpdesc(zpdesc);

mm/zsmalloc.c

889

zpdesc = next_zpdesc;

mm/zsmalloc.c

897

struct zpdesc *zpdescs[])

mm/zsmalloc.c

900

struct zpdesc *zpdesc;

mm/zsmalloc.c

901

struct zpdesc *prev_zpdesc = NULL;

mm/zsmalloc.c

913

zpdesc = zpdescs[i];

mm/zsmalloc.c

914

zpdesc->zspage = zspage;

mm/zsmalloc.c

915

zpdesc->next = NULL;

mm/zsmalloc.c

917

zspage->first_zpdesc = zpdesc;

mm/zsmalloc.c

918

zpdesc_set_first(zpdesc);

mm/zsmalloc.c

923

prev_zpdesc->next = zpdesc;

mm/zsmalloc.c

925

prev_zpdesc = zpdesc;

mm/zsmalloc.c

937

struct zpdesc *zpdescs[ZS_MAX_PAGES_PER_ZSPAGE];

mm/zsmalloc.c

952

struct zpdesc *zpdesc;

mm/zsmalloc.c

954

zpdesc = alloc_zpdesc(gfp, nid);

mm/zsmalloc.c

955

if (!zpdesc) {

mm/zsmalloc.c

963

__zpdesc_set_zsmalloc(zpdesc);

mm/zsmalloc.c

965

zpdesc_inc_zone_page_state(zpdesc);

mm/zsmalloc.c

966

zpdescs[i] = zpdesc;