drivers/iommu/iommu-debug-pagealloc.c

33

static struct iommu_debug_metadata *get_iommu_data(struct page_ext *page_ext)

drivers/iommu/iommu-debug-pagealloc.c

35

return page_ext_data(page_ext, &page_iommu_debug_ops);

drivers/iommu/iommu-debug-pagealloc.c

40

struct page_ext *page_ext = page_ext_from_phys(phys);

drivers/iommu/iommu-debug-pagealloc.c

43

if (!page_ext)

drivers/iommu/iommu-debug-pagealloc.c

46

d = get_iommu_data(page_ext);

drivers/iommu/iommu-debug-pagealloc.c

48

page_ext_put(page_ext);

drivers/iommu/iommu-debug-pagealloc.c

53

struct page_ext *page_ext = page_ext_from_phys(phys);

drivers/iommu/iommu-debug-pagealloc.c

56

if (!page_ext)

drivers/iommu/iommu-debug-pagealloc.c

59

d = get_iommu_data(page_ext);

drivers/iommu/iommu-debug-pagealloc.c

61

page_ext_put(page_ext);

drivers/iommu/iommu-debug-pagealloc.c

80

struct page_ext *page_ext = page_ext_get(page);

drivers/iommu/iommu-debug-pagealloc.c

81

struct iommu_debug_metadata *d = get_iommu_data(page_ext);

drivers/iommu/iommu-debug-pagealloc.c

84

page_ext_put(page_ext);

include/linux/mmzone.h

1484

struct page_ext *node_page_ext;

include/linux/mmzone.h

1994

struct page_ext;

include/linux/mmzone.h

2016

struct page_ext *page_ext;

include/linux/page_ext.h

100

static inline void *page_ext_data(struct page_ext *page_ext,

include/linux/page_ext.h

103

return (void *)(page_ext) + ops->offset;

include/linux/page_ext.h

106

static inline struct page_ext *page_ext_next(struct page_ext *curr)

include/linux/page_ext.h

116

struct page_ext *page_ext;

include/linux/page_ext.h

129

static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter,

include/linux/page_ext.h

137

iter->page_ext = page_ext_lookup(pfn);

include/linux/page_ext.h

139

return iter->page_ext;

include/linux/page_ext.h

151

static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter,

include/linux/page_ext.h

156

if (WARN_ON_ONCE(!iter->page_ext))

include/linux/page_ext.h

164

iter->page_ext = page_ext_next(iter->page_ext);

include/linux/page_ext.h

166

iter->page_ext = page_ext_lookup(pfn);

include/linux/page_ext.h

168

return iter->page_ext;

include/linux/page_ext.h

177

static inline struct page_ext *page_ext_iter_get(const struct page_ext_iter *iter)

include/linux/page_ext.h

179

return iter->page_ext;

include/linux/page_ext.h

198

struct page_ext;

include/linux/page_ext.h

221

static inline struct page_ext *page_ext_get(const struct page *page)

include/linux/page_ext.h

226

static inline struct page_ext *page_ext_from_phys(phys_addr_t phys)

include/linux/page_ext.h

231

static inline void page_ext_put(struct page_ext *page_ext)

include/linux/page_ext.h

95

extern struct page_ext *page_ext_get(const struct page *page);

include/linux/page_ext.h

96

extern struct page_ext *page_ext_from_phys(phys_addr_t phys);

include/linux/page_ext.h

97

extern void page_ext_put(struct page_ext *page_ext);

include/linux/page_ext.h

98

extern struct page_ext *page_ext_lookup(unsigned long pfn);

include/linux/page_idle.h

16

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

19

if (unlikely(!page_ext))

include/linux/page_idle.h

22

page_young = test_bit(PAGE_EXT_YOUNG, &page_ext->flags);

include/linux/page_idle.h

23

page_ext_put(page_ext);

include/linux/page_idle.h

30

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

32

if (unlikely(!page_ext))

include/linux/page_idle.h

35

set_bit(PAGE_EXT_YOUNG, &page_ext->flags);

include/linux/page_idle.h

36

page_ext_put(page_ext);

include/linux/page_idle.h

41

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

44

if (unlikely(!page_ext))

include/linux/page_idle.h

47

page_young = test_and_clear_bit(PAGE_EXT_YOUNG, &page_ext->flags);

include/linux/page_idle.h

48

page_ext_put(page_ext);

include/linux/page_idle.h

55

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

58

if (unlikely(!page_ext))

include/linux/page_idle.h

61

page_idle = test_bit(PAGE_EXT_IDLE, &page_ext->flags);

include/linux/page_idle.h

62

page_ext_put(page_ext);

include/linux/page_idle.h

69

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

71

if (unlikely(!page_ext))

include/linux/page_idle.h

74

set_bit(PAGE_EXT_IDLE, &page_ext->flags);

include/linux/page_idle.h

75

page_ext_put(page_ext);

include/linux/page_idle.h

80

struct page_ext *page_ext = page_ext_get(&folio->page);

include/linux/page_idle.h

82

if (unlikely(!page_ext))

include/linux/page_idle.h

85

clear_bit(PAGE_EXT_IDLE, &page_ext->flags);

include/linux/page_idle.h

86

page_ext_put(page_ext);

include/linux/pgalloc_tag.h

115

struct page_ext *page_ext;

include/linux/pgalloc_tag.h

118

page_ext = page_ext_get(page);

include/linux/pgalloc_tag.h

119

if (!page_ext)

include/linux/pgalloc_tag.h

122

tmp = (union codetag_ref *)page_ext_data(page_ext, &page_alloc_tagging_ops);

mm/page_ext.c

128

page_ext_size = sizeof(struct page_ext);

mm/page_ext.c

156

static inline struct page_ext *get_entry(void *base, unsigned long index)

mm/page_ext.c

172

static struct page_ext *lookup_page_ext(const struct page *page)

mm/page_ext.c

176

struct page_ext *base;

mm/page_ext.c

195

struct page_ext *base;

mm/page_ext.c

247

static bool page_ext_invalid(struct page_ext *page_ext)

mm/page_ext.c

249

return !page_ext || (((unsigned long)page_ext & PAGE_EXT_INVALID) == PAGE_EXT_INVALID);

mm/page_ext.c

252

static struct page_ext *lookup_page_ext(const struct page *page)

mm/page_ext.c

256

struct page_ext *page_ext = READ_ONCE(section->page_ext);

mm/page_ext.c

265

if (page_ext_invalid(page_ext))

mm/page_ext.c

267

return get_entry(page_ext, pfn);

mm/page_ext.c

290

struct page_ext *base;

mm/page_ext.c

295

if (section->page_ext)

mm/page_ext.c

318

section->page_ext = (void *)base - page_ext_size * pfn;

mm/page_ext.c

344

struct page_ext *base;

mm/page_ext.c

347

if (!ms || !ms->page_ext)

mm/page_ext.c

350

base = READ_ONCE(ms->page_ext);

mm/page_ext.c

357

WRITE_ONCE(ms->page_ext, NULL);

mm/page_ext.c

369

if (!ms || !ms->page_ext)

mm/page_ext.c

371

val = (void *)ms->page_ext + PAGE_EXT_INVALID;

mm/page_ext.c

372

WRITE_ONCE(ms->page_ext, val);

mm/page_ext.c

511

struct page_ext *page_ext_lookup(unsigned long pfn)

mm/page_ext.c

527

struct page_ext *page_ext_get(const struct page *page)

mm/page_ext.c

529

struct page_ext *page_ext;

mm/page_ext.c

532

page_ext = lookup_page_ext(page);

mm/page_ext.c

533

if (!page_ext) {

mm/page_ext.c

538

return page_ext;

mm/page_ext.c

554

struct page_ext *page_ext_from_phys(phys_addr_t phys)

mm/page_ext.c

574

void page_ext_put(struct page_ext *page_ext)

mm/page_ext.c

576

if (unlikely(!page_ext))

mm/page_owner.c

150

static inline struct page_owner *get_page_owner(struct page_ext *page_ext)

mm/page_owner.c

152

return page_ext_data(page_ext, &page_owner_ops);

mm/page_owner.c

252

struct page_ext *page_ext;

mm/page_owner.c

256

for_each_page_ext(page, 1 << order, page_ext, iter) {

mm/page_owner.c

257

page_owner = get_page_owner(page_ext);

mm/page_owner.c

267

__set_bit(PAGE_EXT_OWNER, &page_ext->flags);

mm/page_owner.c

268

__set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);

mm/page_owner.c

280

struct page_ext *page_ext;

mm/page_owner.c

284

for_each_page_ext(page, 1 << order, page_ext, iter) {

mm/page_owner.c

285

page_owner = get_page_owner(page_ext);

mm/page_owner.c

288

__clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);

mm/page_owner.c

300

struct page_ext *page_ext;

mm/page_owner.c

306

page_ext = page_ext_get(page);

mm/page_owner.c

307

if (unlikely(!page_ext))

mm/page_owner.c

310

page_owner = get_page_owner(page_ext);

mm/page_owner.c

312

page_ext_put(page_ext);

mm/page_owner.c

350

struct page_ext *page_ext = page_ext_get(&folio->page);

mm/page_owner.c

353

if (unlikely(!page_ext))

mm/page_owner.c

356

page_owner = get_page_owner(page_ext);

mm/page_owner.c

358

page_ext_put(page_ext);

mm/page_owner.c

364

struct page_ext *page_ext;

mm/page_owner.c

368

for_each_page_ext(page, 1 << old_order, page_ext, iter) {

mm/page_owner.c

369

page_owner = get_page_owner(page_ext);

mm/page_owner.c

377

struct page_ext *page_ext;

mm/page_owner.c

383

page_ext = page_ext_get(&old->page);

mm/page_owner.c

384

if (unlikely(!page_ext))

mm/page_owner.c

387

old_page_owner = get_page_owner(page_ext);

mm/page_owner.c

388

page_ext_put(page_ext);

mm/page_owner.c

390

page_ext = page_ext_get(&newfolio->page);

mm/page_owner.c

391

if (unlikely(!page_ext))

mm/page_owner.c

394

new_page_owner = get_page_owner(page_ext);

mm/page_owner.c

395

page_ext_put(page_ext);

mm/page_owner.c

418

for_each_page_ext(&old->page, 1 << new_page_owner->order, page_ext, iter) {

mm/page_owner.c

419

old_page_owner = get_page_owner(page_ext);

mm/page_owner.c

429

struct page_ext *page_ext;

mm/page_owner.c

476

page_ext = page_ext_get(page);

mm/page_owner.c

477

if (unlikely(!page_ext))

mm/page_owner.c

480

if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))

mm/page_owner.c

483

page_owner = get_page_owner(page_ext);

mm/page_owner.c

492

page_ext_put(page_ext);

mm/page_owner.c

497

page_ext_put(page_ext);

mm/page_owner.c

607

struct page_ext *page_ext = page_ext_get((void *)page);

mm/page_owner.c

613

if (unlikely(!page_ext)) {

mm/page_owner.c

618

page_owner = get_page_owner(page_ext);

mm/page_owner.c

622

if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {

mm/page_owner.c

624

page_ext_put(page_ext);

mm/page_owner.c

628

if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))

mm/page_owner.c

656

page_ext_put(page_ext);

mm/page_owner.c

664

struct page_ext *page_ext;

mm/page_owner.c

708

page_ext = page_ext_get(page);

mm/page_owner.c

709

if (unlikely(!page_ext))

mm/page_owner.c

716

if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))

mm/page_owner.c

723

if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))

mm/page_owner.c

726

page_owner = get_page_owner(page_ext);

mm/page_owner.c

747

page_ext_put(page_ext);

mm/page_owner.c

751

page_ext_put(page_ext);

mm/page_owner.c

796

struct page_ext *page_ext;

mm/page_owner.c

819

page_ext = page_ext_get(page);

mm/page_owner.c

820

if (unlikely(!page_ext))

mm/page_owner.c

824

if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))

mm/page_owner.c

833

page_ext_put(page_ext);

mm/page_table_check.c

101

struct page_ext *page_ext;

mm/page_table_check.c

113

for_each_page_ext(page, pgcnt, page_ext, iter) {

mm/page_table_check.c

114

struct page_table_check *ptc = get_page_table_check(page_ext);

mm/page_table_check.c

134

struct page_ext *page_ext;

mm/page_table_check.c

139

for_each_page_ext(page, 1 << order, page_ext, iter) {

mm/page_table_check.c

140

struct page_table_check *ptc = get_page_table_check(page_ext);

mm/page_table_check.c

53

static struct page_table_check *get_page_table_check(struct page_ext *page_ext)

mm/page_table_check.c

55

BUG_ON(!page_ext);

mm/page_table_check.c

56

return page_ext_data(page_ext, &page_table_check_ops);

mm/page_table_check.c

66

struct page_ext *page_ext;

mm/page_table_check.c

78

for_each_page_ext(page, pgcnt, page_ext, iter) {

mm/page_table_check.c

79

struct page_table_check *ptc = get_page_table_check(page_ext);