lruvec
struct lruvec lruvec;
static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
static inline struct lruvec *folio_lruvec(struct folio *folio)
static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
return node_page_state(lruvec_pgdat(lruvec), idx);
static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec,
return node_page_state(lruvec_pgdat(lruvec), idx);
static inline struct lruvec *parent_lruvec(struct lruvec *lruvec)
memcg = lruvec_memcg(lruvec);
return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec));
static inline void lruvec_lock_irq(struct lruvec *lruvec)
spin_lock_irq(&lruvec->lru_lock);
static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec)
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
lruvec = mem_cgroup_lruvec(memcg, pgdat);
spin_lock_irq(&lruvec->lru_lock);
lruvec_lock_irq(lruvec);
return lruvec;
static inline void lruvec_unlock(struct lruvec *lruvec)
spin_unlock(&lruvec->lru_lock);
static inline void lruvec_unlock_irq(struct lruvec *lruvec)
spin_unlock_irq(&lruvec->lru_lock);
static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long flags)
spin_unlock_irqrestore(&lruvec->lru_lock, flags);
struct lruvec *lruvec)
return lruvec_pgdat(lruvec) == folio_pgdat(folio) &&
lruvec_memcg(lruvec) == folio_memcg(folio);
static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio,
struct lruvec *locked_lruvec)
struct lruvec **lruvecp, unsigned long *flags)
static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg,
struct lruvec *lruvec;
lruvec = &pgdat->__lruvec;
lruvec = &mz->lruvec;
if (unlikely(lruvec->pgdat != pgdat))
lruvec->pgdat = pgdat;
return lruvec;
static inline struct lruvec *folio_lruvec(struct folio *folio)
struct lruvec *folio_lruvec_lock(struct folio *folio);
struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx);
unsigned long lruvec_page_state_local(struct lruvec *lruvec,
static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen)
unsigned long max_seq = lruvec->lrugen.max_seq;
static inline void lru_gen_update_size(struct lruvec *lruvec, struct folio *folio,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
if (lru_gen_is_active(lruvec, new_gen))
__update_lru_size(lruvec, lru, zone, delta);
if (lru_gen_is_active(lruvec, old_gen))
__update_lru_size(lruvec, lru, zone, -delta);
if (!lru_gen_is_active(lruvec, old_gen) && lru_gen_is_active(lruvec, new_gen)) {
__update_lru_size(lruvec, lru, zone, -delta);
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, delta);
VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) && !lru_gen_is_active(lruvec, new_gen));
static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec,
const struct lru_gen_folio *lrugen = &lruvec->lrugen;
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
seq = lru_gen_folio_seq(lruvec, folio, reclaiming);
lru_gen_update_size(lruvec, folio, -1, gen);
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0;
lru_gen_update_size(lruvec, folio, gen, -1);
static __always_inline void __update_lru_size(struct lruvec *lruvec,
static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)
VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
if (lru_gen_add_folio(lruvec, folio, false))
update_lru_size(lruvec, lru, folio_zonenum(folio),
list_add(&folio->lru, &lruvec->lists[lru]);
void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)
VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
if (lru_gen_add_folio(lruvec, folio, true))
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
update_lru_size(lruvec, lru, folio_zonenum(folio),
list_add_tail(&folio->lru, &lruvec->lists[lru]);
void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)
VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio);
if (lru_gen_del_folio(lruvec, folio, false))
lockdep_assert_held(&lruvec->lru_lock);
update_lru_size(lruvec, lru, folio_zonenum(folio),
mod_lruvec_state(lruvec, NR_LRU_BASE + lru, nr_pages);
static __always_inline void update_lru_size(struct lruvec *lruvec,
__update_lru_size(lruvec, lru, zid, nr_pages);
mem_cgroup_update_lru_size(lruvec, lru, zid, nr_pages);
struct lruvec __lruvec;
extern void lruvec_init(struct lruvec *lruvec);
static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
return lruvec->pgdat;
return container_of(lruvec, struct pglist_data, __lruvec);
struct lruvec;
struct lruvec *lruvec;
void lru_gen_init_lruvec(struct lruvec *lruvec);
static inline void lru_gen_init_lruvec(struct lruvec *lruvec)
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages);
void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx);
void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
static inline void mod_lruvec_state(struct lruvec *lruvec,
mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
void zswap_lruvec_state_init(struct lruvec *lruvec);
static inline void zswap_lruvec_state_init(struct lruvec *lruvec) {}
struct lruvec;
lruvec = folio_lruvec(folio);
if (lruvec != locked || !locked) {
lruvec = compact_folio_lruvec_lock_irqsave(folio, &flags, cc);
locked = lruvec;
lruvec_del_folio(lruvec, folio);
static struct lruvec *
struct lruvec *lruvec;
lruvec = folio_lruvec(folio);
compact_lock_irqsave(&lruvec->lru_lock, flags, cc);
if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
return lruvec;
struct lruvec *lruvec = NULL;
struct lruvec *locked = NULL;
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(goal->nid));
used_pages = lruvec_page_state(lruvec, NR_ACTIVE_ANON);
used_pages += lruvec_page_state(lruvec, NR_INACTIVE_ANON);
used_pages += lruvec_page_state(lruvec, NR_ACTIVE_FILE);
used_pages += lruvec_page_state(lruvec, NR_INACTIVE_FILE);
struct lruvec *lruvec, struct list_head *list)
lockdep_assert_held(&lruvec->lru_lock);
struct lruvec *lruvec;
lruvec = folio_lruvec_lock(folio);
lru_add_split_folio(folio, new_folio, lruvec, list);
lruvec_unlock(lruvec);
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
anon_cost += mz->lruvec.anon_cost;
file_cost += mz->lruvec.file_cost;
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_lruvec_state(lruvec, idx, val);
struct lruvec *folio_lruvec_lock(struct folio *folio)
struct lruvec *lruvec;
lruvec = folio_lruvec(folio);
spin_lock(&lruvec->lru_lock);
if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
spin_unlock(&lruvec->lru_lock);
return lruvec;
struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
struct lruvec *lruvec;
lruvec = folio_lruvec(folio);
spin_lock_irq(&lruvec->lru_lock);
if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
spin_unlock_irq(&lruvec->lru_lock);
return lruvec;
struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
struct lruvec *lruvec;
lruvec = folio_lruvec(folio);
spin_lock_irqsave(&lruvec->lru_lock, *flags);
if (unlikely(lruvec_memcg(lruvec) != folio_memcg(folio))) {
spin_unlock_irqrestore(&lruvec->lru_lock, *flags);
return lruvec;
void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
__func__, lruvec, lru, nr_pages, size)) {
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_memcg_lruvec_state(lruvec, idx, nr);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_memcg_lruvec_state(lruvec, idx, nr);
lruvec_init(&pn->lruvec);
unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx)
return node_page_state(lruvec_pgdat(lruvec), idx);
static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
return lruvec_page_state(lruvec, item) *
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
size = lruvec_page_state_output(lruvec,
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
unsigned long lruvec_page_state_local(struct lruvec *lruvec,
return node_page_state(lruvec_pgdat(lruvec), idx);
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
struct lruvec *child_lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
struct lruvec *parent_lruvec = mem_cgroup_lruvec(parent, NODE_DATA(nid));
child_pn = container_of(child_lruvec, struct mem_cgroup_per_node, lruvec);
parent_pn = container_of(parent_lruvec, struct mem_cgroup_per_node, lruvec);
static void mod_memcg_lruvec_state(struct lruvec *lruvec,
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
void mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
mod_memcg_lruvec_state(lruvec, idx, val);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_lruvec_state(lruvec, idx, val);
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
mod_lruvec_state(lruvec, PGPROMOTE_SUCCESS, nr_succeeded);
struct lruvec *old_lruvec, *new_lruvec;
return lruvec;
static struct lruvec *__mlock_new_folio(struct folio *folio, struct lruvec *lruvec)
lruvec = folio_lruvec_relock_irq(folio, lruvec);
lruvec_add_folio(lruvec, folio);
return lruvec;
static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec)
lruvec = folio_lruvec_relock_irq(folio, lruvec);
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
return lruvec;
struct lruvec *lruvec = NULL;
lruvec = __mlock_folio(folio, lruvec);
lruvec = __mlock_new_folio(folio, lruvec);
lruvec = __munlock_folio(folio, lruvec);
if (lruvec)
lruvec_unlock_irq(lruvec);
static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec)
return lruvec;
lruvec = folio_lruvec_relock_irq(folio, lruvec);
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
void lruvec_init(struct lruvec *lruvec)
memset(lruvec, 0, sizeof(struct lruvec));
spin_lock_init(&lruvec->lru_lock);
zswap_lruvec_state_init(lruvec);
INIT_LIST_HEAD(&lruvec->lists[lru]);
list_del(&lruvec->lists[LRU_UNEVICTABLE]);
lru_gen_init_lruvec(lruvec);
struct lruvec *lruvec = NULL;
if (lruvec) {
lruvec_unlock_irqrestore(lruvec, flags);
lruvec = NULL;
if (lruvec) {
lruvec_unlock_irqrestore(lruvec, flags);
lruvec = NULL;
__page_cache_release(folio, &lruvec, &flags);
if (lruvec)
lruvec_unlock_irqrestore(lruvec, flags);
static void lruvec_reparent_lru(struct lruvec *child_lruvec,
struct lruvec *parent_lruvec,
typedef void (*move_fn_t)(struct lruvec *lruvec, struct folio *folio);
struct lruvec *child_lruvec, *parent_lruvec;
static void lru_add(struct lruvec *lruvec, struct folio *folio)
lruvec_add_folio(lruvec, folio);
struct lruvec *lruvec = NULL;
folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
move_fn(lruvec, folio);
if (lruvec)
lruvec_unlock_irqrestore(lruvec, flags);
static void lru_move_tail(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
lruvec_add_folio_tail(lruvec, folio);
void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file,
__releases(lruvec->lru_lock)
spin_unlock_irq(&lruvec->lru_lock);
lruvec->file_cost += cost;
lruvec->anon_cost += cost;
lrusize = lruvec_page_state(lruvec, NR_INACTIVE_ANON) +
lruvec_page_state(lruvec, NR_ACTIVE_ANON) +
lruvec_page_state(lruvec, NR_INACTIVE_FILE) +
lruvec_page_state(lruvec, NR_ACTIVE_FILE);
if (lruvec->file_cost + lruvec->anon_cost > lrusize / 4) {
lruvec->file_cost /= 2;
lruvec->anon_cost /= 2;
spin_unlock_irq(&lruvec->lru_lock);
lruvec = parent_lruvec(lruvec);
if (!lruvec) {
spin_lock_irq(&lruvec->lru_lock);
struct lruvec *lruvec;
lruvec = folio_lruvec_lock_irq(folio);
lru_note_cost_unlock_irq(lruvec, folio_is_file_lru(folio),
static void lru_activate(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
count_memcg_events(lruvec_memcg(lruvec), PGACTIVATE, nr_pages);
struct lruvec *lruvec;
lruvec = folio_lruvec_lock_irq(folio);
lru_activate(lruvec, folio);
lruvec_unlock_irq(lruvec);
static void lru_deactivate_file(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
lruvec_add_folio_tail(lruvec, folio);
count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE,
static void lru_deactivate(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_pages);
static void lru_lazyfree(struct lruvec *lruvec, struct folio *folio)
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
count_memcg_events(lruvec_memcg(lruvec), PGLAZYFREE, nr_pages);
static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp,
struct lruvec *lruvec = NULL;
__page_cache_release(folio, &lruvec, &flags);
if (lruvec)
lruvec_unlock_irqrestore(lruvec, flags);
static __always_inline void update_lru_sizes(struct lruvec *lruvec,
update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
struct lruvec *lruvec, struct list_head *dst,
struct list_head *src = &lruvec->lists[lru];
update_lru_sizes(lruvec, lru, nr_zone_taken);
struct lruvec *lruvec;
lruvec = folio_lruvec_lock_irq(folio);
lruvec_del_folio(lruvec, folio);
lruvec_unlock_irq(lruvec);
struct lruvec *lruvec = NULL;
lruvec = folio_lruvec_relock_irq(folio, lruvec);
lruvec_unlock_irq(lruvec);
lruvec = NULL;
lruvec_unlock_irq(lruvec);
lruvec = NULL;
lruvec_add_folio(lruvec, folio);
workingset_age_nonresident(lruvec, nr_pages);
if (lruvec)
lruvec_unlock_irq(lruvec);
struct lruvec *lruvec, struct scan_control *sc,
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
lruvec_lock_irq(lruvec);
nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &folio_list,
mod_lruvec_state(lruvec, item, nr_scanned);
mod_lruvec_state(lruvec, PGSCAN_ANON + file, nr_scanned);
lruvec_unlock_irq(lruvec);
lruvec_memcg(lruvec));
mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
mod_lruvec_state(lruvec, item, nr_reclaimed);
mod_lruvec_state(lruvec, PGSTEAL_ANON + file, nr_reclaimed);
lruvec_lock_irq(lruvec);
lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout,
struct lruvec *lruvec,
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
lruvec_lock_irq(lruvec);
nr_taken = isolate_lru_folios(nr_to_scan, lruvec, &l_hold,
mod_lruvec_state(lruvec, PGREFILL, nr_scanned);
lruvec_unlock_irq(lruvec);
count_memcg_events(lruvec_memcg(lruvec), PGDEACTIVATE, nr_deactivate);
lruvec_lock_irq(lruvec);
lru_note_cost_unlock_irq(lruvec, file, 0, nr_rotated);
struct lruvec *lruvec, struct scan_control *sc)
shrink_active_list(nr_to_scan, lruvec, sc, lru);
return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
static bool inactive_is_low(struct lruvec *lruvec, enum lru_list inactive_lru)
inactive = lruvec_page_state(lruvec, NR_LRU_BASE + inactive_lru);
active = lruvec_page_state(lruvec, NR_LRU_BASE + active_lru);
struct lruvec *target_lruvec;
static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
lruvec_size = lruvec_lru_size(lruvec, lru, sc->reclaim_idx);
static bool can_age_anon_pages(struct lruvec *lruvec,
return can_demote(lruvec_pgdat(lruvec)->node_id, sc,
lruvec_memcg(lruvec));
#define DEFINE_MAX_SEQ(lruvec) \
unsigned long max_seq = READ_ONCE((lruvec)->lrugen.max_seq)
#define DEFINE_MIN_SEQ(lruvec) \
READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_ANON]), \
READ_ONCE((lruvec)->lrugen.min_seq[LRU_GEN_FILE]), \
static struct lruvec *get_lruvec(struct mem_cgroup *memcg, int nid)
struct lruvec *lruvec = &memcg->nodeinfo[nid]->lruvec;
if (!lruvec->pgdat)
lruvec->pgdat = pgdat;
return lruvec;
static int get_swappiness(struct lruvec *lruvec, struct scan_control *sc)
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
static int get_nr_gens(struct lruvec *lruvec, int type)
return lruvec->lrugen.max_seq - lruvec->lrugen.min_seq[type] + 1;
static bool __maybe_unused seq_is_valid(struct lruvec *lruvec)
int n = get_nr_gens(lruvec, type);
static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec)
return &lruvec->mm_state;
struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
struct lruvec *lruvec = get_lruvec(memcg, nid);
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
struct lruvec *lruvec = get_lruvec(memcg, nid);
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
static struct lru_gen_mm_state *get_mm_state(struct lruvec *lruvec)
struct lruvec *lruvec = walk->lruvec;
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
lockdep_assert_held(&get_mm_list(lruvec_memcg(lruvec))->lock);
struct lruvec *lruvec = walk->lruvec;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
static bool iterate_mm_list_nowalk(struct lruvec *lruvec, unsigned long seq)
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
static void read_ctrl_pos(struct lruvec *lruvec, int type, int tier, int gain,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
static void reset_ctrl_pos(struct lruvec *lruvec, int type, bool carryover)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
lockdep_assert_held(&lruvec->lru_lock);
static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
lru_gen_update_size(lruvec, folio, old_gen, new_gen);
struct lruvec *lruvec = lruvec_live_lock_irq(walk->lruvec);
struct lru_gen_folio *lrugen = &lruvec->lrugen;
if (lru_gen_is_active(lruvec, gen))
__update_lru_size(lruvec, lru, zone, delta);
lruvec_unlock_irq(lruvec);
struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
DEFINE_MAX_SEQ(walk->lruvec);
struct mem_cgroup *memcg = lruvec_memcg(walk->lruvec);
struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
DEFINE_MAX_SEQ(walk->lruvec);
struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec);
struct pglist_data *pgdat = lruvec_pgdat(walk->lruvec);
struct lruvec *lruvec = walk->lruvec;
DEFINE_MAX_SEQ(lruvec);
static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
new_gen = folio_inc_gen(lruvec, folio);
reset_ctrl_pos(lruvec, type, true);
static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
DEFINE_MIN_SEQ(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
unsigned long lruvec_lru_size(struct lruvec *lruvec, enum lru_list lru, int zone_idx)
reset_ctrl_pos(lruvec, type, true);
static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
lruvec_lock_irq(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
if (inc_min_seq(lruvec, type, swappiness))
lruvec_unlock_irq(lruvec);
for_each_managed_zone_pgdat(zone, lruvec_pgdat(lruvec), zid, zone_idx) {
__update_lru_size(lruvec, lru, zone, delta);
__update_lru_size(lruvec, lru + LRU_ACTIVE, zone, -delta);
reset_ctrl_pos(lruvec, type, false);
lruvec_unlock_irq(lruvec);
size += mem_cgroup_get_zone_lru_size(lruvec, lru, zid);
static bool try_to_inc_max_seq(struct lruvec *lruvec, unsigned long seq,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
return inc_max_seq(lruvec, seq, swappiness);
success = iterate_mm_list_nowalk(lruvec, seq);
success = iterate_mm_list_nowalk(lruvec, seq);
walk->lruvec = lruvec;
success = inc_max_seq(lruvec, seq, swappiness);
static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
total = lruvec_evictable_size(lruvec, swappiness);
static bool lruvec_is_reclaimable(struct lruvec *lruvec, struct scan_control *sc,
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MIN_SEQ(lruvec);
if (!lruvec_is_sizable(lruvec, sc))
birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
reclaimable = lruvec_is_reclaimable(lruvec, sc, min_ttl);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
max_seq = READ_ONCE((lruvec)->lrugen.max_seq);
mm_state = get_mm_state(lruvec);
static void lru_gen_rotate_memcg(struct lruvec *lruvec, int op)
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
VM_WARN_ON_ONCE(hlist_nulls_unhashed(&lruvec->lrugen.list));
new = old = lruvec->lrugen.gen;
WRITE_ONCE(lruvec->lrugen.seg, seg);
WRITE_ONCE(lruvec->lrugen.gen, new);
hlist_nulls_del_rcu(&lruvec->lrugen.list);
hlist_nulls_add_head_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[new][bin]);
struct lruvec *lruvec = get_lruvec(memcg, nid);
VM_WARN_ON_ONCE(!hlist_nulls_unhashed(&lruvec->lrugen.list));
lruvec->lrugen.gen = gen;
hlist_nulls_add_tail_rcu(&lruvec->lrugen.list, &pgdat->memcg_lru.fifo[gen][bin]);
struct lruvec *lruvec = get_lruvec(memcg, nid);
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_OLD);
struct lruvec *lruvec = get_lruvec(memcg, nid);
if (hlist_nulls_unhashed(&lruvec->lrugen.list))
gen = lruvec->lrugen.gen;
hlist_nulls_del_init_rcu(&lruvec->lrugen.list);
struct lruvec *lruvec = get_lruvec(memcg, nid);
if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_HEAD)
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_HEAD);
struct lruvec *lruvec = get_lruvec(memcg, nid);
if (get_nr_gens(lruvec, type) != MAX_NR_GENS)
struct lruvec *lruvec)
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
DEFINE_MAX_SEQ(lruvec);
inc_max_seq(lruvec, max_seq, swappiness);
struct lruvec *lruvec = get_lruvec(memcg, nid);
while (get_nr_gens(lruvec, type) < MAX_NR_GENS) {
try_to_inc_max_seq_nowalk(memcg, lruvec);
static void __lru_gen_reparent_memcg(struct lruvec *child_lruvec, struct lruvec *parent_lruvec,
struct lruvec *child_lruvec, *parent_lruvec;
static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
success = lru_gen_del_folio(lruvec, folio, true);
lruvec_add_folio(lruvec, folio);
gen = folio_inc_gen(lruvec, folio);
gen = folio_inc_gen(lruvec, folio);
static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct scan_control *sc)
success = lru_gen_del_folio(lruvec, folio, true);
static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
if (sort_folio(lruvec, folio, sc, tier))
else if (isolate_folio(lruvec, folio, sc)) {
mod_lruvec_state(lruvec, item, isolated);
mod_lruvec_state(lruvec, PGREFILL, sorted);
mod_lruvec_state(lruvec, PGSCAN_ANON + type, isolated);
static int get_tier_idx(struct lruvec *lruvec, int type)
read_ctrl_pos(lruvec, type, 0, 2, &sp);
read_ctrl_pos(lruvec, type, tier, 3, &pv);
static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
read_ctrl_pos(lruvec, LRU_GEN_ANON, MAX_NR_TIERS, swappiness, &sp);
read_ctrl_pos(lruvec, LRU_GEN_FILE, MAX_NR_TIERS, MAX_SWAPPINESS - swappiness, &pv);
static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
int type = get_type_to_scan(lruvec, swappiness);
int tier = get_tier_idx(lruvec, type);
scanned = scan_folios(nr_to_scan, lruvec, sc,
static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
lruvec_lock_irq(lruvec);
try_to_inc_min_seq(lruvec, swappiness);
scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
try_to_inc_min_seq(lruvec, swappiness);
lruvec_unlock_irq(lruvec);
DEFINE_MIN_SEQ(lruvec);
if (lru_gen_folio_seq(lruvec, folio, false) == min_seq[type])
walk->lruvec = lruvec;
mod_lruvec_state(lruvec, PGDEMOTE_KSWAPD + reclaimer_offset(sc),
mod_lruvec_state(lruvec, item, reclaimed);
mod_lruvec_state(lruvec, PGSTEAL_ANON + type, reclaimed);
static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
DEFINE_MIN_SEQ(lruvec);
static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
evictable = lruvec_evictable_size(lruvec, swappiness);
static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
struct zone *zone = lruvec_pgdat(lruvec)->node_zones + i;
static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
int swappiness = get_swappiness(lruvec, sc);
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
DEFINE_MAX_SEQ(lruvec);
if (should_run_aging(lruvec, max_seq, sc, swappiness)) {
if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false))
delta = evict_folios(nr_batch, lruvec, sc, swappiness);
if (should_abort_scan(lruvec, sc))
static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
struct pglist_data *pgdat = lruvec_pgdat(lruvec);
if (READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL)
need_rotate = try_to_shrink_lruvec(lruvec, sc);
if (!need_rotate && lruvec_is_sizable(lruvec, sc))
return READ_ONCE(lruvec->lrugen.seg) != MEMCG_LRU_TAIL ?
struct lruvec *lruvec;
lru_gen_rotate_memcg(lruvec, op);
lruvec = container_of(lrugen, struct lruvec, lrugen);
memcg = lruvec_memcg(lruvec);
op = shrink_one(lruvec, sc);
if (should_abort_scan(lruvec, sc))
lru_gen_rotate_memcg(lruvec, op);
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
if (try_to_shrink_lruvec(lruvec, sc))
lru_gen_rotate_memcg(lruvec, MEMCG_LRU_YOUNG);
static bool __maybe_unused state_is_valid(struct lruvec *lruvec)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
if (!list_empty(&lruvec->lists[lru]))
static bool fill_evictable(struct lruvec *lruvec)
struct list_head *head = &lruvec->lists[lru];
lruvec_del_folio(lruvec, folio);
success = lru_gen_add_folio(lruvec, folio, false);
static bool drain_evictable(struct lruvec *lruvec)
struct list_head *head = &lruvec->lrugen.folios[gen][type][zone];
success = lru_gen_del_folio(lruvec, folio, false);
lruvec_add_folio(lruvec, folio);
struct lruvec *lruvec = get_lruvec(memcg, nid);
lruvec_lock_irq(lruvec);
VM_WARN_ON_ONCE(!seq_is_valid(lruvec));
VM_WARN_ON_ONCE(!state_is_valid(lruvec));
lruvec->lrugen.enabled = enabled;
while (!(enabled ? fill_evictable(lruvec) : drain_evictable(lruvec))) {
lruvec_unlock_irq(lruvec);
lruvec_lock_irq(lruvec);
lruvec_unlock_irq(lruvec);
static void lru_gen_seq_show_full(struct seq_file *m, struct lruvec *lruvec,
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
struct lruvec *lruvec = v;
struct lru_gen_folio *lrugen = &lruvec->lrugen;
int nid = lruvec_pgdat(lruvec)->node_id;
struct mem_cgroup *memcg = lruvec_memcg(lruvec);
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
unsigned long birth = READ_ONCE(lruvec->lrugen.timestamps[gen]);
lru_gen_seq_show_full(m, lruvec, max_seq, min_seq, seq);
static int run_aging(struct lruvec *lruvec, unsigned long seq,
DEFINE_MAX_SEQ(lruvec);
return try_to_inc_max_seq(lruvec, max_seq, swappiness, force_scan) ? 0 : -EEXIST;
static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
DEFINE_MAX_SEQ(lruvec);
DEFINE_MIN_SEQ(lruvec);
if (!evict_folios(nr_batch, lruvec, sc, swappiness))
struct lruvec *lruvec;
lruvec = get_lruvec(memcg, nid);
swappiness = get_swappiness(lruvec, sc);
err = run_aging(lruvec, seq, swappiness, opt);
err = run_eviction(lruvec, seq, sc, swappiness, opt);
void lru_gen_init_lruvec(struct lruvec *lruvec)
struct lru_gen_folio *lrugen = &lruvec->lrugen;
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
struct lruvec *lruvec = get_lruvec(memcg, nid);
struct lru_gen_mm_state *mm_state = get_mm_state(lruvec);
VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0,
sizeof(lruvec->lrugen.nr_pages)));
lruvec->lrugen.list.next = LIST_POISON1;
static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
lru_gen_shrink_lruvec(lruvec, sc);
get_scan_count(lruvec, sc, nr);
lruvec, sc);
if (can_age_anon_pages(lruvec, sc) &&
inactive_is_low(lruvec, LRU_INACTIVE_ANON))
shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
shrink_lruvec(lruvec, sc);
struct lruvec *target_lruvec;
struct lruvec *target_lruvec;
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(sc->target_mem_cgroup,
clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags);
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat);
shrink_lruvec(lruvec, &sc);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(NULL, pgdat);
if (!can_age_anon_pages(lruvec, sc))
if (!inactive_is_low(lruvec, LRU_INACTIVE_ANON))
lruvec = mem_cgroup_lruvec(memcg, pgdat);
shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
struct lruvec *lruvec = mem_cgroup_lruvec(NULL, pgdat);
clear_bit(LRUVEC_NODE_CONGESTED, &lruvec->flags);
clear_bit(LRUVEC_CGROUP_CONGESTED, &lruvec->flags);
struct lruvec *lruvec = NULL;
lruvec = folio_lruvec_relock_irq(folio, lruvec);
lruvec_del_folio(lruvec, folio);
lruvec_add_folio(lruvec, folio);
if (lruvec) {
lruvec_unlock_irq(lruvec);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, pgdat);
lrugen = &lruvec->lrugen;
static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
*lruvec = mem_cgroup_lruvec(memcg, pgdat);
max_seq = READ_ONCE((*lruvec)->lrugen.max_seq);
struct lruvec *lruvec;
recent = lru_gen_test_recent(shadow, &lruvec, &token, &workingset, type);
if (lruvec != folio_lruvec(folio))
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + type, delta);
lrugen = &lruvec->lrugen;
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + type, delta);
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + type, delta);
static bool lru_gen_test_recent(void *shadow, struct lruvec **lruvec,
void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages)
atomic_long_add(nr_pages, &lruvec->nonresident_age);
} while ((lruvec = parent_lruvec(lruvec)));
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(target_memcg, pgdat);
memcgid = mem_cgroup_private_id(lruvec_memcg(lruvec));
eviction = atomic_long_read(&lruvec->nonresident_age);
workingset_age_nonresident(lruvec, folio_nr_pages(folio));
struct lruvec *eviction_lruvec;
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(memcg, folio_pgdat(folio));
mod_lruvec_state(lruvec, WORKINGSET_REFAULT_BASE + file, nr);
workingset_age_nonresident(lruvec, nr);
mod_lruvec_state(lruvec, WORKINGSET_ACTIVATE_BASE + file, nr);
mod_lruvec_state(lruvec, WORKINGSET_RESTORE_BASE + file, nr);
struct lruvec *lruvec;
lruvec = mem_cgroup_lruvec(sc->memcg, NODE_DATA(sc->nid));
pages += lruvec_lru_size(lruvec, i, MAX_NR_ZONES - 1);
lruvec, NR_SLAB_RECLAIMABLE_B) >> PAGE_SHIFT;
lruvec, NR_SLAB_UNRECLAIMABLE_B) >> PAGE_SHIFT;
struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(sc->nid));
&lruvec->zswap_lruvec_state.nr_disk_swapins;
void zswap_lruvec_state_init(struct lruvec *lruvec)
atomic_long_set(&lruvec->zswap_lruvec_state.nr_disk_swapins, 0);
struct lruvec *lruvec;
lruvec = folio_lruvec(folio);
atomic_long_inc(&lruvec->zswap_lruvec_state.nr_disk_swapins);