anon_vma
if ((!IS_ENABLED(CONFIG_MMU) || vma->anon_vma) && FILTER(ANON_PRIVATE))
struct anon_vma;
struct anon_vma *anon_vma; /* Serialized by page_table_lock */
struct anon_vma *root; /* Root of this anon_vma tree */
struct anon_vma *anon_vma;
anon_vma = (void *)(mapping - FOLIO_MAPPING_ANON);
VM_WARN_ON_FOLIO(atomic_read(&anon_vma->refcount) == 0, folio);
struct anon_vma *parent; /* Parent of this anon_vma */
struct anon_vma *anon_vma;
struct anon_vma *(*anon_lock)(const struct folio *folio,
struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
vma->anon_vma, vma->vm_ops, vma->vm_pgoff,
vmg->file, vmg->anon_vma, vmg->policy,
VM_BUG_ON_VMA(!vma->anon_vma, vma);
if (!vma->anon_vma)
struct anon_vma *anon_vma = NULL;
anon_vma = folio_get_anon_vma(folio);
if (!anon_vma) {
anon_vma_lock_write(anon_vma);
anon_vma = NULL;
if (anon_vma) {
anon_vma_unlock_write(anon_vma);
put_anon_vma(anon_vma);
struct anon_vma *folio_anon_vma(const struct folio *folio);
static inline void get_anon_vma(struct anon_vma *anon_vma)
atomic_inc(&anon_vma->refcount);
void __put_anon_vma(struct anon_vma *anon_vma);
static inline void put_anon_vma(struct anon_vma *anon_vma)
if (atomic_dec_and_test(&anon_vma->refcount))
__put_anon_vma(anon_vma);
static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
down_write(&anon_vma->root->rwsem);
static inline int anon_vma_trylock_write(struct anon_vma *anon_vma)
return down_write_trylock(&anon_vma->root->rwsem);
static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
up_write(&anon_vma->root->rwsem);
static inline void anon_vma_lock_read(struct anon_vma *anon_vma)
down_read(&anon_vma->root->rwsem);
static inline int anon_vma_trylock_read(struct anon_vma *anon_vma)
return down_read_trylock(&anon_vma->root->rwsem);
static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
up_read(&anon_vma->root->rwsem);
struct anon_vma *folio_get_anon_vma(const struct folio *folio);
if (likely(vma->anon_vma))
anon_vma_lock_write(vma->anon_vma);
anon_vma_unlock_write(vma->anon_vma);
anon_vma_unlock_write(vma->anon_vma);
if (READ_ONCE(vma->anon_vma))
if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
put_anon_vma(rmap_item->anon_vma);
if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
rmap_item->anon_vma = vma->anon_vma;
get_anon_vma(vma->anon_vma);
struct anon_vma *anon_vma; /* when stable */
if (!vma->anon_vma)
if (vma->anon_vma) {
if (vma->anon_vma) {
struct anon_vma *anon_vma = folio_anon_vma(folio);
} else if (!anon_vma) {
anon_vma->root == vma->anon_vma->root) {
struct anon_vma *anon_vma = rmap_item->anon_vma;
if (!anon_vma_trylock_read(anon_vma)) {
anon_vma_lock_read(anon_vma);
anon_vma_interval_tree_foreach(vmac, &anon_vma->rb_root,
anon_vma_unlock_read(anon_vma);
anon_vma_unlock_read(anon_vma);
anon_vma_unlock_read(anon_vma);
struct anon_vma *av = rmap_item->anon_vma;
if (!vma || !(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
put_anon_vma(rmap_item->anon_vma);
put_anon_vma(rmap_item->anon_vma);
VM_WARN_ON_ONCE(!vma->anon_vma &&
prepares_anon_vma(madv_behavior->behavior) && !vma->anon_vma)
struct anon_vma *av;
if (src_vma->anon_vma)
if (likely(vma->anon_vma))
(void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
struct anon_vma *anon_vma)
dst->private = (void *)anon_vma + old_page_state;
struct anon_vma **anon_vmap)
*anon_vmap = (struct anon_vma *)(private & ~PAGE_OLD_STATES);
struct anon_vma *anon_vma,
if (anon_vma)
put_anon_vma(anon_vma);
struct anon_vma *anon_vma = NULL;
anon_vma = folio_get_anon_vma(src);
__migrate_folio_record(dst, old_page_state, anon_vma);
!folio_test_ksm(src) && !anon_vma, src);
__migrate_folio_record(dst, old_page_state, anon_vma);
anon_vma, locked, ret);
struct anon_vma *anon_vma = NULL;
__migrate_folio_extract(dst, &old_page_state, &anon_vma);
if (anon_vma)
put_anon_vma(anon_vma);
__migrate_folio_record(dst, old_page_state, anon_vma);
anon_vma, true, ret);
struct anon_vma *anon_vma = NULL;
anon_vma = folio_get_anon_vma(src);
if (anon_vma)
put_anon_vma(anon_vma);
struct anon_vma *anon_vma = NULL;
__migrate_folio_extract(dst, &old_page_state, &anon_vma);
anon_vma, true, ret_folios);
tmp->anon_vma = NULL;
vma_is_anonymous(vma) && !vma->anon_vma) {
if (vma->anon_vma)
anon_vma_lock_write(vma->anon_vma);
if (vma->anon_vma)
anon_vma_unlock_write(vma->anon_vma);
anon_vma->root = anon_vma;
return anon_vma;
static inline void anon_vma_free(struct anon_vma *anon_vma)
VM_BUG_ON(atomic_read(&anon_vma->refcount));
if (rwsem_is_locked(&anon_vma->root->rwsem)) {
anon_vma_lock_write(anon_vma);
anon_vma_unlock_write(anon_vma);
kmem_cache_free(anon_vma_cachep, anon_vma);
void *anon_vma = vma->anon_vma;
VM_BUG_ON_VMA(!anon_vma, vma);
anon_vma += FOLIO_MAPPING_ANON;
WRITE_ONCE(folio->mapping, anon_vma);
struct anon_vma *anon_vma = vma->anon_vma;
BUG_ON(!anon_vma);
anon_vma = anon_vma->root;
anon_vma = (void *) anon_vma + FOLIO_MAPPING_ANON;
WRITE_ONCE(folio->mapping, (struct address_space *) anon_vma);
VM_BUG_ON_FOLIO(folio_anon_vma(folio)->root != vma->anon_vma->root,
struct anon_vma *anon_vma)
avc->anon_vma = anon_vma;
struct anon_vma *anon_vma, *allocated;
anon_vma = find_mergeable_anon_vma(vma);
if (!anon_vma) {
anon_vma = anon_vma_alloc();
if (unlikely(!anon_vma))
anon_vma->num_children++; /* self-parent link for new root */
allocated = anon_vma;
anon_vma_lock_write(anon_vma);
if (likely(!vma->anon_vma)) {
vma->anon_vma = anon_vma;
anon_vma_chain_assign(vma, avc, anon_vma);
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
anon_vma->num_active_vmas++;
anon_vma_unlock_write(anon_vma);
VM_WARN_ON_ONCE(!src->anon_vma && !list_empty(&src->anon_vma_chain));
VM_WARN_ON_ONCE(!src->anon_vma && dst->anon_vma);
VM_WARN_ON_ONCE(dst->anon_vma && dst->anon_vma != src->anon_vma);
VM_WARN_ON_ONCE(operation != VMA_OP_FORK && src->anon_vma &&
!dst->anon_vma);
struct anon_vma *anon_vma)
if (dst->anon_vma)
if (anon_vma->num_active_vmas > 0)
if (anon_vma->num_children > 1)
dst->anon_vma = anon_vma;
anon_vma->num_active_vmas++;
void __put_anon_vma(struct anon_vma *anon_vma)
struct anon_vma *root = anon_vma->root;
anon_vma_free(anon_vma);
if (root != anon_vma && atomic_dec_and_test(&root->refcount))
static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
struct anon_vma *anon_vma;
anon_vma = folio_anon_vma(folio);
if (!anon_vma)
if (anon_vma_trylock_read(anon_vma))
anon_vma = NULL;
anon_vma_lock_read(anon_vma);
return anon_vma;
struct anon_vma *anon_vma;
anon_vma = folio_anon_vma(folio);
VM_BUG_ON_FOLIO(!anon_vma, folio);
anon_vma = rmap_walk_anon_lock(folio, rwc);
if (!anon_vma)
anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
anon_vma_unlock_read(anon_vma);
struct anon_vma *active_anon_vma = src->anon_vma;
anon_vma_chain_assign(dst, avc, pavc->anon_vma);
anon_vma_lock_write(src->anon_vma);
struct anon_vma *anon_vma = avc->anon_vma;
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
maybe_reuse_anon_vma(dst, anon_vma);
dst->anon_vma->num_active_vmas++;
struct anon_vma *anon_vma;
if (!pvma->anon_vma)
vma->anon_vma = NULL;
anon_vma = anon_vma_alloc();
if (!anon_vma)
put_anon_vma(anon_vma);
if (rc || vma->anon_vma) {
put_anon_vma(anon_vma);
anon_vma->num_active_vmas = 1;
anon_vma->root = pvma->anon_vma->root;
anon_vma->parent = pvma->anon_vma;
get_anon_vma(anon_vma->root);
vma->anon_vma = anon_vma;
anon_vma_chain_assign(vma, avc, anon_vma);
anon_vma_lock_write(anon_vma);
anon_vma_interval_tree_insert(avc, &anon_vma->rb_root);
anon_vma->parent->num_children++;
anon_vma_unlock_write(anon_vma);
vma->anon_vma = NULL;
struct anon_vma *active_anon_vma = vma->anon_vma;
struct anon_vma *anon_vma = avc->anon_vma;
anon_vma_interval_tree_remove(avc, &anon_vma->rb_root);
if (RB_EMPTY_ROOT(&anon_vma->rb_root.rb_root)) {
anon_vma->parent->num_children--;
vma->anon_vma = NULL;
struct anon_vma *anon_vma = avc->anon_vma;
VM_WARN_ON(anon_vma->num_children);
VM_WARN_ON(anon_vma->num_active_vmas);
put_anon_vma(anon_vma);
struct anon_vma *anon_vma = data;
init_rwsem(&anon_vma->rwsem);
atomic_set(&anon_vma->refcount, 0);
anon_vma->rb_root = RB_ROOT_CACHED;
anon_vma_cachep = kmem_cache_create("anon_vma", sizeof(struct anon_vma),
struct anon_vma *folio_get_anon_vma(const struct folio *folio)
struct anon_vma *anon_vma = NULL;
anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
if (!atomic_inc_not_zero(&anon_vma->refcount)) {
anon_vma = NULL;
put_anon_vma(anon_vma);
return anon_vma;
struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio,
struct anon_vma *anon_vma = NULL;
struct anon_vma *root_anon_vma;
anon_vma = (struct anon_vma *) (anon_mapping - FOLIO_MAPPING_ANON);
root_anon_vma = READ_ONCE(anon_vma->root);
anon_vma = NULL;
anon_vma = NULL;
if (!atomic_inc_not_zero(&anon_vma->refcount)) {
anon_vma = NULL;
put_anon_vma(anon_vma);
anon_vma_lock_read(anon_vma);
if (atomic_dec_and_test(&anon_vma->refcount)) {
anon_vma_unlock_read(anon_vma);
__put_anon_vma(anon_vma);
anon_vma = NULL;
return anon_vma;
return anon_vma;
struct anon_vma *anon_vma = folio_anon_vma(folio);
if (!vma->anon_vma || !anon_vma ||
vma->anon_vma->root != anon_vma->root)
static inline struct anon_vma *anon_vma_alloc(void)
struct anon_vma *anon_vma;
anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
if (anon_vma) {
atomic_set(&anon_vma->refcount, 1);
anon_vma->num_children = 0;
anon_vma->num_active_vmas = 0;
anon_vma->parent = anon_vma;
if (vma->anon_vma && !is_vm_hugetlb_page(vma)) {
if (!(vma->vm_flags & VM_SHARED) && unlikely(!vma->anon_vma))
struct anon_vma *folio_anon_vma(const struct folio *folio)
struct anon_vma *tgt_anon = tgt->anon_vma;
struct anon_vma *src_anon = vmg->anon_vma;
VM_WARN_ON(src && src_anon != src->anon_vma);
vp->anon_vma = vma->anon_vma;
if (!vp->anon_vma && adjust)
vp->anon_vma = adjust->anon_vma;
VM_WARN_ON(vp->anon_vma && adjust && adjust->anon_vma &&
vp->anon_vma != adjust->anon_vma);
if (unlikely(vma_is_anonymous(vma) && !vma->anon_vma)) {
static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old,
struct anon_vma *anon_vma = READ_ONCE(old->anon_vma);
if (anon_vma && list_is_singular(&old->anon_vma_chain))
return anon_vma;
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
struct anon_vma *anon_vma = NULL;
anon_vma = reusable_anon_vma(next, vma, next);
if (anon_vma)
return anon_vma;
anon_vma = reusable_anon_vma(prev, prev, vma);
return anon_vma;
static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
if (!test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
down_write_nest_lock(&anon_vma->root->rwsem, &mm->mmap_lock);
&anon_vma->root->rb_root.rb_root.rb_node))
if (vma->anon_vma)
vm_lock_anon_vma(mm, avc->anon_vma);
static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
if (test_bit(0, (unsigned long *) &anon_vma->root->rb_root.rb_root.rb_node)) {
&anon_vma->root->rb_root.rb_root.rb_node))
anon_vma_unlock_write(anon_vma);
if (vma->anon_vma)
vm_unlock_anon_vma(avc->anon_vma);
anon_vma_interval_tree_remove(avc, &avc->anon_vma->rb_root);
anon_vma_interval_tree_insert(avc, &avc->anon_vma->rb_root);
if (vp->anon_vma) {
anon_vma_lock_write(vp->anon_vma);
anon_vma_lock_write(vma->anon_vma);
anon_vma_unlock_write(vma->anon_vma);
anon_vma_lock_write(vma->anon_vma);
anon_vma_unlock_write(vma->anon_vma);
BUG_ON(vma->anon_vma);
if (vp->anon_vma) {
anon_vma_unlock_write(vp->anon_vma);
if (vp->remove->anon_vma)
return !prev->anon_vma || !next->anon_vma ||
prev->anon_vma == next->anon_vma;
if (src->anon_vma && !dst->anon_vma) {
dst->anon_vma = src->anon_vma;
struct anon_vma *anon_vma = vma->anon_vma;
if (anon_vma) {
anon_vma_lock_read(anon_vma);
anon_vma_unlock_read(anon_vma);
return vma && vma->anon_vma && !list_is_singular(&vma->anon_vma_chain);
err = dup_anon_vma(prev, next->anon_vma ? next : middle,
struct anon_vma *anon_vma;
struct anon_vma *anon_vma;
.anon_vma = vma_->anon_vma, \
struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma);
dest->anon_vma = src->anon_vma;
} else if (file && vma->anon_vma) {
struct anon_vma *root;
vma->anon_vma->was_unlinked = true;
if (src->anon_vma != NULL) {
dst->anon_vma = src->anon_vma;
dst->anon_vma->was_cloned = true;
struct anon_vma *anon_vma = calloc(1, sizeof(struct anon_vma));
if (!anon_vma)
anon_vma->root = anon_vma;
vma->anon_vma = anon_vma;
if (likely(vma->anon_vma))
struct anon_vma *anon_vma;
struct anon_vma *anon_vma; /* Serialized by page_table_lock */
static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
struct anon_vma;
static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
struct anon_vma_chain *avc, struct anon_vma *anon_vma)
vma->anon_vma = anon_vma;
avc->anon_vma = vma->anon_vma;
struct anon_vma dummy_anon_vma;
struct anon_vma_chain *avc, struct anon_vma *anon_vma);
extern struct anon_vma dummy_anon_vma;
struct anon_vma dummy_anon_vma_2;
vmg.anon_vma = NULL;
.anon_vma = &dummy_anon_vma,
vma_next->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_TRUE(vma_prev->anon_vma->was_cloned);
vma_next->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_TRUE(vma_prev->anon_vma->was_cloned);
vmg.anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_TRUE(vma_prev->anon_vma->was_cloned);
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_TRUE(vma_prev->anon_vma->was_cloned);
ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
ASSERT_TRUE(vma_next->anon_vma->was_cloned);
vma->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
vma->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
.anon_vma = &dummy_anon_vma,
.anon_vma = &dummy_anon_vma,
.anon_vma = &dummy_anon_vma,
.anon_vma = &dummy_anon_vma,
vma_b->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
vma_d->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
vma_c->anon_vma = &dummy_anon_vma;
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma->anon_vma, &dummy_anon_vma);
struct anon_vma *anon_vma)
vmg->anon_vma = anon_vma;
ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma_next->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);
ASSERT_EQ(vma_prev->anon_vma, &dummy_anon_vma);