@@ -342,6 +342,7 @@ struct vm_area_struct {
struct mempolicy *vm_policy; /* NUMA policy for the VMA */
#endif
struct vm_userfaultfd_ctx vm_userfaultfd_ctx;
+ seqcount_t vm_sequence;
} __randomize_layout;
struct core_thread {
@@ -1380,6 +1380,7 @@ void unmap_page_range(struct mmu_gather *tlb,
unsigned long next;
BUG_ON(addr >= end);
+ write_seqcount_begin(&vma->vm_sequence);
tlb_start_vma(tlb, vma);
pgd = pgd_offset(vma->vm_mm, addr);
do {
@@ -1389,6 +1390,7 @@ void unmap_page_range(struct mmu_gather *tlb,
next = zap_p4d_range(tlb, vma, pgd, addr, next, details);
} while (pgd++, addr = next, addr != end);
tlb_end_vma(tlb, vma);
+ write_seqcount_end(&vma->vm_sequence);
}
@@ -557,6 +557,8 @@ void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
else
mm->highest_vm_end = vm_end_gap(vma);
+ seqcount_init(&vma->vm_sequence);
+
/*
* vma->vm_prev wasn't known when we followed the rbtree to find the
* correct insertion point for that vma. As a result, we could not
@@ -798,6 +800,11 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
}
}
+ write_seqcount_begin(&vma->vm_sequence);
+ if (next && next != vma)
+ write_seqcount_begin_nested(&next->vm_sequence,
+ SINGLE_DEPTH_NESTING);
+
anon_vma = vma->anon_vma;
if (!anon_vma && adjust_next)
anon_vma = next->anon_vma;
@@ -902,6 +909,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
mm->map_count--;
mpol_put(vma_policy(next));
kmem_cache_free(vm_area_cachep, next);
+ write_seqcount_end(&next->vm_sequence);
/*
* In mprotect's case 6 (see comments on vma_merge),
* we must remove another next too. It would clutter
@@ -931,11 +939,14 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
if (remove_next == 2) {
remove_next = 1;
end = next->vm_end;
+ write_seqcount_end(&vma->vm_sequence);
goto again;
- }
- else if (next)
+ } else if (next) {
+ if (next != vma)
+ write_seqcount_begin_nested(&next->vm_sequence,
+ SINGLE_DEPTH_NESTING);
vma_gap_update(next);
- else {
+ } else {
/*
* If remove_next == 2 we obviously can't
* reach this path.
@@ -961,6 +972,10 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
if (insert && file)
uprobe_mmap(insert);
+ if (next && next != vma)
+ write_seqcount_end(&next->vm_sequence);
+ write_seqcount_end(&vma->vm_sequence);
+
validate_mm(mm);
return 0;