@@ -315,10 +315,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec,
* Grow the stack manually; some architectures have a limit on how
* far ahead a user-space access may be in order to grow the stack.
*/
- if (mmap_read_lock_killable(mm))
+ if (mmap_write_lock_killable(mm))
return -EINTR;
- vma = find_extend_vma(mm, bprm->p);
- mmap_read_unlock(mm);
+ vma = find_extend_vma_locked(mm, bprm->p, true);
+ mmap_write_unlock(mm);
if (!vma)
return -EFAULT;
@@ -204,7 +204,8 @@ static struct page *get_arg_page(struct linux_binprm *bprm, unsigned long pos,
#ifdef CONFIG_STACK_GROWSUP
if (write) {
- ret = expand_downwards(bprm->vma, pos);
+ /* We claim to hold the lock - nobody to race with */
+ ret = expand_downwards(bprm->vma, pos, true);
if (ret < 0)
return NULL;
}
@@ -852,7 +853,7 @@ int setup_arg_pages(struct linux_binprm *bprm,
stack_base = vma->vm_end - stack_expand;
#endif
current->mm->start_stack = bprm->p;
- ret = expand_stack(vma, stack_base);
+ ret = expand_stack_locked(vma, stack_base, true);
if (ret)
ret = -EFAULT;
@@ -2941,11 +2941,13 @@ extern vm_fault_t filemap_page_mkwrite(struct vm_fault *vmf);
extern unsigned long stack_guard_gap;
/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
-extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
+int expand_stack_locked(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
+#define expand_stack(vma,addr) expand_stack_locked(vma,addr,false)
/* CONFIG_STACK_GROWSUP still needs to grow downwards at some places */
-extern int expand_downwards(struct vm_area_struct *vma,
- unsigned long address);
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked);
#if VM_GROWSUP
extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
#else
@@ -3046,6 +3048,8 @@ unsigned long change_prot_numa(struct vm_area_struct *vma,
#endif
struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *,
+ unsigned long addr, bool write_locked);
int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
unsigned long pfn, unsigned long size, pgprot_t);
int remap_pfn_range_notrack(struct vm_area_struct *vma, unsigned long addr,
@@ -5362,7 +5362,7 @@ struct vm_area_struct *lock_mm_and_find_vma(struct mm_struct *mm,
goto fail;
}
- if (expand_stack(vma, addr))
+ if (expand_stack_locked(vma, addr, true))
goto fail;
success:
@@ -1945,7 +1945,8 @@ static int acct_stack_growth(struct vm_area_struct *vma,
* PA-RISC uses this for its stack; IA64 for its Register Backing Store.
* vma is the last one with address > vma->vm_end. Have to extend vma.
*/
-int expand_upwards(struct vm_area_struct *vma, unsigned long address)
+int expand_upwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
struct vm_area_struct *next;
@@ -1969,6 +1970,8 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
if (gap_addr < address || gap_addr > TASK_SIZE)
gap_addr = TASK_SIZE;
+ if (!write_locked)
+ return -EAGAIN;
next = find_vma_intersection(mm, vma->vm_end, gap_addr);
if (next && vma_is_accessible(next)) {
if (!(next->vm_flags & VM_GROWSUP))
@@ -2037,7 +2040,8 @@ int expand_upwards(struct vm_area_struct *vma, unsigned long address)
/*
* vma is the first one with address < vma->vm_start. Have to extend vma.
*/
-int expand_downwards(struct vm_area_struct *vma, unsigned long address)
+int expand_downwards(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
struct mm_struct *mm = vma->vm_mm;
MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
@@ -2051,10 +2055,13 @@ int expand_downwards(struct vm_area_struct *vma, unsigned long address)
/* Enforce stack_guard_gap */
prev = mas_prev(&mas, 0);
/* Check that both stack segments have the same anon_vma? */
- if (prev && !(prev->vm_flags & VM_GROWSDOWN) &&
- vma_is_accessible(prev)) {
- if (address - prev->vm_end < stack_guard_gap)
+ if (prev) {
+ if (!(prev->vm_flags & VM_GROWSDOWN) &&
+ vma_is_accessible(prev) &&
+ (address - prev->vm_end < stack_guard_gap))
return -ENOMEM;
+ if (!write_locked && (prev->vm_end == address))
+ return -EAGAIN;
}
if (mas_preallocate(&mas, vma, GFP_KERNEL))
@@ -2132,13 +2139,14 @@ static int __init cmdline_parse_stack_guard_gap(char *p)
__setup("stack_guard_gap=", cmdline_parse_stack_guard_gap);
#ifdef CONFIG_STACK_GROWSUP
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack_locked(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_upwards(vma, address);
+ return expand_upwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma, *prev;
@@ -2146,20 +2154,25 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
vma = find_vma_prev(mm, addr, &prev);
if (vma && (vma->vm_start <= addr))
return vma;
- if (!prev || expand_stack(prev, addr))
+ if (!prev)
+ return NULL;
+ if (expand_stack_locked(prev, addr, write_locked))
return NULL;
if (prev->vm_flags & VM_LOCKED)
populate_vma_page_range(prev, addr, prev->vm_end, NULL);
return prev;
}
#else
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack_locked(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
- return expand_downwards(vma, address);
+ if (unlikely(!(vma->vm_flags & VM_GROWSDOWN)))
+ return -EINVAL;
+ return expand_downwards(vma, address, write_locked);
}
-struct vm_area_struct *
-find_extend_vma(struct mm_struct *mm, unsigned long addr)
+struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm,
+ unsigned long addr, bool write_locked)
{
struct vm_area_struct *vma;
unsigned long start;
@@ -2170,10 +2183,8 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
return NULL;
if (vma->vm_start <= addr)
return vma;
- if (!(vma->vm_flags & VM_GROWSDOWN))
- return NULL;
start = vma->vm_start;
- if (expand_stack(vma, addr))
+ if (expand_stack_locked(vma, addr, write_locked))
return NULL;
if (vma->vm_flags & VM_LOCKED)
populate_vma_page_range(vma, addr, start, NULL);
@@ -2181,6 +2192,11 @@ find_extend_vma(struct mm_struct *mm, unsigned long addr)
}
#endif
+struct vm_area_struct *find_extend_vma(struct mm_struct *mm,
+ unsigned long addr)
+{
+ return find_extend_vma_locked(mm, addr, false);
+}
EXPORT_SYMBOL_GPL(find_extend_vma);
/*
@@ -694,7 +694,8 @@ struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr)
* expand a stack to a given address
* - not supported under NOMMU conditions
*/
-int expand_stack(struct vm_area_struct *vma, unsigned long address)
+int expand_stack_locked(struct vm_area_struct *vma, unsigned long address,
+ bool write_locked)
{
return -ENOMEM;
}