Skip to content

Commit

Permalink
userfaultfd: use maple tree iterator to iterate VMAs
Browse files Browse the repository at this point in the history 
Don't use the mm_struct linked list or the vma->vm_next in prep for
removal.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Alexandre Frade <kernel@xanmod.org>
  • Loading branch information
@howlett @xanmod
howlett authored and xanmod committed Oct 3, 2022
1 parent bf18fab commit 8e8a314
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 25 deletions.
62 changes: 42 additions & 20 deletions fs/userfaultfd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -615,14 +615,16 @@ static void userfaultfd_event_wait_completion(struct userfaultfd_ctx *ctx,
if (release_new_ctx) {
struct vm_area_struct *vma;
struct mm_struct *mm = release_new_ctx->mm;
VMA_ITERATOR(vmi, mm, 0);

/* the various vma->vm_userfaultfd_ctx still points to it */
mmap_write_lock(mm);
for (vma = mm->mmap; vma; vma = vma->vm_next)
for_each_vma(vmi, vma) {
if (vma->vm_userfaultfd_ctx.ctx == release_new_ctx) {
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
vma->vm_flags &= ~__VM_UFFD_FLAGS;
}
}
mmap_write_unlock(mm);

userfaultfd_ctx_put(release_new_ctx);
Expand Down Expand Up @@ -803,11 +805,13 @@ static bool has_unmap_ctx(struct userfaultfd_ctx *ctx, struct list_head *unmaps,
return false;
}

int userfaultfd_unmap_prep(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct list_head *unmaps)
int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
unsigned long end, struct list_head *unmaps)
{
for ( ; vma && vma->vm_start < end; vma = vma->vm_next) {
VMA_ITERATOR(vmi, mm, start);
struct vm_area_struct *vma;

for_each_vma_range(vmi, vma, end) {
struct userfaultfd_unmap_ctx *unmap_ctx;
struct userfaultfd_ctx *ctx = vma->vm_userfaultfd_ctx.ctx;

Expand Down Expand Up @@ -857,6 +861,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
/* len == 0 means wake all */
struct userfaultfd_wake_range range = { .len = 0, };
unsigned long new_flags;
MA_STATE(mas, &mm->mm_mt, 0, 0);

WRITE_ONCE(ctx->released, true);

Expand All @@ -873,7 +878,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
*/
mmap_write_lock(mm);
prev = NULL;
for (vma = mm->mmap; vma; vma = vma->vm_next) {
mas_for_each(&mas, vma, ULONG_MAX) {
cond_resched();
BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
!!(vma->vm_flags & __VM_UFFD_FLAGS));
Expand All @@ -887,10 +892,13 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
vma->vm_file, vma->vm_pgoff,
vma_policy(vma),
NULL_VM_UFFD_CTX, anon_vma_name(vma));
if (prev)
if (prev) {
mas_pause(&mas);
vma = prev;
else
} else {
prev = vma;
}

vma->vm_flags = new_flags;
vma->vm_userfaultfd_ctx = NULL_VM_UFFD_CTX;
}
Expand Down Expand Up @@ -1272,6 +1280,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
bool found;
bool basic_ioctls;
unsigned long start, end, vma_end;
MA_STATE(mas, &mm->mm_mt, 0, 0);

user_uffdio_register = (struct uffdio_register __user *) arg;

Expand Down Expand Up @@ -1314,7 +1323,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
goto out;

mmap_write_lock(mm);
vma = find_vma_prev(mm, start, &prev);
mas_set(&mas, start);
vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;

Expand All @@ -1339,7 +1349,7 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
*/
found = false;
basic_ioctls = false;
for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();

BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
Expand Down Expand Up @@ -1399,8 +1409,10 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
}
BUG_ON(!found);

if (vma->vm_start < start)
prev = vma;
mas_set(&mas, start);
prev = mas_prev(&mas, 0);
if (prev != vma)
mas_next(&mas, ULONG_MAX);

ret = 0;
do {
Expand Down Expand Up @@ -1430,18 +1442,24 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
((struct vm_userfaultfd_ctx){ ctx }),
anon_vma_name(vma));
if (prev) {
/* vma_merge() invalidated the mas */
mas_pause(&mas);
vma = prev;
goto next;
}
if (vma->vm_start < start) {
ret = split_vma(mm, vma, start, 1);
if (ret)
break;
/* split_vma() invalidated the mas */
mas_pause(&mas);
}
if (vma->vm_end > end) {
ret = split_vma(mm, vma, end, 0);
if (ret)
break;
/* split_vma() invalidated the mas */
mas_pause(&mas);
}
next:
/*
Expand All @@ -1458,8 +1476,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
vma = vma->vm_next;
} while (vma && vma->vm_start < end);
vma = mas_next(&mas, end - 1);
} while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
Expand Down Expand Up @@ -1503,6 +1521,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
bool found;
unsigned long start, end, vma_end;
const void __user *buf = (void __user *)arg;
MA_STATE(mas, &mm->mm_mt, 0, 0);

ret = -EFAULT;
if (copy_from_user(&uffdio_unregister, buf, sizeof(uffdio_unregister)))
Expand All @@ -1521,7 +1540,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
goto out;

mmap_write_lock(mm);
vma = find_vma_prev(mm, start, &prev);
mas_set(&mas, start);
vma = mas_find(&mas, ULONG_MAX);
if (!vma)
goto out_unlock;

Expand All @@ -1546,7 +1566,7 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
*/
found = false;
ret = -EINVAL;
for (cur = vma; cur && cur->vm_start < end; cur = cur->vm_next) {
for (cur = vma; cur; cur = mas_next(&mas, end - 1)) {
cond_resched();

BUG_ON(!!cur->vm_userfaultfd_ctx.ctx ^
Expand All @@ -1566,8 +1586,10 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
}
BUG_ON(!found);

if (vma->vm_start < start)
prev = vma;
mas_set(&mas, start);
prev = mas_prev(&mas, 0);
if (prev != vma)
mas_next(&mas, ULONG_MAX);

ret = 0;
do {
Expand Down Expand Up @@ -1636,8 +1658,8 @@ static int userfaultfd_unregister(struct userfaultfd_ctx *ctx,
skip:
prev = vma;
start = vma->vm_end;
vma = vma->vm_next;
} while (vma && vma->vm_start < end);
vma = mas_next(&mas, end - 1);
} while (vma);
out_unlock:
mmap_write_unlock(mm);
mmput(mm);
Expand Down
7 changes: 3 additions & 4 deletions include/linux/userfaultfd_k.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,8 @@ extern bool userfaultfd_remove(struct vm_area_struct *vma,
unsigned long start,
unsigned long end);

extern int userfaultfd_unmap_prep(struct vm_area_struct *vma,
unsigned long start, unsigned long end,
struct list_head *uf);
extern int userfaultfd_unmap_prep(struct mm_struct *mm, unsigned long start,
unsigned long end, struct list_head *uf);
extern void userfaultfd_unmap_complete(struct mm_struct *mm,
struct list_head *uf);

Expand Down Expand Up @@ -258,7 +257,7 @@ static inline bool userfaultfd_remove(struct vm_area_struct *vma,
return true;
}

static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma,
static inline int userfaultfd_unmap_prep(struct mm_struct *mm,
unsigned long start, unsigned long end,
struct list_head *uf)
{
Expand Down
2 changes: 1 addition & 1 deletion mm/mmap.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2542,7 +2542,7 @@ do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
* split, despite we could. This is unlikely enough
* failure that it's not worth optimizing it for.
*/
error = userfaultfd_unmap_prep(vma, start, end, uf);
error = userfaultfd_unmap_prep(mm, start, end, uf);

if (error)
goto userfaultfd_error;
Expand Down

0 comments on commit 8e8a314

Please sign in to comment.