Skip to content

Commit bc70fbf

Browse files
xzpeterakpm00
xzpeter
authored and
akpm00
committed
mm/hugetlb: handle uffd-wp during fork()
Firstly, we'll need to pass in dst_vma into copy_hugetlb_page_range() because for uffd-wp it's the dst vma that matters on deciding how we should treat uffd-wp protected ptes. We should recognize pte markers during fork and do the pte copy if needed. [lkp@intel.com: vma_needs_copy can be static] Link: https://lkml.kernel.org/r/Ylb0CGeFJlc4EzLk@7ec4ff11d4ae Link: https://lkml.kernel.org/r/20220405014918.14932-1-peterx@redhat.com Signed-off-by: Peter Xu <peterx@redhat.com> Cc: Alistair Popple <apopple@nvidia.com> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Axel Rasmussen <axelrasmussen@google.com> Cc: David Hildenbrand <david@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: Jerome Glisse <jglisse@redhat.com> Cc: "Kirill A . Shutemov" <kirill@shutemov.name> Cc: Matthew Wilcox <willy@infradead.org> Cc: Mike Kravetz <mike.kravetz@oracle.com> Cc: Mike Rapoport <rppt@linux.vnet.ibm.com> Cc: Nadav Amit <nadav.amit@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
1 parent 05e90bd commit bc70fbf

File tree

3 files changed

+35
-18
lines changed

3 files changed

+35
-18
lines changed

‎include/linux/hugetlb.h‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ int move_hugetlb_page_tables(struct vm_area_struct *vma,
137137
struct vm_area_struct *new_vma,
138138
unsigned long old_addr, unsigned long new_addr,
139139
unsigned long len);
140-
int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
140+
int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *,
141+
struct vm_area_struct *, struct vm_area_struct *);
141142
long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
142143
struct page **, struct vm_area_struct **,
143144
unsigned long *, unsigned long *, long, unsigned int,
@@ -269,7 +270,9 @@ static inline struct page *follow_huge_addr(struct mm_struct *mm,
269270
}
270271

271272
static inline int copy_hugetlb_page_range(struct mm_struct *dst,
272-
struct mm_struct *src, struct vm_area_struct *vma)
273+
struct mm_struct *src,
274+
struct vm_area_struct *dst_vma,
275+
struct vm_area_struct *src_vma)
273276
{
274277
BUG();
275278
return 0;

‎mm/hugetlb.c‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4719,23 +4719,24 @@ hugetlb_install_page(struct vm_area_struct *vma, pte_t *ptep, unsigned long addr
47194719
}
47204720

47214721
int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
4722-
struct vm_area_struct *vma)
4722+
struct vm_area_struct *dst_vma,
4723+
struct vm_area_struct *src_vma)
47234724
{
47244725
pte_t *src_pte, *dst_pte, entry, dst_entry;
47254726
struct page *ptepage;
47264727
unsigned long addr;
4727-
bool cow = is_cow_mapping(vma->vm_flags);
4728-
struct hstate *h = hstate_vma(vma);
4728+
bool cow = is_cow_mapping(src_vma->vm_flags);
4729+
struct hstate *h = hstate_vma(src_vma);
47294730
unsigned long sz = huge_page_size(h);
47304731
unsigned long npages = pages_per_huge_page(h);
4731-
struct address_space *mapping = vma->vm_file->f_mapping;
4732+
struct address_space *mapping = src_vma->vm_file->f_mapping;
47324733
struct mmu_notifier_range range;
47334734
int ret = 0;
47344735

47354736
if (cow) {
4736-
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, src,
4737-
vma->vm_start,
4738-
vma->vm_end);
4737+
mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, src_vma, src,
4738+
src_vma->vm_start,
4739+
src_vma->vm_end);
47394740
mmu_notifier_invalidate_range_start(&range);
47404741
mmap_assert_write_locked(src);
47414742
raw_write_seqcount_begin(&src->write_protect_seq);
@@ -4749,12 +4750,12 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
47494750
i_mmap_lock_read(mapping);
47504751
}
47514752

4752-
for (addr = vma->vm_start; addr < vma->vm_end; addr += sz) {
4753+
for (addr = src_vma->vm_start; addr < src_vma->vm_end; addr += sz) {
47534754
spinlock_t *src_ptl, *dst_ptl;
47544755
src_pte = huge_pte_offset(src, addr, sz);
47554756
if (!src_pte)
47564757
continue;
4757-
dst_pte = huge_pte_alloc(dst, vma, addr, sz);
4758+
dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz);
47584759
if (!dst_pte) {
47594760
ret = -ENOMEM;
47604761
break;
@@ -4789,6 +4790,7 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
47894790
} else if (unlikely(is_hugetlb_entry_migration(entry) ||
47904791
is_hugetlb_entry_hwpoisoned(entry))) {
47914792
swp_entry_t swp_entry = pte_to_swp_entry(entry);
4793+
bool uffd_wp = huge_pte_uffd_wp(entry);
47924794

47934795
if (!is_readable_migration_entry(swp_entry) && cow) {
47944796
/*
@@ -4798,10 +4800,21 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
47984800
swp_entry = make_readable_migration_entry(
47994801
swp_offset(swp_entry));
48004802
entry = swp_entry_to_pte(swp_entry);
4803+
if (userfaultfd_wp(src_vma) && uffd_wp)
4804+
entry = huge_pte_mkuffd_wp(entry);
48014805
set_huge_swap_pte_at(src, addr, src_pte,
48024806
entry, sz);
48034807
}
4808+
if (!userfaultfd_wp(dst_vma) && uffd_wp)
4809+
entry = huge_pte_clear_uffd_wp(entry);
48044810
set_huge_swap_pte_at(dst, addr, dst_pte, entry, sz);
4811+
} else if (unlikely(is_pte_marker(entry))) {
4812+
/*
4813+
* We copy the pte marker only if the dst vma has
4814+
* uffd-wp enabled.
4815+
*/
4816+
if (userfaultfd_wp(dst_vma))
4817+
set_huge_pte_at(dst, addr, dst_pte, entry);
48054818
} else {
48064819
entry = huge_ptep_get(src_pte);
48074820
ptepage = pte_page(entry);
@@ -4819,20 +4832,21 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
48194832
*/
48204833
if (!PageAnon(ptepage)) {
48214834
page_dup_file_rmap(ptepage, true);
4822-
} else if (page_try_dup_anon_rmap(ptepage, true, vma)) {
4835+
} else if (page_try_dup_anon_rmap(ptepage, true,
4836+
src_vma)) {
48234837
pte_t src_pte_old = entry;
48244838
struct page *new;
48254839

48264840
spin_unlock(src_ptl);
48274841
spin_unlock(dst_ptl);
48284842
/* Do not use reserve as it's private owned */
4829-
new = alloc_huge_page(vma, addr, 1);
4843+
new = alloc_huge_page(dst_vma, addr, 1);
48304844
if (IS_ERR(new)) {
48314845
put_page(ptepage);
48324846
ret = PTR_ERR(new);
48334847
break;
48344848
}
4835-
copy_user_huge_page(new, ptepage, addr, vma,
4849+
copy_user_huge_page(new, ptepage, addr, dst_vma,
48364850
npages);
48374851
put_page(ptepage);
48384852

@@ -4842,13 +4856,13 @@ int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
48424856
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
48434857
entry = huge_ptep_get(src_pte);
48444858
if (!pte_same(src_pte_old, entry)) {
4845-
restore_reserve_on_error(h, vma, addr,
4859+
restore_reserve_on_error(h, dst_vma, addr,
48464860
new);
48474861
put_page(new);
48484862
/* dst_entry won't change as in child */
48494863
goto again;
48504864
}
4851-
hugetlb_install_page(vma, dst_pte, addr, new);
4865+
hugetlb_install_page(dst_vma, dst_pte, addr, new);
48524866
spin_unlock(src_ptl);
48534867
spin_unlock(dst_ptl);
48544868
continue;

‎mm/memory.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1234,7 +1234,7 @@ copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
12341234
* false when we can speed up fork() by allowing lazy page faults later until
12351235
* when the child accesses the memory range.
12361236
*/
1237-
bool
1237+
static bool
12381238
vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
12391239
{
12401240
/*
@@ -1278,7 +1278,7 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
12781278
return 0;
12791279

12801280
if (is_vm_hugetlb_page(src_vma))
1281-
return copy_hugetlb_page_range(dst_mm, src_mm, src_vma);
1281+
return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
12821282

12831283
if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
12841284
/*

0 commit comments

Comments
 (0)