Skip to content

Commit 6b7b90e

Browse files
committed
mm: thp: use single linked list for THP page table page deposit.
The old design uses the double linked list page->lru to chain all deposited page table pages when creating a THP and page->pmd_huge_pte to point to the first page of the list. As the second pointer in page->lru overlaps with page->pmd_huge_pte, the design prevents multi-level page table page deposit, which is useful for PUD and higher level THPs. The new design uses single linked list, where deposit_head points to a single linked list of deposited pages and deposit_node can be used to deposit the page itself to another list. For example, this allows us to have one PUD page points to a list of PMD pages, each of which points a list of PTE pages to support PUD level THP. Signed-off-by: Zi Yan <ziy@nvidia.com>
1 parent ab5018a commit 6b7b90e

4 files changed

Lines changed: 17 additions & 19 deletions

File tree

‎include/linux/mm.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <linux/gfp.h>
1111
#include <linux/bug.h>
1212
#include <linux/list.h>
13+
#include <linux/llist.h>
1314
#include <linux/mmzone.h>
1415
#include <linux/rbtree.h>
1516
#include <linux/atomic.h>
@@ -2260,20 +2261,20 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
22602261
static inline bool pmd_ptlock_init(struct page *page)
22612262
{
22622263
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2263-
page->pmd_huge_pte = NULL;
2264+
init_llist_head(&page->deposit_head);
22642265
#endif
22652266
return ptlock_init(page);
22662267
}
22672268

22682269
static inline void pmd_ptlock_free(struct page *page)
22692270
{
22702271
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2271-
VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
2272+
VM_BUG_ON_PAGE(!llist_empty(&page->deposit_head), page);
22722273
#endif
22732274
ptlock_free(page);
22742275
}
22752276

2276-
#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
2277+
#define huge_pmd_deposit_head(mm, pmd) (pmd_to_page(pmd)->deposit_head)
22772278

22782279
#else
22792280

@@ -2285,7 +2286,7 @@ static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
22852286
static inline bool pmd_ptlock_init(struct page *page) { return true; }
22862287
static inline void pmd_ptlock_free(struct page *page) {}
22872288

2288-
#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
2289+
#define huge_pmd_deposit_head(mm, pmd) ((mm)->deposit_head_pmd)
22892290

22902291
#endif
22912292

‎include/linux/mm_types.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
#include <linux/auxvec.h>
88
#include <linux/list.h>
9+
#include <linux/llist.h>
910
#include <linux/spinlock.h>
1011
#include <linux/rbtree.h>
1112
#include <linux/rwsem.h>
@@ -144,8 +145,8 @@ struct page {
144145
struct list_head deferred_list;
145146
};
146147
struct { /* Page table pages */
147-
unsigned long _pt_pad_1; /* compound_head */
148-
pgtable_t pmd_huge_pte; /* protected by page->ptl */
148+
struct llist_head deposit_head; /* pgtable deposit list head */
149+
struct llist_node deposit_node; /* pgtable deposit list node */
149150
unsigned long _pt_pad_2; /* mapping */
150151
union {
151152
struct mm_struct *pt_mm; /* x86 pgds only */
@@ -526,7 +527,8 @@ struct mm_struct {
526527
struct mmu_notifier_subscriptions *notifier_subscriptions;
527528
#endif
528529
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
529-
pgtable_t pmd_huge_pte; /* protected by page_table_lock */
530+
/* pgtable deposit list head, protected by page_table_lock */
531+
struct llist_head deposit_head_pmd;
530532
#endif
531533
#ifdef CONFIG_NUMA_BALANCING
532534
/*

‎kernel/fork.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ static void check_mm(struct mm_struct *mm)
663663
mm_pgtables_bytes(mm));
664664

665665
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
666-
VM_BUG_ON_MM(mm->pmd_huge_pte, mm);
666+
VM_BUG_ON_MM(!llist_empty(&mm->deposit_head_pmd), mm);
667667
#endif
668668
}
669669

@@ -1029,7 +1029,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
10291029
mmu_notifier_subscriptions_init(mm);
10301030
init_tlb_flush_pending(mm);
10311031
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
1032-
mm->pmd_huge_pte = NULL;
1032+
init_llist_head(&mm->deposit_head_pmd);
10331033
#endif
10341034
mm_init_uprobes_state(mm);
10351035

‎mm/pgtable-generic.c‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -164,11 +164,7 @@ void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
164164
assert_spin_locked(pmd_lockptr(mm, pmdp));
165165

166166
/* FIFO */
167-
if (!pmd_huge_pte(mm, pmdp))
168-
INIT_LIST_HEAD(&pgtable->lru);
169-
else
170-
list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru);
171-
pmd_huge_pte(mm, pmdp) = pgtable;
167+
llist_add(&pgtable->deposit_node, &huge_pmd_deposit_head(mm, pmdp));
172168
}
173169
#endif
174170

@@ -180,12 +176,11 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
180176

181177
assert_spin_locked(pmd_lockptr(mm, pmdp));
182178

179+
/* only withdraw from a non empty list */
180+
VM_BUG_ON(llist_empty(&huge_pmd_deposit_head(mm, pmdp)));
183181
/* FIFO */
184-
pgtable = pmd_huge_pte(mm, pmdp);
185-
pmd_huge_pte(mm, pmdp) = list_first_entry_or_null(&pgtable->lru,
186-
struct page, lru);
187-
if (pmd_huge_pte(mm, pmdp))
188-
list_del(&pgtable->lru);
182+
pgtable = llist_entry(llist_del_first(&huge_pmd_deposit_head(mm, pmdp)),
183+
struct page, deposit_node);
189184
return pgtable;
190185
}
191186
#endif

0 commit comments

Comments
 (0)