Skip to content

Commit c65e774

Browse files
kirylIngo Molnar
authored andcommitted
x86/mm: Make PGDIR_SHIFT and PTRS_PER_P4D variable
For boot-time switching between 4- and 5-level paging we need to be able to fold p4d page table level at runtime. It requires variable PGDIR_SHIFT and PTRS_PER_P4D. The change doesn't affect the kernel image size much: text data bss dec hex filename 8628091 4734304 1368064 14730459 e0c4db vmlinux.before 8628393 4734340 1368064 14730797 e0c62d vmlinux.after Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Borislav Petkov <bp@suse.de> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: linux-mm@kvack.org Link: http://lkml.kernel.org/r/20180214111656.88514-7-kirill.shutemov@linux.intel.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 5c7919b commit c65e774

File tree

13 files changed

+44
-37
lines changed

13 files changed

+44
-37
lines changed

arch/x86/boot/compressed/kaslr.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@
4848

4949
#ifdef CONFIG_X86_5LEVEL
5050
unsigned int pgtable_l5_enabled __ro_after_init = 1;
51+
unsigned int pgdir_shift __ro_after_init = 48;
52+
unsigned int ptrs_per_p4d __ro_after_init = 512;
5153
#endif
5254

5355
extern unsigned long get_cmd_line_ptr(void);

arch/x86/include/asm/pgtable_32.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ static inline void pgtable_cache_init(void) { }
3333
static inline void check_pgt_cache(void) { }
3434
void paging_init(void);
3535

36+
static inline int pgd_large(pgd_t pgd) { return 0; }
37+
3638
/*
3739
* Define this if things work differently on an i386 and an i486:
3840
* it will (on an i486) warn about kernel memory accesses that are

arch/x86/include/asm/pgtable_64_types.h

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ extern unsigned int pgtable_l5_enabled;
2626
#define pgtable_l5_enabled 0
2727
#endif
2828

29+
extern unsigned int pgdir_shift;
30+
extern unsigned int ptrs_per_p4d;
31+
2932
#endif /* !__ASSEMBLY__ */
3033

3134
#define SHARED_KERNEL_PMD 0
@@ -35,16 +38,17 @@ extern unsigned int pgtable_l5_enabled;
3538
/*
3639
* PGDIR_SHIFT determines what a top-level page table entry can map
3740
*/
38-
#define PGDIR_SHIFT 48
41+
#define PGDIR_SHIFT pgdir_shift
3942
#define PTRS_PER_PGD 512
4043

4144
/*
4245
* 4th level page in 5-level paging case
4346
*/
44-
#define P4D_SHIFT 39
45-
#define PTRS_PER_P4D 512
46-
#define P4D_SIZE (_AC(1, UL) << P4D_SHIFT)
47-
#define P4D_MASK (~(P4D_SIZE - 1))
47+
#define P4D_SHIFT 39
48+
#define MAX_PTRS_PER_P4D 512
49+
#define PTRS_PER_P4D ptrs_per_p4d
50+
#define P4D_SIZE (_AC(1, UL) << P4D_SHIFT)
51+
#define P4D_MASK (~(P4D_SIZE - 1))
4852

4953
#define MAX_POSSIBLE_PHYSMEM_BITS 52
5054

@@ -53,8 +57,9 @@ extern unsigned int pgtable_l5_enabled;
5357
/*
5458
* PGDIR_SHIFT determines what a top-level page table entry can map
5559
*/
56-
#define PGDIR_SHIFT 39
57-
#define PTRS_PER_PGD 512
60+
#define PGDIR_SHIFT 39
61+
#define PTRS_PER_PGD 512
62+
#define MAX_PTRS_PER_P4D 1
5863

5964
#endif /* CONFIG_X86_5LEVEL */
6065

arch/x86/kernel/cpu/mcheck/mce.c

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1082,19 +1082,7 @@ void arch_unmap_kpfn(unsigned long pfn)
10821082
* a legal address.
10831083
*/
10841084

1085-
/*
1086-
* Build time check to see if we have a spare virtual bit. Don't want
1087-
* to leave this until run time because most developers don't have a
1088-
* system that can exercise this code path. This will only become a
1089-
* problem if/when we move beyond 5-level page tables.
1090-
*
1091-
* Hard code "9" here because cpp doesn't grok ilog2(PTRS_PER_PGD)
1092-
*/
1093-
#if PGDIR_SHIFT + 9 < 63
10941085
decoy_addr = (pfn << PAGE_SHIFT) + (PAGE_OFFSET ^ BIT(63));
1095-
#else
1096-
#error "no unused virtual bit available"
1097-
#endif
10981086

10991087
if (set_memory_np(decoy_addr, 1))
11001088
pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn);
@@ -2328,6 +2316,12 @@ static __init int mcheck_init_device(void)
23282316
{
23292317
int err;
23302318

2319+
/*
2320+
* Check if we have a spare virtual bit. This will only become
2321+
* a problem if/when we move beyond 5-level page tables.
2322+
*/
2323+
MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);
2324+
23312325
if (!mce_available(&boot_cpu_data)) {
23322326
err = -EIO;
23332327
goto err_out;

arch/x86/kernel/head64.c

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@ pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
4242
#ifdef CONFIG_X86_5LEVEL
4343
unsigned int pgtable_l5_enabled __ro_after_init = 1;
4444
EXPORT_SYMBOL(pgtable_l5_enabled);
45+
unsigned int pgdir_shift __ro_after_init = 48;
46+
EXPORT_SYMBOL(pgdir_shift);
47+
unsigned int ptrs_per_p4d __ro_after_init = 512;
48+
EXPORT_SYMBOL(ptrs_per_p4d);
4549
#endif
4650

4751
#ifdef CONFIG_DYNAMIC_MEMORY_LAYOUT
@@ -336,7 +340,7 @@ asmlinkage __visible void __init x86_64_start_kernel(char * real_mode_data)
336340
BUILD_BUG_ON((__START_KERNEL_map & ~PMD_MASK) != 0);
337341
BUILD_BUG_ON((MODULES_VADDR & ~PMD_MASK) != 0);
338342
BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
339-
BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
343+
MAYBE_BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
340344
(__START_KERNEL & PGDIR_MASK)));
341345
BUILD_BUG_ON(__fix_to_virt(__end_of_fixed_addresses) <= MODULES_END);
342346

arch/x86/mm/dump_pagetables.c

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -428,14 +428,15 @@ static void walk_pud_level(struct seq_file *m, struct pg_state *st, p4d_t addr,
428428
#define p4d_none(a) pud_none(__pud(p4d_val(a)))
429429
#endif
430430

431-
#if PTRS_PER_P4D > 1
432-
433431
static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr, unsigned long P)
434432
{
435433
int i;
436434
p4d_t *start, *p4d_start;
437435
pgprotval_t prot;
438436

437+
if (PTRS_PER_P4D == 1)
438+
return walk_pud_level(m, st, __p4d(pgd_val(addr)), P);
439+
439440
p4d_start = start = (p4d_t *)pgd_page_vaddr(addr);
440441

441442
for (i = 0; i < PTRS_PER_P4D; i++) {
@@ -455,11 +456,8 @@ static void walk_p4d_level(struct seq_file *m, struct pg_state *st, pgd_t addr,
455456
}
456457
}
457458

458-
#else
459-
#define walk_p4d_level(m,s,a,p) walk_pud_level(m,s,__p4d(pgd_val(a)),p)
460-
#define pgd_large(a) p4d_large(__p4d(pgd_val(a)))
461-
#define pgd_none(a) p4d_none(__p4d(pgd_val(a)))
462-
#endif
459+
#define pgd_large(a) (pgtable_l5_enabled ? pgd_large(a) : p4d_large(__p4d(pgd_val(a))))
460+
#define pgd_none(a) (pgtable_l5_enabled ? pgd_none(a) : p4d_none(__p4d(pgd_val(a))))
463461

464462
static inline bool is_hypervisor_range(int idx)
465463
{

arch/x86/mm/init_64.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ void sync_global_pgds(unsigned long start, unsigned long end)
143143
* With folded p4d, pgd_none() is always false, we need to
144144
* handle synchonization on p4d level.
145145
*/
146-
BUILD_BUG_ON(pgd_none(*pgd_ref));
146+
MAYBE_BUILD_BUG_ON(pgd_none(*pgd_ref));
147147
p4d_ref = p4d_offset(pgd_ref, addr);
148148

149149
if (p4d_none(*p4d_ref))

arch/x86/mm/kasan_init_64.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
extern struct range pfn_mapped[E820_MAX_ENTRIES];
2121

22-
static p4d_t tmp_p4d_table[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
22+
static p4d_t tmp_p4d_table[MAX_PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
2323

2424
static __init void *early_alloc(size_t size, int nid, bool panic)
2525
{

arch/x86/platform/efi/efi_64.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -257,8 +257,8 @@ void efi_sync_low_kernel_mappings(void)
257257
* only span a single PGD entry and that the entry also maps
258258
* other important kernel regions.
259259
*/
260-
BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END));
261-
BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) !=
260+
MAYBE_BUILD_BUG_ON(pgd_index(EFI_VA_END) != pgd_index(MODULES_END));
261+
MAYBE_BUILD_BUG_ON((EFI_VA_START & PGDIR_MASK) !=
262262
(EFI_VA_END & PGDIR_MASK));
263263

264264
pgd_efi = efi_pgd + pgd_index(PAGE_OFFSET);

include/asm-generic/5level-fixup.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#define P4D_SHIFT PGDIR_SHIFT
99
#define P4D_SIZE PGDIR_SIZE
1010
#define P4D_MASK PGDIR_MASK
11+
#define MAX_PTRS_PER_P4D 1
1112
#define PTRS_PER_P4D 1
1213

1314
#define p4d_t pgd_t

0 commit comments

Comments
 (0)