Add option to double the kernel thread stack by using larger page size.

This code change adds the kernel option KSTACK_LARGE_PAGE that uses one
16K page for the kernel stack instead of only 4K page sizes. This doubles
the kernel thread stack size without using additional wired TLB entries.
To use this "option KSTACK_LARGE_PAGE" must be added to the kernel conf
file.

Sponsored by:       DARPA, AFRL

sys/conf/options.mips

@@ -97,6 +97,13 @@ PLATFORM_INIT_SECONDARY		opt_global.h
 #
 NF10BMAC_64BIT			opt_netfpga.h
 
+#
+# Options for hardware with PageMask register support
+#
+
+# Use one large page (currently 16K) for the kernel thread stack
+KSTACK_LARGE_PAGE   		opt_global.h
+
 #
 # Options that control the Atheros SoC peripherals
 #

sys/mips/include/param.h

@@ -161,6 +161,22 @@
 
 #define	MAXDUMPPGS		1		/* xxx: why is this only one? */
 
+#ifdef KSTACK_LARGE_PAGE
+/*
+ * For a large kernel stack page the KSTACK_SIZE needs to be a page size
+ * supported by the hardware (e.g. 16K).
+ */
+#define	KSTACK_SIZE		(1 << 14)	/* Single 16K page */
+#define	KSTACK_PAGE_SIZE	KSTACK_SIZE
+#define	KSTACK_PAGE_MASK	(KSTACK_PAGE_SIZE - 1)
+#define	KSTACK_PAGES		(KSTACK_SIZE / PAGE_SIZE)
+#define	KSTACK_TLBMASK_MASK	((KSTACK_PAGE_MASK >> (TLBMASK_SHIFT - 1)) \
+					<< TLBMASK_SHIFT)
+#define	KSTACK_GUARD_PAGES	((KSTACK_PAGE_SIZE * 2) / PAGE_SIZE)
+#define	KSTACK_OBJT		OBJT_PHYS
+
+#else /* ! KSTACK_LARGE_PAGE */
+
 /*
  * The kernel stack needs to be aligned on a (PAGE_SIZE * 2) boundary.
  */
@@ -175,6 +191,7 @@
 #define	KSTACK_PAGE_MASK	(PAGE_SIZE - 1)
 #define	KSTACK_GUARD_PAGES	2	/* pages of kstack guard; 0 disables */
 #define	KSTACK_OBJT		OBJT_DEFAULT
+#endif /* ! KSTACK_LARGE_PAGE */
 
 /*
  * Mach derived conversion macros

sys/mips/mips/cpu.c

@@ -282,6 +282,12 @@ mips_get_identity(struct mips_cpuinfo *cpuinfo)
 	mips_wr_pagemask(~0);
 	cpuinfo->tlb_pgmask = mips_rd_pagemask();
 	mips_wr_pagemask(MIPS3_PGMASK_4K);
+
+#ifdef KSTACK_LARGE_PAGE
+	if ((cpuinfo->tlb_pgmask & MIPS3_PGMASK_16K) == 0)
+		panic("%s: 16K sized pages are not supported by this CPU.",
+		    __func__);
+#endif
 }
 
 void

sys/mips/mips/genassym.c

@@ -107,6 +107,9 @@ ASSYM(TDF_NEEDRESCHED, TDF_NEEDRESCHED);
 ASSYM(TDF_ASTPENDING, TDF_ASTPENDING);
 ASSYM(MAXCOMLEN, MAXCOMLEN);
 ASSYM(MDTD_COP2USED, MDTD_COP2USED);
+#ifdef KSTACK_LARGE_PAGE
+ASSYM(KSTACK_TLBMASK_MASK, KSTACK_TLBMASK_MASK);
+#endif
 
 ASSYM(MIPS_KSEG0_START, MIPS_KSEG0_START);
 ASSYM(MIPS_KSEG1_START, MIPS_KSEG1_START);

sys/mips/mips/machdep.c

@@ -292,9 +292,9 @@ mips_proc0_init(void)
 #endif
 	proc_linkup0(&proc0, &thread0);
 
-	KASSERT((kstack0 & PAGE_MASK) == 0,
-		("kstack0 is not aligned on a page boundary: 0x%0lx",
-		(long)kstack0));
+	KASSERT((kstack0 & ((KSTACK_PAGE_SIZE * 2) - 1)) == 0,
+		("kstack0 is not aligned on a page (0x%0lx) boundary: 0x%0lx",
+		(long)(KSTACK_PAGE_SIZE * 2), (long)kstack0));
 	thread0.td_kstack = kstack0;
 	thread0.td_kstack_pages = KSTACK_PAGES;
 	/* 

sys/mips/mips/pmap.c

@@ -546,9 +546,15 @@ pmap_bootstrap(void)
 	msgbufinit(msgbufp, msgbufsize);
 
 	/*
-	 * Steal thread0 kstack.
+	 * Steal thread0 kstack.  This must be aligned to
+	 * (KSTACK_PAGE_SIZE * 2) so it can mapped to a single TLB entry.
+	 *
+	 * XXX There should be a better way of getting aligned memory
+	 * with pmap_steal_memory().
 	 */
-	kstack0 = pmap_steal_memory(KSTACK_PAGES << PAGE_SHIFT);
+	kstack0 = pmap_steal_memory((KSTACK_PAGES + KSTACK_GUARD_PAGES) \
+					<< PAGE_SHIFT);
+	kstack0 = roundup2(kstack0, (KSTACK_PAGE_SIZE * 2));
 
 	virtual_avail = VM_MIN_KERNEL_ADDRESS;
 	virtual_end = VM_MAX_KERNEL_ADDRESS;

sys/mips/mips/swtch.S

@@ -323,12 +323,14 @@ blocked_loop:
 	PTR_S	a2, PC_CURPCB(a3)
 	PTR_L	v0, TD_KSTACK(a1)		# va of 1st half of kstack
 #if defined(__mips_n64)
+#ifndef KSTACK_LARGE_PAGE
 	PTR_LI	s0,2*PAGE_SIZE
 	PTR_SUB	v1,v0,s0
+#endif /* !KSTACK_LARGE_PAGE */
 	PTR_LI	s0, MIPS_XKSEG_START
 #else
 	PTR_LI	s0, MIPS_KSEG2_START		# If Uarea addr is below kseg2,
-#endif
+#endif /* __mips_n64 */
 	bltu	v0, s0, sw2			# no need to insert in TLB.
 	PTE_L	a1, TD_UPTE + 0(s7)		# a1 = u. pte #0
 	PTE_L	a2, TD_UPTE + PTESIZE(s7)	# a2 = u. pte #1
@@ -346,53 +348,108 @@ blocked_loop:
 	mfc0	s0, MIPS_COP_0_TLB_INDEX
 	HAZARD_DELAY
 
-	PTR_LI	t1, MIPS_KSEG0_START		# invalidate tlb entry
-	bltz	s0, entry0set
+	# MIPS_KSEG0_START + (2 * index * PAGE_SIZE) -> MIPS_COP_0_TLB_HI
+	PTR_LI		t1, MIPS_KSEG0_START		# invalidate tlb entry
+#ifdef KSTACK_LARGE_PAGE
+	bltz		s0, inval_nxt1
+#else
+	bltz		s0, entry0set
+#endif /* KSTACK_LARGE_PAGE */
+	nop
+	sll		s0, PAGE_SHIFT + 1
+	PTR_ADDU	t1, s0
+	MTC0		t1, MIPS_COP_0_TLB_HI
+	PTE_MTC0	zero, MIPS_COP_0_TLB_LO0
+	PTE_MTC0	zero, MIPS_COP_0_TLB_LO1
+	MTC0		zero, MIPS_COP_0_TLB_PG_MASK
+	HAZARD_DELAY
+	tlbwi
+	HAZARD_DELAY
+
+#ifdef KSTACK_LARGE_PAGE
+/*
+ * With a KSTACK_PAGE_SIZE of 16K and PAGE_SIZE of 4K it is possible that
+ * a second TLB entry is currently mapping the kernel thread stack as a
+ * regular 4K sized page(s). Check for this case and, if so, invalidate
+ * that TLB entry as well.
+ */
+#if (PAGE_SIZE != 4096) && (KSTACK_PAGE_SIZE != 16384)
+#error PAGE_SIZE is not 4K or KSTACK_PAGE_SIZE is not 16K.
+#endif
+inval_nxt1:
+	move		v1, v0
+	PTR_ADDU	v1, PAGE_SIZE * 2
+	MTC0		v1, MIPS_COP_0_TLB_HI		# VPN = va
+	HAZARD_DELAY
+	tlbp						# probe VPN
+	HAZARD_DELAY
+	mfc0		s0, MIPS_COP_0_TLB_INDEX
+	HAZARD_DELAY
+
+	# MIPS_KSEG0_START + (2 * index * PAGE_SIZE) -> MIPS_COP_0_TLB_HI
+	PTR_LI		t1, MIPS_KSEG0_START		# invalidate tlb entry
+	bltz		s0, entry0set
 	nop
-	sll	s0, PAGE_SHIFT + 1
-	addu	t1, s0
-	MTC0	t1, MIPS_COP_0_TLB_HI
+	sll		s0, PAGE_SHIFT + 1
+	PTR_ADDU	t1, s0
+	MTC0		t1, MIPS_COP_0_TLB_HI
 	PTE_MTC0	zero, MIPS_COP_0_TLB_LO0
 	PTE_MTC0	zero, MIPS_COP_0_TLB_LO1
+	MTC0		zero, MIPS_COP_0_TLB_PG_MASK
 	HAZARD_DELAY
 	tlbwi
 	HAZARD_DELAY
-	MTC0	v0, MIPS_COP_0_TLB_HI		# set VPN again
+#endif  /* KSTACK_LARGE_PAGE */
 
 entry0set:
+	MTC0		v0, MIPS_COP_0_TLB_HI		# set VPN again
+	HAZARD_DELAY
 /* SMP!! - Works only for  unshared TLB case - i.e. no v-cpus */
-	mtc0	zero, MIPS_COP_0_TLB_INDEX		# TLB entry #0
+	mtc0		zero, MIPS_COP_0_TLB_INDEX	# TLB entry #0
 	HAZARD_DELAY
 	PTE_MTC0	a1, MIPS_COP_0_TLB_LO0		# upte[0]
 	HAZARD_DELAY
 	PTE_MTC0	a2, MIPS_COP_0_TLB_LO1		# upte[1]
+#ifdef KSTACK_LARGE_PAGE
+	HAZARD_DELAY
+	li		t1, KSTACK_TLBMASK_MASK
+	MTC0		t1, MIPS_COP_0_TLB_PG_MASK
+#else
+	MTC0		zero, MIPS_COP_0_TLB_PG_MASK
+#endif /* ! KSTACK_LARGE_PAGE */
 	HAZARD_DELAY
 	tlbwi					# set TLB entry #0
 	HAZARD_DELAY
+#ifdef KSTACK_LARGE_PAGE
+	MTC0		zero, MIPS_COP_0_TLB_PG_MASK
+	HAZARD_DELAY
 
 sw2:
+#else /* ! KSTACK_LARGE_PAGE */
+
 #if defined(__mips_n64)
 /*
  * XXXSS
  * A Workaround for the small kernel thread stack.  We use two wired
  * TLB entries to double the stack size to 16K (- sizeof(struct pcb)).
  */
+sw2:
 	PTR_LI		s0, MIPS_XKSEG_START
-	bltu		v1, s0, sw3						# no need to insert in TLB.
+	bltu		v1, s0, sw3			# no need to insert in TLB.
 	PTE_L		a1, TD_UPTE + 2 * PTESIZE(s7)	# a1 = md_upte[2]
 	PTE_L		a2, TD_UPTE + 3 * PTESIZE(s7)	# a2 = md_upte[3]
 
-	MTC0		v1, MIPS_COP_0_TLB_HI			# VPN = va
+	MTC0		v1, MIPS_COP_0_TLB_HI		# VPN = va
 	HAZARD_DELAY
-	tlbp										# probe VPN
+	tlbp						# probe VPN
 	HAZARD_DELAY
 	mfc0		s0, MIPS_COP_0_TLB_INDEX
 	HAZARD_DELAY
 
 	PTR_LI		t1, MIPS_KSEG0_START
 	bltz		s0, entry1set
 	nop
-	sll			s0, PAGE_SHIFT + 1			# invalidate TLB entry
+	sll		s0, PAGE_SHIFT + 1		# invalidate TLB entry
 	addu		t1, s0
 	MTC0		t1, MIPS_COP_0_TLB_HI
 	PTE_MTC0	zero, MIPS_COP_0_TLB_LO0
@@ -411,13 +468,14 @@ entry1set:
 	HAZARD_DELAY
 	PTE_MTC0	a2, MIPS_COP_0_TLB_LO1		# md_upte[3]
 	HAZARD_DELAY
-	tlbwi									# set TLB entry #1
+	tlbwi						# set TLB entry #1
 	HAZARD_DELAY
+sw3:
 #endif /* __mips_n64 */
+#endif /* ! KSTACK_LARGE_PAGE */
 /*
  * Now running on new u struct.
  */
-sw3:
 	PTR_L	s0, TD_PCB(s7)
 	RESTORE_U_PCB_CONTEXT(sp, PCB_REG_SP, s0)
 	PTR_LA	t1, _C_LABEL(pmap_activate)	# s7 = new proc pointer