9441 kmdb should stash %cr3 in kdiregs

Reviewed by: John Levon <john.levon@joyent.com>
Approved by: Gordon Ross <gwr@nexenta.com>

usr/src/cmd/mdb/intel/mdb/mdb_amd64util.c

@@ -136,6 +136,7 @@ const mdb_tgt_regdesc_t mdb_amd64_kregs[] = {
 	{ "gsbase", KREG_GSBASE, MDB_TGT_R_EXPORT },
 	{ "kgsbase", KREG_KGSBASE, MDB_TGT_R_EXPORT },
 	{ "cr2", KREG_CR2, MDB_TGT_R_EXPORT },
+	{ "cr3", KREG_CR3, MDB_TGT_R_EXPORT },
 	{ NULL, 0, 0 }
 };
 
@@ -194,8 +195,9 @@ mdb_amd64_printregs(const mdb_tgt_gregset_t *gregs)
 	    kregs[KREG_ES], kregs[KREG_FS] & 0xffff);
 	mdb_printf("%%gs = 0x%04x\t%%gsbase = 0x%lx\t%%kgsbase = 0x%lx\n",
 	    kregs[KREG_GS] & 0xffff, kregs[KREG_GSBASE], kregs[KREG_KGSBASE]);
-	mdb_printf("%%trapno = 0x%x\t%%err = 0x%x\t%%cr2 = 0x%lx\n",
-	    kregs[KREG_TRAPNO], kregs[KREG_ERR], kregs[KREG_CR2]);
+	mdb_printf("%%trapno = 0x%x\t%%err = 0x%x\t%%cr2 = 0x%lx\t"
+	    "%%cr3 = 0x%lx\n", kregs[KREG_TRAPNO], kregs[KREG_ERR],
+	    kregs[KREG_CR2], kregs[KREG_CR3]);
 }
 
 int

usr/src/cmd/mdb/intel/mdb/mdb_kreg.h

@@ -80,6 +80,7 @@ typedef uint32_t kreg_t;
 #define	KREG_TRAPNO	KDIREG_TRAPNO
 #define	KREG_ERR	KDIREG_ERR
 #define	KREG_CR2	KDIREG_CR2
+#define	KREG_CR3	KDIREG_CR3
 #define	KREG_RIP	KDIREG_RIP
 #define	KREG_CS		KDIREG_CS
 #define	KREG_RFLAGS	KDIREG_RFLAGS

usr/src/uts/i86pc/ml/kpti_trampolines.s

@@ -539,6 +539,42 @@ tr_intr_ret_start:
 	iretq
 	SET_SIZE(tr_iret_user)
 
+	/*
+	 * This special return trampoline is for KDI's use only (with kmdb).
+	 *
+	 * KDI/kmdb do not use swapgs -- they directly write the GSBASE MSR
+	 * instead. This trampoline runs after GSBASE has already been changed
+	 * back to the userland value (so we can't use %gs).
+	 *
+	 * Instead, the caller gives us a pointer to the kpti_dbg frame in %r13.
+	 * The KPTI_R13 member in the kpti_dbg has already been set to what the
+	 * real %r13 should be before we IRET.
+	 *
+	 * Additionally, KDI keeps a copy of the incoming %cr3 value when it
+	 * took an interrupt, and has put that back in the kpti_dbg area for us
+	 * to use, so we don't do any sniffing of %cs here. This is important
+	 * so that debugging code that changes %cr3 is possible.
+	 */
+	ENTRY_NP(tr_iret_kdi)
+	movq	%r14, KPTI_R14(%r13)	/* %r14 has to be preserved by us */
+
+	movq	%rsp, %r14	/* original %rsp is pointing at IRET frame */
+	leaq	KPTI_TOP(%r13), %rsp
+	pushq	T_FRAMERET_SS(%r14)
+	pushq	T_FRAMERET_RSP(%r14)
+	pushq	T_FRAMERET_RFLAGS(%r14)
+	pushq	T_FRAMERET_CS(%r14)
+	pushq	T_FRAMERET_RIP(%r14)
+
+	movq	KPTI_TR_CR3(%r13), %r14
+	movq	%r14, %cr3
+
+	movq	KPTI_R14(%r13), %r14
+	movq	KPTI_R13(%r13), %r13	/* preserved by our caller */
+
+	iretq
+	SET_SIZE(tr_iret_kdi)
+
 .global	tr_intr_ret_end
 tr_intr_ret_end:
 

usr/src/uts/i86pc/ml/offsets.in

@@ -251,6 +251,8 @@ cpu
 	cpu_m.mcpu_pad2			CPU_KPTI_START
 	cpu_m.mcpu_pad3			CPU_KPTI_END
 
+	cpu_m.mcpu_kpti_dbg		CPU_KPTI_DBG
+
 kpti_frame
 	kf_r14		KPTI_R14
 	kf_r13		KPTI_R13

usr/src/uts/intel/amd64/sys/kdi_regs.h

@@ -58,17 +58,18 @@ extern "C" {
 #define	KDIREG_GSBASE	18
 #define	KDIREG_KGSBASE	19
 #define	KDIREG_CR2	20
-#define	KDIREG_DS	21
-#define	KDIREG_ES	22
-#define	KDIREG_FS	23
-#define	KDIREG_GS	24
-#define	KDIREG_TRAPNO	25
-#define	KDIREG_ERR	26
-#define	KDIREG_RIP	27
-#define	KDIREG_CS	28
-#define	KDIREG_RFLAGS	29
-#define	KDIREG_RSP	30
-#define	KDIREG_SS	31
+#define	KDIREG_CR3	21
+#define	KDIREG_DS	22
+#define	KDIREG_ES	23
+#define	KDIREG_FS	24
+#define	KDIREG_GS	25
+#define	KDIREG_TRAPNO	26
+#define	KDIREG_ERR	27
+#define	KDIREG_RIP	28
+#define	KDIREG_CS	29
+#define	KDIREG_RFLAGS	30
+#define	KDIREG_RSP	31
+#define	KDIREG_SS	32
 
 #define	KDIREG_NGREG	(KDIREG_SS + 1)
 

usr/src/uts/intel/kdi/kdi_asm.s

@@ -283,18 +283,28 @@
 	movl	$MSR_AMD_GSBASE, %ecx
 	wrmsr
 
+	/*
+	 * In the trampoline we stashed the incoming %cr3. Copy this into
+	 * the kdiregs for restoration and later use.
+	 */
+	mov	%gs:(CPU_KPTI_DBG+KPTI_TR_CR3), %rdx
+	mov	%rdx, REG_OFF(KDIREG_CR3)(%rsp)
 	/*
 	 * Switch to the kernel's %cr3. From the early interrupt handler
 	 * until now we've been running on the "paranoid" %cr3 (that of kas
 	 * from early in boot).
 	 *
-	 * Hopefully it's not corrupt!
+	 * If we took the interrupt from somewhere already on the kas/paranoid
+	 * %cr3 though, don't change it (this could happen if kcr3 is corrupt
+	 * and we took a gptrap earlier from this very code).
 	 */
+	cmpq	%rdx, kpti_safe_cr3
+	je	.no_kcr3
 	mov	%gs:CPU_KPTI_KCR3, %rdx
-	cmp	$0, %rdx
-	je	.zero_kcr3
+	cmpq	$0, %rdx
+	je	.no_kcr3
 	mov	%rdx, %cr3
-.zero_kcr3:
+.no_kcr3:
 
 #endif	/* __xpv */
 
@@ -392,6 +402,9 @@
 	subq	$REG_OFF(KDIREG_TRAPNO), %rsp
 	KDI_SAVE_REGS(%rsp)
 
+	movq	%cr3, %rax
+	movq	%rax, REG_OFF(KDIREG_CR3)(%rsp)
+
 	movq	REG_OFF(KDIREG_SS)(%rsp), %rax
 	xchgq	REG_OFF(KDIREG_RIP)(%rsp), %rax
 	movq	%rax, REG_OFF(KDIREG_SS)(%rsp)
@@ -519,14 +532,37 @@
 	KDI_RESTORE_DEBUGGING_STATE
 
 	movq	KRS_GREGS(%rdi), %rsp
+
+#if !defined(__xpv)
+	/*
+	 * If we're going back via tr_iret_kdi, then we want to copy the
+	 * final %cr3 we're going to back into the kpti_dbg area now.
+	 *
+	 * Since the trampoline needs to find the kpti_dbg too, we enter it
+	 * with %r13 set to point at that. The real %r13 (to restore before
+	 * the iret) we stash in the kpti_dbg itself.
+	 */
+	movq	%gs:CPU_SELF, %r13	/* can't leaq %gs:*, use self-ptr */
+	addq	$CPU_KPTI_DBG, %r13
+
+	movq	REG_OFF(KDIREG_R13)(%rsp), %rdx
+	movq	%rdx, KPTI_R13(%r13)
+
+	movq	REG_OFF(KDIREG_CR3)(%rsp), %rdx
+	movq	%rdx, KPTI_TR_CR3(%r13)
+
+	/* The trampoline will undo this later. */
+	movq	%r13, REG_OFF(KDIREG_R13)(%rsp)
+#endif
+
 	KDI_RESTORE_REGS(%rsp)
 	addq	$REG_OFF(KDIREG_RIP), %rsp	/* Discard state, trapno, err */
 	/*
 	 * The common trampoline code will restore %cr3 to the right value
 	 * for either kernel or userland.
 	 */
 #if !defined(__xpv)
-	jmp	tr_iret_auto
+	jmp	tr_iret_kdi
 #else
 	IRET
 #endif