diff --git a/usr/src/cmd/mdb/intel/mdb/mdb_amd64util.c b/usr/src/cmd/mdb/intel/mdb/mdb_amd64util.c index 22c3d1dc6a04..14c81f47fd74 100644 --- a/usr/src/cmd/mdb/intel/mdb/mdb_amd64util.c +++ b/usr/src/cmd/mdb/intel/mdb/mdb_amd64util.c @@ -136,6 +136,7 @@ const mdb_tgt_regdesc_t mdb_amd64_kregs[] = { { "gsbase", KREG_GSBASE, MDB_TGT_R_EXPORT }, { "kgsbase", KREG_KGSBASE, MDB_TGT_R_EXPORT }, { "cr2", KREG_CR2, MDB_TGT_R_EXPORT }, + { "cr3", KREG_CR3, MDB_TGT_R_EXPORT }, { NULL, 0, 0 } }; @@ -194,8 +195,9 @@ mdb_amd64_printregs(const mdb_tgt_gregset_t *gregs) kregs[KREG_ES], kregs[KREG_FS] & 0xffff); mdb_printf("%%gs = 0x%04x\t%%gsbase = 0x%lx\t%%kgsbase = 0x%lx\n", kregs[KREG_GS] & 0xffff, kregs[KREG_GSBASE], kregs[KREG_KGSBASE]); - mdb_printf("%%trapno = 0x%x\t%%err = 0x%x\t%%cr2 = 0x%lx\n", - kregs[KREG_TRAPNO], kregs[KREG_ERR], kregs[KREG_CR2]); + mdb_printf("%%trapno = 0x%x\t%%err = 0x%x\t%%cr2 = 0x%lx\t" + "%%cr3 = 0x%lx\n", kregs[KREG_TRAPNO], kregs[KREG_ERR], + kregs[KREG_CR2], kregs[KREG_CR3]); } int diff --git a/usr/src/cmd/mdb/intel/mdb/mdb_kreg.h b/usr/src/cmd/mdb/intel/mdb/mdb_kreg.h index 8bee68b379f2..a3edf864d725 100644 --- a/usr/src/cmd/mdb/intel/mdb/mdb_kreg.h +++ b/usr/src/cmd/mdb/intel/mdb/mdb_kreg.h @@ -80,6 +80,7 @@ typedef uint32_t kreg_t; #define KREG_TRAPNO KDIREG_TRAPNO #define KREG_ERR KDIREG_ERR #define KREG_CR2 KDIREG_CR2 +#define KREG_CR3 KDIREG_CR3 #define KREG_RIP KDIREG_RIP #define KREG_CS KDIREG_CS #define KREG_RFLAGS KDIREG_RFLAGS diff --git a/usr/src/uts/i86pc/ml/kpti_trampolines.s b/usr/src/uts/i86pc/ml/kpti_trampolines.s index d50e964e62ae..7e72841e323c 100644 --- a/usr/src/uts/i86pc/ml/kpti_trampolines.s +++ b/usr/src/uts/i86pc/ml/kpti_trampolines.s @@ -539,6 +539,42 @@ tr_intr_ret_start: iretq SET_SIZE(tr_iret_user) + /* + * This special return trampoline is for KDI's use only (with kmdb). + * + * KDI/kmdb do not use swapgs -- they directly write the GSBASE MSR + * instead. This trampoline runs after GSBASE has already been changed + * back to the userland value (so we can't use %gs). + * + * Instead, the caller gives us a pointer to the kpti_dbg frame in %r13. + * The KPTI_R13 member in the kpti_dbg has already been set to what the + * real %r13 should be before we IRET. + * + * Additionally, KDI keeps a copy of the incoming %cr3 value when it + * took an interrupt, and has put that back in the kpti_dbg area for us + * to use, so we don't do any sniffing of %cs here. This is important + * so that debugging code that changes %cr3 is possible. + */ + ENTRY_NP(tr_iret_kdi) + movq %r14, KPTI_R14(%r13) /* %r14 has to be preserved by us */ + + movq %rsp, %r14 /* original %rsp is pointing at IRET frame */ + leaq KPTI_TOP(%r13), %rsp + pushq T_FRAMERET_SS(%r14) + pushq T_FRAMERET_RSP(%r14) + pushq T_FRAMERET_RFLAGS(%r14) + pushq T_FRAMERET_CS(%r14) + pushq T_FRAMERET_RIP(%r14) + + movq KPTI_TR_CR3(%r13), %r14 + movq %r14, %cr3 + + movq KPTI_R14(%r13), %r14 + movq KPTI_R13(%r13), %r13 /* preserved by our caller */ + + iretq + SET_SIZE(tr_iret_kdi) + .global tr_intr_ret_end tr_intr_ret_end: diff --git a/usr/src/uts/i86pc/ml/offsets.in b/usr/src/uts/i86pc/ml/offsets.in index 0946b369d980..0e3c63450743 100644 --- a/usr/src/uts/i86pc/ml/offsets.in +++ b/usr/src/uts/i86pc/ml/offsets.in @@ -251,6 +251,8 @@ cpu cpu_m.mcpu_pad2 CPU_KPTI_START cpu_m.mcpu_pad3 CPU_KPTI_END + cpu_m.mcpu_kpti_dbg CPU_KPTI_DBG + kpti_frame kf_r14 KPTI_R14 kf_r13 KPTI_R13 diff --git a/usr/src/uts/intel/amd64/sys/kdi_regs.h b/usr/src/uts/intel/amd64/sys/kdi_regs.h index d7c4e8780756..6fe698551fc2 100644 --- a/usr/src/uts/intel/amd64/sys/kdi_regs.h +++ b/usr/src/uts/intel/amd64/sys/kdi_regs.h @@ -58,17 +58,18 @@ extern "C" { #define KDIREG_GSBASE 18 #define KDIREG_KGSBASE 19 #define KDIREG_CR2 20 -#define KDIREG_DS 21 -#define KDIREG_ES 22 -#define KDIREG_FS 23 -#define KDIREG_GS 24 -#define KDIREG_TRAPNO 25 -#define KDIREG_ERR 26 -#define KDIREG_RIP 27 -#define KDIREG_CS 28 -#define KDIREG_RFLAGS 29 -#define KDIREG_RSP 30 -#define KDIREG_SS 31 +#define KDIREG_CR3 21 +#define KDIREG_DS 22 +#define KDIREG_ES 23 +#define KDIREG_FS 24 +#define KDIREG_GS 25 +#define KDIREG_TRAPNO 26 +#define KDIREG_ERR 27 +#define KDIREG_RIP 28 +#define KDIREG_CS 29 +#define KDIREG_RFLAGS 30 +#define KDIREG_RSP 31 +#define KDIREG_SS 32 #define KDIREG_NGREG (KDIREG_SS + 1) diff --git a/usr/src/uts/intel/kdi/kdi_asm.s b/usr/src/uts/intel/kdi/kdi_asm.s index 9e5bbc110f68..5180dbb1b262 100644 --- a/usr/src/uts/intel/kdi/kdi_asm.s +++ b/usr/src/uts/intel/kdi/kdi_asm.s @@ -283,18 +283,28 @@ movl $MSR_AMD_GSBASE, %ecx wrmsr + /* + * In the trampoline we stashed the incoming %cr3. Copy this into + * the kdiregs for restoration and later use. + */ + mov %gs:(CPU_KPTI_DBG+KPTI_TR_CR3), %rdx + mov %rdx, REG_OFF(KDIREG_CR3)(%rsp) /* * Switch to the kernel's %cr3. From the early interrupt handler * until now we've been running on the "paranoid" %cr3 (that of kas * from early in boot). * - * Hopefully it's not corrupt! + * If we took the interrupt from somewhere already on the kas/paranoid + * %cr3 though, don't change it (this could happen if kcr3 is corrupt + * and we took a gptrap earlier from this very code). */ + cmpq %rdx, kpti_safe_cr3 + je .no_kcr3 mov %gs:CPU_KPTI_KCR3, %rdx - cmp $0, %rdx - je .zero_kcr3 + cmpq $0, %rdx + je .no_kcr3 mov %rdx, %cr3 -.zero_kcr3: +.no_kcr3: #endif /* __xpv */ @@ -392,6 +402,9 @@ subq $REG_OFF(KDIREG_TRAPNO), %rsp KDI_SAVE_REGS(%rsp) + movq %cr3, %rax + movq %rax, REG_OFF(KDIREG_CR3)(%rsp) + movq REG_OFF(KDIREG_SS)(%rsp), %rax xchgq REG_OFF(KDIREG_RIP)(%rsp), %rax movq %rax, REG_OFF(KDIREG_SS)(%rsp) @@ -519,6 +532,29 @@ KDI_RESTORE_DEBUGGING_STATE movq KRS_GREGS(%rdi), %rsp + +#if !defined(__xpv) + /* + * If we're going back via tr_iret_kdi, then we want to copy the + * final %cr3 we're going to back into the kpti_dbg area now. + * + * Since the trampoline needs to find the kpti_dbg too, we enter it + * with %r13 set to point at that. The real %r13 (to restore before + * the iret) we stash in the kpti_dbg itself. + */ + movq %gs:CPU_SELF, %r13 /* can't leaq %gs:*, use self-ptr */ + addq $CPU_KPTI_DBG, %r13 + + movq REG_OFF(KDIREG_R13)(%rsp), %rdx + movq %rdx, KPTI_R13(%r13) + + movq REG_OFF(KDIREG_CR3)(%rsp), %rdx + movq %rdx, KPTI_TR_CR3(%r13) + + /* The trampoline will undo this later. */ + movq %r13, REG_OFF(KDIREG_R13)(%rsp) +#endif + KDI_RESTORE_REGS(%rsp) addq $REG_OFF(KDIREG_RIP), %rsp /* Discard state, trapno, err */ /* @@ -526,7 +562,7 @@ * for either kernel or userland. */ #if !defined(__xpv) - jmp tr_iret_auto + jmp tr_iret_kdi #else IRET #endif