OS-7000 Need Eager FPU

Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Patrick Mooney <patrick.mooney@joyent.com>
Reviewed by: Bryan Cantrill <bryan@joyent.com>
Reviewed by: John Levon <john.levon@joyent.com>
Approved by: Bryan Cantrill <bryan@joyent.com>

usr/src/cmd/mdb/intel/modules/genunix/gcore_isadep.c

@@ -10,6 +10,7 @@
  */
 /*
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2018, Joyent, Inc.
  */
 
 #include <mdb/mdb_modapi.h>
@@ -71,7 +72,7 @@ gcore_getgregs(mdb_klwp_t *lwp, gregset_t grp)
 	grp[REG_R15] = rp->r_r15;
 	grp[REG_FSBASE] = pcb->pcb_fsbase;
 	grp[REG_GSBASE] = pcb->pcb_gsbase;
-	if (pcb->pcb_rupdate == 1) {
+	if (PCB_NEED_UPDATE_SEGS(pcb)) {
 		grp[REG_DS] = pcb->pcb_ds;
 		grp[REG_ES] = pcb->pcb_es;
 		grp[REG_FS] = pcb->pcb_fs;

usr/src/uts/common/brand/lx/syscall/lx_thread_area.c

@@ -21,7 +21,7 @@
 /*
  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
- * Copyright 2016 Joyent, Inc.
+ * Copyright 2018 Joyent, Inc.
  */
 
 #include <sys/types.h>
@@ -65,7 +65,7 @@ lx_arch_prctl(int code, ulong_t addr)
 			/*
 			 * Ensure we go out via update_sregs.
 			 */
-			pcb->pcb_rupdate = 1;
+			PCB_SET_UPDATE_SEGS(pcb);
 		}
 		kpreempt_enable();
 		break;
@@ -87,7 +87,7 @@ lx_arch_prctl(int code, ulong_t addr)
 			/*
 			 * Ensure we go out via update_sregs.
 			 */
-			pcb->pcb_rupdate = 1;
+			PCB_SET_UPDATE_SEGS(pcb);
 		}
 		kpreempt_enable();
 		break;

usr/src/uts/common/brand/solaris10/s10_brand.c

@@ -22,7 +22,7 @@
 /*
  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2017, Joyent, Inc.
+ * Copyright 2018, Joyent, Inc.
  */
 
 #include <sys/errno.h>
@@ -229,7 +229,7 @@ s10_amd64_correct_fsreg(klwp_t *l)
 	if (lwp_getdatamodel(l) == DATAMODEL_NATIVE) {
 		kpreempt_disable();
 		l->lwp_pcb.pcb_fs = LWPFS_SEL;
-		l->lwp_pcb.pcb_rupdate = 1;
+		PCB_SET_UPDATE_SEGS(&l->lwp_pcb);
 		lwptot(l)->t_post_sys = 1;	/* Guarantee update_sregs() */
 		kpreempt_enable();
 	}

usr/src/uts/i86pc/io/vmm/vmm.c

@@ -1198,7 +1198,14 @@ save_guest_fpustate(struct vcpu *vcpu)
 	/* save guest FPU state */
 	fpu_stop_emulating();
 	fpusave(vcpu->guestfpu);
+#ifdef __FreeBSD__
 	fpu_start_emulating();
+#else
+	/*
+	 * When the host state has been restored, we should not re-enable
+	 * CR0.TS on illumos for eager FPU.
+	 */
+#endif
 }
 
 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
@@ -1812,7 +1819,7 @@ vm_run(struct vm *vm, struct vm_run *vmrun)
 	set_pcb_flags(pcb, PCB_FULL_IRET);
 #else
 	/* Force a trip through update_sregs to reload %fs/%gs and friends */
-	ttolwp(curthread)->lwp_pcb.pcb_rupdate = 1;
+	PCB_SET_UPDATE_SEGS(&ttolwp(curthread)->lwp_pcb);
 #endif
 
 #ifndef	__FreeBSD__

usr/src/uts/i86pc/ml/kpti_trampolines.s

@@ -137,6 +137,12 @@
 	DGDEF3(kpti_enable, 8, 8)
 	.fill	1, 8, 1
 
+#if DEBUG
+	.data
+_bad_ts_panic_msg:
+	.string "kpti_trampolines.s: tr_iret_user but CR0.TS set"
+#endif
+
 .section ".text";
 .align MMU_PAGESIZE
 
@@ -523,6 +529,28 @@ tr_intr_ret_start:
 	SET_SIZE(tr_iret_kernel)
 
 	ENTRY_NP(tr_iret_user)
+#if DEBUG
+	/*
+	 * Ensure that we return to user land with CR0.TS clear. We do this
+	 * before we trampoline back and pivot the stack and %cr3. This way
+	 * we're still on the kernel stack and kernel %cr3, though we are on the
+	 * user GSBASE.
+	 */
+	pushq	%rax
+	mov	%cr0, %rax
+	testq	$CR0_TS, %rax
+	jz	1f
+	swapgs
+	popq	%rax
+	leaq	_bad_ts_panic_msg(%rip), %rdi
+	xorl	%eax, %eax
+	pushq	%rbp
+	movq	%rsp, %rbp
+	call	panic
+1:
+	popq	%rax
+#endif
+
 	cmpq	$1, kpti_enable
 	jne	1f
 

usr/src/uts/i86pc/ml/locore.s

@@ -239,6 +239,11 @@ __return_from_main:
 __unsupported_cpu:
 	.string	"486 style cpu detected - no longer supported!"
 
+#if defined(DEBUG)
+_no_pending_updates:
+	.string	"locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
+#endif
+
 #endif	/* !__lint */
 
 #if !defined(__amd64)
@@ -1505,8 +1510,6 @@ _sys_rtt(void)
 
 #else	/* __lint */
 
-#if defined(__amd64)
-
 	ENTRY_NP(lwp_rtt_initial)
 	movq	%gs:CPU_THREAD, %r15
 	movq	T_STACK(%r15), %rsp	/* switch to the thread stack */
@@ -1549,8 +1552,6 @@ _lwp_rtt:
 	movq	%r14, %rdx
 	xorl	%eax, %eax
 	call	panic
-_no_pending_updates:
-	.string	"locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
 1:
 #endif
 
@@ -1570,11 +1571,6 @@ _no_pending_updates:
 	movq	REGOFF_RAX(%rsp), %rdi
 	call	post_syscall		/* post_syscall(rval1, rval2) */
 
-	/*
-	 * set up to take fault on first use of fp
-	 */
-	STTS(%rdi)
-
 	/*
 	 * XXX - may want a fast path that avoids sys_rtt_common in the
 	 * most common case.
@@ -1636,99 +1632,6 @@ _sys_rtt_end:
 	SET_SIZE(sys_rtt_syscall)
 	SET_SIZE(sys_rtt_syscall32)
 
-#elif defined(__i386)
-
-	ENTRY_NP(lwp_rtt_initial)
-	movl	%gs:CPU_THREAD, %eax
-	movl	T_STACK(%eax), %esp	/* switch to the thread stack */
-	movl	%esp, %ebp
-	call	__dtrace_probe___proc_start
-	jmp	_lwp_rtt
-
-	ENTRY_NP(lwp_rtt)
-	movl	%gs:CPU_THREAD, %eax
-	movl	T_STACK(%eax), %esp	/* switch to the thread stack */
-	movl	%esp, %ebp
-_lwp_rtt:
-	call	__dtrace_probe___proc_lwp__start
-
-        /*
-         * If agent lwp, clear %fs and %gs.
-         */
-        movl    %gs:CPU_LWP, %eax
-        movl    LWP_PROCP(%eax), %edx
-
-        cmpl    %eax, P_AGENTTP(%edx)
-        jne     1f
-        movl    $0, REGOFF_FS(%esp)
-        movl    $0, REGOFF_GS(%esp)
-1:
-	call	dtrace_systrace_rtt
-	movl	REGOFF_EDX(%esp), %edx
-	movl	REGOFF_EAX(%esp), %eax
-	pushl	%edx
-	pushl	%eax
-	call	post_syscall		/* post_syscall(rval1, rval2) */
-	addl	$8, %esp
-
-	/*
-	 * set up to take fault on first use of fp
-	 */
-	STTS(%eax)
-
-	/*
-	 * XXX - may want a fast path that avoids sys_rtt_common in the
-	 * most common case.
-	 */
-	ALTENTRY(_sys_rtt)
-	CLI(%eax)			/* disable interrupts */
-	ALTENTRY(_sys_rtt_ints_disabled)
-	pushl	%esp			/* pass rp to sys_rtt_common */
-	call	sys_rtt_common
-	addl	$4, %esp		/* pop arg */
-	testl	%eax, %eax		/* test for return to user mode */
-	jz	sr_sup
-
-	/*
-	 * Return to User.
-	 */
-	ALTENTRY(sys_rtt_syscall)
-	INTR_POP_USER
-
-	/*
-	 * There can be no instructions between this label and IRET or
-	 * we could end up breaking linux brand support. See label usage
-	 * in lx_brand_int80_callback for an example.
-	 */
-	ALTENTRY(nopop_sys_rtt_syscall)
-	IRET
-	/*NOTREACHED*/
-	SET_SIZE(nopop_sys_rtt_syscall)
-
-	ALTENTRY(_sys_rtt_end)
-
-	/*
-	 * Return to supervisor
-	 */
-	ALTENTRY(sr_sup)
-
-	/*
-	 * Restore regs before doing iret to kernel mode
-	 */
-	INTR_POP_KERNEL
-	IRET
-	/*NOTREACHED*/
-
-	SET_SIZE(sr_sup)
-	SET_SIZE(_sys_rtt_end)
-	SET_SIZE(lwp_rtt)
-	SET_SIZE(lwp_rtt_initial)
-	SET_SIZE(_sys_rtt_ints_disabled)
-	SET_SIZE(_sys_rtt)
-	SET_SIZE(sys_rtt_syscall)
-
-#endif	/* __i386 */
-
 #endif	/* __lint */
 
 #if defined(__lint)

usr/src/uts/i86pc/ml/syscall_asm_amd64.s

@@ -271,7 +271,18 @@
  * between entering privileged mode and performing the assertion,
  * otherwise we may perform a context switch on the thread, which
  * will end up setting pcb_rupdate to 1 again.
+ *
+ * ASSERT(%cr0 & CR0_TS == 0);
+ * Preconditions:
+ *	(%rsp is ready for normal call sequence)
+ * Postconditions (if assertion is true):
+ *      (specified register is clobbered)
+ *
+ * Check to make sure that we are returning to user land and that CR0.TS
+ * is not set. This is required as part of the eager FPU (see
+ * uts/intel/ia32/os/fpu.c for more information).
  */
+
 #if defined(DEBUG)
 
 #if !defined(__lint)
@@ -285,6 +296,9 @@ __codesel_msg:
 __no_rupdate_msg:
 	.string	"syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"
 
+__bad_ts_msg:
+	.string "sysscall_asm_amd64.s:%d CR0.TS set on user return"
+
 #endif	/* !__lint */
 
 #define	ASSERT_LWPTOREGS(lwp, rp)			\
@@ -310,9 +324,20 @@ __no_rupdate_msg:
 	call	panic;					\
 8:
 
+#define	ASSERT_CR0TS_ZERO(reg)				\
+	movq	%cr0, reg;				\
+	testq	$CR0_TS, reg;				\
+	jz	9f;					\
+	leaq	__bad_ts_msg(%rip), %rdi;		\
+	movl	$__LINE__, %esi;			\
+	xorl	%eax, %eax;				\
+	call	panic;					\
+9:
+
 #else
 #define	ASSERT_LWPTOREGS(lwp, rp)
 #define	ASSERT_NO_RUPDATE_PENDING(lwp)
+#define	ASSERT_CR0TS_ZERO(reg)
 #endif
 
 /*
@@ -647,6 +672,11 @@ _syscall_after_brand:
 	movq	%r12, REGOFF_RAX(%rsp)
 	movq	%r13, REGOFF_RDX(%rsp)
 
+	/*
+	 * Clobber %r11 as we check CR0.TS.
+	 */
+	ASSERT_CR0TS_ZERO(%r11)
+
 	/*
 	 * To get back to userland, we need the return %rip in %rcx and
 	 * the return %rfl in %r11d.  The sysretq instruction also arranges
@@ -971,6 +1001,11 @@ _syscall32_after_brand:
 	jne	_full_syscall_postsys32
 	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)
 
+	/*
+	 * Clobber %r11 as we check CR0.TS.
+	 */
+	ASSERT_CR0TS_ZERO(%r11)
+
 	/*
 	 * To get back to userland, we need to put the return %rip in %rcx and
 	 * the return %rfl in %r11d.  The sysret instruction also arranges
@@ -1262,6 +1297,11 @@ sys_sysenter()
 	 */
 	andq	$_BITNOT(PS_IE), REGOFF_RFL(%rsp)
 
+	/*
+	 * Clobber %r11 as we check CR0.TS.
+	 */
+	ASSERT_CR0TS_ZERO(%r11)
+
 	/*
 	 * (There's no point in loading up %edx because the sysexit
 	 * mechanism smashes it.)

usr/src/uts/i86pc/os/fpu_subr.c

@@ -148,8 +148,7 @@ fpu_probe(void)
 		ENABLE_SSE();
 
 		if (is_x86_feature(x86_featureset, X86FSET_AVX)) {
-			ASSERT(is_x86_feature(x86_featureset,
-			    X86FSET_XSAVE));
+			ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
 			fp_kind |= __FP_AVX;
 		}
 
@@ -180,7 +179,7 @@ fpu_probe(void)
 					fpsave_ctxt = xsave_ctxt;
 				}
 			}
-			patch_xsave();
+			fprestore_ctxt = xrestore_ctxt;
 			fpsave_cachep = kmem_cache_create("xsave_cache",
 			    cpuid_get_xsave_size(), XSAVE_ALIGN,
 			    NULL, NULL, NULL, NULL, NULL, 0);