16413 Post-barrier Return Stack Buffer (PBRSB) fixes can be detected …

…in HW

Reviewed by: Robert Mustacchi <rm@fingolfin.org>
Reviewed by: Richard Lowe <richlowe@richlowe.net>
Approved by: Gordon Ross <Gordon.W.Ross@gmail.com>

usr/src/uts/intel/io/vmm/intel/vmx_support.S

@@ -36,7 +36,7 @@
  *
  * Copyright 2013 Pluribus Networks Inc.
  * Copyright 2018 Joyent, Inc.
- * Copyright 2022 MNX Cloud, Inc.
+ * Copyright 2024 MNX Cloud, Inc.
  */
 
 #include <sys/asm_linkage.h>
@@ -226,7 +226,7 @@ ENTRY_NP(vmx_exit_guest)
 	 * NOTE: If RSB mitigations are disabled (see cpuid.c), this call is
 	 * entirely a NOP.
 	 */
-	call	x86_rsb_stuff
+	call	x86_rsb_stuff_vmexit
 
 	/*
 	 * This will return to the caller of 'vmx_enter_guest()' with a return

usr/src/uts/intel/ml/retpoline.S

@@ -11,6 +11,7 @@
 
 /*
  * Copyright 2019 Joyent, Inc.
+ * Copyright 2024 MNX Cloud, Inc.
  */
 
 	.file	"retpoline.s"
@@ -119,18 +120,23 @@
 	RETPOLINE_MKJUMP(r15)
 
 	/*
-	 * The x86_rsb_stuff function is called from pretty arbitrary
-	 * contexts. It's much easier for us to save and restore all the
-	 * registers we touch rather than clobber them for callers. You must
-	 * preserve this property or the system will panic at best.
-	 */
-	ENTRY(x86_rsb_stuff)
-	/*
-	 * These nops are present so we can patch a ret instruction if we need
-	 * to disable RSB stuffing because enhanced IBRS is present or we're
-	 * disabling mitigations.
+	 * The x86_rsb_stuff{,_vmexit} functions can be called from pretty
+	 * arbitrary contexts. It's much easier for us to save and restore all
+	 * the registers we touch rather than clobber them for callers. You
+	 * must preserve this property or the system will panic at best.
+	 *
+	 * The reason for two entry points is because the need to RSB stuff
+	 * on Intel depends greatly on factors that are different in the
+	 * VMEXIT case, vs. the other context-switching cases
+	 *
+	 * See cpuid.c's cpuid_patch_rsb() for where the two entry points'
+	 * NOPs actually get patched with one-byte RETs as need be, and the
+	 * rules we use to determine which gets disabled with a RET, and which
+	 * maintain their NOP to proceed to executing the stuffing sequence.
 	 */
+	ENTRY_NP(x86_rsb_stuff_vmexit)
 	nop
+	ALTENTRY(x86_rsb_stuff)
 	nop
 	pushq	%rdi
 	pushq	%rax
@@ -154,6 +160,7 @@ rsb_loop:
 	popq	%rdi
 	ret
 	SET_SIZE(x86_rsb_stuff)
+	SET_SIZE(x86_rsb_stuff_vmexit)
 
 #elif defined(__i386)
 

usr/src/uts/intel/os/cpuid.c

@@ -1203,10 +1203,14 @@
  * enhanced IBRS. When eIBRS is present and enabled, then there should be
  * nothing else that we need to do to protect the kernel at this time.
  *
- * Unfortunately, eIBRS or not, we need to manually overwrite the contents of
- * the return stack buffer. We do this through the x86_rsb_stuff() function.
- * Currently this is employed on context switch and vmx_exit. The
- * x86_rsb_stuff() function is disabled only when mitigations in general are.
+ * Unfortunately, not all eIBRS implementations are sufficient to guard
+ * against RSB manipulations, so we still need to manually overwrite the
+ * contents of the return stack buffer unless the hardware specifies we are
+ * covered. We do this through the x86_rsb_stuff() function.  Currently this
+ * is employed on context switch and vmx_exit. The x86_rsb_stuff() function is
+ * disabled only when mitigations in general are, or if we have hardware
+ * indicating no need for post-barrier RSB protections, either in one place
+ * (old hardware), or on both (newer hardware).
  *
  * If SMEP is not present, then we would have to stuff the RSB every time we
  * transitioned from user mode to the kernel, which isn't very practical right
@@ -1675,7 +1679,8 @@ static char *x86_feature_names[NUM_X86_FEATURES] = {
 	"avx512_bf16",
 	"auto_ibrs",
 	"rfds_no",
-	"rfds_clear"
+	"rfds_clear",
+	"pbrsb_no"
 };
 
 boolean_t
@@ -2968,12 +2973,40 @@ cpuid_update_l1d_flush(cpu_t *cpu, uchar_t *featureset)
 }
 
 /*
- * We default to enabling RSB mitigations.
- *
- * NOTE: We used to skip RSB mitigations with eIBRS, but developments around
- * post-barrier RSB guessing suggests we should enable RSB mitigations always
- * unless specifically instructed not to.
- *
+ * We default to enabling Return Stack Buffer (RSB) mitigations.
+ *
+ * We used to skip RSB mitigations with Intel eIBRS, but developments around
+ * post-barrier RSB (PBRSB) guessing suggests we should enable Intel RSB
+ * mitigations always unless explicitly bypassed, or unless hardware indicates
+ * the bug has been fixed.
+ *
+ * The current decisions for using, or ignoring, a RSB software stuffing
+ * sequence are expressed by the following table:
+ *
+ * +-------+------------+-----------------+--------+
+ * | eIBRS |  PBRSB_NO  |  context switch | vmexit |
+ * +-------+------------+-----------------+--------+
+ * |   Yes |     No     |  stuff          | stuff  |
+ * |   Yes |     Yes    |  ignore         | ignore |
+ * |   No  |     No     |  stuff          | ignore |
+ * +-------+------------+-----------------+--------+
+ *
+ * Note that if an Intel CPU has no eIBRS, it will never enumerate PBRSB_NO,
+ * because machines with no eIBRS do not have a problem with PBRSB overflow.
+ * See the Intel document cited below for details.
+ *
+ * Also note that AMD AUTO_IBRS has no PBRSB problem, so it is not included in
+ * the table above, and that there is no situation where vmexit stuffing is
+ * needed, but context-switch stuffing isn't.
+ */
+
+/* BEGIN CSTYLED */
+/*
+ * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/post-barrier-return-stack-buffer-predictions.html
+ */
+/* END CSTYLED */
+
+/*
  * AMD indicates that when Automatic IBRS is enabled we do not need to implement
  * return stack buffer clearing for VMEXIT as it takes care of it. The manual
  * also states that as long as SMEP and we maintain at least one page between
@@ -2982,17 +3015,41 @@ cpuid_update_l1d_flush(cpu_t *cpu, uchar_t *featureset)
  * present.
  */
 static void
-cpuid_patch_rsb(x86_spectrev2_mitigation_t mit)
+cpuid_patch_rsb(x86_spectrev2_mitigation_t mit, bool intel_pbrsb_no)
 {
 	const uint8_t ret = RET_INSTR;
 	uint8_t *stuff = (uint8_t *)x86_rsb_stuff;
+	uint8_t *vmx_stuff = (uint8_t *)x86_rsb_stuff_vmexit;
 
 	switch (mit) {
 	case X86_SPECTREV2_AUTO_IBRS:
 	case X86_SPECTREV2_DISABLED:
+		/* Don't bother with any RSB stuffing! */
 		*stuff = ret;
+		*vmx_stuff = ret;
+		break;
+	case X86_SPECTREV2_RETPOLINE:
+		/*
+		 * The Intel document on Post-Barrier RSB says that processors
+		 * without eIBRS do not have PBRSB problems upon VMEXIT.
+		 */
+		VERIFY(!intel_pbrsb_no);
+		VERIFY3U(*stuff, !=, ret);
+		*vmx_stuff = ret;
 		break;
 	default:
+		/*
+		 * eIBRS is all that's left.  If CPU claims PBRSB is fixed,
+		 * don't use the RSB mitigation in either case.  Otherwise
+		 * both vmexit and context-switching require the software
+		 * mitigation.
+		 */
+		if (intel_pbrsb_no) {
+			/* CPU claims PBRSB problems are fixed. */
+			*stuff = ret;
+			*vmx_stuff = ret;
+		}
+		VERIFY3U(*stuff, ==, *vmx_stuff);
 		break;
 	}
 }
@@ -3267,6 +3324,10 @@ cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
 					add_x86_feature(featureset,
 					    X86FSET_RFDS_CLEAR);
 				}
+				if (reg & IA32_ARCH_CAP_PBRSB_NO) {
+					add_x86_feature(featureset,
+					    X86FSET_PBRSB_NO);
+				}
 			}
 			no_trap();
 		}
@@ -3327,7 +3388,7 @@ cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
 	}
 
 	cpuid_patch_retpolines(v2mit);
-	cpuid_patch_rsb(v2mit);
+	cpuid_patch_rsb(v2mit, is_x86_feature(featureset, X86FSET_PBRSB_NO));
 	x86_spectrev2_mitigation = v2mit;
 	membar_producer();
 

usr/src/uts/intel/sys/x86_archext.h

@@ -934,6 +934,7 @@ extern "C" {
 #define	X86FSET_AUTO_IBRS	108
 #define	X86FSET_RFDS_NO		109
 #define	X86FSET_RFDS_CLEAR	110
+#define	X86FSET_PBRSB_NO	111
 
 /*
  * Intel Deep C-State invariant TSC in leaf 0x80000007.
@@ -1586,7 +1587,7 @@ typedef enum x86_uarchrev {
 
 #if defined(_KERNEL) || defined(_KMEMUSER)
 
-#define	NUM_X86_FEATURES	111
+#define	NUM_X86_FEATURES	112
 extern uchar_t x86_featureset[];
 
 extern void free_x86_featureset(void *featureset);
@@ -1609,6 +1610,7 @@ extern uint_t pentiumpro_bug4046376;
  */
 extern void (*spec_uarch_flush)(void);
 extern void x86_rsb_stuff(void);
+extern void x86_rsb_stuff_vmexit(void);
 extern void x86_md_clear(void);
 
 #endif