MFC r337785:

Provide part of the mitigation for L1TF-VMM.

Security:	CVE-2018-3646
Approved by:	so (insta-MFC)

sys/amd64/vmm/intel/vmx.c

@@ -185,6 +185,12 @@ static u_int vpid_alloc_failed;
 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 	    &vpid_alloc_failed, 0, NULL);
 
+static int guest_l1d_flush;
+SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD,
+    &guest_l1d_flush, 0, NULL);
+
+uint64_t vmx_msr_flush_cmd;
+
 /*
  * Use the last page below 4GB as the APIC access address. This address is
  * occupied by the boot firmware so it is guaranteed that it will not conflict
@@ -720,6 +726,12 @@ vmx_init(int ipinum)
 		return (error);
 	}
 
+	guest_l1d_flush = (cpu_ia32_arch_caps & IA32_ARCH_CAP_RDCL_NO) == 0;
+	TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush);
+	if (guest_l1d_flush &&
+	    (cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) != 0)
+		vmx_msr_flush_cmd = IA32_FLUSH_CMD_L1D;
+
 	/*
 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 	 */

sys/amd64/vmm/intel/vmx_genassym.c

@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
+#include <vm/vm_param.h>
 
 #include <machine/vmm.h>
 #include "vmx_cpufunc.h"
@@ -86,3 +87,6 @@ ASSYM(PM_EPTGEN, offsetof(struct pmap, pm_eptgen));
 
 ASSYM(KERNEL_SS, GSEL(GDATA_SEL, SEL_KPL));
 ASSYM(KERNEL_CS, GSEL(GCODE_SEL, SEL_KPL));
+
+ASSYM(PAGE_SIZE, PAGE_SIZE);
+ASSYM(KERNBASE, KERNBASE);

sys/amd64/vmm/intel/vmx_support.S

@@ -30,6 +30,7 @@
  */
 
 #include <machine/asmacros.h>
+#include <machine/specialreg.h>
 
 #include "vmx_assym.h"
 
@@ -175,9 +176,47 @@ ENTRY(vmx_enter_guest)
 	jbe	invept_error		/* Check invept instruction error */
 
 guest_restore:
-	cmpl	$0, %edx
-	je	do_launch
 
+	/*
+	 * Flush L1D cache if requested.  Use IA32_FLUSH_CMD MSR if available,
+	 * otherwise load enough of the data from the zero_region to flush
+	 * existing L1D content.
+	 */
+#define	L1D_FLUSH_SIZE	(64 * 1024)
+	movl	%edx, %r8d
+	cmpb	$0, guest_l1d_flush(%rip)
+	je	after_l1d
+	movq	vmx_msr_flush_cmd(%rip), %rax
+	testq	%rax, %rax
+	jz	1f
+	movq	%rax, %rdx
+	shrq	$32, %rdx
+	movl	$MSR_IA32_FLUSH_CMD, %ecx
+	wrmsr
+	jmp	after_l1d
+1:	movq	$KERNBASE, %r9
+	movq	$-L1D_FLUSH_SIZE, %rcx
+	/*
+	 * pass 1: Preload TLB.
+	 * Kernel text is mapped using superpages.  TLB preload is
+	 * done for the benefit of older CPUs which split 2M page
+	 * into 4k TLB entries.
+	 */
+2:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
+	addq	$PAGE_SIZE, %rcx
+	jne	2b
+	xorl	%eax, %eax
+	cpuid
+	movq	$-L1D_FLUSH_SIZE, %rcx
+	/* pass 2: Read each cache line */
+3:	movb	L1D_FLUSH_SIZE(%r9, %rcx), %al
+	addq	$64, %rcx
+	jne	3b
+	lfence
+#undef	L1D_FLUSH_SIZE
+after_l1d:
+	cmpl	$0, %r8d
+	je	do_launch
 	VMX_GUEST_RESTORE
 	vmresume
 	/*