diff --git a/sys/arch/amd64/amd64/amd64_trap.S b/sys/arch/amd64/amd64/amd64_trap.S index 326f9bc5a9b7b..0476ed8cabd8b 100644 --- a/sys/arch/amd64/amd64/amd64_trap.S +++ b/sys/arch/amd64/amd64/amd64_trap.S @@ -1,4 +1,4 @@ -/* $NetBSD: amd64_trap.S,v 1.46 2019/02/11 14:59:32 cherry Exp $ */ +/* $NetBSD: amd64_trap.S,v 1.47 2019/05/14 16:59:25 maxv Exp $ */ /* * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc. @@ -281,6 +281,7 @@ IDTVEC(trap02) call _C_LABEL(nmitrap) .Lnmileave: + MDS_LEAVE SVS_LEAVE_NMI IBRS_LEAVE INTR_RESTORE_GPRS @@ -369,6 +370,7 @@ IDTVEC(trap08) incq CPUVAR(NTRAP) call _C_LABEL(doubletrap) + MDS_LEAVE SVS_LEAVE_ALTSTACK IBRS_LEAVE INTR_RESTORE_GPRS diff --git a/sys/arch/amd64/amd64/locore.S b/sys/arch/amd64/amd64/locore.S index 969d7cf68256d..9e048ec0ba727 100644 --- a/sys/arch/amd64/amd64/locore.S +++ b/sys/arch/amd64/amd64/locore.S @@ -1,4 +1,4 @@ -/* $NetBSD: locore.S,v 1.180 2019/03/09 08:42:25 maxv Exp $ */ +/* $NetBSD: locore.S,v 1.181 2019/05/14 16:59:25 maxv Exp $ */ /* * Copyright-o-rama! @@ -1483,6 +1483,7 @@ IDTVEC_END(osyscall) TEXT_USER_BEGIN _ALIGN_TEXT LABEL(syscall_sysret) + MDS_LEAVE SVS_LEAVE IBRS_LEAVE INTR_RESTORE_GPRS @@ -1574,6 +1575,7 @@ END(pagezero) _ALIGN_TEXT LABEL(intrfastexit) NOT_XEN(cli;) + MDS_LEAVE SVS_LEAVE IBRS_LEAVE INTR_RESTORE_GPRS @@ -1721,3 +1723,18 @@ LABEL(noibrs_enter_end) LABEL(noibrs_leave) NOIBRS_LEAVE LABEL(noibrs_leave_end) + + .globl mds_leave, mds_leave_end + +LABEL(mds_leave) + testb $SEL_UPL,TF_CS(%rsp) + jz 1234f + pushq $GSEL(GDATA_SEL, SEL_KPL) + verw (%rsp) + addq $8,%rsp +1234: +LABEL(mds_leave_end) + +LABEL(nomds_leave) + NOMDS_LEAVE +LABEL(nomds_leave_end) diff --git a/sys/arch/amd64/include/frameasm.h b/sys/arch/amd64/include/frameasm.h index 52029c1730c08..6fab07e3bd37d 100644 --- a/sys/arch/amd64/include/frameasm.h +++ b/sys/arch/amd64/include/frameasm.h @@ -1,4 +1,4 @@ -/* $NetBSD: frameasm.h,v 1.42 2019/02/11 14:59:32 cherry Exp $ */ +/* $NetBSD: frameasm.h,v 1.43 2019/05/14 16:59:25 maxv Exp $ */ #ifndef _AMD64_MACHINE_FRAMEASM_H #define _AMD64_MACHINE_FRAMEASM_H @@ -48,6 +48,7 @@ #define HP_NAME_IBRS_LEAVE 10 #define HP_NAME_SVS_ENTER_NMI 11 #define HP_NAME_SVS_LEAVE_NMI 12 +#define HP_NAME_MDS_LEAVE 13 #define HOTPATCH(name, size) \ 123: ; \ @@ -85,6 +86,18 @@ .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \ .fill (IBRS_LEAVE_BYTES-2),1,0xCC +/* + * MDS + */ + +#define MDS_LEAVE_BYTES 20 +#define MDS_LEAVE \ + HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \ + NOMDS_LEAVE +#define NOMDS_LEAVE \ + .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \ + .fill (MDS_LEAVE_BYTES-2),1,0xCC + #define SWAPGS NOT_XEN(swapgs) /* diff --git a/sys/arch/x86/include/specialreg.h b/sys/arch/x86/include/specialreg.h index 5200bc97500c7..437f30a6cecdd 100644 --- a/sys/arch/x86/include/specialreg.h +++ b/sys/arch/x86/include/specialreg.h @@ -1,4 +1,4 @@ -/* $NetBSD: specialreg.h,v 1.143 2019/03/13 05:22:07 msaitoh Exp $ */ +/* $NetBSD: specialreg.h,v 1.144 2019/05/14 16:59:26 maxv Exp $ */ /*- * Copyright (c) 1991 The Regents of the University of California. @@ -426,6 +426,7 @@ /* %edx */ #define CPUID_SEF_AVX512_4VNNIW __BIT(2) #define CPUID_SEF_AVX512_4FMAPS __BIT(3) +#define CPUID_SEF_MD_CLEAR __BIT(10) #define CPUID_SEF_TSX_FORCE_ABORT __BIT(13) /* MSR_TSX_FORCE_ABORT bit 0 */ #define CPUID_SEF_IBRS __BIT(26) /* IBRS / IBPB Speculation Control */ #define CPUID_SEF_STIBP __BIT(27) /* STIBP Speculation Control */ @@ -747,6 +748,7 @@ #define IA32_ARCH_RSBA 0x04 #define IA32_ARCH_SKIP_L1DFL_VMENTRY 0x08 #define IA32_ARCH_SSB_NO 0x10 +#define IA32_ARCH_MDS_NO 0x20 #define MSR_IA32_FLUSH_CMD 0x10b #define IA32_FLUSH_CMD_L1D_FLUSH 0x01 #define MSR_TSX_FORCE_ABORT 0x10f diff --git a/sys/arch/x86/x86/spectre.c b/sys/arch/x86/x86/spectre.c index cc1a26876a5c2..cd976bc1f4814 100644 --- a/sys/arch/x86/x86/spectre.c +++ b/sys/arch/x86/x86/spectre.c @@ -1,4 +1,4 @@ -/* $NetBSD: spectre.c,v 1.26 2019/04/27 10:40:17 maxv Exp $ */ +/* $NetBSD: spectre.c,v 1.27 2019/05/14 16:59:26 maxv Exp $ */ /* * Copyright (c) 2018 NetBSD Foundation, Inc. @@ -34,7 +34,7 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.26 2019/04/27 10:40:17 maxv Exp $"); +__KERNEL_RCSID(0, "$NetBSD: spectre.c,v 1.27 2019/05/14 16:59:26 maxv Exp $"); #include "opt_spectre.h" @@ -549,6 +549,226 @@ sysctl_machdep_spectreV4_mitigated(SYSCTLFN_ARGS) /* -------------------------------------------------------------------------- */ +enum mds_mitigation { + MDS_MITIGATION_NONE, + MDS_MITIGATION_VERW, + MDS_MITIGATION_MDS_NO +}; + +static char mds_mitigation_name[64] = "(none)"; + +static enum mds_mitigation mds_mitigation_method = MDS_MITIGATION_NONE; +static bool mds_mitigation_enabled __read_mostly = false; + +static volatile unsigned long mds_cpu_barrier1 __cacheline_aligned; +static volatile unsigned long mds_cpu_barrier2 __cacheline_aligned; + +#ifdef __x86_64__ +static void +mds_disable_hotpatch(void) +{ + extern uint8_t nomds_leave, nomds_leave_end; + u_long psl, cr0; + uint8_t *bytes; + size_t size; + + x86_patch_window_open(&psl, &cr0); + + bytes = &nomds_leave; + size = (size_t)&nomds_leave_end - (size_t)&nomds_leave; + x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size); + + x86_patch_window_close(psl, cr0); +} + +static void +mds_enable_hotpatch(void) +{ + extern uint8_t mds_leave, mds_leave_end; + u_long psl, cr0; + uint8_t *bytes; + size_t size; + + x86_patch_window_open(&psl, &cr0); + + bytes = &mds_leave; + size = (size_t)&mds_leave_end - (size_t)&mds_leave; + x86_hotpatch(HP_NAME_MDS_LEAVE, bytes, size); + + x86_patch_window_close(psl, cr0); +} +#else +/* MDS not supported on i386 */ +static void +mds_disable_hotpatch(void) +{ + panic("%s: impossible", __func__); +} +static void +mds_enable_hotpatch(void) +{ + panic("%s: impossible", __func__); +} +#endif + +static void +mitigation_mds_apply_cpu(struct cpu_info *ci, bool enabled) +{ + switch (mds_mitigation_method) { + case MDS_MITIGATION_NONE: + case MDS_MITIGATION_MDS_NO: + panic("impossible"); + case MDS_MITIGATION_VERW: + /* cpu0 is the one that does the hotpatch job */ + if (ci == &cpu_info_primary) { + if (enabled) { + mds_enable_hotpatch(); + } else { + mds_disable_hotpatch(); + } + } + break; + } +} + +static void +mitigation_mds_change_cpu(void *arg1, void *arg2) +{ + struct cpu_info *ci = curcpu(); + bool enabled = (bool)arg1; + u_long psl = 0; + + /* Rendez-vous 1. */ + psl = x86_read_psl(); + x86_disable_intr(); + + atomic_dec_ulong(&mds_cpu_barrier1); + while (atomic_cas_ulong(&mds_cpu_barrier1, 0, 0) != 0) { + x86_pause(); + } + + mitigation_mds_apply_cpu(ci, enabled); + + /* Rendez-vous 2. */ + atomic_dec_ulong(&mds_cpu_barrier2); + while (atomic_cas_ulong(&mds_cpu_barrier2, 0, 0) != 0) { + x86_pause(); + } + + /* Write back and invalidate cache, flush pipelines. */ + wbinvd(); + x86_flush(); + + x86_write_psl(psl); +} + +static void +mds_detect_method(void) +{ + u_int descs[4]; + uint64_t msr; + + if (cpu_vendor != CPUVENDOR_INTEL) { + mds_mitigation_method = MDS_MITIGATION_MDS_NO; + return; + } + + x86_cpuid(0x7, descs); + if (descs[3] & CPUID_SEF_ARCH_CAP) { + msr = rdmsr(MSR_IA32_ARCH_CAPABILITIES); + if (msr & IA32_ARCH_MDS_NO) { + mds_mitigation_method = MDS_MITIGATION_MDS_NO; + return; + } + } + +#ifdef __x86_64__ + if (descs[3] & CPUID_SEF_MD_CLEAR) { + mds_mitigation_method = MDS_MITIGATION_VERW; + } +#endif +} + +static void +mds_set_name(void) +{ + char name[64] = ""; + + if (!mds_mitigation_enabled) { + strlcat(name, "(none)", sizeof(name)); + } else { + switch (mds_mitigation_method) { + case MDS_MITIGATION_NONE: + panic("%s: impossible", __func__); + case MDS_MITIGATION_MDS_NO: + strlcat(name, "[MDS_NO]", sizeof(name)); + break; + case MDS_MITIGATION_VERW: + strlcat(name, "[VERW]", sizeof(name)); + break; + } + } + + strlcpy(mds_mitigation_name, name, + sizeof(mds_mitigation_name)); +} + +static int +mitigation_mds_change(bool enabled) +{ + uint64_t xc; + + mds_detect_method(); + + switch (mds_mitigation_method) { + case MDS_MITIGATION_NONE: + printf("[!] No mitigation available\n"); + return EOPNOTSUPP; + case MDS_MITIGATION_VERW: + /* Initialize the barriers */ + mds_cpu_barrier1 = ncpu; + mds_cpu_barrier2 = ncpu; + + printf("[+] %s MDS Mitigation...", + enabled ? "Enabling" : "Disabling"); + xc = xc_broadcast(XC_HIGHPRI, mitigation_mds_change_cpu, + (void *)enabled, NULL); + xc_wait(xc); + printf(" done!\n"); + mds_mitigation_enabled = enabled; + mds_set_name(); + return 0; + case MDS_MITIGATION_MDS_NO: + printf("[+] The CPU is not affected by MDS\n"); + return 0; + default: + panic("impossible"); + } +} + +static int +sysctl_machdep_mds_mitigated(SYSCTLFN_ARGS) +{ + struct sysctlnode node; + int error; + bool val; + + val = *(bool *)rnode->sysctl_data; + + node = *rnode; + node.sysctl_data = &val; + + error = sysctl_lookup(SYSCTLFN_CALL(&node)); + if (error != 0 || newp == NULL) + return error; + + if (val == mds_mitigation_enabled) + return 0; + return mitigation_mds_change(val); +} + +/* -------------------------------------------------------------------------- */ + void speculation_barrier(struct lwp *, struct lwp *); void @@ -628,6 +848,23 @@ cpu_speculation_init(struct cpu_info *ci) } } #endif + + /* + * Microarchectural Data Sampling. + * + * cpu0 is the one that detects the method and sets the global + * variable. + */ + if (ci == &cpu_info_primary) { + mds_detect_method(); + mds_mitigation_enabled = + (mds_mitigation_method != MDS_MITIGATION_NONE); + mds_set_name(); + } + if (mds_mitigation_method != MDS_MITIGATION_NONE && + mds_mitigation_method != MDS_MITIGATION_MDS_NO) { + mitigation_mds_apply_cpu(ci, true); + } } void sysctl_speculation_init(struct sysctllog **); @@ -705,4 +942,26 @@ sysctl_speculation_init(struct sysctllog **clog) NULL, 0, v4_mitigation_name, 0, CTL_CREATE, CTL_EOL); + + /* Microarchitectural Data Sampling */ + spec_rnode = NULL; + sysctl_createv(clog, 0, NULL, &spec_rnode, + CTLFLAG_PERMANENT, + CTLTYPE_NODE, "mds", NULL, + NULL, 0, NULL, 0, + CTL_MACHDEP, CTL_CREATE); + sysctl_createv(clog, 0, &spec_rnode, NULL, + CTLFLAG_READWRITE, + CTLTYPE_BOOL, "mitigated", + SYSCTL_DESCR("Whether MDS is mitigated"), + sysctl_machdep_mds_mitigated, 0, + &mds_mitigation_enabled, 0, + CTL_CREATE, CTL_EOL); + sysctl_createv(clog, 0, &spec_rnode, NULL, + CTLFLAG_PERMANENT, + CTLTYPE_STRING, "method", + SYSCTL_DESCR("Mitigation method in use"), + NULL, 0, + mds_mitigation_name, 0, + CTL_CREATE, CTL_EOL); }