mirrored from git://xenbits.xen.org/xen.git
-
Notifications
You must be signed in to change notification settings - Fork 325
/
tsx.c
288 lines (256 loc) · 10.3 KB
/
tsx.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
#include <xen/init.h>
#include <xen/param.h>
#include <asm/msr.h>
/*
* Valid values:
* 1 => Explicit tsx=1
* 0 => Explicit tsx=0
* -1 => Default, altered to 0/1 (if unspecified) by:
* - TAA heuristics/settings for speculative safety
* - "TSX vs PCR3" select for TSX memory ordering safety
* -3 => Implicit tsx=1 (feed-through from spec-ctrl=0)
*
* This is arranged such that the bottom bit encodes whether TSX is actually
* disabled, while identifying various explicit (>=0) and implicit (<0)
* conditions.
*
* This option only has any effect on systems presenting a mechanism of
* controlling TSX behaviour, and where TSX isn't force-disabled by firmware.
*/
int8_t __read_mostly opt_tsx = -1;
bool __read_mostly rtm_disabled;
static int __init cf_check parse_tsx(const char *s)
{
int rc = 0, val = parse_bool(s, NULL);
if ( val >= 0 )
opt_tsx = val;
else
rc = -EINVAL;
return rc;
}
custom_param("tsx", parse_tsx);
void tsx_init(void)
{
static bool __read_mostly once;
/*
* This function is first called between microcode being loaded, and
* CPUID being scanned generally. early_cpu_init() has already prepared
* the feature bits needed here. And early_microcode_init() has ensured
* they are not stale after the microcode update.
*/
if ( unlikely(!once) )
{
bool has_rtm_always_abort;
once = true;
has_rtm_always_abort = cpu_has_rtm_always_abort;
if ( cpu_has_tsx_ctrl && cpu_has_srbds_ctrl )
{
/*
* On a TAA-vulnerable or later part with at least the May 2020
* microcode mitigating SRBDS.
*/
uint64_t val;
rdmsrl(MSR_MCU_OPT_CTRL, val);
/*
* Probe for the February 2022 microcode which de-features TSX on
* TAA-vulnerable client parts - WHL-R/CFL-R.
*
* RTM_ALWAYS_ABORT (read above) enumerates the new functionality,
* but is read as zero if MCU_OPT_CTRL.RTM_ALLOW has been set
* before we run. Undo this.
*/
if ( val & MCU_OPT_CTRL_RTM_ALLOW )
has_rtm_always_abort = true;
if ( has_rtm_always_abort )
{
if ( val & MCU_OPT_CTRL_RTM_LOCKED )
{
/*
* If RTM_LOCKED is set, TSX is disabled because SGX is
* enabled, and there is nothing we can do. Override with
* tsx=0 so all other logic takes sensible actions.
*/
printk(XENLOG_WARNING "TSX locked by firmware - disabling\n");
opt_tsx = 0;
}
else
{
/*
* Otherwise, set RTM_ALLOW. Not because we necessarily
* intend to enable RTM, but it prevents
* MSR_TSX_CTRL.RTM_DISABLE from being ignored, thus
* allowing the rest of the TSX selection logic to work as
* before.
*/
val |= MCU_OPT_CTRL_RTM_ALLOW;
}
set_in_mcu_opt_ctrl(
MCU_OPT_CTRL_RTM_LOCKED | MCU_OPT_CTRL_RTM_ALLOW, val);
/*
* If no explicit tsx= option is provided, pick a default.
*
* With RTM_ALWAYS_ABORT, the default ucode behaviour is to
* disable, so match that. This does not override explicit user
* choices, or implicit choices as a side effect of spec-ctrl=0.
*/
if ( opt_tsx == -1 )
opt_tsx = 0;
}
}
if ( cpu_has_tsx_force_abort )
{
/*
* On an early TSX-enable Skylake part subject to the memory
* ordering erratum, with at least the March 2019 microcode.
*/
/*
* Probe for the June 2021 microcode which de-features TSX on
* client parts. (Note - this is a subset of parts impacted by
* the memory ordering errata.)
*
* RTM_ALWAYS_ABORT enumerates the new functionality, but is also
* read as zero if TSX_FORCE_ABORT.ENABLE_RTM has been set before
* we run.
*/
if ( !has_rtm_always_abort )
{
uint64_t val;
rdmsrl(MSR_TSX_FORCE_ABORT, val);
if ( val & TSX_ENABLE_RTM )
has_rtm_always_abort = true;
}
/*
* If no explicit tsx= option is provided, pick a default.
*
* This deliberately overrides the implicit opt_tsx=-3 from
* `spec-ctrl=0` because:
* - parse_spec_ctrl() ran before any CPU details where know.
* - We now know we're running on a CPU not affected by TAA (as
* TSX_FORCE_ABORT is enumerated).
* - When RTM_ALWAYS_ABORT is enumerated, TSX malfunctions, so we
* only ever want it enabled by explicit user choice.
*
* Without RTM_ALWAYS_ABORT, leave TSX active. In particular,
* this includes SKX where TSX is still supported.
*
* With RTM_ALWAYS_ABORT, disable TSX.
*/
if ( opt_tsx < 0 )
opt_tsx = !has_rtm_always_abort;
}
/*
* Always force RTM_ALWAYS_ABORT, even if it currently visible. If
* the user explicitly opts to enable TSX, we'll set the appropriate
* RTM_ENABLE bit and cause RTM_ALWAYS_ABORT to be hidden from the
* general CPUID scan later.
*/
if ( has_rtm_always_abort )
setup_force_cpu_cap(X86_FEATURE_RTM_ALWAYS_ABORT);
/*
* The TSX features (HLE/RTM) are handled specially. They both
* enumerate features but, on certain parts, have mechanisms to be
* hidden without disrupting running software.
*
* At the moment, we're running in an unknown context (WRT hiding -
* particularly if another fully fledged kernel ran before us) and
* depending on user settings, may elect to continue hiding them from
* native CPUID instructions.
*
* Xen doesn't use TSX itself, but use cpu_has_{hle,rtm} for various
* system reasons, mostly errata detection, so the meaning is more
* useful as "TSX infrastructure available", as opposed to "features
* advertised and working".
*
* Force the features to be visible in Xen's view if we see any of the
* infrastructure capable of hiding them.
*/
if ( cpu_has_tsx_ctrl || cpu_has_tsx_force_abort )
{
setup_force_cpu_cap(X86_FEATURE_HLE);
setup_force_cpu_cap(X86_FEATURE_RTM);
}
}
/*
* Note: MSR_TSX_CTRL is enumerated on TSX-enabled MDS_NO and later parts.
* MSR_TSX_FORCE_ABORT is enumerated on TSX-enabled pre-MDS_NO Skylake
* parts only. The two features are on a disjoint set of CPUs, and not
* offered to guests by hypervisors.
*/
if ( cpu_has_tsx_ctrl )
{
/*
* On a TAA-vulnerable part with at least the November 2019 microcode,
* or newer part with TAA fixed.
*
* Notes:
* - With the February 2022 microcode, if SGX has caused TSX to be
* locked off, opt_tsx is overridden to 0. TSX_CTRL.RTM_DISABLE is
* an ignored bit, but we write it such that it matches the
* behaviour enforced by microcode.
* - Otherwise, if SGX isn't enabled and TSX is available to be
* controlled, we have or will set MSR_MCU_OPT_CTRL.RTM_ALLOW to
* let TSX_CTRL.RTM_DISABLE be usable.
*/
uint32_t hi, lo;
rdmsr(MSR_TSX_CTRL, lo, hi);
/* Check bottom bit only. Higher bits are various sentinels. */
rtm_disabled = !(opt_tsx & 1);
lo &= ~(TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR);
if ( rtm_disabled )
lo |= TSX_CTRL_RTM_DISABLE | TSX_CTRL_CPUID_CLEAR;
wrmsr(MSR_TSX_CTRL, lo, hi);
}
else if ( cpu_has_tsx_force_abort )
{
/*
* On an early TSX-enable Skylake part subject to the memory ordering
* erratum, with at least the March 2019 microcode.
*/
uint32_t hi, lo;
rdmsr(MSR_TSX_FORCE_ABORT, lo, hi);
/* Check bottom bit only. Higher bits are various sentinels. */
rtm_disabled = !(opt_tsx & 1);
lo &= ~(TSX_FORCE_ABORT_RTM | TSX_CPUID_CLEAR | TSX_ENABLE_RTM);
if ( cpu_has_rtm_always_abort )
{
/*
* June 2021 microcode, on a client part with TSX de-featured:
* - There are no mitigations for the TSX memory ordering errata.
* - Performance counter 3 works. (I.e. it isn't being used by
* microcode to work around the memory ordering errata.)
* - TSX_FORCE_ABORT.FORCE_ABORT_RTM is fixed read1/write-discard.
* - TSX_FORCE_ABORT.TSX_CPUID_CLEAR can be used to hide the
* HLE/RTM CPUID bits.
* - TSX_FORCE_ABORT.ENABLE_RTM may be used to opt in to
* re-enabling RTM, at the users own risk.
*/
lo |= rtm_disabled ? TSX_CPUID_CLEAR : TSX_ENABLE_RTM;
}
else
{
/*
* Either a server part where TSX isn't de-featured, or pre-June
* 2021 microcode:
* - By default, the TSX memory ordering errata is worked around
* in microcode at the cost of Performance Counter 3.
* - "Working TSX" vs "Working PCR3" can be selected by way of
* setting TSX_FORCE_ABORT.FORCE_ABORT_RTM.
*/
if ( rtm_disabled )
lo |= TSX_FORCE_ABORT_RTM;
}
wrmsr(MSR_TSX_FORCE_ABORT, lo, hi);
}
else if ( opt_tsx >= 0 )
printk_once(XENLOG_WARNING
"TSX controls not available - Ignoring tsx= setting\n");
}
/*
* Local variables:
* mode: C
* c-file-style: "BSD"
* c-basic-offset: 4
* tab-width: 4
* indent-tabs-mode: nil
* End:
*/