Skip to content

Commit 48204ab

Browse files
ardbiesheuvelbp3tk0v
ardbiesheuvel
authored and
bp3tk0v
committed
x86/sme: Move early SME kernel encryption handling into .head.text
The .head.text section is the initial primary entrypoint of the core kernel, and is entered with the CPU executing from a 1:1 mapping of memory. Such code must never access global variables using absolute references, as these are based on the kernel virtual mapping which is not active yet at this point. Given that the SME startup code is also called from this early execution context, move it into .head.text as well. This will allow more thorough build time checks in the future to ensure that early startup code only uses RIP-relative references to global variables. Also replace some occurrences of __pa_symbol() [which relies on the compiler generating an absolute reference, which is not guaranteed] and an open coded RIP-relative access with RIP_REL_REF(). Signed-off-by: Ard Biesheuvel <ardb@kernel.org> Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de> Tested-by: Tom Lendacky <thomas.lendacky@amd.com> Link: https://lore.kernel.org/r/20240227151907.387873-18-ardb+git@google.com
1 parent cd0d9d9 commit 48204ab

File tree

2 files changed

+21
-29
lines changed

2 files changed

+21
-29
lines changed

‎arch/x86/include/asm/mem_encrypt.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ void __init sme_unmap_bootdata(char *real_mode_data);
4747

4848
void __init sme_early_init(void);
4949

50-
void __init sme_encrypt_kernel(struct boot_params *bp);
51-
void __init sme_enable(struct boot_params *bp);
50+
void sme_encrypt_kernel(struct boot_params *bp);
51+
void sme_enable(struct boot_params *bp);
5252

5353
int __init early_set_memory_decrypted(unsigned long vaddr, unsigned long size);
5454
int __init early_set_memory_encrypted(unsigned long vaddr, unsigned long size);
@@ -81,8 +81,8 @@ static inline void __init sme_unmap_bootdata(char *real_mode_data) { }
8181

8282
static inline void __init sme_early_init(void) { }
8383

84-
static inline void __init sme_encrypt_kernel(struct boot_params *bp) { }
85-
static inline void __init sme_enable(struct boot_params *bp) { }
84+
static inline void sme_encrypt_kernel(struct boot_params *bp) { }
85+
static inline void sme_enable(struct boot_params *bp) { }
8686

8787
static inline void sev_es_init_vc_handling(void) { }
8888

‎arch/x86/mm/mem_encrypt_identity.c‎

Lines changed: 17 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#include <linux/mem_encrypt.h>
4242
#include <linux/cc_platform.h>
4343

44+
#include <asm/init.h>
4445
#include <asm/setup.h>
4546
#include <asm/sections.h>
4647
#include <asm/coco.h>
@@ -94,7 +95,7 @@ struct sme_populate_pgd_data {
9495
*/
9596
static char sme_workarea[2 * PMD_SIZE] __section(".init.scratch");
9697

97-
static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
98+
static void __head sme_clear_pgd(struct sme_populate_pgd_data *ppd)
9899
{
99100
unsigned long pgd_start, pgd_end, pgd_size;
100101
pgd_t *pgd_p;
@@ -109,7 +110,7 @@ static void __init sme_clear_pgd(struct sme_populate_pgd_data *ppd)
109110
memset(pgd_p, 0, pgd_size);
110111
}
111112

112-
static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
113+
static pud_t __head *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
113114
{
114115
pgd_t *pgd;
115116
p4d_t *p4d;
@@ -146,7 +147,7 @@ static pud_t __init *sme_prepare_pgd(struct sme_populate_pgd_data *ppd)
146147
return pud;
147148
}
148149

149-
static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
150+
static void __head sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
150151
{
151152
pud_t *pud;
152153
pmd_t *pmd;
@@ -162,7 +163,7 @@ static void __init sme_populate_pgd_large(struct sme_populate_pgd_data *ppd)
162163
set_pmd(pmd, __pmd(ppd->paddr | ppd->pmd_flags));
163164
}
164165

165-
static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
166+
static void __head sme_populate_pgd(struct sme_populate_pgd_data *ppd)
166167
{
167168
pud_t *pud;
168169
pmd_t *pmd;
@@ -188,7 +189,7 @@ static void __init sme_populate_pgd(struct sme_populate_pgd_data *ppd)
188189
set_pte(pte, __pte(ppd->paddr | ppd->pte_flags));
189190
}
190191

191-
static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
192+
static void __head __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
192193
{
193194
while (ppd->vaddr < ppd->vaddr_end) {
194195
sme_populate_pgd_large(ppd);
@@ -198,7 +199,7 @@ static void __init __sme_map_range_pmd(struct sme_populate_pgd_data *ppd)
198199
}
199200
}
200201

201-
static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
202+
static void __head __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
202203
{
203204
while (ppd->vaddr < ppd->vaddr_end) {
204205
sme_populate_pgd(ppd);
@@ -208,7 +209,7 @@ static void __init __sme_map_range_pte(struct sme_populate_pgd_data *ppd)
208209
}
209210
}
210211

211-
static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
212+
static void __head __sme_map_range(struct sme_populate_pgd_data *ppd,
212213
pmdval_t pmd_flags, pteval_t pte_flags)
213214
{
214215
unsigned long vaddr_end;
@@ -232,22 +233,22 @@ static void __init __sme_map_range(struct sme_populate_pgd_data *ppd,
232233
__sme_map_range_pte(ppd);
233234
}
234235

235-
static void __init sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
236+
static void __head sme_map_range_encrypted(struct sme_populate_pgd_data *ppd)
236237
{
237238
__sme_map_range(ppd, PMD_FLAGS_ENC, PTE_FLAGS_ENC);
238239
}
239240

240-
static void __init sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
241+
static void __head sme_map_range_decrypted(struct sme_populate_pgd_data *ppd)
241242
{
242243
__sme_map_range(ppd, PMD_FLAGS_DEC, PTE_FLAGS_DEC);
243244
}
244245

245-
static void __init sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
246+
static void __head sme_map_range_decrypted_wp(struct sme_populate_pgd_data *ppd)
246247
{
247248
__sme_map_range(ppd, PMD_FLAGS_DEC_WP, PTE_FLAGS_DEC_WP);
248249
}
249250

250-
static unsigned long __init sme_pgtable_calc(unsigned long len)
251+
static unsigned long __head sme_pgtable_calc(unsigned long len)
251252
{
252253
unsigned long entries = 0, tables = 0;
253254

@@ -284,7 +285,7 @@ static unsigned long __init sme_pgtable_calc(unsigned long len)
284285
return entries + tables;
285286
}
286287

287-
void __init sme_encrypt_kernel(struct boot_params *bp)
288+
void __head sme_encrypt_kernel(struct boot_params *bp)
288289
{
289290
unsigned long workarea_start, workarea_end, workarea_len;
290291
unsigned long execute_start, execute_end, execute_len;
@@ -319,9 +320,8 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
319320
* memory from being cached.
320321
*/
321322

322-
/* Physical addresses gives us the identity mapped virtual addresses */
323-
kernel_start = __pa_symbol(_text);
324-
kernel_end = ALIGN(__pa_symbol(_end), PMD_SIZE);
323+
kernel_start = (unsigned long)RIP_REL_REF(_text);
324+
kernel_end = ALIGN((unsigned long)RIP_REL_REF(_end), PMD_SIZE);
325325
kernel_len = kernel_end - kernel_start;
326326

327327
initrd_start = 0;
@@ -338,14 +338,6 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
338338
}
339339
#endif
340340

341-
/*
342-
* We're running identity mapped, so we must obtain the address to the
343-
* SME encryption workarea using rip-relative addressing.
344-
*/
345-
asm ("lea sme_workarea(%%rip), %0"
346-
: "=r" (workarea_start)
347-
: "p" (sme_workarea));
348-
349341
/*
350342
* Calculate required number of workarea bytes needed:
351343
* executable encryption area size:
@@ -355,7 +347,7 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
355347
* pagetable structures for the encryption of the kernel
356348
* pagetable structures for workarea (in case not currently mapped)
357349
*/
358-
execute_start = workarea_start;
350+
execute_start = workarea_start = (unsigned long)RIP_REL_REF(sme_workarea);
359351
execute_end = execute_start + (PAGE_SIZE * 2) + PMD_SIZE;
360352
execute_len = execute_end - execute_start;
361353

@@ -498,7 +490,7 @@ void __init sme_encrypt_kernel(struct boot_params *bp)
498490
native_write_cr3(__native_read_cr3());
499491
}
500492

501-
void __init sme_enable(struct boot_params *bp)
493+
void __head sme_enable(struct boot_params *bp)
502494
{
503495
unsigned int eax, ebx, ecx, edx;
504496
unsigned long feature_mask;

0 commit comments

Comments
 (0)