Skip to content

Commit

Permalink
x86/fred: Add __attribute__((no_callee_saved_registers))
Browse files Browse the repository at this point in the history 
Add __attribute__((no_callee_saved_registers)) to avoid saving registers
used in callee.

Suggested-by: H. Peter Anvin (Intel) <hpa@zytor.com>
Signed-off-by: Xin Li (Intel) <xin@zytor.com>
  • Loading branch information
@xinli-intel
xinli-intel committed Jan 17, 2024
1 parent 09124f7 commit 12c3814
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 35 deletions.
30 changes: 20 additions & 10 deletions arch/x86/entry/entry_fred.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
#define FRED_SYSCALL 1
#define FRED_SYSENTER 2

static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
static noinstr __attribute__((no_callee_saved_registers))
void fred_bad_type(struct pt_regs *regs, unsigned long error_code)
{
irqentry_state_t irq_state = irqentry_nmi_enter(regs);

Expand Down Expand Up @@ -51,7 +52,8 @@ static noinstr void fred_bad_type(struct pt_regs *regs, unsigned long error_code
irqentry_nmi_exit(regs, irq_state);
}

static noinstr void fred_intx(struct pt_regs *regs)
static noinstr __attribute__((no_callee_saved_registers))
void fred_intx(struct pt_regs *regs)
{
switch (regs->fred_ss.vector) {
/* Opcode 0xcd, 0x3, NOT INT3 (opcode 0xcc) */
Expand All @@ -73,7 +75,8 @@ static noinstr void fred_intx(struct pt_regs *regs)
}
}

static __always_inline void fred_other(struct pt_regs *regs)
static __always_inline __attribute__((no_callee_saved_registers))
void fred_other(struct pt_regs *regs)
{
/* The compiler can fold these conditions into a single test */
if (likely(regs->fred_ss.vector == FRED_SYSCALL && regs->fred_ss.lm)) {
Expand Down Expand Up @@ -132,7 +135,8 @@ void __init fred_install_sysvec(unsigned int sysvec, idtentry_t handler)
sysvec_table[sysvec - FIRST_SYSTEM_VECTOR] = handler;
}

static noinstr void fred_handle_spurious_interrupt(struct pt_regs *regs)
static noinstr __attribute__((no_callee_saved_registers))
void fred_handle_spurious_interrupt(struct pt_regs *regs)
{
spurious_interrupt(regs, regs->fred_ss.vector);
}
Expand All @@ -153,7 +157,8 @@ void __init fred_complete_exception_setup(void)
fred_setup_done = true;
}

static noinstr void fred_extint(struct pt_regs *regs)
static noinstr __attribute__((no_callee_saved_registers))
void fred_extint(struct pt_regs *regs)
{
unsigned int vector = regs->fred_ss.vector;
unsigned int index = array_index_nospec(vector - FIRST_SYSTEM_VECTOR,
Expand All @@ -174,7 +179,8 @@ static noinstr void fred_extint(struct pt_regs *regs)
}
}

static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
static noinstr __attribute__((no_callee_saved_registers))
void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
{
/* Optimize for #PF. That's the only exception which matters performance wise */
if (likely(regs->fred_ss.vector == X86_TRAP_PF))
Expand Down Expand Up @@ -209,7 +215,8 @@ static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)

}

static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
static noinstr __attribute__((no_callee_saved_registers))
void fred_swexc(struct pt_regs *regs, unsigned long error_code)
{
switch (regs->fred_ss.vector) {
case X86_TRAP_BP: return exc_int3(regs);
Expand All @@ -218,7 +225,8 @@ static noinstr void fred_swexc(struct pt_regs *regs, unsigned long error_code)
}
}

__visible noinstr void fred_entry_from_user(struct pt_regs *regs)
__visible noinstr __attribute__((no_callee_saved_registers))
void fred_entry_from_user(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;

Expand Down Expand Up @@ -250,7 +258,8 @@ __visible noinstr void fred_entry_from_user(struct pt_regs *regs)
return fred_bad_type(regs, error_code);
}

__visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
__visible noinstr __attribute__((no_callee_saved_registers))
void fred_entry_from_kernel(struct pt_regs *regs)
{
unsigned long error_code = regs->orig_ax;

Expand Down Expand Up @@ -279,7 +288,8 @@ __visible noinstr void fred_entry_from_kernel(struct pt_regs *regs)
}

#if IS_ENABLED(CONFIG_KVM_INTEL)
__visible noinstr void __fred_entry_from_kvm(struct pt_regs *regs)
__visible noinstr __attribute__((no_callee_saved_registers))
void __fred_entry_from_kvm(struct pt_regs *regs)
{
switch (regs->fred_ss.type) {
case EVENT_TYPE_EXTINT:
Expand Down
9 changes: 6 additions & 3 deletions arch/x86/include/asm/fred.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,12 @@ void asm_fred_entrypoint_user(void);
void asm_fred_entrypoint_kernel(void);
void asm_fred_entry_from_kvm(struct fred_ss);

__visible void fred_entry_from_user(struct pt_regs *regs);
__visible void fred_entry_from_kernel(struct pt_regs *regs);
__visible void __fred_entry_from_kvm(struct pt_regs *regs);
__visible noinstr __attribute__((no_callee_saved_registers))
void fred_entry_from_user(struct pt_regs *regs);
__visible noinstr __attribute__((no_callee_saved_registers))
void fred_entry_from_kernel(struct pt_regs *regs);
__visible noinstr __attribute__((no_callee_saved_registers))
void __fred_entry_from_kvm(struct pt_regs *regs);

/* Can be called from noinstr code, thus __always_inline */
static __always_inline void fred_entry_from_kvm(unsigned int type, unsigned int vector)
Expand Down
62 changes: 40 additions & 22 deletions arch/x86/include/asm/idtentry.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@

#include <asm/irq_stack.h>

typedef void (*idtentry_t)(struct pt_regs *regs);
typedef __attribute__((no_callee_saved_registers))
void (*idtentry_t)(struct pt_regs *regs);

/**
* DECLARE_IDTENTRY - Declare functions for simple IDT entry points
Expand All @@ -34,8 +35,10 @@ typedef void (*idtentry_t)(struct pt_regs *regs);
#define DECLARE_IDTENTRY(vector, func) \
asmlinkage void asm_##func(void); \
asmlinkage void xen_asm_##func(void); \
__attribute__((no_callee_saved_registers)) \
void fred_##func(struct pt_regs *regs); \
__visible void func(struct pt_regs *regs)
__visible __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs)

/**
* DEFINE_IDTENTRY - Emit code for simple IDT entry points
Expand All @@ -53,7 +56,8 @@ typedef void (*idtentry_t)(struct pt_regs *regs);
#define DEFINE_IDTENTRY(func) \
static __always_inline void __##func(struct pt_regs *regs); \
\
__visible noinstr void func(struct pt_regs *regs) \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs) \
{ \
irqentry_state_t state = irqentry_enter(regs); \
\
Expand Down Expand Up @@ -86,7 +90,8 @@ static __always_inline void __##func(struct pt_regs *regs)
#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
asmlinkage void asm_##func(void); \
asmlinkage void xen_asm_##func(void); \
__visible void func(struct pt_regs *regs, unsigned long error_code)
__visible __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, unsigned long error_code)

/**
* DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points
Expand All @@ -99,8 +104,8 @@ static __always_inline void __##func(struct pt_regs *regs)
static __always_inline void __##func(struct pt_regs *regs, \
unsigned long error_code); \
\
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, unsigned long error_code) \
{ \
irqentry_state_t state = irqentry_enter(regs); \
\
Expand Down Expand Up @@ -139,7 +144,8 @@ static __always_inline void __##func(struct pt_regs *regs, \
* is required before the enter/exit() helpers are invoked.
*/
#define DEFINE_IDTENTRY_RAW(func) \
__visible noinstr void func(struct pt_regs *regs)
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs)

/**
* DEFINE_FREDENTRY_RAW - Emit code for raw FRED entry points
Expand All @@ -150,7 +156,8 @@ __visible noinstr void func(struct pt_regs *regs)
* See @DEFINE_IDTENTRY_RAW for further details.
*/
#define DEFINE_FREDENTRY_RAW(func) \
noinstr void fred_##func(struct pt_regs *regs)
noinstr __attribute__((no_callee_saved_registers)) \
void fred_##func(struct pt_regs *regs)

/**
* DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points
Expand Down Expand Up @@ -178,7 +185,8 @@ noinstr void fred_##func(struct pt_regs *regs)
* is required before the enter/exit() helpers are invoked.
*/
#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \
__visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, unsigned long error_code)

/**
* DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry
Expand Down Expand Up @@ -206,8 +214,8 @@ __visible noinstr void func(struct pt_regs *regs, unsigned long error_code)
#define DEFINE_IDTENTRY_IRQ(func) \
static void __##func(struct pt_regs *regs, u32 vector); \
\
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code) \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, unsigned long error_code) \
{ \
irqentry_state_t state = irqentry_enter(regs); \
u32 vector = (u32)(u8)error_code; \
Expand Down Expand Up @@ -254,7 +262,8 @@ static __always_inline void instr_##func(struct pt_regs *regs) \
run_sysvec_on_irqstack_cond(__##func, regs); \
} \
\
__visible noinstr void func(struct pt_regs *regs) \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs) \
{ \
irqentry_state_t state = irqentry_enter(regs); \
\
Expand All @@ -264,6 +273,7 @@ __visible noinstr void func(struct pt_regs *regs) \
irqentry_exit(regs, state); \
} \
\
__attribute__((no_callee_saved_registers)) \
void fred_##func(struct pt_regs *regs) \
{ \
instr_##func (regs); \
Expand Down Expand Up @@ -293,7 +303,8 @@ static __always_inline void instr_##func(struct pt_regs *regs) \
__irq_exit_raw(); \
} \
\
__visible noinstr void func(struct pt_regs *regs) \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs) \
{ \
irqentry_state_t state = irqentry_enter(regs); \
\
Expand All @@ -303,6 +314,7 @@ __visible noinstr void func(struct pt_regs *regs) \
irqentry_exit(regs, state); \
} \
\
__attribute__((no_callee_saved_registers)) \
void fred_##func(struct pt_regs *regs) \
{ \
instr_##func (regs); \
Expand Down Expand Up @@ -337,7 +349,8 @@ static __always_inline void __##func(struct pt_regs *regs)
*/
#define DECLARE_IDTENTRY_IST(vector, func) \
DECLARE_IDTENTRY_RAW(vector, func); \
__visible void noist_##func(struct pt_regs *regs)
__visible __attribute__((no_callee_saved_registers)) \
void noist_##func(struct pt_regs *regs)

/**
* DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
Expand All @@ -349,8 +362,11 @@ static __always_inline void __##func(struct pt_regs *regs)
*/
#define DECLARE_IDTENTRY_VC(vector, func) \
DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func); \
__visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code); \
__visible noinstr void user_##func(struct pt_regs *regs, unsigned long error_code)
__visible noinstr __attribute__((no_callee_saved_registers)) \
void kernel_##func(struct pt_regs *regs, \
unsigned long error_code); \
__visible noinstr __attribute__((no_callee_saved_registers)) \
void user_##func(struct pt_regs *regs, unsigned long error_code)

/**
* DEFINE_IDTENTRY_IST - Emit code for IST entry points
Expand Down Expand Up @@ -424,9 +440,10 @@ static __always_inline void __##func(struct pt_regs *regs)
*/
#define DECLARE_IDTENTRY_DF(vector, func) \
asmlinkage void asm_##func(void); \
__visible void func(struct pt_regs *regs, \
unsigned long error_code, \
unsigned long address)
__visible __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, \
unsigned long error_code, \
unsigned long address)

/**
* DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit
Expand All @@ -436,9 +453,10 @@ static __always_inline void __##func(struct pt_regs *regs)
* cr2 in the address argument.
*/
#define DEFINE_IDTENTRY_DF(func) \
__visible noinstr void func(struct pt_regs *regs, \
unsigned long error_code, \
unsigned long address)
__visible noinstr __attribute__((no_callee_saved_registers)) \
void func(struct pt_regs *regs, \
unsigned long error_code, \
unsigned long address)

#endif /* !CONFIG_X86_64 */

Expand Down

0 comments on commit 12c3814

Please sign in to comment.