Skip to content

Commit 7e92e01

Browse files
rohanmclurempe
rohanmclure
authored and
mpe
committed
powerpc: Provide syscall wrapper
Implement syscall wrapper as per s390, x86, arm64. When enabled cause handlers to accept parameters from a stack frame rather than from user scratch register state. This allows for user registers to be safely cleared in order to reduce caller influence on speculation within syscall routine. The wrapper is a macro that emits syscall handler symbols that call into the target handler, obtaining its parameters from a struct pt_regs on the stack. As registers are already saved to the stack prior to calling system_call_exception, it appears that this function is executed more efficiently with the new stack-pointer convention than with parameters passed by registers, avoiding the allocation of a stack frame for this method. On a 32-bit system, we see >20% performance increases on the null_syscall microbenchmark, and on a Power 8 the performance gains amortise the cost of clearing and restoring registers which is implemented at the end of this series, seeing final result of ~5.6% performance improvement on null_syscall. Syscalls are wrapped in this fashion on all platforms except for the Cell processor as this commit does not provide SPU support. This can be quickly fixed in a successive patch, but requires spu_sys_callback to allocate a pt_regs structure to satisfy the wrapped calling convention. Co-developed-by: Andrew Donnellan <ajd@linux.ibm.com> Signed-off-by: Andrew Donnellan <ajd@linux.ibm.com> Signed-off-by: Rohan McLure <rmclure@linux.ibm.com> Reviewed-by: Nicholas Piggin <npiggin@gmai.com> [mpe: Make incompatible with COMPAT to retain clearing of high bits of args] Signed-off-by: Michael Ellerman <mpe@ellerman.id.au> Link: https://lore.kernel.org/r/20220921065605.1051927-22-rmclure@linux.ibm.com
1 parent f8971c6 commit 7e92e01

File tree

7 files changed

+105
-18
lines changed

7 files changed

+105
-18
lines changed

‎arch/powerpc/Kconfig‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ config PPC
137137
select ARCH_HAS_STRICT_KERNEL_RWX if (PPC_BOOK3S || PPC_8xx || 40x) && !HIBERNATION
138138
select ARCH_HAS_STRICT_KERNEL_RWX if PPC_85xx && !HIBERNATION && !RANDOMIZE_BASE
139139
select ARCH_HAS_STRICT_MODULE_RWX if ARCH_HAS_STRICT_KERNEL_RWX
140+
select ARCH_HAS_SYSCALL_WRAPPER if !SPU_BASE && !COMPAT
140141
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
141142
select ARCH_HAS_UACCESS_FLUSHCACHE
142143
select ARCH_HAS_UBSAN_SANITIZE_ALL

‎arch/powerpc/include/asm/syscall.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,12 @@
1414
#include <linux/sched.h>
1515
#include <linux/thread_info.h>
1616

17+
#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
18+
typedef long (*syscall_fn)(const struct pt_regs *);
19+
#else
1720
typedef long (*syscall_fn)(unsigned long, unsigned long, unsigned long,
1821
unsigned long, unsigned long, unsigned long);
22+
#endif
1923

2024
/* ftrace syscalls requires exporting the sys_call_table */
2125
extern const syscall_fn sys_call_table[];

‎arch/powerpc/include/asm/syscall_wrapper.h‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* syscall_wrapper.h - powerpc specific wrappers to syscall definitions
4+
*
5+
* Based on arch/{x86,arm64}/include/asm/syscall_wrapper.h
6+
*/
7+
8+
#ifndef __ASM_POWERPC_SYSCALL_WRAPPER_H
9+
#define __ASM_POWERPC_SYSCALL_WRAPPER_H
10+
11+
struct pt_regs;
12+
13+
#define SC_POWERPC_REGS_TO_ARGS(x, ...) \
14+
__MAP(x,__SC_ARGS \
15+
,,regs->gpr[3],,regs->gpr[4],,regs->gpr[5] \
16+
,,regs->gpr[6],,regs->gpr[7],,regs->gpr[8])
17+
18+
#define __SYSCALL_DEFINEx(x, name, ...) \
19+
long __powerpc_sys##name(const struct pt_regs *regs); \
20+
ALLOW_ERROR_INJECTION(__powerpc_sys##name, ERRNO); \
21+
static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \
22+
static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \
23+
long __powerpc_sys##name(const struct pt_regs *regs) \
24+
{ \
25+
return __se_sys##name(SC_POWERPC_REGS_TO_ARGS(x,__VA_ARGS__)); \
26+
} \
27+
static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \
28+
{ \
29+
long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \
30+
__MAP(x,__SC_TEST,__VA_ARGS__); \
31+
__PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \
32+
return ret; \
33+
} \
34+
static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
35+
36+
#define SYSCALL_DEFINE0(sname) \
37+
SYSCALL_METADATA(_##sname, 0); \
38+
long __powerpc_sys_##sname(const struct pt_regs *__unused); \
39+
ALLOW_ERROR_INJECTION(__powerpc_sys_##sname, ERRNO); \
40+
long __powerpc_sys_##sname(const struct pt_regs *__unused)
41+
42+
#define COND_SYSCALL(name) \
43+
long __powerpc_sys_##name(const struct pt_regs *regs); \
44+
long __weak __powerpc_sys_##name(const struct pt_regs *regs) \
45+
{ \
46+
return sys_ni_syscall(); \
47+
}
48+
49+
#define SYS_NI(name) SYSCALL_ALIAS(__powerpc_sys_##name, sys_ni_posix_timers);
50+
51+
#endif // __ASM_POWERPC_SYSCALL_WRAPPER_H

‎arch/powerpc/include/asm/syscalls.h‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,12 @@
1515
#include <asm/unistd.h>
1616
#include <asm/ucontext.h>
1717

18+
#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
19+
long sys_ni_syscall(void);
20+
#else
21+
long sys_ni_syscall(const struct pt_regs *regs);
22+
#endif
23+
1824
struct rtas_args;
1925

2026
/*
@@ -29,12 +35,12 @@ struct rtas_args;
2935
#define merge_64(high, low) (((u64)high << 32) | low)
3036
#endif
3137

32-
long sys_ni_syscall(void);
33-
3438
/*
3539
* PowerPC architecture-specific syscalls
3640
*/
3741

42+
#ifndef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
43+
3844
long sys_rtas(struct rtas_args __user *uargs);
3945

4046
#ifdef CONFIG_PPC64
@@ -114,5 +120,19 @@ long sys_ppc_fadvise64_64(int fd, int advice,
114120
u32 len_high, u32 len_low);
115121
#endif
116122

123+
#else
124+
125+
#define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native)
126+
#define __SYSCALL(nr, entry) \
127+
long __powerpc_##entry(const struct pt_regs *regs);
128+
129+
#ifdef CONFIG_PPC64
130+
#include <asm/syscall_table_64.h>
131+
#else
132+
#include <asm/syscall_table_32.h>
133+
#endif /* CONFIG_PPC64 */
134+
135+
#endif /* CONFIG_ARCH_HAS_SYSCALL_WRAPPER */
136+
117137
#endif /* __KERNEL__ */
118138
#endif /* __ASM_POWERPC_SYSCALLS_H */

‎arch/powerpc/kernel/syscall.c‎

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
/* Has to run notrace because it is entered not completely "reconciled" */
1616
notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
1717
{
18-
unsigned long r3, r4, r5, r6, r7, r8;
1918
long ret;
2019
syscall_fn f;
2120

@@ -145,31 +144,34 @@ notrace long system_call_exception(struct pt_regs *regs, unsigned long r0)
145144
return -ENOSYS;
146145
}
147146

148-
r3 = regs->gpr[3];
149-
r4 = regs->gpr[4];
150-
r5 = regs->gpr[5];
151-
r6 = regs->gpr[6];
152-
r7 = regs->gpr[7];
153-
r8 = regs->gpr[8];
154-
155147
/* May be faster to do array_index_nospec? */
156148
barrier_nospec();
157149

150+
#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
151+
// No COMPAT if we have SYSCALL_WRAPPER, see Kconfig
152+
f = (void *)sys_call_table[r0];
153+
ret = f(regs);
154+
#else
158155
if (unlikely(is_compat_task())) {
156+
unsigned long r3, r4, r5, r6, r7, r8;
157+
159158
f = (void *)compat_sys_call_table[r0];
160159

161-
r3 &= 0x00000000ffffffffULL;
162-
r4 &= 0x00000000ffffffffULL;
163-
r5 &= 0x00000000ffffffffULL;
164-
r6 &= 0x00000000ffffffffULL;
165-
r7 &= 0x00000000ffffffffULL;
166-
r8 &= 0x00000000ffffffffULL;
160+
r3 = regs->gpr[3] & 0x00000000ffffffffULL;
161+
r4 = regs->gpr[4] & 0x00000000ffffffffULL;
162+
r5 = regs->gpr[5] & 0x00000000ffffffffULL;
163+
r6 = regs->gpr[6] & 0x00000000ffffffffULL;
164+
r7 = regs->gpr[7] & 0x00000000ffffffffULL;
165+
r8 = regs->gpr[8] & 0x00000000ffffffffULL;
167166

167+
ret = f(r3, r4, r5, r6, r7, r8);
168168
} else {
169169
f = (void *)sys_call_table[r0];
170-
}
171170

172-
ret = f(r3, r4, r5, r6, r7, r8);
171+
ret = f(regs->gpr[3], regs->gpr[4], regs->gpr[5],
172+
regs->gpr[6], regs->gpr[7], regs->gpr[8]);
173+
}
174+
#endif
173175

174176
/*
175177
* Ultimately, this value will get limited by KSTACK_OFFSET_MAX(),

‎arch/powerpc/kernel/systbl.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,20 @@
1515
#include <asm/unistd.h>
1616
#include <asm/syscalls.h>
1717

18+
#undef __SYSCALL_WITH_COMPAT
1819
#define __SYSCALL_WITH_COMPAT(nr, entry, compat) __SYSCALL(nr, entry)
1920

21+
#undef __SYSCALL
22+
#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
23+
#define __SYSCALL(nr, entry) [nr] = __powerpc_##entry,
24+
#define __powerpc_sys_ni_syscall sys_ni_syscall
25+
#else
2026
/*
2127
* Coerce syscall handlers with arbitrary parameters to common type
2228
* requires cast to void* to avoid -Wcast-function-type.
2329
*/
2430
#define __SYSCALL(nr, entry) [nr] = (void *) entry,
31+
#endif
2532

2633
const syscall_fn sys_call_table[] = {
2734
#ifdef CONFIG_PPC64

‎arch/powerpc/kernel/vdso.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@
3939
extern char vdso32_start, vdso32_end;
4040
extern char vdso64_start, vdso64_end;
4141

42+
long sys_ni_syscall(void);
43+
4244
/*
4345
* The vdso data page (aka. systemcfg for old ppc64 fans) is here.
4446
* Once the early boot kernel code no longer needs to muck around

0 commit comments

Comments
 (0)