Skip to content

Commit

Permalink
powerpc/64e: remove implicit soft-masking and interrupt exit restart …
Browse files Browse the repository at this point in the history 
…logic

The implicit soft-masking to speed up interrupt return was going to be
used by 64e as well, but it has not been extensively tested on that
platform and is not considered ready. It was intended to be disabled
before merge. Disable it for now.

Most of the restart code is common with 64s, so with more correctness
and performance testing this could be re-enabled again by adding the
extra soft-mask checks to interrupt handlers and flipping
exit_must_hard_disable().

Fixes: 9d1988c ("powerpc/64: treat low kernel text as irqs soft-masked")
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210630074621.2109197-4-npiggin@gmail.com
  • Loading branch information
@npiggin @mpe
npiggin authored and mpe committed Jun 30, 2021
1 parent fce01ac commit 9b69d48
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 23 deletions.
33 changes: 24 additions & 9 deletions arch/powerpc/include/asm/interrupt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,20 +73,34 @@
#include <asm/kprobes.h>
#include <asm/runlatch.h>

#ifdef CONFIG_PPC64
#ifdef CONFIG_PPC_BOOK3S_64
extern char __end_soft_masked[];
unsigned long search_kernel_restart_table(unsigned long addr);
#endif

#ifdef CONFIG_PPC_BOOK3S_64
DECLARE_STATIC_KEY_FALSE(interrupt_exit_not_reentrant);

static inline bool is_implicit_soft_masked(struct pt_regs *regs)
{
if (regs->msr & MSR_PR)
return false;

if (regs->nip >= (unsigned long)__end_soft_masked)
return false;

return true;
}

static inline void srr_regs_clobbered(void)
{
local_paca->srr_valid = 0;
local_paca->hsrr_valid = 0;
}
#else
static inline bool is_implicit_soft_masked(struct pt_regs *regs)
{
return false;
}

static inline void srr_regs_clobbered(void)
{
}
Expand Down Expand Up @@ -150,11 +164,13 @@ static inline void interrupt_enter_prepare(struct pt_regs *regs, struct interrup
*/
if (TRAP(regs) != INTERRUPT_PROGRAM) {
CT_WARN_ON(ct_state() != CONTEXT_KERNEL);
BUG_ON(regs->nip < (unsigned long)__end_soft_masked);
BUG_ON(is_implicit_soft_masked(regs));
}
#ifdef CONFIG_PPC_BOOK3S
/* Move this under a debugging check */
if (arch_irq_disabled_regs(regs))
BUG_ON(search_kernel_restart_table(regs->nip));
#endif
}
#endif

Expand Down Expand Up @@ -244,10 +260,9 @@ static inline void interrupt_nmi_enter_prepare(struct pt_regs *regs, struct inte
local_paca->irq_soft_mask = IRQS_ALL_DISABLED;
local_paca->irq_happened |= PACA_IRQ_HARD_DIS;

if (IS_ENABLED(CONFIG_PPC_BOOK3S_64) && !(regs->msr & MSR_PR) &&
regs->nip < (unsigned long)__end_soft_masked) {
// Kernel code running below __end_soft_masked is
// implicitly soft-masked.
if (is_implicit_soft_masked(regs)) {
// Adjust regs->softe soft implicit soft-mask, so
// arch_irq_disabled_regs(regs) behaves as expected.
regs->softe = IRQS_ALL_DISABLED;
}

Expand Down Expand Up @@ -282,14 +297,14 @@ static inline void interrupt_nmi_exit_prepare(struct pt_regs *regs, struct inter
*/

#ifdef CONFIG_PPC64
#ifdef CONFIG_PPC_BOOK3S
if (arch_irq_disabled_regs(regs)) {
unsigned long rst = search_kernel_restart_table(regs->nip);
if (rst)
regs_set_return_ip(regs, rst);
}
#endif

#ifdef CONFIG_PPC64
if (nmi_disables_ftrace(regs))
this_cpu_set_ftrace_enabled(state->ftrace_enabled);

Expand Down
12 changes: 1 addition & 11 deletions arch/powerpc/kernel/exceptions-64e.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,17 +342,7 @@ ret_from_mc_except:
#define PROLOG_ADDITION_MASKABLE_GEN(n) \
lbz r10,PACAIRQSOFTMASK(r13); /* are irqs soft-masked? */ \
andi. r10,r10,IRQS_DISABLED; /* yes -> go out of line */ \
bne masked_interrupt_book3e_##n; \
/* Kernel code below __end_soft_masked is implicitly masked */ \
andi. r10,r11,MSR_PR; \
bne 1f; /* user -> not masked */ \
std r14,PACA_EXGEN+EX_R14(r13); \
LOAD_REG_IMMEDIATE_SYM(r14, r10, __end_soft_masked); \
mfspr r10,SPRN_SRR0; \
cmpld r10,r14; \
ld r14,PACA_EXGEN+EX_R14(r13); \
blt masked_interrupt_book3e_##n; \
1:
bne masked_interrupt_book3e_##n

/*
* Additional regs must be re-loaded from paca before EXCEPTION_COMMON* is
Expand Down
2 changes: 1 addition & 1 deletion arch/powerpc/kernel/interrupt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ static inline bool exit_must_hard_disable(void)
#else
static inline bool exit_must_hard_disable(void)
{
return IS_ENABLED(CONFIG_PPC32);
return true;
}
#endif

Expand Down
16 changes: 14 additions & 2 deletions arch/powerpc/kernel/interrupt_64.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ _ASM_NOKPROBE_SYMBOL(system_call_vectored_emulate)
li r10,IRQS_ALL_DISABLED
stb r10,PACAIRQSOFTMASK(r13)
b system_call_vectored_common
#endif
#endif /* CONFIG_PPC_BOOK3S */

.balign IFETCH_ALIGN_BYTES
.globl system_call_common_real
Expand Down Expand Up @@ -320,10 +320,12 @@ END_BTB_FLUSH_SECTION
li r5,0 /* !scv */
bl syscall_exit_prepare
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Lsyscall_rst_start:
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- syscall_restart
#endif
li r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
li r11,0
Expand Down Expand Up @@ -396,6 +398,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
b .Lsyscall_restore_regs_cont
.Lsyscall_rst_end:

#ifdef CONFIG_PPC_BOOK3S
syscall_restart:
GET_PACA(r13)
ld r1,PACA_EXIT_SAVE_R1(r13)
Expand All @@ -409,6 +412,7 @@ syscall_restart:
b .Lsyscall_rst_start

RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
#endif

#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
tabort_syscall:
Expand Down Expand Up @@ -504,10 +508,12 @@ _ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
bne- .Lrestore_nvgprs_\srr
.Lrestore_nvgprs_\srr\()_cont:
std r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Linterrupt_return_\srr\()_user_rst_start:
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- interrupt_return_\srr\()_user_restart
#endif
li r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
li r11,0
Expand Down Expand Up @@ -590,6 +596,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
REST_NVGPRS(r1)
b .Lrestore_nvgprs_\srr\()_cont

#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_user_restart:
GET_PACA(r13)
ld r1,PACA_EXIT_SAVE_R1(r13)
Expand All @@ -602,6 +609,7 @@ interrupt_return_\srr\()_user_restart:
b .Linterrupt_return_\srr\()_user_rst_start

RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
#endif

.balign IFETCH_ALIGN_BYTES
.Lkernel_interrupt_return_\srr\():
Expand All @@ -615,9 +623,11 @@ RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr
cmpwi r11,IRQS_ENABLED
stb r11,PACAIRQSOFTMASK(r13)
bne 1f
#ifdef CONFIG_PPC_BOOK3S
lbz r11,PACAIRQHAPPENED(r13)
andi. r11,r11,(~PACA_IRQ_HARD_DIS)@l
bne- interrupt_return_\srr\()_kernel_restart
#endif
li r11,0
stb r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS
1:
Expand Down Expand Up @@ -717,6 +727,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
b . /* prevent speculative execution */
.Linterrupt_return_\srr\()_kernel_rst_end:

#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_kernel_restart:
GET_PACA(r13)
ld r1,PACA_EXIT_SAVE_R1(r13)
Expand All @@ -729,14 +740,15 @@ interrupt_return_\srr\()_kernel_restart:
b .Linterrupt_return_\srr\()_kernel_rst_start

RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
#endif

.endm

interrupt_return_macro srr
#ifdef CONFIG_PPC_BOOK3S
interrupt_return_macro hsrr
#endif /* CONFIG_PPC_BOOK3S */

.globl __end_soft_masked
__end_soft_masked:
DEFINE_FIXED_SYMBOL(__end_soft_masked)
#endif /* CONFIG_PPC_BOOK3S */

0 comments on commit 9b69d48

Please sign in to comment.