diff --git a/arch/riscv32/core/CMakeLists.txt b/arch/riscv32/core/CMakeLists.txt index 69c8e2693f46a..846c54c70ad83 100644 --- a/arch/riscv32/core/CMakeLists.txt +++ b/arch/riscv32/core/CMakeLists.txt @@ -3,9 +3,20 @@ zephyr_sources( fatal.c irq_manage.c irq_offload.c - isr.S prep_c.c reset.S - swap.S + thread_entry_wrapper.S thread.c ) + +# SMP-only sources +zephyr_sources_ifdef(CONFIG_SMP curr_cpu.c) +zephyr_sources_ifdef(CONFIG_SMP start_cpu.c) + +# Use SMP ISR if SMP is configured +zephyr_sources_ifndef(CONFIG_SMP isr.S) +zephyr_sources_ifdef(CONFIG_SMP isr_smp.S) + +# Use swap or switch depending on the USE_SWITCH config +zephyr_sources_ifndef(CONFIG_USE_SWITCH swap.S) +zephyr_sources_ifdef(CONFIG_USE_SWITCH switch.S) diff --git a/arch/riscv32/core/curr_cpu.c b/arch/riscv32/core/curr_cpu.c new file mode 100644 index 0000000000000..7d88a0e9ad90e --- /dev/null +++ b/arch/riscv32/core/curr_cpu.c @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +inline _cpu_t *_arch_curr_cpu(void) +{ + /* On hart init, the mscratch csr is set to the pointer for the + * _kernel.cpus[] struct for that hart */ + u32_t mscratch; + + __asm__ volatile("csrr %0, mscratch" : "=r" (mscratch)); + + return (_cpu_t *) mscratch; +} + diff --git a/arch/riscv32/core/fatal.c b/arch/riscv32/core/fatal.c index 078fbfa6ba1f0..a173fe3268b61 100644 --- a/arch/riscv32/core/fatal.c +++ b/arch/riscv32/core/fatal.c @@ -91,6 +91,7 @@ FUNC_NORETURN void _NanoFatalErrorHandler(unsigned int reason, break; } +#ifndef CONFIG_SMP printk("Current thread ID = %p\n" "Faulting instruction address = 0x%x\n" " ra: 0x%x gp: 0x%x tp: 0x%x t0: 0x%x\n" @@ -105,6 +106,24 @@ FUNC_NORETURN void _NanoFatalErrorHandler(unsigned int reason, esf->t5, esf->t6, esf->a0, esf->a1, esf->a2, esf->a3, esf->a4, esf->a5, esf->a6, esf->a7); +#else + printk("Hart: %d\n" + "Current thread ID = %p\n" + "Faulting instruction address = 0x%x\n" + " ra: 0x%x gp: 0x%x tp: 0x%x t0: 0x%x\n" + " t1: 0x%x t2: 0x%x t3: 0x%x t4: 0x%x\n" + " t5: 0x%x t6: 0x%x a0: 0x%x a1: 0x%x\n" + " a2: 0x%x a3: 0x%x a4: 0x%x a5: 0x%x\n" + " a6: 0x%x a7: 0x%x\n", + _current_cpu->id, + k_current_get(), + (esf->mepc == 0xdeadbaad) ? 0xdeadbaad : esf->mepc, + esf->ra, esf->gp, esf->tp, esf->t0, + esf->t1, esf->t2, esf->t3, esf->t4, + esf->t5, esf->t6, esf->a0, esf->a1, + esf->a2, esf->a3, esf->a4, esf->a5, + esf->a6, esf->a7); +#endif _SysFatalErrorHandler(reason, esf); /* spin forever */ diff --git a/arch/riscv32/core/isr_smp.S b/arch/riscv32/core/isr_smp.S new file mode 100644 index 0000000000000..565ac108a188c --- /dev/null +++ b/arch/riscv32/core/isr_smp.S @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +/* imports */ +GDATA(_sw_isr_table) +GTEXT(__soc_is_irq) +GTEXT(__soc_handle_irq) +GTEXT(_get_next_switch_handle) +GTEXT(_Fault) + +#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE +GTEXT(__soc_save_context) +GTEXT(__soc_restore_context) +#endif + +#ifdef CONFIG_TRACING +GTEXT(z_sys_trace_thread_switched_in) +GTEXT(z_sys_trace_isr_enter) +#endif + +#ifdef CONFIG_IRQ_OFFLOAD +GTEXT(_offload_routine) +#endif + +#ifdef CONFIG_TIMESLICING +GTEXT(_update_time_slice_before_swap) +#endif + +/* exports */ +GTEXT(__irq_wrapper) + +SECTION_FUNC(exception.entry, __irq_wrapper) + /* Allocate space on thread stack to save registers */ + addi sp, sp, -__NANO_ESF_SIZEOF + + /* + * Save caller and callee-saved registers onto the stack + */ + sw ra, __NANO_ESF_ra_OFFSET(sp) + sw gp, __NANO_ESF_gp_OFFSET(sp) + sw tp, __NANO_ESF_tp_OFFSET(sp) + sw t0, __NANO_ESF_t0_OFFSET(sp) + sw t1, __NANO_ESF_t1_OFFSET(sp) + sw t2, __NANO_ESF_t2_OFFSET(sp) + sw t3, __NANO_ESF_t3_OFFSET(sp) + sw t4, __NANO_ESF_t4_OFFSET(sp) + sw t5, __NANO_ESF_t5_OFFSET(sp) + sw t6, __NANO_ESF_t6_OFFSET(sp) + sw a0, __NANO_ESF_a0_OFFSET(sp) + sw a1, __NANO_ESF_a1_OFFSET(sp) + sw a2, __NANO_ESF_a2_OFFSET(sp) + sw a3, __NANO_ESF_a3_OFFSET(sp) + sw a4, __NANO_ESF_a4_OFFSET(sp) + sw a5, __NANO_ESF_a5_OFFSET(sp) + sw a6, __NANO_ESF_a6_OFFSET(sp) + sw a7, __NANO_ESF_a7_OFFSET(sp) + sw s0, __NANO_ESF_s0_OFFSET(sp) + sw s1, __NANO_ESF_s1_OFFSET(sp) + sw s2, __NANO_ESF_s2_OFFSET(sp) + sw s3, __NANO_ESF_s3_OFFSET(sp) + sw s4, __NANO_ESF_s4_OFFSET(sp) + sw s5, __NANO_ESF_s5_OFFSET(sp) + sw s6, __NANO_ESF_s6_OFFSET(sp) + sw s7, __NANO_ESF_s7_OFFSET(sp) + sw s8, __NANO_ESF_s8_OFFSET(sp) + sw s9, __NANO_ESF_s9_OFFSET(sp) + sw s10, __NANO_ESF_s10_OFFSET(sp) + sw s11, __NANO_ESF_s11_OFFSET(sp) + +#ifdef CONFIG_EXECUTION_BENCHMARKING + call read_timer_start_of_isr +#endif + /* Save MEPC register */ + csrr t0, mepc + sw t0, __NANO_ESF_mepc_OFFSET(sp) + + /* Save SOC-specific MSTATUS register */ + csrr t0, SOC_MSTATUS_REG + sw t0, __NANO_ESF_mstatus_OFFSET(sp) + +#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE + /* Handle context saving at SOC level. */ + jal ra, __soc_save_context +#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */ + +switch_to_interrupt_stack: + /* Get cpu context */ + csrr t0, mscratch + + /* Increment nested interrupt count */ + lw t1, _cpu_offset_to_nested(t0) + addi t2, t1, 1 + sw t2, _cpu_offset_to_nested(t0) + + /* Save thread stack pointer to temp register t2 */ + mv t2, sp + + /* If the nested count was nonzero, we're already on the + * interrupt stack */ + bnez t1, on_interrupt_stack + + /* Switch to interrupt stack */ + lw sp, _cpu_offset_to_irq_stack(t0) + +on_interrupt_stack: + /* + * Save thread stack pointer on interrupt stack + * In RISC-V, stack pointer needs to be 16-byte aligned + */ + addi sp, sp, -16 + sw t2, 0x00(sp) + + /* + * Check if exception is the result of an interrupt or not. + * (SOC dependent). Following the RISC-V architecture spec, the MSB + * of the mcause register is used to indicate whether an exception + * is the result of an interrupt or an exception/fault. But for some + * SOCs (like pulpino or riscv-qemu), the MSB is never set to indicate + * interrupt. Hence, check for interrupt/exception via the __soc_is_irq + * function (that needs to be implemented by each SOC). The result is + * returned via register a0 (1: interrupt, 0 exception) + */ + jal ra, __soc_is_irq + bnez a0, is_interrupt + + /* + * If the exception is the result of an ECALL, check whether to + * perform a context-switch or an IRQ offload. Otherwise call _Fault + * to report the exception. + */ + csrr t0, mcause + li t2, SOC_MCAUSE_EXP_MASK + and t0, t0, t2 + + /* + * If mcause == SOC_MCAUSE_ECALL_EXP, handle system call, + * otherwise handle fault + */ + li t1, SOC_MCAUSE_ECALL_EXP + beq t0, t1, is_syscall + + /* + * Call _Fault to handle exception. + * Load the stored stack frame off of the interrupt stack and pass it + * to _Fault (via register a0). + * + * If _Fault returns, set return address to leave_interrupt_stack + * to restore stack. + */ + lw a0, 0x00(sp) + la ra, leave_interrupt_stack + tail _Fault + +is_syscall: + /* + * A syscall is the result of an ecall instruction, in which case the + * MEPC will contain the address of the ecall instruction. + * Increment saved MEPC by 4 to prevent triggering the same ecall + * again upon exiting the ISR. + * + * It's safe to always increment by 4, even with compressed + * instructions, because the ecall instruction is always 4 bytes. + */ + lw t0, __NANO_ESF_mepc_OFFSET(sp) + addi t0, t0, 4 + sw t0, __NANO_ESF_mepc_OFFSET(sp) + + j leave_interrupt_stack + +is_interrupt: +#ifdef CONFIG_TRACING + call z_sys_trace_isr_enter +#endif + + /* Get IRQ causing interrupt */ + csrr a0, mcause + li t0, SOC_MCAUSE_EXP_MASK + and a0, a0, t0 + + /* + * Clear pending IRQ generating the interrupt at SOC level + * Pass IRQ number to __soc_handle_irq via register a0 + */ + jal ra, __soc_handle_irq + + /* + * Load corresponding registered function in _sw_isr_table. + * (table is 8-bytes wide, we should shift index by 3) + */ + la t0, _sw_isr_table + slli a0, a0, 3 + add t0, t0, a0 + + /* Load argument in a0 register */ + lw a0, 0x00(t0) + + /* Load ISR function address in register t1 */ + lw t1, 0x04(t0) + + /* Call ISR function */ + jalr ra, t1 + +#ifdef CONFIG_EXECUTION_BENCHMARKING + call read_timer_end_of_isr +#endif + +leave_interrupt_stack: + /* Get cpu context */ + csrr t0, mscratch + + /* Decrement nested count */ + lw t1, _cpu_offset_to_nested(t0) + addi t1, t1, -1 + sw t1, _cpu_offset_to_nested(t0) + + /* Pop saved thread stack pointer off of the interrupt stack and into + * the argument to _get_next_switch_handle */ + lw a0, 0x00(sp) + addi sp, sp, 16 + + /* Ask the scheduler for the next thread to run */ + call _get_next_switch_handle + + /* Switch to the new stack */ + mv sp, a0 + +restore_next_thread: +#ifdef CONFIG_RISCV_SOC_CONTEXT_SAVE + /* Handle context saving at SOC level. */ + jal ra, __soc_save_context +#endif /* CONFIG_RISCV_SOC_CONTEXT_SAVE */ + + /* Restore MEPC register */ + lw t0, __NANO_ESF_mepc_OFFSET(sp) + csrw mepc, t0 + + /* Restore SOC-specific MSTATUS register */ + lw t0, __NANO_ESF_mstatus_OFFSET(sp) + csrw SOC_MSTATUS_REG, t0 + + /* Load saved registers */ + lw ra, __NANO_ESF_ra_OFFSET(sp) + lw gp, __NANO_ESF_gp_OFFSET(sp) + lw tp, __NANO_ESF_tp_OFFSET(sp) + lw t0, __NANO_ESF_t0_OFFSET(sp) + lw t1, __NANO_ESF_t1_OFFSET(sp) + lw t2, __NANO_ESF_t2_OFFSET(sp) + lw t3, __NANO_ESF_t3_OFFSET(sp) + lw t4, __NANO_ESF_t4_OFFSET(sp) + lw t5, __NANO_ESF_t5_OFFSET(sp) + lw t6, __NANO_ESF_t6_OFFSET(sp) + lw a0, __NANO_ESF_a0_OFFSET(sp) + lw a1, __NANO_ESF_a1_OFFSET(sp) + lw a2, __NANO_ESF_a2_OFFSET(sp) + lw a3, __NANO_ESF_a3_OFFSET(sp) + lw a4, __NANO_ESF_a4_OFFSET(sp) + lw a5, __NANO_ESF_a5_OFFSET(sp) + lw a6, __NANO_ESF_a6_OFFSET(sp) + lw a7, __NANO_ESF_a7_OFFSET(sp) + lw s0, __NANO_ESF_s0_OFFSET(sp) + lw s1, __NANO_ESF_s1_OFFSET(sp) + lw s2, __NANO_ESF_s2_OFFSET(sp) + lw s3, __NANO_ESF_s3_OFFSET(sp) + lw s4, __NANO_ESF_s4_OFFSET(sp) + lw s5, __NANO_ESF_s5_OFFSET(sp) + lw s6, __NANO_ESF_s6_OFFSET(sp) + lw s7, __NANO_ESF_s7_OFFSET(sp) + lw s8, __NANO_ESF_s8_OFFSET(sp) + lw s9, __NANO_ESF_s9_OFFSET(sp) + lw s10, __NANO_ESF_s10_OFFSET(sp) + lw s11, __NANO_ESF_s11_OFFSET(sp) + + /* Free stack space */ + addi sp, sp, __NANO_ESF_SIZEOF + + /* Return from interrupt */ + SOC_ERET + diff --git a/arch/riscv32/core/offsets/offsets.c b/arch/riscv32/core/offsets/offsets.c index eb6ba821cd217..9254d5da41db5 100644 --- a/arch/riscv32/core/offsets/offsets.c +++ b/arch/riscv32/core/offsets/offsets.c @@ -54,6 +54,20 @@ GEN_OFFSET_SYM(NANO_ESF, a4); GEN_OFFSET_SYM(NANO_ESF, a5); GEN_OFFSET_SYM(NANO_ESF, a6); GEN_OFFSET_SYM(NANO_ESF, a7); +#ifdef CONFIG_USE_SWITCH +GEN_OFFSET_SYM(NANO_ESF, s0); +GEN_OFFSET_SYM(NANO_ESF, s1); +GEN_OFFSET_SYM(NANO_ESF, s2); +GEN_OFFSET_SYM(NANO_ESF, s3); +GEN_OFFSET_SYM(NANO_ESF, s4); +GEN_OFFSET_SYM(NANO_ESF, s5); +GEN_OFFSET_SYM(NANO_ESF, s6); +GEN_OFFSET_SYM(NANO_ESF, s7); +GEN_OFFSET_SYM(NANO_ESF, s8); +GEN_OFFSET_SYM(NANO_ESF, s9); +GEN_OFFSET_SYM(NANO_ESF, s10); +GEN_OFFSET_SYM(NANO_ESF, s11); +#endif GEN_OFFSET_SYM(NANO_ESF, mepc); GEN_OFFSET_SYM(NANO_ESF, mstatus); diff --git a/arch/riscv32/core/start_cpu.c b/arch/riscv32/core/start_cpu.c new file mode 100644 index 0000000000000..3a24ffddd8345 --- /dev/null +++ b/arch/riscv32/core/start_cpu.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include + +#ifdef CONFIG_SMP + +u32_t __init_riscv_smp_mscratch[CONFIG_MP_NUM_CPUS]; +u32_t __init_riscv_smp_go[CONFIG_MP_NUM_CPUS]; +u32_t __init_riscv_smp_stacks[CONFIG_MP_NUM_CPUS]; +u32_t __init_riscv_smp_entry[CONFIG_MP_NUM_CPUS]; +u32_t __init_riscv_smp_keys[CONFIG_MP_NUM_CPUS]; +u32_t __init_riscv_smp_start_flags[CONFIG_MP_NUM_CPUS]; + +void _arch_start_cpu(int cpu_num, k_thread_stack_t *stack, int sz, + void (*fn)(int, void *), void *arg) +{ + /* Set up data */ + __init_riscv_smp_mscratch[cpu_num] = (u32_t) &(_kernel.cpus[cpu_num]); + __init_riscv_smp_stacks[cpu_num] = (u32_t) stack + sz; + __init_riscv_smp_entry[cpu_num] = (u32_t) fn; + __init_riscv_smp_keys[cpu_num] = (u32_t) 0; /* TODO */ + __init_riscv_smp_start_flags[cpu_num] = (u32_t) arg; + + /* Push the go button */ + __init_riscv_smp_go[cpu_num] = 1; +} + +#endif /* CONFIG_SMP */ + diff --git a/arch/riscv32/core/swap.S b/arch/riscv32/core/swap.S index ed18cd6d6fe2a..0802aece5129d 100644 --- a/arch/riscv32/core/swap.S +++ b/arch/riscv32/core/swap.S @@ -10,7 +10,6 @@ /* exports */ GTEXT(__swap) -GTEXT(_thread_entry_wrapper) /* Use ABI name of registers for the sake of simplicity */ @@ -101,20 +100,3 @@ SECTION_FUNC(exception.other, __swap) /* Return */ jalr x0, ra - -/* - * void _thread_entry_wrapper(k_thread_entry_t, void *, void *, void *) - */ -SECTION_FUNC(TEXT, _thread_entry_wrapper) - /* - * _thread_entry_wrapper is called for every new thread upon the return - * of __swap or ISR. Its address, as well as its input function - * arguments thread_entry_t, void *, void *, void * are restored from - * the thread stack (initialized via function _thread). - * In this case, thread_entry_t, * void *, void * and void * are stored - * in registers a0, a1, a2 and a3. These registers are used as arguments - * to function _thread_entry. Hence, just call _thread_entry with - * return address set to 0 to indicate a non-returning function call. - */ - - jal x0, _thread_entry diff --git a/arch/riscv32/core/switch.S b/arch/riscv32/core/switch.S new file mode 100644 index 0000000000000..0acf7991fe5b0 --- /dev/null +++ b/arch/riscv32/core/switch.S @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +/* exports */ +GTEXT(_riscv_switch) + +/* Use ABI name of registers for the sake of simplicity */ + +/* + * void _riscv_switch(void *switch_to, void **switch_from) + */ +SECTION_FUNC(exception.other, _riscv_switch) + /* Allocate space on thread stack to save registers */ + addi sp, sp, -__NANO_ESF_SIZEOF + + /* + * Save caller-saved registers on current thread stack. + * NOTE: need to be updated to account for floating-point registers + * floating-point registers should be accounted for when corresponding + * config variable is set + */ + sw ra, __NANO_ESF_ra_OFFSET(sp) + sw gp, __NANO_ESF_gp_OFFSET(sp) + sw tp, __NANO_ESF_tp_OFFSET(sp) + sw t0, __NANO_ESF_t0_OFFSET(sp) + sw t1, __NANO_ESF_t1_OFFSET(sp) + sw t2, __NANO_ESF_t2_OFFSET(sp) + sw t3, __NANO_ESF_t3_OFFSET(sp) + sw t4, __NANO_ESF_t4_OFFSET(sp) + sw t5, __NANO_ESF_t5_OFFSET(sp) + sw t6, __NANO_ESF_t6_OFFSET(sp) + sw a0, __NANO_ESF_a0_OFFSET(sp) + sw a1, __NANO_ESF_a1_OFFSET(sp) + sw a2, __NANO_ESF_a2_OFFSET(sp) + sw a3, __NANO_ESF_a3_OFFSET(sp) + sw a4, __NANO_ESF_a4_OFFSET(sp) + sw a5, __NANO_ESF_a5_OFFSET(sp) + sw a6, __NANO_ESF_a6_OFFSET(sp) + sw a7, __NANO_ESF_a7_OFFSET(sp) + sw s0, __NANO_ESF_s0_OFFSET(sp) + sw s1, __NANO_ESF_s1_OFFSET(sp) + sw s2, __NANO_ESF_s2_OFFSET(sp) + sw s3, __NANO_ESF_s3_OFFSET(sp) + sw s4, __NANO_ESF_s4_OFFSET(sp) + sw s5, __NANO_ESF_s5_OFFSET(sp) + sw s6, __NANO_ESF_s6_OFFSET(sp) + sw s7, __NANO_ESF_s7_OFFSET(sp) + sw s8, __NANO_ESF_s8_OFFSET(sp) + sw s9, __NANO_ESF_s9_OFFSET(sp) + sw s10, __NANO_ESF_s10_OFFSET(sp) + sw s11, __NANO_ESF_s11_OFFSET(sp) + + /* Set MEPC register to return from _riscv_switch */ + la t0, leave_riscv_switch + sw t0, __NANO_ESF_mepc_OFFSET(sp) + + /* + * Emulate the behavior of the MSTATUS register when an interrupt is taken + * by setting the MPIE bit if the MIE bit is set and clearing the MIE bit, + * then setting MPP to the previous privilege mode (machine mode) + */ + csrr t0, SOC_MSTATUS_REG + + /* Extract the interrupt enable bit */ + li t1, SOC_MSTATUS_IEN + and t1, t0, t1 + + /* Set the MPIE bit if the IEN bit was set */ + beqz t1, switch_no_set_mpie + li t1, SOC_MSTATUS_MPIE + or t0, t0, t1 + +switch_no_set_mpie: + /* Clear the MIE bit */ + li t1, SOC_MSTATUS_IEN + not t1, t1 + and t0, t0, t1 + + /* Set the previous privilege mode to M-mode */ + li t1, SOC_MSTATUS_MPP_M_MODE + or t0, t0, t1 + + /* Save the new value of MSTATUS onto the stack frame */ + sw t0, __NANO_ESF_mstatus_OFFSET(sp) + + /* Store stack pointer in switch_from */ + sw sp, 0(a1) + + /* Load stack pointer from switch_to */ + mv sp, a0 + + /* Restore MEPC register */ + lw t0, __NANO_ESF_mepc_OFFSET(sp) + csrw mepc, t0 + + /* Restore SOC-specific MSTATUS register */ + lw t0, __NANO_ESF_mstatus_OFFSET(sp) + csrw SOC_MSTATUS_REG, t0 + + /* Load caller-saved registers */ + lw ra, __NANO_ESF_ra_OFFSET(sp) + lw gp, __NANO_ESF_gp_OFFSET(sp) + lw tp, __NANO_ESF_tp_OFFSET(sp) + lw t0, __NANO_ESF_t0_OFFSET(sp) + lw t1, __NANO_ESF_t1_OFFSET(sp) + lw t2, __NANO_ESF_t2_OFFSET(sp) + lw t3, __NANO_ESF_t3_OFFSET(sp) + lw t4, __NANO_ESF_t4_OFFSET(sp) + lw t5, __NANO_ESF_t5_OFFSET(sp) + lw t6, __NANO_ESF_t6_OFFSET(sp) + lw a0, __NANO_ESF_a0_OFFSET(sp) + lw a1, __NANO_ESF_a1_OFFSET(sp) + lw a2, __NANO_ESF_a2_OFFSET(sp) + lw a3, __NANO_ESF_a3_OFFSET(sp) + lw a4, __NANO_ESF_a4_OFFSET(sp) + lw a5, __NANO_ESF_a5_OFFSET(sp) + lw a6, __NANO_ESF_a6_OFFSET(sp) + lw a7, __NANO_ESF_a7_OFFSET(sp) + lw s0, __NANO_ESF_s0_OFFSET(sp) + lw s1, __NANO_ESF_s1_OFFSET(sp) + lw s2, __NANO_ESF_s2_OFFSET(sp) + lw s3, __NANO_ESF_s3_OFFSET(sp) + lw s4, __NANO_ESF_s4_OFFSET(sp) + lw s5, __NANO_ESF_s5_OFFSET(sp) + lw s6, __NANO_ESF_s6_OFFSET(sp) + lw s7, __NANO_ESF_s7_OFFSET(sp) + lw s8, __NANO_ESF_s8_OFFSET(sp) + lw s9, __NANO_ESF_s9_OFFSET(sp) + lw s10, __NANO_ESF_s10_OFFSET(sp) + lw s11, __NANO_ESF_s11_OFFSET(sp) + + /* Free allocated stack space */ + addi sp, sp, __NANO_ESF_SIZEOF + + SOC_ERET + +leave_riscv_switch: + ret + diff --git a/arch/riscv32/core/thread.c b/arch/riscv32/core/thread.c index 2fd3357ad00db..ec47e768f205d 100644 --- a/arch/riscv32/core/thread.c +++ b/arch/riscv32/core/thread.c @@ -63,5 +63,9 @@ void _new_thread(struct k_thread *thread, k_thread_stack_t *stack, stack_init->mstatus = SOC_MSTATUS_DEF_RESTORE; stack_init->mepc = (u32_t)_thread_entry_wrapper; +#ifdef CONFIG_USE_SWITCH + thread->switch_handle = (void *)stack_init; +#else thread->callee_saved.sp = (u32_t)stack_init; +#endif /* CONFIG_USE_SWITCH */ } diff --git a/arch/riscv32/core/thread_entry_wrapper.S b/arch/riscv32/core/thread_entry_wrapper.S new file mode 100644 index 0000000000000..02c0656e1369d --- /dev/null +++ b/arch/riscv32/core/thread_entry_wrapper.S @@ -0,0 +1,30 @@ + +/* + * Copyright (c) 2016 Jean-Paul Etienne + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +/* exports */ +GTEXT(_thread_entry_wrapper) + +/* + * void _thread_entry_wrapper(k_thread_entry_t, void *, void *, void *) + */ +SECTION_FUNC(TEXT, _thread_entry_wrapper) + /* + * _thread_entry_wrapper is called for every new thread upon the return + * of __swap or ISR. Its address, as well as its input function + * arguments thread_entry_t, void *, void *, void * are restored from + * the thread stack (initialized via function _thread). + * In this case, thread_entry_t, * void *, void * and void * are stored + * in registers a0, a1, a2 and a3. These registers are used as arguments + * to function _thread_entry. Hence, just call _thread_entry with + * return address set to 0 to indicate a non-returning function call. + */ + + jal x0, _thread_entry diff --git a/arch/riscv32/include/kernel_arch_func.h b/arch/riscv32/include/kernel_arch_func.h index 50f1fc2a8d369..eb00f555c15d8 100644 --- a/arch/riscv32/include/kernel_arch_func.h +++ b/arch/riscv32/include/kernel_arch_func.h @@ -16,6 +16,7 @@ #define ZEPHYR_ARCH_RISCV32_INCLUDE_KERNEL_ARCH_FUNC_H_ #include +#include #ifdef __cplusplus extern "C" { @@ -24,18 +25,36 @@ extern "C" { #ifndef _ASMLANGUAGE void k_cpu_idle(void); void k_cpu_atomic_idle(unsigned int key); +_cpu_t *_arch_curr_cpu(void); static ALWAYS_INLINE void kernel_arch_init(void) { +#ifdef CONFIG_SMP + _kernel.cpus[0].irq_stack = + K_THREAD_STACK_BUFFER(_interrupt_stack) + CONFIG_ISR_STACK_SIZE; + + /* Init pointer to _kernel.cpus[0] in mscratch for hart 0 */ + u32_t cput = (u32_t) &(_kernel.cpus[0]); + __asm__ volatile("csrw mscratch, %[cput]" :: [cput] "r" (cput)); +#else _kernel.irq_stack = K_THREAD_STACK_BUFFER(_interrupt_stack) + CONFIG_ISR_STACK_SIZE; +#endif } +#ifndef CONFIG_USE_SWITCH +/* If we're using new-style _arch_switch for context switching, this function + * is no longer architecture-specific */ static ALWAYS_INLINE void _set_thread_return_value(struct k_thread *thread, unsigned int value) { thread->arch.swap_return_value = value; } +# else +/* Point _arch_switch at our architecture-specific switch function */ +extern void _riscv_switch(void *switch_to, void **switch_from); +#define _arch_switch _riscv_switch +#endif /* CONFIG_USE_SWITCH */ static inline void _IntLibInit(void) { @@ -47,8 +66,11 @@ static inline void _IntLibInit(void) FUNC_NORETURN void _NanoFatalErrorHandler(unsigned int reason, const NANO_ESF *esf); - -#define _is_in_isr() (_kernel.nested != 0U) +#ifdef CONFIG_SMP +#define _is_in_isr() (_arch_curr_cpu()->nested != 0) +#else +#define _is_in_isr() (_kernel.nested != 0) +#endif #ifdef CONFIG_IRQ_OFFLOAD int _irq_do_offload(void); diff --git a/boards/riscv32/sifive-multicore/CMakeLists.txt b/boards/riscv32/sifive-multicore/CMakeLists.txt new file mode 100644 index 0000000000000..d78027f97558c --- /dev/null +++ b/boards/riscv32/sifive-multicore/CMakeLists.txt @@ -0,0 +1,4 @@ +zephyr_library() +zephyr_library_sources(pinmux.c) +zephyr_library_sources(clock.c) +zephyr_library_include_directories(${ZEPHYR_BASE}/drivers) diff --git a/boards/riscv32/sifive-multicore/Kconfig.board b/boards/riscv32/sifive-multicore/Kconfig.board new file mode 100644 index 0000000000000..5e2389bc1a338 --- /dev/null +++ b/boards/riscv32/sifive-multicore/Kconfig.board @@ -0,0 +1,4 @@ +config BOARD_SIFIVE_MULTICORE + bool "Multicore Generic Sifive Target" + depends on SOC_RISCV32_SIFIVE_FREEDOM + select HAS_DTS diff --git a/boards/riscv32/sifive-multicore/Kconfig.defconfig b/boards/riscv32/sifive-multicore/Kconfig.defconfig new file mode 100644 index 0000000000000..b6d93d4fa66d9 --- /dev/null +++ b/boards/riscv32/sifive-multicore/Kconfig.defconfig @@ -0,0 +1,6 @@ +if BOARD_SIFIVE_MULTICORE + +config BOARD + default "sifive-multicore" + +endif diff --git a/boards/riscv32/sifive-multicore/board.cmake b/boards/riscv32/sifive-multicore/board.cmake new file mode 100644 index 0000000000000..9cb1cd7514c07 --- /dev/null +++ b/boards/riscv32/sifive-multicore/board.cmake @@ -0,0 +1,11 @@ +set(EMU_PLATFORM qemu) + +set(QEMU_CPU_TYPE_${ARCH} riscv32) + +set(QEMU_FLAGS_${ARCH} + -nographic + -machine sifive_e + -smp ${CONFIG_MP_NUM_CPUS} + ) + +set(BOARD_DEBUG_RUNNER qemu) diff --git a/boards/riscv32/sifive-multicore/board.h b/boards/riscv32/sifive-multicore/board.h new file mode 100644 index 0000000000000..7f321e7635210 --- /dev/null +++ b/boards/riscv32/sifive-multicore/board.h @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2017 Jean-Paul Etienne + * Copyright (c) 2017 Palmer Dabbelt + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef __INC_BOARD_H +#define __INC_BOARD_H + +#include + +#endif /* __INC_BOARD_H */ diff --git a/boards/riscv32/sifive-multicore/clock.c b/boards/riscv32/sifive-multicore/clock.c new file mode 100644 index 0000000000000..a7f3edacb0edb --- /dev/null +++ b/boards/riscv32/sifive-multicore/clock.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 Jean-Paul Etienne + * Copyright (c) 2017 Palmer Dabbelt + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include "prci.h" + +/* Selects the 16MHz oscilator on the HiFive1 board, which provides a clock + * that's accurate enough to actually drive serial ports off of. + */ +static int sifive_clock_init(struct device *dev) +{ + ARG_UNUSED(dev); + + PRCI_REG(PRCI_PLLCFG) = PLL_REFSEL(1) | PLL_BYPASS(1); + PRCI_REG(PRCI_PLLDIV) = (PLL_FINAL_DIV_BY_1(1) | PLL_FINAL_DIV(0)); + PRCI_REG(PRCI_PLLCFG) |= PLL_SEL(1); + PRCI_REG(PRCI_HFROSCCFG) &= ~ROSC_EN(1); + return 0; +} + +SYS_INIT(sifive_clock_init, PRE_KERNEL_1, CONFIG_PINMUX_INIT_PRIORITY); diff --git a/boards/riscv32/sifive-multicore/dts_fixup.h b/boards/riscv32/sifive-multicore/dts_fixup.h new file mode 100644 index 0000000000000..a800e2c28a2a6 --- /dev/null +++ b/boards/riscv32/sifive-multicore/dts_fixup.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +/* GPIO 0 */ +#define CONFIG_SIFIVE_GPIO_0_BASE_ADDR SIFIVE_GPIO0_10012000_BASE_ADDRESS +#define CONFIG_SIFIVE_GPIO_0_IRQ_0 SIFIVE_GPIO0_10012000_IRQ_0 +#define CONFIG_SIFIVE_GPIO_0_IRQ_1 SIFIVE_GPIO0_10012000_IRQ_1 +#define CONFIG_SIFIVE_GPIO_0_IRQ_2 SIFIVE_GPIO0_10012000_IRQ_2 +#define CONFIG_SIFIVE_GPIO_0_IRQ_3 SIFIVE_GPIO0_10012000_IRQ_3 +#define CONFIG_SIFIVE_GPIO_0_IRQ_4 SIFIVE_GPIO0_10012000_IRQ_4 +#define CONFIG_SIFIVE_GPIO_0_IRQ_5 SIFIVE_GPIO0_10012000_IRQ_5 +#define CONFIG_SIFIVE_GPIO_0_IRQ_6 SIFIVE_GPIO0_10012000_IRQ_6 +#define CONFIG_SIFIVE_GPIO_0_IRQ_7 SIFIVE_GPIO0_10012000_IRQ_7 +#define CONFIG_SIFIVE_GPIO_0_IRQ_8 SIFIVE_GPIO0_10012000_IRQ_8 +#define CONFIG_SIFIVE_GPIO_0_IRQ_9 SIFIVE_GPIO0_10012000_IRQ_9 +#define CONFIG_SIFIVE_GPIO_0_IRQ_10 SIFIVE_GPIO0_10012000_IRQ_10 +#define CONFIG_SIFIVE_GPIO_0_IRQ_11 SIFIVE_GPIO0_10012000_IRQ_11 +#define CONFIG_SIFIVE_GPIO_0_IRQ_12 SIFIVE_GPIO0_10012000_IRQ_12 +#define CONFIG_SIFIVE_GPIO_0_IRQ_13 SIFIVE_GPIO0_10012000_IRQ_13 +#define CONFIG_SIFIVE_GPIO_0_IRQ_14 SIFIVE_GPIO0_10012000_IRQ_14 +#define CONFIG_SIFIVE_GPIO_0_IRQ_15 SIFIVE_GPIO0_10012000_IRQ_15 +#define CONFIG_SIFIVE_GPIO_0_IRQ_16 SIFIVE_GPIO0_10012000_IRQ_16 +#define CONFIG_SIFIVE_GPIO_0_IRQ_17 SIFIVE_GPIO0_10012000_IRQ_17 +#define CONFIG_SIFIVE_GPIO_0_IRQ_18 SIFIVE_GPIO0_10012000_IRQ_18 +#define CONFIG_SIFIVE_GPIO_0_IRQ_19 SIFIVE_GPIO0_10012000_IRQ_19 +#define CONFIG_SIFIVE_GPIO_0_IRQ_20 SIFIVE_GPIO0_10012000_IRQ_20 +#define CONFIG_SIFIVE_GPIO_0_IRQ_21 SIFIVE_GPIO0_10012000_IRQ_21 +#define CONFIG_SIFIVE_GPIO_0_IRQ_22 SIFIVE_GPIO0_10012000_IRQ_22 +#define CONFIG_SIFIVE_GPIO_0_IRQ_23 SIFIVE_GPIO0_10012000_IRQ_23 +#define CONFIG_SIFIVE_GPIO_0_IRQ_24 SIFIVE_GPIO0_10012000_IRQ_24 +#define CONFIG_SIFIVE_GPIO_0_IRQ_25 SIFIVE_GPIO0_10012000_IRQ_25 +#define CONFIG_SIFIVE_GPIO_0_IRQ_26 SIFIVE_GPIO0_10012000_IRQ_26 +#define CONFIG_SIFIVE_GPIO_0_IRQ_27 SIFIVE_GPIO0_10012000_IRQ_27 +#define CONFIG_SIFIVE_GPIO_0_IRQ_28 SIFIVE_GPIO0_10012000_IRQ_28 +#define CONFIG_SIFIVE_GPIO_0_IRQ_29 SIFIVE_GPIO0_10012000_IRQ_29 +#define CONFIG_SIFIVE_GPIO_0_IRQ_30 SIFIVE_GPIO0_10012000_IRQ_30 +#define CONFIG_SIFIVE_GPIO_0_IRQ_31 SIFIVE_GPIO0_10012000_IRQ_31 +#define CONFIG_SIFIVE_GPIO_0_SIZE SIFIVE_GPIO0_10012000_SIZE + +/* UART 0 */ +#define CONFIG_SIFIVE_UART_0_BASE_ADDR SIFIVE_UART0_10013000_BASE_ADDRESS +#define CONFIG_SIFIVE_UART_0_CURRENT_SPEED SIFIVE_UART0_10013000_CURRENT_SPEED +#define CONFIG_SIFIVE_UART_0_IRQ_0 SIFIVE_UART0_10013000_IRQ_0 +#define CONFIG_SIFIVE_UART_0_LABEL SIFIVE_UART0_10013000_LABEL +#define CONFIG_SIFIVE_UART_0_SIZE SIFIVE_UART0_10013000_SIZE + +/* UART 1 */ +#define CONFIG_SIFIVE_UART_1_BASE_ADDR SIFIVE_UART0_10023000_BASE_ADDRESS +#define CONFIG_SIFIVE_UART_1_CURRENT_SPEED SIFIVE_UART0_10023000_CURRENT_SPEED +#define CONFIG_SIFIVE_UART_1_IRQ_0 SIFIVE_UART0_10023000_IRQ_0 +#define CONFIG_SIFIVE_UART_1_SIZE SIFIVE_UART0_10023000_SIZE + diff --git a/boards/riscv32/sifive-multicore/pinmux.c b/boards/riscv32/sifive-multicore/pinmux.c new file mode 100644 index 0000000000000..22f0c41b57e23 --- /dev/null +++ b/boards/riscv32/sifive-multicore/pinmux.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2017 Jean-Paul Etienne + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +static int sifive_pinmux_init(struct device *dev) +{ + ARG_UNUSED(dev); + + struct device *p = device_get_binding(CONFIG_PINMUX_SIFIVE_0_NAME); + + /* UART0 RX */ + pinmux_pin_set(p, 16, SIFIVE_PINMUX_IOF0); + + /* UART0 TX */ + pinmux_pin_set(p, 17, SIFIVE_PINMUX_IOF0); + + return 0; +} + +SYS_INIT(sifive_pinmux_init, PRE_KERNEL_1, CONFIG_PINMUX_INIT_PRIORITY); diff --git a/boards/riscv32/sifive-multicore/prci.h b/boards/riscv32/sifive-multicore/prci.h new file mode 100644 index 0000000000000..d180d028cfbbd --- /dev/null +++ b/boards/riscv32/sifive-multicore/prci.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 SiFive Inc + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _SIFIVE_PRCI_H +#define _SIFIVE_PRCI_H + +#define _REG32(p, i) (*(volatile uint32_t *) ((p) + (i))) +#define PRCI_REG(offset) _REG32(PRCI_BASE_ADDR, offset) + +/* Register offsets */ + +#define PRCI_HFROSCCFG (0x0000) +#define PRCI_HFXOSCCFG (0x0004) +#define PRCI_PLLCFG (0x0008) +#define PRCI_PLLDIV (0x000C) +#define PRCI_PROCMONCFG (0x00F0) + +/* Fields */ +#define ROSC_DIV(x) (((x) & 0x2F) << 0) +#define ROSC_TRIM(x) (((x) & 0x1F) << 16) +#define ROSC_EN(x) (((x) & 0x1) << 30) +#define ROSC_RDY(x) (((x) & 0x1) << 31) + +#define XOSC_EN(x) (((x) & 0x1) << 30) +#define XOSC_RDY(x) (((x) & 0x1) << 31) + +#define PLL_R(x) (((x) & 0x7) << 0) +/* single reserved bit for F LSB. */ +#define PLL_F(x) (((x) & 0x3F) << 4) +#define PLL_Q(x) (((x) & 0x3) << 10) +#define PLL_SEL(x) (((x) & 0x1) << 16) +#define PLL_REFSEL(x) (((x) & 0x1) << 17) +#define PLL_BYPASS(x) (((x) & 0x1) << 18) +#define PLL_LOCK(x) (((x) & 0x1) << 31) + +#define PLL_R_default 0x1 +#define PLL_F_default 0x1F +#define PLL_Q_default 0x3 + +#define PLL_REFSEL_HFROSC 0x0 +#define PLL_REFSEL_HFXOSC 0x1 + +#define PLL_SEL_HFROSC 0x0 +#define PLL_SEL_PLL 0x1 + +#define PLL_FINAL_DIV(x) (((x) & 0x3F) << 0) +#define PLL_FINAL_DIV_BY_1(x) (((x) & 0x1) << 8) + +#define PROCMON_DIV(x) (((x) & 0x1F) << 0) +#define PROCMON_TRIM(x) (((x) & 0x1F) << 8) +#define PROCMON_EN(x) (((x) & 0x1) << 16) +#define PROCMON_SEL(x) (((x) & 0x3) << 24) +#define PROCMON_NT_EN(x) (((x) & 0x1) << 28) + +#define PROCMON_SEL_HFCLK 0 +#define PROCMON_SEL_HFXOSCIN 1 +#define PROCMON_SEL_PLLOUTDIV 2 +#define PROCMON_SEL_PROCMON 3 + +#endif /* _SIFIVE_PRCI_H */ diff --git a/boards/riscv32/sifive-multicore/sifive-multicore.dts b/boards/riscv32/sifive-multicore/sifive-multicore.dts new file mode 100644 index 0000000000000..51be7403dca97 --- /dev/null +++ b/boards/riscv32/sifive-multicore/sifive-multicore.dts @@ -0,0 +1,26 @@ +/dts-v1/; + +#include + +/ { + model = "SiFive Multicore"; + compatible = "sifive,rocket0"; + chosen { + zephyr,console = &uart0; + }; +}; + +&gpio0 { + status = "ok"; +}; + +&uart0 { + status = "ok"; + current-speed = <115200>; + clock-frequency = <16000000>; +}; + +&spi0 { + status = "ok"; +}; + diff --git a/boards/riscv32/sifive-multicore/sifive-multicore.yaml b/boards/riscv32/sifive-multicore/sifive-multicore.yaml new file mode 100644 index 0000000000000..47f8b54b28412 --- /dev/null +++ b/boards/riscv32/sifive-multicore/sifive-multicore.yaml @@ -0,0 +1,12 @@ +identifier: sifive-multicore +name: SiFive E Multicore +type: mcu +arch: riscv32 +simulation: qemu +toolchain: + - zephyr +ram: 16 +testing: + ignore_tags: + - net + - bluetooth diff --git a/boards/riscv32/sifive-multicore/sifive-multicore_defconfig b/boards/riscv32/sifive-multicore/sifive-multicore_defconfig new file mode 100644 index 0000000000000..d8ea6d93081ed --- /dev/null +++ b/boards/riscv32/sifive-multicore/sifive-multicore_defconfig @@ -0,0 +1,22 @@ +CONFIG_RISCV32=y +CONFIG_SOC_SERIES_RISCV32_SIFIVE_FREEDOM=y +CONFIG_SOC_RISCV32_SIFIVE_FREEDOM=y +CONFIG_BOARD_SIFIVE_MULTICORE=y +CONFIG_SMP=y +CONFIG_TICKLESS_KERNEL=n +CONFIG_TICKLESS_IDLE=n +CONFIG_USE_SWITCH=y +CONFIG_MP_NUM_CPUS=2 +CONFIG_CONSOLE=y +CONFIG_PRINTK=y +CONFIG_SERIAL=y +CONFIG_UART_SIFIVE=y +CONFIG_UART_SIFIVE_PORT_0=y +CONFIG_UART_CONSOLE=y +CONFIG_PLIC=y +CONFIG_PINMUX=y +CONFIG_PINMUX_SIFIVE=y +CONFIG_RISCV_MACHINE_TIMER=y +CONFIG_GPIO=y +CONFIG_GPIO_SIFIVE=y +CONFIG_BOOT_BANNER=y diff --git a/drivers/timer/riscv_machine_timer.c b/drivers/timer/riscv_machine_timer.c index c6b6f6427b4ac..ded16758113c2 100644 --- a/drivers/timer/riscv_machine_timer.c +++ b/drivers/timer/riscv_machine_timer.c @@ -3,13 +3,16 @@ * * SPDX-License-Identifier: Apache-2.0 */ +#include +#include +#include #include #include #include #include -#define CYC_PER_TICK ((u32_t)((u64_t)CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC \ - / (u64_t)CONFIG_SYS_CLOCK_TICKS_PER_SEC)) +#define CYC_PER_TICK ((u64_t)CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC \ + / (u64_t)CONFIG_SYS_CLOCK_TICKS_PER_SEC) #define MAX_TICKS ((0xffffffffu - CYC_PER_TICK) / CYC_PER_TICK) #define MIN_DELAY 1000 @@ -17,21 +20,37 @@ !IS_ENABLED(CONFIG_QEMU_TICKLESS_WORKAROUND)) static struct k_spinlock lock; -static u64_t last_count; +#ifdef CONFIG_SMP +static u64_t last_count[CONFIG_MP_NUM_CPUS]; +#else +static u64_t last_count[1]; +#endif static void set_mtimecmp(u64_t time) { + /* Get a pointer to the MTIMECMP array */ volatile u32_t *r = (u32_t *)RISCV_MTIMECMP_BASE; + /* Get the current hart id */ +#ifdef CONFIG_SMP + u32_t hartid = _current_cpu->id; +#else + u32_t hartid = 0; +#endif + + /* Calculate the indexes into the mtimecmp array for the current hart */ + u32_t low_index = hartid * 2; + u32_t high_index = low_index + 1; + /* Per spec, the RISC-V MTIME/MTIMECMP registers are 64 bit, * but are NOT internally latched for multiword transfers. So * we have to be careful about sequencing to avoid triggering * spurious interrupts: always set the high word to a max * value first. */ - r[1] = 0xffffffff; - r[0] = (u32_t)time; - r[1] = (u32_t)(time >> 32); + r[high_index] = 0xffffffff; + r[low_index] = (u32_t)time; + r[high_index] = (u32_t)(time >> 32); } static u64_t mtime(void) @@ -51,30 +70,39 @@ static u64_t mtime(void) static void timer_isr(void *arg) { ARG_UNUSED(arg); +#ifdef CONFIG_SMP + u32_t hartid = _current_cpu->id; +#else + u32_t hartid = 0; +#endif k_spinlock_key_t key = k_spin_lock(&lock); u64_t now = mtime(); - u32_t dticks = (u32_t)((now - last_count) / CYC_PER_TICK); + u64_t dticks = (now - last_count[hartid]) / CYC_PER_TICK; - last_count += dticks * CYC_PER_TICK; + last_count[hartid] += dticks * CYC_PER_TICK; if (!TICKLESS) { - u64_t next = last_count + CYC_PER_TICK; + u64_t next = last_count[hartid] + CYC_PER_TICK; - if ((s64_t)(next - now) < MIN_DELAY) { + while((s64_t)(next - now) < MIN_DELAY) { next += CYC_PER_TICK; } set_mtimecmp(next); } k_spin_unlock(&lock, key); - z_clock_announce(dticks); + + if(hartid == 0) { + z_clock_announce(dticks); + } } int z_clock_driver_init(struct device *device) { + last_count[0] = mtime(); IRQ_CONNECT(RISCV_MACHINE_TIMER_IRQ, 0, timer_isr, NULL, 0); - set_mtimecmp(mtime() + CYC_PER_TICK); + set_mtimecmp(last_count[0] + CYC_PER_TICK); irq_enable(RISCV_MACHINE_TIMER_IRQ); return 0; } @@ -94,6 +122,12 @@ void z_clock_set_timeout(s32_t ticks, bool idle) return; } +#ifdef CONFIG_SMP + u32_t hartid = _current_cpu->id; +#else + u32_t hartid = 0; +#endif + ticks = ticks == K_FOREVER ? MAX_TICKS : ticks; ticks = max(min(ticks - 1, (s32_t)MAX_TICKS), 0); @@ -104,14 +138,14 @@ void z_clock_set_timeout(s32_t ticks, bool idle) /* Round up to next tick boundary. Note use of 32 bit math, * max_ticks is calibrated to permit this. */ - cyc += (u32_t)(now - last_count) + (CYC_PER_TICK - 1); + cyc += (u32_t)(now - last_count[hartid]) + (CYC_PER_TICK - 1); cyc = (cyc / CYC_PER_TICK) * CYC_PER_TICK; - if ((s32_t)(cyc + last_count - now) < MIN_DELAY) { + if ((s32_t)(cyc + last_count[hartid] - now) < MIN_DELAY) { cyc += CYC_PER_TICK; } - set_mtimecmp(cyc + last_count); + set_mtimecmp(cyc + last_count[hartid]); k_spin_unlock(&lock, key); #endif } @@ -122,8 +156,14 @@ u32_t z_clock_elapsed(void) return 0; } +#ifdef CONFIG_SMP + u32_t hartid = _current_cpu->id; +#else + u32_t hartid = 0; +#endif + k_spinlock_key_t key = k_spin_lock(&lock); - u32_t ret = ((u32_t)mtime() - (u32_t)last_count) / CYC_PER_TICK; + u32_t ret = ((u32_t)mtime() - (u32_t)last_count[hartid]) / CYC_PER_TICK; k_spin_unlock(&lock, key); return ret; @@ -133,3 +173,14 @@ u32_t _timer_cycle_get_32(void) { return (u32_t)mtime(); } + +#ifdef CONFIG_SMP +void smp_timer_init(void) { + u64_t now = mtime(); + u32_t mhartid = _current_cpu->id; + last_count[mhartid] = now; + set_mtimecmp(now + CYC_PER_TICK); + irq_enable(RISCV_MACHINE_TIMER_IRQ); +} +#endif /* CONFIG_SMP */ + diff --git a/include/arch/riscv32/exp.h b/include/arch/riscv32/exp.h index ec91ce59425a1..443ae7d99cb4f 100644 --- a/include/arch/riscv32/exp.h +++ b/include/arch/riscv32/exp.h @@ -44,6 +44,22 @@ struct __esf { u32_t a6; /* function argument */ u32_t a7; /* function argument */ +#ifdef CONFIG_USE_SWITCH + /* USE_SWITCH on RISC-V stores callee-saved registers on the stack */ + u32_t s0; /* Callee-saved register */ + u32_t s1; /* Callee-saved register */ + u32_t s2; /* Callee-saved register */ + u32_t s3; /* Callee-saved register */ + u32_t s4; /* Callee-saved register */ + u32_t s5; /* Callee-saved register */ + u32_t s6; /* Callee-saved register */ + u32_t s7; /* Callee-saved register */ + u32_t s8; /* Callee-saved register */ + u32_t s9; /* Callee-saved register */ + u32_t s10; /* Callee-saved register */ + u32_t s11; /* Callee-saved register */ +#endif + u32_t mepc; /* machine exception program counter */ u32_t mstatus; /* machine status register */ diff --git a/kernel/include/offsets_short.h b/kernel/include/offsets_short.h index bd4907417762b..fff8b44b58da5 100644 --- a/kernel/include/offsets_short.h +++ b/kernel/include/offsets_short.h @@ -32,6 +32,21 @@ #define _kernel_offset_to_ready_q_cache \ (___kernel_t_ready_q_OFFSET + ___ready_q_t_cache_OFFSET) +#define _cpu_offset_to_nested \ + (___cpu_t_nested_OFFSET) + +#define _cpu_offset_to_irq_stack \ + (___cpu_t_irq_stack_OFFSET) + +#define _cpu_offset_to_current \ + (___cpu_t_current_OFFSET) + +#define _cpu_offset_to_idle \ + (___cpu_t_idle_OFFSET) + +#define _cpu_offset_to_current_fp \ + (___cpu_t_current_fp_OFFSET) + /* end - kernel */ /* threads */ diff --git a/kernel/sched.c b/kernel/sched.c index c959cb1fd91d2..323c091064385 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -166,7 +166,7 @@ static struct k_thread *next_up(void) * queue such that we don't want to re-add it". */ int queued = _is_thread_queued(_current); - int active = !_is_thread_prevented_from_running(_current); + int active = _is_thread_ready(_current); /* Choose the best thread that is not current */ struct k_thread *th = _priq_run_best(&_kernel.ready_q.runq); @@ -283,8 +283,10 @@ static void update_cache(int preempt_ok) void _add_thread_to_ready_q(struct k_thread *thread) { LOCKED(&sched_lock) { - _priq_run_add(&_kernel.ready_q.runq, thread); - _mark_thread_as_queued(thread); + if(!_is_thread_queued(thread)) { + _priq_run_add(&_kernel.ready_q.runq, thread); + _mark_thread_as_queued(thread); + } update_cache(0); } } @@ -292,7 +294,12 @@ void _add_thread_to_ready_q(struct k_thread *thread) void _move_thread_to_end_of_prio_q(struct k_thread *thread) { LOCKED(&sched_lock) { - _priq_run_remove(&_kernel.ready_q.runq, thread); + /* In SMP, the current thread is not in the queue, and if we call + * remove on the current thread, it corrupts the ready queue, so check + * that the thread we're moving is queued before removing it */ + if(_is_thread_queued(thread)) { + _priq_run_remove(&_kernel.ready_q.runq, thread); + } _priq_run_add(&_kernel.ready_q.runq, thread); _mark_thread_as_queued(thread); update_cache(thread == _current); @@ -317,7 +324,9 @@ static void pend(struct k_thread *thread, _wait_q_t *wait_q, s32_t timeout) if (wait_q != NULL) { thread->base.pended_on = wait_q; - _priq_wait_add(&wait_q->waitq, thread); + LOCKED(&sched_lock) { + _priq_wait_add(&wait_q->waitq, thread); + } } if (timeout != K_FOREVER) { @@ -415,12 +424,18 @@ void _thread_priority_set(struct k_thread *thread, int prio) bool need_sched = 0; LOCKED(&sched_lock) { - need_sched = _is_thread_ready(thread); + need_sched = _is_thread_ready(thread) && _is_thread_queued(thread); if (need_sched) { + /* Remove thread from run queue */ _priq_run_remove(&_kernel.ready_q.runq, thread); + + /* Adjust thread priority */ thread->base.prio = prio; + + /* Add thread back to queue (in the right place) */ _priq_run_add(&_kernel.ready_q.runq, thread); + _mark_thread_as_queued(thread); update_cache(1); } else { thread->base.prio = prio; @@ -563,6 +578,9 @@ void _priq_dumb_remove(sys_dlist_t *pq, struct k_thread *thread) struct k_thread *_priq_dumb_best(sys_dlist_t *pq) { + if(sys_dlist_is_empty(pq)) + return NULL; + return CONTAINER_OF(sys_dlist_peek_head(pq), struct k_thread, base.qnode_dlist); } @@ -777,8 +795,11 @@ void _impl_k_yield(void) if (!_is_idle(_current)) { LOCKED(&sched_lock) { - _priq_run_remove(&_kernel.ready_q.runq, _current); + if(_is_thread_queued(_current)) { + _priq_run_remove(&_kernel.ready_q.runq, _current); + } _priq_run_add(&_kernel.ready_q.runq, _current); + _mark_thread_as_queued(_current); update_cache(1); } } diff --git a/kernel/smp.c b/kernel/smp.c index 85aa46b73c388..5e5188b6cc404 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -92,26 +92,26 @@ static void smp_init_top(int key, void *arg) } #endif +static atomic_t _smp_start_flag; + void smp_init(void) { - atomic_t start_flag; - - (void)atomic_clear(&start_flag); + atomic_clear(&_smp_start_flag); #if defined(CONFIG_SMP) && CONFIG_MP_NUM_CPUS > 1 _arch_start_cpu(1, _interrupt_stack1, CONFIG_ISR_STACK_SIZE, - smp_init_top, &start_flag); + smp_init_top, &_smp_start_flag); #endif #if defined(CONFIG_SMP) && CONFIG_MP_NUM_CPUS > 2 _arch_start_cpu(2, _interrupt_stack2, CONFIG_ISR_STACK_SIZE, - smp_init_top, &start_flag); + smp_init_top, &_smp_start_flag); #endif #if defined(CONFIG_SMP) && CONFIG_MP_NUM_CPUS > 3 _arch_start_cpu(3, _interrupt_stack3, CONFIG_ISR_STACK_SIZE, - smp_init_top, &start_flag); + smp_init_top, &_smp_start_flag); #endif - (void)atomic_set(&start_flag, 1); + atomic_set(&_smp_start_flag, 1); } diff --git a/samples/philosophers/src/main.c b/samples/philosophers/src/main.c index bd3e03d01eb9a..e0a2ca929e09d 100644 --- a/samples/philosophers/src/main.c +++ b/samples/philosophers/src/main.c @@ -106,6 +106,8 @@ #define fork(x) (forks[x]) +K_MUTEX_DEFINE(print_state_mutex); + static void set_phil_state_pos(int id) { #if !DEBUG_PRINTF @@ -118,6 +120,8 @@ static void print_phil_state(int id, const char *fmt, s32_t delay) { int prio = k_thread_priority_get(k_current_get()); + k_mutex_lock(&print_state_mutex, K_FOREVER); + set_phil_state_pos(id); PRINTF("Philosopher %d [%s:%s%d] ", @@ -132,6 +136,8 @@ static void print_phil_state(int id, const char *fmt, s32_t delay) } PRINTF("\n"); + + k_mutex_unlock(&print_state_mutex); } static s32_t get_random_delay(int id, int period_in_ms) diff --git a/soc/riscv32/riscv-privilege/common/CMakeLists.txt b/soc/riscv32/riscv-privilege/common/CMakeLists.txt index 30dd1522d131a..38ffc047ee69d 100644 --- a/soc/riscv32/riscv-privilege/common/CMakeLists.txt +++ b/soc/riscv32/riscv-privilege/common/CMakeLists.txt @@ -5,4 +5,6 @@ zephyr_sources( soc_irq.S soc_common_irq.c vector.S + atomics.c + atomic_cas.S ) diff --git a/soc/riscv32/riscv-privilege/common/atomic_cas.S b/soc/riscv32/riscv-privilege/common/atomic_cas.S new file mode 100644 index 0000000000000..1426cee147f54 --- /dev/null +++ b/soc/riscv32/riscv-privilege/common/atomic_cas.S @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2018 SiFive Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +/* exports */ +GTEXT(atomic_cas) + +/** + * + * @brief Atomic compare-and-set primitive + * + * This routine provides the compare-and-set operator. If the original value at + * equals , then is stored at and the + * function returns 1. + * + * If the original value at does not equal , then the store + * is not done and the function returns 0. + * + * The reading of the original value at , the comparison, + * and the write of the new value (if it occurs) all happen atomically with + * respect to both interrupts and accesses of other processors to . + * + * @param target address to be tested + * @param old_value value to compare against + * @param new_value value to store + * @return Returns 1 if is written, 0 otherwise. + */ +SECTION_FUNC(text, atomic_cas) + /* Adapted from the Compare-and-swap example from page 62 + of The RISC-V Reader by Patterson and Waterman (2017) */ + + /* a0 contains the atomic target + a1 contains the expected old value + a2 contains the new value to store */ + + /* Load and create a memory reservation */ + lr.w a3, (a0) + + /* If the value doesn't match, fail */ + bne a3, a1, cas_fail + + /* Try to store-conditional */ + sc.w a4, a2, (a0) + + /* If we couldn't store, fail */ + bnez a4, cas_fail + + /* Return 1 for success */ + li a0, 1 + ret + +cas_fail: + /* Return 0 for failure */ + li a0, 0 + ret + diff --git a/soc/riscv32/riscv-privilege/common/atomics.c b/soc/riscv32/riscv-privilege/common/atomics.c new file mode 100644 index 0000000000000..c4dbb0c43dfc5 --- /dev/null +++ b/soc/riscv32/riscv-privilege/common/atomics.c @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2018 SiFive, Inc. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +/** + * + * @brief Atomic addition primitive + * + * This routine provides the atomic addition operator. The is + * atomically added to the value at , placing the result at , + * and the old value from is returned. + * + * @param target memory location to add to + * @param value the value to add + * + * @return The previous value from + */ +atomic_val_t atomic_add(atomic_t *target, atomic_val_t value) +{ + atomic_val_t old_value; + + __asm__ volatile("amoadd.w %[old], %[incr], (%[atomic])" + : [old] "=r" (old_value) + : [atomic] "r" (target), [incr] "r" (value) + : "memory"); + + return old_value; +} + +/** + * + * @brief Atomic subtraction primitive + * + * This routine provides the atomic subtraction operator. The is + * atomically subtracted from the value at , placing the result at + * , and the old value from is returned. + * + * @param target the memory location to subtract from + * @param value the value to subtract + * + * @return The previous value from + */ +atomic_val_t atomic_sub(atomic_t *target, atomic_val_t value) +{ + return atomic_add(target, -value); +} + +/** + * + * @brief Atomic increment primitive + * + * @param target memory location to increment + * + * This routine provides the atomic increment operator. The value at + * is atomically incremented by 1, and the old value from is returned. + * + * @return The value from before the increment + */ +atomic_val_t atomic_inc(atomic_t *target) +{ + return atomic_add(target, 1); +} + +/** + * + * @brief Atomic decrement primitive + * + * @param target memory location to decrement + * + * This routine provides the atomic decrement operator. The value at + * is atomically decremented by 1, and the old value from is returned. + * + * @return The value from prior to the decrement + */ +atomic_val_t atomic_dec(atomic_t *target) +{ + return atomic_add(target, -1); +} + +/** + * + * @brief Atomic get primitive + * + * @param target memory location to read from + * + * This routine provides the atomic get primitive to atomically read + * a value from . It simply does an ordinary load. Note that + * is expected to be aligned to a 4-byte boundary. + * + * @return The value read from + */ +atomic_val_t atomic_get(const atomic_t *target) +{ + return *target; +} + +/** + * + * @brief Atomic get-and-set primitive + * + * This routine provides the atomic set operator. The is atomically + * written at and the previous value at is returned. + * + * @param target the memory location to write to + * @param value the value to write + * + * @return The previous value from + */ +atomic_val_t atomic_set(atomic_t *target, atomic_val_t value) +{ + atomic_val_t old_value; + + __asm__ volatile("amoswap.w %[old], %[new], (%[mem])" + : [old] "=r" (old_value) + : [new] "r" (value), [mem] "r" (target) + : "memory"); + + return old_value; +} + +/** + * + * @brief Atomic clear primitive + * + * This routine provides the atomic clear operator. The value of 0 is atomically + * written at and the previous value at is returned. (Hence, + * atomic_clear(pAtomicVar) is equivalent to atomic_set(pAtomicVar, 0).) + * + * @param target the memory location to write + * + * @return The previous value from + */ +atomic_val_t atomic_clear(atomic_t *target) +{ + return atomic_set(target, 0); +} + +/** + * + * @brief Atomic bitwise inclusive OR primitive + * + * This routine provides the atomic bitwise inclusive OR operator. The + * is atomically bitwise OR'ed with the value at , placing the result + * at , and the previous value at is returned. + * + * @param target the memory location to be modified + * @param value the value to OR + * + * @return The previous value from + */ +atomic_val_t atomic_or(atomic_t *target, atomic_val_t value) +{ + atomic_val_t old_value; + + __asm__ volatile("amoor.w %[old], %[new], (%[mem])" + : [old] "=r" (old_value) + : [new] "r" (value), [mem] "r" (target) + : "memory"); + + return old_value; +} + +/** + * + * @brief Atomic bitwise exclusive OR (XOR) primitive + * + * This routine provides the atomic bitwise exclusive OR operator. The + * is atomically bitwise XOR'ed with the value at , placing the result + * at , and the previous value at is returned. + * + * @param target the memory location to be modified + * @param value the value to XOR + * + * @return The previous value from + */ +atomic_val_t atomic_xor(atomic_t *target, atomic_val_t value) +{ + atomic_val_t old_value; + + __asm__ volatile("amoxor.w %[old], %[new], (%[mem])" + : [old] "=r" (old_value) + : [new] "r" (value), [mem] "r" (target) + : "memory"); + + return old_value; +} + +/** + * + * @brief Atomic bitwise AND primitive + * + * This routine provides the atomic bitwise AND operator. The is + * atomically bitwise AND'ed with the value at , placing the result + * at , and the previous value at is returned. + * + * @param target the memory location to be modified + * @param value the value to AND + * + * @return The previous value from + */ +atomic_val_t atomic_and(atomic_t *target, atomic_val_t value) +{ + atomic_val_t old_value; + + __asm__ volatile("amoand.w %[old], %[new], (%[mem])" + : [old] "=r" (old_value) + : [new] "r" (value), [mem] "r" (target) + : "memory"); + + return old_value; +} + +/** + * + * @brief Atomic bitwise NAND primitive + * + * This routine provides the atomic bitwise NAND operator. The is + * atomically bitwise NAND'ed with the value at , placing the result + * at , and the previous value at is returned. + * + * @param target the memory location to be modified + * @param value the value to NAND + * + * @return The previous value from + */ +atomic_val_t atomic_nand(atomic_t *target, atomic_val_t value) +{ + /* TODO there's no atomic NAND instruction in RISC-V. What other + * solutions are appropriate? */ + atomic_val_t old_value = *target; + + *target = ~(old_value & value); + + return old_value; +} diff --git a/soc/riscv32/riscv-privilege/common/soc_common_irq.c b/soc/riscv32/riscv-privilege/common/soc_common_irq.c index e2dcc9cbeff70..0019c4eb08203 100644 --- a/soc/riscv32/riscv-privilege/common/soc_common_irq.c +++ b/soc/riscv32/riscv-privilege/common/soc_common_irq.c @@ -69,7 +69,7 @@ int _arch_irq_is_enabled(unsigned int irq) void soc_interrupt_init(void) { /* ensure that all interrupts are disabled */ - (void)irq_lock(); + (void)_arch_irq_lock(); __asm__ volatile ("csrwi mie, 0\n" "csrwi mip, 0\n"); diff --git a/soc/riscv32/riscv-privilege/common/vector.S b/soc/riscv32/riscv-privilege/common/vector.S index a6decc522d098..7be8b9fd7eb8a 100644 --- a/soc/riscv32/riscv-privilege/common/vector.S +++ b/soc/riscv32/riscv-privilege/common/vector.S @@ -24,5 +24,76 @@ SECTION_FUNC(vectors, __start) la t0, __irq_wrapper csrw mtvec, t0 - /* Jump to __initialize */ + /* Read the hart ID */ + csrr t0, mhartid + +#ifndef CONFIG_SMP + /* If we're not running with SMP enabled, then make sure any other harts + * are parked forever */ + bnez t0, park_hart_forever +#else /* CONFIG_SMP */ + /* If the hart ID is >= the configured number of CPUs, park forever + * + * Greater than or equal to because if CONFIG_NUM_CPUS == 3, then the + * last hart will be hart 2. */ + li t1, CONFIG_MP_NUM_CPUS + bgeu t0, t1, park_hart_forever + + /* If we're not hart 0, wait for _arch_start_cpu to start our hart */ + bnez t0, secondary_harts +#endif /* CONFIG_SMP */ + + /* Hart 0 jumps to __initialize */ tail __initialize + +park_hart_forever: + wfi + j park_hart_forever + +#ifdef CONFIG_SMP +secondary_harts: + /* multiply the hartid by 4 */ + slli t0, t0, 2 + + /* load the address of the go flag */ + la t1, __init_riscv_smp_go + add t1, t1, t0 + +secondary_harts_wait: + /* load the go flag and see if we've been signaled */ + lw t2, (t1) + bnez t2, secondary_harts_go + j secondary_harts_wait + +secondary_harts_go: + /* load our start point and jump into it */ + + /* load the cpu context into mscratch */ + la t2, __init_riscv_smp_mscratch + add t2, t2, t0 + lw t2, 0(t2) + csrw mscratch, t2 + + /* load entry function into t1 */ + la t1, __init_riscv_smp_entry + add t1, t1, t0 + lw t1, 0(t1) + + /* load the stack pointer */ + la sp, __init_riscv_smp_stacks + add sp, sp, t0 + lw sp, 0(sp) + + /* load the irq_unlock key into a0 */ + la a0, __init_riscv_smp_keys + add a0, a0, t0 + lw a0, 0(a0) + + /* load the start flag into a1 */ + la a1, __init_riscv_smp_start_flags + add a1, a1, t0 + lw a1, 0(a1) + + /* call the entry function */ + jr t1 +#endif /* CONFIG_SMP */ diff --git a/soc/riscv32/riscv-privilege/sifive-freedom/Kconfig.soc b/soc/riscv32/riscv-privilege/sifive-freedom/Kconfig.soc index 68444ae516956..6e51fd9da7fed 100644 --- a/soc/riscv32/riscv-privilege/sifive-freedom/Kconfig.soc +++ b/soc/riscv32/riscv-privilege/sifive-freedom/Kconfig.soc @@ -11,6 +11,5 @@ depends on SOC_SERIES_RISCV32_SIFIVE_FREEDOM config SOC_RISCV32_SIFIVE_FREEDOM bool "SiFive Freedom SOC implementation" - select ATOMIC_OPERATIONS_C endchoice