diff --git a/include/rtdef.h b/include/rtdef.h index 06fcfcb8b88d..f6ad7f73e315 100644 --- a/include/rtdef.h +++ b/include/rtdef.h @@ -731,8 +731,10 @@ struct rt_cpu struct rt_thread *current_thread; rt_uint8_t irq_switch_flag:1; - rt_uint8_t critical_switch_flag:1; rt_uint8_t sched_lock_flag:1; +#ifndef ARCH_USING_HW_THREAD_SELF + rt_uint8_t critical_switch_flag:1; +#endif /* ARCH_USING_HW_THREAD_SELF */ rt_uint8_t current_priority; rt_list_t priority_table[RT_THREAD_PRIORITY_MAX]; diff --git a/include/rtsched.h b/include/rtsched.h index 9e313200953d..ab8d6a132685 100644 --- a/include/rtsched.h +++ b/include/rtsched.h @@ -56,6 +56,10 @@ struct rt_sched_thread_ctx rt_uint8_t sched_flag_locked:1; /**< calling thread have the scheduler locked */ rt_uint8_t sched_flag_ttmr_set:1; /**< thread timer is start */ +#ifdef ARCH_USING_HW_THREAD_SELF + rt_uint8_t critical_switch_flag:1; /**< critical switch pending */ +#endif /* ARCH_USING_HW_THREAD_SELF */ + #ifdef RT_USING_SMP rt_uint8_t bind_cpu; /**< thread is bind to cpu */ rt_uint8_t oncpu; /**< process on cpu */ @@ -138,6 +142,8 @@ rt_bool_t rt_sched_is_locked(void); #define RT_SCHED_DEBUG_IS_LOCKED #define RT_SCHED_DEBUG_IS_UNLOCKED + +extern struct rt_thread *rt_current_thread; #endif /* RT_USING_SMP */ /** diff --git a/libcpu/Kconfig b/libcpu/Kconfig index 540794c43f13..8c211de23660 100644 --- a/libcpu/Kconfig +++ b/libcpu/Kconfig @@ -14,6 +14,8 @@ if ARCH_ARMV8 && ARCH_CPU_64BIT default y config ARCH_USING_GENERIC_CPUID bool "Using generic cpuid implemenation" + select ARCH_USING_HW_THREAD_SELF + default y if RT_USING_OFW default n endmenu endif @@ -270,3 +272,7 @@ config ARCH_HOST_SIMULATOR config ARCH_CPU_STACK_GROWS_UPWARD bool default n + +config ARCH_USING_HW_THREAD_SELF + bool + default n diff --git a/libcpu/aarch64/common/context_gcc.S b/libcpu/aarch64/common/context_gcc.S index f27144b764bb..7850c467ad7c 100644 --- a/libcpu/aarch64/common/context_gcc.S +++ b/libcpu/aarch64/common/context_gcc.S @@ -8,6 +8,7 @@ * 2021-05-18 Jesven the first version * 2023-06-24 WangXiaoyao Support backtrace for user thread * 2024-01-06 Shell Fix barrier on irq_disable/enable + * 2024-01-18 Shell fix implicit dependency of cpuid management */ #ifndef __ASSEMBLY__ @@ -27,15 +28,28 @@ rt_thread_switch_interrupt_flag: .zero 8 #endif .text -.weak rt_hw_cpu_id_set + .type rt_hw_cpu_id_set, @function rt_hw_cpu_id_set: - mrs x0, mpidr_el1 /* MPIDR_EL1: Multi-Processor Affinity Register */ +#ifdef ARCH_USING_GENERIC_CPUID + .globl rt_hw_cpu_id_set +#else /* !ARCH_USING_GENERIC_CPUID */ + .weak rt_hw_cpu_id_set +#endif /* ARCH_USING_GENERIC_CPUID */ + +#ifndef RT_USING_OFW + mrs x0, mpidr_el1 /* MPIDR_EL1: Multi-Processor Affinity Register */ #ifdef ARCH_ARM_CORTEX_A55 - lsr x0, x0, #8 + lsr x0, x0, #8 +#endif /* ARCH_ARM_CORTEX_A55 */ + and x0, x0, #15 +#endif /* !RT_USING_OFW */ + +#ifdef ARCH_USING_GENERIC_CPUID + msr tpidrro_el0, x0 +#else + msr tpidr_el1, x0 #endif - and x0, x0, #15 - msr tpidr_el1, x0 ret /* diff --git a/libcpu/aarch64/common/cpu.c b/libcpu/aarch64/common/cpu.c index 72ccccac2449..b9fedefdd307 100644 --- a/libcpu/aarch64/common/cpu.c +++ b/libcpu/aarch64/common/cpu.c @@ -231,6 +231,24 @@ int rt_hw_cpu_boot_secondary(int num_cpus, rt_uint64_t *cpu_hw_ids, struct cpu_o #endif /*RT_USING_SMP*/ +/** + * Generic hw-cpu-id + */ +#ifdef ARCH_USING_GENERIC_CPUID + +int rt_hw_cpu_id(void) +{ +#if RT_CPUS_NR > 1 + long cpuid; + __asm__ volatile("mrs %0, tpidrro_el0":"=r"(cpuid)); + return cpuid; +#else + return 0; +#endif /* RT_CPUS_NR > 1 */ +} + +#endif /* ARCH_USING_GENERIC_CPUID */ + /** * @addtogroup ARM CPU */ diff --git a/libcpu/aarch64/common/cpuport.h b/libcpu/aarch64/common/cpuport.h index 15500894db91..217f477aaf13 100644 --- a/libcpu/aarch64/common/cpuport.h +++ b/libcpu/aarch64/common/cpuport.h @@ -7,6 +7,7 @@ * Date Author Notes * 2023-10-25 Shell Move ffs to cpuport, add general implementation * by inline assembly + * 2024-01-18 Shell support rt_hw_thread_self to improve overall performance */ #ifndef CPUPORT_H__ @@ -27,31 +28,6 @@ typedef struct rt_uint32_t value; } rt_hw_spinlock_t; -/** - * Generic hw-cpu-id - */ -#ifdef ARCH_USING_GENERIC_CPUID - -#if RT_CPUS_NR > 1 - -rt_inline int rt_hw_cpu_id(void) -{ - long cpuid; - __asm__ volatile("mrs %0, tpidr_el1":"=r"(cpuid)); - return cpuid; -} - -#else - -rt_inline int rt_hw_cpu_id(void) -{ - return 0; -} - -#endif /* RT_CPUS_NR > 1 */ - -#endif /* ARCH_USING_GENERIC_CPUID */ - #endif /* RT_USING_SMP */ #define rt_hw_barrier(cmd, ...) \ @@ -107,4 +83,20 @@ rt_inline int __rt_ffs(int value) #endif /* RT_USING_CPU_FFS */ +#ifdef ARCH_USING_HW_THREAD_SELF +rt_inline struct rt_thread *rt_hw_thread_self(void) +{ + struct rt_thread *thread; + __asm__ volatile ("mrs %0, tpidr_el1":"=r"(thread)); + + return thread; +} + +rt_inline void rt_hw_thread_set_self(struct rt_thread *thread) +{ + __asm__ volatile ("msr tpidr_el1, %0"::"r"(thread)); +} + +#endif /* ARCH_USING_HW_THREAD_SELF */ + #endif /*CPUPORT_H__*/ diff --git a/libcpu/aarch64/cortex-a/entry_point.S b/libcpu/aarch64/cortex-a/entry_point.S index 361c9d4f6751..7a6f4f62c519 100644 --- a/libcpu/aarch64/cortex-a/entry_point.S +++ b/libcpu/aarch64/cortex-a/entry_point.S @@ -7,6 +7,7 @@ * 2020-01-15 bigmagic the first version * 2020-08-10 SummerGift support clang compiler * 2023-04-29 GuEe-GUI support kernel's ARM64 boot header + * 2024-01-18 Shell fix implicit dependency of cpuid management */ #ifndef __ASSEMBLY__ @@ -94,7 +95,12 @@ _start: /* Save cpu stack */ get_phy stack_top, .boot_cpu_stack_top + /* Save cpu id temp */ +#ifdef ARCH_USING_HW_THREAD_SELF + msr tpidrro_el0, xzr + /* Save thread self */ +#endif /* ARCH_USING_HW_THREAD_SELF */ msr tpidr_el1, xzr bl init_cpu_el @@ -149,11 +155,10 @@ _secondary_cpu_entry: /* Get cpu id success */ sub x0, x2, #1 - msr tpidr_el1, x0 /* Save cpu id global */ -#else - bl rt_hw_cpu_id_set - mrs x0, tpidr_el1 #endif /* RT_USING_OFW */ + /* Save cpu id global */ + bl rt_hw_cpu_id_set + bl rt_hw_cpu_id /* Set current cpu's stack top */ sub x0, x0, #1 diff --git a/src/scheduler_mp.c b/src/scheduler_mp.c index c1626c5f99db..57d32c7990a6 100644 --- a/src/scheduler_mp.c +++ b/src/scheduler_mp.c @@ -33,6 +33,7 @@ * 2023-12-10 xqyjlj use rt_hw_spinlock * 2024-01-05 Shell Fixup of data racing in rt_critical_level * 2024-01-18 Shell support rt_sched_thread of scheduling status for better mt protection + * 2024-01-18 Shell support rt_hw_thread_self to improve overall performance */ #include @@ -99,6 +100,14 @@ static struct rt_spinlock _mp_scheduler_lock; rt_hw_local_irq_enable(level); \ } while (0) +#ifdef ARCH_USING_HW_THREAD_SELF +#define CRITICAL_SWITCH_FLAG(pcpu, curthr) (RT_SCHED_CTX(curthr).critical_switch_flag) + +#else /* !ARCH_USING_HW_THREAD_SELF */ +#define CRITICAL_SWITCH_FLAG(pcpu, curthr) ((pcpu)->critical_switch_flag) + +#endif /* ARCH_USING_HW_THREAD_SELF */ + static rt_uint32_t rt_thread_ready_priority_group; #if RT_THREAD_PRIORITY_MAX > 32 /* Maximum priority level, 256 */ @@ -749,7 +758,7 @@ rt_err_t rt_sched_unlock_n_resched(rt_sched_lock_level_t level) /* leaving critical region of global context since we can't schedule */ SCHEDULER_CONTEXT_UNLOCK(pcpu); - pcpu->critical_switch_flag = 1; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 1; error = -RT_ESCHEDLOCKED; SCHEDULER_EXIT_CRITICAL(current_thread); @@ -757,7 +766,7 @@ rt_err_t rt_sched_unlock_n_resched(rt_sched_lock_level_t level) else { /* flush critical switch flag since a scheduling is done */ - pcpu->critical_switch_flag = 0; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 0; /* pick the highest runnable thread, and pass the control to it */ to_thread = _prepare_context_switch_locked(cpu_id, pcpu, current_thread); @@ -828,7 +837,7 @@ void rt_schedule(void) /* whether caller had locked the local scheduler already */ if (RT_SCHED_CTX(current_thread).critical_lock_nest > 1) { - pcpu->critical_switch_flag = 1; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 1; SCHEDULER_EXIT_CRITICAL(current_thread); @@ -837,7 +846,7 @@ void rt_schedule(void) else { /* flush critical switch flag since a scheduling is done */ - pcpu->critical_switch_flag = 0; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 0; pcpu->irq_switch_flag = 0; /** @@ -912,13 +921,13 @@ void rt_scheduler_do_irq_switch(void *context) /* whether caller had locked the local scheduler already */ if (RT_SCHED_CTX(current_thread).critical_lock_nest > 1) { - pcpu->critical_switch_flag = 1; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 1; SCHEDULER_EXIT_CRITICAL(current_thread); } else if (rt_atomic_load(&(pcpu->irq_nest)) == 0) { /* flush critical & irq switch flag since a scheduling is done */ - pcpu->critical_switch_flag = 0; + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 0; pcpu->irq_switch_flag = 0; SCHEDULER_CONTEXT_LOCK(pcpu); @@ -1056,6 +1065,9 @@ void rt_sched_post_ctx_switch(struct rt_thread *thread) } /* safe to access since irq is masked out */ pcpu->current_thread = thread; +#ifdef ARCH_USING_HW_THREAD_SELF + rt_hw_thread_set_self(thread); +#endif /* ARCH_USING_HW_THREAD_SELF */ } #ifdef RT_DEBUGING_CRITICAL @@ -1096,14 +1108,28 @@ void rt_exit_critical_safe(rt_base_t critical_level) #endif /* RT_DEBUGING_CRITICAL */ RTM_EXPORT(rt_exit_critical_safe); +#ifdef ARCH_USING_HW_THREAD_SELF +#define FREE_THREAD_SELF(lvl) + +#else /* !ARCH_USING_HW_THREAD_SELF */ +#define FREE_THREAD_SELF(lvl) \ + do \ + { \ + rt_hw_local_irq_enable(lvl); \ + } while (0) + +#endif /* ARCH_USING_HW_THREAD_SELF */ + /** * @brief This function will lock the thread scheduler. */ rt_base_t rt_enter_critical(void) { - rt_base_t level; rt_base_t critical_level; struct rt_thread *current_thread; + +#ifndef ARCH_USING_HW_THREAD_SELF + rt_base_t level; struct rt_cpu *pcpu; /* disable interrupt */ @@ -1118,12 +1144,22 @@ rt_base_t rt_enter_critical(void) return -RT_EINVAL; } +#else /* !ARCH_USING_HW_THREAD_SELF */ + + current_thread = rt_hw_thread_self(); + if (!current_thread) + { + /* scheduler unavailable */ + return -RT_EINVAL; + } + +#endif /* ARCH_USING_HW_THREAD_SELF */ + /* critical for local cpu */ RT_SCHED_CTX(current_thread).critical_lock_nest++; critical_level = RT_SCHED_CTX(current_thread).critical_lock_nest; - /* enable interrupt */ - rt_hw_local_irq_enable(level); + FREE_THREAD_SELF(level); return critical_level; } @@ -1134,9 +1170,11 @@ RTM_EXPORT(rt_enter_critical); */ void rt_exit_critical(void) { - rt_base_t level; struct rt_thread *current_thread; rt_bool_t need_resched; + +#ifndef ARCH_USING_HW_THREAD_SELF + rt_base_t level; struct rt_cpu *pcpu; /* disable interrupt */ @@ -1150,6 +1188,15 @@ void rt_exit_critical(void) return; } +#else /* !ARCH_USING_HW_THREAD_SELF */ + + current_thread = rt_hw_thread_self(); + if (!current_thread) + { + return; + } +#endif /* ARCH_USING_HW_THREAD_SELF */ + /* the necessary memory barrier is done on irq_(dis|en)able */ RT_SCHED_CTX(current_thread).critical_lock_nest--; @@ -1157,11 +1204,10 @@ void rt_exit_critical(void) if (RT_SCHED_CTX(current_thread).critical_lock_nest == 0) { /* is there any scheduling request unfinished? */ - need_resched = pcpu->critical_switch_flag; - pcpu->critical_switch_flag = 0; + need_resched = CRITICAL_SWITCH_FLAG(pcpu, current_thread); + CRITICAL_SWITCH_FLAG(pcpu, current_thread) = 0; - /* enable interrupt */ - rt_hw_local_irq_enable(level); + FREE_THREAD_SELF(level); if (need_resched) rt_schedule(); @@ -1171,8 +1217,7 @@ void rt_exit_critical(void) /* each exit_critical is strictly corresponding to an enter_critical */ RT_ASSERT(RT_SCHED_CTX(current_thread).critical_lock_nest > 0); - /* enable interrupt */ - rt_hw_local_irq_enable(level); + FREE_THREAD_SELF(level); } } RTM_EXPORT(rt_exit_critical); diff --git a/src/thread.c b/src/thread.c index 8d4752409d41..02a8e0d73b77 100644 --- a/src/thread.c +++ b/src/thread.c @@ -348,27 +348,38 @@ rt_err_t rt_thread_init(struct rt_thread *thread, } RTM_EXPORT(rt_thread_init); -/** - * @brief This function will return self thread object. - * - * @return The self thread object. - */ -rt_thread_t rt_thread_self(void) -{ #ifdef RT_USING_SMP - rt_base_t lock; +static rt_thread_t _mp_thread_self(void) +{ +#ifdef ARCH_USING_HW_THREAD_SELF + return rt_hw_thread_self(); + +#else /* !ARCH_USING_HW_THREAD_SELF */ rt_thread_t self; + rt_base_t lock; lock = rt_hw_local_irq_disable(); self = rt_cpu_self()->current_thread; rt_hw_local_irq_enable(lock); + return self; +#endif /* ARCH_USING_HW_THREAD_SELF */ +} +#define THREAD_SELF _mp_thread_self() #else /* !RT_USING_SMP */ - extern rt_thread_t rt_current_thread; +#define THREAD_SELF rt_current_thread - return rt_current_thread; #endif /* RT_USING_SMP */ + +/** + * @brief This function will return self thread object. + * + * @return The self thread object. + */ +rt_thread_t rt_thread_self(void) +{ + return THREAD_SELF; } RTM_EXPORT(rt_thread_self);