Merge branch 'feature/freertos_xtensa_disable_port_using_mpu_wrappers…

…' into 'master'

FreeRTOS: Fix FPU bugs on Amazon SMP FreeRTOS, port over FPU updates to IDF FreeRTOS

Closes IDF-5803

See merge request espressif/esp-idf!20280

components/freertos/FreeRTOS-Kernel-SMP/portable/riscv/port.c

@@ -537,6 +537,11 @@ FORCE_INLINE_ATTR UBaseType_t uxInitialiseStackFrame(UBaseType_t uxStackPointer,
 
 StackType_t *pxPortInitialiseStack(StackType_t *pxTopOfStack, TaskFunction_t pxCode, void *pvParameters)
 {
+#ifdef __clang_analyzer__
+    // Teach clang-tidy that pxTopOfStack cannot be a pointer to const
+    volatile StackType_t * pxTemp = pxTopOfStack;
+    pxTopOfStack = pxTemp;
+#endif /*__clang_analyzer__ */
     /*
     HIGH ADDRESS
     |---------------------------| <- pxTopOfStack on entry

components/freertos/FreeRTOS-Kernel-SMP/portable/xtensa/include/freertos/xtensa_rtos.h

@@ -170,6 +170,15 @@ The implementation may use only a2-4, a15 (all other regs must be preserved).
 // void* XT_RTOS_CP_STATE(void)
 #define XT_RTOS_CP_STATE    _frxt_task_coproc_state
 
+/*
+RTOS provided hook function that is called on every coprocessor exception. May
+only be called from assembly code and by the 'call0' instruction.
+The implementation may use only a2-4, a15 (all other regs must be preserved).
+*/
+// void XT_RTOS_CP_EXC_HOOK(void)
+#if XCHAL_CP_NUM > 0
+#define XT_RTOS_CP_EXC_HOOK   _frxt_coproc_exc_hook
+#endif
 
 /*******************************************************************************
 

components/freertos/FreeRTOS-Kernel-SMP/portable/xtensa/port.c

@@ -41,9 +41,25 @@
 
 _Static_assert(portBYTE_ALIGNMENT == 16, "portBYTE_ALIGNMENT must be set to 16");
 
-/*
-OS state variables
-*/
+/* ---------------------------------------------------- Variables ------------------------------------------------------
+ * - Various variables used to maintain the FreeRTOS port's state. Used from both port.c and various .S files
+ * - Constant offsets are used by assembly to jump to particular TCB members or a stack area (such as the CPSA). We use
+ *   C constants instead of preprocessor macros due to assembly lacking "offsetof()".
+ * ------------------------------------------------------------------------------------------------------------------ */
+
+#if XCHAL_CP_NUM > 0
+/* Offsets used to navigate to a task's CPSA on the stack */
+const DRAM_ATTR uint32_t offset_pxEndOfStack = offsetof(StaticTask_t, pxDummy8);
+const DRAM_ATTR uint32_t offset_cpsa = XT_CP_SIZE;  /* Offset to start of the CPSA area on the stack. See uxInitialiseStackCPSA(). */
+#if configNUM_CORES > 1
+/* Offset to TCB_t.uxCoreAffinityMask member. Used to pin unpinned tasks that use the FPU. */
+const DRAM_ATTR uint32_t offset_uxCoreAffinityMask = offsetof(StaticTask_t, uxDummy25);
+#if configUSE_CORE_AFFINITY != 1
+#error "configUSE_CORE_AFFINITY must be 1 on multicore targets with coprocessor support"
+#endif
+#endif /* configNUM_CORES > 1 */
+#endif /* XCHAL_CP_NUM > 0 */
+
 volatile unsigned port_xSchedulerRunning[portNUM_PROCESSORS] = {0}; // Indicates whether scheduler is running on a per-core basis
 unsigned int port_interruptNesting[portNUM_PROCESSORS] = {0};  // Interrupt nesting level. Increased/decreased in portasm.c, _frxt_int_enter/_frxt_int_exit
 //FreeRTOS SMP Locks
@@ -423,12 +439,6 @@ static void vPortTaskWrapper(TaskFunction_t pxCode, void *pvParameters)
 }
 #endif
 
-const DRAM_ATTR uint32_t offset_pxEndOfStack = offsetof(StaticTask_t, pxDummy8);
-#if ( configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1 )
-const DRAM_ATTR uint32_t offset_uxCoreAffinityMask = offsetof(StaticTask_t, uxDummy25);
-#endif // ( configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1 )
-const DRAM_ATTR uint32_t offset_cpsa = XT_CP_SIZE;
-
 /**
  * @brief Align stack pointer in a downward growing stack
  *
@@ -674,10 +684,15 @@ StackType_t * pxPortInitialiseStack( StackType_t * pxTopOfStack,
                                      void * pvParameters )
 #endif
 {
+#ifdef __clang_analyzer__
+    // Teach clang-tidy that pxTopOfStack cannot be a pointer to const
+    volatile StackType_t * pxTemp = pxTopOfStack;
+    pxTopOfStack = pxTemp;
+#endif /*__clang_analyzer__ */
     /*
     HIGH ADDRESS
     |---------------------------| <- pxTopOfStack on entry
-    | Coproc Save Area          |
+    | Coproc Save Area          | (CPSA MUST BE FIRST)
     | ------------------------- |
     | TLS Variables             |
     | ------------------------- | <- Start of useable stack
@@ -697,7 +712,7 @@ StackType_t * pxPortInitialiseStack( StackType_t * pxTopOfStack,
     configASSERT((uxStackPointer & portBYTE_ALIGNMENT_MASK) == 0);
 
 #if XCHAL_CP_NUM > 0
-    // Initialize the coprocessor save area
+    // Initialize the coprocessor save area. THIS MUST BE THE FIRST AREA due to access from _frxt_task_coproc_state()
     uxStackPointer = uxInitialiseStackCPSA(uxStackPointer);
     configASSERT((uxStackPointer & portBYTE_ALIGNMENT_MASK) == 0);
 #endif /* XCHAL_CP_NUM > 0 */
@@ -717,25 +732,25 @@ StackType_t * pxPortInitialiseStack( StackType_t * pxTopOfStack,
 // -------------------- Co-Processor -----------------------
 #if ( XCHAL_CP_NUM > 0 && configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1 )
 
-void _xt_coproc_release(volatile void *coproc_sa_base, BaseType_t xCoreID);
+void _xt_coproc_release(volatile void *coproc_sa_base, BaseType_t xTargetCoreID);
 
 void vPortCleanUpCoprocArea( void *pxTCB )
 {
-    StackType_t *coproc_area;
-    BaseType_t xCoreID;
+    UBaseType_t uxCoprocArea;
+    BaseType_t xTargetCoreID;
 
-    /* Calculate the coproc save area in the stack from the TCB base */
-    coproc_area = ( StackType_t * ) ( ( uint32_t ) ( pxTCB + offset_pxEndOfStack ));
-    coproc_area = ( StackType_t * ) ( ( ( portPOINTER_SIZE_TYPE ) coproc_area ) & ( ~( ( portPOINTER_SIZE_TYPE ) portBYTE_ALIGNMENT_MASK ) ) );
-    coproc_area = ( StackType_t * ) ( ( ( uint32_t ) coproc_area - XT_CP_SIZE ) & ~0xf );
+    /* Get pointer to the task's coprocessor save area from TCB->pxEndOfStack. See uxInitialiseStackCPSA() */
+    uxCoprocArea = ( UBaseType_t ) ( ( ( StaticTask_t * ) pxTCB )->pxDummy8 );  /* Get TCB_t.pxEndOfStack */
+    uxCoprocArea = STACKPTR_ALIGN_DOWN(16, uxCoprocArea - XT_CP_SIZE);
 
     /* Extract core ID from the affinity mask */
-    xCoreID = __builtin_ffs( * ( UBaseType_t * ) ( pxTCB + offset_uxCoreAffinityMask ) );
-    assert( xCoreID >= 1 );
-    xCoreID -= 1;
+    xTargetCoreID = ( ( StaticTask_t * ) pxTCB )->uxDummy25 ;
+    xTargetCoreID = ( BaseType_t ) __builtin_ffs( ( int ) xTargetCoreID );
+    assert( xTargetCoreID >= 1 ); // __builtin_ffs always returns first set index + 1
+    xTargetCoreID -= 1;
 
     /* If task has live floating point registers somewhere, release them */
-    _xt_coproc_release( coproc_area, xCoreID );
+    _xt_coproc_release( (void *)uxCoprocArea, xTargetCoreID );
 }
 #endif // ( XCHAL_CP_NUM > 0 && configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1 )
 

components/freertos/FreeRTOS-Kernel-SMP/portable/xtensa/portasm.S

@@ -34,32 +34,45 @@
 #define TOPOFSTACK_OFFS                 0x00    /* StackType_t *pxTopOfStack */
 
 .extern pxCurrentTCBs
+#if XCHAL_CP_NUM > 0
+/* Offsets used to get a task's coprocessor save area (CPSA) from its TCB */
 .extern offset_pxEndOfStack
 .extern offset_cpsa
+#if configNUM_CORES > 1
+/* Offset to TCB_t.uxCoreAffinityMask member. Used to pin unpinned tasks that use the FPU. */
+.extern offset_uxCoreAffinityMask
+#endif /* configNUM_CORES > 1 */
+#endif /* XCHAL_CP_NUM > 0 */
 
 /*
-Macro to get a task's coprocessor save area (CPSA) from its TCB
+--------------------------------------------------------------------------------
+  Macro get_cpsa_from_tcb - get the pointer to a task's CPSA form its TCB
+
+  Entry:
+  - "reg_A" contains a pointer to the task's TCB
+  Exit:
+  - "reg_A" contains pointer the the task's CPSA
+  - "reg_B" clobbered
 
-Entry:
-- reg_A contains a pointer to the TCB
-Exit:
-- reg_A contains a pointer to the CPSA
-- reg_B destroyed
+  The two arguments must be different AR registers.
+--------------------------------------------------------------------------------
 */
+#if XCHAL_CP_NUM > 0
     .macro get_cpsa_from_tcb reg_A reg_B
-    // Get TCB.pxEndOfStack from reg_A
+    /* Get TCB.pxEndOfStack from reg_A */
     movi    \reg_B, offset_pxEndOfStack     /* Move &offset_pxEndOfStack into reg_B */
     l32i    \reg_B, \reg_B, 0               /* Load offset_pxEndOfStack into reg_B */
     add     \reg_A, \reg_A, \reg_B          /* Calculate &pxEndOfStack to reg_A (&TCB + offset_pxEndOfStack) */
     l32i    \reg_A, \reg_A, 0               /* Load TCB.pxEndOfStack into reg_A */
-    //Offset to start of coproc save area
+    /* Offset to start of CP save area */
     movi    \reg_B, offset_cpsa             /* Move &offset_cpsa into reg_B */
     l32i    \reg_B, \reg_B, 0               /* Load offset_cpsa into reg_B */
     sub     \reg_A, \reg_A, \reg_B          /* Subtract offset_cpsa from pxEndOfStack to get to start of CP save area (unaligned) */
-    //Align down start of CP save area to 16 byte boundary
+    /* Align down start of CP save area to 16 byte boundary */
     movi    \reg_B, ~(0xF)
-    and     \reg_A, \reg_A, \reg_B          /* Align CPSA pointer to 16 bytes */
+    and     \reg_A, \reg_A, \reg_B          /* Align CP save area pointer to 16 bytes */
     .endm
+#endif /* XCHAL_CP_NUM > 0 */
 
     .global     port_IntStack
     .global     port_switch_flag    //Required by sysview_tracing build
@@ -692,3 +705,54 @@ _frxt_task_coproc_state:
 2:  ret
 
 #endif /* XCHAL_CP_NUM > 0 */
+
+/*
+**********************************************************************************************************
+*                                        _frxt_coproc_exc_hook
+*                                   void _frxt_coproc_exc_hook(void)
+*
+* Implements the Xtensa RTOS porting layer's XT_RTOS_CP_EXC_HOOK function for FreeRTOS.
+*
+* May only be called from assembly code by the 'call0' instruction. Does NOT obey ABI conventions.
+* May only only use a2-4, a15 (all other regs must be preserved).
+* See the detailed description of the XT_RTOS_ENTER macro in xtensa_rtos.h.
+*
+**********************************************************************************************************
+*/
+#if XCHAL_CP_NUM > 0
+
+    .globl  _frxt_coproc_exc_hook
+    .type   _frxt_coproc_exc_hook,@function
+    .align  4
+_frxt_coproc_exc_hook:
+
+    #if configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1
+    getcoreid a2                            /* a2 = xCurCoreID */
+    /* if (port_xSchedulerRunning[xCurCoreID] == 0) */
+    movi    a3, port_xSchedulerRunning
+    addx4   a3, a2, a3
+    l32i    a3, a3, 0
+    beqz    a3, 1f                          /* Scheduler hasn't started yet. Return. */
+    /* if (port_interruptNesting[xCurCoreID] != 0) */
+    movi    a3, port_interruptNesting
+    addx4   a3, a2, a3
+    l32i    a3, a3, 0
+    bnez    a3, 1f                          /* We are in an interrupt. Return*/
+    /* CP operations are incompatible with unpinned tasks. Thus we pin the task
+    to the current running core by updating its TCB.uxCoreAffinityMask field. */
+    movi    a3, pxCurrentTCBs
+    addx4   a3, a2, a3
+    l32i    a3, a3, 0                       /* a3 = pxCurrentTCBs[xCurCoreID] */
+    movi    a4, offset_uxCoreAffinityMask
+    l32i    a4, a4, 0                       /* a4 = offset_uxCoreAffinityMask */
+    add     a3, a3, a4                      /* a3 = &TCB.uxCoreAffinityMask */
+    ssl     a2                              /* Use xCurCoreID as left shift amount */
+    movi    a4, 1
+    sll     a4, a4                          /* a4 = (1 << xCurCoreID) */
+    s32i    a4, a3, 0                       /* TCB.uxCoreAffinityMask = a4 = (1 << xCurCoreID) */
+1:
+    #endif /* configUSE_CORE_AFFINITY == 1 && configNUM_CORES > 1 */
+
+    ret
+
+#endif /* XCHAL_CP_NUM > 0 */

components/freertos/FreeRTOS-Kernel-SMP/portable/xtensa/xt_asm_utils.h

@@ -50,26 +50,88 @@
 
 .macro SPILL_ALL_WINDOWS
 #if XCHAL_NUM_AREGS == 64
-	and a12, a12, a12
-	rotw 3
-	and a12, a12, a12
-	rotw 3
-	and a12, a12, a12
-	rotw 3
-	and a12, a12, a12
-	rotw 3
-	and a12, a12, a12
-	rotw 4
+    and a12, a12, a12
+    rotw 3
+    and a12, a12, a12
+    rotw 3
+    and a12, a12, a12
+    rotw 3
+    and a12, a12, a12
+    rotw 3
+    and a12, a12, a12
+    rotw 4
 #elif XCHAL_NUM_AREGS == 32
-	and a12, a12, a12
-	rotw 3
-	and a12, a12, a12
-	rotw 3
-	and a4, a4, a4
-	rotw 2
+    and a12, a12, a12
+    rotw 3
+    and a12, a12, a12
+    rotw 3
+    and a4, a4, a4
+    rotw 2
 #else
 #error Unrecognized XCHAL_NUM_AREGS
 #endif
 .endm
 
-#endif
+/*
+--------------------------------------------------------------------------------
+  Macro spinlock_take
+
+  This macro will repeatedley attempt to atomically set a spinlock variable
+  using the s32c1i instruciton. A spinlock is considered free if its value is 0.
+
+  Entry:
+  - "reg_A/B" as scratch registers
+  - "lock_var" spinlock variable's symbol
+  - Interrupts must already be disabled by caller
+  Exit:
+  - Spinlock set to current core's ID (PRID)
+  - "reg_A/B" clobbered
+--------------------------------------------------------------------------------
+*/
+
+#if portNUM_PROCESSORS > 1
+
+    .macro  spinlock_take       reg_A reg_B lock_var
+
+    movi    \reg_A, \lock_var               /* reg_A = &lock_var */
+.L_spinlock_loop:
+    movi    \reg_B, 0                       /* Load spinlock free value (0) into SCOMPARE1 */
+    wsr     \reg_B, SCOMPARE1
+    rsync                                   /* Ensure that SCOMPARE1 is set before s32c1i executes */
+    rsr     \reg_B, PRID                    /* Load the current core's ID into reg_B */
+    s32c1i  \reg_B, \reg_A, 0               /* Attempt *lock_var = reg_B */
+    bnez    \reg_B, .L_spinlock_loop        /* If the write was successful (i.e., lock was free), 0 will have been written back to reg_B */
+
+    .endm
+
+#endif /* portNUM_PROCESSORS > 1 */
+
+/*
+--------------------------------------------------------------------------------
+  Macro spinlock_release
+
+  This macro will release a spinlock variable previously taken by the
+  spinlock_take macro.
+
+  Entry:
+  - "reg_A/B" as scratch registers
+  - "lock_var" spinlock variable's symbol
+  - Interrupts must already be disabled by caller
+  Exit:
+  - "reg_A/B" clobbered
+--------------------------------------------------------------------------------
+*/
+
+#if portNUM_PROCESSORS > 1
+
+    .macro spinlock_release     reg_A reg_B lock_var
+
+    movi    \reg_A, \lock_var               /* reg_A = &lock_var */
+    movi    \reg_B, 0
+    s32i    \reg_B, \reg_A, 0               /* Release the spinlock (*reg_A = 0) */
+
+    .endm
+
+#endif /* portNUM_PROCESSORS > 1 */
+
+#endif /* __XT_ASM_UTILS_H */