[release/7.0] Prevent unwinding through stack bottom (#81804)

* Prevent unwinding through stack bottom

When processing unhandled exception on the most recent Alpine 3.17,
the libunwind doesn't stop at the bottom
frame of the main thread (the caller of `main`) and tries to unwind
further. The reason is that the method is missing dwarf unwind
information, so the libunwind falls back to using RBP chain, but the RBP
points to a garbage and so it ends up crashing with SIGSEGV.

While the missing DWARF unwind info seems to be a bug in the Alpine 3.17
(older ones work fine), we can prevent issues like this by stopping at
the hosting API boundary and not trying to unwind past that. This is
what this PR does.

* Fix bug introduced by preventing unwind through stack bottom (#81956)

The irecent fix to prevent unwinding through stack bottom was
incorrect for secondary threads, as it just compared the SP
being above the frame of the hosting API. However, other threads
can have their stacks anywhere in the memory, thus this
sometimes broke exception handling on secondary threads.

I have also found that there was one more case where the
unwind through the hosting API need to be checked - the
Thread::VirtualUnwindToFirstManagedCallFrame.

I have decided to use the return address of the hosting
API for the checks instead of the frame address. That
makes the check work properly.

---------

Co-authored-by: Jan Vorlicek <janvorli@microsoft.com>

src/coreclr/dlls/mscoree/exports.cpp

@@ -23,6 +23,25 @@
 
 #define ASSERTE_ALL_BUILDS(expr) _ASSERTE_ALL_BUILDS((expr))
 
+#ifdef TARGET_UNIX
+#define NO_HOSTING_API_RETURN_ADDRESS ((void*)ULONG_PTR_MAX)
+void* g_hostingApiReturnAddress = NO_HOSTING_API_RETURN_ADDRESS;
+
+class HostingApiFrameHolder
+{
+public:
+    HostingApiFrameHolder(void* returnAddress)
+    {
+        g_hostingApiReturnAddress = returnAddress;
+    }
+
+    ~HostingApiFrameHolder()
+    {
+        g_hostingApiReturnAddress = NO_HOSTING_API_RETURN_ADDRESS;
+    }
+};
+#endif // TARGET_UNIX
+
 // Holder for const wide strings
 typedef NewArrayHolder<const WCHAR> ConstWStringHolder;
 
@@ -194,6 +213,7 @@ extern "C" int coreclr_create_delegate(void*, unsigned int, const char*, const c
 //  HRESULT indicating status of the operation. S_OK if the assembly was successfully executed
 //
 extern "C"
+NOINLINE
 DLLEXPORT
 int coreclr_initialize(
             const char* exePath,
@@ -212,6 +232,10 @@ int coreclr_initialize(
     bool hostPolicyEmbedded = false;
     PInvokeOverrideFn* pinvokeOverride = nullptr;
 
+#ifdef TARGET_UNIX
+    HostingApiFrameHolder apiFrameHolder(_ReturnAddress());
+#endif
+
     ConvertConfigPropertiesToUnicode(
         propertyKeys,
         propertyValues,
@@ -420,6 +444,7 @@ int coreclr_create_delegate(
 //  HRESULT indicating status of the operation. S_OK if the assembly was successfully executed
 //
 extern "C"
+NOINLINE
 DLLEXPORT
 int coreclr_execute_assembly(
             void* hostHandle,
@@ -435,6 +460,10 @@ int coreclr_execute_assembly(
     }
     *exitCode = -1;
 
+#ifdef TARGET_UNIX
+    HostingApiFrameHolder apiFrameHolder(_ReturnAddress());
+#endif
+
     ICLRRuntimeHost4* host = reinterpret_cast<ICLRRuntimeHost4*>(hostHandle);
 
     ConstWStringArrayHolder argvW;

src/coreclr/vm/exceptionhandling.cpp

@@ -4523,6 +4523,8 @@ VOID UnwindManagedExceptionPass2(PAL_SEHException& ex, CONTEXT* unwindStartConte
     EEPOLICY_HANDLE_FATAL_ERROR(COR_E_EXECUTIONENGINE);
 }
 
+extern void* g_hostingApiReturnAddress;
+
 //---------------------------------------------------------------------------------------
 //
 // This functions performs dispatching of a managed exception.
@@ -4724,7 +4726,8 @@ VOID DECLSPEC_NORETURN UnwindManagedExceptionPass1(PAL_SEHException& ex, CONTEXT
 
             STRESS_LOG2(LF_EH, LL_INFO100, "Processing exception at native frame: IP = %p, SP = %p \n", controlPc, sp);
 
-            if (controlPc == 0)
+            // Consider the exception unhandled if the unwinding cannot proceed further or if it went past the coreclr_initialize or coreclr_execute_assembly
+            if ((controlPc == 0) || (controlPc == (UINT_PTR)g_hostingApiReturnAddress))
             {
                 if (!GetThread()->HasThreadStateNC(Thread::TSNC_ProcessedUnhandledException))
                 {

src/coreclr/vm/stackwalk.cpp

@@ -709,6 +709,8 @@ PCODE Thread::VirtualUnwindNonLeafCallFrame(T_CONTEXT* pContext, KNONVOLATILE_CO
     return uControlPc;
 }
 
+extern void* g_hostingApiReturnAddress;
+
 // static
 UINT_PTR Thread::VirtualUnwindToFirstManagedCallFrame(T_CONTEXT* pContext)
 {
@@ -751,8 +753,9 @@ UINT_PTR Thread::VirtualUnwindToFirstManagedCallFrame(T_CONTEXT* pContext)
 
         uControlPc = GetIP(pContext);
 
-        if (uControlPc == 0)
+        if ((uControlPc == 0) || (uControlPc == (PCODE)g_hostingApiReturnAddress))
         {
+            uControlPc = 0;
             break;
         }
 #endif // !TARGET_UNIX