WIP: Moves Thread state tracking to frontend

The FEXCore ThreadManagement object has been a byproduct of leaking
Linux details in to FEXCore for a long time. Since we have moved most of
the thread management to the frontend, we can finally move this out of
FEXCore and start taking advantage of the `FrontendPtr`.

FEXCore/Source/Interface/Context/Context.h

@@ -130,7 +130,7 @@ namespace FEXCore::Context {
        *    - HandleCallback(Thread, RIP);
        */
 
-      FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) override;
+      FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState) override;
 
       // Public for threading
       void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) override;
@@ -277,7 +277,6 @@ namespace FEXCore::Context {
     static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
       auto Thread = Frame->Thread;
 
-      LOGMAN_THROW_A_FMT(Thread->ThreadManager.GetTID() == FHU::Syscalls::gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), FHU::Syscalls::gettid());
       auto lk = GuardSignalDeferringSection(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
       ThreadRemoveCodeEntry(Thread, GuestRIP);

FEXCore/Source/Interface/Core/Core.cpp

@@ -38,7 +38,6 @@ desc: Glues Frontend, OpDispatcher and IR Opts & Compilation, LookupCache, Dispa
 #include <FEXCore/Debug/InternalThreadState.h>
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <FEXCore/HLE/SourcecodeResolver.h>
-#include <FEXCore/HLE/Linux/ThreadManagement.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/RegisterAllocationData.h>
@@ -354,10 +353,6 @@ namespace FEXCore::Context {
 
 
   void ContextImpl::InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread) {
-    // Let's do some initial bookkeeping here
-    Thread->ThreadManager.TID = FHU::Syscalls::gettid();
-    Thread->ThreadManager.PID = ::getpid();
-
     if (ThunkHandler) {
       ThunkHandler->RegisterTLSState(Thread);
     }
@@ -402,7 +397,7 @@ namespace FEXCore::Context {
     Thread->PassManager->Finalize();
   }
 
-  FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID) {
+  FEXCore::Core::InternalThreadState* ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState) {
     FEXCore::Core::InternalThreadState *Thread = new FEXCore::Core::InternalThreadState{};
 
     Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
@@ -413,8 +408,6 @@ namespace FEXCore::Context {
       memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState));
     }
 
-    // Set up the thread manager state
-    Thread->ThreadManager.parent_tid = ParentTID;
     Thread->CurrentFrame->Thread = Thread;
 
     InitializeCompiler(Thread);
@@ -909,16 +902,6 @@ namespace FEXCore::Context {
 
     // If it is the parent thread that died then just leave
     FEX_TODO("This doesn't make sense when the parent thread doesn't outlive its children");
-
-    if (Thread->ThreadManager.parent_tid == 0) {
-      CoreShuttingDown.store(true);
-      Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_SHUTDOWN;
-
-      if (CustomExitHandler) {
-        CustomExitHandler(Thread->ThreadManager.TID, Thread->ExitReason);
-      }
-    }
-
 #ifndef _WIN32
     Alloc::OSAllocator::UninstallTLSData(Thread);
 #endif

FEXCore/include/FEXCore/Core/Context.h

@@ -201,7 +201,7 @@ namespace FEXCore::Context {
        * @return A new InternalThreadState object for using with a new guest thread.
        */
 
-      FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState = nullptr, uint64_t ParentTID = 0) = 0;
+      FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState = nullptr) = 0;
 
       FEX_DEFAULT_VISIBILITY virtual void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void DestroyThread(FEXCore::Core::InternalThreadState *Thread, bool NeedsTLSUninstall = false) = 0;

FEXCore/include/FEXCore/Core/CoreState.h

@@ -3,7 +3,6 @@
 
 #include "FEXCore/Core/X86Enums.h"
 #include "FEXCore/IR/IR.h"
-#include <FEXCore/HLE/Linux/ThreadManagement.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/Telemetry.h>
 #include <FEXCore/Core/CPUBackend.h>

FEXCore/include/FEXCore/Debug/InternalThreadState.h

@@ -108,7 +108,6 @@ namespace FEXCore::Core {
 
     fextl::unique_ptr<FEXCore::Frontend::Decoder> FrontendDecoder;
     fextl::unique_ptr<FEXCore::IR::PassManager> PassManager;
-    FEXCore::HLE::ThreadManagement ThreadManager;
     fextl::unique_ptr<JITSymbolBuffer> SymbolBuffer;
 
     int StatusCode{};

Source/Tools/FEXLoader/FEXLoader.cpp

@@ -560,7 +560,7 @@ int main(int argc, char **argv, char **const envp) {
       FEX::AOT::AOTGenSection(CTX.get(), Section);
     }
   } else {
-    CTX->RunUntilExit(ParentThread);
+    CTX->RunUntilExit(ParentThread->Thread);
   }
 
   if (AOTEnabled) {
@@ -581,10 +581,10 @@ int main(int argc, char **argv, char **const envp) {
     }
   }
 
-  auto ProgramStatus = ParentThread->StatusCode;
+  auto ProgramStatus = ParentThread->Thread->StatusCode;
 
   SignalDelegation->UninstallTLSState(ParentThread);
-  CTX->DestroyThread(ParentThread);
+  SyscallHandler->TM.DestroyThread(ParentThread);
 
   DebugServer.reset();
   SyscallHandler.reset();

Source/Tools/LinuxEmulation/LinuxSyscalls/GdbServer.cpp

@@ -22,7 +22,6 @@ desc: Provides a gdb interface to the guest state
 #include <FEXCore/Core/SignalDelegator.h>
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Debug/InternalThreadState.h>
-#include <FEXCore/HLE/Linux/ThreadManagement.h>
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/FileLoading.h>
@@ -341,29 +340,29 @@ fextl::string GdbServer::readRegs() {
   FEXCore::Core::CPUState state{};
 
   auto Threads = SyscallHandler->TM.GetThreads();
-  FEXCore::Core::InternalThreadState *CurrentThread { Threads->at(0) };
+  FEX::HLE::ThreadStateObject *CurrentThread { Threads->at(0) };
   bool Found = false;
 
   for (auto &Thread : *Threads) {
     if (Thread->ThreadManager.GetTID() != CurrentDebuggingThread) {
       continue;
     }
-    memcpy(&state, Thread->CurrentFrame, sizeof(state));
+    memcpy(&state, Thread->Thread->CurrentFrame, sizeof(state));
     CurrentThread = Thread;
     Found = true;
     break;
   }
 
   if (!Found) {
     // If set to an invalid thread then just get the parent thread ID
-    memcpy(&state, CurrentThread->CurrentFrame, sizeof(state));
+    memcpy(&state, CurrentThread->Thread->CurrentFrame, sizeof(state));
   }
 
   // Encode the GDB context definition
   memcpy(&GDB.gregs[0], &state.gregs[0], sizeof(GDB.gregs));
   memcpy(&GDB.rip, &state.rip, sizeof(GDB.rip));
 
-  GDB.eflags = CTX->ReconstructCompactedEFLAGS(CurrentThread, false, nullptr, 0);
+  GDB.eflags = CTX->ReconstructCompactedEFLAGS(CurrentThread->Thread, false, nullptr, 0);
 
   for (size_t i = 0; i < FEXCore::Core::CPUState::NUM_MMS; ++i) {
     memcpy(&GDB.mm[i], &state.mm[i], sizeof(GDB.mm));
@@ -392,22 +391,22 @@ GdbServer::HandledPacketType GdbServer::readReg(const fextl::string& packet) {
   FEXCore::Core::CPUState state{};
 
   auto Threads = SyscallHandler->TM.GetThreads();
-  FEXCore::Core::InternalThreadState *CurrentThread { Threads->at(0) };
+  FEX::HLE::ThreadStateObject *CurrentThread { Threads->at(0) };
   bool Found = false;
 
   for (auto &Thread : *Threads) {
     if (Thread->ThreadManager.GetTID() != CurrentDebuggingThread) {
       continue;
     }
-    memcpy(&state, Thread->CurrentFrame, sizeof(state));
+    memcpy(&state, Thread->Thread->CurrentFrame, sizeof(state));
     CurrentThread = Thread;
     Found = true;
     break;
   }
 
   if (!Found) {
     // If set to an invalid thread then just get the parent thread ID
-    memcpy(&state, CurrentThread->CurrentFrame, sizeof(state));
+    memcpy(&state, CurrentThread->Thread->CurrentFrame, sizeof(state));
   }
 
 
@@ -419,7 +418,7 @@ GdbServer::HandledPacketType GdbServer::readReg(const fextl::string& packet) {
     return {encodeHex((unsigned char *)(&state.rip), sizeof(uint64_t)), HandledPacketType::TYPE_ACK};
   }
   else if (addr == offsetof(GDBContextDefinition, eflags)) {
-    uint32_t eflags = CTX->ReconstructCompactedEFLAGS(CurrentThread, false, nullptr, 0);
+    uint32_t eflags = CTX->ReconstructCompactedEFLAGS(CurrentThread->Thread, false, nullptr, 0);
 
     return {encodeHex((unsigned char *)(&eflags), sizeof(uint32_t)), HandledPacketType::TYPE_ACK};
   }

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -14,7 +14,6 @@ desc: Handles host -> host and host -> guest signal routing, emulates procmask &
 #include <FEXCore/Core/SignalDelegator.h>
 #include <FEXCore/Core/X86Enums.h>
 #include <FEXCore/Debug/InternalThreadState.h>
-#include <FEXCore/HLE/Linux/ThreadManagement.h>
 #include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/LogManager.h>
 #include <FEXCore/Utils/MathUtils.h>
@@ -57,7 +56,7 @@ namespace FEX::HLE {
   static SignalDelegator *GlobalDelegator{};
 
   struct ThreadState {
-    FEXCore::Core::InternalThreadState *Thread{};
+    FEX::HLE::ThreadStateObject *Thread{};
 
     void *AltStackPtr{};
     stack_t GuestAltStack {
@@ -153,12 +152,14 @@ namespace FEX::HLE {
 
   void SignalDelegator::HandleSignal(int Signal, void *Info, void *UContext) {
     // Let the host take first stab at handling the signal
-    auto Thread = GetTLSThread();
+    auto ThreadDataObject = GetTLSThread();
 
-    if (!Thread) {
+    if (!ThreadDataObject) {
       LogMan::Msg::AFmt("[{}] Thread has received a signal and hasn't registered itself with the delegate! Programming error!", FHU::Syscalls::gettid());
     }
     else {
+      auto Thread = ThreadDataObject->Thread;
+
       SignalHandler &Handler = HostHandlers[Signal];
       for (auto &HandlerFunc : Handler.Handlers) {
         if (HandlerFunc(Thread, Signal, Info, UContext)) {
@@ -1387,7 +1388,9 @@ namespace FEX::HLE {
       return;
     }
     Thread->SignalReason.store(Event);
-    FHU::Syscalls::tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
+    auto ThreadObject = static_cast<FEX::HLE::ThreadStateObject *>(Thread->FrontendPtr);
+
+    FHU::Syscalls::tgkill(ThreadObject->ThreadManager.PID, ThreadObject->ThreadManager.TID, SignalDelegator::SIGNAL_FOR_PAUSE);
   }
 
   /**  @} */
@@ -1809,11 +1812,11 @@ namespace FEX::HLE {
     GlobalDelegator = nullptr;
   }
 
-  FEXCore::Core::InternalThreadState *SignalDelegator::GetTLSThread() {
+  FEX::HLE::ThreadStateObject *SignalDelegator::GetTLSThread() {
     return ThreadData.Thread;
   }
 
-  void SignalDelegator::RegisterTLSState(FEXCore::Core::InternalThreadState *Thread) {
+  void SignalDelegator::RegisterTLSState(FEX::HLE::ThreadStateObject *Thread) {
     ThreadData.Thread = Thread;
 
     // Set up our signal alternative stack
@@ -1834,14 +1837,14 @@ namespace FEX::HLE {
     // Get the current host signal mask
     ::syscall(SYS_rt_sigprocmask, 0, nullptr, &ThreadData.CurrentSignalMask.Val, 8);
 
-    if (Thread != (FEXCore::Core::InternalThreadState*)UINTPTR_MAX) {
+    if (Thread->Thread) {
       // Reserve a small amount of deferred signal frames. Usually the stack won't be utilized beyond
       // 1 or 2 signals but add a few more just in case.
-      Thread->DeferredSignalFrames.reserve(8);
+      Thread->Thread->DeferredSignalFrames.reserve(8);
     }
   }
 
-  void SignalDelegator::UninstallTLSState(FEXCore::Core::InternalThreadState *Thread) {
+  void SignalDelegator::UninstallTLSState(FEX::HLE::ThreadStateObject *Thread) {
     FEXCore::Allocator::munmap(ThreadData.AltStackPtr, SIGSTKSZ * 16);
 
     ThreadData.AltStackPtr = nullptr;
@@ -1944,7 +1947,7 @@ namespace FEX::HLE {
     bool UsingAltStack{};
     uint64_t AltStackBase = reinterpret_cast<uint64_t>(ThreadData.GuestAltStack.ss_sp);
     uint64_t AltStackEnd = AltStackBase + ThreadData.GuestAltStack.ss_size;
-    uint64_t GuestSP = Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RSP];
+    uint64_t GuestSP = Thread->Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RSP];
 
     if (!(ThreadData.GuestAltStack.ss_flags & SS_DISABLE) &&
         GuestSP >= AltStackBase &&
@@ -2004,7 +2007,7 @@ namespace FEX::HLE {
     if (PendingSignals != 0) {
       for (int i = 0; i < 64; ++i) {
         if (PendingSignals & (1ULL << i)) {
-          FHU::Syscalls::tgkill(Thread->ThreadManager.PID, Thread->ThreadManager.TID, i + 1);
+          FHU::Syscalls::tgkill(ThreadData.Thread->ThreadManager.PID, ThreadData.Thread->ThreadManager.TID, i + 1);
           // We might not even return here which is spooky
         }
       }
@@ -2053,7 +2056,7 @@ namespace FEX::HLE {
       *oldset = OldSet;
     }
 
-    CheckForPendingSignals(GetTLSThread());
+    CheckForPendingSignals(GetTLSThread()->Thread);
 
     return 0;
   }
@@ -2116,7 +2119,7 @@ namespace FEX::HLE {
     // then this is safe-ish
     ThreadData.CurrentSignalMask = ThreadData.PreviousSuspendMask;
 
-    CheckForPendingSignals(GetTLSThread());
+    CheckForPendingSignals(GetTLSThread()->Thread);
 
     return Result == -1 ? -errno : Result;
 

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.h

@@ -35,6 +35,7 @@ namespace Core {
 }
 
 namespace FEX::HLE {
+  struct ThreadStateObject;
   using HostSignalDelegatorFunction = std::function<bool(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext)>;
   using HostSignalDelegatorFunctionForGuest = std::function<bool(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack)>;
 
@@ -54,8 +55,8 @@ namespace FEX::HLE {
     // Called from the signal trampoline function.
     void HandleSignal(int Signal, void *Info, void *UContext);
 
-    void RegisterTLSState(FEXCore::Core::InternalThreadState *Thread);
-    void UninstallTLSState(FEXCore::Core::InternalThreadState *Thread);
+    void RegisterTLSState(FEX::HLE::ThreadStateObject *Thread);
+    void UninstallTLSState(FEX::HLE::ThreadStateObject *Thread);
 
     /**
      * @brief Registers a signal handler for the host to handle a signal
@@ -130,7 +131,7 @@ namespace FEX::HLE {
 
     void SaveTelemetry();
   private:
-    FEXCore::Core::InternalThreadState *GetTLSThread();
+    FEX::HLE::ThreadStateObject *GetTLSThread();
 
     // Called from the thunk handler to handle the signal
     void HandleGuestSignal(FEXCore::Core::InternalThreadState *Thread, int Signal, void *Info, void *UContext);