FEXCore: Moves XID check to the frontend

The XID signal handler is OS specific and needs to be delegated to being
handled in the frontend. This only happens when we create a thread with
pthread rather than clone/fork/vfork.

The first created pthread will reset the XID signal handler in glibc,
which we need to check the first time this occurs. A little bit disjoint
while the actual thread creation still happens in the backend but will
come together once the thread creation gets moved to the frontend.

The only game I am aware of that requires this to work is SOMA since its
one of its middleware libraries tries asking for root privileges.

Retested the game to make sure it still works as expected.

FEXCore/Source/Interface/Context/Context.h

@@ -254,7 +254,6 @@ namespace FEXCore::Context {
     FEXCore::Core::InternalThreadState* ParentThread{};
     fextl::vector<FEXCore::Core::InternalThreadState*> Threads;
     std::atomic_bool CoreShuttingDown{false};
-    bool NeedToCheckXID{true};
 
     std::mutex IdleWaitMutex;
     std::condition_variable IdleWaitCV;

FEXCore/Source/Interface/Core/Core.cpp

@@ -530,21 +530,10 @@ namespace FEXCore::Context {
     ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
     Arg->This = this;
     Arg->Thread = Thread;
-    Thread->StartPaused = NeedToCheckXID;
     Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
 
     // Wait for the thread to have started
     Thread->ThreadWaiting.Wait();
-
-    if (NeedToCheckXID) {
-      // The first time an application creates a thread, GLIBC installs their SETXID signal handler.
-      // FEX needs to capture all signals and defer them to the guest.
-      // Once FEX creates its first guest thread, overwrite the GLIBC SETXID handler *again* to ensure
-      // FEX maintains control of the signal handler on this signal.
-      NeedToCheckXID = false;
-      SignalDelegation->CheckXIDHandler();
-      Thread->StartRunning.NotifyAll();
-    }
   }
 
   void ContextImpl::InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread) {

FEXCore/include/FEXCore/Core/SignalDelegator.h

@@ -47,14 +47,6 @@ namespace Core {
     virtual void RegisterTLSState(FEXCore::Core::InternalThreadState *Thread) = 0;
     virtual void UninstallTLSState(FEXCore::Core::InternalThreadState *Thread) = 0;
 
-    /**
-     * @brief Check to ensure the XID handler is still set to the FEX handler
-     *
-     * On a new thread GLIBC will set the XID handler underneath us.
-     * After the first thread is created check this.
-     */
-    virtual void CheckXIDHandler() = 0;
-
     struct SignalDelegatorConfig {
       bool StaticRegisterAllocation{};
       bool SupportsAVX{};

Source/Tools/CommonTools/DummyHandlers.h

@@ -25,8 +25,6 @@ class DummySyscallHandler: public FEXCore::HLE::SyscallHandler, public FEXCore::
 
 class DummySignalDelegator final : public FEXCore::SignalDelegator, public FEXCore::Allocator::FEXAllocOperators {
   public:
-  void CheckXIDHandler() override {}
-
   void SignalThread(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::SignalEvent Event) override {}
 
   FEXCore::Core::InternalThreadState *GetBackingTLSThread() {

Source/Tools/FEXLoader/LinuxSyscalls/SignalDelegator.h

@@ -89,7 +89,13 @@ namespace FEX::HLE {
       uint64_t GuestSignalFD(int fd, const uint64_t *set, size_t sigsetsize , int flags);
     /**  @} */
 
-      void CheckXIDHandler() override;
+    /**
+     * @brief Check to ensure the XID handler is still set to the FEX handler
+     *
+     * On a new thread GLIBC will set the XID handler underneath us.
+     * After the first thread is created check this.
+     */
+      void CheckXIDHandler();
 
       void UninstallHostHandler(int Signal);
       FEXCore::Context::Context *CTX;

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls.h

@@ -225,6 +225,9 @@ public:
 
   SourcecodeResolver *GetSourcecodeResolver() override { return this; }
 
+  bool NeedXIDCheck() const { return NeedToCheckXID; }
+  void DisableXIDCheck() { NeedToCheckXID = false; }
+
 protected:
   SyscallHandler(FEXCore::Context::Context *_CTX, FEX::HLE::SignalDelegator *_SignalDelegation);
 
@@ -250,6 +253,7 @@ private:
   std::mutex FutexMutex;
   std::mutex SyscallMutex;
   FEXCore::CodeLoader *LocalLoader{};
+  bool NeedToCheckXID{true};
 
   #ifdef DEBUG_STRACE
     void Strace(FEXCore::HLE::SyscallArguments *Args, uint64_t Ret);

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp

@@ -59,8 +59,19 @@ namespace FEX::HLE {
     }
 
     auto NewThread = CTX->CreateThread(0, 0, &NewThreadState, args->args.parent_tid);
+    bool NeedsXIDCheck = FEX::HLE::_SyscallHandler->NeedXIDCheck();
+    NewThread->StartPaused = NeedsXIDCheck;
     CTX->InitializeThread(NewThread);
 
+    if (NeedsXIDCheck) {
+      // The first time an application creates a thread, GLIBC installs their SETXID signal handler.
+      // FEX needs to capture all signals and defer them to the guest.
+      // Once FEX creates its first guest thread, overwrite the GLIBC SETXID handler *again* to ensure
+      // FEX maintains control of the signal handler on this signal.
+      FEX::HLE::_SyscallHandler->GetSignalDelegator()->CheckXIDHandler();
+      FEX::HLE::_SyscallHandler->DisableXIDCheck();
+    }
+
     if (FEX::HLE::_SyscallHandler->Is64BitMode()) {
       if (flags & CLONE_SETTLS) {
         x64::SetThreadArea(NewThread->CurrentFrame, reinterpret_cast<void*>(args->args.tls));