Merge pull request #205 from Sonicadvance1/threaded_compilation_fix

Makes threaded compilation not rely on a global mutex

External/FEXCore/Source/Interface/Context/Context.h

@@ -75,8 +75,6 @@ namespace FEXCore::Context {
     Event PauseWait;
     bool Running{};
     CoreRunningMode RunningMode {CoreRunningMode::MODE_RUN};
-    FEXCore::Frontend::Decoder FrontendDecoder;
-    FEXCore::IR::PassManager PassManager;
 
     FEXCore::CPUIDEmu CPUID;
     std::unique_ptr<FEXCore::SyscallHandler> SyscallHandler;
@@ -133,8 +131,6 @@ namespace FEXCore::Context {
     void RunThread(FEXCore::Core::InternalThreadState *Thread);
 
   protected:
-    IR::RegisterAllocationPass *GetRegisterAllocatorPass();
-    bool HasRegisterAllocationPass() const { return RAPass != nullptr; }
     void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
 
   private:
@@ -157,7 +153,6 @@ namespace FEXCore::Context {
     std::set<uint64_t> EntryList;
     std::vector<uint64_t> InitLocations;
     uint64_t StartingRIP;
-    IR::RegisterAllocationPass *RAPass {};
     std::mutex ExitMutex;
     std::unique_ptr<GdbServer> DebugServer;
 

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -124,14 +124,8 @@ namespace DefaultFallbackCore {
 }
 
 namespace FEXCore::Context {
-  Context::Context()
-    : FrontendDecoder {this} {
+  Context::Context() {
     FallbackCPUFactory = FEXCore::Core::DefaultFallbackCore::CPUCreationFactory;
-    PassManager.RegisterExitHandler([this]() {
-      ShouldStop = true;
-    });
-    PassManager.AddDefaultPasses();
-    PassManager.AddDefaultValidationPasses();
 #ifdef BLOCKSTATS
     BlockData = std::make_unique<FEXCore::BlockSamplingData>();
 #endif
@@ -234,7 +228,6 @@ namespace FEXCore::Context {
 
   bool Context::InitCore(FEXCore::CodeLoader *Loader) {
     SyscallHandler.reset(FEXCore::CreateHandler(Config.Is64BitMode ? OperatingMode::MODE_64BIT : OperatingMode::MODE_32BIT, this));
-    PassManager.RegisterSyscallHandler(SyscallHandler.get());
     LocalLoader = Loader;
     using namespace FEXCore::Core;
     FEXCore::Core::CPUState NewThreadState{};
@@ -389,9 +382,9 @@ namespace FEXCore::Context {
     Thread->IntBackend->Initialize();
     Thread->FallbackBackend->Initialize();
 
-    auto IRHandler = [this, Thread](uint64_t Addr, IR::IREmitter *IR) -> void {
+    auto IRHandler = [Thread](uint64_t Addr, IR::IREmitter *IR) -> void {
       // Run the passmanager over the IR from the dispatcher
-      PassManager.Run(IR);
+      Thread->PassManager->Run(IR);
       Thread->IRLists.try_emplace(Addr, IR->CreateIRCopy());
     };
 
@@ -433,6 +426,15 @@ namespace FEXCore::Context {
     Thread->OpDispatcher = std::make_unique<FEXCore::IR::OpDispatchBuilder>(this);
     Thread->OpDispatcher->SetMultiblock(Config.Multiblock);
     Thread->BlockCache = std::make_unique<FEXCore::BlockCache>(this);
+    Thread->FrontendDecoder = std::make_unique<FEXCore::Frontend::Decoder>(this);
+    Thread->PassManager = std::make_unique<FEXCore::IR::PassManager>();
+    Thread->PassManager->RegisterExitHandler([this]() {
+      ShouldStop = true;
+    });
+    Thread->PassManager->AddDefaultPasses();
+    Thread->PassManager->AddDefaultValidationPasses();
+    Thread->PassManager->RegisterSyscallHandler(SyscallHandler.get());
+
     Thread->CTX = this;
 
     // Copy over the new thread state to the new object
@@ -447,7 +449,10 @@ namespace FEXCore::Context {
       Thread->CPUBackend.reset(FEXCore::CPU::CreateInterpreterCore(this));
       Thread->IntBackend = Thread->CPUBackend;
       break;
-    case FEXCore::Config::CONFIG_IRJIT:       Thread->CPUBackend.reset(FEXCore::CPU::CreateJITCore(this, Thread)); break;
+    case FEXCore::Config::CONFIG_IRJIT:
+      Thread->PassManager->InsertRAPass(IR::CreateRegisterAllocationPass());
+      Thread->CPUBackend.reset(FEXCore::CPU::CreateJITCore(this, Thread));
+      break;
     case FEXCore::Config::CONFIG_LLVMJIT:     Thread->CPUBackend.reset(FEXCore::CPU::CreateLLVMCore(Thread)); break;
     case FEXCore::Config::CONFIG_CUSTOM:      Thread->CPUBackend.reset(CustomCPUFactory(this, &Thread->State)); break;
     default: LogMan::Msg::A("Unknown core configuration");
@@ -463,15 +468,6 @@ namespace FEXCore::Context {
     return Thread;
   }
 
-  IR::RegisterAllocationPass *Context::GetRegisterAllocatorPass() {
-    if (!RAPass) {
-      RAPass = IR::CreateRegisterAllocationPass();
-      PassManager.InsertRAPass(RAPass);
-    }
-
-    return RAPass;
-  }
-
   uintptr_t Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
     auto BlockMapPtr = Thread->BlockCache->AddBlockMapping(Address, Ptr);
     if (BlockMapPtr == 0) {
@@ -502,10 +498,6 @@ namespace FEXCore::Context {
   }
 
   uintptr_t Context::CompileBlock(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
-    // XXX: Threaded mutex hack until we support proper threaded compilation. Issue #13
-    static std::mutex SyscallMutex;
-    std::scoped_lock<std::mutex> lk(SyscallMutex);
-
     void *CodePtr {nullptr};
     uint8_t const *GuestCode{};
     if (Thread->CTX->Config.UnifiedMemory) {
@@ -526,7 +518,7 @@ namespace FEXCore::Context {
       uint64_t TotalInstructions {0};
       uint64_t TotalInstructionsLength {0};
 
-      if (!FrontendDecoder.DecodeInstructionsAtEntry(GuestCode, GuestRIP)) {
+      if (!Thread->FrontendDecoder->DecodeInstructionsAtEntry(GuestCode, GuestRIP)) {
         if (Config.BreakOnFrontendFailure) {
            LogMan::Msg::E("Had Frontend decoder error");
            ShouldStop = true;
@@ -534,7 +526,7 @@ namespace FEXCore::Context {
         return 0;
       }
 
-      auto CodeBlocks = FrontendDecoder.GetDecodedBlocks();
+      auto CodeBlocks = Thread->FrontendDecoder->GetDecodedBlocks();
 
       Thread->OpDispatcher->BeginFunction(GuestRIP, CodeBlocks);
 
@@ -598,12 +590,12 @@ namespace FEXCore::Context {
       Thread->OpDispatcher->Finalize();
 
       // Run the passmanager over the IR from the dispatcher
-      PassManager.Run(Thread->OpDispatcher.get());
+      Thread->PassManager->Run(Thread->OpDispatcher.get());
 
       if (Thread->OpDispatcher->ShouldDump) {
         std::stringstream out;
         auto NewIR = Thread->OpDispatcher->ViewIR();
-        FEXCore::IR::Dump(&out, &NewIR, RAPass);
+        FEXCore::IR::Dump(&out, &NewIR, Thread->PassManager->GetRAPass());
         printf("IR 0x%lx:\n%s\n@@@@@\n", GuestRIP, out.str().c_str());
       }
 

External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -258,8 +258,7 @@ JITCore::JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadSt
   // XXX: Set this to a real minimum feature set in the future
   SetCPUFeatures(vixl::CPUFeatures::All());
 
-  bool HadRA = CTX->HasRegisterAllocationPass();
-  RAPass = CTX->GetRegisterAllocatorPass();
+  RAPass = Thread->PassManager->GetRAPass();
 
   // Just set the entire range as executable
   auto Buffer = GetBuffer();
@@ -285,20 +284,18 @@ JITCore::JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadSt
     UsedRegisterCount -= NumCalleeGPRs + NumCalleeGPRPairs;
   }
 
-  if (!HadRA) {
-    RAPass->AllocateRegisterSet(UsedRegisterCount, RegisterClasses);
+  RAPass->AllocateRegisterSet(UsedRegisterCount, RegisterClasses);
 
-    RAPass->AddRegisters(FEXCore::IR::GPRClass, NumUsedGPRs);
-    RAPass->AddRegisters(FEXCore::IR::FPRClass, NumFPRs);
-    RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumUsedGPRPairs);
+  RAPass->AddRegisters(FEXCore::IR::GPRClass, NumUsedGPRs);
+  RAPass->AddRegisters(FEXCore::IR::FPRClass, NumFPRs);
+  RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumUsedGPRPairs);
 
-    RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRClass, NumUsedGPRs);
-    RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRPairClass, NumUsedGPRs);
+  RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRClass, NumUsedGPRs);
+  RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRPairClass, NumUsedGPRs);
 
-    for (uint32_t i = 0; i < NumUsedGPRPairs; ++i) {
-      RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2,     FEXCore::IR::GPRPairClass, i);
-      RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
-    }
+  for (uint32_t i = 0; i < NumUsedGPRPairs; ++i) {
+    RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2,     FEXCore::IR::GPRPairClass, i);
+    RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
   }
 }
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp

@@ -123,22 +123,19 @@ JITCore::JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadSt
   , ThreadState {Thread} {
   Stack.resize(9000 * 16 * 64);
 
-  bool HadRA = CTX->HasRegisterAllocationPass();
-  RAPass = CTX->GetRegisterAllocatorPass();
+  RAPass = Thread->PassManager->GetRAPass();
 
-  if (!HadRA) {
-    RAPass->AllocateRegisterSet(RegisterCount, RegisterClasses);
-    RAPass->AddRegisters(FEXCore::IR::GPRClass, NumGPRs);
-    RAPass->AddRegisters(FEXCore::IR::FPRClass, NumXMMs);
-    RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumGPRPairs);
+  RAPass->AllocateRegisterSet(RegisterCount, RegisterClasses);
+  RAPass->AddRegisters(FEXCore::IR::GPRClass, NumGPRs);
+  RAPass->AddRegisters(FEXCore::IR::FPRClass, NumXMMs);
+  RAPass->AddRegisters(FEXCore::IR::GPRPairClass, NumGPRPairs);
 
-    RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRClass, NumGPRs);
-    RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRPairClass, NumGPRs);
+  RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRClass, NumGPRs);
+  RAPass->AllocateRegisterConflicts(FEXCore::IR::GPRPairClass, NumGPRs);
 
-    for (uint32_t i = 0; i < NumGPRPairs; ++i) {
-      RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2,     FEXCore::IR::GPRPairClass, i);
-      RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
-    }
+  for (uint32_t i = 0; i < NumGPRPairs; ++i) {
+    RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2,     FEXCore::IR::GPRPairClass, i);
+    RAPass->AddRegisterConflict(FEXCore::IR::GPRClass, i * 2 + 1, FEXCore::IR::GPRPairClass, i);
   }
 
   CreateCustomDispatch(Thread);

External/FEXCore/Source/Interface/IR/PassManager.h

@@ -40,6 +40,7 @@ public:
     Passes.emplace_back(Pass);
   }
   void InsertRAPass(Pass *Pass) {
+    RAPass = Pass;
     InsertPass(Pass);
   }
 

External/FEXCore/include/FEXCore/Debug/InternalThreadState.h

@@ -14,8 +14,14 @@ namespace FEXCore {
 namespace FEXCore::Context {
   struct Context;
 }
+
+namespace FEXCore::Frontend {
+  class Decoder;
+}
+
 namespace FEXCore::IR{
   class OpDispatchBuilder;
+  class PassManager;
 }
 
 namespace FEXCore::Core {
@@ -57,6 +63,10 @@ namespace FEXCore::Core {
 
     std::map<uint64_t, std::unique_ptr<FEXCore::IR::IRListView<true>>> IRLists;
     std::map<uint64_t, FEXCore::Core::DebugData> DebugData;
+
+    std::unique_ptr<FEXCore::Frontend::Decoder> FrontendDecoder;
+    std::unique_ptr<FEXCore::IR::PassManager> PassManager;
+
     RuntimeStats Stats{};
 
     int StatusCode{};