Core: Add support for process-wide code invalidation, rename IR invalidate op to do thread specific invalidation

External/FEXCore/Source/Interface/Context/Context.h

@@ -155,11 +155,13 @@ namespace FEXCore::Context {
     void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
     void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
 
-    static void RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
+    // Must be called from owning thread
+    static void RemoveThreadCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
 
     // Wrapper which takes CpuStateFrame instead of InternalThreadState
-    static void RemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
-      RemoveCodeEntry(Frame->Thread, GuestRIP);
+    // Must be called from owning thread
+    static void RemoveThreadCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
+      RemoveThreadCodeEntry(Frame->Thread, GuestRIP);
     }
 
     // Debugger interface

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -447,6 +447,7 @@ namespace FEXCore::Context {
           : nullptr),
         decltype(Entry.DebugData)(new Core::DebugData())
       };
+      std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
       Thread->LocalIRCache.insert({Addr, std::move(Entry)});
     };
 
@@ -626,6 +627,8 @@ namespace FEXCore::Context {
   }
 
   void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache) {
+    std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
+
     Thread->LookupCache->ClearCache();
     Thread->CPUBackend->ClearCache();
 
@@ -738,7 +741,7 @@ namespace FEXCore::Context {
           Thread->OpDispatcher->SetTrueJumpTarget(InvalidateCodeCond, CodeWasChangedBlock);
 
           Thread->OpDispatcher->SetCurrentCodeBlock(CodeWasChangedBlock);
-          Thread->OpDispatcher->_RemoveCodeEntry();
+          Thread->OpDispatcher->_RemoveThreadCodeEntry();
           Thread->OpDispatcher->_ExitFunction(Thread->OpDispatcher->_EntrypointOffset(Block.Entry + BlockInstructionsLength - GuestRIP, GPRSize));
 
           auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
@@ -847,6 +850,7 @@ namespace FEXCore::Context {
     uint64_t StartAddr {};
     uint64_t Length {};
 
+    std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
     // Do we already have this in the IR cache?
     auto LocalEntry = Thread->LocalIRCache.find(GuestRIP);
 
@@ -1033,21 +1037,31 @@ namespace FEXCore::Context {
     }
   }
 
-  void FlushCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length) {
+  static void FlushThreadCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length) {
+    std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
 
-    if (Thread->CTX->Config.SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) {
-      auto lower = Thread->LookupCache->CodePages.lower_bound(Start >> 12);
-      auto upper = Thread->LookupCache->CodePages.upper_bound((Start + Length) >> 12);
+    auto lower = Thread->LookupCache->CodePages.lower_bound(Start >> 12);
+    auto upper = Thread->LookupCache->CodePages.upper_bound((Start + Length) >> 12);
 
-      for (auto it = lower; it != upper; it++) {
-        for (auto Address: it->second)
-          Context::RemoveCodeEntry(Thread, Address);
-        it->second.clear();
+    for (auto it = lower; it != upper; it++) {
+      for (auto Address: it->second)
+      Context::RemoveThreadCodeEntry(Thread, Address);
+      it->second.clear();
+    }
+  }
+
+  void FlushCodeRange(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length) {
+    std::lock_guard<std::mutex> lk(CTX->ThreadCreationMutex);
+    for (auto &Thread : CTX->Threads) {
+      if (Thread->RunningEvents.Running.load()) {
+        FlushThreadCodeRange(Thread, Start, Length);
       }
     }
   }
 
-  void Context::RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
+  void Context::RemoveThreadCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP) {
+    std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
+
     Thread->LocalIRCache.erase(GuestRIP);
     Thread->LookupCache->Erase(GuestRIP);
   }
@@ -1058,7 +1072,7 @@ namespace FEXCore::Context {
     Thread->CurrentFrame->State.rip = RIP;
 
     // Erase the RIP from all the storage backings if it exists
-    RemoveCodeEntry(Thread, RIP);
+    RemoveThreadCodeEntry(Thread, RIP);
 
     // We don't care if compilation passes or not
     CompileBlock(Thread->CurrentFrame, RIP);
@@ -1075,6 +1089,7 @@ namespace FEXCore::Context {
   }
 
   bool Context::GetDebugDataForRIP(uint64_t RIP, FEXCore::Core::DebugData *Data) {
+    std::lock_guard<std::recursive_mutex> lk(ParentThread->LookupCache->WriteLock);
     auto it = ParentThread->LocalIRCache.find(RIP);
     if (it == ParentThread->LocalIRCache.end()) {
       return false;

External/FEXCore/Source/Interface/Core/Interpreter/BranchOps.cpp

@@ -147,8 +147,8 @@ DEF_OP(ValidateCode) {
   }
 }
 
-DEF_OP(RemoveCodeEntry) {
-  Data->State->CTX->RemoveCodeEntry(Data->State, Data->CurrentEntry);
+DEF_OP(RemoveThreadCodeEntry) {
+  Data->State->CTX->RemoveThreadCodeEntry(Data->State, Data->CurrentEntry);
 }
 
 DEF_OP(CPUID) {

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -123,7 +123,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(INLINESYSCALL,          InlineSyscall);
   REGISTER_OP(THUNK,                  Thunk);
   REGISTER_OP(VALIDATECODE,           ValidateCode);
-  REGISTER_OP(REMOVECODEENTRY,        RemoveCodeEntry);
+  REGISTER_OP(REMOVETHREADCODEENTRY,        RemoveThreadCodeEntry);
   REGISTER_OP(CPUID,                  CPUID);
 
   // Conversion ops

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -153,7 +153,7 @@ namespace FEXCore::CPU {
   DEF_OP(InlineSyscall);
   DEF_OP(Thunk);
   DEF_OP(ValidateCode);
-  DEF_OP(RemoveCodeEntry);
+  DEF_OP(RemoveThreadCodeEntry);
   DEF_OP(CPUID);
 
   ///< Conversion ops

External/FEXCore/Source/Interface/Core/JIT/Arm64/BranchOps.cpp

@@ -442,7 +442,7 @@ DEF_OP(ValidateCode) {
   }
 }
 
-DEF_OP(RemoveCodeEntry) {
+DEF_OP(RemoveThreadCodeEntry) {
   // Arguments are passed as follows:
   // X0: Thread
   // X1: RIP
@@ -452,7 +452,7 @@ DEF_OP(RemoveCodeEntry) {
   mov(x0, STATE);
   LoadConstant(x1, Entry);
 
-  ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.RemoveCodeEntryFromJIT)));
+  ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.AArch64.RemoveThreadCodeEntryFromJIT)));
   SpillStaticRegs();
   blr(x2);
   FillStaticRegs();
@@ -500,7 +500,7 @@ void Arm64JITCore::RegisterBranchHandlers() {
   REGISTER_OP(INLINESYSCALL,     InlineSyscall);
   REGISTER_OP(THUNK,             Thunk);
   REGISTER_OP(VALIDATECODE,      ValidateCode);
-  REGISTER_OP(REMOVECODEENTRY,   RemoveCodeEntry);
+  REGISTER_OP(REMOVETHREADCODEENTRY,   RemoveThreadCodeEntry);
   REGISTER_OP(CPUID,             CPUID);
 #undef REGISTER_OP
 }

External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -453,7 +453,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
     Pointers.LREM = reinterpret_cast<uint64_t>(LREM);
     Pointers.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
     Pointers.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
-    Pointers.RemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit);
+    Pointers.RemoveThreadCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveThreadCodeEntryFromJit);
     Pointers.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
 
     {

External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h

@@ -278,7 +278,7 @@ private:
   DEF_OP(InlineSyscall);
   DEF_OP(Thunk);
   DEF_OP(ValidateCode);
-  DEF_OP(RemoveCodeEntry);
+  DEF_OP(RemoveThreadCodeEntry);
   DEF_OP(CPUID);
 
   ///< Conversion ops

External/FEXCore/Source/Interface/Core/JIT/x86_64/BranchOps.cpp

@@ -259,7 +259,7 @@ DEF_OP(ValidateCode) {
   }
 }
 
-DEF_OP(RemoveCodeEntry) {
+DEF_OP(RemoveThreadCodeEntry) {
   auto NumPush = RA64.size();
 
   for (auto &Reg : RA64)
@@ -272,7 +272,7 @@ DEF_OP(RemoveCodeEntry) {
   mov(rax, Entry); // imm64 move
   mov(rsi, rax);
 
-  call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.X86.RemoveCodeEntryFromJIT)]);
+  call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.X86.RemoveThreadCodeEntryFromJIT)]);
 
   if (NumPush & 1)
     add(rsp, 8); // Align
@@ -330,7 +330,7 @@ void X86JITCore::RegisterBranchHandlers() {
   REGISTER_OP(SYSCALL,           Syscall);
   REGISTER_OP(THUNK,             Thunk);
   REGISTER_OP(VALIDATECODE,      ValidateCode);
-  REGISTER_OP(REMOVECODEENTRY,   RemoveCodeEntry);
+  REGISTER_OP(REMOVETHREADCODEENTRY,   RemoveThreadCodeEntry);
   REGISTER_OP(CPUID,             CPUID);
 #undef REGISTER_OP
 }

External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp

@@ -371,7 +371,7 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
     // Process specific
     Pointers.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
     Pointers.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
-    Pointers.RemoveCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit);
+    Pointers.RemoveThreadCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveThreadCodeEntryFromJit);
     Pointers.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
 
     {

External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h

@@ -276,7 +276,7 @@ private:
   DEF_OP(Syscall);
   DEF_OP(Thunk);
   DEF_OP(ValidateCode);
-  DEF_OP(RemoveCodeEntry);
+  DEF_OP(RemoveThreadCodeEntry);
   DEF_OP(CPUID);
 
   ///< Conversion ops

External/FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -61,6 +61,7 @@ void LookupCache::HintUsedRange(uint64_t Address, uint64_t Size) {
 }
 
 void LookupCache::ClearL2Cache() {
+  std::lock_guard<std::recursive_mutex> lk(WriteLock);
   // Clear out the page memory
   madvise(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8, MADV_DONTNEED);
   madvise(reinterpret_cast<void*>(PageMemory), CODE_SIZE, MADV_DONTNEED);
@@ -68,6 +69,8 @@ void LookupCache::ClearL2Cache() {
 }
 
 void LookupCache::ClearCache() {
+  std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
   // Clear L1
   madvise(reinterpret_cast<void*>(L1Pointer), L1_SIZE, MADV_DONTNEED);
   // Clear L2

External/FEXCore/Source/Interface/Core/LookupCache.h

@@ -7,6 +7,7 @@
 #include <stddef.h>
 #include <utility>
 #include <vector>
+#include <mutex>
 
 namespace FEXCore {
 namespace Context {
@@ -28,24 +29,51 @@ public:
   uintptr_t End() { return 0; }
 
   uintptr_t FindBlock(uint64_t Address) {
-    auto HostCode = FindCodePointerForAddress(Address);
-    if (HostCode) {
-      return HostCode;
-    } else {
-      auto HostCode = BlockList.find(Address);
+    // Try L1, no lock needed
+    auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
+    if (L1Entry.GuestCode == Address) {
+      return L1Entry.HostCode;
+    }
 
-      if (HostCode != BlockList.end()) {
-        CacheBlockMapping(Address, HostCode->second);
-        return HostCode->second;
-      } else {
-        return 0;
+    // L2 and L3 need to be locked
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
+    // Try L2
+    auto PageIndex = Address & (VirtualMemSize -1);
+
+    uint64_t PageOffset = Address & (0x0FFF);
+    PageIndex >>= 12;
+    uintptr_t *Pointers = reinterpret_cast<uintptr_t*>(PagePointer);
+    uint64_t LocalPagePointer = Pointers[PageIndex];
+    // Do we a page pointer for this address?
+    if (LocalPagePointer) {
+      // Find there pointer for the address in the blocks
+      auto BlockPointers = reinterpret_cast<LookupCacheEntry*>(LocalPagePointer);
+
+      if (BlockPointers[PageOffset].GuestCode == Address)
+      {
+        L1Entry.GuestCode = Address;
+        return L1Entry.HostCode = BlockPointers[PageOffset].HostCode;
       }
     }
+    
+    // Try L3
+    auto HostCode = BlockList.find(Address);
+
+    if (HostCode != BlockList.end()) {
+      CacheBlockMapping(Address, HostCode->second);
+      return HostCode->second;
+    }
+      
+    // Failed to find
+    return 0;
   }
 
   std::map<uint64_t, std::vector<uint64_t>> CodePages;
 
-  void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) { 
+  void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+    
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
     auto InsertPoint =
 #endif
@@ -65,6 +93,8 @@ public:
 
   void Erase(uint64_t Address) {
 
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
     // Sever any links to this block
     auto lower = BlockLinks.lower_bound({Address, 0});
     auto upper = BlockLinks.upper_bound({Address, UINTPTR_MAX});
@@ -78,7 +108,10 @@ public:
     // Do L1
     auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
     if (L1Entry.GuestCode == Address) {
-      L1Entry.GuestCode = L1Entry.HostCode = 0;
+      L1Entry.GuestCode = 0;
+      // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
+      // This is a soft guarantee for cross thread invalidation, as atomics are not used
+      // and it hasn't been thoroughly tested
     }
 
     // Do full map
@@ -101,6 +134,8 @@ public:
 
 
   void AddBlockLink(uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
     BlockLinks.insert({{GuestDestination, HostLink}, delinker});
   }
 
@@ -116,8 +151,19 @@ public:
   constexpr static size_t L1_ENTRIES = 1 * 1024 * 1024; // Must be a power of 2
   constexpr static size_t L1_ENTRIES_MASK = L1_ENTRIES - 1;
 
+  // This needs to be taken before reads or writes to L2, L3, CodePages, Thread::LocalIRCache,
+  // and before writes to L1. Concurrent access from a thread that this LookupCache doesn't belong to 
+  // can only happen during cross thread invalidation (::Erase).
+  // All other operations must be done from the owning thread.
+  // Some care is taken so that L1 lookups can be done without locks, and even tearing happens
+  // it is unlikely to lead to a crash. This approach has not been fully vetted yet.
+  // Also note that L1 lookups might be inlined in the JIT Dispatcher and/or block ends.
+  std::recursive_mutex WriteLock;
+
 private:
   void CacheBlockMapping(uint64_t Address, uintptr_t HostCode) { 
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
     // Do L1
     auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
     L1Entry.GuestCode = Address;
@@ -175,6 +221,8 @@ private:
       return L1Entry.HostCode;
     }
 
+    std::lock_guard<std::recursive_mutex> lk(WriteLock);
+
     auto FullAddress = Address;
     Address = Address & (VirtualMemSize -1);
 

External/FEXCore/Source/Interface/IR/AOTIR.cpp

@@ -1,5 +1,6 @@
 #include "Interface/Context/Context.h"
 #include "Interface/IR/AOTIR.h"
+#include "Interface/Core/LookupCache.h"
 
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/RegisterAllocationData.h>
@@ -349,6 +350,8 @@ namespace FEXCore::IR {
         if (Thread->CPUBackend->NeedsRetainedIRCopy()) {
           // Add to thread local ir cache
           Core::LocalIREntry Entry = {StartAddr, Length, decltype(Entry.IR)(IRList), decltype(Entry.RAData)(RAData), decltype(Entry.DebugData)(DebugData)};
+          
+          std::lock_guard<std::recursive_mutex> lk(Thread->LookupCache->WriteLock);
           Thread->LocalIRCache.insert({GuestRIP, std::move(Entry)});
         }
         else {

External/FEXCore/Source/Interface/IR/IR.json

@@ -187,7 +187,7 @@
         "DestSize": "8"
       },
 
-      "RemoveCodeEntry": {
+      "RemoveThreadCodeEntry": {
         "HasSideEffects": true
       },
 

External/FEXCore/include/FEXCore/Core/Context.h

@@ -244,7 +244,7 @@ namespace FEXCore::Context {
 
   FEX_DEFAULT_VISIBILITY void FinalizeAOTIRCache(FEXCore::Context::Context *CTX);
   FEX_DEFAULT_VISIBILITY void WriteFilesWithCode(FEXCore::Context::Context *CTX, std::function<void(const std::string& fileid, const std::string& filename)> Writer);
-  FEX_DEFAULT_VISIBILITY void FlushCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length);
+  FEX_DEFAULT_VISIBILITY void FlushCodeRange(FEXCore::Context::Context *CTX, uint64_t Start, uint64_t Length);
 
   FEX_DEFAULT_VISIBILITY void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress);
 }

External/FEXCore/include/FEXCore/Core/CoreState.h

@@ -101,7 +101,7 @@ namespace FEXCore::Core {
       uint64_t LREM{};
       uint64_t PrintValue{};
       uint64_t PrintVectorValue{};
-      uint64_t RemoveCodeEntryFromJIT{};
+      uint64_t RemoveThreadCodeEntryFromJIT{};
       uint64_t CPUIDObj{};
       uint64_t CPUIDFunction{};
       uint64_t SyscallHandlerObj{};
@@ -134,7 +134,7 @@ namespace FEXCore::Core {
       // Process specific
       uint64_t PrintValue{};
       uint64_t PrintVectorValue{};
-      uint64_t RemoveCodeEntryFromJIT{};
+      uint64_t RemoveThreadCodeEntryFromJIT{};
       uint64_t CPUIDObj{};
       uint64_t CPUIDFunction{};
       uint64_t SyscallHandlerObj{};

Source/Tests/LinuxSyscalls/x32/Memory.cpp

@@ -38,14 +38,14 @@ namespace FEX::HLE::x32 {
       auto Result = (uint64_t)static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
         mmap(reinterpret_cast<void*>(arg->addr), arg->len, arg->prot, arg->flags, arg->fd, arg->offset);
 
-      auto Thread = Frame->Thread;
       if (!FEX::HLE::HasSyscallError(Result)) {
+        auto CTX = Frame->Thread->CTX;
         if (!(arg->flags & MAP_ANONYMOUS)) {
           auto filename = get_fdpath(arg->fd);
 
-          FEXCore::Context::AddNamedRegion(Thread->CTX, Result, arg->len, arg->offset, filename);
+          FEXCore::Context::AddNamedRegion(CTX, Result, arg->len, arg->offset, filename);
         }
-        FEXCore::Context::FlushCodeRange(Thread, (uintptr_t)Result, arg->len);
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)Result, arg->len);
       }
       return Result;
     });
@@ -54,14 +54,14 @@ namespace FEX::HLE::x32 {
       auto Result = (uint64_t)static_cast<FEX::HLE::x32::x32SyscallHandler*>(FEX::HLE::_SyscallHandler)->GetAllocator()->
         mmap(reinterpret_cast<void*>(addr), length, prot,flags, fd, (uint64_t)pgoffset * 0x1000);
 
-      auto Thread = Frame->Thread;
       if (!FEX::HLE::HasSyscallError(Result)) {
+        auto CTX = Frame->Thread->CTX;
         if (!(flags & MAP_ANONYMOUS)) {
           auto filename = get_fdpath(fd);
 
-          FEXCore::Context::AddNamedRegion(Thread->CTX, Result, length, pgoffset * 0x1000, filename);
+          FEXCore::Context::AddNamedRegion(CTX, Result, length, pgoffset * 0x1000, filename);
         }
-        FEXCore::Context::FlushCodeRange(Thread, (uintptr_t)Result, length);
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)Result, length);
       }
 
       return Result;
@@ -72,8 +72,9 @@ namespace FEX::HLE::x32 {
         munmap(addr, length);
 
       if (Result == 0) {
-        FEXCore::Context::RemoveNamedRegion(Frame->Thread->CTX, (uintptr_t)addr, length);
-        FEXCore::Context::FlushCodeRange(Frame->Thread, (uintptr_t)addr, length);
+        auto CTX = Frame->Thread->CTX;
+        FEXCore::Context::RemoveNamedRegion(CTX, (uintptr_t)addr, length);
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)addr, length);
       }
 
       return Result;
@@ -81,8 +82,10 @@ namespace FEX::HLE::x32 {
 
     REGISTER_SYSCALL_IMPL_X32(mprotect, [](FEXCore::Core::CpuStateFrame *Frame, void *addr, uint32_t len, int prot) -> uint64_t {
       uint64_t Result = ::mprotect(addr, len, prot);
+
       if (Result != -1 && prot & PROT_EXEC) {
-        FEXCore::Context::FlushCodeRange(Frame->Thread, (uintptr_t)addr, len);
+        auto CTX = Frame->Thread->CTX;
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)addr, len);
       }
       SYSCALL_ERRNO();
     });

Source/Tests/LinuxSyscalls/x64/Memory.cpp

@@ -50,10 +50,10 @@ namespace FEX::HLE::x64 {
         Result = FEXCore::Allocator::munmap(addr, length);
       }
 
-      auto Thread = Frame->Thread;
       if (Result != -1) {
-        FEXCore::Context::RemoveNamedRegion(Thread->CTX, (uintptr_t)addr, length);
-        FEXCore::Context::FlushCodeRange(Thread, (uintptr_t)addr, length);
+        auto CTX = Frame->Thread->CTX;
+        FEXCore::Context::RemoveNamedRegion(CTX, (uintptr_t)addr, length);
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)addr, length);
       }
       SYSCALL_ERRNO();
     });
@@ -78,14 +78,14 @@ namespace FEX::HLE::x64 {
         Result = reinterpret_cast<uint64_t>(FEXCore::Allocator::mmap(addr, length, prot, flags, fd, offset));
       }
 
-      auto Thread = Frame->Thread;
       if (Result != -1) {
+        auto CTX = Frame->Thread->CTX;
         if (!(flags & MAP_ANONYMOUS)) {
           auto filename = get_fdpath(fd);
 
-          FEXCore::Context::AddNamedRegion(Thread->CTX, Result, length, offset, filename);
+          FEXCore::Context::AddNamedRegion(CTX, Result, length, offset, filename);
         }
-        FEXCore::Context::FlushCodeRange(Thread, (uintptr_t)Result, length);
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)Result, length);
       }
       SYSCALL_ERRNO();
     });
@@ -100,9 +100,9 @@ namespace FEX::HLE::x64 {
       [](FEXCore::Core::CpuStateFrame *Frame, void *addr, size_t len, int prot) -> uint64_t {
       uint64_t Result = ::mprotect(addr, len, prot);
 
-      auto Thread = Frame->Thread;
       if (Result != -1 && prot & PROT_EXEC) {
-        FEXCore::Context::FlushCodeRange(Thread, (uintptr_t)addr, len);
+        auto CTX = Frame->Thread->CTX;
+        FEXCore::Context::FlushCodeRange(CTX, (uintptr_t)addr, len);
       }
       SYSCALL_ERRNO();
     });