Merge pull request #5006 from bylaws/fasterrrrr

Introduce two-pass code invalidation model

FEXCore/Source/Interface/Context/Context.h

@@ -61,7 +61,7 @@ struct CustomIRResult {
     , Data(Data) {}
 };
 
-using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record);
+using BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record);
 constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz
 
 class CodeCache : public AbstractCodeCache {
@@ -154,10 +154,10 @@ public:
     return CodeCache;
   }
 
-  void OnCodeBufferAllocated(CPU::CodeBuffer&) override;
+  void OnCodeBufferAllocated(const std::shared_ptr<CPU::CodeBuffer>&) override;
   void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override;
-  void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start,
-                                uint64_t Length) override;
+  void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override;
+  void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override;
   FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override {
     return CodeInvalidationMutex;
   }
@@ -229,8 +229,6 @@ public:
 
   ContextImpl(const FEXCore::HostFeatures& Features);
 
-  static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, const FEXCore::LookupCacheWriteLockToken& lk);
-
   static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP);
 
   // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations
@@ -346,5 +344,8 @@ private:
 
   bool MonoDetected = false;
   std::atomic<uint64_t> MonoBackpatcherBlock;
+
+  std::mutex CodeBufferListLock;
+  fextl::vector<std::weak_ptr<CPU::CodeBuffer>> CodeBufferList;
 };
 } // namespace FEXCore::Context

FEXCore/Source/Interface/Core/CPUBackend.cpp

@@ -400,7 +400,7 @@ namespace CPU {
     Latest = Buffer;
     LatestOffset = 0;
 
-    OnCodeBufferAllocated(*Buffer);
+    OnCodeBufferAllocated(Buffer);
 
     return Buffer;
   }

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -81,7 +81,7 @@ namespace CPU {
     // Protects writes to the latest CodeBuffer and changes to LatestOffset
     FEXCore::ForkableUniqueMutex CodeBufferWriteMutex;
 
-    virtual void OnCodeBufferAllocated(CodeBuffer&) {};
+    virtual void OnCodeBufferAllocated(const std::shared_ptr<CodeBuffer>&) {};
 
   private:
     fextl::shared_ptr<CodeBuffer> Latest;

FEXCore/Source/Interface/Core/Core.cpp

@@ -459,9 +459,14 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) {
 }
 #endif
 
-void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) {
+void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr<CPU::CodeBuffer>& Buffer) {
   if (Config.GlobalJITNaming()) {
-    Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
+    Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->Size);
+  }
+
+  {
+    std::scoped_lock lk {CodeBufferListLock};
+    CodeBufferList.emplace_back(Buffer);
   }
 }
 
@@ -833,10 +838,14 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
     Thread->CPUBackend->ClearRelocations();
   }
 
+  fextl::vector<uint64_t> CodePages;
+
   if (NeedsAddGuestCodeRanges) {
     // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously
     // contain code, inform the frontend so it can setup SMC detection.
     auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo();
+    CodePages.reserve(BlockInfo->CodePages.size());
+    CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end());
     for (auto CodePage : BlockInfo->CodePages) {
       if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) {
         SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE);
@@ -845,8 +854,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_
   }
 
   // Insert to lookup cache
+
   for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) {
-    Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, HostAddr);
+    Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr);
   }
 
   return (uintptr_t)CodePtr;
@@ -873,50 +883,37 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui
   return (uintptr_t)CodePtr;
 }
 
-static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
-                                           uint64_t Start, uint64_t Length) {
+void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) {
+  FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange");
+
+  LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
+  std::scoped_lock lk {CodeBufferListLock};
+  auto it = CodeBufferList.begin();
+  while (it != CodeBufferList.end()) {
+    if (auto Strong = it->lock()) {
+      Strong->LookupCache->InvalidateRange(Start, Length);
+      it++;
+    } else {
+      it = CodeBufferList.erase(it);
+    }
+  }
+}
+
+void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) {
+  LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here");
+
   // Ensures now-modified mappings aren't cached as being in their previous non-executable state.
   // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here.
   Thread->FrontendDecoder->ResetExecutableRangeCache();
 
-  auto lk = Thread->LookupCache->AcquireWriteLock();
-  auto& CodePages = Thread->LookupCache->Shared->CodePages;
+  if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) {
+    FEXCORE_PROFILE_SCOPED("InvalidateCallRet");
 
-  auto lower = CodePages.lower_bound(Start >> 12);
-  auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
-
-  for (auto it = lower; it != upper; it++) {
-    Accumulator.emplace_back(std::move(it->second));
-  }
-
-  bool InvalidatedAnyEntries = false;
-  for (const auto& PageEntries : Accumulator) {
-    for (const auto& Entry : PageEntries) {
-      if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry, lk)) {
-        InvalidatedAnyEntries = true;
-      }
-    }
-  }
-
-  if (InvalidatedAnyEntries) {
     // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend
     Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE);
   }
 }
 
-void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator,
-                                           uint64_t Start, uint64_t Length) {
-  InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length);
-}
-
-bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP,
-                                        const FEXCore::LookupCacheWriteLockToken& lk) {
-  LogMan::Throw::AFmt(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to "
-                                                                                                         "be unique_locked here");
-
-  return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP, lk);
-}
-
 void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) {
   static_cast<ContextImpl*>(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1);
 }

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -493,7 +493,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) {
   }
 }
 
-static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
+static void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) {
   uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
   uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset;
   auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4;
@@ -511,7 +511,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co
   ARMEmitter::Emitter::ClearICache(reinterpret_cast<void*>(CallerAddress), 4);
 }
 
-static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
+static void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) {
   uintptr_t JumpThunkStartAddress = reinterpret_cast<uintptr_t>(Record) - 0x10;
   uint32_t BranchInst = 0;
   ARMEmitter::Emitter BranchEmit(reinterpret_cast<uint8_t*>(&BranchInst), 4);
@@ -577,16 +577,11 @@ uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEX
     if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) {
       BranchEmit.bl(BranchOffset);
       Thread->LookupCache->AddBlockLink(
-        GuestRip, Record,
-        [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Frame, Record, true); }, lk);
+        GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk);
     } else {
       BranchEmit.b(BranchOffset);
       Thread->LookupCache->AddBlockLink(
-        GuestRip, Record,
-        [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) {
-          DirectBlockDelinker(Frame, Record, false);
-        },
-        lk);
+        GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, false); }, lk);
     }
 
     std::atomic_ref<uint32_t>(*reinterpret_cast<uint32_t*>(CallerAddress)).store(BranchInst, std::memory_order::relaxed);

FEXCore/Source/Interface/Core/LookupCache.cpp

@@ -75,7 +75,7 @@ LookupCache::~LookupCache() {
   // These will get freed when their memory allocators are deallocated.
 }
 
-void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) {
+void LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) {
   // Clear out the page memory
   // PagePointer and PageMemory are sequential with each other. Clear both at once.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer),
@@ -86,12 +86,12 @@ void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) {
 void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) {
   // Clear L1 and L2 by clearing the full cache.
   FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
+  CachedCodePages.clear();
 }
 
 void LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) {
   // Clear L1 and L2 by clearing the full cache.
-  FEXCore::Allocator::VirtualDontNeed(reinterpret_cast<void*>(PagePointer), TotalCacheSize, false);
-
+  ClearThreadLocalCaches(lk);
   Shared->ClearCache(lk);
 }
 

FEXCore/Source/Interface/Core/LookupCache.h

@@ -8,6 +8,7 @@
 #include <FEXCore/fextl/map.h>
 #include <FEXCore/fextl/memory_resource.h>
 #include <FEXCore/fextl/robin_map.h>
+#include <FEXCore/fextl/robin_set.h>
 #include <FEXCore/fextl/vector.h>
 #include <FEXCore/fextl/memory_resource.h>
 
@@ -17,7 +18,13 @@
 #include <mutex>
 
 namespace FEXCore {
-struct LookupCacheWriteLockToken {
+struct LookupCacheBaseLockToken {
+protected:
+  // Protected constructor - only derived classes can construct
+  LookupCacheBaseLockToken() = default;
+};
+
+struct LookupCacheWriteLockToken : public LookupCacheBaseLockToken {
 private:
   // Only constructible by GuestToHostMap
   friend struct GuestToHostMap;
@@ -26,7 +33,7 @@ private:
   std::lock_guard<FEXCore::Utils::WritePriorityMutex::Mutex> Lock;
 };
 
-struct LookupCacheReadLockToken {
+struct LookupCacheReadLockToken : public LookupCacheBaseLockToken {
 private:
   // Only constructible by GuestToHostMap
   friend struct GuestToHostMap;
@@ -74,42 +81,61 @@ struct GuestToHostMap {
   fextl::unique_ptr<std::pmr::polymorphic_allocator<std::byte>> BlockLinks_pma;
   BlockLinksMapType* BlockLinks;
 
-  fextl::robin_map<uint64_t, uint64_t> BlockList;
+  struct BlockEntry {
+    uint64_t HostCode;
+    fextl::vector<uint64_t> CodePages;
+  };
+
+  fextl::robin_map<uint64_t, BlockEntry> BlockList;
 
   fextl::map<uint64_t, fextl::vector<uint64_t>> CodePages;
 
   GuestToHostMap();
 
   // Adds to Guest -> Host code mapping
-  void AddBlockMapping(uint64_t Address, void* HostCode, const LookupCacheWriteLockToken&) {
+  const BlockEntry& AddBlockMapping(uint64_t Address, const fextl::vector<uint64_t>& CodePages, void* HostCode, const LookupCacheWriteLockToken&) {
     // This may replace an existing mapping
     // NOTE: Generally no previous entry should exist, however there is one exception:
     //       If the backend updates the active thread's CodeBuffer, the new associated LookupCache
     //       may already contain the block address. Since is comparatively rare, we'll just leak
     //       one of the two blocks in this case.
-    BlockList[Address] = (uintptr_t)HostCode;
+    return BlockList.insert_or_assign(Address, BlockEntry {(uintptr_t)HostCode, CodePages}).first->second;
   }
 
-  std::optional<uintptr_t> FindBlock(uint64_t Address, const LookupCacheReadLockToken&) {
+  const BlockEntry* FindBlock(uint64_t Address, const LookupCacheReadLockToken&) {
     auto HostCode = BlockList.find(Address);
     if (HostCode == BlockList.end()) {
-      return std::nullopt;
+      return nullptr;
     }
-    return HostCode->second;
+    return &HostCode->second;
   }
 
-  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken&) {
+  bool Erase(uint64_t Address, const LookupCacheWriteLockToken&) {
     // Sever any links to this block
     auto lower = BlockLinks->lower_bound({Address, nullptr});
     auto upper = BlockLinks->upper_bound({Address, reinterpret_cast<FEXCore::Context::ExitFunctionLinkData*>(UINTPTR_MAX)});
     for (auto it = lower; it != upper; it = BlockLinks->erase(it)) {
-      it->second(Frame, it->first.HostLink);
+      it->second(it->first.HostLink);
     }
 
     // Remove from BlockList
     return BlockList.erase(Address) != 0;
   }
 
+  void InvalidateRange(uint64_t Start, uint64_t Length) {
+    auto lk = AcquireWriteLock();
+
+    auto lower = CodePages.lower_bound(Start >> 12);
+    auto upper = CodePages.upper_bound((Start + Length - 1) >> 12);
+
+    for (auto it = lower; it != upper; it++) {
+      for (const auto& Entry : it->second) {
+        Erase(Entry, lk);
+      }
+    }
+    CodePages.erase(lower, upper);
+  }
+
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
                     const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken&) {
     BlockLinks->insert({{GuestDestination, HostLink}, delinker});
@@ -185,10 +211,10 @@ public:
 
       if (!HostPtr) {
         // Try L3
-        auto HostCode = Shared->FindBlock(Address, lk);
-        if (HostCode) {
-          CacheBlockMapping(Address, HostCode.value(), lk);
-          HostPtr = HostCode.value();
+        auto Entry = Shared->FindBlock(Address, lk);
+        if (Entry) {
+          CacheBlockMapping(Address, *Entry, false, lk);
+          HostPtr = Entry->HostCode;
         }
       }
     }
@@ -259,32 +285,25 @@ public:
   }
 
   // Adds to Guest -> Host code mapping
-  void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, void* HostCode) {
+  void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, const fextl::vector<uint64_t>& CodePages, void* HostCode) {
     std::optional<FEXCore::SHMStats::AccumulationBlock<uint64_t>> LockTime(
       Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr);
     auto lk = Shared->AcquireWriteLock();
     LockTime.reset();
 
-    Shared->AddBlockMapping(Address, HostCode, lk);
+    const auto& Entry = Shared->AddBlockMapping(Address, CodePages, HostCode, lk);
 
     // There is no need to update L1 or L2, they will get updated on first lookup
     // However, adding to L1 here increases performance
-    auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
-    L1Entry.GuestCode = Address;
-    L1Entry.HostCode = (uintptr_t)HostCode;
+    CacheBlockMapping(Address, Entry, true, lk);
   }
 
-  // NOTE: It's the caller's responsibility to call Erase() for all other
-  //       GuestToHostMaps that share the same LookupCache. Otherwise, the
-  //       L1/L2 caches will contain stale references to deallocated memory.
-  bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken& lk) {
-    bool ErasedAny = Shared->Erase(Frame, Address, lk);
-
+  // Invalidates L1/L2 for a given guest block
+  void InvalidateCache(uint64_t Address, const LookupCacheWriteLockToken& lk) {
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
     if (L1Entry.GuestCode == Address) {
       L1Entry.GuestCode = 0;
-      ErasedAny = true;
       // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
       // This is a soft guarantee for cross thread invalidation, as atomics are not used
       // and it hasn't been thoroughly tested
@@ -300,7 +319,7 @@ public:
       uint64_t LocalPagePointer = Pointers[Address];
       if (!LocalPagePointer) {
         // Page for this code didn't even exist, nothing to do
-        return ErasedAny;
+        return;
       }
 
       // Page exists, just set the offset to zero
@@ -308,7 +327,22 @@ public:
       BlockPointers[PageOffset].GuestCode = 0;
       BlockPointers[PageOffset].HostCode = 0;
     }
-    return true;
+  }
+
+  // Invalidates all L1/L2 entries for all guest block that intersect the given range
+  bool InvalidateCacheRange(uint64_t Start, uint64_t Length) {
+    auto lk = Shared->AcquireWriteLock();
+
+    auto lower = CachedCodePages.lower_bound(Start >> 12);
+    auto upper = CachedCodePages.upper_bound((Start + Length - 1) >> 12);
+
+    for (auto it = lower; it != upper; it++) {
+      for (const auto& Entry : it->second) {
+        InvalidateCache(Entry, lk);
+      }
+    }
+    CachedCodePages.erase(lower, upper);
+    return upper != lower;
   }
 
   void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink,
@@ -317,7 +351,7 @@ public:
   }
 
   void ClearCache(const LookupCacheWriteLockToken&);
-  void ClearL2Cache(const LookupCacheReadLockToken&);
+  void ClearL2Cache(const LookupCacheBaseLockToken&);
   void ClearThreadLocalCaches(const LookupCacheWriteLockToken&);
 
   uintptr_t GetL1Pointer() const {
@@ -345,13 +379,17 @@ public:
   }
 
 private:
-  void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheReadLockToken& lk) {
+  void CacheBlockMapping(uint64_t Address, const GuestToHostMap::BlockEntry& Entry, bool L1Only, const LookupCacheBaseLockToken& lk) {
+    for (const auto& CodePage : Entry.CodePages) {
+      CachedCodePages[CodePage >> 12].insert(Address);
+    }
+
     // Do L1
     auto& L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1PointerMask];
     L1Entry.GuestCode = Address;
-    L1Entry.HostCode = HostCode;
+    L1Entry.HostCode = Entry.HostCode;
 
-    if (!DisableL2Cache()) {
+    if (!DisableL2Cache() && !L1Only) {
       // Do ful map
       auto FullAddress = Address;
       Address = Address & (VirtualMemSize - 1);
@@ -368,7 +406,7 @@ private:
         if (!NewPageBacking) {
           // Couldn't allocate, clear L2 and retry
           ClearL2Cache(lk);
-          CacheBlockMapping(Address, HostCode, lk);
+          CacheBlockMapping(Address, Entry, false, lk);
           return;
         }
         Pointers[Address] = NewPageBacking;
@@ -380,7 +418,7 @@ private:
 
       // This silently replaces existing mappings
       BlockPointers[PageOffset].GuestCode = FullAddress;
-      BlockPointers[PageOffset].HostCode = HostCode;
+      BlockPointers[PageOffset].HostCode = Entry.HostCode;
     }
   }
 
@@ -398,6 +436,9 @@ private:
     return PageMemory + NewBase;
   }
 
+  // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2
+  fextl::map<uint64_t, fextl::robin_set<uint64_t>> CachedCodePages;
+
   uintptr_t PagePointer;
   uintptr_t PageMemory;
   uintptr_t L1Pointer;

FEXCore/include/FEXCore/Core/Context.h

@@ -42,9 +42,6 @@ enum OperatingMode {
 
 using CodeRangeInvalidationFn = std::function<void(uint64_t start, uint64_t Length)>;
 
-// Nested vector of guest block entrypoints
-using InvalidatedEntryAccumulator = fextl::vector<fextl::vector<uint64_t>>;
-
 using CustomIREntrypointHandler = std::function<void(uintptr_t Entrypoint, IR::IREmitter*)>;
 
 using ExitHandler = std::function<void(Core::InternalThreadState* Thread)>;
@@ -141,8 +138,9 @@ public:
   virtual AbstractCodeCache& GetCodeCache() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0;
-  FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange(
-    FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) = 0;
+  FEX_DEFAULT_VISIBILITY virtual void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) = 0;
+  FEX_DEFAULT_VISIBILITY virtual void
+  InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) = 0;
   FEX_DEFAULT_VISIBILITY virtual FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() = 0;
 
   FEX_DEFAULT_VISIBILITY virtual void

FEXCore/include/FEXCore/fextl/robin_set.h

@@ -0,0 +1,10 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <FEXCore/fextl/allocator.h>
+
+#include <tsl/robin_set.h>
+
+namespace fextl {
+template<class Key, class Hash = std::hash<Key>, class KeyEqual = std::equal_to<Key>, class Allocator = fextl::FEXAlloc<Key>>
+using robin_set = tsl::robin_set<Key, Hash, KeyEqual, Allocator>;
+}

Source/Tools/CodeSizeValidation/Main.cpp

@@ -52,8 +52,8 @@ public:
 
     {
       auto CodeInvalidationlk = FEXCore::GuardSignalDeferringSection(CTX->GetCodeInvalidationMutex(), Thread);
-      FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-      CTX->InvalidateGuestCodeRange(Thread, Accumulator, reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
+      CTX->InvalidateCodeBuffersCodeRange(reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
+      CTX->InvalidateThreadCachedCodeRange(Thread, reinterpret_cast<uint64_t>(CodeStart), MAX_CODE_SIZE);
     }
 
     ClearStats();

Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h

@@ -208,10 +208,9 @@ public:
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
     auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread);
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-
+    CTX->InvalidateCodeBuffersCodeRange(Start, Length);
     for (auto& Thread : Threads) {
-      CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length);
+      CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length);
     }
   }
 
@@ -223,10 +222,9 @@ public:
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
     auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread);
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-
+    CTX->InvalidateCodeBuffersCodeRange(Start, Length);
     for (auto& Thread : Threads) {
-      CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length);
+      CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length);
     }
 
     // Callback while holding the locks.

Source/Windows/Common/InvalidationTracker.cpp

@@ -56,11 +56,7 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u
 
   if (NeedsInvalidate) {
     // IntervalsLock cannot be held during invalidation
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize);
-    }
+    InvalidateIntervalInternal(AlignedBase, AlignedSize);
   }
 }
 
@@ -115,13 +111,8 @@ InvalidationTracker::InvalidateContainingSectionResult InvalidationTracker::Inva
          reinterpret_cast<uint64_t>(Info.AllocationBase) == SectionBase) {
     SectionSize += Info.RegionSize;
   }
-  {
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, SectionBase, SectionSize);
-    }
-  }
+
+  InvalidateIntervalInternal(SectionBase, SectionSize);
 
   if (Free) {
     std::unique_lock Lock(IntervalsLock);
@@ -141,13 +132,7 @@ void InvalidationTracker::InvalidateAlignedInterval(uint64_t Address, uint64_t S
   const auto AlignedBase = Address & FEXCore::Utils::FEX_PAGE_MASK;
   const auto AlignedSize = std::max(Size, (Address - AlignedBase + Size + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK);
 
-  {
-    std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-    FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-    for (auto Thread : Threads) {
-      CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize);
-    }
-  }
+  InvalidateIntervalInternal(AlignedBase, AlignedSize);
 
   if (Free) {
     std::unique_lock Lock(IntervalsLock);
@@ -197,14 +182,8 @@ bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThread
   }(FaultAddress);
 
   if (NeedsInvalidate) {
-    {
-      // IntervalsLock cannot be held during invalidation
-      std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
-      FEXCore::Context::InvalidatedEntryAccumulator Accumulator;
-      for (auto Thread : Threads) {
-        CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE);
-      }
-    }
+    // IntervalsLock cannot be held during invalidation
+    InvalidateIntervalInternal(FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE);
     DetectMonoBackpatcherBlock(Thread, HostPc);
     return true;
   }
@@ -274,4 +253,12 @@ void InvalidationTracker::DisableSMCDetection() {
   } while (Query.Size);
 }
 
+void InvalidationTracker::InvalidateIntervalInternal(uint64_t Address, uint64_t Size) {
+  std::scoped_lock Lock(CTX.GetCodeInvalidationMutex());
+  CTX.InvalidateCodeBuffersCodeRange(Address, Size);
+  for (auto Thread : Threads) {
+    CTX.InvalidateThreadCachedCodeRange(Thread.second, Address, Size);
+  }
+}
+
 } // namespace FEX::Windows

Source/Windows/Common/InvalidationTracker.h

@@ -4,6 +4,7 @@
 #include <FEXCore/Utils/IntervalList.h>
 #include <FEXCore/HLE/SyscallHandler.h>
 #include <mutex>
+#include <shared_mutex>
 #include <unordered_map>
 #include <string_view>
 
@@ -37,6 +38,8 @@ public:
 private:
   void DetectMonoBackpatcherBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC);
   void DisableSMCDetection();
+  void InvalidateIntervalInternal(uint64_t Address, uint64_t Size);
+
 
   FEXCore::IntervalList<uint64_t> XIntervals;
   FEXCore::IntervalList<uint64_t> RWXIntervals;