From 99ad7ea45c5db36cbf70cb9b3902c6e58159d83a Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Tue, 28 Oct 2025 21:55:42 +0000 Subject: [PATCH 1/6] LookupCache: Drop unused state frame argument for delinker cbs --- FEXCore/Source/Interface/Context/Context.h | 2 +- FEXCore/Source/Interface/Core/JIT/JIT.cpp | 13 ++++--------- FEXCore/Source/Interface/Core/LookupCache.h | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 6a7abb1db..38431f030 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -61,7 +61,7 @@ struct CustomIRResult { , Data(Data) {} }; -using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record); +using BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record); constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz class CodeCache : public AbstractCodeCache { diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 8be3e83fa..a0ffd06c7 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -493,7 +493,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { } } -static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) { +static void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) { uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10; uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset; auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4; @@ -511,7 +511,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co ARMEmitter::Emitter::ClearICache(reinterpret_cast(CallerAddress), 4); } -static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { +static void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) { uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10; uint32_t BranchInst = 0; ARMEmitter::Emitter BranchEmit(reinterpret_cast(&BranchInst), 4); @@ -577,16 +577,11 @@ uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEX if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) { BranchEmit.bl(BranchOffset); Thread->LookupCache->AddBlockLink( - GuestRip, Record, - [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Frame, Record, true); }, lk); + GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk); } else { BranchEmit.b(BranchOffset); Thread->LookupCache->AddBlockLink( - GuestRip, Record, - [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { - DirectBlockDelinker(Frame, Record, false); - }, - lk); + GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, false); }, lk); } std::atomic_ref(*reinterpret_cast(CallerAddress)).store(BranchInst, std::memory_order::relaxed); diff --git a/FEXCore/Source/Interface/Core/LookupCache.h b/FEXCore/Source/Interface/Core/LookupCache.h index c0e2b4e39..3c1f5d04d 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.h +++ b/FEXCore/Source/Interface/Core/LookupCache.h @@ -103,7 +103,7 @@ struct GuestToHostMap { auto lower = BlockLinks->lower_bound({Address, nullptr}); auto upper = BlockLinks->upper_bound({Address, reinterpret_cast(UINTPTR_MAX)}); for (auto it = lower; it != upper; it = BlockLinks->erase(it)) { - it->second(Frame, it->first.HostLink); + it->second(it->first.HostLink); } // Remove from BlockList From efd95efb4081057f485899da69ebaa4c78b62b30 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Tue, 28 Oct 2025 22:17:17 +0000 Subject: [PATCH 2/6] FEXCore: Keep a list of weak refs to all allocated codebuffers We currently rely on the frontend to keep track of threads and then iterate over all threads to perform per-codebuffer operations. However as codebuffers are shared between many threads (the common case is a single code buffer across all) this ends up being inefficient. Introduce a list of codebuffers to solve that (new codebuffers are very rare, so a vector is plenty fine here for erasing invalid weak refs). --- FEXCore/Source/Interface/Context/Context.h | 5 ++++- FEXCore/Source/Interface/Core/CPUBackend.cpp | 2 +- FEXCore/Source/Interface/Core/CPUBackend.h | 2 +- FEXCore/Source/Interface/Core/Core.cpp | 9 +++++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 38431f030..e20391616 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -154,7 +154,7 @@ public: return CodeCache; } - void OnCodeBufferAllocated(CPU::CodeBuffer&) override; + void OnCodeBufferAllocated(const std::shared_ptr &) override; void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override; void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) override; @@ -346,5 +346,8 @@ private: bool MonoDetected = false; std::atomic MonoBackpatcherBlock; + + std::mutex CodeBufferListLock; + fextl::vector> CodeBufferList; }; } // namespace FEXCore::Context diff --git a/FEXCore/Source/Interface/Core/CPUBackend.cpp b/FEXCore/Source/Interface/Core/CPUBackend.cpp index 4d1ec87b9..a5aa13f68 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.cpp +++ b/FEXCore/Source/Interface/Core/CPUBackend.cpp @@ -400,7 +400,7 @@ namespace CPU { Latest = Buffer; LatestOffset = 0; - OnCodeBufferAllocated(*Buffer); + OnCodeBufferAllocated(Buffer); return Buffer; } diff --git a/FEXCore/Source/Interface/Core/CPUBackend.h b/FEXCore/Source/Interface/Core/CPUBackend.h index ab9f13b54..9eb9e0639 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.h +++ b/FEXCore/Source/Interface/Core/CPUBackend.h @@ -81,7 +81,7 @@ namespace CPU { // Protects writes to the latest CodeBuffer and changes to LatestOffset FEXCore::ForkableUniqueMutex CodeBufferWriteMutex; - virtual void OnCodeBufferAllocated(CodeBuffer&) {}; + virtual void OnCodeBufferAllocated(const std::shared_ptr&) {}; private: fextl::shared_ptr Latest; diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 3a1e6120b..6d95635d4 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -459,9 +459,14 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) { } #endif -void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) { +void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr& Buffer) { if (Config.GlobalJITNaming()) { - Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size); + Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->Size); + } + + { + std::scoped_lock lk{CodeBufferListLock}; + CodeBufferList.emplace_back(Buffer); } } From 85c8e7f1bb56b7f0809e9ebb316e53e608fae4cc Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Thu, 20 Nov 2025 00:05:12 +0000 Subject: [PATCH 3/6] fextl: Wrap tsl::robin_set --- FEXCore/include/FEXCore/fextl/robin_set.h | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 FEXCore/include/FEXCore/fextl/robin_set.h diff --git a/FEXCore/include/FEXCore/fextl/robin_set.h b/FEXCore/include/FEXCore/fextl/robin_set.h new file mode 100644 index 000000000..9bdb935e5 --- /dev/null +++ b/FEXCore/include/FEXCore/fextl/robin_set.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT +#pragma once +#include + +#include + +namespace fextl { +template, class KeyEqual = std::equal_to, class Allocator = fextl::FEXAlloc> +using robin_set = tsl::robin_set; +} From cf4478eeeef909733aac3036020a62829443a16c Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Tue, 28 Oct 2025 22:24:07 +0000 Subject: [PATCH 4/6] LookupCache: Introduce two-pass code invalidation model Shared code buffer support introduced the concept of having a single GuestToHostMaps shared across many threads. In the common case all threads will share one however if e.g. a resize recently occured and specific thread is yet to compile any code with the new codebuffer it will still use the old GuestToHostMap. The current invalidation approach handles this by repeatedly calling erase for every single thread's GuestToHostMap, even if it is repeated. An accumulator is used to ensure when two threads share a map, the L1/L2 cache entries in the second thread will still be invalidated even if the the iteration for the first thread removed them from the map. Unfortunately this is incredibly slow in cases with many threads, as a significant number of redundant map lookups and L1/L2 cache erasures on threads that never even observed a given block can occur. Solve this by introducing a two-pass model: - First, all active codebuffers (and their associated GuestToHostMaps) have their entries invalidated for the given range, these codebuffers are tracked internally within FEXCore. It is at this point that delinking callbacks are ran. - Second, each thread will have its caches invalidated. But rather than naively invalidating the L1/L2 caches for every invalidated block for every thread, threads now track on their own what specific entries have been potentially fetched into their L1/L2 caches. This is aided by GuestToHostMap now tracking the pages each block touches. (an inverse CodePages so to speak). --- FEXCore/Source/Interface/Context/Context.h | 8 +- FEXCore/Source/Interface/Core/Core.cpp | 64 +++++------ FEXCore/Source/Interface/Core/LookupCache.cpp | 6 +- FEXCore/Source/Interface/Core/LookupCache.h | 107 ++++++++++++------ FEXCore/include/FEXCore/Core/Context.h | 8 +- 5 files changed, 111 insertions(+), 82 deletions(-) diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index e20391616..0f6339984 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -154,10 +154,10 @@ public: return CodeCache; } - void OnCodeBufferAllocated(const std::shared_ptr &) override; + void OnCodeBufferAllocated(const std::shared_ptr&) override; void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override; - void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, - uint64_t Length) override; + void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override; + void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override; FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override { return CodeInvalidationMutex; } @@ -229,8 +229,6 @@ public: ContextImpl(const FEXCore::HostFeatures& Features); - static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, const FEXCore::LookupCacheWriteLockToken& lk); - static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP); // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 6d95635d4..6ff1deb9b 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -465,7 +465,7 @@ void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr } { - std::scoped_lock lk{CodeBufferListLock}; + std::scoped_lock lk {CodeBufferListLock}; CodeBufferList.emplace_back(Buffer); } } @@ -838,10 +838,14 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ Thread->CPUBackend->ClearRelocations(); } + fextl::vector CodePages; + if (NeedsAddGuestCodeRanges) { // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously // contain code, inform the frontend so it can setup SMC detection. auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo(); + CodePages.reserve(BlockInfo->CodePages.size()); + CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end()); for (auto CodePage : BlockInfo->CodePages) { if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) { SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE); @@ -850,8 +854,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } // Insert to lookup cache + for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) { - Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, HostAddr); + Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr); } return (uintptr_t)CodePtr; @@ -878,50 +883,37 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui return (uintptr_t)CodePtr; } -static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, - uint64_t Start, uint64_t Length) { +void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) { + FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange"); + + LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + std::scoped_lock lk {CodeBufferListLock}; + auto it = CodeBufferList.begin(); + while (it != CodeBufferList.end()) { + if (auto Strong = it->lock(); Strong) { + Strong->LookupCache->InvalidateRange(Start, Length); + it++; + } else { + it = CodeBufferList.erase(it); + } + } +} + +void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) { + LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + // Ensures now-modified mappings aren't cached as being in their previous non-executable state. // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here. Thread->FrontendDecoder->ResetExecutableRangeCache(); - auto lk = Thread->LookupCache->AcquireWriteLock(); - auto& CodePages = Thread->LookupCache->Shared->CodePages; + if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) { + FEXCORE_PROFILE_SCOPED("InvalidateCallRet"); - auto lower = CodePages.lower_bound(Start >> 12); - auto upper = CodePages.upper_bound((Start + Length - 1) >> 12); - - for (auto it = lower; it != upper; it++) { - Accumulator.emplace_back(std::move(it->second)); - } - - bool InvalidatedAnyEntries = false; - for (const auto& PageEntries : Accumulator) { - for (const auto& Entry : PageEntries) { - if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry, lk)) { - InvalidatedAnyEntries = true; - } - } - } - - if (InvalidatedAnyEntries) { // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE); } } -void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, - uint64_t Start, uint64_t Length) { - InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length); -} - -bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, - const FEXCore::LookupCacheWriteLockToken& lk) { - LogMan::Throw::AFmt(static_cast(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to " - "be unique_locked here"); - - return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP, lk); -} - void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) { static_cast(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1); } diff --git a/FEXCore/Source/Interface/Core/LookupCache.cpp b/FEXCore/Source/Interface/Core/LookupCache.cpp index 41efed911..b8fec2618 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.cpp +++ b/FEXCore/Source/Interface/Core/LookupCache.cpp @@ -75,7 +75,7 @@ LookupCache::~LookupCache() { // These will get freed when their memory allocators are deallocated. } -void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) { +void LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) { // Clear out the page memory // PagePointer and PageMemory are sequential with each other. Clear both at once. FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), @@ -86,12 +86,12 @@ void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) { void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) { // Clear L1 and L2 by clearing the full cache. FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false); + CachedCodePages.clear(); } void LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) { // Clear L1 and L2 by clearing the full cache. - FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false); - + ClearThreadLocalCaches(lk); Shared->ClearCache(lk); } diff --git a/FEXCore/Source/Interface/Core/LookupCache.h b/FEXCore/Source/Interface/Core/LookupCache.h index 3c1f5d04d..33e7ed657 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.h +++ b/FEXCore/Source/Interface/Core/LookupCache.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -17,7 +18,13 @@ #include namespace FEXCore { -struct LookupCacheWriteLockToken { +struct LookupCacheBaseLockToken { +protected: + // Protected constructor - only derived classes can construct + LookupCacheBaseLockToken() = default; +}; + +struct LookupCacheWriteLockToken : public LookupCacheBaseLockToken { private: // Only constructible by GuestToHostMap friend struct GuestToHostMap; @@ -26,7 +33,7 @@ private: std::lock_guard Lock; }; -struct LookupCacheReadLockToken { +struct LookupCacheReadLockToken : public LookupCacheBaseLockToken { private: // Only constructible by GuestToHostMap friend struct GuestToHostMap; @@ -74,31 +81,36 @@ struct GuestToHostMap { fextl::unique_ptr> BlockLinks_pma; BlockLinksMapType* BlockLinks; - fextl::robin_map BlockList; + struct BlockEntry { + uint64_t HostCode; + fextl::vector CodePages; + }; + + fextl::robin_map BlockList; fextl::map> CodePages; GuestToHostMap(); // Adds to Guest -> Host code mapping - void AddBlockMapping(uint64_t Address, void* HostCode, const LookupCacheWriteLockToken&) { + const BlockEntry& AddBlockMapping(uint64_t Address, const fextl::vector& CodePages, void* HostCode, const LookupCacheWriteLockToken&) { // This may replace an existing mapping // NOTE: Generally no previous entry should exist, however there is one exception: // If the backend updates the active thread's CodeBuffer, the new associated LookupCache // may already contain the block address. Since is comparatively rare, we'll just leak // one of the two blocks in this case. - BlockList[Address] = (uintptr_t)HostCode; + return BlockList.insert_or_assign(Address, BlockEntry {(uintptr_t)HostCode, CodePages}).first->second; } - std::optional FindBlock(uint64_t Address, const LookupCacheReadLockToken&) { + const BlockEntry* FindBlock(uint64_t Address, const LookupCacheReadLockToken&) { auto HostCode = BlockList.find(Address); if (HostCode == BlockList.end()) { - return std::nullopt; + return nullptr; } - return HostCode->second; + return &HostCode->second; } - bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken&) { + bool Erase(uint64_t Address, const LookupCacheWriteLockToken&) { // Sever any links to this block auto lower = BlockLinks->lower_bound({Address, nullptr}); auto upper = BlockLinks->upper_bound({Address, reinterpret_cast(UINTPTR_MAX)}); @@ -110,6 +122,20 @@ struct GuestToHostMap { return BlockList.erase(Address) != 0; } + void InvalidateRange(uint64_t Start, uint64_t Length) { + auto lk = AcquireWriteLock(); + + auto lower = CodePages.lower_bound(Start >> 12); + auto upper = CodePages.upper_bound((Start + Length - 1) >> 12); + + for (auto it = lower; it != upper; it++) { + for (const auto& Entry : it->second) { + Erase(Entry, lk); + } + } + CodePages.erase(lower, upper); + } + void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken&) { BlockLinks->insert({{GuestDestination, HostLink}, delinker}); @@ -185,10 +211,10 @@ public: if (!HostPtr) { // Try L3 - auto HostCode = Shared->FindBlock(Address, lk); - if (HostCode) { - CacheBlockMapping(Address, HostCode.value(), lk); - HostPtr = HostCode.value(); + auto Entry = Shared->FindBlock(Address, lk); + if (Entry) { + CacheBlockMapping(Address, *Entry, false, lk); + HostPtr = Entry->HostCode; } } } @@ -259,32 +285,25 @@ public: } // Adds to Guest -> Host code mapping - void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, void* HostCode) { + void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, const fextl::vector& CodePages, void* HostCode) { std::optional> LockTime( Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr); auto lk = Shared->AcquireWriteLock(); LockTime.reset(); - Shared->AddBlockMapping(Address, HostCode, lk); + const auto& Entry = Shared->AddBlockMapping(Address, CodePages, HostCode, lk); // There is no need to update L1 or L2, they will get updated on first lookup // However, adding to L1 here increases performance - auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; - L1Entry.GuestCode = Address; - L1Entry.HostCode = (uintptr_t)HostCode; + CacheBlockMapping(Address, Entry, true, lk); } - // NOTE: It's the caller's responsibility to call Erase() for all other - // GuestToHostMaps that share the same LookupCache. Otherwise, the - // L1/L2 caches will contain stale references to deallocated memory. - bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken& lk) { - bool ErasedAny = Shared->Erase(Frame, Address, lk); - + // Invalidates L1/L2 for a given guest block + void InvalidateCache(uint64_t Address, const LookupCacheWriteLockToken& lk) { // Do L1 auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; if (L1Entry.GuestCode == Address) { L1Entry.GuestCode = 0; - ErasedAny = true; // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer // This is a soft guarantee for cross thread invalidation, as atomics are not used // and it hasn't been thoroughly tested @@ -300,7 +319,7 @@ public: uint64_t LocalPagePointer = Pointers[Address]; if (!LocalPagePointer) { // Page for this code didn't even exist, nothing to do - return ErasedAny; + return; } // Page exists, just set the offset to zero @@ -308,7 +327,22 @@ public: BlockPointers[PageOffset].GuestCode = 0; BlockPointers[PageOffset].HostCode = 0; } - return true; + } + + // Invalidates all L1/L2 entries for all guest block that intersect the given range + bool InvalidateCacheRange(uint64_t Start, uint64_t Length) { + auto lk = Shared->AcquireWriteLock(); + + auto lower = CachedCodePages.lower_bound(Start >> 12); + auto upper = CachedCodePages.upper_bound((Start + Length - 1) >> 12); + + for (auto it = lower; it != upper; it++) { + for (const auto& Entry : it->second) { + InvalidateCache(Entry, lk); + } + } + CachedCodePages.erase(lower, upper); + return upper != lower; } void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, @@ -317,7 +351,7 @@ public: } void ClearCache(const LookupCacheWriteLockToken&); - void ClearL2Cache(const LookupCacheReadLockToken&); + void ClearL2Cache(const LookupCacheBaseLockToken&); void ClearThreadLocalCaches(const LookupCacheWriteLockToken&); uintptr_t GetL1Pointer() const { @@ -345,13 +379,17 @@ public: } private: - void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheReadLockToken& lk) { + void CacheBlockMapping(uint64_t Address, const GuestToHostMap::BlockEntry& Entry, bool L1Only, const LookupCacheBaseLockToken& lk) { + for (const auto& CodePage : Entry.CodePages) { + CachedCodePages[CodePage >> 12].insert(Address); + } + // Do L1 auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; L1Entry.GuestCode = Address; - L1Entry.HostCode = HostCode; + L1Entry.HostCode = Entry.HostCode; - if (!DisableL2Cache()) { + if (!DisableL2Cache() && !L1Only) { // Do ful map auto FullAddress = Address; Address = Address & (VirtualMemSize - 1); @@ -368,7 +406,7 @@ private: if (!NewPageBacking) { // Couldn't allocate, clear L2 and retry ClearL2Cache(lk); - CacheBlockMapping(Address, HostCode, lk); + CacheBlockMapping(Address, Entry, false, lk); return; } Pointers[Address] = NewPageBacking; @@ -380,7 +418,7 @@ private: // This silently replaces existing mappings BlockPointers[PageOffset].GuestCode = FullAddress; - BlockPointers[PageOffset].HostCode = HostCode; + BlockPointers[PageOffset].HostCode = Entry.HostCode; } } @@ -398,6 +436,9 @@ private: return PageMemory + NewBase; } + // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2 + fextl::map> CachedCodePages; + uintptr_t PagePointer; uintptr_t PageMemory; uintptr_t L1Pointer; diff --git a/FEXCore/include/FEXCore/Core/Context.h b/FEXCore/include/FEXCore/Core/Context.h index 3f12d6017..6b44a30bd 100644 --- a/FEXCore/include/FEXCore/Core/Context.h +++ b/FEXCore/include/FEXCore/Core/Context.h @@ -42,9 +42,6 @@ enum OperatingMode { using CodeRangeInvalidationFn = std::function; -// Nested vector of guest block entrypoints -using InvalidatedEntryAccumulator = fextl::vector>; - using CustomIREntrypointHandler = std::function; using ExitHandler = std::function; @@ -141,8 +138,9 @@ public: virtual AbstractCodeCache& GetCodeCache() = 0; FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0; - FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange( - FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) = 0; + FEX_DEFAULT_VISIBILITY virtual void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) = 0; + FEX_DEFAULT_VISIBILITY virtual void + InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) = 0; FEX_DEFAULT_VISIBILITY virtual FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() = 0; FEX_DEFAULT_VISIBILITY virtual void From 8c00ac78b145a581cdafb3df87a53b5183128e46 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Tue, 28 Oct 2025 22:24:35 +0000 Subject: [PATCH 5/6] Linux: Support new two-stage invalidation model --- FEXCore/Source/Interface/Core/Core.cpp | 6 +++--- FEXCore/Source/Interface/Core/LookupCache.h | 4 ++-- Source/Tools/CodeSizeValidation/Main.cpp | 4 ++-- .../Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h | 10 ++++------ 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 6ff1deb9b..282ecabba 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -886,11 +886,11 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) { FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange"); - LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); std::scoped_lock lk {CodeBufferListLock}; auto it = CodeBufferList.begin(); while (it != CodeBufferList.end()) { - if (auto Strong = it->lock(); Strong) { + if (auto Strong = it->lock()) { Strong->LookupCache->InvalidateRange(Start, Length); it++; } else { @@ -900,7 +900,7 @@ void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length } void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) { - LogMan::Throw::AFmt(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); // Ensures now-modified mappings aren't cached as being in their previous non-executable state. // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here. diff --git a/FEXCore/Source/Interface/Core/LookupCache.h b/FEXCore/Source/Interface/Core/LookupCache.h index 33e7ed657..fab666eb2 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.h +++ b/FEXCore/Source/Interface/Core/LookupCache.h @@ -8,8 +8,8 @@ #include #include #include +#include #include -#include #include #include @@ -437,7 +437,7 @@ private: } // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2 - fextl::map> CachedCodePages; + fextl::map> CachedCodePages; uintptr_t PagePointer; uintptr_t PageMemory; diff --git a/Source/Tools/CodeSizeValidation/Main.cpp b/Source/Tools/CodeSizeValidation/Main.cpp index 1562c19ed..07380659d 100644 --- a/Source/Tools/CodeSizeValidation/Main.cpp +++ b/Source/Tools/CodeSizeValidation/Main.cpp @@ -52,8 +52,8 @@ public: { auto CodeInvalidationlk = FEXCore::GuardSignalDeferringSection(CTX->GetCodeInvalidationMutex(), Thread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - CTX->InvalidateGuestCodeRange(Thread, Accumulator, reinterpret_cast(CodeStart), MAX_CODE_SIZE); + CTX->InvalidateCodeBuffersCodeRange(reinterpret_cast(CodeStart), MAX_CODE_SIZE); + CTX->InvalidateThreadCachedCodeRange(Thread, reinterpret_cast(CodeStart), MAX_CODE_SIZE); } ClearStats(); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h index e19f43224..541a07cc7 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h @@ -208,10 +208,9 @@ public: // Thread object isn't valid very early in frontend's initialization. // To be more optimal the frontend should provide this code with a valid Thread object earlier. auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - + CTX->InvalidateCodeBuffersCodeRange(Start, Length); for (auto& Thread : Threads) { - CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length); + CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length); } } @@ -223,10 +222,9 @@ public: // Thread object isn't valid very early in frontend's initialization. // To be more optimal the frontend should provide this code with a valid Thread object earlier. auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - + CTX->InvalidateCodeBuffersCodeRange(Start, Length); for (auto& Thread : Threads) { - CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length); + CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length); } // Callback while holding the locks. From 9a7285dca4aa70769200bd9b5049daf192047b82 Mon Sep 17 00:00:00 2001 From: Billy Laws Date: Tue, 28 Oct 2025 22:24:51 +0000 Subject: [PATCH 6/6] Windows: Support new two-stage invalidation model --- Source/Windows/Common/InvalidationTracker.cpp | 41 +++++++------------ Source/Windows/Common/InvalidationTracker.h | 3 ++ 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/Source/Windows/Common/InvalidationTracker.cpp b/Source/Windows/Common/InvalidationTracker.cpp index e6ce26aa8..5bd2930e0 100644 --- a/Source/Windows/Common/InvalidationTracker.cpp +++ b/Source/Windows/Common/InvalidationTracker.cpp @@ -56,11 +56,7 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u if (NeedsInvalidate) { // IntervalsLock cannot be held during invalidation - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize); - } + InvalidateIntervalInternal(AlignedBase, AlignedSize); } } @@ -115,13 +111,8 @@ InvalidationTracker::InvalidateContainingSectionResult InvalidationTracker::Inva reinterpret_cast(Info.AllocationBase) == SectionBase) { SectionSize += Info.RegionSize; } - { - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, SectionBase, SectionSize); - } - } + + InvalidateIntervalInternal(SectionBase, SectionSize); if (Free) { std::unique_lock Lock(IntervalsLock); @@ -141,13 +132,7 @@ void InvalidationTracker::InvalidateAlignedInterval(uint64_t Address, uint64_t S const auto AlignedBase = Address & FEXCore::Utils::FEX_PAGE_MASK; const auto AlignedSize = std::max(Size, (Address - AlignedBase + Size + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK); - { - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize); - } - } + InvalidateIntervalInternal(AlignedBase, AlignedSize); if (Free) { std::unique_lock Lock(IntervalsLock); @@ -197,14 +182,8 @@ bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThread }(FaultAddress); if (NeedsInvalidate) { - { - // IntervalsLock cannot be held during invalidation - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE); - } - } + // IntervalsLock cannot be held during invalidation + InvalidateIntervalInternal(FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE); DetectMonoBackpatcherBlock(Thread, HostPc); return true; } @@ -274,4 +253,12 @@ void InvalidationTracker::DisableSMCDetection() { } while (Query.Size); } +void InvalidationTracker::InvalidateIntervalInternal(uint64_t Address, uint64_t Size) { + std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); + CTX.InvalidateCodeBuffersCodeRange(Address, Size); + for (auto Thread : Threads) { + CTX.InvalidateThreadCachedCodeRange(Thread.second, Address, Size); + } +} + } // namespace FEX::Windows diff --git a/Source/Windows/Common/InvalidationTracker.h b/Source/Windows/Common/InvalidationTracker.h index 39fc382a7..88c2ee1af 100644 --- a/Source/Windows/Common/InvalidationTracker.h +++ b/Source/Windows/Common/InvalidationTracker.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,8 @@ public: private: void DetectMonoBackpatcherBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC); void DisableSMCDetection(); + void InvalidateIntervalInternal(uint64_t Address, uint64_t Size); + FEXCore::IntervalList XIntervals; FEXCore::IntervalList RWXIntervals;