diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index 6a7abb1db..0f6339984 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -61,7 +61,7 @@ struct CustomIRResult { , Data(Data) {} }; -using BlockDelinkerFunc = void (*)(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record); +using BlockDelinkerFunc = void (*)(FEXCore::Context::ExitFunctionLinkData* Record); constexpr uint32_t TSC_SCALE_MAXIMUM = 1'000'000'000; ///< 1Ghz class CodeCache : public AbstractCodeCache { @@ -154,10 +154,10 @@ public: return CodeCache; } - void OnCodeBufferAllocated(CPU::CodeBuffer&) override; + void OnCodeBufferAllocated(const std::shared_ptr&) override; void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) override; - void InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, - uint64_t Length) override; + void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) override; + void InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) override; FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() override { return CodeInvalidationMutex; } @@ -229,8 +229,6 @@ public: ContextImpl(const FEXCore::HostFeatures& Features); - static bool ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, const FEXCore::LookupCacheWriteLockToken& lk); - static void ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP); // This is used as a replacement for the SMC writes in the mono callsite backpatcher that avoids atomic operations @@ -346,5 +344,8 @@ private: bool MonoDetected = false; std::atomic MonoBackpatcherBlock; + + std::mutex CodeBufferListLock; + fextl::vector> CodeBufferList; }; } // namespace FEXCore::Context diff --git a/FEXCore/Source/Interface/Core/CPUBackend.cpp b/FEXCore/Source/Interface/Core/CPUBackend.cpp index 4d1ec87b9..a5aa13f68 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.cpp +++ b/FEXCore/Source/Interface/Core/CPUBackend.cpp @@ -400,7 +400,7 @@ namespace CPU { Latest = Buffer; LatestOffset = 0; - OnCodeBufferAllocated(*Buffer); + OnCodeBufferAllocated(Buffer); return Buffer; } diff --git a/FEXCore/Source/Interface/Core/CPUBackend.h b/FEXCore/Source/Interface/Core/CPUBackend.h index ab9f13b54..9eb9e0639 100644 --- a/FEXCore/Source/Interface/Core/CPUBackend.h +++ b/FEXCore/Source/Interface/Core/CPUBackend.h @@ -81,7 +81,7 @@ namespace CPU { // Protects writes to the latest CodeBuffer and changes to LatestOffset FEXCore::ForkableUniqueMutex CodeBufferWriteMutex; - virtual void OnCodeBufferAllocated(CodeBuffer&) {}; + virtual void OnCodeBufferAllocated(const std::shared_ptr&) {}; private: fextl::shared_ptr Latest; diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 3a1e6120b..282ecabba 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -459,9 +459,14 @@ void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) { } #endif -void ContextImpl::OnCodeBufferAllocated(CPU::CodeBuffer& Buffer) { +void ContextImpl::OnCodeBufferAllocated(const fextl::shared_ptr& Buffer) { if (Config.GlobalJITNaming()) { - Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size); + Symbols.RegisterJITSpace(Buffer->Ptr, Buffer->Size); + } + + { + std::scoped_lock lk {CodeBufferListLock}; + CodeBufferList.emplace_back(Buffer); } } @@ -833,10 +838,14 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ Thread->CPUBackend->ClearRelocations(); } + fextl::vector CodePages; + if (NeedsAddGuestCodeRanges) { // Track in the guest to host map all entrypoints for all pages the compiled block touches, if any page didn't previously // contain code, inform the frontend so it can setup SMC detection. auto BlockInfo = Thread->FrontendDecoder->GetDecodedBlockInfo(); + CodePages.reserve(BlockInfo->CodePages.size()); + CodePages.insert(CodePages.end(), BlockInfo->CodePages.begin(), BlockInfo->CodePages.end()); for (auto CodePage : BlockInfo->CodePages) { if (Thread->LookupCache->AddBlockExecutableRange(Thread, BlockInfo->EntryPoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE)) { SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE); @@ -845,8 +854,9 @@ uintptr_t ContextImpl::CompileBlock(FEXCore::Core::CpuStateFrame* Frame, uint64_ } // Insert to lookup cache + for (auto [GuestAddr, HostAddr] : CompiledCode.EntryPoints) { - Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, HostAddr); + Thread->LookupCache->AddBlockMapping(Thread, GuestAddr, CodePages, HostAddr); } return (uintptr_t)CodePtr; @@ -873,50 +883,37 @@ uintptr_t ContextImpl::CompileSingleStep(FEXCore::Core::CpuStateFrame* Frame, ui return (uintptr_t)CodePtr; } -static void InvalidateGuestThreadCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, - uint64_t Start, uint64_t Length) { +void ContextImpl::InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) { + FEXCORE_PROFILE_SCOPED("InvalidateCodeBuffersCodeRange"); + + LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + std::scoped_lock lk {CodeBufferListLock}; + auto it = CodeBufferList.begin(); + while (it != CodeBufferList.end()) { + if (auto Strong = it->lock()) { + Strong->LookupCache->InvalidateRange(Start, Length); + it++; + } else { + it = CodeBufferList.erase(it); + } + } +} + +void ContextImpl::InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) { + LOGMAN_THROW_A_FMT(CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to be unique_locked here"); + // Ensures now-modified mappings aren't cached as being in their previous non-executable state. // Accessing FrontendDecoder is safe as the thread's code invalidation mutex must be locked here. Thread->FrontendDecoder->ResetExecutableRangeCache(); - auto lk = Thread->LookupCache->AcquireWriteLock(); - auto& CodePages = Thread->LookupCache->Shared->CodePages; + if (Thread->LookupCache->InvalidateCacheRange(Start, Length)) { + FEXCORE_PROFILE_SCOPED("InvalidateCallRet"); - auto lower = CodePages.lower_bound(Start >> 12); - auto upper = CodePages.upper_bound((Start + Length - 1) >> 12); - - for (auto it = lower; it != upper; it++) { - Accumulator.emplace_back(std::move(it->second)); - } - - bool InvalidatedAnyEntries = false; - for (const auto& PageEntries : Accumulator) { - for (const auto& Entry : PageEntries) { - if (ContextImpl::ThreadRemoveCodeEntry(Thread, Entry, lk)) { - InvalidatedAnyEntries = true; - } - } - } - - if (InvalidatedAnyEntries) { // This may cause access violations in the thread on Windows as zeroing is not atomic, this is handled by the frontend Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE); } } -void ContextImpl::InvalidateGuestCodeRange(FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, - uint64_t Start, uint64_t Length) { - InvalidateGuestThreadCodeRange(Thread, Accumulator, Start, Length); -} - -bool ContextImpl::ThreadRemoveCodeEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t GuestRIP, - const FEXCore::LookupCacheWriteLockToken& lk) { - LogMan::Throw::AFmt(static_cast(Thread->CTX)->CodeInvalidationMutex.try_lock() == false, "CodeInvalidationMutex needs to " - "be unique_locked here"); - - return Thread->LookupCache->Erase(Thread->CurrentFrame, GuestRIP, lk); -} - void ContextImpl::ThreadRemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame* Frame, uint64_t GuestRIP) { static_cast(Frame->Thread->CTX)->SyscallHandler->InvalidateGuestCodeRange(Frame->Thread, GuestRIP, 1); } diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index 8be3e83fa..a0ffd06c7 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -493,7 +493,7 @@ void Arm64JITCore::Op_Unhandled(const IR::IROp_Header* IROp, IR::Ref Node) { } } -static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record, bool Call) { +static void DirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record, bool Call) { uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10; uintptr_t CallerAddress = JumpThunkStartAddress + Record->CallerOffset; auto BranchOffset = JumpThunkStartAddress / 4 - CallerAddress / 4; @@ -511,7 +511,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co ARMEmitter::Emitter::ClearICache(reinterpret_cast(CallerAddress), 4); } -static void IndirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { +static void IndirectBlockDelinker(FEXCore::Context::ExitFunctionLinkData* Record) { uintptr_t JumpThunkStartAddress = reinterpret_cast(Record) - 0x10; uint32_t BranchInst = 0; ARMEmitter::Emitter BranchEmit(reinterpret_cast(&BranchInst), 4); @@ -577,16 +577,11 @@ uint64_t Arm64JITCore::ExitFunctionLink(FEXCore::Core::CpuStateFrame* Frame, FEX if (KnownCallMarkerInst == ExpectedKnownCallMarkerInst) { BranchEmit.bl(BranchOffset); Thread->LookupCache->AddBlockLink( - GuestRip, Record, - [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Frame, Record, true); }, lk); + GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, true); }, lk); } else { BranchEmit.b(BranchOffset); Thread->LookupCache->AddBlockLink( - GuestRip, Record, - [](FEXCore::Core::CpuStateFrame* Frame, FEXCore::Context::ExitFunctionLinkData* Record) { - DirectBlockDelinker(Frame, Record, false); - }, - lk); + GuestRip, Record, [](FEXCore::Context::ExitFunctionLinkData* Record) { DirectBlockDelinker(Record, false); }, lk); } std::atomic_ref(*reinterpret_cast(CallerAddress)).store(BranchInst, std::memory_order::relaxed); diff --git a/FEXCore/Source/Interface/Core/LookupCache.cpp b/FEXCore/Source/Interface/Core/LookupCache.cpp index 41efed911..b8fec2618 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.cpp +++ b/FEXCore/Source/Interface/Core/LookupCache.cpp @@ -75,7 +75,7 @@ LookupCache::~LookupCache() { // These will get freed when their memory allocators are deallocated. } -void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) { +void LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) { // Clear out the page memory // PagePointer and PageMemory are sequential with each other. Clear both at once. FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), @@ -86,12 +86,12 @@ void LookupCache::ClearL2Cache(const FEXCore::LookupCacheReadLockToken& lk) { void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) { // Clear L1 and L2 by clearing the full cache. FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false); + CachedCodePages.clear(); } void LookupCache::ClearCache(const LookupCacheWriteLockToken& lk) { // Clear L1 and L2 by clearing the full cache. - FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false); - + ClearThreadLocalCaches(lk); Shared->ClearCache(lk); } diff --git a/FEXCore/Source/Interface/Core/LookupCache.h b/FEXCore/Source/Interface/Core/LookupCache.h index c0e2b4e39..fab666eb2 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.h +++ b/FEXCore/Source/Interface/Core/LookupCache.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -17,7 +18,13 @@ #include namespace FEXCore { -struct LookupCacheWriteLockToken { +struct LookupCacheBaseLockToken { +protected: + // Protected constructor - only derived classes can construct + LookupCacheBaseLockToken() = default; +}; + +struct LookupCacheWriteLockToken : public LookupCacheBaseLockToken { private: // Only constructible by GuestToHostMap friend struct GuestToHostMap; @@ -26,7 +33,7 @@ private: std::lock_guard Lock; }; -struct LookupCacheReadLockToken { +struct LookupCacheReadLockToken : public LookupCacheBaseLockToken { private: // Only constructible by GuestToHostMap friend struct GuestToHostMap; @@ -74,42 +81,61 @@ struct GuestToHostMap { fextl::unique_ptr> BlockLinks_pma; BlockLinksMapType* BlockLinks; - fextl::robin_map BlockList; + struct BlockEntry { + uint64_t HostCode; + fextl::vector CodePages; + }; + + fextl::robin_map BlockList; fextl::map> CodePages; GuestToHostMap(); // Adds to Guest -> Host code mapping - void AddBlockMapping(uint64_t Address, void* HostCode, const LookupCacheWriteLockToken&) { + const BlockEntry& AddBlockMapping(uint64_t Address, const fextl::vector& CodePages, void* HostCode, const LookupCacheWriteLockToken&) { // This may replace an existing mapping // NOTE: Generally no previous entry should exist, however there is one exception: // If the backend updates the active thread's CodeBuffer, the new associated LookupCache // may already contain the block address. Since is comparatively rare, we'll just leak // one of the two blocks in this case. - BlockList[Address] = (uintptr_t)HostCode; + return BlockList.insert_or_assign(Address, BlockEntry {(uintptr_t)HostCode, CodePages}).first->second; } - std::optional FindBlock(uint64_t Address, const LookupCacheReadLockToken&) { + const BlockEntry* FindBlock(uint64_t Address, const LookupCacheReadLockToken&) { auto HostCode = BlockList.find(Address); if (HostCode == BlockList.end()) { - return std::nullopt; + return nullptr; } - return HostCode->second; + return &HostCode->second; } - bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken&) { + bool Erase(uint64_t Address, const LookupCacheWriteLockToken&) { // Sever any links to this block auto lower = BlockLinks->lower_bound({Address, nullptr}); auto upper = BlockLinks->upper_bound({Address, reinterpret_cast(UINTPTR_MAX)}); for (auto it = lower; it != upper; it = BlockLinks->erase(it)) { - it->second(Frame, it->first.HostLink); + it->second(it->first.HostLink); } // Remove from BlockList return BlockList.erase(Address) != 0; } + void InvalidateRange(uint64_t Start, uint64_t Length) { + auto lk = AcquireWriteLock(); + + auto lower = CodePages.lower_bound(Start >> 12); + auto upper = CodePages.upper_bound((Start + Length - 1) >> 12); + + for (auto it = lower; it != upper; it++) { + for (const auto& Entry : it->second) { + Erase(Entry, lk); + } + } + CodePages.erase(lower, upper); + } + void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, const FEXCore::Context::BlockDelinkerFunc& delinker, const LookupCacheWriteLockToken&) { BlockLinks->insert({{GuestDestination, HostLink}, delinker}); @@ -185,10 +211,10 @@ public: if (!HostPtr) { // Try L3 - auto HostCode = Shared->FindBlock(Address, lk); - if (HostCode) { - CacheBlockMapping(Address, HostCode.value(), lk); - HostPtr = HostCode.value(); + auto Entry = Shared->FindBlock(Address, lk); + if (Entry) { + CacheBlockMapping(Address, *Entry, false, lk); + HostPtr = Entry->HostCode; } } } @@ -259,32 +285,25 @@ public: } // Adds to Guest -> Host code mapping - void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, void* HostCode) { + void AddBlockMapping(FEXCore::Core::InternalThreadState* Thread, uint64_t Address, const fextl::vector& CodePages, void* HostCode) { std::optional> LockTime( Thread->ThreadStats ? &Thread->ThreadStats->AccumulatedCacheWriteLockTime : nullptr); auto lk = Shared->AcquireWriteLock(); LockTime.reset(); - Shared->AddBlockMapping(Address, HostCode, lk); + const auto& Entry = Shared->AddBlockMapping(Address, CodePages, HostCode, lk); // There is no need to update L1 or L2, they will get updated on first lookup // However, adding to L1 here increases performance - auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; - L1Entry.GuestCode = Address; - L1Entry.HostCode = (uintptr_t)HostCode; + CacheBlockMapping(Address, Entry, true, lk); } - // NOTE: It's the caller's responsibility to call Erase() for all other - // GuestToHostMaps that share the same LookupCache. Otherwise, the - // L1/L2 caches will contain stale references to deallocated memory. - bool Erase(FEXCore::Core::CpuStateFrame* Frame, uint64_t Address, const LookupCacheWriteLockToken& lk) { - bool ErasedAny = Shared->Erase(Frame, Address, lk); - + // Invalidates L1/L2 for a given guest block + void InvalidateCache(uint64_t Address, const LookupCacheWriteLockToken& lk) { // Do L1 auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; if (L1Entry.GuestCode == Address) { L1Entry.GuestCode = 0; - ErasedAny = true; // Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer // This is a soft guarantee for cross thread invalidation, as atomics are not used // and it hasn't been thoroughly tested @@ -300,7 +319,7 @@ public: uint64_t LocalPagePointer = Pointers[Address]; if (!LocalPagePointer) { // Page for this code didn't even exist, nothing to do - return ErasedAny; + return; } // Page exists, just set the offset to zero @@ -308,7 +327,22 @@ public: BlockPointers[PageOffset].GuestCode = 0; BlockPointers[PageOffset].HostCode = 0; } - return true; + } + + // Invalidates all L1/L2 entries for all guest block that intersect the given range + bool InvalidateCacheRange(uint64_t Start, uint64_t Length) { + auto lk = Shared->AcquireWriteLock(); + + auto lower = CachedCodePages.lower_bound(Start >> 12); + auto upper = CachedCodePages.upper_bound((Start + Length - 1) >> 12); + + for (auto it = lower; it != upper; it++) { + for (const auto& Entry : it->second) { + InvalidateCache(Entry, lk); + } + } + CachedCodePages.erase(lower, upper); + return upper != lower; } void AddBlockLink(uint64_t GuestDestination, FEXCore::Context::ExitFunctionLinkData* HostLink, @@ -317,7 +351,7 @@ public: } void ClearCache(const LookupCacheWriteLockToken&); - void ClearL2Cache(const LookupCacheReadLockToken&); + void ClearL2Cache(const LookupCacheBaseLockToken&); void ClearThreadLocalCaches(const LookupCacheWriteLockToken&); uintptr_t GetL1Pointer() const { @@ -345,13 +379,17 @@ public: } private: - void CacheBlockMapping(uint64_t Address, uintptr_t HostCode, const LookupCacheReadLockToken& lk) { + void CacheBlockMapping(uint64_t Address, const GuestToHostMap::BlockEntry& Entry, bool L1Only, const LookupCacheBaseLockToken& lk) { + for (const auto& CodePage : Entry.CodePages) { + CachedCodePages[CodePage >> 12].insert(Address); + } + // Do L1 auto& L1Entry = reinterpret_cast(L1Pointer)[Address & L1PointerMask]; L1Entry.GuestCode = Address; - L1Entry.HostCode = HostCode; + L1Entry.HostCode = Entry.HostCode; - if (!DisableL2Cache()) { + if (!DisableL2Cache() && !L1Only) { // Do ful map auto FullAddress = Address; Address = Address & (VirtualMemSize - 1); @@ -368,7 +406,7 @@ private: if (!NewPageBacking) { // Couldn't allocate, clear L2 and retry ClearL2Cache(lk); - CacheBlockMapping(Address, HostCode, lk); + CacheBlockMapping(Address, Entry, false, lk); return; } Pointers[Address] = NewPageBacking; @@ -380,7 +418,7 @@ private: // This silently replaces existing mappings BlockPointers[PageOffset].GuestCode = FullAddress; - BlockPointers[PageOffset].HostCode = HostCode; + BlockPointers[PageOffset].HostCode = Entry.HostCode; } } @@ -398,6 +436,9 @@ private: return PageMemory + NewBase; } + // Maps from a page index to all blocks in the page that have at some point been fetched into L1/L2 + fextl::map> CachedCodePages; + uintptr_t PagePointer; uintptr_t PageMemory; uintptr_t L1Pointer; diff --git a/FEXCore/include/FEXCore/Core/Context.h b/FEXCore/include/FEXCore/Core/Context.h index 3f12d6017..6b44a30bd 100644 --- a/FEXCore/include/FEXCore/Core/Context.h +++ b/FEXCore/include/FEXCore/Core/Context.h @@ -42,9 +42,6 @@ enum OperatingMode { using CodeRangeInvalidationFn = std::function; -// Nested vector of guest block entrypoints -using InvalidatedEntryAccumulator = fextl::vector>; - using CustomIREntrypointHandler = std::function; using ExitHandler = std::function; @@ -141,8 +138,9 @@ public: virtual AbstractCodeCache& GetCodeCache() = 0; FEX_DEFAULT_VISIBILITY virtual void ClearCodeCache(FEXCore::Core::InternalThreadState* Thread, bool NewCodeBuffer = true) = 0; - FEX_DEFAULT_VISIBILITY virtual void InvalidateGuestCodeRange( - FEXCore::Core::InternalThreadState* Thread, InvalidatedEntryAccumulator& Accumulator, uint64_t Start, uint64_t Length) = 0; + FEX_DEFAULT_VISIBILITY virtual void InvalidateCodeBuffersCodeRange(uint64_t Start, uint64_t Length) = 0; + FEX_DEFAULT_VISIBILITY virtual void + InvalidateThreadCachedCodeRange(FEXCore::Core::InternalThreadState* Thread, uint64_t Start, uint64_t Length) = 0; FEX_DEFAULT_VISIBILITY virtual FEXCore::ForkableSharedMutex& GetCodeInvalidationMutex() = 0; FEX_DEFAULT_VISIBILITY virtual void diff --git a/FEXCore/include/FEXCore/fextl/robin_set.h b/FEXCore/include/FEXCore/fextl/robin_set.h new file mode 100644 index 000000000..9bdb935e5 --- /dev/null +++ b/FEXCore/include/FEXCore/fextl/robin_set.h @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: MIT +#pragma once +#include + +#include + +namespace fextl { +template, class KeyEqual = std::equal_to, class Allocator = fextl::FEXAlloc> +using robin_set = tsl::robin_set; +} diff --git a/Source/Tools/CodeSizeValidation/Main.cpp b/Source/Tools/CodeSizeValidation/Main.cpp index 1562c19ed..07380659d 100644 --- a/Source/Tools/CodeSizeValidation/Main.cpp +++ b/Source/Tools/CodeSizeValidation/Main.cpp @@ -52,8 +52,8 @@ public: { auto CodeInvalidationlk = FEXCore::GuardSignalDeferringSection(CTX->GetCodeInvalidationMutex(), Thread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - CTX->InvalidateGuestCodeRange(Thread, Accumulator, reinterpret_cast(CodeStart), MAX_CODE_SIZE); + CTX->InvalidateCodeBuffersCodeRange(reinterpret_cast(CodeStart), MAX_CODE_SIZE); + CTX->InvalidateThreadCachedCodeRange(Thread, reinterpret_cast(CodeStart), MAX_CODE_SIZE); } ClearStats(); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h index e19f43224..541a07cc7 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/ThreadManager.h @@ -208,10 +208,9 @@ public: // Thread object isn't valid very early in frontend's initialization. // To be more optimal the frontend should provide this code with a valid Thread object earlier. auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - + CTX->InvalidateCodeBuffersCodeRange(Start, Length); for (auto& Thread : Threads) { - CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length); + CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length); } } @@ -223,10 +222,9 @@ public: // Thread object isn't valid very early in frontend's initialization. // To be more optimal the frontend should provide this code with a valid Thread object earlier. auto CodeInvalidationlk = GuardSignalDeferringSectionWithFallback(CTX->GetCodeInvalidationMutex(), CallingThread); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - + CTX->InvalidateCodeBuffersCodeRange(Start, Length); for (auto& Thread : Threads) { - CTX->InvalidateGuestCodeRange(Thread->Thread, Accumulator, Start, Length); + CTX->InvalidateThreadCachedCodeRange(Thread->Thread, Start, Length); } // Callback while holding the locks. diff --git a/Source/Windows/Common/InvalidationTracker.cpp b/Source/Windows/Common/InvalidationTracker.cpp index e6ce26aa8..5bd2930e0 100644 --- a/Source/Windows/Common/InvalidationTracker.cpp +++ b/Source/Windows/Common/InvalidationTracker.cpp @@ -56,11 +56,7 @@ void InvalidationTracker::HandleMemoryProtectionNotification(uint64_t Address, u if (NeedsInvalidate) { // IntervalsLock cannot be held during invalidation - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize); - } + InvalidateIntervalInternal(AlignedBase, AlignedSize); } } @@ -115,13 +111,8 @@ InvalidationTracker::InvalidateContainingSectionResult InvalidationTracker::Inva reinterpret_cast(Info.AllocationBase) == SectionBase) { SectionSize += Info.RegionSize; } - { - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, SectionBase, SectionSize); - } - } + + InvalidateIntervalInternal(SectionBase, SectionSize); if (Free) { std::unique_lock Lock(IntervalsLock); @@ -141,13 +132,7 @@ void InvalidationTracker::InvalidateAlignedInterval(uint64_t Address, uint64_t S const auto AlignedBase = Address & FEXCore::Utils::FEX_PAGE_MASK; const auto AlignedSize = std::max(Size, (Address - AlignedBase + Size + FEXCore::Utils::FEX_PAGE_SIZE - 1) & FEXCore::Utils::FEX_PAGE_MASK); - { - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, AlignedBase, AlignedSize); - } - } + InvalidateIntervalInternal(AlignedBase, AlignedSize); if (Free) { std::unique_lock Lock(IntervalsLock); @@ -197,14 +182,8 @@ bool InvalidationTracker::HandleRWXAccessViolation(FEXCore::Core::InternalThread }(FaultAddress); if (NeedsInvalidate) { - { - // IntervalsLock cannot be held during invalidation - std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); - FEXCore::Context::InvalidatedEntryAccumulator Accumulator; - for (auto Thread : Threads) { - CTX.InvalidateGuestCodeRange(Thread.second, Accumulator, FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE); - } - } + // IntervalsLock cannot be held during invalidation + InvalidateIntervalInternal(FaultAddress & FEXCore::Utils::FEX_PAGE_MASK, FEXCore::Utils::FEX_PAGE_SIZE); DetectMonoBackpatcherBlock(Thread, HostPc); return true; } @@ -274,4 +253,12 @@ void InvalidationTracker::DisableSMCDetection() { } while (Query.Size); } +void InvalidationTracker::InvalidateIntervalInternal(uint64_t Address, uint64_t Size) { + std::scoped_lock Lock(CTX.GetCodeInvalidationMutex()); + CTX.InvalidateCodeBuffersCodeRange(Address, Size); + for (auto Thread : Threads) { + CTX.InvalidateThreadCachedCodeRange(Thread.second, Address, Size); + } +} + } // namespace FEX::Windows diff --git a/Source/Windows/Common/InvalidationTracker.h b/Source/Windows/Common/InvalidationTracker.h index 39fc382a7..88c2ee1af 100644 --- a/Source/Windows/Common/InvalidationTracker.h +++ b/Source/Windows/Common/InvalidationTracker.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,8 @@ public: private: void DetectMonoBackpatcherBlock(FEXCore::Core::InternalThreadState* Thread, uint64_t HostPC); void DisableSMCDetection(); + void InvalidateIntervalInternal(uint64_t Address, uint64_t Size); + FEXCore::IntervalList XIntervals; FEXCore::IntervalList RWXIntervals;