From 5be0dc9fc5c53f6915d0274b9a0ff40967cb24ff Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Wed, 22 Apr 2026 14:44:19 +0200 Subject: [PATCH] CodeCache: Implement lazy code loading --- .../Source/Interface/Config/Config.json.in | 7 + FEXCore/Source/Interface/Context/Context.cpp | 2 +- FEXCore/Source/Interface/Context/Context.h | 8 +- FEXCore/Source/Interface/Core/CodeCache.cpp | 482 ++++++++++++------ FEXCore/Source/Interface/Core/Core.cpp | 1 - FEXCore/Source/Interface/Core/LookupCache.cpp | 3 + FEXCore/include/FEXCore/Core/CodeCache.h | 91 +++- .../include/FEXCore/Utils/SignalScopeGuards.h | 3 + .../LinuxEmulation/LinuxSyscalls/Syscalls.cpp | 18 +- .../LinuxEmulation/LinuxSyscalls/Syscalls.h | 5 +- .../LinuxSyscalls/SyscallsSMCTracking.cpp | 95 +++- .../LinuxSyscalls/SyscallsVMATracking.cpp | 28 +- .../LinuxSyscalls/SyscallsVMATracking.h | 19 +- Source/Windows/Common/ImageTracker.cpp | 3 +- 14 files changed, 563 insertions(+), 202 deletions(-) diff --git a/FEXCore/Source/Interface/Config/Config.json.in b/FEXCore/Source/Interface/Config/Config.json.in index a3ddd6de8..fa56db44c 100644 --- a/FEXCore/Source/Interface/Config/Config.json.in +++ b/FEXCore/Source/Interface/Config/Config.json.in @@ -23,6 +23,13 @@ "Enable the code caching subsystem" ] }, + "EnableLazyCodeCachingWIP": { + "Type": "bool", + "Default": "false", + "Desc": [ + "Enable lazy loading of chunks in code caches" + ] + }, "EnableCodeCacheValidation": { "Type": "bool", "Default": "false", diff --git a/FEXCore/Source/Interface/Context/Context.cpp b/FEXCore/Source/Interface/Context/Context.cpp index d4af939cc..de8415b0a 100644 --- a/FEXCore/Source/Interface/Context/Context.cpp +++ b/FEXCore/Source/Interface/Context/Context.cpp @@ -53,6 +53,6 @@ FEXCore::CPUID::FunctionResults FEXCore::Context::ContextImpl::RunCPUIDFunctionN } bool FEXCore::Context::ContextImpl::IsAddressInCodeBuffer(FEXCore::Core::InternalThreadState* Thread, uintptr_t Address) const { - return Thread->CPUBackend->IsAddressInCodeBuffer(Address); + return Thread->CPUBackend->IsAddressInCodeBuffer(Address) || CodeCache.IsAddressInMappedCodeBuffer(Address); } } // namespace FEXCore::Context diff --git a/FEXCore/Source/Interface/Context/Context.h b/FEXCore/Source/Interface/Context/Context.h index e195f6b9e..b3868ca22 100644 --- a/FEXCore/Source/Interface/Context/Context.h +++ b/FEXCore/Source/Interface/Context/Context.h @@ -76,11 +76,17 @@ public: bool IsGeneratingCache = false; FEX_CONFIG_OPT(EnableCodeCaching, ENABLECODECACHINGWIP); + FEX_CONFIG_OPT(EnableLazyCodeCaching, ENABLELAZYCODECACHINGWIP); FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION); uint64_t ComputeCodeMapId(std::string_view Filename, int FD) override; bool SaveData(Core::InternalThreadState&, int TargetFD, const ExecutableFileSectionInfo&, uint64_t SerializedBaseAddress) override; - bool LoadData(Core::InternalThreadState*, std::byte* MappedCacheFile, const ExecutableFileSectionInfo&) override; + + fextl::unique_ptr LoadCache(std::span CacheFile, const ExecutableFileInfo&, uint64_t FileStartVA) override; + + bool EnableLoadedSection(Core::InternalThreadState*, MappedCodeCacheFile&, const ExecutableFileSectionInfo&) override; + + void FinalizeCodePages(MappedCodeCacheFile&, std::span CodeRange) override; /** * Performs expensive extra validation on the loaded code cache data. diff --git a/FEXCore/Source/Interface/Core/CodeCache.cpp b/FEXCore/Source/Interface/Core/CodeCache.cpp index b80d0d5a0..9d23b311f 100644 --- a/FEXCore/Source/Interface/Core/CodeCache.cpp +++ b/FEXCore/Source/Interface/Core/CodeCache.cpp @@ -1,4 +1,9 @@ // SPDX-License-Identifier: MIT +#include "FEXCore/Utils/LogManager.h" +#include "FEXCore/Utils/MathUtils.h" +#include "FEXCore/Utils/TypeDefines.h" +#include "FEXCore/fextl/memory.h" +#include #include #include @@ -16,10 +21,14 @@ #include +#include #include +#include #include +#include + #include namespace FEXCore { @@ -32,6 +41,32 @@ ExecutableFileInfo::ExecutableFileInfo(fextl::unique_ptr Map #endif ExecutableFileInfo::~ExecutableFileInfo() = default; +MappedCodeCacheFile::~MappedCodeCacheFile() { + if (CacheManager) { + CacheManager->UnregisterMappedCodeBuffer(*this); + } +} + +void AbstractCodeCache::RegisterMappedCodeBuffer(MappedCodeCacheFile& Code) { + MappedCodeBuffers.push_back(Code.CodeBuffer); + // Unregister on destruction of Code + Code.CacheManager = this; +} + +void AbstractCodeCache::UnregisterMappedCodeBuffer(MappedCodeCacheFile& Code) { + std::erase_if(MappedCodeBuffers, [&](const auto& Elem) { return Elem.data() == Code.CodeBuffer.data(); }); +} + +bool AbstractCodeCache::IsAddressInMappedCodeBuffer(uintptr_t Address) const { + for (const auto& Range : MappedCodeBuffers) { + auto Start = reinterpret_cast(Range.data()); + if (Address >= Start && Address < Start + Range.size_bytes()) { + return true; + } + } + return false; +} + fextl::string CodeMap::GetBaseFilename(const ExecutableFileInfo& MainExecutable, bool AddNombSuffix) { auto FileId = MainExecutable.FileId; @@ -339,167 +374,6 @@ bool CodeCache::SaveData(Core::InternalThreadState& Thread, int fd, const Execut return true; } -bool CodeCache::LoadData(Core::InternalThreadState* Thread, std::byte* MappedCacheFile, const ExecutableFileSectionInfo& BinarySection) { - if (!EnableCodeCaching) { - return true; - } - - namespace ranges = std::ranges; - - // Read file header - CodeCacheHeader header {}; - ::memcpy(&header, MappedCacheFile, sizeof(header)); - MappedCacheFile += sizeof(header); - - LogMan::Msg::IFmt("Cache load: {:5} blocks; base={:#14x}; off={:#9x}-{:#09x}; {:016x} {}", header.NumBlocks, BinarySection.FileStartVA, - BinarySection.BeginVA - BinarySection.FileStartVA, BinarySection.EndVA - BinarySection.FileStartVA, - BinarySection.FileInfo.FileId, BinarySection.FileInfo.Filename); - - if (!ranges::equal(header.Magic, header.ExpectedMagic)) { - LogMan::Msg::EFmt("Invalid cache file header"); - return false; - } - - if (!ranges::equal(header.FEXVersion, GIT_HASH)) { - LogMan::Msg::IFmt("Cache generated from old FEX version {:02x}, current is {:02x}; skipping", fmt::join(header.FEXVersion, ""), - fmt::join(GIT_HASH, "")); - return false; - } - - if (header.NumBlocks == 0) { - // Valid caches are never empty - LogMan::Msg::IFmt("Code cache empty, aborting"); - return false; - } - - // Read guest<->host block mappings - using BlockListEntry = decltype(GuestToHostMap::BlockList)::value_type; - fextl::vector BlockList(header.NumBlocks); - { - for (auto& BlockPtr : BlockList) { - ::memcpy(&BlockPtr.first, MappedCacheFile, sizeof(BlockPtr.first)); - MappedCacheFile += sizeof(BlockPtr.first); - ::memcpy(&BlockPtr.second.HostCode, MappedCacheFile, sizeof(BlockPtr.second.HostCode)); - MappedCacheFile += sizeof(BlockPtr.second.HostCode); - uint64_t NumGuestPages; - ::memcpy(&NumGuestPages, MappedCacheFile, sizeof(NumGuestPages)); - MappedCacheFile += sizeof(NumGuestPages); - - BlockPtr.second.CodePages.resize(NumGuestPages); - ::memcpy(BlockPtr.second.CodePages.data(), MappedCacheFile, std::span {BlockPtr.second.CodePages}.size_bytes()); - MappedCacheFile += std::span {BlockPtr.second.CodePages}.size_bytes(); - } - - // Constrain BlockList to the given ExecutableFileSectionInfo - LOGMAN_THROW_A_FMT(ranges::is_sorted(BlockList, [](auto& a, auto& b) { return a.first < b.first; }), "Expected sorted block list"); - auto begin = ranges::lower_bound(BlockList, BinarySection.BeginVA - BinarySection.FileStartVA, std::less {}, &BlockListEntry::first); - auto end = - ranges::upper_bound(begin, BlockList.end(), BinarySection.EndVA - BinarySection.FileStartVA - 1, std::less {}, &BlockListEntry::first); - if (begin == end) { - // Not an error since there is just no data to load - LogMan::Msg::IFmt("No blocks cached in this range, aborting"); - return true; - } - BlockList.erase(end, BlockList.end()); - BlockList.erase(BlockList.begin(), begin); - } - - // Read relocations - fextl::vector Relocations(header.NumRelocations, FEXCore::CPU::Relocation::Default()); - ::memcpy(Relocations.data(), MappedCacheFile, Relocations.size() * sizeof(Relocations[0])); - MappedCacheFile += Relocations.size() * sizeof(Relocations[0]); - - // Pad to next page in file, which contains CodeBuffer data - MappedCacheFile = reinterpret_cast(AlignUp(reinterpret_cast(MappedCacheFile), Utils::FEX_PAGE_SIZE)); - - // Prepare CodeBuffer: Page aligned and big enough to hold all cached data - auto Lock = std::unique_lock {CTX.CodeBufferWriteMutex}; - if (Thread) { - if (auto Prev = Thread->CPUBackend->CheckCodeBufferUpdate()) { - Allocator::VirtualDontNeed(Thread->CallRetStackBase, FEXCore::Core::InternalThreadState::CALLRET_STACK_SIZE); - auto lk = Thread->LookupCache->AcquireWriteLock(); - Thread->LookupCache->ChangeGuestToHostMapping(*Prev, *CTX.GetLatest()->LookupCache, lk); - } - } - - auto CodeBuffer = CTX.GetLatest(); - LOGMAN_THROW_A_FMT(reinterpret_cast(CodeBuffer->Ptr) % 0x1000 == 0, "Expected CodeBuffer base to be page-aligned"); - const auto Delta = AlignUp(CTX.LatestOffset, 0x1000) - CTX.LatestOffset; - CTX.LatestOffset += Delta; - - while (CTX.LatestOffset + header.CodeBufferSize > CodeBuffer->UsableSize()) { - if (Thread) { - CTX.ClearCodeCache(Thread); - CodeBuffer = CTX.GetLatest(); - LogMan::Msg::IFmt("Increased code buffer size to {} MiB for cache load", CodeBuffer->AllocatedSize / 1024 / 1024); - } else { - ERROR_AND_DIE_FMT("Cannot extend codebuffer without thread!"); - } - } - - // Read CodeBuffer data from file. Make sure the destination is page-aligned. - // TODO: Only load the data needed for the selected section - auto CodeBufferRange = - std::as_writable_bytes(std::span {CodeBuffer->Ptr, CodeBuffer->UsableSize()}).subspan(CTX.LatestOffset, header.CodeBufferSize); - ::memcpy(CodeBufferRange.data(), MappedCacheFile, header.CodeBufferSize); - MappedCacheFile += header.CodeBufferSize; - CTX.LatestOffset += header.CodeBufferSize; - - // Apply FEX relocations - auto Ret = ApplyCodeRelocations(BinarySection.FileStartVA, CodeBufferRange, Relocations, false); - LOGMAN_THROW_A_FMT(Ret == true, "Failed to apply code cache relocations"); - - { - auto& LookupCache = *CodeBuffer->LookupCache; - auto WriteLock = LookupCache.AcquireWriteLock(); - - // Register blocks to LookupCache - for (auto& [Guest, Host] : BlockList) { - for (auto& CodePage : Host.CodePages) { - CodePage += BinarySection.FileStartVA; - } - auto HostCode = reinterpret_cast(Host.HostCode + reinterpret_cast(CodeBufferRange.data())); - LookupCache.AddBlockMapping(Guest + BinarySection.FileStartVA, std::move(Host.CodePages), HostCode, WriteLock); - } - - // Register loaded code ranges - fextl::vector Entrypoints; - for (uint32_t i = 0; i < header.NumCodePages; ++i) { - uint64_t CodePage; - memcpy(&CodePage, MappedCacheFile, sizeof(CodePage)); - CodePage += BinarySection.FileStartVA; - MappedCacheFile += sizeof(CodePage); - - uint64_t NumEntrypoints; - memcpy(&NumEntrypoints, MappedCacheFile, sizeof(NumEntrypoints)); - MappedCacheFile += sizeof(NumEntrypoints); - - Entrypoints.resize(NumEntrypoints); - memcpy(Entrypoints.data(), MappedCacheFile, NumEntrypoints * sizeof(Entrypoints[0])); - MappedCacheFile += NumEntrypoints * sizeof(Entrypoints[0]); - for (auto& Entrypoint : Entrypoints) { - Entrypoint += BinarySection.FileStartVA; - } - - if (LookupCache.AddBlockExecutableRange(Entrypoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE, WriteLock)) { - CTX.SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE); - } - } - } - - if (EnableCodeCacheValidation) { - fextl::set GuestBlocks, HostBlocks; - for (auto& [Guest, Host] : BlockList) { - GuestBlocks.insert(Guest + BinarySection.FileStartVA); - HostBlocks.insert(Host.HostCode); - } - - Validate(BinarySection, std::move(GuestBlocks), HostBlocks, CodeBufferRange); - } - - return true; -} - void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::set GuestBlocks, const fextl::set& HostBlocks, std::span CachedCode) { LOGMAN_THROW_A_FMT(!HostBlocks.empty(), "Tried to validate without any host blocks"); @@ -612,7 +486,7 @@ void CodeCache::Validate(const ExecutableFileSectionInfo& Section, fextl::setLookupCache->ClearCache(ValidationThread->LookupCache->AcquireWriteLock()); ValidationCTX->LatestOffset = 0; - LogMan::Msg::IFmt("\tSuccessfully validated cache"); + LogMan::Msg::IFmt(" successfully validated cache"); } bool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span Code, @@ -659,4 +533,288 @@ bool CodeCache::ApplyCodeRelocations(uint64_t GuestEntry, std::span C return true; } +fextl::unique_ptr +CodeCache::LoadCache(std::span CacheFile, const ExecutableFileInfo& FileInfo, uint64_t FileStartVA) { + if (!EnableCodeCaching) { + return nullptr; + } + + FEXCORE_PROFILE_SCOPED("LoadCache"); + + // Read file header + CodeCacheHeader header {}; + ::memcpy(&header, CacheFile.data(), sizeof(header)); + + if (!std::ranges::equal(header.Magic, header.ExpectedMagic)) { + LogMan::Msg::EFmt("Invalid cache file header"); + return nullptr; + } + + if (!std::ranges::equal(header.FEXVersion, GIT_HASH)) { + LogMan::Msg::IFmt("Cache generated from old FEX version {:02x}, current is {:02x}; skipping", fmt::join(header.FEXVersion, ""), + fmt::join(GIT_HASH, "")); + return nullptr; + } + + if (header.NumBlocks == 0) { + // Valid caches are never empty + LogMan::Msg::IFmt("Code cache empty, aborting"); + return nullptr; + } + + // Skip over BlockEntry data since it won't be used until EnableLoadedSection + // TODO: Store direct offset to relocations in the header + auto* BlockListStart = CacheFile.data() + sizeof(header); + auto* Cursor = BlockListStart; + for (uint32_t i = 0; i < header.NumBlocks; ++i) { + Cursor += sizeof(uint64_t); // guest address + Cursor += sizeof(uint64_t); // host code address + uint64_t NumGuestCodePages; + ::memcpy(&NumGuestCodePages, Cursor, sizeof(NumGuestCodePages)); + Cursor += sizeof(NumGuestCodePages); + Cursor += NumGuestCodePages * sizeof(uint64_t); + } + + auto Relocations = std::span {reinterpret_cast(Cursor), header.NumRelocations}; + Cursor += Relocations.size_bytes(); + + // Pad to next page to get the code buffer data + Cursor = reinterpret_cast(AlignUp(reinterpret_cast(Cursor), Utils::FEX_PAGE_SIZE)); + auto CodeDataInFile = std::span {Cursor, header.CodeBufferSize}; + + // Make code data inaccessible until finalized + Allocator::VirtualProtect(CodeDataInFile.data(), header.CodeBufferSize, Allocator::ProtectOptions::None); + + // Group relocations by page + size_t NumPages = header.CodeBufferSize / Utils::FEX_PAGE_SIZE; + fextl::vector PageRelocationRanges(NumPages, {0, 0}); + auto RelocBaseOffset = std::as_bytes(Relocations).data() - CacheFile.data(); + auto RelocIt = Relocations.begin(); + for (size_t Page = 0; Page < NumPages; ++Page) { + auto EndRelocIt = std::upper_bound(RelocIt, Relocations.end(), Page, + [](auto& Page, auto& Reloc) { return Page < Reloc.Header.Offset / Utils::FEX_PAGE_SIZE; }); + PageRelocationRanges.at(Page) = {static_cast(RelocBaseOffset + (RelocIt - Relocations.begin()) * sizeof(CPU::Relocation)), + static_cast(EndRelocIt - RelocIt)}; + RelocIt = EndRelocIt; + } + + auto Storage = FEXCore::Allocator::aligned_alloc(alignof(MappedCodeCacheFile), sizeof(MappedCodeCacheFile)); + return fextl::unique_ptr( + new (Storage) MappedCodeCacheFile {this, CacheFile, CodeDataInFile, BlockListStart, header.NumBlocks, header.NumCodePages, + std::move(PageRelocationRanges), fextl::vector(NumPages), FileStartVA}); +} + +bool CodeCache::EnableLoadedSection(Core::InternalThreadState* Thread, MappedCodeCacheFile& Code, const ExecutableFileSectionInfo& BinarySection) { + if (!EnableCodeCaching) { + return true; + } + + namespace ranges = std::ranges; + + FEXCORE_PROFILE_SCOPED("EnableLoadedSection"); + + // Read block list from cache file + // TODO: Store section-ized BlockLists in cache file + using BlockListEntry = decltype(GuestToHostMap::BlockList)::value_type; + fextl::vector BlockList(Code.NumBlocks); + { + auto* Cursor = Code.BlockListInFile; + for (auto& BlockPtr : BlockList) { + ::memcpy(&BlockPtr.first, Cursor, sizeof(BlockPtr.first)); + Cursor += sizeof(BlockPtr.first); + ::memcpy(&BlockPtr.second.HostCode, Cursor, sizeof(BlockPtr.second.HostCode)); + Cursor += sizeof(BlockPtr.second.HostCode); + uint64_t NumGuestPages; + ::memcpy(&NumGuestPages, Cursor, sizeof(NumGuestPages)); + Cursor += sizeof(NumGuestPages); + + BlockPtr.second.CodePages.resize(NumGuestPages); + ::memcpy(BlockPtr.second.CodePages.data(), Cursor, std::span {BlockPtr.second.CodePages}.size_bytes()); + Cursor += std::span {BlockPtr.second.CodePages}.size_bytes(); + } + + // Constrain BlockList to the given ExecutableFileSectionInfo + LOGMAN_THROW_A_FMT(ranges::is_sorted(BlockList, [](auto& a, auto& b) { return a.first < b.first; }), "Expected sorted block list"); + auto begin = ranges::lower_bound(BlockList, BinarySection.BeginVA - BinarySection.FileStartVA, std::less {}, &BlockListEntry::first); + auto end = + ranges::upper_bound(begin, BlockList.end(), BinarySection.EndVA - BinarySection.FileStartVA - 1, std::less {}, &BlockListEntry::first); + if (begin == end) { + LogMan::Msg::IFmt("No blocks cached in this range, aborting"); + return true; + } + BlockList.erase(end, BlockList.end()); + BlockList.erase(BlockList.begin(), begin); + } + + LogMan::Msg::IFmt("Cache load: {:5} blocks; base={:#14x}; off={:#9x}-{:#09x}; {:016x} {}", BlockList.size(), BinarySection.FileStartVA, + BinarySection.BeginVA - BinarySection.FileStartVA, BinarySection.EndVA - BinarySection.FileStartVA, + BinarySection.FileInfo.FileId, BinarySection.FileInfo.Filename); + + if (EnableLazyCodeCaching) { + LogMan::Msg::IFmt(" lazy mapping: base={:#14x} -> host={}; cache_source={}", BinarySection.FileStartVA, + fmt::ptr(Code.CodeBuffer.data()), fmt::ptr(Code.MappedFile.data())); + } + // Register blocks to LookupCache. + // The host addresses will point into the protected code buffer, so that FEX + // can lazily apply relocations on first execution of each page. + auto CodeBuffer = CTX.GetLatest(); + { + FEXCORE_PROFILE_SCOPED("Decode"); + auto& LookupCache = *CodeBuffer->LookupCache; + auto WriteLock = LookupCache.AcquireWriteLock(); + + for (auto& [Guest, Block] : BlockList) { + for (auto& CodePage : Block.CodePages) { + CodePage += BinarySection.FileStartVA; + } + LOGMAN_THROW_A_FMT(Block.HostCode < Code.CodeBuffer.size_bytes(), "Host offset {:#x} out of range ({:#x})", Block.HostCode, + Code.CodeBuffer.size_bytes()); + auto HostCode = &Code.CodeBuffer[Block.HostCode]; + LookupCache.AddBlockMapping(Guest + BinarySection.FileStartVA, std::move(Block.CodePages), HostCode, WriteLock); + } + + // Guest code pages + auto* Cursor = Code.CodeBuffer.data() + Code.CodeBuffer.size_bytes(); + fextl::vector Entrypoints; + for (uint32_t i = 0; i < Code.NumCodePages; ++i) { + uint64_t CodePage; + memcpy(&CodePage, Cursor, sizeof(CodePage)); + CodePage += BinarySection.FileStartVA; + Cursor += sizeof(CodePage); + + uint64_t NumEntrypoints; + memcpy(&NumEntrypoints, Cursor, sizeof(NumEntrypoints)); + Cursor += sizeof(NumEntrypoints); + + Entrypoints.resize(NumEntrypoints); + memcpy(Entrypoints.data(), Cursor, std::span {Entrypoints}.size_bytes()); + Cursor += std::span {Entrypoints}.size_bytes(); + for (auto& Entrypoint : Entrypoints) { + Entrypoint += BinarySection.FileStartVA; + } + + if (LookupCache.AddBlockExecutableRange(Entrypoints, CodePage, FEXCore::Utils::FEX_PAGE_SIZE, WriteLock)) { + CTX.SyscallHandler->MarkGuestExecutableRange(Thread, CodePage, FEXCore::Utils::FEX_PAGE_SIZE); + } + } + } + + if (!EnableLazyCodeCaching || EnableCodeCacheValidation) { + auto Range = SelectCodeRangeToFinalize(Code, 0, Code.CodeBuffer.size_bytes() / Utils::FEX_PAGE_SIZE); + FinalizeCodePages(Code, Range); + } + + if (EnableCodeCacheValidation) { + fextl::set GuestBlocks, HostBlocks; + for (auto& [Guest, Host] : BlockList) { + GuestBlocks.insert(Guest + BinarySection.FileStartVA); + HostBlocks.insert(Host.HostCode); + } + + Validate(BinarySection, std::move(GuestBlocks), HostBlocks, Code.CodeBuffer); + } + + return true; +} + +} // namespace FEXCore::Context + +namespace FEXCore { + +static std::span SpanPageRelocations(const MappedCodeCacheFile& Code, size_t PageIndex) { + auto [Offset, Count] = Code.PageRelocationRanges.at(PageIndex); + return std::span {reinterpret_cast(Code.MappedFile.data() + Offset), Count}; +} + +std::span AbstractCodeCache::SelectCodeRangeToFinalize(MappedCodeCacheFile& Code, size_t StartPage, size_t EndPage) { + // First, check if we were racing another thread in loading this range + if (std::find(Code.LoadedPages.begin() + StartPage, Code.LoadedPages.begin() + EndPage, false) == Code.LoadedPages.begin() + EndPage) { + return {}; + } + + LOGMAN_THROW_A_FMT(StartPage < EndPage, "Invalid page range [{}, {})", StartPage, EndPage); + LOGMAN_THROW_A_FMT(EndPage <= Code.NumPages(), "End page {} out of range ({})", EndPage, Code.NumPages()); + + // Include any pages that have relocations or block link records crossing + // into the current page range. This ensures we don't attempt to finalize + // any page twice, partially apply FEX relocations, or trigger page loads + // during block linking. + while (EndPage < Code.NumPages()) { + auto PageRelocs = SpanPageRelocations(Code, EndPage - 1); + if (!PageRelocs.empty()) { + auto It = std::prev(PageRelocs.end()); + size_t RelocEnd = It->Header.Offset + 16 /* Upper bound for relocation size */; + if (RelocEnd > EndPage * Utils::FEX_PAGE_SIZE) { + ++EndPage; + continue; + } + } + + // Check for trailing block link + { + auto PageRelocs = SpanPageRelocations(Code, EndPage); + if (!PageRelocs.empty() && PageRelocs.begin()->Header.Offset < EndPage * Utils::FEX_PAGE_SIZE + 0x18) { + ++EndPage; + continue; + } + } + break; + }; + while (StartPage != 0) { + auto PageRelocs = SpanPageRelocations(Code, StartPage - 1); + if (!PageRelocs.empty()) { + auto It = std::prev(PageRelocs.end()); + size_t RelocEnd = It->Header.Offset + 16 /* Upper bound for relocation size */; + if (RelocEnd > StartPage * Utils::FEX_PAGE_SIZE) { + --StartPage; + continue; + } + } + + // Check for trailing block link + { + auto PageRelocs = SpanPageRelocations(Code, StartPage); + if (!PageRelocs.empty() && PageRelocs.begin()->Header.Offset < StartPage * Utils::FEX_PAGE_SIZE + 0x18) { + --StartPage; + continue; + } + } + break; + }; + + return Code.CodeBuffer.subspan(StartPage * Utils::FEX_PAGE_SIZE, (EndPage - StartPage) * Utils::FEX_PAGE_SIZE); +} +} // namespace FEXCore + +namespace FEXCore::Context { + +void CodeCache::FinalizeCodePages(MappedCodeCacheFile& Code, std::span CodeRange) { + const size_t StartOffset = CodeRange.data() - Code.CodeBuffer.data(); + const auto StartPage = StartOffset / Utils::FEX_PAGE_SIZE; + const auto EndPage = StartPage + CodeRange.size_bytes() / Utils::FEX_PAGE_SIZE; + + // None of the selected pages should be loaded at all; otherwise, SelectCodeRangeToFinalize returned inconsistent ranges + LOGMAN_THROW_A_FMT(std::find(Code.LoadedPages.begin() + StartPage, Code.LoadedPages.begin() + EndPage, true) == Code.LoadedPages.begin() + EndPage, + "Inconsistent page load state"); + + FEXCORE_PROFILE_SCOPED("FinalizeCodePages"); + + // Map writeable to allow applying relocations + // TODO: Are there any remaining race conditions here? + Allocator::VirtualProtect(CodeRange.data(), CodeRange.size_bytes(), Allocator::ProtectOptions::Write); + + // Apply relocations + for (size_t i = StartPage; i < EndPage; ++i) { + auto PageRelocations = SpanPageRelocations(Code, i); + (void)ApplyCodeRelocations(Code.GuestBase, Code.CodeBuffer, PageRelocations, false); + Code.LoadedPages[i] = true; + } + + ARMEmitter::Emitter::ClearICache(CodeRange.data(), CodeRange.size_bytes()); + if (!Allocator::VirtualProtect(CodeRange.data(), CodeRange.size_bytes(), + Allocator::ProtectOptions::Read | Allocator::ProtectOptions::Write | Allocator::ProtectOptions::Exec)) { + ERROR_AND_DIE_FMT("VirtualProtect failed"); + } +} + } // namespace FEXCore::Context diff --git a/FEXCore/Source/Interface/Core/Core.cpp b/FEXCore/Source/Interface/Core/Core.cpp index 718482700..813a363fb 100644 --- a/FEXCore/Source/Interface/Core/Core.cpp +++ b/FEXCore/Source/Interface/Core/Core.cpp @@ -456,7 +456,6 @@ void ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread if (Config.StrictInProcessSplitLocks) { FEXCore::Utils::SpinWaitLock::unlock(&StrictSplitLockMutex); } - return; } } diff --git a/FEXCore/Source/Interface/Core/LookupCache.cpp b/FEXCore/Source/Interface/Core/LookupCache.cpp index aaa0c2424..0b230306f 100644 --- a/FEXCore/Source/Interface/Core/LookupCache.cpp +++ b/FEXCore/Source/Interface/Core/LookupCache.cpp @@ -87,8 +87,11 @@ void LookupCache::ClearL2Cache(const FEXCore::LookupCacheBaseLockToken& lk) { } void LookupCache::ClearThreadLocalCaches(const LookupCacheWriteLockToken&) { + // TODO: Preserve code cache entries? // Clear L1 and L2 by clearing the full cache. FEXCore::Allocator::VirtualDontNeed(reinterpret_cast(PagePointer), TotalCacheSize, false); + + // TODO: Rename this member to avoid confusion with code caching CachedCodePages.clear(); } diff --git a/FEXCore/include/FEXCore/Core/CodeCache.h b/FEXCore/include/FEXCore/Core/CodeCache.h index 21d408721..eaa20a4ac 100644 --- a/FEXCore/include/FEXCore/Core/CodeCache.h +++ b/FEXCore/include/FEXCore/Core/CodeCache.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -178,7 +179,54 @@ private: CodeMapOpener& FileOpener; }; +class AbstractCodeCache; + +/** + * Manages runtime state associated with a mapped code cache file. + * + * The mapped file pointer is managed by the frontend and must be valid + * throughout the lifetime of this object. + */ +struct MappedCodeCacheFile { + // Calls UnregisterMappedCodeBuffer internally, see its docstring about synchronization requirements + ~MappedCodeCacheFile(); + + // If not nullptr, the MappedCodeCacheFile will be unregistered from this on destruction + AbstractCodeCache* CacheManager; + + std::span MappedFile; // Mapped data of the whole cache file + std::span CodeBuffer; // Subspan of cached ARM64 data within MappedFile + std::byte* BlockListInFile; // Pointer to BlockListEntry data within MappedFile + uint32_t NumBlocks; // Number of BlockListEntry objects + uint32_t NumCodePages; // Number of code page entrypoint mappings + + struct PageRelocationRange { + uint32_t Offset; // In bytes from start of file + uint32_t Length; // Number of relocations + }; + + // List of relocation ranges in the mapped cache file, grouped by the code page they apply to. + // This vector is indexed by the relative page offset from the start of the ARM64 code data. + // + // For example PageRelocationRanges[1] == { 0x100, 0x20 } means: + // - there are 0x20 bytes of relocation data at offset 0x100 in the cache file + // - these 0x20 bytes of relocation data will patch data at CodeBuffer[0x1000..0x2000] + fextl::vector PageRelocationRanges; + fextl::vector LoadedPages; + + uint64_t GuestBase {}; // Guest base address for relocation application + + // Helper member to prevent moving/copying without disallowing aggregate-construction + std::atomic disallow_copy_or_move; + + size_t NumPages() const { + return CodeBuffer.size_bytes() / FEXCore::Utils::FEX_PAGE_SIZE; + } +}; + class AbstractCodeCache { + fextl::vector> MappedCodeBuffers; + public: virtual ~AbstractCodeCache() = default; @@ -190,13 +238,6 @@ public: */ virtual uint64_t ComputeCodeMapId(std::string_view Filename, int FD) = 0; - /** - * Loads a code cache from mapped memory and appends it to the current Core state. - * TODO: Optionally recompiles all contained code blocks at runtime for validation. - * Returns false if the provided cache file is invalid, and true otherwise. - */ - virtual bool LoadData(Core::InternalThreadState*, std::byte* MappedCacheFile, const ExecutableFileSectionInfo&) = 0; - /** * Bundles the current Core state (CodeBuffer, GuestToHostMapping, ...) to a code cache and writes it to the given file descriptor. * Returns true on success. @@ -207,6 +248,42 @@ public: * Function to be called before compiling any code for caching purposes */ virtual void InitiateCacheGeneration() = 0; + + /** + * Loads a code cache from mapped memory. + * + * Code sections must be enabled in a second step (see EnableLoadedSection). + * Afterwards, individual code pages must be finalized using FinalizeCodePages. + * + * On success, this returns a MappedCodeCacheFile that must be kept alive + * as long the cache is in use. + */ + virtual fextl::unique_ptr LoadCache(std::span CacheFile, const ExecutableFileInfo&, uint64_t FileStartVA) = 0; + + /** + * Registers cached blocks for the given file section to the LookupCache. + * + * Also runs extended cache validation if enabled. + */ + virtual bool EnableLoadedSection(Core::InternalThreadState*, MappedCodeCacheFile&, const ExecutableFileSectionInfo&) = 0; + + /** + * Extend the given code range so that it can be safely finalized. + * + * This is required for example to avoid dangling page-crossing FEX relocations on the edges + * + * StartPage and EndPage a 0-based relative page offsets into the cached code. + */ + static std::span SelectCodeRangeToFinalize(MappedCodeCacheFile&, size_t StartPage, size_t EndPage); + + /** + * Finalize code pages in the given range (see SelectCodePagesToFinalize) for execution. + */ + virtual void FinalizeCodePages(MappedCodeCacheFile&, std::span CodeRange) = 0; + + void RegisterMappedCodeBuffer(MappedCodeCacheFile&); + void UnregisterMappedCodeBuffer(MappedCodeCacheFile&); + bool IsAddressInMappedCodeBuffer(uintptr_t Address) const; }; } // namespace FEXCore diff --git a/FEXCore/include/FEXCore/Utils/SignalScopeGuards.h b/FEXCore/include/FEXCore/Utils/SignalScopeGuards.h index d4ceeab89..6589cf2d3 100644 --- a/FEXCore/include/FEXCore/Utils/SignalScopeGuards.h +++ b/FEXCore/include/FEXCore/Utils/SignalScopeGuards.h @@ -35,6 +35,9 @@ public: const auto Result = pthread_mutex_lock(&Mutex); LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result); } + bool try_lock() { + return pthread_mutex_trylock(&Mutex) == 0; + } void unlock() { const auto Result = pthread_mutex_unlock(&Mutex); LOGMAN_THROW_A_FMT(Result == 0, "{} failed to unlock with {}", __func__, Result); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp index bf6a5a0fe..b726033e9 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.cpp @@ -899,9 +899,19 @@ uint64_t UnimplementedSyscallSafe(FEXCore::Core::CpuStateFrame* Frame, uint64_t } void SyscallHandler::LockBeforeFork(FEXCore::Core::InternalThreadState* Thread) { - TM.LockBeforeFork(); - Thread->CTX->LockBeforeFork(Thread); - VMATracking.Mutex.lock(); + while (true) { + TM.LockBeforeFork(); + Thread->CTX->LockBeforeFork(Thread); + if (std::try_lock(CodeCachePatchingMutex, VMATracking.Mutex) == -1) { + break; + } + + // Lock failed: Another thread has temporarily acquired these mutexes. + // Release them to a void a deadlock and retry later + CTX->UnlockAfterFork(Thread, false); + TM.UnlockAfterFork(Thread, false); + std::this_thread::sleep_for(std::chrono::milliseconds {10}); + }; } void SyscallHandler::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThread, bool Child) { @@ -910,8 +920,10 @@ void SyscallHandler::UnlockAfterFork(FEXCore::Core::InternalThreadState* LiveThr FM.SetProtectedCodeMapFD(-1); VMATracking.Mutex.StealAndDropActiveLocks(); + CodeCachePatchingMutex.StealAndDropActiveLocks(); } else { VMATracking.Mutex.unlock(); + CodeCachePatchingMutex.unlock(); } CTX->UnlockAfterFork(LiveThread, Child); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h index 9810099ef..6b7a30602 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h @@ -265,7 +265,7 @@ public: if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) { TM.InvalidateGuestCodeRange(Thread, Base, Length); } - if (CheckPendingVMAResources) { + if (CheckPendingVMAResources && Thread) { auto lk = FEXCore::GuardSignalDeferringSection(VMATracking.Mutex, Thread); VMATracking.FlushPendingResourceDeletions(); } @@ -363,6 +363,9 @@ private: std::mutex FutexMutex; std::mutex SyscallMutex; + // std::mutex CodeCachePatchingMutex; + FEXCore::ForkableUniqueMutex CodeCachePatchingMutex; + FEX::CodeLoader* LocalLoader {}; bool NeedToCheckXID {true}; diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp index 228928188..69fe6a163 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsSMCTracking.cpp @@ -30,7 +30,21 @@ $end_info$ #include namespace FEX::HLE { -// SMC interactions +static void HandleSegfaultForCodeCacheFinalization(FEXCore::Core::InternalThreadState& Thread, FEXCore::MappedCodeCacheFile& Code, + uintptr_t FaultAddress) { + FEXCORE_PROFILE_SCOPED("Load code cache page"); + size_t PageIdx = (reinterpret_cast(FaultAddress) - Code.CodeBuffer.data()) / FEXCore::Utils::FEX_PAGE_SIZE; + + auto RangeToFinalize = Thread.CTX->GetCodeCache().SelectCodeRangeToFinalize(Code, PageIdx, PageIdx + 1); + if (!RangeToFinalize.empty()) { + Thread.CTX->GetCodeCache().FinalizeCodePages(Code, RangeToFinalize); + } +} + +// Handles segfaults from: +// - call-ret shadow stack overflow +// - guest-side self-modifying code (SMC) +// - lazy loading of mapped code cache pages bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState* Thread, int Signal, void* info, void* ucontext) { const auto FaultAddress = (uintptr_t)((siginfo_t*)info)->si_addr; @@ -46,13 +60,25 @@ bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState* Thread, // Can't use the deferred signal lock in the SIGSEGV handler. auto lk = FEXCore::MaskSignalsAndLockMutex(_SyscallHandler->VMATracking.Mutex); - auto VMATracking = &_SyscallHandler->VMATracking; + auto& VMATracking = _SyscallHandler->VMATracking; // If the write spans two pages, they will be flushed one at a time (generating two faults) - auto Entry = VMATracking->FindVMAEntry(FaultAddress); + auto Entry = VMATracking.FindVMAEntry(FaultAddress); - // If an untracked address, or the mapping wasn't writable, it can't be handled here - if (Entry == VMATracking->VMAs.end() || !Entry->second.Prot.Writable) { + if (Entry == VMATracking.VMAs.end()) { + // Not a guest page; check mapped code cache pages + auto* Code = VMATracking.FindMappedCodeCacheByHostAddress(FaultAddress); + if (!Code) { + // Untracked address; not handled here + return false; + } + std::lock_guard lk(_SyscallHandler->CodeCachePatchingMutex); + HandleSegfaultForCodeCacheFinalization(*Thread, *Code, FaultAddress); + return true; + } + + // If the mapping wasn't writable, it can't be handled here + if (!Entry->second.Prot.Writable) { return false; } @@ -233,30 +259,44 @@ static ReadELFHeadersResult ReadELFHeaders(int FD, std::span HeaderDa return ReadELFHeadersResult {std::move(Parser.phdrs), std::move(Relocations), HasCodeRelocations}; } -static void LoadCodeCache(FEXCore::Core::InternalThreadState& Thread, FEXCore::ExecutableFileSectionInfo& Section, uint64_t CodeCacheConfigId) { +static fextl::unique_ptr +LoadCodeCache(FEXCore::Core::InternalThreadState& Thread, VMATracking::VMATracking& VMATracking, + const FEXCore::ExecutableFileInfo& FileInfo, uint64_t CodeCacheConfigId, uint64_t FileStartVA) { + auto& CodeCache = Thread.CTX->GetCodeCache(); + auto CacheFilename = fextl::fmt::format("{}cache/{}-{:016x}", FEX::Config::GetCacheDirectory(), - FEXCore::CodeMap::GetBaseFilename(Section.FileInfo, false), CodeCacheConfigId); + FEXCore::CodeMap::GetBaseFilename(FileInfo, false), CodeCacheConfigId); int CacheFD = open(CacheFilename.c_str(), O_RDONLY); if (CacheFD == -1) { LogMan::Msg::IFmt("Cache file does not exist: {}", CacheFilename); - return; + return nullptr; } struct stat buf; if (fstat(CacheFD, &buf) != 0) { LogMan::Msg::EFmt("Invalid cache file: {}", CacheFilename); close(CacheFD); - return; + return nullptr; } - auto CacheFileSize = buf.st_size; - auto MappedCache = (std::byte*)FEXCore::Allocator::mmap(nullptr, CacheFileSize, PROT_READ, MAP_PRIVATE, CacheFD, 0); - LOGMAN_THROW_A_FMT(MappedCache, "Failed to map code cache into memory"); - if (!Thread.CTX->GetCodeCache().LoadData(&Thread, MappedCache, Section)) { - // TODO: Delete this cache file - } - FEXCore::Allocator::munmap(MappedCache, CacheFileSize); + auto CacheFileSize = static_cast(buf.st_size); + auto MappedCache = (std::byte*)FEXCore::Allocator::mmap(nullptr, CacheFileSize, PROT_READ | PROT_WRITE, MAP_PRIVATE, CacheFD, 0); close(CacheFD); + if (!MappedCache || MappedCache == MAP_FAILED) { + LogMan::Msg::EFmt("Failed to map code cache into memory"); + return nullptr; + } + + auto Result = CodeCache.LoadCache(std::span {MappedCache, CacheFileSize}, FileInfo, FileStartVA); + if (!Result) { + FEXCore::Allocator::munmap(MappedCache, CacheFileSize); + return nullptr; + } + + // NOTE: This is synchronized by acquiring VMATracking.Mutex at call site + CodeCache.RegisterMappedCodeBuffer(*Result); + + return Result; } void* SyscallHandler::GuestMmap(bool Is64Bit, FEXCore::Core::InternalThreadState* Thread, void* addr, size_t length, int prot, int flags, @@ -302,7 +342,8 @@ void* SyscallHandler::GuestMmap(bool Is64Bit, FEXCore::Core::InternalThreadState } if (EnableCodeCaching && CachedSection) { - LoadCodeCache(*Thread, *CachedSection, CodeCacheConfigId); + Thread->CTX->GetCodeCache().EnableLoadedSection( + Thread, *static_cast(CachedSection->FileInfo).MappedCache, *CachedSection); } return reinterpret_cast(Result); @@ -386,7 +427,7 @@ void SyscallHandler::TriggerGuestLibWrapperCodeCacheLoad(FEXCore::Core::Internal } auto SectionInfo = BuildSectionInfo(*VMAEntry->second.Resource, VMA->Base, VMA->Length); - LoadCodeCache(Thread, SectionInfo, CodeCacheConfigId); + LoadCodeCache(Thread, VMATracking, SectionInfo.FileInfo, CodeCacheConfigId, SectionInfo.FileStartVA); } } @@ -461,9 +502,12 @@ uint64_t SyscallHandler::GuestMprotect(FEXCore::Core::InternalThreadState* Threa } // Trigger delayed cache load. This must be done separately since - // LoadCodeCache will call interfaces that acquire the VMATracking mutex. + // EnableLoadedSection will call interfaces that acquire the VMATracking mutex. for (auto& CachedSection : CachedSections) { - LoadCodeCache(*Thread, CachedSection, CodeCacheConfigId); + auto Cache = static_cast(CachedSection.FileInfo).MappedCache.get(); + if (Cache) { + Thread->CTX->GetCodeCache().EnableLoadedSection(Thread, *Cache, CachedSection); + } } return Result; @@ -565,7 +609,7 @@ SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState* Thread, uint64_t a if (PathLength != -1 && S_ISREG(buf.st_mode) && (buf.st_mode & S_IXUSR)) { // ELF files that are mapped multiple times get a separate MappedResource for each base virtual address if ((prot & PROT_READ) && Inserted) { - Resource->MappedFile = fextl::make_unique(); + Resource->MappedFile = fextl::make_unique(); Resource->MappedFile->Filename = fextl::string(Tmp, PathLength); Resource->MappedFile->FileId = CTX->GetCodeCache().ComputeCodeMapId(Resource->MappedFile->Filename, fd); @@ -651,7 +695,14 @@ SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState* Thread, uint64_t a // Load code cache if present. // FEXServer was requested to generate library caches on program launch. if (EnableCodeCaching && Resource && Resource->MappedFile && VMATracking::VMAProt::fromProt(prot).Executable) { - if (Resource->MappedFile->Filename.ends_with("-guest.so")) { + if (!Resource->MappedFile->AttemptedCacheLoad) { + Resource->MappedFile->MappedCache = LoadCodeCache(*Thread, VMATracking, *Resource->MappedFile, CodeCacheConfigId, Resource->FirstVMA->Base); + Resource->MappedFile->AttemptedCacheLoad = true; + } + + if (!Resource->MappedFile->MappedCache) { + // No cache present + } else if (Resource->MappedFile->Filename.ends_with("-guest.so")) { // For guest library wrappers, cache loading must be delayed until LoadLib is called. // Before that, we can't patch up the SHA256 function identifiers. LogMan::Msg::IFmt("Delaying code cache load for {}", Resource->MappedFile->Filename); diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.cpp b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.cpp index ad5146cde..62490bfdc 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.cpp +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.cpp @@ -7,10 +7,20 @@ desc: VMA Tracking $end_info$ */ +#include "FEXCore/Utils/MathUtils.h" #include "LinuxSyscalls/Syscalls.h" #include namespace FEX::HLE::VMATracking { + +ExecutableFileState::~ExecutableFileState() { + if (MappedCache && MappedCache->MappedFile.data()) { + auto ret = FEXCore::Allocator::munmap(MappedCache->MappedFile.data(), + FEXCore::AlignUp(MappedCache->MappedFile.size_bytes(), FEXCore::Utils::FEX_PAGE_SIZE)); + LOGMAN_THROW_A_FMT(ret == 0, "Error unmapping cache for {}: {} {}", Filename, errno, strerror(errno)); + } +} + /// Helpers /// auto VMAProt::fromProt(int Prot) -> VMAProt { return VMAProt { @@ -238,7 +248,7 @@ void VMATracking::DeleteVMARange(FEXCore::Context::Context* CTX, uintptr_t Base, auto Iter = Current->Resource->Iterator; // Defer deletion if the resource has mapped code cache data, so its code buffer // outlives code cache invalidation (which runs after the VMA lock is released). - if (false) { // TODO: Consider unconditionally deferring deletion? + if (Current->Resource->MappedFile && Current->Resource->MappedFile->MappedCache) { PendingResourceDeletions.push_back(std::move(*Current->Resource)); } MappedResources.erase(Iter); @@ -554,7 +564,7 @@ uintptr_t VMATracking::DeleteSHMRegion(FEXCore::Context::Context* CTX, uintptr_t if (Entry->second.Resource == Resource) { if (ListRemove(&Entry->second)) { auto Iter = Entry->second.Resource->Iterator; - if (false) { // TODO: Consider unconditionally deferring deletion? + if (Entry->second.Resource->MappedFile && Entry->second.Resource->MappedFile->MappedCache) { PendingResourceDeletions.push_back(std::move(*Entry->second.Resource)); } MappedResources.erase(Iter); @@ -567,4 +577,18 @@ uintptr_t VMATracking::DeleteSHMRegion(FEXCore::Context::Context* CTX, uintptr_t return ShmLength; } + +FEXCore::MappedCodeCacheFile* VMATracking::FindMappedCodeCacheByHostAddress(uintptr_t HostAddr) const { + for (auto& [_, Resource] : MappedResources) { + if (Resource.MappedFile && Resource.MappedFile->MappedCache) { + auto* Code = Resource.MappedFile->MappedCache.get(); + auto BufferStart = reinterpret_cast(Code->CodeBuffer.data()); + if (HostAddr >= BufferStart && HostAddr < BufferStart + Code->CodeBuffer.size_bytes()) { + return Code; + } + } + } + return nullptr; +} + } // namespace FEX::HLE::VMATracking diff --git a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.h b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.h index e6f7f826b..8998fdb84 100644 --- a/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.h +++ b/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.h @@ -4,12 +4,18 @@ #include #include +#include "FEXCore/Core/CodeCache.h" #include #include #include #include +namespace FEXCore { +struct ExecutableFileInfo; +struct MappedCodeCacheFile; +} // namespace FEXCore + namespace FEX::HLE::VMATracking { ///// VMA (Virtual Memory Area) tracking ///// @@ -32,6 +38,13 @@ struct MRID { struct VMAEntry; +struct ExecutableFileState : FEXCore::ExecutableFileInfo { + ~ExecutableFileState(); + + bool AttemptedCacheLoad = false; + fextl::unique_ptr MappedCache; +}; + /** * Meta data associated to one system resource. * @@ -43,7 +56,7 @@ struct VMAEntry; struct MappedResource { using ContainerType = fextl::multimap; - fextl::unique_ptr MappedFile; + fextl::unique_ptr MappedFile; // Pointer to lowest memory range this file is mapped to VMAEntry* FirstVMA; uint64_t Length; // 0 if not fixed size @@ -136,6 +149,10 @@ struct VMATracking { return MappedResources.equal_range(mrid); } + // Find any MappedCodeCacheFile that contains the given host code address. + // - Mutex must be shared_locked before calling + FEXCore::MappedCodeCacheFile* FindMappedCodeCacheByHostAddress(uintptr_t HostAddr) const; + bool HasPendingResourceDeletions() const { return !PendingResourceDeletions.empty(); } diff --git a/Source/Windows/Common/ImageTracker.cpp b/Source/Windows/Common/ImageTracker.cpp index 01d5fa90c..972d561a0 100644 --- a/Source/Windows/Common/ImageTracker.cpp +++ b/Source/Windows/Common/ImageTracker.cpp @@ -172,7 +172,7 @@ FEXCore::ExecutableFileSectionInfo ImageTracker::HandleImageMap(std::string_view auto AOTImage = AOTImages.find(ID); if (AOTImage != AOTImages.end()) { - CTX.GetCodeCache().LoadData(nullptr, AOTImage->second.Data, ImageInfo->SectionInfo); + // TODO: CodeCache::EnableLoadedSection } } @@ -277,6 +277,7 @@ void ImageTracker::LoadAOTImages(MappedImageInfo& ImageInfo) { RtlUnicodeToMultiByteN(UniqueId.data(), AnsiLength, NULL, Info->FileName, Info->FileNameLength); AOTImages[UniqueId] = {.Data = static_cast(LoadAddress)}; + // TODO: CodeCache::LoadCache, CodeCache::RegisterMappedCodeBuffer LogMan::Msg::IFmt("Loaded cache: {}", UniqueId); } }