SMC: Add mprotect based tracking, --smc=mtrack, make default

External/FEXCore/Source/Interface/Config/Config.json.in

@@ -234,14 +234,15 @@
     "Hacks": {
       "SMCChecks": {
         "Type": "uint8",
-        "Default": "FEXCore::Config::CONFIG_SMC_MMAN",
-        "TextDefault": "mman",
+        "Default": "FEXCore::Config::CONFIG_SMC_MTRACK",
+        "TextDefault": "mtrack",
         "ArgumentHandler": "SMCCheckHandler",
         "Desc": [
           "Checks code for modification before execution.",
           "\tnone: No checks",
-          "\tmman: Invalidate on mmap, mprotect, munmap",
-          "\tfull: Validate code before every run (slow)"
+          "\tmtrack: Page tracking based invalidation",
+          "\tfull: Validate code before every run (slow)",
+          "\tmman: Invalidate on mmap, mprotect, munmap (deprecated, use mtrack)"
         ]
       },
       "TSOEnabled": {

External/FEXCore/Source/Interface/Context/Context.cpp

@@ -149,7 +149,7 @@ namespace FEXCore::Context {
   void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
     CTX->CleanupAfterFork(Thread);
   }
-
+  
   void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation) {
     CTX->SignalDelegation = SignalDelegation;
   }
@@ -186,11 +186,11 @@ namespace FEXCore::Context {
     CTX->WriteFilesWithCode(Writer);
   }
 
-  void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name) {
-    return CTX->AddNamedRegion(Base, Length, Offset, Name);
+  IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string &Name) {
+    return CTX->LoadAOTIRCacheEntry(Name);
   }
-  void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length) {
-    return CTX->RemoveNamedRegion(Base, Length);
+  void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, IR::AOTIRCacheEntry *Entry) {
+    return CTX->UnloadAOTIRCacheEntry(Entry);
   }
 
 namespace Debug {

External/FEXCore/Source/Interface/Context/Context.h

@@ -266,8 +266,8 @@ namespace FEXCore::Context {
 
     uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; }
 
-    void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
-    void RemoveNamedRegion(uintptr_t Base, uintptr_t Size);
+    IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename);
+    void UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry);
 
     FEXCore::JITSymbols Symbols;
 

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -352,6 +352,9 @@ namespace FEXCore::Context {
       // Walk the threads and tell them to clear their caches
       // Useful when our block size is set to a large number and we need to step a single instruction
       for (auto &Thread : Threads) {
+        // Wait for thread to be fully constructed
+        // XXX: Look into thread partial construction issues
+        while(Thread->RunningEvents.WaitingToStart.load()) ;
         ClearCodeCache(Thread, true);
       }
     }
@@ -632,7 +635,10 @@ namespace FEXCore::Context {
   }
 
   void Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length) {
-    Thread->LookupCache->AddBlockMapping(Address, Ptr, Start, Length);
+    // Only call MarkGuestExecutableRange if new pages are marked as containing code
+    if (Thread->LookupCache->AddBlockMapping(Address, Ptr, Start, Length)) {
+      Thread->CTX->SyscallHandler->MarkGuestExecutableRange(Start, Length);
+    }
   }
 
   void Context::ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache) {
@@ -1177,26 +1183,19 @@ namespace FEXCore::Context {
     return Result;
   }
 
-  void Context::AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) {
-    IRCaptureCache.AddNamedRegion(Base, Size, Offset, filename);
+  IR::AOTIRCacheEntry *Context::LoadAOTIRCacheEntry(const std::string &filename) {
+    auto rv = IRCaptureCache.LoadAOTIRCacheEntry(filename);
     if (DebugServer) {
       DebugServer->AlertLibrariesChanged();
     }
-
-    if (CodeObjectCacheService) {
-      CodeObjectCacheService->AsyncAddNamedRegionJob(Base, Size, Offset, filename);
-    }
+    return rv;
   }
 
-  void Context::RemoveNamedRegion(uintptr_t Base, uintptr_t Size) {
-    IRCaptureCache.RemoveNamedRegion(Base, Size);
+  void Context::UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry) {
+    IRCaptureCache.UnloadAOTIRCacheEntry(Entry);
     if (DebugServer) {
       DebugServer->AlertLibrariesChanged();
     }
-
-    if (CodeObjectCacheService) {
-      CodeObjectCacheService->AsyncRemoveNamedRegionJob(Base, Size);
-    }
   }
 
   void ConfigureAOTGen(FEXCore::Core::InternalThreadState *Thread, std::set<uint64_t> *ExternalBranches, uint64_t SectionMaxAddress) {

External/FEXCore/Source/Interface/Core/LookupCache.h

@@ -72,7 +72,8 @@ public:
 
   std::map<uint64_t, std::vector<uint64_t>> CodePages;
 
-  void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
+  // Returns true if new pages are marked as containing code
+  bool AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
     std::lock_guard<std::recursive_mutex> lk(WriteLock);
     
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
@@ -81,7 +82,10 @@ public:
     BlockList.emplace(Address, (uintptr_t)HostCode);
     LOGMAN_THROW_A_FMT(InsertPoint.second == true, "Dupplicate block mapping added");
 
+    bool rv = false;
+
     for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) {
+      rv |= CodePages[CurrentPage].size() == 0;
       CodePages[CurrentPage].push_back(Address);
     }
 
@@ -90,6 +94,8 @@ public:
     auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
     L1Entry.GuestCode = Address;
     L1Entry.HostCode = (uintptr_t)HostCode;
+
+    return rv;
   }
 
   void Erase(uint64_t Address) {

External/FEXCore/Source/Interface/IR/AOTIR.cpp

@@ -1,10 +1,11 @@
 #include "Interface/Context/Context.h"
 #include "Interface/IR/AOTIR.h"
-#include "Interface/Core/LookupCache.h"
 
 #include <FEXCore/IR/IntrusiveIRList.h>
 #include <FEXCore/IR/RegisterAllocationData.h>
 #include <FEXCore/Utils/Allocator.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <Interface/Core/LookupCache.h>
 
 #include <cstddef>
 #include <cstdint>
@@ -78,7 +79,7 @@ namespace FEXCore::IR {
       return true;
   }
 
-  bool LoadAOTIRCache(AOTCacheType *AOTIRCache, int streamfd) {
+  static bool LoadAOTIRCache(AOTIRCacheEntry *Entry, int streamfd) {
     uint64_t tag;
 
     if (!readAll(streamfd, (char*)&tag, sizeof(tag)) || tag != FEXCore::IR::AOTIR_COOKIE)
@@ -100,6 +101,10 @@ namespace FEXCore::IR {
     if (!readAll(streamfd,  (char*)&Module[0], Module.size()))
       return false;
 
+    if (Entry->FileId != Module) {
+      return false;
+    }
+
     lseek(streamfd, -sizeof(ModSize) - ModSize - sizeof(IndexSize), SEEK_END);
 
     if (!readAll(streamfd,  (char*)&IndexSize, sizeof(IndexSize)))
@@ -120,19 +125,16 @@ namespace FEXCore::IR {
 
     auto Array = (AOTIRInlineIndex *)((char*)FilePtr + IndexOffset);
 
-    AOTIRCache->insert({Module, {Array, FilePtr, Size}});
+    LOGMAN_THROW_A_FMT(Entry->Array == nullptr && Entry->FilePtr == nullptr, "Entry must not be initialized here");
+    Entry->Array = Array;
+    Entry->FilePtr = FilePtr;
+    Entry->Size = Size;
 
     LogMan::Msg::DFmt("AOTIR: Module {} has {} functions", Module, Array->Count);
 
     return true;
   }
 
-  AOTIRCaptureCache::~AOTIRCaptureCache() {
-    for (auto &Mod: AOTIRCache) {
-      FEXCore::Allocator::munmap(Mod.second.mapping, Mod.second.size);
-    }
-  }
-
   void AOTIRCaptureCache::FinalizeAOTIRCache() {
     AOTIRCaptureCacheWriteoutQueue_Flush();
 
@@ -231,39 +233,27 @@ namespace FEXCore::IR {
 
   void AOTIRCaptureCache::WriteFilesWithCode(std::function<void(const std::string& fileid, const std::string& filename)> Writer) {
     std::shared_lock lk(AOTIRCacheLock);
-    for( const auto &File: FilesWithCode) {
-      Writer(File.first, File.second);
+    for( const auto &Entry: AOTIRCache) {
+      if (Entry.second.ContainsCode) {
+        Writer(Entry.second.FileId, Entry.second.Filename);
+      }
     }
   }
 
   AOTIRCaptureCache::PreGenerateIRFetchResult AOTIRCaptureCache::PreGenerateIRFetch(uint64_t GuestRIP, FEXCore::IR::IRListView *IRList) {
-    {
-      std::shared_lock lk(AOTIRCacheLock);
-      auto file = AddrToFile.lower_bound(GuestRIP);
-      if (file != AddrToFile.begin()) {
-        --file;
-        if (!file->second.ContainsCode) {
-          file->second.ContainsCode = true;
-          FilesWithCode[file->second.fileid] = file->second.filename;
-        }
-      }
-    }
-
+    auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(GuestRIP);
+    
     PreGenerateIRFetchResult Result{};
-    if (IRList == nullptr && CTX->Config.AOTIRLoad()) {
-      std::shared_lock lk(AOTIRCacheLock);
-      auto file = AddrToFile.lower_bound(GuestRIP);
-      if (file != AddrToFile.begin()) {
-        --file;
-        auto Mod = (FEXCore::IR::AOTIRInlineIndex*)file->second.CachedFileEntry;
+    
+    if (AOTIRCacheEntry.Entry) {
+      AOTIRCacheEntry.Entry->ContainsCode = true;
 
-        if (Mod == nullptr) {
-          file->second.CachedFileEntry = Mod = AOTIRCache[file->second.fileid].Array;
-        }
+      if (IRList == nullptr && CTX->Config.AOTIRLoad()) {
+        auto Mod = AOTIRCacheEntry.Entry->Array;
 
         if (Mod != nullptr)
         {
-          auto AOTEntry = Mod->Find(GuestRIP - file->second.Start + file->second.Offset);
+          auto AOTEntry = Mod->Find(GuestRIP - AOTIRCacheEntry.Offset);
 
           if (AOTEntry) {
             // verify hash
@@ -303,14 +293,12 @@ namespace FEXCore::IR {
     bool GeneratedIR) {
     // Both generated ir and LibraryJITName need a named region lookup
     if (GeneratedIR || CTX->Config.LibraryJITNaming()) {
-      std::shared_lock lk(AOTIRCacheLock);
 
-      auto file = FindAddrForFile(StartAddr, Length);
+      auto AOTIRCacheEntry = CTX->SyscallHandler->LookupAOTIRCacheEntry(GuestRIP);
 
-      // Only go down this path if we actually found a library region
-      if (file != AddrToFile.end()) {
+      if (AOTIRCacheEntry.Entry) {
         if (DebugData && CTX->Config.LibraryJITNaming()) {
-          CTX->Symbols.RegisterNamedRegion(CodePtr, DebugData->HostCodeSize, file->second.filename);
+          CTX->Symbols.RegisterNamedRegion(CodePtr, DebugData->HostCodeSize, AOTIRCacheEntry.Entry->Filename);
         }
 
         // Add to AOT cache if aot generation is enabled
@@ -319,16 +307,20 @@ namespace FEXCore::IR {
 
           auto hash = XXH3_64bits((void*)StartAddr, Length);
 
-          auto LocalRIP = GuestRIP - file->second.Start + file->second.Offset;
-          auto LocalStartAddr = StartAddr - file->second.Start + file->second.Offset;
-          auto fileid = file->second.fileid;
+          auto LocalRIP = GuestRIP - AOTIRCacheEntry.Offset;
+          auto LocalStartAddr = StartAddr - AOTIRCacheEntry.Offset;
+          auto FileId = AOTIRCacheEntry.Entry->FileId;
           auto RADataCopy = RAData->CreateCopy();
           auto IRListCopy = IRList->CreateCopy();
-          AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRListCopy, RADataCopy, fileid]() {
-            auto *AotFile = &AOTIRCaptureCacheMap[fileid];
+          AOTIRCaptureCacheWriteoutQueue_Append([this, LocalRIP, LocalStartAddr, Length, hash, IRListCopy, RADataCopy, FileId]() {
+
+            // It is guaranteed via AOTIRCaptureCacheWriteoutLock and AOTIRCaptureCacheWriteoutFlusing that this will not run concurrently
+            // Memory coherency is guaranteed via AOTIRCaptureCacheWriteoutLock
+            
+            auto *AotFile = &AOTIRCaptureCacheMap[FileId];
 
             if (!AotFile->Stream) {
-              AotFile->Stream = AOTIRWriter(fileid);
+              AotFile->Stream = AOTIRWriter(FileId);
               uint64_t tag = FEXCore::IR::AOTIR_COOKIE;
               AotFile->Stream->write((char*)&tag, sizeof(tag));
             }
@@ -366,21 +358,7 @@ namespace FEXCore::IR {
     return false;
   }
 
-  AOTIRCaptureCache::AddrToFileMapType::iterator AOTIRCaptureCache::FindAddrForFile(uint64_t Entry, uint64_t Length) {
-    // Thread safety here! We are returning an iterator to the map object
-    // This needs the AOTIRCacheLock locked prior to coming in to the function
-    auto file = AddrToFile.lower_bound(Entry);
-    if (file != AddrToFile.begin()) {
-      --file;
-      if (file->second.Start <= Entry && (file->second.Start + file->second.Len) >= (Entry + Length)) {
-        return file;
-      }
-    }
-    return AddrToFile.end();
-  }
-
-  void AOTIRCaptureCache::AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) {
-    // TODO: Support overlapping maps and region splitting
+  AOTIRCacheEntry *AOTIRCaptureCache::LoadAOTIRCacheEntry(const std::string &filename) {
     auto base_filename = std::filesystem::path(filename).filename().string();
 
     if (!base_filename.empty()) {
@@ -396,21 +374,32 @@ namespace FEXCore::IR {
 
       std::unique_lock lk(AOTIRCacheLock);
 
-      AddrToFile.insert({ Base, { Base, Size, Offset, fileid, filename, nullptr, false} });
+      auto Inserted = AOTIRCache.insert({fileid, AOTIRCacheEntry{0, 0, 0, fileid, filename, false}});
+      auto Entry = &(Inserted.first->second);
 
-      if (CTX->Config.AOTIRLoad && !AOTIRCache.contains(fileid) && AOTIRLoader) {
+      LOGMAN_THROW_A_FMT(Entry->Array == nullptr, "Duplicate LoadAOTIRCacheEntry");
+
+      if (CTX->Config.AOTIRLoad && AOTIRLoader) {
         auto streamfd = AOTIRLoader(fileid);
         if (streamfd != -1) {
-          FEXCore::IR::LoadAOTIRCache(&AOTIRCache, streamfd);
+          FEXCore::IR::LoadAOTIRCache(Entry, streamfd);
           close(streamfd);
         }
       }
+      return Entry;
     }
+
+    return nullptr;
   }
 
-  void AOTIRCaptureCache::RemoveNamedRegion(uintptr_t Base, uintptr_t Size) {
-    std::unique_lock lk(AOTIRCacheLock);
-    // TODO: Support partial removing
-    AddrToFile.erase(Base);
+  void AOTIRCaptureCache::UnloadAOTIRCacheEntry(AOTIRCacheEntry *Entry) {
+    LOGMAN_THROW_A_FMT(Entry != nullptr, "Removing not existing entry");
+
+    if (Entry->Array) {
+      FEXCore::Allocator::munmap(Entry->FilePtr, Entry->Size);
+      Entry->Array = nullptr;
+      Entry->FilePtr = nullptr;
+      Entry->Size = 0;
+    }
   }
 }

External/FEXCore/Source/Interface/IR/AOTIR.h

@@ -72,18 +72,19 @@ namespace FEXCore::IR {
 
   struct AOTIRCacheEntry {
     AOTIRInlineIndex *Array;
-    void *mapping;
-    size_t size;
+    void *FilePtr;
+    size_t Size;
+    std::string FileId;
+    std::string Filename;
+    bool ContainsCode;
   };
 
   using AOTCacheType = std::unordered_map<std::string, FEXCore::IR::AOTIRCacheEntry>;
-  bool LoadAOTIRCache(AOTCacheType *AOTIRCache, int streamfd);
 
   class AOTIRCaptureCache final {
     public:
 
       AOTIRCaptureCache(FEXCore::Context::Context *ctx) : CTX {ctx} {}
-      ~AOTIRCaptureCache();
 
       void FinalizeAOTIRCache();
       void AOTIRCaptureCacheWriteoutQueue_Flush();
@@ -110,8 +111,8 @@ namespace FEXCore::IR {
         FEXCore::Core::DebugData *DebugData,
         bool GeneratedIR);
 
-      void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
-      void RemoveNamedRegion(uintptr_t Base, uintptr_t Size);
+      AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename);
+      void UnloadAOTIRCacheEntry(AOTIRCacheEntry *Entry);
 
       // Callbacks
       void SetAOTIRLoader(std::function<int(const std::string&)> CacheReader) {
@@ -135,27 +136,11 @@ namespace FEXCore::IR {
 
       std::queue<std::function<void()>> AOTIRCaptureCacheWriteoutQueue;
 
-      std::map<std::string, std::string> FilesWithCode;
-
-      struct AddrToFileEntry {
-        uint64_t Start;
-        uint64_t Len;
-        uint64_t Offset;
-        std::string fileid;
-        std::string filename;
-        void *CachedFileEntry;
-        bool ContainsCode;
-      };
-
-      using AddrToFileMapType = std::map<uint64_t, AddrToFileEntry>;
-      AddrToFileMapType AddrToFile;
       FEXCore::IR::AOTCacheType AOTIRCache;
 
       std::function<int(const std::string&)> AOTIRLoader;
       std::function<std::unique_ptr<std::ofstream>(const std::string&)> AOTIRWriter;
       std::function<void(const std::string&)> AOTIRRenamer;
       std::unordered_map<std::string, FEXCore::IR::AOTIRCaptureCacheEntry> AOTIRCaptureCacheMap;
-
-      AddrToFileMapType::iterator FindAddrForFile(uint64_t Entry, uint64_t Length);
   };
 }

External/FEXCore/include/FEXCore/Config/Config.h

@@ -27,10 +27,12 @@ namespace Handler {
   static inline std::string_view SMCCheckHandler(std::string_view Value) {
     if (Value == "none")
       return "0";
-    else if (Value == "mman")
+    else if (Value == "mtrack")
       return "1";
     else if (Value == "full")
       return "2";
+    else if (Value == "mman")
+      return "3";
     return "0";
   }
   static inline std::string_view CacheObjectCodeHandler(std::string_view Value) {
@@ -57,8 +59,9 @@ namespace Handler {
 
   enum ConfigSMCChecks {
     CONFIG_SMC_NONE,
-    CONFIG_SMC_MMAN,
+    CONFIG_SMC_MTRACK,
     CONFIG_SMC_FULL,
+    CONFIG_SMC_MMAN,
   };
 
   enum ConfigObjectCodeHandler {

External/FEXCore/include/FEXCore/Core/Context.h

@@ -31,6 +31,10 @@ namespace FEXCore::HLE {
   class SyscallHandler;
 }
 
+namespace FEXCore::IR {
+  struct AOTIRCacheEntry;
+}
+
 namespace FEXCore::Context {
   struct Context;
   enum ExitReason {
@@ -236,8 +240,9 @@ namespace FEXCore::Context {
   FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf);
   FEX_DEFAULT_VISIBILITY FEXCore::CPUID::FunctionResults RunCPUIDFunctionName(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf, uint32_t CPU);
 
-  FEX_DEFAULT_VISIBILITY void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name);
-  FEX_DEFAULT_VISIBILITY void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length);
+  FEX_DEFAULT_VISIBILITY FEXCore::IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string& Name);
+  FEX_DEFAULT_VISIBILITY void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, FEXCore::IR::AOTIRCacheEntry *Entry);
+
   FEX_DEFAULT_VISIBILITY void SetAOTIRLoader(FEXCore::Context::Context *CTX, std::function<int(const std::string&)> CacheReader);
   FEX_DEFAULT_VISIBILITY void SetAOTIRWriter(FEXCore::Context::Context *CTX, std::function<std::unique_ptr<std::ofstream>(const std::string&)> CacheWriter);
   FEX_DEFAULT_VISIBILITY void SetAOTIRRenamer(FEXCore::Context::Context *CTX, std::function<void(const std::string&)> CacheRenamer);

External/FEXCore/include/FEXCore/HLE/SyscallHandler.h

@@ -1,13 +1,19 @@
 #pragma once
 #include <cstdint>
 #include <string>
+#include <shared_mutex>
 
 #include <FEXCore/IR/IR.h>
+#include <FEXHeaderUtils/ScopedSignalMask.h>
 
 namespace FEXCore {
   class CodeLoader;
 }
 
+namespace FEXCore::IR {
+  struct AOTIRCacheEntry;
+}
+
 namespace FEXCore::Context {
   struct Context;
 }
@@ -43,6 +49,24 @@ namespace FEXCore::HLE {
     OS_HANGOVER,
   };
 
+  class SyscallHandler;
+  struct AOTIRCacheEntryLookupResult {
+    AOTIRCacheEntryLookupResult(FEXCore::IR::AOTIRCacheEntry *Entry, uintptr_t Offset, FHU::ScopedSignalMaskWithSharedLock &&lk)
+      : Entry(Entry), Offset(Offset), lk(std::move(lk))
+    {
+
+    }
+
+    AOTIRCacheEntryLookupResult(AOTIRCacheEntryLookupResult&&) = default;
+
+    FEXCore::IR::AOTIRCacheEntry *Entry;
+    uintptr_t Offset;
+
+    friend class SyscallHandler;
+    protected:
+    FHU::ScopedSignalMaskWithSharedLock lk;
+  };
+
   class SyscallHandler {
   public:
     virtual ~SyscallHandler() = default;
@@ -53,6 +77,8 @@ namespace FEXCore::HLE {
 
     SyscallOSABI GetOSABI() const { return OSABI; }
     virtual FEXCore::CodeLoader *GetCodeLoader() const { return nullptr; }
+    virtual void MarkGuestExecutableRange(uint64_t Start, uint64_t Length) { }
+    virtual AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) = 0;
 
   protected:
     SyscallOSABI OSABI;

Source/Tests/IRLoader.cpp

@@ -15,6 +15,7 @@ $end_info$
 #include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Utils/Allocator.h>
 #include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/HLE/SyscallHandler.h>
 
 #include <functional>
 #include <memory>
@@ -119,6 +120,25 @@ private:
   constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024;
 };
 
+class DummySyscallHandler: public FEXCore::HLE::SyscallHandler {
+  public:
+
+  uint64_t HandleSyscall(FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args) override {
+    LOGMAN_MSG_A_FMT("Syscalls not implemented");
+    return 0;
+  }
+
+  FEXCore::HLE::SyscallABI GetSyscallABI(uint64_t Syscall) override {
+    LOGMAN_MSG_A_FMT("Syscalls not implemented");
+    return {0, false, 0 };
+  }
+
+  std::shared_mutex Mutex;
+  FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) override {
+    return {0, 0, FHU::ScopedSignalMaskWithSharedLock {Mutex}};
+  }
+};
+
 int main(int argc, char **argv, char **const envp)
 {
   LogMan::Throw::InstallHandler(AssertHandler);
@@ -141,6 +161,7 @@ int main(int argc, char **argv, char **const envp)
   std::unique_ptr<FEX::HLE::SignalDelegator> SignalDelegation = std::make_unique<FEX::HLE::SignalDelegator>();
 
   FEXCore::Context::SetSignalDelegator(CTX, SignalDelegation.get());
+  FEXCore::Context::SetSyscallHandler(CTX, new DummySyscallHandler());
 
   FEX::IRLoader::Loader Loader(Args[0], Args[1]);
 

Source/Tests/LinuxSyscalls/CMakeLists.txt

@@ -6,6 +6,8 @@ add_library(LinuxEmulation STATIC
     LinuxAllocator.cpp
     SignalDelegator.cpp
     Syscalls.cpp
+    SyscallsSMCTracking.cpp
+    SyscallsVMATracking.cpp
     x32/Syscalls.cpp
     x32/EPoll.cpp
     x32/FD.cpp

Source/Tests/LinuxSyscalls/Syscalls.cpp

@@ -27,6 +27,9 @@ $end_info$
 #include <FEXCore/Utils/MathUtils.h>
 #include <FEXCore/Utils/Threads.h>
 #include <FEXHeaderUtils/Syscalls.h>
+#include <FEXHeaderUtils/ScopedSignalMask.h>
+#include <FEXHeaderUtils/TypeDefines.h>
+#include <Tests/LinuxSyscalls/SignalDelegator.h>
 
 #include <algorithm>
 #include <alloca.h>
@@ -586,6 +589,10 @@ SyscallHandler::SyscallHandler(FEXCore::Context::Context *_CTX, FEX::HLE::Signal
   HostKernelVersion = CalculateHostKernelVersion();
   GuestKernelVersion = CalculateGuestKernelVersion();
   Alloc32Handler = FEX::HLE::Create32BitAllocator();
+
+  if (SMCChecks == FEXCore::Config::CONFIG_SMC_MTRACK) {
+    SignalDelegation->RegisterHostSignalHandler(SIGSEGV, HandleSegfault, true);
+  }
 }
 
 SyscallHandler::~SyscallHandler() {
@@ -669,54 +676,22 @@ uint64_t UnimplementedSyscallSafe(FEXCore::Core::CpuStateFrame *Frame, uint64_t
   return -ENOSYS;
 }
 
+void SyscallHandler::LockBeforeFork() {
+  FM.GetFDLock()->lock();
 
-//// VMA Tracking ////
-static std::string get_fdpath(int fd)
-{
-      std::error_code ec;
-      return std::filesystem::canonical(std::filesystem::path("/proc/self/fd") / std::to_string(fd), ec).string();
+  // XXX shared_mutex has issues with locking and forks
+  // VMATracking.Mutex.lock();
+
+  // Add other mutexes here
 }
 
-void SyscallHandler::TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset) {
-  if (!(Flags & MAP_ANONYMOUS)) {
-    auto filename = get_fdpath(fd);
+void SyscallHandler::UnlockAfterFork() {
+  // Add other mutexes here
 
-    FEXCore::Context::AddNamedRegion(CTX, Base, Size, Offset, filename);
-  }
-  if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) {
-    FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size);
-  }
-}
-
-void SyscallHandler::TrackMunmap(uintptr_t Base, uintptr_t Size) {  
-  if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) {
-    FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size);
-  }
-}
-
-void SyscallHandler::TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot) {
-  if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN && Prot & PROT_EXEC) {
-    FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size);
-  }
-}
-
-void SyscallHandler::TrackMremap(uintptr_t OldAddress, size_t OldSize, size_t NewSize, int flags, uintptr_t NewAddress) {
-  if (SMCChecks == FEXCore::Config::CONFIG_SMC_MMAN) {
-    FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress, OldSize);
-    FEXCore::Context::InvalidateGuestCodeRange(CTX, NewAddress, NewSize);
-  }
-}
-
-void SyscallHandler::TrackShmat(int shmid, uintptr_t Base, int shmflg) {
-  // TODO
-}
-
-void SyscallHandler::TrackShmdt(uintptr_t Base) {
-  // TODO
-}
-
-void SyscallHandler::TrackMadvise(uintptr_t Base, uintptr_t Size, int advice) {
-  // TODO
+  // XXX shared_mutex has issues with locking and forks
+  // VMATracking.Mutex.unlock();
+  
+  FM.GetFDLock()->unlock(); 
 }
 
 }

Source/Tests/LinuxSyscalls/Syscalls.h

@@ -16,11 +16,14 @@ $end_info$
 #include <FEXCore/Utils/CompilerDefs.h>
 
 #include <mutex>
+#include <shared_mutex>
 
 #include <errno.h>
 #include <stdint.h>
 #include <type_traits>
 #include <vector>
+#include <list>
+#include <map>
 #ifdef _M_X86_64
 #define SYSCALL_ARCH_NAME x64
 #elif _M_ARM_64
@@ -170,6 +173,7 @@ public:
   // does a guest munmap as if done via a guest syscall
   virtual int GuestMunmap(void *addr, uint64_t length) = 0;
 
+  ///// Memory Manager tracking /////
   void TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset);
   void TrackMunmap(uintptr_t Base, uintptr_t Size);
   void TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot);
@@ -178,6 +182,14 @@ public:
   void TrackShmdt(uintptr_t Base);
   void TrackMadvise(uintptr_t Base, uintptr_t Size, int advice);
   
+  ///// VMA (Virtual Memory Area) tracking /////
+  static bool HandleSegfault(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext);
+  virtual void MarkGuestExecutableRange(uint64_t Start, uint64_t Length) override;
+
+  ///// FORK tracking /////
+  void LockBeforeFork();
+  void UnlockAfterFork();
+  
 protected:
   SyscallHandler(FEXCore::Context::Context *_CTX, FEX::HLE::SignalDelegator *_SignalDelegation);
 
@@ -209,6 +221,109 @@ private:
   #endif
 
   std::unique_ptr<FEX::HLE::MemAllocator> Alloc32Handler{};
+  
+  ///// VMA (Virtual Memory Area) tracking /////
+
+  // AOTIRCacheEntryLookupResult also includes a shared lock guard, so the pointed AOTIRCacheEntry return can be safely used
+  FEXCore::HLE::AOTIRCacheEntryLookupResult LookupAOTIRCacheEntry(uint64_t GuestAddr) final override;
+
+
+  struct SpecialDev {
+    static constexpr uint64_t Anon = 0x1'0000'0000; // Anonymous shared mapping, id is incrementing allocation number
+    static constexpr uint64_t SHM = 0x2'0000'0000;  // sys-v shm, id is shmid
+  };
+
+  // Memory Resource ID
+  // An id that can be used to identify when shared mappings actually have the same backing storage
+  // when dev != SpecialDev::Anon, this is unique system wide
+  struct MRID {
+    uint64_t dev; // kernel dev_t is actually 32-bits, we use the extra bits to track SpecialDevs
+    uint64_t id;
+
+    bool operator<(const MRID& other) const {
+      return std::tie(dev, id) < std::tie(other.dev, other.id);
+    }
+  };
+
+  struct VMAEntry;
+
+  // Used to all MAP_SHARED VMAs of a system resource.
+  struct MappedResource {
+    using ContainerType = std::map<MRID, MappedResource>;
+
+    FEXCore::IR::AOTIRCacheEntry *AOTIRCacheEntry;
+    VMAEntry *FirstVMA;
+    uint64_t Length; // 0 if not fixed size
+    ContainerType::iterator Iterator;
+  };
+
+  union VMAProt {
+    struct {
+      bool Readable: 1;
+      bool Writable: 1;
+      bool Executable: 1;
+    };
+    uint8_t All: 3;
+    
+
+    static VMAProt fromProt(int Prot);
+    static VMAProt fromSHM(int SHMFlg);
+  };
+
+  struct VMAFlags {
+    bool Shared: 1;
+
+    static VMAFlags fromFlags(int Flags);
+  };
+
+  struct VMAEntry {
+    MappedResource *Resource;
+
+    // these are for Intrusive linked list tracking, starting from Resource->FirstVMA
+    VMAEntry *ResourcePrevVMA;
+    VMAEntry *ResourceNextVMA;
+
+    uint64_t Base;
+    uint64_t Offset;
+    uint64_t Length;
+
+    VMAFlags Flags;
+    VMAProt Prot;
+  };
+
+  struct VMATracking {
+    using VMAEntry = SyscallHandler::VMAEntry;
+    std::shared_mutex Mutex;
+    
+    // Memory ranges indexed by page aligned starting address
+    std::map<uint64_t, VMAEntry> VMAs;
+
+    using VMACIterator = decltype(VMAs)::const_iterator;
+
+    MappedResource::ContainerType MappedResources;
+
+    // Mutex must be at least shared_locked before calling
+    VMACIterator LookupVMAUnsafe(uint64_t GuestAddr) const;
+
+    // Mutex must be unique_locked before calling
+    void SetUnsafe(FEXCore::Context::Context *Ctx, MappedResource *MappedResource, uintptr_t Base, uintptr_t Offset, uintptr_t Length, VMAFlags Flags, VMAProt Prot);
+    
+    // Mutex must be unique_locked before calling
+    void ClearUnsafe(FEXCore::Context::Context *Ctx, uintptr_t Base, uintptr_t Length, MappedResource *PreservedMappedResource = nullptr);
+
+    // Mutex must be unique_locked before calling
+    void ChangeUnsafe(uintptr_t Base, uintptr_t Length, VMAProt Prot);
+
+    // Mutex must be unique_locked before calling
+    // Returns the Size fo the Shm or 0 if not found
+    uintptr_t ClearShmUnsafe(FEXCore::Context::Context *Ctx, uintptr_t Base);
+  private:
+    bool ListRemove(VMAEntry *Mapping);
+    void ListReplace(VMAEntry *Mapping, VMAEntry *NewMapping);
+    void ListInsertAfter(VMAEntry *Mapping, VMAEntry *NewMapping);
+    void ListPrepend(MappedResource *Resource, VMAEntry *NewVMA);
+    static void ListCheckVMALinks(VMAEntry *VMA);
+  } VMATracking;
 };
 
 uint64_t HandleSyscall(SyscallHandler *Handler, FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args);

Source/Tests/LinuxSyscalls/Syscalls/Thread.cpp

@@ -171,11 +171,9 @@ namespace FEX::HLE {
   }
 
   uint64_t ForkGuest(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::CpuStateFrame *Frame, uint32_t flags, void *stack, pid_t *parent_tid, pid_t *child_tid, void *tls) {
-    // Just before we fork, pull the Filemanagement mutex so it is in a safe state
-    // Locking the mutex will mean that both processes will end up with a locked mutex
-    auto Mutex = FEX::HLE::_SyscallHandler->FM.GetFDLock();
-    Mutex->lock();
-
+    // Just before we fork, we lock all syscall mutexes so that both processes will end up with a locked mutex
+    FEX::HLE::_SyscallHandler->LockBeforeFork();
+    
     pid_t Result{};
     if (flags & CLONE_VFORK) {
       // XXX: We don't currently support a vfork as it causes problems.
@@ -186,8 +184,8 @@ namespace FEX::HLE {
       Result = fork();
     }
 
-    // Unlock the mutex on both sides of the fork
-    Mutex->unlock();
+    // Unlock the mutexes on both sides of the fork
+    FEX::HLE::_SyscallHandler->UnlockAfterFork();
 
     if (Result == 0) {
       // Child

Source/Tests/LinuxSyscalls/SyscallsSMCTracking.cpp

@@ -0,0 +1,354 @@
+/*
+$info$
+category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough
+tags: LinuxSyscalls|common
+desc: SMC/MMan Tracking
+$end_info$
+*/
+
+#include <filesystem>
+#include <sys/shm.h>
+#include <sys/mman.h>
+
+#include "Tests/LinuxSyscalls/Syscalls.h"
+
+#include <FEXHeaderUtils/TypeDefines.h>
+#include <FEXCore/Debug/InternalThreadState.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/MathUtils.h>
+
+namespace FEX::HLE {
+
+/// Helpers ///
+auto SyscallHandler::VMAProt::fromProt (int Prot) -> VMAProt {
+  return VMAProt {
+    .Readable = (Prot & PROT_READ) != 0,
+    .Writable = (Prot & PROT_WRITE) != 0,
+    .Executable = (Prot & PROT_EXEC) != 0,
+  };
+}
+
+auto SyscallHandler::VMAProt::fromSHM (int SHMFlg) -> VMAProt {
+  return VMAProt {
+    .Readable = true,
+    .Writable = SHMFlg & SHM_RDONLY ? false : true,
+    .Executable = false,
+  };
+}
+
+auto SyscallHandler::VMAFlags::fromFlags(int Flags) -> VMAFlags {
+  return VMAFlags {
+    .Shared = (Flags & MAP_SHARED) != 0, // also includes MAP_SHARED_VALIDATE
+  };
+}
+
+// SMC interactions
+bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) {
+  auto CTX = Thread->CTX;
+
+  const auto FaultAddress = (uintptr_t)((siginfo_t *)info)->si_addr;
+
+  {
+    FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex);
+    
+    auto VMATracking = &_SyscallHandler->VMATracking;
+
+    // If the write spans two pages, they will be flushed one at a time (generating two faults)
+    auto Entry = VMATracking->LookupVMAUnsafe(FaultAddress);
+
+    // If an untracked address, or the mapping wasn't writable, it can't be handled here
+    if (Entry == VMATracking->VMAs.end() || !Entry->second.Prot.Writable) {
+      return false;
+    }
+
+    auto FaultBase = FEXCore::AlignDown(FaultAddress, FHU::FEX_PAGE_SIZE);
+
+    if (Entry->second.Flags.Shared) {
+      LOGMAN_THROW_A_FMT(Entry->second.Resource, "VMA tracking error");
+
+      auto Offset = FaultBase - Entry->first + Entry->second.Offset;
+
+      auto VMA = Entry->second.Resource->FirstVMA;
+      LOGMAN_THROW_A_FMT(VMA, "VMA tracking error");
+
+      // Flush all mirrors, remap the page writable as needed
+      do {
+        if (VMA->Offset <= Offset && (VMA->Offset + VMA->Length) > Offset) {
+          auto FaultBaseMirrored = Offset - VMA->Offset + VMA->Base;
+          if (VMA->Prot.Writable) {
+            auto rv = mprotect((void *)FaultBaseMirrored, FHU::FEX_PAGE_SIZE, PROT_READ | PROT_WRITE);
+            LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", FaultBaseMirrored, FHU::FEX_PAGE_SIZE);
+          }
+          FEXCore::Context::InvalidateGuestCodeRange(CTX, FaultBaseMirrored, FHU::FEX_PAGE_SIZE);
+        }
+      } while ((VMA = VMA->ResourceNextVMA));
+    } else {
+      // Mark as read write before flush, so that if code is compiled after the Flush but before returning, the segfault will be
+      // re-raised
+      auto rv = mprotect((void *)FaultBase, FHU::FEX_PAGE_SIZE, PROT_READ | PROT_WRITE);
+      LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", FaultBase, FHU::FEX_PAGE_SIZE);
+      FEXCore::Context::InvalidateGuestCodeRange(CTX, FaultBase, FHU::FEX_PAGE_SIZE);
+    }
+
+    return true;
+  }
+}
+
+void SyscallHandler::MarkGuestExecutableRange(uint64_t Start, uint64_t Length) {
+  const auto Base = Start & FHU::FEX_PAGE_MASK;
+  const auto Top = FEXCore::AlignUp(Start + Length, FHU::FEX_PAGE_SIZE);
+
+  {
+    if (SMCChecks != FEXCore::Config::CONFIG_SMC_MTRACK) {
+      return;
+    }
+
+    FHU::ScopedSignalMaskWithSharedLock lk(VMATracking.Mutex);
+
+    // Find the first mapping at or after the range ends, or ::end().
+    // Top points to the address after the end of the range
+    auto Mapping = VMATracking.VMAs.lower_bound(Top);
+
+    while (Mapping != VMATracking.VMAs.begin()) {
+      Mapping--;
+
+      const auto MapBase = Mapping->first;
+      const auto MapTop = MapBase + Mapping->second.Length;
+
+      if (MapTop <= Base) {
+        // Mapping ends before the Range start, exit
+        break;
+      } else {
+        const auto ProtectBase = std::max(MapBase, Base);
+        const auto ProtectSize = std::min(MapTop, Top) - ProtectBase;
+
+        if (Mapping->second.Flags.Shared) {
+          LOGMAN_THROW_A_FMT(Mapping->second.Resource, "VMA tracking error");
+
+          const auto OffsetBase = ProtectBase - Mapping->first + Mapping->second.Offset;
+          const auto OffsetTop = OffsetBase + ProtectSize;
+
+          auto VMA = Mapping->second.Resource->FirstVMA;
+          LOGMAN_THROW_A_FMT(VMA, "VMA tracking error");
+
+          do {
+            auto VMAOffsetBase = VMA->Offset;
+            auto VMAOffsetTop = VMA->Offset + VMA->Length;
+            auto VMABase = VMA->Base;
+
+            if (VMA->Prot.Writable && VMAOffsetBase < OffsetTop && VMAOffsetTop > OffsetBase) {
+
+              const auto MirroredBase = std::max(VMAOffsetBase, OffsetBase);
+              const auto MirroredSize = std::min(OffsetTop, VMAOffsetTop) - MirroredBase;
+
+              auto rv = mprotect((void *)(MirroredBase - VMAOffsetBase + VMABase), MirroredSize, PROT_READ);
+              LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", MirroredBase, MirroredSize);
+            }
+          } while ((VMA = VMA->ResourceNextVMA));
+
+        } else if (Mapping->second.Prot.Writable) {
+          int rv = mprotect((void *)ProtectBase, ProtectSize, PROT_READ);
+
+          LogMan::Throw::AFmt(rv == 0, "mprotect({}, {}) failed", ProtectBase, ProtectSize);
+        }
+      }
+    }
+  }
+}
+
+// Used for AOT
+FEXCore::HLE::AOTIRCacheEntryLookupResult SyscallHandler::LookupAOTIRCacheEntry(uint64_t GuestAddr) {
+  FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex);
+  auto rv = FEXCore::HLE::AOTIRCacheEntryLookupResult(nullptr, 0, std::move(lk));
+
+  // Get the first mapping after GuestAddr, or end
+  // GuestAddr is inclusive
+  // If the write spans two pages, they will be flushed one at a time (generating two faults)
+  auto Entry = _SyscallHandler->VMATracking.LookupVMAUnsafe(GuestAddr);
+
+  if (Entry != _SyscallHandler->VMATracking.VMAs.end()) {
+    rv.Entry = Entry->second.Resource ? Entry->second.Resource->AOTIRCacheEntry : nullptr;
+    rv.Offset = Entry->second.Base - Entry->second.Offset;
+  }
+
+  return rv;
+}
+
+// MMan Tracking
+static std::string get_fdpath(int fd) {
+  std::error_code ec;
+  return std::filesystem::canonical(std::filesystem::path("/proc/self/fd") / std::to_string(fd), ec).string();
+}
+
+void SyscallHandler::TrackMmap(uintptr_t Base, uintptr_t Size, int Prot, int Flags, int fd, off_t Offset) {
+	Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
+
+  {
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+    static uint64_t AnonSharedId = 1;
+
+    MappedResource *Resource = nullptr;
+
+    if (!(Flags & MAP_ANONYMOUS)) {
+      struct stat64 buf;
+      fstat64(fd, &buf);
+      MRID mrid {buf.st_dev, buf.st_ino};
+
+      auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource {nullptr, nullptr, 0});
+      Resource = &Iter->second;
+
+      if (Inserted) {
+        auto filename = get_fdpath(fd);
+        Resource->AOTIRCacheEntry = FEXCore::Context::LoadAOTIRCacheEntry(CTX, filename);
+        Resource->Iterator = Iter;
+      }
+
+    } else if (Flags & MAP_SHARED) {
+      MRID mrid{SpecialDev::Anon, AnonSharedId++};
+
+      auto [Iter, Inserted] = VMATracking.MappedResources.emplace(mrid, MappedResource{nullptr, nullptr, 0});
+      LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error");
+      Resource = &Iter->second;
+      Resource->Iterator = Iter;
+    } else {
+      Resource = nullptr;
+    }
+
+    VMATracking.SetUnsafe(CTX, Resource, Base, Offset, Size, VMAFlags::fromFlags(Flags), VMAProt::fromProt(Prot));
+  }
+
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size);
+  }
+}
+
+void SyscallHandler::TrackMunmap(uintptr_t Base, uintptr_t Size) {
+	Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
+
+  {
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+    VMATracking.ClearUnsafe(CTX, Base, Size);
+  }
+
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    FEXCore::Context::InvalidateGuestCodeRange(CTX, (uintptr_t)Base, Size);
+  }
+}
+
+void SyscallHandler::TrackMprotect(uintptr_t Base, uintptr_t Size, int Prot) {
+	Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
+
+  {
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+    VMATracking.ChangeUnsafe(Base, Size, VMAProt::fromProt(Prot));
+  }
+
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Size);
+  }
+}
+
+void SyscallHandler::TrackMremap(uintptr_t OldAddress, size_t OldSize, size_t NewSize, int flags, uintptr_t NewAddress) {
+  OldSize = FEXCore::AlignUp(OldSize, FHU::FEX_PAGE_SIZE);
+  NewSize = FEXCore::AlignUp(NewSize, FHU::FEX_PAGE_SIZE);
+
+  {
+
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+    const auto OldVMA = VMATracking.LookupVMAUnsafe(OldAddress);
+
+    const auto OldResource = OldVMA->second.Resource;
+    const auto OldOffset = OldVMA->second.Offset + OldAddress - OldVMA->first;
+    const auto OldFlags = OldVMA->second.Flags;
+    const auto OldProt = OldVMA->second.Prot;
+
+    LOGMAN_THROW_A_FMT(OldVMA != VMATracking.VMAs.end(), "VMA Tracking corruption");
+
+    if (OldSize == 0) {
+      // Mirror existing mapping
+      // must be a shared mapping
+      LOGMAN_THROW_A_FMT(OldResource != nullptr, "VMA Tracking error");
+      LOGMAN_THROW_A_FMT(OldFlags.Shared, "VMA Tracking error");
+      VMATracking.SetUnsafe(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt);
+    } else {
+
+// MREMAP_DONTUNMAP is kernel 5.7+
+#ifdef MREMAP_DONTUNMAP
+      if (!(flags & MREMAP_DONTUNMAP))
+#endif
+      {
+        VMATracking.ClearUnsafe(CTX, OldAddress, OldSize, OldResource);
+      }
+
+      // Make anonymous mapping
+      VMATracking.SetUnsafe(CTX, OldResource, NewAddress, OldOffset, NewSize, OldFlags, OldProt);
+    }
+  }
+
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    if (OldAddress != NewAddress) {
+      if (OldSize != 0) {
+        // This also handles the MREMAP_DONTUNMAP case
+        FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress, OldSize);
+      }
+    } else {
+      // If mapping shrunk, flush the unmapped region
+      if (OldSize > NewSize) {
+        FEXCore::Context::InvalidateGuestCodeRange(CTX, OldAddress + NewSize, OldSize - NewSize);
+      }
+    }
+  }
+}
+
+void SyscallHandler::TrackShmat(int shmid, uintptr_t Base, int shmflg) {
+  shmid_ds stat;
+
+  auto res = shmctl(shmid, IPC_STAT, &stat);
+  LOGMAN_THROW_A_FMT(res != -1, "shmctl IPC_STAT failed");
+
+  uint64_t Length = stat.shm_segsz;
+
+  {
+    FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+    // TODO
+    MRID mrid{SpecialDev::SHM, static_cast<uint64_t>(shmid)};
+
+    auto ResourceInserted = VMATracking.MappedResources.insert({mrid, {nullptr, nullptr, Length}});
+    auto Resource = &ResourceInserted.first->second;
+    if (ResourceInserted.second) {
+      Resource->Iterator = ResourceInserted.first;
+    }
+    VMATracking.SetUnsafe(CTX, Resource, Base, 0, Length, VMAFlags::fromFlags(MAP_SHARED), 
+      VMAProt::fromProt((shmflg & SHM_RDONLY) ? PROT_READ : (PROT_READ | PROT_WRITE))
+    );
+  }
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Length);
+  }
+}
+
+void SyscallHandler::TrackShmdt(uintptr_t Base) {
+  FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+
+  auto Length = VMATracking.ClearShmUnsafe(CTX, Base);
+
+  if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    // This might over flush if the shm has holes in it
+    FEXCore::Context::InvalidateGuestCodeRange(CTX, Base, Length);
+  }
+}
+
+void SyscallHandler::TrackMadvise(uintptr_t Base, uintptr_t Size, int advice) {
+	Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
+	{
+		FHU::ScopedSignalMaskWithUniqueLock lk(_SyscallHandler->VMATracking.Mutex);
+  	// TODO
+	}
+}
+
+}

Source/Tests/LinuxSyscalls/SyscallsVMATracking.cpp

@@ -0,0 +1,366 @@
+/*
+$info$
+category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough
+tags: LinuxSyscalls|common
+desc: VMA Tracking
+$end_info$
+*/
+
+#include "Tests/LinuxSyscalls/Syscalls.h"
+
+namespace FEX::HLE {
+/// List Operations ///
+
+inline void SyscallHandler::VMATracking::ListCheckVMALinks(VMAEntry *VMA) {
+  if (VMA) {
+    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA != VMA, "VMA tracking error");
+    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA != VMA, "VMA tracking error");
+  }
+}
+
+// Removes a VMA from corresponding MappedResource list
+// Returns true if list is empty
+bool SyscallHandler::VMATracking::ListRemove(VMAEntry *VMA) {
+  LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error");
+
+  // if it has prev, make prev to next
+  if (VMA->ResourcePrevVMA) {
+    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error");
+    VMA->ResourcePrevVMA->ResourceNextVMA = VMA->ResourceNextVMA;
+  } else {
+    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error");
+  }
+
+  // if it has next, make next to prev
+  if (VMA->ResourceNextVMA) {
+    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error");
+    VMA->ResourceNextVMA->ResourcePrevVMA = VMA->ResourcePrevVMA;
+  }
+
+  // If it is the first in the list, make Next the first in the list
+  if (VMA->Resource && VMA->Resource->FirstVMA == VMA) {
+    LOGMAN_THROW_A_FMT(!VMA->ResourceNextVMA || VMA->ResourceNextVMA->ResourcePrevVMA == nullptr, "VMA tracking error");
+
+    VMA->Resource->FirstVMA = VMA->ResourceNextVMA;
+  }
+
+  ListCheckVMALinks(VMA);
+  ListCheckVMALinks(VMA->ResourceNextVMA);
+  ListCheckVMALinks(VMA->ResourcePrevVMA);
+
+  // Return true if list is empty
+  return VMA->Resource->FirstVMA == nullptr;
+}
+
+// Replaces a VMA in corresponding MappedResource list
+// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
+void SyscallHandler::VMATracking::ListReplace(VMAEntry *VMA, VMAEntry *NewVMA) {
+  LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error");
+
+  LOGMAN_THROW_A_FMT(VMA->Resource == NewVMA->Resource, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == VMA->ResourcePrevVMA, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == VMA->ResourceNextVMA, "VMA tracking error");
+
+  if (VMA->ResourcePrevVMA) {
+    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA != VMA, "VMA tracking error");
+    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error");
+    VMA->ResourcePrevVMA->ResourceNextVMA = NewVMA;
+  } else {
+    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error");
+    VMA->Resource->FirstVMA = NewVMA;
+  }
+
+  if (VMA->ResourceNextVMA) {
+    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error");
+    VMA->ResourceNextVMA->ResourcePrevVMA = NewVMA;
+  }
+
+  ListCheckVMALinks(VMA);
+  ListCheckVMALinks(NewVMA);
+  ListCheckVMALinks(VMA->ResourceNextVMA);
+  ListCheckVMALinks(VMA->ResourcePrevVMA);
+}
+
+// Inserts a VMA in corresponding MappedResource list
+// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
+void SyscallHandler::VMATracking::ListInsertAfter(VMAEntry *AfterVMA, VMAEntry *NewVMA) {
+  LOGMAN_THROW_A_FMT(NewVMA->Resource != nullptr, "VMA tracking error");
+
+  LOGMAN_THROW_A_FMT(AfterVMA->Resource == NewVMA->Resource, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == AfterVMA->ResourceNextVMA, "VMA tracking error");
+
+  if (AfterVMA->ResourceNextVMA) {
+    LOGMAN_THROW_A_FMT(AfterVMA->ResourceNextVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error");
+    AfterVMA->ResourceNextVMA->ResourcePrevVMA = NewVMA;
+  }
+  AfterVMA->ResourceNextVMA = NewVMA;
+
+  ListCheckVMALinks(AfterVMA);
+  ListCheckVMALinks(NewVMA);
+  ListCheckVMALinks(AfterVMA->ResourceNextVMA);
+  ListCheckVMALinks(AfterVMA->ResourcePrevVMA);
+}
+
+// Prepends a VMA
+// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
+void SyscallHandler::VMATracking::ListPrepend(MappedResource *Resource, VMAEntry *NewVMA) {
+  LOGMAN_THROW_A_FMT(Resource != nullptr, "VMA tracking error");
+
+  LOGMAN_THROW_A_FMT(NewVMA->Resource == Resource, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == nullptr, "VMA tracking error");
+  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == Resource->FirstVMA, "VMA tracking error");
+
+  if (Resource->FirstVMA) {
+    LOGMAN_THROW_A_FMT(Resource->FirstVMA->ResourcePrevVMA == nullptr, "VMA tracking error");
+    Resource->FirstVMA->ResourcePrevVMA = NewVMA;
+  }
+
+  Resource->FirstVMA = NewVMA;
+
+  ListCheckVMALinks(NewVMA);
+  ListCheckVMALinks(NewVMA->ResourceNextVMA);
+  ListCheckVMALinks(NewVMA->ResourcePrevVMA);
+}
+
+/// VMA tracking ///
+
+// Lookup a VMA by address
+SyscallHandler::VMATracking::VMACIterator SyscallHandler::VMATracking::LookupVMAUnsafe(uint64_t GuestAddr) const {
+  auto Entry = VMAs.upper_bound(GuestAddr);
+
+  if (Entry != VMAs.begin()) {
+    --Entry;
+
+    if (Entry->first <= GuestAddr && (Entry->first + Entry->second.Length) > GuestAddr) {
+      return Entry;
+    }
+  }
+
+  return VMAs.end();
+}
+
+// Set or Replace mappings in a range with a new mapping
+void SyscallHandler::VMATracking::SetUnsafe(FEXCore::Context::Context *CTX, MappedResource *MappedResource, uintptr_t Base,
+                                            uintptr_t Offset, uintptr_t Length, VMAFlags Flags, VMAProt Prot) {
+  ClearUnsafe(CTX, Base, Length, MappedResource);
+
+  auto [Iter, Inserted] = VMAs.emplace(
+      Base, VMAEntry{MappedResource, nullptr, MappedResource ? MappedResource->FirstVMA : nullptr, Base, Offset, Length, Flags, Prot});
+      
+  LOGMAN_THROW_A_FMT(Inserted == true, "VMA Tracking corruption");
+
+  if (MappedResource) {
+    // Insert to the front of the linked list
+    ListPrepend(MappedResource, &Iter->second);
+  }
+}
+
+// Remove mappings in a range, possibly splitting them if needed and 
+// freeing their associated MappedResource unless it is equal to PreservedMappedResource
+void SyscallHandler::VMATracking::ClearUnsafe(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length,
+                                              MappedResource *PreservedMappedResource) {
+  const auto Top = Base + Length;
+
+  // find the first Mapping at or after the Range ends, or ::end()
+  // Top is the address after the end
+  auto CurrentIter = VMAs.lower_bound(Top);
+
+  // Iterate backwards all mappings
+  while (CurrentIter != VMAs.begin()) {
+    CurrentIter--;
+
+    const auto Current = &CurrentIter->second;
+    const auto MapBase = Current->Base;
+    const auto MapTop = MapBase + Current->Length;
+    const auto OffsetDiff = Current->Offset - MapBase;
+
+    if (MapTop <= Base) {
+      // Mapping ends before the Range start, exit
+      break;
+    } else {
+      const bool HasFirstPart = MapBase < Base;
+      const bool HasTrailingPart = MapTop > Top;
+
+      // (1) HasFirstPart, !HasTrailingPart -> trim
+      // (2) HasFirstPart, HasTrailingPart -> trim, insert trailing, list add after first part
+      // (3) !HasFirstPart, !HasTrailing part -> list remove, erase
+      // (4) !HasFirstPart, HasTrailing part -> insert trailing, list replace first part, erase
+
+      if (HasFirstPart) {
+        // Handle trim for (1) & (2)
+        Current->Length = Base - MapBase;
+      } else if (!HasTrailingPart) {
+        // Handle all of (3)
+        // Mapping is included or equal to Range, delete
+
+        // If linked to a Mapped Resource, remove from linked list and possibly delete the Mapped Resource
+        if (Current->Resource) {
+          if (ListRemove(Current) && Current->Resource != PreservedMappedResource) {
+            if (Current->Resource->AOTIRCacheEntry) {
+              FEXCore::Context::UnloadAOTIRCacheEntry(CTX, Current->Resource->AOTIRCacheEntry);
+            }
+            MappedResources.erase(Current->Resource->Iterator);
+          }
+        }
+
+        // returns next element, so -- is safe at loop
+        CurrentIter = VMAs.erase(CurrentIter);
+        continue; // we're done
+      }
+
+      const bool ReplaceAndErase = !HasFirstPart;
+
+      if (HasTrailingPart) {
+        // Handle insert of (2), (4)
+
+        // insert trailing part, link it after Mapping
+        auto NewOffset = OffsetDiff + Top;
+        auto NewLength = MapTop - Top;
+
+        auto [Iter, Inserted] = VMAs.emplace(
+            Top, VMAEntry{Current->Resource, ReplaceAndErase ?  Current->ResourcePrevVMA : Current, Current->ResourceNextVMA, Top, NewOffset, NewLength, Current->Flags, Current->Prot});
+        LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error");
+        auto TrailingPart = &Iter->second;
+        if (Current->Resource) {
+          if (ReplaceAndErase) {
+            // Handle list replace of (4)
+            ListReplace(Current, TrailingPart);
+          } else {
+            // Handle list insert (2)
+            ListInsertAfter(Current, TrailingPart);
+          }
+        }
+      }
+
+      if (ReplaceAndErase) {
+        // Handle erase of (4)
+        // returns next element, so -- is safe at loop
+        CurrentIter = VMAs.erase(CurrentIter);
+      }
+    }
+  }
+}
+
+// Change flags of mappings in a range and split the mappings if needed
+void SyscallHandler::VMATracking::ChangeUnsafe(uintptr_t Base, uintptr_t Length, VMAProt NewProt) {
+  const auto Top = Base + Length;
+
+  // find the first Mapping at or after the Range ends, or ::end()
+  // Top is the address after the end
+  auto MappingIter = VMAs.lower_bound(Top);
+
+  // Iterate backwards all mappings
+  while (MappingIter != VMAs.begin()) {
+    MappingIter--;
+
+    auto Current = &MappingIter->second;
+    const auto MapBase = Current->Base;
+    const auto MapTop = MapBase + Current->Length;
+    const auto MapFlags = Current->Flags;
+    const auto MapProt = Current->Prot;
+
+    const auto OffsetDiff = Current->Offset - MapBase;
+
+    if (MapTop <= Base) {
+      // Mapping ends before the Range start, exit
+      break;
+    } else if (MapProt.All == NewProt.All) {
+      // Mapping already has the needed prots
+      continue;
+    } else {
+      const bool HasFirstPart = MapBase < Base;
+      const bool HasTrailingPart = MapTop > Top;
+
+      if (HasFirstPart) {
+        // Mapping starts before range, split first part
+
+        // Trim end of original mapping
+        Current->Length = Base - MapBase;
+
+        // Make new VMA with new flags, insert for length of range
+        auto NewOffset = OffsetDiff + Base;
+        auto NewLength = Top - Base;
+
+        auto [Iter, Inserted] =
+            VMAs.emplace(Base, VMAEntry{Current->Resource, Current, Current->ResourceNextVMA, Base, NewOffset, NewLength, MapFlags, NewProt});
+        LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error");
+        auto RestOfMapping = &Iter->second;
+
+        if (Current->Resource) {
+          ListInsertAfter(Current, RestOfMapping);
+        }
+
+        Current = RestOfMapping;
+      } else {
+        // Mapping starts in range, just change Prot
+        Current->Prot = NewProt;
+      }
+
+      if (HasTrailingPart) {
+        // ends after Range, split last part and insert with original flags
+
+        // Trim the mapping (possibly already trimmed)
+        Current->Length = Top - Current->Base;
+
+        // prot has already been changed
+
+        // Make new VMA with original flags, insert for remaining length
+        auto NewOffset = OffsetDiff + Top;
+        auto NewLength = MapTop - Top;
+
+        auto [Iter, Inserted] =
+            VMAs.emplace(Top, VMAEntry{Current->Resource, Current, Current->ResourceNextVMA, Top, NewOffset, NewLength, MapFlags, MapProt});
+        LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error");
+        auto TrailingMapping = &Iter->second;
+
+        if (Current->Resource) {
+          ListInsertAfter(Current, TrailingMapping);
+        }
+      }
+    }
+  }
+}
+
+// This matches the peculiarities algorithm used in linux ksys_shmdt (linux kernel 5.16, ipc/shm.c)
+uintptr_t SyscallHandler::VMATracking::ClearShmUnsafe(FEXCore::Context::Context *CTX, uintptr_t Base) {
+
+  // Find first VMA at or after Base
+  // Iterate until first SHM VMA, with matching offset, get length
+  // Then, erase any later occurrences of this SHM
+
+  // returns first element that is greater or equal or ::end
+  auto Entry = VMAs.lower_bound(Base);
+
+  for (; Entry != VMAs.end(); ++Entry) {
+    LOGMAN_THROW_A_FMT(Entry->second.Base >= Base, "VMA tracking corruption");
+    if (Entry->second.Base - Base == Entry->second.Offset && Entry->second.Resource &&
+        Entry->second.Resource->Iterator->first.dev == SpecialDev::SHM) {
+        break;
+    }
+  }
+
+  if (Entry == VMAs.end()) {
+    return 0;
+  }
+
+  const auto ShmLength = Entry->second.Resource->Iterator->second.Length;
+  const auto Resource = Entry->second.Resource;
+
+  do {
+    if (Entry->second.Resource == Resource) {
+      if (ListRemove(&Entry->second)) {
+        if (Entry->second.Resource->AOTIRCacheEntry) {
+          FEXCore::Context::UnloadAOTIRCacheEntry(CTX, Entry->second.Resource->AOTIRCacheEntry);
+        }
+        MappedResources.erase(Entry->second.Resource->Iterator);
+      }
+      Entry = VMAs.erase(Entry);
+    } else {
+      Entry++;
+    }
+  } while (Entry != VMAs.end() && (Entry->second.Base + Entry->second.Length - Base) <= ShmLength);
+
+  return ShmLength;
+}
+}

Source/Tools/FEXConfig/Main.cpp

@@ -568,16 +568,19 @@ namespace {
         if (**Value == "0") {
           SMCChecks = FEXCore::Config::CONFIG_SMC_NONE;
         } else if (**Value == "1") {
-          SMCChecks = FEXCore::Config::CONFIG_SMC_MMAN;
+          SMCChecks = FEXCore::Config::CONFIG_SMC_MTRACK;
         } else if (**Value == "2") {
           SMCChecks = FEXCore::Config::CONFIG_SMC_FULL;
+        } else if (**Value == "3") {
+          SMCChecks = FEXCore::Config::CONFIG_SMC_MMAN;
         }
       }
 
       bool SMCChanged = false;
       SMCChanged |= ImGui::RadioButton("None", &SMCChecks, FEXCore::Config::CONFIG_SMC_NONE); ImGui::SameLine();
-      SMCChanged |= ImGui::RadioButton("MMan", &SMCChecks, FEXCore::Config::CONFIG_SMC_MMAN); ImGui::SameLine();
+      SMCChanged |= ImGui::RadioButton("MTrack (Default)", &SMCChecks, FEXCore::Config::CONFIG_SMC_MTRACK); ImGui::SameLine();
       SMCChanged |= ImGui::RadioButton("Full", &SMCChecks, FEXCore::Config::CONFIG_SMC_FULL);
+      SMCChanged |= ImGui::RadioButton("MMan (Deprecated)", &SMCChecks, FEXCore::Config::CONFIG_SMC_MMAN); ImGui::SameLine();
 
       if (SMCChanged) {
         LoadedConfig->EraseSet(FEXCore::Config::ConfigOption::CONFIG_SMCCHECKS, std::to_string(SMCChecks));