Merge pull request #6039 from Sonicadvance1/274

arm64ec: Have synchronous faults skip fault overhead

CMakeLists.txt

@@ -618,11 +618,21 @@ if (BUILD_TESTING)
   endif()
   set(TEST_JOB_FLAG "-j${TEST_JOB_COUNT}")
 
+  # When running "ninja tests", defined below, it is useful to intentionally
+  # oversubscribe the CPU cores since many of the tests spend their time idle.
+  if ("$CACHE{TEST_JOB_COUNT}" STREQUAL "")
+    execute_process(COMMAND "nproc" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE TESTS_TARGET_JOB_COUNT)
+    math(EXPR TESTS_TARGET_JOB_COUNT "${TESTS_TARGET_JOB_COUNT} * 3 / 2")
+    set(TESTS_TARGET_JOB_FLAG "-j${TESTS_TARGET_JOB_COUNT}")
+  else()
+    set(TESTS_TARGET_JOB_FLAG ${TEST_JOB_FLAG})
+  endif()
+
   # Runs the whole test suite, with large pages enabled to reduce the cost of fork() in ctest.
   add_custom_target(tests
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     USES_TERMINAL
-    COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1 ctest "--progress" "--timeout" "302" ${TEST_JOB_FLAG})
+    COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1 ctest "--progress" "--timeout" "302" ${TESTS_TARGET_JOB_FLAG})
 endif()
 
 add_subdirectory(External/SoftFloat-3e/)

FEXCore/Source/Interface/Core/DiskCache.cpp

@@ -1026,9 +1026,8 @@ namespace DiskCache {
     const size_t TotalSize = GuestCodeOffset + GuestCode.size();
 
     // we'll copy everything into here and pass it to the Writer, then return to caller quickly
-    fextl::vector<uint8_t> Blob;
-    Blob.resize(TotalSize);
-    uint8_t* BlobData = Blob.data();
+    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(TotalSize);
+    uint8_t* BlobData = BlobRef->data();
 
     BlobFixedHeader Header {
       .GuestSize = (uint32_t)GuestCode.size(),
@@ -1182,7 +1181,6 @@ namespace DiskCache {
       MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
 
     // blob done, publish to index as in-memory for now, flush to disk below
-    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(std::move(Blob));
     struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
     NewEntry.GuestExtents = ExactGuestCodeExtents;
     {

FEXCore/Source/Interface/Core/DiskCache.h

@@ -249,7 +249,7 @@ namespace DiskCache {
   // The current version of the diskcache.
   // This must be changed any time codegen changes occur!
   // Be aware of the impact of changing this frequently!
-  static constexpr uint16_t FormatVersion = 30;
+  static constexpr uint16_t FormatVersion = 31;
 
   static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 500;
 

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -439,6 +439,26 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
+  {
+    // Faultless synchronous signal.
+    // Some games abuse SIGILL/SIGTRAP/SIGSEGV heavily and the fault overhead is visible.
+    // Instead of double faulting, clean up state and jump to the frontend to handle this.
+    // `FEXCore::Core::CpuStateFrame::SynchronousFaultData` needs to be setup at this point.
+    GuestSignal_SynchronousFault = GetCursorAddress<uint64_t>();
+
+    SpillStaticRegs(TMP1);
+
+    ldr(ARMEmitter::XReg::x0, &l_CTX);
+    mov(ARMEmitter::XReg::x1, STATE);
+    ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.SynchronousExceptionHandler));
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
+    } else {
+      br(ARMEmitter::Reg::r2);
+    }
+  }
+
   {
     ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
     SpillStaticRegs(TMP1);
@@ -2690,6 +2710,7 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread)
     Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;
     Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
     Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
+    Ptrs.GuestSignal_SynchronousFault = GuestSignal_SynchronousFault;
     Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;
     Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
     Ptrs.LUDIVHandler = LUDIVHandlerAddress;

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -89,6 +89,7 @@ private:
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
 
   uint64_t PauseReturnInstruction {};
   std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -76,23 +76,28 @@ DEF_OP(Break) {
   LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
   str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
 
-  switch (Op->Reason.Signal) {
-  case Core::FAULT_SIGILL:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+  if (CTX->HostFeatures.SupportsFaultFreeSynchronousFaults) {
+    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SynchronousFault));
     br(TMP1);
-    break;
-  case Core::FAULT_SIGTRAP:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
-  case Core::FAULT_SIGSEGV:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
-    br(TMP1);
-    break;
-  default:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
+  } else {
+    switch (Op->Reason.Signal) {
+    case Core::FAULT_SIGILL:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGTRAP:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGSEGV:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
+      br(TMP1);
+      break;
+    default:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    }
   }
 }
 

FEXCore/Source/Utils/Allocator.cpp

@@ -23,7 +23,9 @@
 #include <fcntl.h>
 #ifndef _WIN32
 #include <sys/mman.h>
+#include <sys/resource.h>
 #include <sys/user.h>
+#include <unistd.h>
 #endif
 
 namespace fextl::pmr {
@@ -214,6 +216,52 @@ fextl::vector<MemoryRegion> CollectMemoryGaps(uintptr_t Begin, uintptr_t End, in
   FEX_UNREACHABLE;
 }
 
+// Tracks the region below the main thread stack, which is reserved for stack growth,
+// as well as the guard region directly below the stack window.
+static uintptr_t MainThreadStackReserveBegin {};
+static uintptr_t MainThreadStackWindowBegin {};
+static uintptr_t MainThreadStackWindowEnd {};
+static size_t MainThreadStackGuardSize {};
+static uintptr_t MainThreadStackGuardBegin {};
+
+// Currently, guest signal handlers leak space on the main stack
+// if they exit via longjmp instead of sigreturn. Because of this memory leak
+// we must allow the main thread stack to grow without bound. This technically
+// violates RLIMIT_STACK, but until we fix the leak, we don't have much choice.
+// TODO: Once the leak is fixed, remove this function entirely.
+bool TryGrowMainThreadStack(uintptr_t FaultAddress) {
+  // Only grow the stack when the fault is in the guard region.
+  if (FaultAddress < MainThreadStackGuardBegin || FaultAddress >= MainThreadStackWindowBegin) {
+    return false;
+  }
+
+  // Compute the required size of the stack region.
+  // Double the current stack, up to 1GB, then just add 1GB at a time.
+  constexpr size_t OneGB = 1ULL << 30;
+  const size_t RequiredSize = MainThreadStackWindowEnd - FaultAddress;
+  size_t NewSize = MainThreadStackWindowEnd - MainThreadStackWindowBegin;
+
+  while (NewSize < RequiredSize) {
+    if (NewSize < OneGB) {
+      NewSize = std::min(NewSize * 2, OneGB);
+    } else {
+      NewSize += OneGB;
+    }
+  }
+
+  // Don't grow past the bottom of the reserved region,
+  // which is where the first dynamically loaded libraries live.
+  const uintptr_t NewWindowBegin = std::max(MainThreadStackWindowEnd - NewSize, MainThreadStackReserveBegin);
+  if (mprotect(reinterpret_cast<void*>(NewWindowBegin), MainThreadStackWindowBegin - NewWindowBegin, PROT_READ | PROT_WRITE) != 0) {
+    return false;
+  }
+
+  // Move the window and the guard region below it down.
+  MainThreadStackWindowBegin = NewWindowBegin;
+  MainThreadStackGuardBegin = std::max(NewWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+  return true;
+}
+
 fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   const uintptr_t StackLocation_u64 = reinterpret_cast<uintptr_t>(alloca(0));
 
@@ -231,7 +279,7 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   // If the memory bounds include the stack, blocking all memory regions will
   // limit the stack size to the current value. To allow some stack growth,
   // we don't block the memory gap directly below the stack memory but
-  // instead map it as readable+writable.
+  // instead reserve it for the stack, see TryGrowMainThreadStack.
   {
     auto StackRegionIt = std::find_if(Regions.begin(), Regions.end(), [StackLocation_u64](auto& Region) {
       return reinterpret_cast<uintptr_t>(Region.Ptr) + Region.Size > StackLocation_u64;
@@ -243,16 +291,40 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
 
     if (IsStackMapping && StackRegionIt != Regions.begin() &&
         reinterpret_cast<uintptr_t>(std::prev(StackRegionIt)->Ptr) + std::prev(StackRegionIt)->Size <= End) {
-      // Allocate the region under the stack as READ | WRITE so the stack can still grow
       --StackRegionIt;
 
-      auto Alloc =
-        ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
+      // Get the page size, so we can make sure to align on page boundaries.
+      const auto PageSize = sysconf(_SC_PAGESIZE);
+      const size_t HostPageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE;
+
+      // Allocate RLIMIT_STACK worth of memory for the main thread stack, or 64MB if no limit is set.
+      struct rlimit StackLimit {};
+      size_t Size = 1ULL << 26;
+      if (getrlimit(RLIMIT_STACK, &StackLimit) == 0 && StackLimit.rlim_cur != RLIM_INFINITY) {
+        Size = std::max<size_t>(FEXCore::AlignUp(StackLimit.rlim_cur, HostPageSize), HostPageSize);
+      }
+      Size = std::min(Size, StackRegionIt->Size);
+
+      // Reserve all of the memory between the top of the stack, and the start of the next region.
+      // We must map it with PROT_NONE, otherwise the MM will assume these pages are "in use"
+      // and charge this usage towards VmData. In practice this is multiple TBs of VA space.
+      auto Alloc = ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
 
       LogMan::Throw::AFmt(Alloc != MAP_FAILED, "StealMemoryRegion:Stack: mmap({}, {:x}) failed: {}", fmt::ptr(StackRegionIt->Ptr),
                           StackRegionIt->Size, errno);
       LogMan::Throw::AFmt(Alloc == StackRegionIt->Ptr, "mmap returned {} instead of {}", Alloc, fmt::ptr(StackRegionIt->Ptr));
 
+      // Now determine the bounds for the stack region, and its guard region, which is used by TryGrowMainThreadStack().
+      MainThreadStackReserveBegin = reinterpret_cast<uintptr_t>(StackRegionIt->Ptr);
+      MainThreadStackWindowEnd = MainThreadStackReserveBegin + StackRegionIt->Size;
+      MainThreadStackWindowBegin = MainThreadStackWindowEnd - Size;
+      MainThreadStackGuardSize = 256 * HostPageSize;
+      MainThreadStackGuardBegin = std::max(MainThreadStackWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+
+      // Protect the stack window.
+      const auto Result = mprotect(reinterpret_cast<void*>(MainThreadStackWindowBegin), Size, PROT_READ | PROT_WRITE);
+      LogMan::Throw::AFmt(Result == 0, "StealMemoryRegion:Stack: mprotect({:x}, {:x}) failed: {}", MainThreadStackWindowBegin, Size, errno);
+
       Regions.erase(StackRegionIt);
     }
   }

FEXCore/Source/Utils/AllocatorHooks.cpp

@@ -27,9 +27,13 @@ using vma_name_hook_type = void (*)(const char* name, const void* address, size_
 #ifdef ENABLE_FEX_ALLOCATOR
 typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);
 typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);
+typedef void (*rp_mcommit_hook_type)(void* address, size_t size);
+typedef void (*rp_mdecommit_hook_type)(void* address, size_t size);
 typedef void (*vma_name_hook_type)(const char* name, const void* address, size_t size);
 extern "C" rp_mmap_hook_type rp_mmap_hook;
 extern "C" rp_munmap_hook_type rp_munmap_hook;
+extern "C" rp_mcommit_hook_type rp_mcommit_hook;
+extern "C" rp_mdecommit_hook_type rp_mdecommit_hook;
 extern "C" vma_name_hook_type rp_name_hook;
 
 #ifndef _WIN32
@@ -112,8 +116,11 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
     alignment = 0;
   }
 
+  // Protect with PROT_NONE so that the whole heap is not charged against VmData.
+  // In the memory commit callback, FEX_rp_memory_commit, we reprotect with
+  // PROT_READ | PROT_WRITE to commit memory on demand.
   size_t map_size = AlignUp(size + alignment, global_config.page_size);
-  auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  auto ptr = fex_mmap_hook(0, map_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 
   if (ptr == MAP_FAILED) {
     ptr = nullptr;
@@ -149,7 +156,9 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
 }
 
 static void FEX_rp_memory_commit(void* address, size_t size) {
-  // NOP-implementation.
+  if (mprotect(address, size, PROT_READ | PROT_WRITE)) {
+    fprintf(stderr, "Failed to commit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_decommit(void* address, size_t size) {
@@ -160,6 +169,11 @@ static void FEX_rp_memory_decommit(void* address, size_t size) {
   if (madvise(address, size, MADV_DONTNEED)) {
     fprintf(stderr, "Failed to decommit VMA region.");
   }
+
+  // Drop the protections back to PROT_NONE so the pages are no longer charged towards VmData.
+  if (mprotect(address, size, PROT_NONE)) {
+    fprintf(stderr, "Failed to decommit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {
@@ -190,6 +204,8 @@ void InitializeAllocator(size_t PageSize) {
   rpmalloc_initialize_config(&global_interface, &global_config);
   rp_mmap_hook = FEX_rp_mmap;
   rp_munmap_hook = FEX_rp_memory_unmap;
+  rp_mcommit_hook = FEX_rp_memory_commit;
+  rp_mdecommit_hook = FEX_rp_memory_decommit;
   rp_name_hook = LocalVirtualName;
 }
 #else

FEXCore/include/FEXCore/Core/CoreState.h

@@ -353,8 +353,14 @@ struct JITPointers {
   uint64_t RDRANDFallback {};
   uint64_t ThunkCallbackRet {};
 
+  /**
+   * @name Frontend handler pointers.
+   * @{ */
   // Handles returning/calling ARM64EC code from the JIT, expects the target PC in TMP3
   uint64_t ExitFunctionEC {};
+  // Handles a synchronous exception without faulting to jump in to the handler. Frontend specific.
+  uint64_t SynchronousExceptionHandler {};
+  /**  @} */
 
   FallbackABIInfo FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX];
   uint64_t NamedVectorConstantPointers[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX];
@@ -376,6 +382,7 @@ struct JITPointers {
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
   uint64_t SignalReturnHandler {};
   uint64_t SignalReturnHandlerRT {};
   uint64_t L2Pointer {};

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -83,9 +83,10 @@ struct HostFeatures {
   uint32_t PreferZVAForVZero           : 1 {};
   uint32_t SupportsAFP                 : 1 {};
   uint32_t SupportsFloatExceptions     : 1 {};
+  uint32_t SupportsFaultFreeSynchronousFaults : 1 {};
   // Flag if this is InstCountCI
   uint32_t IsInstCountCI : 1 {};
-  uint32_t pad           : 26 {};
+  uint32_t pad           : 25 {};
 
   // This affects codegen, but it isn't machine state
   HostTypeEnum HostType {};

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -79,5 +79,6 @@ FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(s
 #ifndef _WIN32
 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);
 FEX_DEFAULT_VISIBILITY void UninstallTLSData(FEXCore::Core::InternalThreadState* Thread);
+FEX_DEFAULT_VISIBILITY bool TryGrowMainThreadStack(uintptr_t FaultAddress);
 #endif
 } // namespace FEXCore::Allocator

Scripts/guest_test_runner.py

@@ -1,11 +1,13 @@
 #!/usr/bin/python3
-import json
 import os
 import sys
-import subprocess
+# NB: This script is run with the "-S" flag, and therefore it can only use the Python standard library.
 
 def DoesFEXSupportAVX(mode):
     # Check if FEX indicates support for AVX
+    
+    # Import this here lazily to save on startup time
+    import subprocess
     fex_interpreter_path = os.path.dirname(sys.argv[7]) + "/FEX"
 
     args = list()
@@ -30,6 +32,8 @@ def TestRequiresAVXSupport():
     try:
         with open(json_path) as json_file:
             try:
+                # Import this here lazily to save on startup time
+                import json
                 json_data = json.load(json_file)
                 if not isinstance(json_data, dict):
                     raise TypeError('JSON data must be a dict')
@@ -130,9 +134,9 @@ if ResultCode == 0:
     for Try in range(TryCount):
         # Run the test and wait for it to end to get the result
         print(RunnerArgs)
-        Process = subprocess.Popen(RunnerArgs)
-        Process.wait()
-        ResultCode = Process.returncode
+        Process = os.posix_spawnp(RunnerArgs[0], RunnerArgs, os.environ)
+        _, Status = os.waitpid(Process, 0)
+        ResultCode = os.waitstatus_to_exitcode(Status)
 
         # Break if the expected output is the result code
         if (expected_output[test_name] == ResultCode):

Scripts/testharness_runner.py

@@ -1,8 +1,9 @@
 #!/usr/bin/python3
+import os
 import sys
-import subprocess
 from os import path
 from shutil import which
+# NB: This script is run with the "-S" flag, and therefore it can only use the Python standard library.
 
 # Args: <Known Failures file> <Known Failures Type File> <DisabledTestsFile> <DisabledTestsTypeFile> <DisabledTestsRunnerFile> <TestName> <FullTestName> <Test Harness Executable> <Args>...
 
@@ -59,9 +60,9 @@ if (disabled_tests.get(current_test)):
     sys.exit(125)
 
 # Run the test and wait for it to end to get the result
-Process = subprocess.Popen(RunnerArgs)
-Process.wait()
-ResultCode = Process.returncode
+Process = os.posix_spawnp(RunnerArgs[0], RunnerArgs, os.environ)
+_, Status = os.waitpid(Process, 0)
+ResultCode = os.waitstatus_to_exitcode(Status)
 
 # Check for known failures - try full test name first, then partial test name
 is_known_failure = known_failures.get(full_test_name) or known_failures.get(current_test)

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -400,6 +400,20 @@ bool SignalDelegator::HandleDispatcherGuestSignal(FEXCore::Core::InternalThreadS
   // Set our state register to point to our guest thread data
   ArchHelpers::Context::SetState(ucontext, reinterpret_cast<uint64_t>(Frame));
 
+  if (GuestAction->sa_flags & SA_RESETHAND) {
+
+    {
+      // If SA_RESETHAND is set, then the signal handler is immediately reset to SIG_DFL.
+      std::lock_guard lk(GuestDelegatorMutex);
+      GuestAction->sigaction_handler.handler = SIG_DFL;
+    }
+
+    // If this signal is not required by FEX then we can fully uninstall the host handler as well.
+    if (HostHandlers[Signal].Required.load(std::memory_order_relaxed) != true) {
+      UninstallHostHandler(Signal);
+    }
+  }
+
   return true;
 }
 
@@ -586,6 +600,11 @@ bool SignalDelegator::HandleFrontendSIGSEGV(FEXCore::Core::InternalThreadState*
   }
 #endif
 
+  // If this fault is in the main thread stack guard region, we should try to grow the main thread stack.
+  if (SigInfo.si_code == SEGV_ACCERR && FEXCore::Allocator::TryGrowMainThreadStack(reinterpret_cast<uintptr_t>(SigInfo.si_addr))) {
+    return true;
+  }
+
   return false;
 }
 

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -408,6 +408,10 @@ private:
     if (Result == 0) return -errno; \
     return Result;                  \
   } while (0)
+#define SYSCALL_EFAULT_IF_NULL(Ptr)       \
+  do {                                    \
+    if ((Ptr) == nullptr) return -EFAULT; \
+  } while (0)
 
 extern FEX::HLE::SyscallHandler* _SyscallHandler;
 

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/FD.cpp

@@ -315,6 +315,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -332,6 +333,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -349,6 +351,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -360,6 +363,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -370,6 +374,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = ::fstat(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -380,6 +385,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -390,6 +396,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -401,6 +408,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
 
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -411,6 +419,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat64 host_stat;
     uint64_t Result = ::fstat64(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -421,6 +430,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -431,6 +441,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = ::fstatfs(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -443,6 +454,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs64 host_stat;
     uint64_t Result = ::fstatfs64(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -455,6 +467,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -564,6 +577,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat64 host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.NewFSStatAt64(dirfd, pathname, &host_stat, flag);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }

Source/Tools/LinuxEmulation/LinuxSyscalls/x64/FD.cpp

@@ -63,32 +63,37 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     });
 
   REGISTER_SYSCALL_IMPL_X64(stat, [](FEXCore::Core::CpuStateFrame* Frame, const char* pathname, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(pathname);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(pathname, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
   });
 
   REGISTER_SYSCALL_IMPL_X64(fstat, [](FEXCore::Core::CpuStateFrame* Frame, int fd, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = ::fstat(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
   });
 
   REGISTER_SYSCALL_IMPL_X64(lstat, [](FEXCore::Core::CpuStateFrame* Frame, const char* path, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(path);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(path, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
@@ -96,11 +101,13 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
 
   REGISTER_SYSCALL_IMPL_X64(
     newfstatat, [](FEXCore::Core::CpuStateFrame* Frame, int dirfd, const char* pathname, FEX::HLE::x64::guest_stat* buf, int flag) -> uint64_t {
+      SYSCALL_EFAULT_IF_NULL(pathname);
       FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(pathname, PATH_MAX);
-      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       struct stat host_stat;
       uint64_t Result = FEX::HLE::_SyscallHandler->FM.NewFSStatAt(dirfd, pathname, &host_stat, flag);
       if (Result != -1) {
+        SYSCALL_EFAULT_IF_NULL(buf);
+        FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
         *buf = host_stat;
       }
       SYSCALL_ERRNO();
@@ -133,8 +140,8 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
   });
 
   REGISTER_SYSCALL_IMPL_X64(statfs, [](FEXCore::Core::CpuStateFrame* Frame, const char* path, struct statfs* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(path);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(path, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, buf);
     SYSCALL_ERRNO();
   });

Source/Windows/ARM64EC/Module.S

@@ -158,9 +158,9 @@ DIRECT_SYSCALL_WRAPPER("#NtProtectVirtualMemoryNative", WineNtProtectVirtualMemo
   subs x16, x11, x9
   b.lo end
   adrp x17, NtDllRedirectionLUTSize
-  ldr x17, [x17, #:lo12:NtDllRedirectionLUTSize]
+  ldr w17, [x17, #:lo12:NtDllRedirectionLUTSize]
   cmp x16, x17
-  b.hi end
+  b.hs end
   adrp x17, NtDllRedirectionLUT
   ldr x17, [x17, #:lo12:NtDllRedirectionLUT]
   ldr w11, [x17, x16, lsl #2]

Source/Windows/ARM64EC/Module.cpp

@@ -542,17 +542,9 @@ static ARM64_NT_CONTEXT StoreStateToPackedECContext(FEXCore::Core::InternalThrea
   return ECContext;
 }
 
-static void RethrowGuestException(const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& Context) {
-  auto* Thread = GetCPUArea().ThreadState();
+static void RaiseException(FEXCore::Core::InternalThreadState* Thread, const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& GuestContext) {
   auto& Fault = Thread->CurrentFrame->SynchronousFaultData;
 
-  LogMan::Msg::DFmt("Reconstructing context");
-  if (!IsDispatcherAddress(Context.Pc)) {
-    ReconstructThreadState(Thread, Context);
-  }
-  ARM64_NT_CONTEXT GuestContext = StoreStateToPackedECContext(Thread, Context.Fpcr, Context.Fpsr);
-  LogMan::Msg::DFmt("pc: {:X} rip: {:X}", Context.Pc, GuestContext.Pc);
-
   // X64 Windows always clears TF, DF and AF when handling an exception, restoring after.
   // Current ARM64EC windows can only restore NZCV+SS when returning from an exception and other flags are left untouched from the handler context.
   // TODO: Can extend wine to support this by mapping the remaining EFlags into reserved cpsr members.
@@ -572,6 +564,40 @@ static void RethrowGuestException(const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT&
   NtRaiseExceptionNative(&GuestRec, &GuestContext, FirstChance);
 }
 
+static void RethrowGuestException(FEXCore::Core::InternalThreadState* Thread, const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& HostContext) {
+  LogMan::Msg::DFmt("Reconstructing context");
+  if (!IsDispatcherAddress(HostContext.Pc)) {
+    ReconstructThreadState(Thread, HostContext);
+  }
+  auto GuestContext = StoreStateToPackedECContext(Thread, HostContext.Fpcr, HostContext.Fpsr);
+  LogMan::Msg::DFmt("pc: {:X} rip: {:X}", HostContext.Pc, GuestContext.Pc);
+
+  RaiseException(Thread, Rec, GuestContext);
+}
+
+[[noreturn]]
+static void SynchronousExceptionHandler(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
+  // Coming in to this function, the guest state is synchronized and the host thread is in a safe location.
+  auto Thread = Frame->Thread;
+  auto& Fault = Thread->CurrentFrame->SynchronousFaultData;
+
+  uint64_t FPCR, FPSR;
+  __asm volatile("mrs %[fpcr], fpcr" : [fpcr] "=r"(FPCR));
+  __asm volatile("mrs %[fpsr], fpsr" : [fpsr] "=r"(FPSR));
+
+  auto Context = StoreStateToPackedECContext(Thread, static_cast<uint32_t>(FPCR), static_cast<uint32_t>(FPSR));
+  EXCEPTION_RECORD Record = {};
+  switch (Fault.Signal) {
+  case FEXCore::Core::FAULT_SIGILL: Record.ExceptionCode = EXCEPTION_ILLEGAL_INSTRUCTION; break;
+  case FEXCore::Core::FAULT_SIGTRAP: Record.ExceptionCode = EXCEPTION_BREAKPOINT; break;
+  case FEXCore::Core::FAULT_SIGSEGV: Record.ExceptionCode = EXCEPTION_ACCESS_VIOLATION; break;
+  default: ERROR_AND_DIE_FMT("Unknown fault signal"); break;
+  }
+
+  RaiseException(Thread, Record, Context);
+  ERROR_AND_DIE_FMT("This should always noreturn");
+}
+
 class ECSyscallHandler : public FEXCore::HLE::SyscallHandler, public FEXCore::Allocator::FEXAllocOperators {
 public:
   ECSyscallHandler() = default;
@@ -805,7 +831,7 @@ bool ResetToConsistentStateImpl(const ThreadCPUArea CPUArea, EXCEPTION_RECORD* E
   }
 
   if (IsEmulatorStackAddress(CPUArea, reinterpret_cast<uint64_t>(__builtin_frame_address(0)))) {
-    Exception::RethrowGuestException(*Exception, *NativeContext);
+    Exception::RethrowGuestException(Thread, *Exception, *NativeContext);
     LogMan::Msg::DFmt("Rethrowing onto guest stack: {:X}", NativeContext->Sp);
     return true;
   } else {
@@ -1038,6 +1064,7 @@ NTSTATUS ThreadInit() {
 
   FEX::Windows::CallRetStack::InitializeThread(Thread);
   Thread->CurrentFrame->Pointers.ExitFunctionEC = reinterpret_cast<uintptr_t>(&ExitFunctionEC);
+  Thread->CurrentFrame->Pointers.SynchronousExceptionHandler = reinterpret_cast<uintptr_t>(&Exception::SynchronousExceptionHandler);
   CPUArea.StateFrame() = Thread->CurrentFrame;
 
   uint64_t EnterEC = Thread->CurrentFrame->Pointers.DispatcherLoopTopEnterEC;

Source/Windows/Common/CPUFeatures.cpp

@@ -89,7 +89,11 @@ FEXCore::HostFeatures CPUFeatures::FetchHostFeatures(bool IsWine, FEXCore::HostF
   if (HostType == FEXCore::HostFeatures::HostTypeEnum::Wow64) {
     // AVX is unsupported for WOW64
     HostFeatures.SupportsAVX = false;
+  } else {
+    // arm64ec supports synchronous faults without a double-fault dance.
+    HostFeatures.SupportsFaultFreeSynchronousFaults = true;
   }
+
   return HostFeatures;
 }
 

ThunkLibs/Generator/gen.cpp

@@ -840,6 +840,8 @@ bool GenerateThunkLibsActionFactory::runInvocation(std::shared_ptr<clang::Compil
 
   const bool Success = Compiler.ExecuteAction(Action);
 
+#if LLVM_VERSION_MAJOR < 23
   Files->clearStatCache();
+#endif
   return Success;
 }

unittests/32Bit_ASM/CMakeLists.txt

@@ -80,7 +80,7 @@ foreach(ASM_SRC ${ASM_SOURCES})
     set(TEST_NAME "${TEST_DESC}/Test_32Bit_${REL_TEST_ASM}")
     string(REPLACE " " ";" FEX_ARGS_LIST ${FEX_ARGS})
     add_test(NAME ${TEST_NAME}
-      COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
+      COMMAND ${PYTHON_TEST_RUNNER} "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
       "${CMAKE_SOURCE_DIR}/unittests/32Bit_ASM/Known_Failures"
       "${CMAKE_SOURCE_DIR}/unittests/32Bit_ASM/Known_Failures_${TEST_TYPE}"
       "${CMAKE_SOURCE_DIR}/unittests/32Bit_ASM/Disabled_Tests"

unittests/ASM/CMakeLists.txt

@@ -87,7 +87,7 @@ foreach(ASM_SRC ${ASM_SOURCES})
     endif()
 
     add_test(NAME ${TEST_NAME}
-      COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
+      COMMAND ${PYTHON_TEST_RUNNER} "${CMAKE_SOURCE_DIR}/Scripts/testharness_runner.py"
       "${CMAKE_SOURCE_DIR}/unittests/ASM/Known_Failures"
       "${CMAKE_SOURCE_DIR}/unittests/ASM/Known_Failures_${TEST_TYPE}"
       "${CMAKE_SOURCE_DIR}/unittests/ASM/Disabled_Tests"

unittests/ASM/PrimaryGroup/2_C0_07_3.asm

@@ -0,0 +1,20 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x414243444546F848"
+  }
+}
+%endif
+
+mov rdx, 0xe0000000
+
+mov rax, 0x414243444546C348
+mov [rdx + 8 * 0], rax
+mov rax, 0x5152535455565758
+mov [rdx + 8 * 1], rax
+
+sar byte [rdx + 8 * 0 + 1], 3
+
+mov rax, [rdx + 8 * 0]
+
+hlt

unittests/ASM/PrimaryGroup/2_C0_07_4.asm

@@ -0,0 +1,20 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x414243444546FF48"
+  }
+}
+%endif
+
+mov rdx, 0xe0000000
+
+mov rax, 0x4142434445468148
+mov [rdx + 8 * 0], rax
+mov rax, 0x5152535455565758
+mov [rdx + 8 * 1], rax
+
+sar byte [rdx + 8 * 0 + 1], 12
+
+mov rax, [rdx + 8 * 0]
+
+hlt

unittests/ASM/PrimaryGroup/2_C1_07_2.asm

@@ -0,0 +1,20 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x41424344FE2A4748"
+  }
+}
+%endif
+
+mov rdx, 0xe0000000
+
+mov rax, 0x41424344C5464748
+mov [rdx + 8 * 0], rax
+mov rax, 0x5152535455565758
+mov [rdx + 8 * 1], rax
+
+sar word [rdx + 8 * 0 + 2], 5
+
+mov rax, [rdx + 8 * 0]
+
+hlt

unittests/ASM/PrimaryGroup/2_D0_07_2.asm

@@ -0,0 +1,20 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x414243444546D048"
+  }
+}
+%endif
+
+mov rdx, 0xe0000000
+
+mov rax, 0x414243444546A048
+mov [rdx + 8 * 0], rax
+mov rax, 0x5152535455565758
+mov [rdx + 8 * 1], rax
+
+sar byte [rdx + 8 * 0 + 1], 1
+
+mov rax, [rdx + 8 * 0]
+
+hlt

unittests/CMakeLists.txt

@@ -1,3 +1,8 @@
+# -S skips importing the "site" package which cuts down on interpreter startup time.
+# This means that these scripts can only use the python standard library, since "site"
+# is what resolves third-party packages from pip or apt.
+set(PYTHON_TEST_RUNNER "python3" "-S")
+
 if (NOT MINGW)
   add_subdirectory(APITests/)
   add_subdirectory(POSIX/)

unittests/FEXLinuxTests/CMakeLists.txt

@@ -47,7 +47,7 @@ function(AddTests Tests BinDirectory Bitness)
 
     # Add jit test case
     add_test(NAME "${TEST_CASE}.jit.flt"
-      COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
+      COMMAND ${PYTHON_TEST_RUNNER} "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
       "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures"
       "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
       "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests"
@@ -76,7 +76,7 @@ function(AddTests Tests BinDirectory Bitness)
     if (ARCHITECTURE_x86_64 AND NOT TEST_NAME STREQUAL "thunk_testlib" AND NOT TEST_NAME STREQUAL "vulkan_procaddr")
       # Add host test case
       add_test(NAME "${TEST_CASE}.host.flt"
-        COMMAND "python3" "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
+        COMMAND ${PYTHON_TEST_RUNNER} "${CMAKE_SOURCE_DIR}/Scripts/guest_test_runner.py"
         "${CMAKE_CURRENT_SOURCE_DIR}/Known_Failures_Host"
         "${CMAKE_CURRENT_SOURCE_DIR}/Expected_Output"
         "${CMAKE_CURRENT_SOURCE_DIR}/Disabled_Tests_Host"

unittests/FEXLinuxTests/tests/CMakeLists.txt

@@ -33,6 +33,8 @@ endforeach()
 
 target_link_libraries(pthread_cancel.${BITNESS} PRIVATE pthread)
 
+target_link_libraries(rlimit_data_pthread.${BITNESS} PRIVATE pthread)
+
 target_link_options(smc-1-dynamic.${BITNESS} PRIVATE -z execstack)
 
 target_link_libraries(smc-mt-1.${BITNESS} PRIVATE pthread)

unittests/FEXLinuxTests/tests/signal/sigtest_resethandler.cpp

@@ -0,0 +1,184 @@
+#include <catch2/catch_test_macros.hpp>
+
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/wait.h>
+#include <thread>
+
+void sig_handler(int signum, siginfo_t* info, void* context) {
+  printf("Inside handler function\n");
+}
+
+TEST_CASE("Signals: Check waitpid works") {
+  int pid = ::fork();
+  if (pid == 0) {
+    _exit(0);
+  }
+
+  int status {};
+  auto res = ::waitpid(pid, &status, 0);
+  REQUIRE(res != -1);
+  CHECK(WIFEXITED(status));
+  CHECK(WEXITSTATUS(status) == 0);
+}
+
+TEST_CASE("Signals: Check terminate works.") {
+  int pid = ::fork();
+  if (pid == 0) {
+    // Reset Catch2's signal handler for this fork.
+    struct sigaction act = {};
+    act.sa_flags = 0;
+    act.sa_handler = SIG_DFL;
+    REQUIRE(sigaction(SIGTERM, &act, nullptr) == 0);
+
+    ::tgkill(::getpid(), ::gettid(), SIGTERM);
+  }
+
+  int status {};
+  auto res = ::waitpid(pid, &status, 0);
+  REQUIRE(res != -1);
+  CHECK(!WIFEXITED(status));
+  CHECK(WIFSIGNALED(status));
+  CHECK(WTERMSIG(status) == SIGTERM);
+}
+
+TEST_CASE("Signals: Check flags.") {
+  constexpr static uint32_t SIGNAL_FLAGS = SA_SIGINFO | SA_NODEFER;
+
+  int pid = ::fork();
+  if (pid == 0) {
+    static struct sigaction act = {};
+    auto sig_handler = [](int signum, siginfo_t* info, void* context) -> void {
+      REQUIRE(sigaction(SIGTERM, nullptr, &act) == 0);
+    };
+
+    // Reset Catch2's signal handler for this fork.
+    act.sa_flags = SIGNAL_FLAGS;
+    act.sa_sigaction = sig_handler;
+    REQUIRE(sigaction(SIGTERM, &act, nullptr) == 0);
+
+    ::tgkill(::getpid(), ::gettid(), SIGTERM);
+
+#define SA_RESTORER 0x04000000
+    if ((act.sa_flags & ~SA_RESTORER) != SIGNAL_FLAGS) {
+      _exit(1);
+    }
+
+    if (act.sa_sigaction != sig_handler) {
+      _exit(1);
+    }
+
+    _exit(0);
+  }
+
+  int status {};
+  auto res = ::waitpid(pid, &status, 0);
+  REQUIRE(res != -1);
+  CHECK(WIFEXITED(status));
+  CHECK(WEXITSTATUS(status) == 0);
+}
+
+TEST_CASE("Signals: Check flags reset handler.") {
+  constexpr static uint32_t SIGNAL_FLAGS = SA_SIGINFO | SA_NODEFER | SA_RESETHAND;
+
+  int pid = ::fork();
+  if (pid == 0) {
+    static struct sigaction act = {};
+    auto sig_handler = [](int signum, siginfo_t* info, void* context) -> void {
+      REQUIRE(sigaction(SIGTERM, nullptr, &act) == 0);
+    };
+
+    // Reset Catch2's signal handler for this fork.
+    act.sa_flags = SIGNAL_FLAGS;
+    act.sa_sigaction = sig_handler;
+    REQUIRE(sigaction(SIGTERM, &act, nullptr) == 0);
+
+    ::tgkill(::getpid(), ::gettid(), SIGTERM);
+
+#define SA_RESTORER 0x04000000
+    // With SA_RESETHAND, sa_handler resets to SIG_DFL but everything else stays the same.
+    REQUIRE((act.sa_flags & ~SA_RESTORER) == SIGNAL_FLAGS);
+    REQUIRE(act.sa_handler == SIG_DFL);
+    _exit(0);
+  }
+
+  int status {};
+  auto res = ::waitpid(pid, &status, 0);
+  REQUIRE(res != -1);
+  CHECK(WIFEXITED(status));
+  CHECK(WEXITSTATUS(status) == 0);
+}
+
+TEST_CASE("Signals: Check flags reset handler ignore.") {
+  // SIGWINCH by default is type SIG_DFL but that means SIG_IGN in the kernel.
+  // Ensure SA_RESETHAND resets it to SIG_DFL, not SIG_IGN.
+  constexpr static uint32_t SIGNAL_FLAGS = SA_SIGINFO | SA_NODEFER | SA_RESETHAND;
+
+  int pid = ::fork();
+  if (pid == 0) {
+    static struct sigaction act = {};
+    auto sig_handler = [](int signum, siginfo_t* info, void* context) -> void {
+      REQUIRE(sigaction(SIGWINCH, nullptr, &act) == 0);
+    };
+
+    // Reset Catch2's signal handler for this fork.
+    act.sa_flags = SIGNAL_FLAGS;
+    act.sa_sigaction = sig_handler;
+    REQUIRE(sigaction(SIGWINCH, &act, nullptr) == 0);
+
+    ::tgkill(::getpid(), ::gettid(), SIGWINCH);
+
+#define SA_RESTORER 0x04000000
+    // With SA_RESETHAND, sa_handler resets to SIG_DFL but everything else stays the same.
+    REQUIRE((act.sa_flags & ~SA_RESTORER) == SIGNAL_FLAGS);
+    REQUIRE(act.sa_handler == SIG_DFL);
+    _exit(0);
+  }
+
+  int status {};
+  auto res = ::waitpid(pid, &status, 0);
+  REQUIRE(res != -1);
+  CHECK(WIFEXITED(status));
+  CHECK(WEXITSTATUS(status) == 0);
+}
+
+TEST_CASE("Signals: Check flags reset handler thread.") {
+  constexpr static uint32_t SIGNAL_FLAGS = SA_SIGINFO | SA_NODEFER | SA_RESETHAND;
+
+  static struct sigaction act = {};
+  static struct sigaction orig_act = {};
+
+  auto sig_handler = [](int signum, siginfo_t* info, void* context) -> void {
+    REQUIRE(sigaction(SIGTERM, nullptr, &act) == 0);
+  };
+
+  // Reset Catch2's signal handler for this fork.
+  act.sa_flags = SIGNAL_FLAGS;
+  act.sa_sigaction = sig_handler;
+  REQUIRE(sigaction(SIGTERM, &act, &orig_act) == 0);
+
+  std::thread t([]() {
+    ::tgkill(::getpid(), ::gettid(), SIGTERM);
+
+#define SA_RESTORER 0x04000000
+    // With SA_RESETHAND, sa_handler resets to SIG_DFL but everything else stays the same.
+    REQUIRE((act.sa_flags & ~SA_RESTORER) == SIGNAL_FLAGS);
+    REQUIRE(act.sa_handler == SIG_DFL);
+  });
+
+  // DFL switch should be visible from parent thread.
+  static struct sigaction act_parent = {};
+  while (sigaction(SIGTERM, nullptr, &act_parent) == 0) {
+    if (act_parent.sa_handler == SIG_DFL) {
+      fprintf(stderr, "Hit DFL!\n");
+      break;
+    }
+  }
+
+  t.join();
+
+  // Set the original handler back for any remaining tests.
+  REQUIRE(sigaction(SIGTERM, &orig_act, nullptr) == 0);
+}

unittests/FEXLinuxTests/tests/syscalls/rlimit_data_pthread.cpp

@@ -0,0 +1,42 @@
+#include <catch2/catch_test_macros.hpp>
+
+#include <atomic>
+#include <pthread.h>
+#include <sys/resource.h>
+#include <unistd.h>
+
+static void* ThreadFunc(void* Arg) {
+  auto Semaphore = static_cast<std::atomic<bool>*>(Arg);
+  while (*Semaphore) {
+    // Sleep for a bit to avoid thrashing.
+    usleep(100'000);
+  }
+  return nullptr;
+}
+
+// NB: At present, empirical testing shows that we can
+// actually set this to ~45, and still pass. To give us wiggle room for
+// any other per thread overhead let's set it to a nice round number instead.
+#define NUM_THREADS 32
+
+// Chromium's sandbox lowers RLIMIT_DATA to 16GB and then spawns threads.
+// Historically, FEX would allocate a several TB writtable region
+// for its host stack, which would blow past this limit.
+TEST_CASE("pthread_create after lowering RLIMIT_DATA") {
+  rlimit64 Limit {
+    .rlim_cur = 16ULL * 1024 * 1024 * 1024,
+    .rlim_max = 16ULL * 1024 * 1024 * 1024,
+  };
+  REQUIRE(setrlimit64(RLIMIT_DATA, &Limit) == 0);
+
+  pthread_t Threads[NUM_THREADS];
+  std::atomic<bool> Semaphore = true;
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    REQUIRE(pthread_create(&Threads[i], nullptr, ThreadFunc, &Semaphore) == 0);
+  }
+
+  Semaphore = false;
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    CHECK(pthread_join(Threads[i], nullptr) == 0);
+  }
+}

unittests/FEXLinuxTests/tests/syscalls/stat_efault.32.cpp

@@ -0,0 +1,49 @@
+#include <catch2/catch_test_macros.hpp>
+
+#include <cerrno>
+#include <fcntl.h>
+#include <sys/statfs.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#define CHECK_EFAULT(Expr)  \
+  do {                      \
+    REQUIRE((Expr) == -1);  \
+    CHECK(errno == EFAULT); \
+  } while (0)
+
+#define CHECK_ENOENT(Expr)  \
+  do {                      \
+    REQUIRE((Expr) == -1);  \
+    CHECK(errno == ENOENT); \
+  } while (0)
+
+TEST_CASE("stat et al. returns EFAULT for null pointers") {
+  int FD = ::open("/", O_RDONLY);
+  CHECK_EFAULT(::syscall(SYS_oldstat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_oldfstat, FD, nullptr));
+  CHECK_EFAULT(::syscall(SYS_oldlstat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_stat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstat, FD, nullptr));
+  CHECK_EFAULT(::syscall(SYS_lstat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_stat64, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_lstat64, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstat64, FD, nullptr));
+  CHECK_EFAULT(::syscall(SYS_statfs, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstatfs, FD, nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstatfs64, FD, sizeof(struct statfs64), nullptr));
+  CHECK_EFAULT(::syscall(SYS_statfs64, "/", sizeof(struct statfs64), nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstatat64, AT_FDCWD, "/", nullptr, 0));
+}
+
+TEST_CASE("stat et al. returns ENOENT for missing paths with null buffers") {
+  CHECK_ENOENT(::syscall(SYS_oldstat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_oldlstat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_stat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_lstat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_stat64, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_lstat64, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_statfs, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_statfs64, "/does/not/exist", sizeof(struct statfs64), nullptr));
+  CHECK_ENOENT(::syscall(SYS_fstatat64, AT_FDCWD, "/does/not/exist", nullptr, 0));
+}

unittests/FEXLinuxTests/tests/syscalls/stat_efault.64.cpp

@@ -0,0 +1,41 @@
+#include <catch2/catch_test_macros.hpp>
+
+#include <cerrno>
+#include <fcntl.h>
+#include <sys/syscall.h>
+#include <unistd.h>
+
+#define CHECK_EFAULT(Expr)  \
+  do {                      \
+    REQUIRE((Expr) == -1);  \
+    CHECK(errno == EFAULT); \
+  } while (0)
+
+#define CHECK_ENOENT(Expr)  \
+  do {                      \
+    REQUIRE((Expr) == -1);  \
+    CHECK(errno == ENOENT); \
+  } while (0)
+
+// This should be more than big enough
+static char Buffer[2048];
+
+TEST_CASE("stat et al. returns EFAULT for null pointers") {
+  int FD = ::open("/", O_RDONLY);
+  CHECK_EFAULT(::syscall(SYS_stat, nullptr, Buffer));
+  CHECK_EFAULT(::syscall(SYS_stat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_fstat, FD, nullptr));
+  CHECK_EFAULT(::syscall(SYS_lstat, nullptr, Buffer));
+  CHECK_EFAULT(::syscall(SYS_lstat, "/", nullptr));
+  CHECK_EFAULT(::syscall(SYS_newfstatat, AT_FDCWD, nullptr, Buffer, 0));
+  CHECK_EFAULT(::syscall(SYS_newfstatat, AT_FDCWD, "/", nullptr, 0));
+  CHECK_EFAULT(::syscall(SYS_statfs, nullptr, Buffer));
+  CHECK_EFAULT(::syscall(SYS_statfs, "/", nullptr));
+}
+
+TEST_CASE("stat et al. returns ENOENT for missing paths with null buffers") {
+  CHECK_ENOENT(::syscall(SYS_stat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_lstat, "/does/not/exist", nullptr));
+  CHECK_ENOENT(::syscall(SYS_newfstatat, AT_FDCWD, "/does/not/exist", nullptr, 0));
+  CHECK_ENOENT(::syscall(SYS_statfs, "/does/not/exist", nullptr));
+}

unittests/InstructionCountCI/AFP/H0F3A.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "roundss xmm0, xmm1, 00000000b": {

unittests/InstructionCountCI/AFP/SVE256/Secondary.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtpi2ps xmm0, [rax]": {

unittests/InstructionCountCI/AFP/SVE256/Secondary_REP.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "RPRES"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtsi2ss xmm0, eax": {

unittests/InstructionCountCI/AFP/SVE256/Secondary_REPNE.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtsi2sd xmm0, eax": {

unittests/InstructionCountCI/AFP/Secondary.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtpi2ps xmm0, [rax]": {

unittests/InstructionCountCI/AFP/Secondary_REP.json

@@ -9,7 +9,7 @@
       "SVE256",
       "RPRES"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtsi2ss xmm0, eax": {

unittests/InstructionCountCI/AFP/Secondary_REPNE.json

@@ -8,7 +8,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "cvtsi2sd xmm0, eax": {

unittests/InstructionCountCI/AFP/VEX_map1.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "RPRES"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vsqrtss xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AFP/VEX_map3.json

@@ -7,7 +7,7 @@
       "AFP"
     ],
     "DisabledHostFeatures": [],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vroundss xmm0, xmm1, 00000000b": {

unittests/InstructionCountCI/AVX128/FMA4.json

@@ -7,7 +7,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vfmaddsubps xmm0, xmm1, xmm2, xmm3": {
@@ -20,7 +20,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -34,7 +34,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -48,7 +48,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -62,7 +62,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -76,7 +76,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -90,7 +90,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -104,7 +104,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -118,7 +118,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -132,7 +132,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -146,7 +146,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -160,7 +160,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -174,7 +174,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -188,7 +188,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -202,7 +202,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -216,7 +216,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -230,7 +230,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -244,7 +244,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -258,7 +258,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -272,7 +272,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -286,7 +286,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -300,7 +300,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -314,7 +314,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -328,7 +328,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -342,7 +342,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -356,7 +356,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -370,7 +370,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -384,7 +384,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -398,7 +398,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -412,7 +412,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     },
@@ -426,7 +426,7 @@
         "str x20, [x28, #24]",
         "mov w0, #0x401",
         "str x0, [x28, #1496]",
-        "ldr x0, [x28, #3000]",
+        "ldr x0, [x28, #3008]",
         "br x0"
       ]
     }

unittests/InstructionCountCI/AVX128/VEX_map1.json

@@ -13,7 +13,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vmovups xmm0, xmm0": {
@@ -486,7 +486,7 @@
       ],
       "ExpectedArm64ASM": [
         "ushr v2.4s, v16.4s, #31",
-        "ldr q3, [x28, #3328]",
+        "ldr q3, [x28, #3344]",
         "ushl v2.4s, v2.4s, v3.4s",
         "addv s2, v2.4s",
         "mov w4, v2.s[0]"
@@ -501,7 +501,7 @@
         "ldr q2, [x28, #192]",
         "uzp2 v2.8h, v16.8h, v2.8h",
         "ushr v2.8h, v2.8h, #15",
-        "ldr q3, [x28, #3152]",
+        "ldr q3, [x28, #3168]",
         "ushl v2.8h, v2.8h, v3.8h",
         "addv h2, v2.8h",
         "umov w4, v2.h[0]"
@@ -528,7 +528,7 @@
         "ldr q2, [x28, #192]",
         "uzp2 v2.4s, v16.4s, v2.4s",
         "ushr v2.4s, v2.4s, #31",
-        "ldr q3, [x28, #3328]",
+        "ldr q3, [x28, #3344]",
         "ushl v2.4s, v2.4s, v3.4s",
         "addv s2, v2.4s",
         "mov w4, v2.s[0]"
@@ -1116,7 +1116,7 @@
         "Map 1 0b01 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q2, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1128,7 +1128,7 @@
         "Map 1 0b01 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q2, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1140,7 +1140,7 @@
         "Map 1 0b01 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q2, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1165,7 +1165,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q3, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1179,7 +1179,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q3, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1193,7 +1193,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q3, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1217,7 +1217,7 @@
         "Map 1 0b10 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q2, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1229,7 +1229,7 @@
         "Map 1 0b10 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q2, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1241,7 +1241,7 @@
         "Map 1 0b10 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q2, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1268,7 +1268,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q3, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1282,7 +1282,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q3, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1296,7 +1296,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2744]",
+        "ldr x0, [x28, #2752]",
         "ldr q3, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1320,7 +1320,7 @@
         "Map 1 0b11 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q2, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1332,7 +1332,7 @@
         "Map 1 0b11 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q2, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1344,7 +1344,7 @@
         "Map 1 0b11 0x70 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q2, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -1371,7 +1371,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q3, [x0, #16]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1385,7 +1385,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q3, [x0, #32]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -1399,7 +1399,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
-        "ldr x0, [x28, #2736]",
+        "ldr x0, [x28, #2744]",
         "ldr q3, [x0, #48]",
         "tbl v16.16b, {v17.16b}, v3.16b",
         "tbl v2.16b, {v2.16b}, v3.16b",
@@ -2283,7 +2283,7 @@
         "Map 1 0b00 0xC6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q2, [x0, #16]",
         "tbl v16.16b, {v17.16b, v18.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -2297,7 +2297,7 @@
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
         "ldr q3, [x28, #224]",
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q4, [x0, #16]",
         "tbl v16.16b, {v17.16b, v18.16b}, v4.16b",
         "tbl v2.16b, {v2.16b, v3.16b}, v4.16b",
@@ -2310,7 +2310,7 @@
         "Map 1 0b00 0xC6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q2, [x0, #32]",
         "tbl v16.16b, {v17.16b, v18.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -2324,7 +2324,7 @@
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
         "ldr q3, [x28, #224]",
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q4, [x0, #32]",
         "tbl v16.16b, {v17.16b, v18.16b}, v4.16b",
         "tbl v2.16b, {v2.16b, v3.16b}, v4.16b",
@@ -2337,7 +2337,7 @@
         "Map 1 0b00 0xC6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q2, [x0, #48]",
         "tbl v16.16b, {v17.16b, v18.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
@@ -2351,7 +2351,7 @@
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
         "ldr q3, [x28, #224]",
-        "ldr x0, [x28, #2760]",
+        "ldr x0, [x28, #2768]",
         "ldr q4, [x0, #48]",
         "tbl v16.16b, {v17.16b, v18.16b}, v4.16b",
         "tbl v2.16b, {v2.16b, v3.16b}, v4.16b",
@@ -3936,7 +3936,7 @@
         "Map 1 0b01 0xd0 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v18.16b, v2.16b",
         "fadd v16.2d, v17.2d, v2.2d",
         "stp xzr, xzr, [x28, #192]"
@@ -3950,7 +3950,7 @@
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
         "ldr q3, [x28, #224]",
-        "ldr q4, [x28, #3232]",
+        "ldr q4, [x28, #3248]",
         "eor v5.16b, v18.16b, v4.16b",
         "fadd v16.2d, v17.2d, v5.2d",
         "eor v3.16b, v3.16b, v4.16b",
@@ -3964,7 +3964,7 @@
         "Map 1 0b11 0xd0 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v18.16b, v2.16b",
         "fadd v16.4s, v17.4s, v2.4s",
         "stp xzr, xzr, [x28, #192]"
@@ -3978,7 +3978,7 @@
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #208]",
         "ldr q3, [x28, #224]",
-        "ldr q4, [x28, #3200]",
+        "ldr q4, [x28, #3216]",
         "eor v5.16b, v18.16b, v4.16b",
         "fadd v16.4s, v17.4s, v5.4s",
         "eor v3.16b, v3.16b, v4.16b",
@@ -4149,7 +4149,7 @@
         "Map 1 0b01 0xd7 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3456]",
+        "ldr q2, [x28, #3472]",
         "cmlt v3.16b, v16.16b, #0",
         "and v2.16b, v3.16b, v2.16b",
         "addp v2.16b, v2.16b, v2.16b",
@@ -4165,7 +4165,7 @@
       ],
       "ExpectedArm64ASM": [
         "ldr q2, [x28, #192]",
-        "ldr q3, [x28, #3456]",
+        "ldr q3, [x28, #3472]",
         "cmlt v4.16b, v16.16b, #0",
         "and v4.16b, v4.16b, v3.16b",
         "addp v4.16b, v4.16b, v4.16b",

unittests/InstructionCountCI/AVX128/VEX_map1_FCMA.json

@@ -9,7 +9,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vaddsubpd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map1_SVE128.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vmovntps [rax], xmm0": {

unittests/InstructionCountCI/AVX128/VEX_map1_flagm.json

@@ -10,7 +10,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vucomiss xmm0, xmm1": {

unittests/InstructionCountCI/AVX128/VEX_map2.json

@@ -11,7 +11,7 @@
       "I8MM",
       "DOTPROD"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vpshufb xmm0, xmm1, xmm2": {
@@ -1902,7 +1902,7 @@
         "Map 2 0b01 0x41 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3152]",
+        "ldr q2, [x28, #3168]",
         "zip1 v3.8h, v2.8h, v17.8h",
         "zip2 v2.8h, v2.8h, v17.8h",
         "umin v2.4s, v3.4s, v2.4s",
@@ -4695,7 +4695,7 @@
         "Map 2 0b01 0x96 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -4712,7 +4712,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -4728,7 +4728,7 @@
         "Map 2 0b01 0x96 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -4745,7 +4745,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -4761,7 +4761,7 @@
         "Map 2 0b01 0x97 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -4778,7 +4778,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -4794,7 +4794,7 @@
         "Map 2 0b01 0x97 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -4811,7 +4811,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -5751,7 +5751,7 @@
         "Map 2 0b01 0xa6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -5768,7 +5768,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -5784,7 +5784,7 @@
         "Map 2 0b01 0xa6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -5801,7 +5801,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -5817,7 +5817,7 @@
         "Map 2 0b01 0xa7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -5834,7 +5834,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -5850,7 +5850,7 @@
         "Map 2 0b01 0xa7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -5867,7 +5867,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -5883,7 +5883,7 @@
         "Map 2 0b01 0xb6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -5899,7 +5899,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -5914,7 +5914,7 @@
         "Map 2 0b01 0xb6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -5930,7 +5930,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -5945,7 +5945,7 @@
         "Map 2 0b01 0xb7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -5961,7 +5961,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -5976,7 +5976,7 @@
         "Map 2 0b01 0xb7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -5992,7 +5992,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.2d, v17.2d, v18.2d",

unittests/InstructionCountCI/AVX128/VEX_map2_AFP.json

@@ -8,7 +8,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vfmadd132ss xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_DOTPROD.json

@@ -12,7 +12,7 @@
       "SVE256",
       "I8MM"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vpdpbusd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_I8MM.json

@@ -11,7 +11,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vpdpbusd xmm0, xmm1, xmm2": {

unittests/InstructionCountCI/AVX128/VEX_map2_SVE128.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vmovntdqa xmm0, [rax]": {
@@ -2844,7 +2844,7 @@
         "Map 2 0b01 0x96 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -2861,7 +2861,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -2877,7 +2877,7 @@
         "Map 2 0b01 0x96 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -2894,7 +2894,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -2910,7 +2910,7 @@
         "Map 2 0b01 0x97 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -2927,7 +2927,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v16.4s, v18.4s",
@@ -2943,7 +2943,7 @@
         "Map 2 0b01 0x97 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v17.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -2960,7 +2960,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v17.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v16.2d, v18.2d",
@@ -3880,7 +3880,7 @@
         "Map 2 0b01 0xa6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -3897,7 +3897,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -3913,7 +3913,7 @@
         "Map 2 0b01 0xa6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -3930,7 +3930,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -3946,7 +3946,7 @@
         "Map 2 0b01 0xa7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -3963,7 +3963,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.4s, v17.4s, v16.4s",
@@ -3979,7 +3979,7 @@
         "Map 2 0b01 0xa7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v18.16b, v2.16b",
         "mov v0.16b, v2.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -3996,7 +3996,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v18.16b, v5.16b",
         "mov v0.16b, v6.16b",
         "fmla v0.2d, v17.2d, v16.2d",
@@ -4012,7 +4012,7 @@
         "Map 2 0b01 0xb6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3200]",
+        "ldr q2, [x28, #3216]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -4028,7 +4028,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3200]",
+        "ldr q5, [x28, #3216]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -4043,7 +4043,7 @@
         "Map 2 0b01 0xb6 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3232]",
+        "ldr q2, [x28, #3248]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -4059,7 +4059,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3232]",
+        "ldr q5, [x28, #3248]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -4074,7 +4074,7 @@
         "Map 2 0b01 0xb7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3264]",
+        "ldr q2, [x28, #3280]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -4090,7 +4090,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3264]",
+        "ldr q5, [x28, #3280]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.4s, v17.4s, v18.4s",
@@ -4105,7 +4105,7 @@
         "Map 2 0b01 0xb7 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3296]",
+        "ldr q2, [x28, #3312]",
         "eor v2.16b, v16.16b, v2.16b",
         "mov v16.16b, v2.16b",
         "fmla v16.2d, v17.2d, v18.2d",
@@ -4121,7 +4121,7 @@
         "ldr q2, [x28, #192]",
         "ldr q3, [x28, #208]",
         "ldr q4, [x28, #224]",
-        "ldr q5, [x28, #3296]",
+        "ldr q5, [x28, #3312]",
         "eor v6.16b, v16.16b, v5.16b",
         "mov v16.16b, v6.16b",
         "fmla v16.2d, v17.2d, v18.2d",

unittests/InstructionCountCI/AVX128/VEX_map2_flagm.json

@@ -11,7 +11,7 @@
       "SVE256",
       "SVEBITPERM"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vtestps xmm0, xmm1": {

unittests/InstructionCountCI/AVX128/VEX_map3.json

@@ -7,7 +7,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vpermq ymm0, ymm1, 1": {
@@ -338,7 +338,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3360]",
+        "ldr q2, [x28, #3376]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -349,7 +349,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3376]",
+        "ldr q2, [x28, #3392]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -370,7 +370,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3392]",
+        "ldr q2, [x28, #3408]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -392,7 +392,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3408]",
+        "ldr q2, [x28, #3424]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -413,7 +413,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3424]",
+        "ldr q2, [x28, #3440]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -424,7 +424,7 @@
         "Map 3 0b01 0x02 128-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3440]",
+        "ldr q2, [x28, #3456]",
         "tbx v16.16b, {v17.16b}, v2.16b",
         "stp xzr, xzr, [x28, #192]"
       ]
@@ -3494,7 +3494,7 @@
       ],
       "ExpectedArm64ASM": [
         "movi v2.2d, #0x0",
-        "ldr q3, [x28, #3344]",
+        "ldr q3, [x28, #3360]",
         "mov v16.16b, v17.16b",
         "aese v16.16b, v2.16b",
         "tbl v16.16b, {v16.16b}, v3.16b",
@@ -3508,7 +3508,7 @@
       ],
       "ExpectedArm64ASM": [
         "movi v2.2d, #0x0",
-        "ldr q3, [x28, #3344]",
+        "ldr q3, [x28, #3360]",
         "mov v16.16b, v17.16b",
         "aese v16.16b, v2.16b",
         "tbl v16.16b, {v16.16b}, v3.16b",

unittests/InstructionCountCI/AVX128/VEX_map3_SVE128.json

@@ -8,7 +8,7 @@
       "AFP",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vblendvps xmm0, xmm1, xmm2, xmm3": {

unittests/InstructionCountCI/AVX128/VEX_map_group.json

@@ -9,7 +9,7 @@
       "SVE256",
       "SVE128"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vpsrlw xmm0, xmm1, 0": {

unittests/InstructionCountCI/Atomics.json

@@ -9,7 +9,7 @@
       "FLAGM",
       "FLAGM2"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "lock add byte [rax], cl": {

unittests/InstructionCountCI/Crypto/H0F38.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "sha1nexte xmm0, xmm1": {
@@ -32,7 +32,7 @@
         "0x66 0x0f 0x38 0xca"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q2, [x0, #432]",
         "tbl v3.16b, {v16.16b}, v2.16b",
         "tbl v4.16b, {v17.16b}, v2.16b",

unittests/InstructionCountCI/Crypto/H0F38_SVE128.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "sha1nexte xmm0, xmm1": {

unittests/InstructionCountCI/Crypto/H0F3A.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "pclmulqdq xmm0, xmm1, 00000b": {
@@ -56,7 +56,7 @@
         "0x66 0x0f 0x3a 0xdf"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3344]",
+        "ldr q2, [x28, #3360]",
         "movi v3.2d, #0x0",
         "mov v16.16b, v17.16b",
         "aese v16.16b, v3.16b",
@@ -69,7 +69,7 @@
         "0x66 0x0f 0x3a 0xdf"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3344]",
+        "ldr q2, [x28, #3360]",
         "movi v3.2d, #0x0",
         "mov v16.16b, v17.16b",
         "aese v16.16b, v3.16b",
@@ -85,9 +85,9 @@
         "0x66 0x0f 0x3a 0xcc"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3760]",
+        "ldr q2, [x28, #3776]",
         "movi v3.2d, #0x0",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q4, [x0, #432]",
         "tbl v5.16b, {v16.16b}, v4.16b",
         "tbl v6.16b, {v17.16b}, v4.16b",
@@ -102,9 +102,9 @@
         "0x66 0x0f 0x3a 0xcc"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3776]",
+        "ldr q2, [x28, #3792]",
         "movi v3.2d, #0x0",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q4, [x0, #432]",
         "tbl v5.16b, {v16.16b}, v4.16b",
         "tbl v6.16b, {v17.16b}, v4.16b",
@@ -119,9 +119,9 @@
         "0x66 0x0f 0x3a 0xcc"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3792]",
+        "ldr q2, [x28, #3808]",
         "movi v3.2d, #0x0",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q4, [x0, #432]",
         "tbl v5.16b, {v16.16b}, v4.16b",
         "tbl v6.16b, {v17.16b}, v4.16b",
@@ -136,9 +136,9 @@
         "0x66 0x0f 0x3a 0xcc"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3808]",
+        "ldr q2, [x28, #3824]",
         "movi v3.2d, #0x0",
-        "ldr x0, [x28, #2752]",
+        "ldr x0, [x28, #2760]",
         "ldr q4, [x0, #432]",
         "tbl v5.16b, {v16.16b}, v4.16b",
         "tbl v6.16b, {v17.16b}, v4.16b",

unittests/InstructionCountCI/DDD.json

@@ -10,7 +10,7 @@
       "AFP",
       "RPRES"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These 3DNow! instructions are optimal assuming that FEX doesn't SRA MMX registers",

unittests/InstructionCountCI/FEXOpt/AddressingLimitations.json

@@ -8,7 +8,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "Instructions that explicitly push against the limits of ARM's loadstore instructions"

unittests/InstructionCountCI/FEXOpt/AddressingLimitations_32Bit.json

@@ -8,7 +8,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "Instructions that explicitly push against the limits of ARM's loadstore instructions"

unittests/InstructionCountCI/FEXOpt/MultiInst.json

@@ -13,7 +13,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_32bit.json

@@ -11,7 +11,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_AFP.json

@@ -9,7 +9,7 @@
       "SVE256",
       "RPRES"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_TSO.json

@@ -14,7 +14,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/MultiInst_TSO_32bit.json

@@ -14,7 +14,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "These are instruction combinations that could be more optimal if FEX optimized for them"

unittests/InstructionCountCI/FEXOpt/Peephole.json

@@ -11,7 +11,7 @@
       "I8MM",
       "DOTPROD"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "This file tracks the translations for the 'peephole' optimizations in FEX.",

unittests/InstructionCountCI/FEXOpt/Peephole_SVE128.json

@@ -10,7 +10,7 @@
       "FLAGM2",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [
     "This file tracks the translations for the 'peephole' optimizations in FEX.",

unittests/InstructionCountCI/FEXOpt/libnss.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Comment": [],
   "Instructions": {

unittests/InstructionCountCI/FlagM/Atomics.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "lock add byte [rax], cl": {

unittests/InstructionCountCI/FlagM/FlagOpts.json

@@ -9,7 +9,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "Chained add": {

unittests/InstructionCountCI/FlagM/H0F38.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "ptest xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/HotBlocks.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "The Witcher 3": {
@@ -892,7 +892,7 @@
         "Comes from vcruntime140.dll wcsstr"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3152]",
+        "ldr q2, [x28, #3168]",
         "cmeq v3.8h, v16.8h, #0",
         "movi v4.8h, #0x8",
         "bsl v3.16b, v2.16b, v4.16b",

unittests/InstructionCountCI/FlagM/HotBlocks_32Bit.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "Sonic Mania movie player": {

unittests/InstructionCountCI/FlagM/HotBlocks_AFP.json

@@ -10,7 +10,7 @@
       "SVE128",
       "SVE256"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "FMOD scalar loop": {

unittests/InstructionCountCI/FlagM/HotBlocks_TSO_32Bit.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "The Sims 1 hot block": {

unittests/InstructionCountCI/FlagM/Primary.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "add bl, cl": {

unittests/InstructionCountCI/FlagM/PrimaryGroup.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "add al, 1": {

unittests/InstructionCountCI/FlagM/Primary_32Bit.json

@@ -10,7 +10,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "push es": {

unittests/InstructionCountCI/FlagM/Secondary.json

@@ -11,7 +11,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "ucomiss xmm0, xmm1": {
@@ -1610,7 +1610,7 @@
         "mov w20, #0xffff",
         "strb w20, [x28, #1202]",
         "ldr d2, [x28, #1056]",
-        "ldr d3, [x28, #3456]",
+        "ldr d3, [x28, #3472]",
         "cmlt v2.16b, v2.16b, #0",
         "and v2.8b, v2.8b, v3.8b",
         "addp v2.16b, v2.16b, v2.16b",

unittests/InstructionCountCI/FlagM/SecondaryGroup.json

@@ -11,7 +11,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "sgdt [rax]": {

unittests/InstructionCountCI/FlagM/SecondaryModRM.json

@@ -11,7 +11,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "xgetbv": {

unittests/InstructionCountCI/FlagM/Secondary_OpSize.json

@@ -11,7 +11,7 @@
       "AFP",
       "FCMA"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "ucomisd xmm0, xmm1": {
@@ -38,7 +38,7 @@
       "ExpectedInstructionCount": 7,
       "Comment": "0x66 0x0f 0xd7",
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3456]",
+        "ldr q2, [x28, #3472]",
         "cmlt v3.16b, v16.16b, #0",
         "and v2.16b, v3.16b, v2.16b",
         "addp v2.16b, v2.16b, v2.16b",

unittests/InstructionCountCI/FlagM/Secondary_REP.json

@@ -12,7 +12,7 @@
       "AFP",
       "CSSC"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "popcnt ax, bx": {

unittests/InstructionCountCI/FlagM/Secondary_REP_CSSC.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "popcnt ax, bx": {

unittests/InstructionCountCI/FlagM/VEX_map1.json

@@ -12,7 +12,7 @@
       "RPRES",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vucomiss xmm0, xmm1": {
@@ -69,7 +69,7 @@
         "Map 1 0b01 0xd7 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr q2, [x28, #3456]",
+        "ldr q2, [x28, #3472]",
         "cmlt v3.16b, v16.16b, #0",
         "and v2.16b, v3.16b, v2.16b",
         "addp v2.16b, v2.16b, v2.16b",
@@ -84,7 +84,7 @@
         "Map 1 0b01 0xd7 256-bit"
       ],
       "ExpectedArm64ASM": [
-        "ldr x0, [x28, #2552]",
+        "ldr x0, [x28, #2560]",
         "ld1b {z2.b}, p7/z, [x0]",
         "mrs x0, nzcv",
         "mov z0.d, #0",

unittests/InstructionCountCI/FlagM/VEX_map2.json

@@ -10,7 +10,7 @@
       "AFP",
       "SVEBITPERM"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "vtestps xmm0, xmm1": {

unittests/InstructionCountCI/FlagM/VEX_map_group.json

@@ -9,7 +9,7 @@
     "DisabledHostFeatures": [
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "blsr eax, ebx": {

unittests/InstructionCountCI/FlagM/x87_f64-Crysis2Max-fmodel.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64-HalfLife.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "Block1": {

unittests/InstructionCountCI/FlagM/x87_f64-Oblivion.json

@@ -13,7 +13,7 @@
       "SVE256",
       "AFP"
     ],
-    "BinaryCacheVersion": 30
+    "BinaryCacheVersion": 31
   },
   "Instructions": {
     "Block1": {