Merge pull request #4985 from Sonicadvance1/fex-atomic

Support (downstream) kernel-side unaligned atomic handling (The rebase sequel)

FEXCore/Source/Interface/Config/Config.json.in

@@ -443,6 +443,13 @@
           "This is required to ensure a split-lock doesn't tear inside the process"
         ]
       },
+      "KernelUnalignedAtomicBackpatching": {
+        "Type": "bool",
+        "Default": "true",
+        "Desc": [
+          "When the kernel unaligned atomic handler is enabled, use backpatching to reduce kernel context switches."
+        ]
+      },
       "VolatileMetadata": {
         "Type": "bool",
         "Default": "true",

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -222,6 +222,26 @@ void CheckForGCS() {
 }
 } // namespace FEX::GCS
 
+namespace FEX::UnalignedAtomic {
+void SetupKernelUnalignedAtomics() {
+#ifndef PR_ARM64_SET_UNALIGN_ATOMIC
+#define PR_ARM64_SET_UNALIGN_ATOMIC 0x46455849
+#define PR_ARM64_UNALIGN_ATOMIC_EMULATE (1UL << 0)
+#define PR_ARM64_UNALIGN_ATOMIC_BACKPATCH (1UL << 1)
+#define PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1UL << 2)
+#endif
+
+  // Interfaces with downstream FEX kernel patches to control unaligned atomic handling
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
+  FEX_CONFIG_OPT(KernelUnalignedAtomicBackpatching, KERNELUNALIGNEDATOMICBACKPATCHING);
+
+  uint64_t Flags = (StrictInProcessSplitLocks() ? PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                   (KernelUnalignedAtomicBackpatching() ? PR_ARM64_UNALIGN_ATOMIC_BACKPATCH : 0) | PR_ARM64_UNALIGN_ATOMIC_EMULATE;
+
+  prctl(PR_ARM64_SET_UNALIGN_ATOMIC, Flags, 0, 0, 0);
+}
+} // namespace FEX::UnalignedAtomic
+
 /**
  * @brief Get an FD from an environment variable and then unset the environment variable.
  *
@@ -458,6 +478,7 @@ int main(int argc, char** argv, char** const envp) {
 
   // Setup TSO hardware emulation immediately after initializing the context.
   FEX::TSO::SetupTSOEmulation(CTX.get());
+  FEX::UnalignedAtomic::SetupKernelUnalignedAtomics();
 
   if (!Loader.Is64BitMode()) {
     // Tell the kernel we want to use the compat input syscalls even though we're

Source/Windows/ARM64EC/Module.cpp

@@ -640,7 +640,6 @@ NTSTATUS ProcessInit() {
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<Exception::ECSyscallHandler>();
-  Exception::HandlerConfig.emplace();
 
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
@@ -654,7 +653,7 @@ NTSTATUS ProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Exception::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   HandleImageMap(NtDllBase);

Source/Windows/Common/TSOHandlerConfig.h

@@ -1,18 +1,34 @@
 // SPDX-License-Identifier: MIT
 #pragma once
 
+#include <FEXCore/Core/Context.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Utils/ArchHelpers/Arm64.h>
 
 namespace FEX::Windows {
 class TSOHandlerConfig final {
 public:
-  TSOHandlerConfig() {
+  TSOHandlerConfig(FEXCore::Context::Context& CTX) {
     if (HalfBarrierTSOEnabled()) {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier;
     } else {
       UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic;
     }
+
+    if (TSOEnabled()) {
+      BOOL Enable = TRUE;
+      NTSTATUS Status = NtSetInformationProcess(NtCurrentProcess(), ProcessFexHardwareTso, &Enable, sizeof(Enable));
+      if (Status == STATUS_SUCCESS) {
+        CTX.SetHardwareTSOSupport(true);
+      }
+    }
+
+    uint64_t Flags = (StrictInProcessSplitLocks() ? FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) |
+                     (KernelUnalignedAtomicBackpatching() ? FEX_UNALIGN_ATOMIC_BACKPATCH : 0) | FEX_UNALIGN_ATOMIC_EMULATE;
+
+    if (NtSetInformationProcess(NtCurrentProcess(), ProcessFexUnalignAtomic, &Flags, sizeof(Flags)) == STATUS_SUCCESS) {
+      LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!");
+    }
   }
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const {
@@ -20,7 +36,10 @@ public:
   }
 
 private:
+  FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);
   FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED);
+  FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
+  FEX_CONFIG_OPT(KernelUnalignedAtomicBackpatching, KERNELUNALIGNEDATOMICBACKPATCHING);
 
   FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier};
 };

Source/Windows/WOW64/Module.cpp

@@ -523,7 +523,6 @@ void BTCpuProcessInit() {
 
   SignalDelegator = fextl::make_unique<FEX::DummyHandlers::DummySignalDelegator>();
   SyscallHandler = fextl::make_unique<WowSyscallHandler>();
-  Context::HandlerConfig.emplace();
   const auto NtDll = GetModuleHandle("ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
   OvercommitTracker.emplace(IsWine);
@@ -538,7 +537,7 @@ void BTCpuProcessInit() {
   CTX->SetSignalDelegator(SignalDelegator.get());
   CTX->SetSyscallHandler(SyscallHandler.get());
   CTX->InitCore();
-
+  Context::HandlerConfig.emplace(*CTX);
   InvalidationTracker.emplace(*CTX, Threads);
 
   auto NtDllX86 = reinterpret_cast<SYSTEM_DLL_INIT_BLOCK*>(GetProcAddress(NtDll, "LdrSystemDllInitBlock"))->ntdll_handle;

Source/Windows/include/winternl.h

@@ -455,6 +455,12 @@ typedef enum _MEMORY_INFORMATION_CLASS {
 #define SystemEmulationBasicInformation (SYSTEM_INFORMATION_CLASS)62
 
 #define ProcessFexHardwareTso (PROCESSINFOCLASS)2000
+#define ProcessFexUnalignAtomic (PROCESSINFOCLASS)2001
+
+// These match the prctl flag values
+#define FEX_UNALIGN_ATOMIC_EMULATE (1ULL << 0)
+#define FEX_UNALIGN_ATOMIC_BACKPATCH (1ULL << 1)
+#define FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1ULL << 2)
 
 typedef enum _KEY_VALUE_INFORMATION_CLASS {
   KeyValueBasicInformation,