diff --git a/FEXCore/Source/Interface/Config/Config.json.in b/FEXCore/Source/Interface/Config/Config.json.in index 9668a6074..710d52b39 100644 --- a/FEXCore/Source/Interface/Config/Config.json.in +++ b/FEXCore/Source/Interface/Config/Config.json.in @@ -443,6 +443,13 @@ "This is required to ensure a split-lock doesn't tear inside the process" ] }, + "KernelUnalignedAtomicBackpatching": { + "Type": "bool", + "Default": "true", + "Desc": [ + "When the kernel unaligned atomic handler is enabled, use backpatching to reduce kernel context switches." + ] + }, "VolatileMetadata": { "Type": "bool", "Default": "true", diff --git a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp index 915c24d11..d11aff1ff 100644 --- a/Source/Tools/FEXInterpreter/FEXInterpreter.cpp +++ b/Source/Tools/FEXInterpreter/FEXInterpreter.cpp @@ -222,6 +222,26 @@ void CheckForGCS() { } } // namespace FEX::GCS +namespace FEX::UnalignedAtomic { +void SetupKernelUnalignedAtomics() { +#ifndef PR_ARM64_SET_UNALIGN_ATOMIC +#define PR_ARM64_SET_UNALIGN_ATOMIC 0x46455849 +#define PR_ARM64_UNALIGN_ATOMIC_EMULATE (1UL << 0) +#define PR_ARM64_UNALIGN_ATOMIC_BACKPATCH (1UL << 1) +#define PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1UL << 2) +#endif + + // Interfaces with downstream FEX kernel patches to control unaligned atomic handling + FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS); + FEX_CONFIG_OPT(KernelUnalignedAtomicBackpatching, KERNELUNALIGNEDATOMICBACKPATCHING); + + uint64_t Flags = (StrictInProcessSplitLocks() ? PR_ARM64_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) | + (KernelUnalignedAtomicBackpatching() ? PR_ARM64_UNALIGN_ATOMIC_BACKPATCH : 0) | PR_ARM64_UNALIGN_ATOMIC_EMULATE; + + prctl(PR_ARM64_SET_UNALIGN_ATOMIC, Flags, 0, 0, 0); +} +} // namespace FEX::UnalignedAtomic + /** * @brief Get an FD from an environment variable and then unset the environment variable. * @@ -458,6 +478,7 @@ int main(int argc, char** argv, char** const envp) { // Setup TSO hardware emulation immediately after initializing the context. FEX::TSO::SetupTSOEmulation(CTX.get()); + FEX::UnalignedAtomic::SetupKernelUnalignedAtomics(); if (!Loader.Is64BitMode()) { // Tell the kernel we want to use the compat input syscalls even though we're diff --git a/Source/Windows/ARM64EC/Module.cpp b/Source/Windows/ARM64EC/Module.cpp index 723465800..e60ae29a4 100644 --- a/Source/Windows/ARM64EC/Module.cpp +++ b/Source/Windows/ARM64EC/Module.cpp @@ -640,7 +640,6 @@ NTSTATUS ProcessInit() { SignalDelegator = fextl::make_unique(); SyscallHandler = fextl::make_unique(); - Exception::HandlerConfig.emplace(); const auto NtDll = GetModuleHandle("ntdll.dll"); const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); @@ -654,7 +653,7 @@ NTSTATUS ProcessInit() { CTX->SetSignalDelegator(SignalDelegator.get()); CTX->SetSyscallHandler(SyscallHandler.get()); CTX->InitCore(); - + Exception::HandlerConfig.emplace(*CTX); InvalidationTracker.emplace(*CTX, Threads); HandleImageMap(NtDllBase); diff --git a/Source/Windows/Common/TSOHandlerConfig.h b/Source/Windows/Common/TSOHandlerConfig.h index fbd84a571..1a0a888ab 100644 --- a/Source/Windows/Common/TSOHandlerConfig.h +++ b/Source/Windows/Common/TSOHandlerConfig.h @@ -1,18 +1,34 @@ // SPDX-License-Identifier: MIT #pragma once +#include #include #include namespace FEX::Windows { class TSOHandlerConfig final { public: - TSOHandlerConfig() { + TSOHandlerConfig(FEXCore::Context::Context& CTX) { if (HalfBarrierTSOEnabled()) { UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier; } else { UnalignedHandlerType = FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::NonAtomic; } + + if (TSOEnabled()) { + BOOL Enable = TRUE; + NTSTATUS Status = NtSetInformationProcess(NtCurrentProcess(), ProcessFexHardwareTso, &Enable, sizeof(Enable)); + if (Status == STATUS_SUCCESS) { + CTX.SetHardwareTSOSupport(true); + } + } + + uint64_t Flags = (StrictInProcessSplitLocks() ? FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS : 0) | + (KernelUnalignedAtomicBackpatching() ? FEX_UNALIGN_ATOMIC_BACKPATCH : 0) | FEX_UNALIGN_ATOMIC_EMULATE; + + if (NtSetInformationProcess(NtCurrentProcess(), ProcessFexUnalignAtomic, &Flags, sizeof(Flags)) == STATUS_SUCCESS) { + LogMan::Msg::IFmt("FEX: Kernel unaligned atomics enabled!"); + } } FEXCore::ArchHelpers::Arm64::UnalignedHandlerType GetUnalignedHandlerType() const { @@ -20,7 +36,10 @@ public: } private: + FEX_CONFIG_OPT(TSOEnabled, TSOENABLED); FEX_CONFIG_OPT(HalfBarrierTSOEnabled, HALFBARRIERTSOENABLED); + FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS); + FEX_CONFIG_OPT(KernelUnalignedAtomicBackpatching, KERNELUNALIGNEDATOMICBACKPATCHING); FEXCore::ArchHelpers::Arm64::UnalignedHandlerType UnalignedHandlerType {FEXCore::ArchHelpers::Arm64::UnalignedHandlerType::HalfBarrier}; }; diff --git a/Source/Windows/WOW64/Module.cpp b/Source/Windows/WOW64/Module.cpp index 4d539ca7e..c51f4f902 100644 --- a/Source/Windows/WOW64/Module.cpp +++ b/Source/Windows/WOW64/Module.cpp @@ -523,7 +523,6 @@ void BTCpuProcessInit() { SignalDelegator = fextl::make_unique(); SyscallHandler = fextl::make_unique(); - Context::HandlerConfig.emplace(); const auto NtDll = GetModuleHandle("ntdll.dll"); const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version"); OvercommitTracker.emplace(IsWine); @@ -538,7 +537,7 @@ void BTCpuProcessInit() { CTX->SetSignalDelegator(SignalDelegator.get()); CTX->SetSyscallHandler(SyscallHandler.get()); CTX->InitCore(); - + Context::HandlerConfig.emplace(*CTX); InvalidationTracker.emplace(*CTX, Threads); auto NtDllX86 = reinterpret_cast(GetProcAddress(NtDll, "LdrSystemDllInitBlock"))->ntdll_handle; diff --git a/Source/Windows/include/winternl.h b/Source/Windows/include/winternl.h index 9607087f4..efd211c5a 100644 --- a/Source/Windows/include/winternl.h +++ b/Source/Windows/include/winternl.h @@ -455,6 +455,12 @@ typedef enum _MEMORY_INFORMATION_CLASS { #define SystemEmulationBasicInformation (SYSTEM_INFORMATION_CLASS)62 #define ProcessFexHardwareTso (PROCESSINFOCLASS)2000 +#define ProcessFexUnalignAtomic (PROCESSINFOCLASS)2001 + +// These match the prctl flag values +#define FEX_UNALIGN_ATOMIC_EMULATE (1ULL << 0) +#define FEX_UNALIGN_ATOMIC_BACKPATCH (1ULL << 1) +#define FEX_UNALIGN_ATOMIC_STRICT_SPLIT_LOCKS (1ULL << 2) typedef enum _KEY_VALUE_INFORMATION_CLASS { KeyValueBasicInformation,