From cf5afbcd8d0f801f4368349d64bbde7a3a3b9a1b Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Sun, 30 Aug 2020 00:34:49 -0700 Subject: [PATCH] Some minor optimizations in the AArch64 ASM dispatcher Ensures the top of the loop is 16byte aligned to give Cortex's prefetcher the best chance of loading the code. If the address size is a power of two (We always do 64GB or 32GB VM atm) then do the address size masking in one instruction instead of loading a 64bit constant first Move some accessed constants to a literal pool at the end of the ASM dispatcher. --- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 103 ++++++++++++------ 1 file changed, 72 insertions(+), 31 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 22b3be7df..809b8d183 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -789,6 +789,42 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { // } // } + + uint64_t VirtualMemorySize = Thread->BlockCache->GetVirtualMemorySize(); + Literal l_VirtualMemory {VirtualMemorySize}; + Literal l_PagePtr {Thread->BlockCache->GetPagePointer()}; + Literal l_CTX {reinterpret_cast(CTX)}; + Literal l_Interpreter {reinterpret_cast(State->IntBackend->CompileCode(nullptr, nullptr))}; + Literal l_Sleep {reinterpret_cast(SleepThread)}; + + uintptr_t CompileBlockPtr{}; + uintptr_t CompileFallbackPtr{}; + { + using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t); + union PtrCast { + ClassPtrType ClassPtr; + uintptr_t Data; + }; + + PtrCast Ptr; + Ptr.ClassPtr = &FEXCore::Context::Context::CompileBlock; + CompileBlockPtr = Ptr.Data; + } + { + using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t); + union PtrCast { + ClassPtrType ClassPtr; + uintptr_t Data; + }; + + PtrCast Ptr; + Ptr.ClassPtr = &FEXCore::Context::Context::CompileFallbackBlock; + CompileFallbackPtr = Ptr.Data; + } + + Literal l_CompileBlock {CompileBlockPtr}; + Literal l_CompileFallback {CompileFallbackPtr}; + // Push all the register we need to save PushCalleeSavedRegisters(); @@ -802,24 +838,39 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { add(x0, sp, 0); str(x0, MemOperand(STATE, offsetof(FEXCore::Core::ThreadState, ReturningStackLocation))); + auto Align16B = [&Buffer, this]() { + uint64_t CurrentOffset = Buffer->GetOffsetAddress(GetCursorOffset()); + for (uint64_t i = (16 - (CurrentOffset & 0xF)); i != 0; i -= 4) { + nop(); + } + }; + + // We want to ensure that we are 16 byte aligned at the top of this loop + Align16B(); + bind(&LoopTop); AbsoluteLoopTopAddress = GetLabelAddress(&LoopTop); + // This is the block cache lookup routine + // It matches what is going on it BlockCache.h::FindBlock + ldr(x0, &l_PagePtr); + // Load in our RIP // Don't modify x2 since it contains our RIP once the block doesn't exist ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::ThreadState, State.rip))); auto RipReg = x2; // Mask the address by the virtual address size so we can check for aliases - LoadConstant(x3, Thread->BlockCache->GetVirtualMemorySize() - 1); - and_(x3, RipReg, x3); + if (__builtin_popcountl(VirtualMemorySize) == 1) { + and_(x3, RipReg, Thread->BlockCache->GetVirtualMemorySize() - 1); + } + else { + ldr(x3, &l_VirtualMemory); + and_(x3, RipReg, x3); + } aarch64::Label NoBlock; { - // This is the block cache lookup routine - // It matches what is going on it BlockCache.h::FindBlock - LoadConstant(x0, Thread->BlockCache->GetPagePointer()); - // Offset the address and add to our page pointer lsr(x1, x3, 12); @@ -886,16 +937,7 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::InternalThreadState, CTX))); mov(x1, STATE); - - using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t); - union PtrCast { - ClassPtrType ClassPtr; - uintptr_t Data; - }; - - PtrCast Ptr; - Ptr.ClassPtr = &FEXCore::Context::Context::CompileBlock; - LoadConstant(x3, Ptr.Data); + ldr(x3, &l_CompileBlock); // X2 contains our guest RIP blr(x3); // { CTX, ThreadState, RIP} @@ -911,18 +953,9 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { { bind(&FallbackCore); - LoadConstant(x0, reinterpret_cast(CTX)); + ldr(x0, &l_CTX); mov(x1, STATE); - - using ClassPtrType = uintptr_t (FEXCore::Context::Context::*)(FEXCore::Core::InternalThreadState *, uint64_t); - union PtrCast { - ClassPtrType ClassPtr; - uintptr_t Data; - }; - - PtrCast Ptr; - Ptr.ClassPtr = &FEXCore::Context::Context::CompileFallbackBlock; - LoadConstant(x3, Ptr.Data); + ldr(x3, &l_CompileFallback); // X2 contains our guest RIP blr(x3); // {ThreadState, RIP} @@ -949,9 +982,9 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { bind(&InterpreterFallback); InterpreterFallbackHelperAddress = Buffer->GetOffsetAddress(GetCursorOffset()); mov(x0, STATE); - LoadConstant(x1, reinterpret_cast(State->IntBackend->CompileCode(nullptr, nullptr))); - blr(x1); + ldr(x1, &l_Interpreter); + blr(x1); b(&LoopTop); } @@ -962,9 +995,9 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { // We will have faulted and jumped to this location at this point // Call our sleep handler - LoadConstant(x0, reinterpret_cast(CTX)); + ldr(x0, &l_CTX); mov(x1, STATE); - LoadConstant(x2, reinterpret_cast(SleepThread)); + ldr(x2, &l_Sleep); blr(x2); PauseReturnInstruction = Buffer->GetOffsetAddress(GetCursorOffset()); @@ -973,6 +1006,14 @@ void JITCore::CreateCustomDispatch(FEXCore::Core::InternalThreadState *Thread) { hlt(0); } + place(&l_VirtualMemory); + place(&l_PagePtr); + place(&l_CTX); + place(&l_Interpreter); + place(&l_Sleep); + place(&l_CompileBlock); + place(&l_CompileFallback); + FinalizeCode(); uint64_t CodeEnd = Buffer->GetOffsetAddress(GetCursorOffset()); CPU.EnsureIAndDCacheCoherency(reinterpret_cast(DispatchPtr), CodeEnd - reinterpret_cast(DispatchPtr));