From 35dbf6c44b2231ecc66d58345bea95a080b8deff Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 21 Nov 2022 22:13:41 -0800 Subject: [PATCH] OpcodeDispatcher: Moves all GPR and XMM accesses to direct register accesses This is a bit of a large commit since it is an all or nothing sort of change. Instead of doing LoadContext and StoreContext for GPRs and FPRs, any that are statically allocated (GPR and XMM) should use {Load,Store}Register directly. This is now enforced that {Load,Store}Context{,Indexed} will assert if trying to access these ranges. This is the first step towards accelerating our JIT compile times in that it removes the need for the SRA pass to convert all the IR over. This ensures that from the Dispatcher directly we are handling SRA accesses as "registers", ensuring that future changes don't need to run in to this problem. Even though it wasn't a target of this change, in a simple test application this removes ~12% of the total JIT compile time. --- .../Interface/Core/OpcodeDispatcher.cpp | 703 ++++++++++-------- .../Source/Interface/Core/OpcodeDispatcher.h | 11 +- .../Core/OpcodeDispatcher/Crypto.cpp | 3 +- .../Core/OpcodeDispatcher/Vector.cpp | 67 +- .../Source/Interface/HLE/Thunks/Thunks.cpp | 2 +- 5 files changed, 433 insertions(+), 353 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index eeb0e07f3..98a731197 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -98,7 +98,7 @@ void OpDispatchBuilder::SyscallOp(OpcodeArgs) { InvalidNode, }; for (size_t i = 0; i < NumArguments; ++i) { - Arguments[i] = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs) + GPRIndicesRef[i] * 8); + Arguments[i] = LoadGPRRegister(GPRIndicesRef[i]); } auto SyscallOp = _Syscall( @@ -114,7 +114,7 @@ void OpDispatchBuilder::SyscallOp(OpcodeArgs) { if (OSABI != FEXCore::HLE::SyscallOSABI::OS_HANGOVER) { // Hangover doesn't want us returning a result here // syscall is being abused as a thunk for now. - _StoreContext(GPRSize, GPRClass, SyscallOp, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, SyscallOp); } } @@ -122,32 +122,31 @@ void OpDispatchBuilder::ThunkOp(OpcodeArgs) { // Calculate flags early. CalculateDeferredFlags(); - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t GPRSize = CTX->GetGPRSize(); uint8_t *sha256 = (uint8_t *)(Op->PC + 2); if (CTX->Config.Is64BitMode) { // x86-64 ABI puts the function argument in RDI _Thunk( - _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)), + LoadGPRRegister(X86State::REG_RDI), *reinterpret_cast(sha256) ); } else { // x86 fastcall ABI puts the function argument in ECX _Thunk( - _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)), + LoadGPRRegister(X86State::REG_RCX), *reinterpret_cast(sha256) ); } auto Constant = _Constant(GPRSize); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewRIP = _LoadMem(GPRClass, GPRSize, OldSP, GPRSize); OrderedNode *NewSP = _Add(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store the new RIP _ExitFunction(NewRIP); @@ -156,17 +155,24 @@ void OpDispatchBuilder::ThunkOp(OpcodeArgs) { void OpDispatchBuilder::LEAOp(OpcodeArgs) { // LEA specifically ignores segment prefixes + const auto SrcSize = GetSrcSize(Op); + if (CTX->Config.Is64BitMode) { - uint32_t DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? 2 : + const uint32_t DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? 2 : X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? 8 : 4; - OrderedNode *Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], GetSrcSize(Op), Op->Flags, -1, false); + auto Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags, -1, false); + if (DstSize != SrcSize) { + // If the SrcSize isn't the DstSize then we need to zero extend. + const uint8_t GPRSize = CTX->GetGPRSize(); + Src = _Bfe(GPRSize, SrcSize * 8, 0, Src); + } StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, DstSize, -1); } else { uint32_t DstSize = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0) == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? 2 : 4; - OrderedNode *Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], GetSrcSize(Op), Op->Flags, -1, false); + auto Src = LoadSource_WithOpSize(GPRClass, Op, Op->Src[0], SrcSize, Op->Flags, -1, false); StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, DstSize, -1); } } @@ -175,7 +181,6 @@ void OpDispatchBuilder::NOPOp(OpcodeArgs) { } void OpDispatchBuilder::RETOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t GPRSize = CTX->GetGPRSize(); // ABI Optimization: Flags don't survive calls or rets @@ -190,7 +195,7 @@ void OpDispatchBuilder::RETOp(OpcodeArgs) { } auto Constant = _Constant(GPRSize); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewRIP = _LoadMem(GPRClass, GPRSize, OldSP, GPRSize); OrderedNode *NewSP; @@ -203,7 +208,7 @@ void OpDispatchBuilder::RETOp(OpcodeArgs) { } // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store the new RIP _ExitFunction(NewRIP); @@ -230,12 +235,11 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) { // Calculate flags early. CalculateDeferredFlags(); - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t GPRSize = CTX->GetGPRSize(); auto Constant = _Constant(GPRSize); - OrderedNode* SP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto SP = LoadGPRRegister(X86State::REG_RSP); // RIP (64/32/16 bits) auto NewRIP = _LoadMem(GPRClass, GPRSize, SP, GPRSize); @@ -254,7 +258,8 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) { if (CTX->Config.Is64BitMode) { // RSP and SS only happen in 64-bit mode or if this is a CPL mode jump! // FEX doesn't support a CPL mode switch, so don't need to worry about this on 32-bit - _StoreContext(GPRSize, GPRClass, _LoadMem(GPRClass, GPRSize, SP, GPRSize), RSPOffset); + StoreGPRRegister(X86State::REG_RSP, _LoadMem(GPRClass, GPRSize, SP, GPRSize)); + SP = _Add(SP, Constant); // ss auto NewSegmentSS = _LoadMem(GPRClass, GPRSize, SP, GPRSize); @@ -265,7 +270,7 @@ void OpDispatchBuilder::IRETOp(OpcodeArgs) { } else { // Store the stack in 32-bit mode - _StoreContext(GPRSize, GPRClass, SP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, SP); } _ExitFunction(NewRIP); @@ -331,14 +336,14 @@ void OpDispatchBuilder::SecondaryALUOp(OpcodeArgs) { }; #undef OPD // X86 basic ALU ops just do the operation between the destination and a single source - OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, -1); + auto Src = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, -1); uint8_t Size = GetDstSize(Op); OrderedNode *Result{}; OrderedNode *Dest{}; if (DestIsLockedMem(Op)) { HandledLock = true; - OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false); + auto DestMem = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1, false); DestMem = AppendSegmentOffset(DestMem, Op->Flags); switch (IROp) { case FEXCore::IR::IROps::OP_ADD: { @@ -471,36 +476,32 @@ void OpDispatchBuilder::SBBOp(OpcodeArgs) { } void OpDispatchBuilder::PUSHOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t Size = GetSrcSize(Op); - const uint8_t GPRSize = CTX->GetGPRSize(); OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store our value to the new stack location _StoreMem(GPRClass, Size, NewSP, Src, Size); } void OpDispatchBuilder::PUSHREGOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t Size = GetSrcSize(Op); - const uint8_t GPRSize = CTX->GetGPRSize(); OrderedNode *Src = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1); auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store our value to the new stack location _StoreMem(GPRClass, Size, NewSP, Src, Size); @@ -508,12 +509,10 @@ void OpDispatchBuilder::PUSHREGOp(OpcodeArgs) { void OpDispatchBuilder::PUSHAOp(OpcodeArgs) { // 32bit only - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t Size = GetSrcSize(Op); - const uint8_t GPRSize = 4; auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); // PUSHA order: // Tmp = SP @@ -529,53 +528,51 @@ void OpDispatchBuilder::PUSHAOp(OpcodeArgs) { OrderedNode *Src{}; OrderedNode *NewSP = OldSP; NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); + Src = LoadGPRRegister(X86State::REG_RAX); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RCX)); + Src = LoadGPRRegister(X86State::REG_RCX); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + Src = LoadGPRRegister(X86State::REG_RDX); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RBX)); + Src = LoadGPRRegister(X86State::REG_RBX); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); _StoreMem(GPRClass, Size, NewSP, OldSP, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RBP)); + Src = LoadGPRRegister(X86State::REG_RBP); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RSI)); + Src = LoadGPRRegister(X86State::REG_RSI); _StoreMem(GPRClass, Size, NewSP, Src, Size); NewSP = _Sub(NewSP, Constant); - Src = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDI)); + Src = LoadGPRRegister(X86State::REG_RDI); _StoreMem(GPRClass, Size, NewSP, Src, Size); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP, 4); } template void OpDispatchBuilder::PUSHSegmentOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t SrcSize = GetSrcSize(Op); const uint8_t DstSize = GetDstSize(Op); - const uint8_t GPRSize = CTX->GetGPRSize(); auto Constant = _Constant(DstSize); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); OrderedNode *Src{}; if (!CTX->Config.Is64BitMode()) { @@ -632,18 +629,15 @@ void OpDispatchBuilder::PUSHSegmentOp(OpcodeArgs) { } void OpDispatchBuilder::POPOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t Size = GetSrcSize(Op); - const uint8_t GPRSize = CTX->GetGPRSize(); auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); - + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewGPR = _LoadMem(GPRClass, Size, OldSP, Size); auto NewSP = _Add(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store what we loaded from the stack StoreResult(GPRClass, Op, NewGPR, -1); @@ -651,12 +645,11 @@ void OpDispatchBuilder::POPOp(OpcodeArgs) { void OpDispatchBuilder::POPAOp(OpcodeArgs) { // 32bit only - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t Size = GetSrcSize(Op); const uint8_t GPRSize = 4; auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); // POPA order: // pop EDI @@ -671,52 +664,50 @@ void OpDispatchBuilder::POPAOp(OpcodeArgs) { OrderedNode *Src{}; OrderedNode *NewSP = OldSP; Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, Src, GPRSize); NewSP = _Add(NewSP, Constant); Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RSI)); + StoreGPRRegister(X86State::REG_RSI, Src, GPRSize); NewSP = _Add(NewSP, Constant); Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RBP)); + StoreGPRRegister(X86State::REG_RBP, Src, GPRSize); NewSP = _Add(NewSP, _Constant(Size * 2)); // Skip SP loading Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RBX)); + StoreGPRRegister(X86State::REG_RBX, Src, GPRSize); NewSP = _Add(NewSP, Constant); Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RDX, Src, GPRSize); NewSP = _Add(NewSP, Constant); Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, Src, GPRSize); NewSP = _Add(NewSP, Constant); Src = _LoadMem(GPRClass, Size, NewSP, Size); - _StoreContext(Size, GPRClass, Src, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, Src, GPRSize); NewSP = _Add(NewSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); } template void OpDispatchBuilder::POPSegmentOp(OpcodeArgs) { const uint8_t SrcSize = GetSrcSize(Op); const uint8_t DstSize = GetDstSize(Op); - const uint8_t GPRSize = CTX->GetGPRSize(); auto Constant = _Constant(SrcSize); - auto OldSP = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSP)); - + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSegment = _LoadMem(GPRClass, SrcSize, OldSP, SrcSize); auto NewSP = _Add(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, GPROffset(X86State::REG_RSP)); + StoreGPRRegister(X86State::REG_RSP, NewSP); switch (SegmentReg) { case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX: @@ -744,27 +735,23 @@ void OpDispatchBuilder::POPSegmentOp(OpcodeArgs) { } void OpDispatchBuilder::LEAVEOp(OpcodeArgs) { - const uint32_t RBPOffset = GPROffset(X86State::REG_RBP); - const uint8_t GPRSize = CTX->GetGPRSize(); - // First we move RBP in to RSP and then behave effectively like a pop const uint8_t Size = GetSrcSize(Op); auto Constant = _Constant(Size); - auto OldBP = _LoadContext(GPRSize, GPRClass, RBPOffset); + auto OldBP = LoadGPRRegister(X86State::REG_RBP); auto NewGPR = _LoadMem(GPRClass, Size, OldBP, Size); auto NewSP = _Add(OldBP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, GPROffset(X86State::REG_RSP)); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store what we loaded to RBP - _StoreContext(GPRSize, GPRClass, NewGPR, RBPOffset); + StoreGPRRegister(X86State::REG_RBP, NewGPR); } void OpDispatchBuilder::CALLOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t GPRSize = CTX->GetGPRSize(); BlockSetRIP = true; @@ -788,12 +775,11 @@ void OpDispatchBuilder::CALLOp(OpcodeArgs) { auto ConstantPCReturn = GetRelocatedPC(Op); auto ConstantSize = _Constant(GPRSize); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); - + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, ConstantSize); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); _StoreMem(GPRClass, GPRSize, NewSP, ConstantPCReturn, GPRSize); @@ -802,9 +788,6 @@ void OpDispatchBuilder::CALLOp(OpcodeArgs) { } void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); - const uint8_t GPRSize = CTX->GetGPRSize(); - // Calculate flags early. CalculateDeferredFlags(); @@ -816,12 +799,11 @@ void OpDispatchBuilder::CALLAbsoluteOp(OpcodeArgs) { auto ConstantPCReturn = GetRelocatedPC(Op); auto ConstantSize = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); - + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, ConstantSize); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); _StoreMem(GPRClass, Size, NewSP, ConstantPCReturn, Size); @@ -1176,7 +1158,7 @@ void OpDispatchBuilder::CondJUMPRCXOp(OpcodeArgs) { uint64_t Target = Op->PC + Op->InstSize + Op->Src[0].Data.Literal.Value; - OrderedNode *CondReg = _LoadContext(JcxGPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *CondReg = LoadGPRRegister(X86State::REG_RCX, JcxGPRSize); auto TrueBlock = JumpTargets.find(Target); auto FalseBlock = JumpTargets.find(Op->PC + Op->InstSize); @@ -1526,8 +1508,8 @@ void OpDispatchBuilder::CDQOp(OpcodeArgs) { } void OpDispatchBuilder::SAHFOp(OpcodeArgs) { - const uint32_t RAXOffset = GPROffset(X86State::REG_RAX); - OrderedNode *Src = _LoadContext(1, GPRClass, RAXOffset + 1); + // Extract AH + OrderedNode *Src = LoadGPRRegister(X86State::REG_RAX, 1, 8); // Clear bits that aren't supposed to be set Src = _And(Src, _Constant(~0b101000)); @@ -1540,10 +1522,10 @@ void OpDispatchBuilder::SAHFOp(OpcodeArgs) { } void OpDispatchBuilder::LAHFOp(OpcodeArgs) { // Load the lower 8 bits of the Rflags register - auto RFLAG = GetPackedRFLAG(true); + auto RFLAG = GetPackedRFLAG(true); // Store the lower 8 bits of the rflags register in to AH - _StoreContext(1, GPRClass, RFLAG, GPROffset(X86State::REG_RAX) + 1); + StoreGPRRegister(X86State::REG_RAX, RFLAG, 1, 8); } void OpDispatchBuilder::FLAGControlOp(OpcodeArgs) { @@ -1751,20 +1733,18 @@ void OpDispatchBuilder::MOVOffsetOp(OpcodeArgs) { } void OpDispatchBuilder::CPUIDOp(OpcodeArgs) { - const uint8_t GPRSize = CTX->GetGPRSize(); - OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); - OrderedNode *Leaf = _LoadContext(4, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Leaf = LoadGPRRegister(X86State::REG_RCX); auto Res = _CPUID(Src, Leaf); OrderedNode *Result_Lower = _ExtractElementPair(Res, 0); OrderedNode *Result_Upper = _ExtractElementPair(Res, 1); - _StoreContext(GPRSize, GPRClass, _Bfe(32, 0, Result_Lower), GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, _Bfe(32, 32, Result_Lower), GPROffset(X86State::REG_RBX)); - _StoreContext(GPRSize, GPRClass, _Bfe(32, 32, Result_Upper), GPROffset(X86State::REG_RDX)); - _StoreContext(GPRSize, GPRClass, _Bfe(32, 0, Result_Upper), GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RAX, _Bfe(32, 0, Result_Lower)); + StoreGPRRegister(X86State::REG_RBX, _Bfe(32, 32, Result_Lower)); + StoreGPRRegister(X86State::REG_RDX, _Bfe(32, 32, Result_Upper)); + StoreGPRRegister(X86State::REG_RCX, _Bfe(32, 0, Result_Upper)); } template @@ -1991,7 +1971,7 @@ void OpDispatchBuilder::SHRDOp(OpcodeArgs) { OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); OrderedNode *Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1); - OrderedNode *Shift = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Shift = LoadGPRRegister(X86State::REG_RCX); const auto Size = GetDstBitSize(Op); @@ -2454,11 +2434,10 @@ void OpDispatchBuilder::RORX(OpcodeArgs) { void OpDispatchBuilder::MULX(OpcodeArgs) { // RDX is the implied source operand in the instruction - const auto RDXOffset = offsetof(FEXCore::Core::CPUState, gregs[FEXCore::X86State::REG_RDX]); const auto OperandSize = GetSrcSize(Op); OrderedNode* Src1 = LoadSource(GPRClass, Op, Op->Src[1], Op->Flags, -1); - OrderedNode* Src2 = _LoadContext(OperandSize, GPRClass, RDXOffset); + OrderedNode* Src2 = LoadGPRRegister(X86State::REG_RDX, OperandSize); OrderedNode* ResultLo = _UMul(Src1, Src2); OrderedNode* ResultHi = _UMulH(Src1, Src2); @@ -3207,12 +3186,10 @@ void OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) { } void OpDispatchBuilder::IMULOp(OpcodeArgs) { - const uint32_t RAXOffset = GPROffset(X86State::REG_RAX); - const uint32_t RDXOffset = GPROffset(X86State::REG_RDX); const uint8_t Size = GetSrcSize(Op); OrderedNode *Src1 = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, RAXOffset); + OrderedNode* Src2 = LoadGPRRegister(X86State::REG_RAX); if (Size != 8) { Src1 = _Sext(Size * 8, Src1); @@ -3223,15 +3200,15 @@ void OpDispatchBuilder::IMULOp(OpcodeArgs) { OrderedNode *ResultHigh{}; if (Size == 1) { // Result is stored in AX - _StoreContext(2, GPRClass, Result, RAXOffset); + StoreGPRRegister(X86State::REG_RAX, Result, 2); ResultHigh = _Sbfe(8, 8, Result); } else if (Size == 2) { // 16bits stored in AX // 16bits stored in DX - _StoreContext(Size, GPRClass, Result, RAXOffset); + StoreGPRRegister(X86State::REG_RAX, Result, Size); ResultHigh = _Sbfe(16, 16, Result); - _StoreContext(Size, GPRClass, ResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size); } else if (Size == 4) { // 32bits stored in EAX @@ -3241,8 +3218,8 @@ void OpDispatchBuilder::IMULOp(OpcodeArgs) { auto LocalResultHigh = _Bfe(32, 32, Result); ResultHigh = _Sbfe(32, 32, Result); Result = _Sbfe(32, 0, Result); - _StoreContext(8, GPRClass, LocalResult, RAXOffset); - _StoreContext(8, GPRClass, LocalResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RAX, LocalResult); + StoreGPRRegister(X86State::REG_RDX, LocalResultHigh); } else if (Size == 8) { if (!CTX->Config.Is64BitMode) { @@ -3253,21 +3230,20 @@ void OpDispatchBuilder::IMULOp(OpcodeArgs) { // 64bits stored in RAX // 64bits stored in RDX ResultHigh = _MulH(Src1, Src2); - _StoreContext(8, GPRClass, Result, RAXOffset); - _StoreContext(8, GPRClass, ResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RAX, Result); + StoreGPRRegister(X86State::REG_RDX, ResultHigh); } GenerateFlags_MUL(Op, Result, ResultHigh); } void OpDispatchBuilder::MULOp(OpcodeArgs) { - const uint32_t RAXOffset = GPROffset(X86State::REG_RAX); - const uint32_t RDXOffset = GPROffset(X86State::REG_RDX); const uint8_t Size = GetSrcSize(Op); const uint8_t GPRSize = CTX->GetGPRSize(); OrderedNode *Src1 = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, RAXOffset); + OrderedNode* Src2 = LoadGPRRegister(X86State::REG_RAX); + if (Size != 8) { Src1 = _Bfe(8, Size * 8, 0, Src1); Src2 = _Bfe(8, Size * 8, 0, Src2); @@ -3276,24 +3252,24 @@ void OpDispatchBuilder::MULOp(OpcodeArgs) { OrderedNode *ResultHigh{}; if (Size == 1) { - // Result is stored in AX - _StoreContext(2, GPRClass, Result, RAXOffset); + // Result is stored in AX + StoreGPRRegister(X86State::REG_RAX, Result, 2); ResultHigh = _Bfe(8, 8, Result); } else if (Size == 2) { // 16bits stored in AX // 16bits stored in DX - _StoreContext(Size, GPRClass, Result, RAXOffset); + StoreGPRRegister(X86State::REG_RAX, Result, Size); ResultHigh = _Bfe(16, 16, Result); - _StoreContext(Size, GPRClass, ResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RDX, ResultHigh, Size); } else if (Size == 4) { // 32bits stored in EAX // 32bits stored in EDX OrderedNode *ResultLow = _Bfe(GPRSize, 32, 0, Result); ResultHigh = _Bfe(GPRSize, 32, 32, Result); - _StoreContext(GPRSize, GPRClass, ResultLow, RAXOffset); - _StoreContext(GPRSize, GPRClass, ResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RAX, ResultLow); + StoreGPRRegister(X86State::REG_RDX, ResultHigh); } else if (Size == 8) { if (!CTX->Config.Is64BitMode) { @@ -3304,8 +3280,8 @@ void OpDispatchBuilder::MULOp(OpcodeArgs) { // 64bits stored in RAX // 64bits stored in RDX ResultHigh = _UMulH(Src1, Src2); - _StoreContext(8, GPRClass, Result, RAXOffset); - _StoreContext(8, GPRClass, ResultHigh, RDXOffset); + StoreGPRRegister(X86State::REG_RAX, Result); + StoreGPRRegister(X86State::REG_RDX, ResultHigh); } GenerateFlags_UMUL(Op, ResultHigh); @@ -3384,10 +3360,11 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) { CalculateDeferredFlags(); auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); auto AF = GetRFLAG(FEXCore::X86State::RFLAG_AF_LOC); - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); + SetRFLAG(_Constant(0)); - auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); + auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock()); auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock); auto EndBlock = CreateNewCodeBlockAfter(TrueBlock); @@ -3401,7 +3378,7 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) { SetCurrentCodeBlock(TrueBlock); { auto NewAL = _Add(AL, _Constant(0x6)); - _StoreContext(1, GPRClass, NewAL, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAL, 1); CalculateDeferredFlags(); auto NewCF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); SetRFLAG(_Or(CF, NewCF)); @@ -3422,15 +3399,17 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) { } SetCurrentCodeBlock(TrueBlock); { - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); + auto NewAL = _Add(AL, _Constant(0x60)); - _StoreContext(1, GPRClass, NewAL, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAL, 1); SetRFLAG(_Constant(1)); _Jump(EndBlock); } SetCurrentCodeBlock(EndBlock); // Update Flags - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); + SetRFLAG(_Select(FEXCore::IR::COND_UGE, _And(AL, _Constant(0x80)), _Constant(0), _Constant(1), _Constant(0))); SetRFLAG(_Select(FEXCore::IR::COND_EQ, _And(AL, _Constant(0xFF)), _Constant(0), _Constant(1), _Constant(0))); auto EightBitMask = _Constant(0xFF); @@ -3443,10 +3422,11 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) { CalculateDeferredFlags(); auto CF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); auto AF = GetRFLAG(FEXCore::X86State::RFLAG_AF_LOC); - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); + SetRFLAG(_Constant(0)); - auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xf)), _Constant(9), _Constant(1), _Constant(0))); + auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xf)), _Constant(9), _Constant(1), _Constant(0))); auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock()); auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock); auto EndBlock = CreateNewCodeBlockAfter(TrueBlock); @@ -3460,7 +3440,7 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) { SetCurrentCodeBlock(TrueBlock); { auto NewAL = _Sub(AL, _Constant(0x6)); - _StoreContext(1, GPRClass, NewAL, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAL, 1); CalculateDeferredFlags(); auto NewCF = GetRFLAG(FEXCore::X86State::RFLAG_CF_LOC); SetRFLAG(_Or(CF, NewCF)); @@ -3481,15 +3461,15 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) { } SetCurrentCodeBlock(TrueBlock); { - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); auto NewAL = _Sub(AL, _Constant(0x60)); - _StoreContext(1, GPRClass, NewAL, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAL, 1); SetRFLAG(_Constant(1)); _Jump(EndBlock); } SetCurrentCodeBlock(EndBlock); // Update Flags - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); SetRFLAG(_Select(FEXCore::IR::COND_UGE, _And(AL, _Constant(0x80)), _Constant(0), _Constant(1), _Constant(0))); SetRFLAG(_Select(FEXCore::IR::COND_EQ, _And(AL, _Constant(0xFF)), _Constant(0), _Constant(1), _Constant(0))); auto EightBitMask = _Constant(0xFF); @@ -3500,9 +3480,9 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) { void OpDispatchBuilder::AAAOp(OpcodeArgs) { auto AF = GetRFLAG(FEXCore::X86State::RFLAG_AF_LOC); - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); - auto AX = _LoadContext(2, GPRClass, GPROffset(X86State::REG_RAX)); - auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); + auto AX = LoadGPRRegister(X86State::REG_RAX, 2); + auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock()); auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock); @@ -3512,7 +3492,7 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) { SetCurrentCodeBlock(FalseBlock); { auto NewAX = _And(AX, _Constant(0xFF0F)); - _StoreContext(2, GPRClass, NewAX, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAX, 2); SetRFLAG(_Constant(0)); SetRFLAG(_Constant(0)); _Jump(EndBlock); @@ -3521,7 +3501,7 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) { { auto NewAX = _Add(AX, _Constant(0x106)); auto Result = _And(NewAX, _Constant(0xFF0F)); - _StoreContext(2, GPRClass, Result, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, Result, 2); SetRFLAG(_Constant(1)); SetRFLAG(_Constant(1)); _Jump(EndBlock); @@ -3531,9 +3511,9 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) { void OpDispatchBuilder::AASOp(OpcodeArgs) { auto AF = GetRFLAG(FEXCore::X86State::RFLAG_AF_LOC); - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); - auto AX = _LoadContext(2, GPRClass, GPROffset(X86State::REG_RAX)); - auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); + auto AX = LoadGPRRegister(X86State::REG_RAX, 2); + auto Cond = _Or(AF, _Select(FEXCore::IR::COND_UGT, _And(AL, _Constant(0xF)), _Constant(9), _Constant(1), _Constant(0))); auto FalseBlock = CreateNewCodeBlockAfter(GetCurrentBlock()); auto TrueBlock = CreateNewCodeBlockAfter(FalseBlock); @@ -3543,7 +3523,7 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) { SetCurrentCodeBlock(FalseBlock); { auto NewAX = _And(AX, _Constant(0xFF0F)); - _StoreContext(2, GPRClass, NewAX, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, NewAX, 2); SetRFLAG(_Constant(0)); SetRFLAG(_Constant(0)); _Jump(EndBlock); @@ -3553,7 +3533,7 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) { auto NewAX = _Sub(AX, _Constant(6)); NewAX = _Sub(NewAX, _Constant(0x100)); auto Result = _And(NewAX, _Constant(0xFF0F)); - _StoreContext(2, GPRClass, Result, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, Result, 2); SetRFLAG(_Constant(1)); SetRFLAG(_Constant(1)); _Jump(EndBlock); @@ -3562,15 +3542,16 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) { } void OpDispatchBuilder::AAMOp(OpcodeArgs) { - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); auto Imm8 = _Constant(Op->Src[0].Data.Literal.Value & 0xFF); auto UDivOp = _UDiv(AL, Imm8); auto URemOp = _URem(AL, Imm8); auto AH = _Lshl(UDivOp, _Constant(8)); auto AX = _Add(AH, URemOp); - _StoreContext(2, GPRClass, AX, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, AX, 2); + // Update Flags - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); SetRFLAG(_Select(FEXCore::IR::COND_UGE, _And(AL, _Constant(0x80)), _Constant(0), _Constant(1), _Constant(0))); SetRFLAG(_Select(FEXCore::IR::COND_EQ, _And(AL, _Constant(0xFF)), _Constant(0), _Constant(1), _Constant(0))); auto EightBitMask = _Constant(0xFF); @@ -3580,14 +3561,15 @@ void OpDispatchBuilder::AAMOp(OpcodeArgs) { } void OpDispatchBuilder::AADOp(OpcodeArgs) { - auto AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); - auto AH = _Lshr(_LoadContext(2, GPRClass, GPROffset(X86State::REG_RAX)), _Constant(8)); + auto AL = LoadGPRRegister(X86State::REG_RAX, 1); + auto AH = _Lshr(LoadGPRRegister(X86State::REG_RAX, 2), _Constant(8)); auto Imm8 = _Constant(Op->Src[0].Data.Literal.Value & 0xFF); auto NewAL = _Add(AL, _Mul(AH, Imm8)); auto Result = _And(NewAL, _Constant(0xFF)); - _StoreContext(2, GPRClass, Result, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, Result, 2); + // Update Flags - AL = _LoadContext(1, GPRClass, GPROffset(X86State::REG_RAX)); + AL = LoadGPRRegister(X86State::REG_RAX, 1); SetRFLAG(_Select(FEXCore::IR::COND_UGE, _And(AL, _Constant(0x80)), _Constant(0), _Constant(1), _Constant(0))); SetRFLAG(_Select(FEXCore::IR::COND_EQ, _And(AL, _Constant(0xFF)), _Constant(0), _Constant(1), _Constant(0))); auto EightBitMask = _Constant(0xFF); @@ -3597,19 +3579,15 @@ void OpDispatchBuilder::AADOp(OpcodeArgs) { } void OpDispatchBuilder::XLATOp(OpcodeArgs) { - const uint32_t RAXOffset = GPROffset(X86State::REG_RAX); - const uint32_t RBXOffset = GPROffset(X86State::REG_RBX); - const uint8_t GPRSize = CTX->GetGPRSize(); - - OrderedNode *Src = _LoadContext(GPRSize, GPRClass, RBXOffset); - OrderedNode *Offset = _LoadContext(1, GPRClass, RAXOffset); + OrderedNode *Src = LoadGPRRegister(X86State::REG_RBX); + OrderedNode *Offset = LoadGPRRegister(X86State::REG_RAX, 1); Src = AppendSegmentOffset(Src, Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX); Src = _Add(Src, Offset); auto Res = _LoadMemAutoTSO(GPRClass, 1, Src, 1); - _StoreContext(1, GPRClass, Res, RAXOffset); + StoreGPRRegister(X86State::REG_RAX, Res, 1); } template @@ -3643,8 +3621,6 @@ void OpDispatchBuilder::WriteSegmentReg(OpcodeArgs) { } void OpDispatchBuilder::EnterOp(OpcodeArgs) { - const uint32_t RBPOffset = GPROffset(X86State::REG_RBP); - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); const uint8_t GPRSize = CTX->GetGPRSize(); LOGMAN_THROW_A_FMT(Op->Src[0].IsLiteral(), "Src1 needs to be literal here"); @@ -3654,17 +3630,17 @@ void OpDispatchBuilder::EnterOp(OpcodeArgs) { const uint8_t Level = (Value >> 16) & 0x1F; const auto PushValue = [&](uint8_t Size, OrderedNode *Src) { - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, _Constant(Size)); _StoreMem(GPRClass, Size, NewSP, Src, Size); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); return NewSP; }; - auto OldBP = _LoadContext(GPRSize, GPRClass, RBPOffset); + auto OldBP = LoadGPRRegister(X86State::REG_RBP); auto NewSP = PushValue(GPRSize, OldBP); auto temp_RBP = NewSP; @@ -3678,19 +3654,16 @@ void OpDispatchBuilder::EnterOp(OpcodeArgs) { NewSP = PushValue(GPRSize, temp_RBP); } NewSP = _Sub(NewSP, _Constant(AllocSpace)); - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); - - _StoreContext(GPRSize, GPRClass, temp_RBP, RBPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); + StoreGPRRegister(X86State::REG_RBP, temp_RBP); } void OpDispatchBuilder::RDTSCOp(OpcodeArgs) { - const uint8_t GPRSize = CTX->GetGPRSize(); - auto Counter = _CycleCounter(); auto CounterLow = _Bfe(32, 0, Counter); auto CounterHigh = _Bfe(32, 32, Counter); - _StoreContext(GPRSize, GPRClass, CounterLow, GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, CounterHigh, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, CounterLow); + StoreGPRRegister(X86State::REG_RDX, CounterHigh); } void OpDispatchBuilder::INCOp(OpcodeArgs) { @@ -3769,13 +3742,12 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) { return; } - const auto GPRSize = CTX->GetGPRSize(); const auto Size = GetSrcSize(Op); const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0; if (!Repeat) { OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); - OrderedNode *Dest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *Dest = LoadGPRRegister(X86State::REG_RDI); // Only ES prefix Dest = AppendSegmentOffset(Dest, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); @@ -3793,9 +3765,10 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) { SizeConst, NegSizeConst); // Offset the pointer - OrderedNode *TailDest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *TailDest = LoadGPRRegister(X86State::REG_RDI); TailDest = _Add(TailDest, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest, GPROffset(X86State::REG_RDI)); + + StoreGPRRegister(X86State::REG_RDI, TailDest); } else { // Calculate deffered flags. @@ -3827,7 +3800,8 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) { SetCurrentCodeBlock(LoopHead); { - OrderedNode *Counter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX); + // Can we end the block? _CondJump(Counter, LoopEnd, LoopTail, {COND_EQ}); } @@ -3835,7 +3809,7 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) { SetCurrentCodeBlock(LoopTail); { OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); - OrderedNode *Dest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *Dest = LoadGPRRegister(X86State::REG_RDI); // Only ES prefix Dest = AppendSegmentOffset(Dest, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); @@ -3843,18 +3817,19 @@ void OpDispatchBuilder::STOSOp(OpcodeArgs) { // Store to memory where RDI points _StoreMemAutoTSO(GPRClass, Size, Dest, Src, Size); - OrderedNode *TailCounter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); - OrderedNode *TailDest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX); + OrderedNode *TailDest = LoadGPRRegister(X86State::REG_RDI); // Decrement counter TailCounter = _Sub(TailCounter, _Constant(1)); // Store the counter so we don't have to deal with PHI here - _StoreContext(GPRSize, GPRClass, TailCounter, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, TailCounter); + // Offset the pointer TailDest = _Add(TailDest, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, TailDest); // Jump back to the start, we have more work to do _Jump(LoopHead); @@ -3874,7 +3849,6 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) { } // RA now can handle these to be here, to avoid DF accesses - const auto GPRSize = CTX->GetGPRSize(); const auto Size = GetSrcSize(Op); auto SizeConst = _Constant(Size); auto NegSizeConst = _Constant(-Size); @@ -3902,14 +3876,14 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) { SetCurrentCodeBlock(LoopHead); { - OrderedNode *Counter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX); _CondJump(Counter, LoopEnd, LoopTail, {COND_EQ}); } SetCurrentCodeBlock(LoopTail); { - OrderedNode *Src = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); - OrderedNode *Dest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *Src = LoadGPRRegister(X86State::REG_RSI); + OrderedNode *Dest = LoadGPRRegister(X86State::REG_RDI); Dest = AppendSegmentOffset(Dest, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); Src = AppendSegmentOffset(Src, Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX); @@ -3918,21 +3892,23 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) { // Store to memory where RDI points _StoreMemAutoTSO(GPRClass, Size, Dest, Src, Size); - OrderedNode *TailCounter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX); // Decrement counter TailCounter = _Sub(TailCounter, _Constant(1)); // Store the counter so we don't have to deal with PHI here - _StoreContext(GPRSize, GPRClass, TailCounter, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, TailCounter); + // Offset the pointer - OrderedNode *TailSrc = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); - OrderedNode *TailDest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *TailSrc = LoadGPRRegister(X86State::REG_RSI); + OrderedNode *TailDest = LoadGPRRegister(X86State::REG_RDI); + TailSrc = _Add(TailSrc, PtrDir); TailDest = _Add(TailDest, PtrDir); - _StoreContext(GPRSize, GPRClass, TailSrc, GPROffset(X86State::REG_RSI)); - _StoreContext(GPRSize, GPRClass, TailDest, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RSI, TailSrc); + StoreGPRRegister(X86State::REG_RDI, TailDest); // Jump back to the start, we have more work to do _Jump(LoopHead); @@ -3943,8 +3919,8 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) { SetCurrentCodeBlock(LoopEnd); } else { - OrderedNode *RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); - OrderedNode *RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *RSI = LoadGPRRegister(X86State::REG_RSI); + OrderedNode *RDI = LoadGPRRegister(X86State::REG_RDI); RDI= AppendSegmentOffset(RDI, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); RSI = AppendSegmentOffset(RSI, Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX); @@ -3956,8 +3932,8 @@ void OpDispatchBuilder::MOVSOp(OpcodeArgs) { RSI = _Add(RSI, PtrDir); RDI = _Add(RDI, PtrDir); - _StoreContext(GPRSize, GPRClass, RSI, GPROffset(X86State::REG_RSI)); - _StoreContext(GPRSize, GPRClass, RDI, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RSI, RSI); + StoreGPRRegister(X86State::REG_RDI, RDI); } } @@ -3968,13 +3944,12 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) { return; } - const auto GPRSize = CTX->GetGPRSize(); const auto Size = GetSrcSize(Op); bool Repeat = Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX); if (!Repeat) { - OrderedNode *Dest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); - OrderedNode *Dest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *Dest_RSI = LoadGPRRegister(X86State::REG_RSI); + OrderedNode *Dest_RDI = LoadGPRRegister(X86State::REG_RDI); // Only ES prefix Dest_RDI = AppendSegmentOffset(Dest_RDI, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); @@ -3997,11 +3972,11 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) { // Offset the pointer Dest_RDI = _Add(Dest_RDI, PtrDir); - _StoreContext(GPRSize, GPRClass, Dest_RDI, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, Dest_RDI); // Offset second pointer Dest_RSI = _Add(Dest_RSI, PtrDir); - _StoreContext(GPRSize, GPRClass, Dest_RSI, GPROffset(X86State::REG_RSI)); + StoreGPRRegister(X86State::REG_RSI, Dest_RSI); } else { // Calculate flags early. @@ -4021,7 +3996,7 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) { SetJumpTarget(JumpStart, LoopStart); SetCurrentCodeBlock(LoopStart); - OrderedNode *Counter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX); // Can we end the block? auto CondJump = _CondJump(Counter, {COND_EQ}); @@ -4033,8 +4008,8 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) { // Working loop { - OrderedNode *Dest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); - OrderedNode *Dest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *Dest_RSI = LoadGPRRegister(X86State::REG_RSI); + OrderedNode *Dest_RDI = LoadGPRRegister(X86State::REG_RDI); // Only ES prefix Dest_RDI = AppendSegmentOffset(Dest_RDI, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); @@ -4053,21 +4028,21 @@ void OpDispatchBuilder::CMPSOp(OpcodeArgs) { // Calculate flags early. CalculateDeferredFlags(); - OrderedNode *TailCounter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX); // Decrement counter TailCounter = _Sub(TailCounter, _Constant(1)); // Store the counter so we don't have to deal with PHI here - _StoreContext(GPRSize, GPRClass, TailCounter, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, TailCounter); // Offset the pointer Dest_RDI = _Add(Dest_RDI, PtrDir); - _StoreContext(GPRSize, GPRClass, Dest_RDI, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, Dest_RDI); // Offset second pointer Dest_RSI = _Add(Dest_RSI, PtrDir); - _StoreContext(GPRSize, GPRClass, Dest_RSI, GPROffset(X86State::REG_RSI)); + StoreGPRRegister(X86State::REG_RSI, Dest_RSI); OrderedNode *ZF = GetRFLAG(FEXCore::X86State::RFLAG_ZF_LOC); InternalCondJump = _CondJump(ZF, {REPE ? COND_NEQ : COND_EQ}); @@ -4093,12 +4068,11 @@ void OpDispatchBuilder::LODSOp(OpcodeArgs) { return; } - const auto GPRSize = CTX->GetGPRSize(); const auto Size = GetSrcSize(Op); const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX)) != 0; if (!Repeat) { - OrderedNode *Dest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *Dest_RSI = LoadGPRRegister(X86State::REG_RSI); Dest_RSI = AppendSegmentOffset(Dest_RSI, Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX); auto Src = _LoadMemAutoTSO(GPRClass, Size, Dest_RSI, Size); @@ -4114,9 +4088,10 @@ void OpDispatchBuilder::LODSOp(OpcodeArgs) { SizeConst, NegSizeConst); // Offset the pointer - OrderedNode *TailDest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *TailDest_RSI = LoadGPRRegister(X86State::REG_RSI); + TailDest_RSI = _Add(TailDest_RSI, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest_RSI, GPROffset(X86State::REG_RSI)); + StoreGPRRegister(X86State::REG_RSI, TailDest_RSI); } else { // Calculate flags early. because end of block @@ -4143,7 +4118,7 @@ void OpDispatchBuilder::LODSOp(OpcodeArgs) { SetJumpTarget(JumpStart, LoopStart); SetCurrentCodeBlock(LoopStart); - OrderedNode *Counter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX); // Can we end the block? @@ -4156,25 +4131,26 @@ void OpDispatchBuilder::LODSOp(OpcodeArgs) { // Working loop { - OrderedNode *Dest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *Dest_RSI = LoadGPRRegister(X86State::REG_RSI); + Dest_RSI = AppendSegmentOffset(Dest_RSI, Op->Flags, FEXCore::X86Tables::DecodeFlags::FLAG_DS_PREFIX); auto Src = _LoadMemAutoTSO(GPRClass, Size, Dest_RSI, Size); StoreResult(GPRClass, Op, Src, -1); - OrderedNode *TailCounter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); - OrderedNode *TailDest_RSI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RSI)); + OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX); + OrderedNode *TailDest_RSI = LoadGPRRegister(X86State::REG_RSI); // Decrement counter TailCounter = _Sub(TailCounter, _Constant(1)); // Store the counter so we don't have to deal with PHI here - _StoreContext(GPRSize, GPRClass, TailCounter, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, TailCounter); // Offset the pointer TailDest_RSI = _Add(TailDest_RSI, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest_RSI, GPROffset(X86State::REG_RSI)); + StoreGPRRegister(X86State::REG_RSI, TailDest_RSI); // Jump back to the start, we have more work to do _Jump(LoopStart); @@ -4193,12 +4169,11 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) { return; } - const auto GPRSize = CTX->GetGPRSize(); const auto Size = GetSrcSize(Op); const bool Repeat = (Op->Flags & (FEXCore::X86Tables::DecodeFlags::FLAG_REPNE_PREFIX | FEXCore::X86Tables::DecodeFlags::FLAG_REP_PREFIX)) != 0; if (!Repeat) { - OrderedNode *Dest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *Dest_RDI = LoadGPRRegister(X86State::REG_RDI); Dest_RDI = AppendSegmentOffset(Dest_RDI, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); auto Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); @@ -4218,9 +4193,10 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) { SizeConst, NegSizeConst); // Offset the pointer - OrderedNode *TailDest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *TailDest_RDI = LoadGPRRegister(X86State::REG_RDI); + TailDest_RDI = _Add(TailDest_RDI, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest_RDI, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, TailDest_RDI); } else { // Calculate flags early. because end of block @@ -4244,7 +4220,7 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) { SetJumpTarget(JumpStart, LoopStart); SetCurrentCodeBlock(LoopStart); - OrderedNode *Counter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Counter = LoadGPRRegister(X86State::REG_RCX); // Can we end the block? // We leave if RCX = 0 @@ -4257,7 +4233,8 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) { // Working loop { - OrderedNode *Dest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *Dest_RDI = LoadGPRRegister(X86State::REG_RDI); + Dest_RDI = AppendSegmentOffset(Dest_RDI, 0, FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX, true); auto Src1 = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); @@ -4272,18 +4249,18 @@ void OpDispatchBuilder::SCASOp(OpcodeArgs) { // Calculate flags early. CalculateDeferredFlags(); - OrderedNode *TailCounter = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RCX)); - OrderedNode *TailDest_RDI = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + OrderedNode *TailCounter = LoadGPRRegister(X86State::REG_RCX); + OrderedNode *TailDest_RDI = LoadGPRRegister(X86State::REG_RDI); // Decrement counter TailCounter = _Sub(TailCounter, _Constant(1)); // Store the counter so we don't have to deal with PHI here - _StoreContext(GPRSize, GPRClass, TailCounter, GPROffset(X86State::REG_RCX)); + StoreGPRRegister(X86State::REG_RCX, TailCounter); // Offset the pointer TailDest_RDI = _Add(TailDest_RDI, PtrDir); - _StoreContext(GPRSize, GPRClass, TailDest_RDI, GPROffset(X86State::REG_RDI)); + StoreGPRRegister(X86State::REG_RDI, TailDest_RDI); OrderedNode *ZF = GetRFLAG(FEXCore::X86State::RFLAG_ZF_LOC); InternalCondJump = _CondJump(ZF, {REPE ? COND_NEQ : COND_EQ}); @@ -4315,8 +4292,6 @@ void OpDispatchBuilder::BSWAPOp(OpcodeArgs) { } void OpDispatchBuilder::PUSHFOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); - const uint8_t GPRSize = CTX->GetGPRSize(); const uint8_t Size = GetSrcSize(Op); OrderedNode *Src = GetPackedRFLAG(false); @@ -4325,29 +4300,26 @@ void OpDispatchBuilder::PUSHFOp(OpcodeArgs) { } auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); + auto OldSP = LoadGPRRegister(X86State::REG_RSP); auto NewSP = _Sub(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Store our value to the new stack location _StoreMem(GPRClass, Size, NewSP, Src, Size); } void OpDispatchBuilder::POPFOp(OpcodeArgs) { - const uint32_t RSPOffset = GPROffset(X86State::REG_RSP); - const uint8_t GPRSize = CTX->GetGPRSize(); const uint8_t Size = GetSrcSize(Op); auto Constant = _Constant(Size); - auto OldSP = _LoadContext(GPRSize, GPRClass, RSPOffset); - + auto OldSP = LoadGPRRegister(X86State::REG_RSP); OrderedNode *Src = _LoadMem(GPRClass, Size, OldSP, Size); auto NewSP = _Add(OldSP, Constant); // Store the new stack pointer - _StoreContext(GPRSize, GPRClass, NewSP, RSPOffset); + StoreGPRRegister(X86State::REG_RSP, NewSP); // Add back our flag constants // Bit 1 is always 1 @@ -4395,32 +4367,33 @@ void OpDispatchBuilder::DIVOp(OpcodeArgs) { const auto Size = GetSrcSize(Op); if (Size == 1) { - OrderedNode *Src1 = _LoadContext(2, GPRClass, GPROffset(X86State::REG_RAX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, 2); auto UDivOp = _UDiv(Src1, Divisor); auto URemOp = _URem(Src1, Divisor); - _StoreContext(Size, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(Size, GPRClass, URemOp, GPROffset(X86State::REG_RAX) + 1); + // AX[15:0] = concat + auto ResultAX = _Bfi(GPRSize, 8, 8, UDivOp, URemOp); + StoreGPRRegister(X86State::REG_RAX, ResultAX, 2); } else if (Size == 2) { - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, Size); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX, Size); auto UDivOp = _LUDiv(Src1, Src2, Divisor); auto URemOp = _LURem(Src1, Src2, Divisor); - _StoreContext(Size, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(Size, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp, Size); + StoreGPRRegister(X86State::REG_RDX, URemOp, Size); } else if (Size == 4) { - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, Size); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX, Size); OrderedNode *UDivOp = _Bfe(Size * 8, 0, _LUDiv(Src1, Src2, Divisor)); OrderedNode *URemOp = _Bfe(Size * 8, 0, _LURem(Src1, Src2, Divisor)); - _StoreContext(GPRSize, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp); + StoreGPRRegister(X86State::REG_RDX, URemOp); } else if (Size == 8) { if (!CTX->Config.Is64BitMode) { @@ -4428,14 +4401,14 @@ void OpDispatchBuilder::DIVOp(OpcodeArgs) { DecodeFailure = true; return; } - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX); auto UDivOp = _LUDiv(Src1, Src2, Divisor); auto URemOp = _LURem(Src1, Src2, Divisor); - _StoreContext(8, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(8, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp); + StoreGPRRegister(X86State::REG_RDX, URemOp); } } @@ -4447,34 +4420,35 @@ void OpDispatchBuilder::IDIVOp(OpcodeArgs) { const auto Size = GetSrcSize(Op); if (Size == 1) { - OrderedNode *Src1 = _LoadContext(2, GPRClass, GPROffset(X86State::REG_RAX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, 2); Src1 = _Sbfe(16, 0, Src1); Divisor = _Sbfe(8, 0, Divisor); auto UDivOp = _Div(Src1, Divisor); auto URemOp = _Rem(Src1, Divisor); - _StoreContext(Size, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(Size, GPRClass, URemOp, GPROffset(X86State::REG_RAX) + 1); + // AX[15:0] = concat + auto ResultAX = _Bfi(GPRSize, 8, 8, UDivOp, URemOp); + StoreGPRRegister(X86State::REG_RAX, ResultAX, 2); } else if (Size == 2) { - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, Size); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX, Size); auto UDivOp = _LDiv(Src1, Src2, Divisor); auto URemOp = _LRem(Src1, Src2, Divisor); - _StoreContext(Size, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(Size, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp, Size); + StoreGPRRegister(X86State::REG_RDX, URemOp, Size); } else if (Size == 4) { - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX, Size); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX, Size); OrderedNode *UDivOp = _Bfe(Size * 8, 0, _LDiv(Src1, Src2, Divisor)); OrderedNode *URemOp = _Bfe(Size * 8, 0, _LRem(Src1, Src2, Divisor)); - _StoreContext(GPRSize, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp); + StoreGPRRegister(X86State::REG_RDX, URemOp); } else if (Size == 8) { if (!CTX->Config.Is64BitMode) { @@ -4482,14 +4456,14 @@ void OpDispatchBuilder::IDIVOp(OpcodeArgs) { DecodeFailure = true; return; } - OrderedNode *Src1 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Src2 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Src1 = LoadGPRRegister(X86State::REG_RAX); + OrderedNode *Src2 = LoadGPRRegister(X86State::REG_RDX); auto UDivOp = _LDiv(Src1, Src2, Divisor); auto URemOp = _LRem(Src1, Src2, Divisor); - _StoreContext(8, GPRClass, UDivOp, GPROffset(X86State::REG_RAX)); - _StoreContext(8, GPRClass, URemOp, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, UDivOp); + StoreGPRRegister(X86State::REG_RDX, URemOp); } } @@ -4568,14 +4542,19 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) { OrderedNode *Src3Lower{}; if (GPRSize == 8 && Size == 4) { Src1 = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, GPRSize, Op->Flags, -1); - Src1Lower = _Bfe(4, 32, 0, Src1); - Src3 = _LoadContext(8, GPRClass, GPROffset(X86State::REG_RAX)); - Src3Lower = _Bfe(4, 32, 0, Src3); + Src3 = LoadGPRRegister(X86State::REG_RAX); } else { Src1 = LoadSource_WithOpSize(GPRClass, Op, Op->Dest, Size, Op->Flags, -1); + Src3 = LoadGPRRegister(X86State::REG_RAX); + } + + if (Size != GPRSize) { + Src1Lower = _Bfe(GPRSize, Size * 8, 0, Src1); + Src3Lower = _Bfe(GPRSize, Size * 8, 0, Src3); + } + else { Src1Lower = Src1; - Src3 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); Src3Lower = Src3; } @@ -4603,11 +4582,11 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) { Src3, Src1Lower); // When the size is 4 we need to make sure not zext the GPR when the comparison fails - _StoreContext(GPRSize, GPRClass, RAXResult, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, RAXResult); StoreResult_WithOpSize(GPRClass, Op, Op->Dest, DestResult, GPRSize, -1); } else { - _StoreContext(Size, GPRClass, CASResult, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, CASResult, Size); StoreResult(GPRClass, Op, DestResult, -1); } @@ -4626,11 +4605,11 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) { OrderedNode *Src3{}; OrderedNode *Src3Lower{}; if (GPRSize == 8 && Size == 4) { - Src3 = _LoadContext(8, GPRClass, GPROffset(X86State::REG_RAX)); + Src3 = LoadGPRRegister(X86State::REG_RAX); Src3Lower = _Bfe(4, 32, 0, Src3); } else { - Src3 = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); + Src3 = LoadGPRRegister(X86State::REG_RAX, Size); Src3Lower = Src3; } // If this is a memory location then we want the pointer to it @@ -4643,7 +4622,7 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) { // This will write to memory! Careful! // Third operand must be a calculated guest memory address OrderedNode *CASResult = _CAS(Src3Lower, Src2, Src1); - OrderedNode *RAXResult = CASResult; + OrderedNode *RAXResult = CASResult; if (GPRSize == 8 && Size == 4) { // This allows us to only hit the ZEXT case on failure @@ -4654,7 +4633,8 @@ void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) { } // RAX gets the result of the CAS op - _StoreContext(Size, GPRClass, RAXResult, GPROffset(X86State::REG_RAX)); + StoreGPRRegister(X86State::REG_RAX, RAXResult, Size); + const auto Size = GetDstBitSize(Op); @@ -4671,7 +4651,6 @@ void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) { // Calculate flags early. CalculateDeferredFlags(); - const uint8_t GPRSize = CTX->GetGPRSize(); // REX.W used to determine if it is 16byte or 8byte // Unlike CMPXCHG, the destination can only be a memory location uint8_t Size = Op->Flags & FEXCore::X86Tables::DecodeFlags::FLAG_REX_WIDENING ? 8 : 4; @@ -4682,12 +4661,12 @@ void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) { Src1 = AppendSegmentOffset(Src1, Op->Flags); - OrderedNode *Expected_Lower = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RAX)); - OrderedNode *Expected_Upper = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RDX)); + OrderedNode *Expected_Lower = LoadGPRRegister(X86State::REG_RAX, Size); + OrderedNode *Expected_Upper = LoadGPRRegister(X86State::REG_RDX, Size); OrderedNode *Expected = _CreateElementPair(Expected_Lower, Expected_Upper); - OrderedNode *Desired_Lower = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RBX)); - OrderedNode *Desired_Upper = _LoadContext(Size, GPRClass, GPROffset(X86State::REG_RCX)); + OrderedNode *Desired_Lower = LoadGPRRegister(X86State::REG_RBX, Size); + OrderedNode *Desired_Upper = LoadGPRRegister(X86State::REG_RCX, Size); OrderedNode *Desired = _CreateElementPair(Desired_Lower, Desired_Upper); // ssa0 = Expected @@ -4725,8 +4704,8 @@ void OpDispatchBuilder::CMPXCHGPairOp(OpcodeArgs) { SetFalseJumpTarget(CondJump, JumpTarget); SetCurrentCodeBlock(JumpTarget); - _StoreContext(GPRSize, GPRClass, Result_Lower, GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, Result_Upper, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, Result_Lower); + StoreGPRRegister(X86State::REG_RDX, Result_Upper); auto Jump = _Jump(); auto NextJumpTarget = CreateNewCodeBlockAfter(JumpTarget); @@ -4936,35 +4915,64 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass } else if (Operand.IsGPR()) { const auto gpr = Operand.Data.GPR.GPR; + const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0; + if (gpr >= FEXCore::X86State::REG_MM_0) { Src = _LoadContext(OpSize, FPRClass, offsetof(FEXCore::Core::CPUState, mm[gpr - FEXCore::X86State::REG_MM_0])); } else if (gpr >= FEXCore::X86State::REG_XMM_0) { const auto gprIndex = gpr - X86State::REG_XMM_0; - const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0; - if (CTX->HostFeatures.SupportsAVX) { - Src = _LoadContext(OpSize, FPRClass, offsetof(Core::CPUState, xmm.avx.data[gprIndex][highIndex])); - } else { - Src = _LoadContext(OpSize, FPRClass, offsetof(Core::CPUState, xmm.sse.data[gprIndex][highIndex])); + const auto regSize = CTX->HostFeatures.SupportsAVX ? + Core::CPUState::XMM_AVX_REG_SIZE : + Core::CPUState::XMM_SSE_REG_SIZE; + + const auto VectorOffset = CTX->HostFeatures.SupportsAVX ? + offsetof(Core::CPUState, xmm.avx.data[gprIndex][0]) : + offsetof(Core::CPUState, xmm.sse.data[gprIndex][0]); + + // Load the full register size if it is a XMM register source. + Src = _LoadRegister(false, VectorOffset, FPRClass, FPRFixedClass, regSize); + + // If we are wanting a high-index then we need to extract an element from the upper half of the reg. + // We can only extract an element size here. + // TODO: Have the instruction doing this load do the extract instead of here. + // We don't have enough information here to know if we can avoid this dup. + if (highIndex && OpSize < Core::CPUState::XMM_SSE_REG_SIZE) { + Src = _VDupElement(regSize, OpSize, Src, 1); + } + + // Now extract the subregister if it was a partial load /smaller/ than SSE size + // TODO: Instead of doing the VMov implicitly on load, hunt down all use cases that require partial loads and do it after load. + // We don't have information here to know if the operation needs zero upper bits or can contain data. + if (OpSize < Core::CPUState::XMM_SSE_REG_SIZE) { + Src = _VMov(OpSize, Src); + } + + // OpSize of 16 is special in that it is expected to zero the upper bits of the 256-bit operation. + // TODO: Longer term we should enforce the difference between zero and insert. + if (regSize == Core::CPUState::XMM_AVX_REG_SIZE && OpSize == Core::CPUState::XMM_SSE_REG_SIZE) { + Src = _VMov(OpSize, Src); } } else { - Src = _LoadContext(OpSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[gpr]) + (Operand.Data.GPR.HighBits ? 1 : 0)); + Src = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[gpr]) + (highIndex ? 1 : 0), GPRClass, GPRFixedClass, OpSize); } } else if (Operand.IsGPRDirect()) { - Src = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR])); + Src = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR]), GPRClass, GPRFixedClass, GPRSize); + LoadableType = true; if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { AccessType = MemoryAccessType::ACCESS_NONTSO; } } else if (Operand.IsGPRIndirect()) { - auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR])); + auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR]), GPRClass, GPRFixedClass, GPRSize); + auto Constant = _Constant(GPRSize * 8, Operand.Data.GPRIndirect.Displacement); - Src = _Add(GPR, Constant); + Src = _Add(GPR, Constant); LoadableType = true; if (Operand.Data.GPRIndirect.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { @@ -4985,7 +4993,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass else if (Operand.IsSIB()) { OrderedNode *Tmp {}; if (Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) { - Tmp = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index])); + Tmp = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index]), GPRClass, GPRFixedClass, GPRSize); if (Operand.Data.SIB.Scale != 1) { auto Constant = _Constant(GPRSize * 8, Operand.Data.SIB.Scale); @@ -4997,7 +5005,7 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass } if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) { - auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base])); + auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base]), GPRClass, GPRFixedClass, GPRSize); if (Tmp != nullptr) { Tmp = _Add(Tmp, GPR); @@ -5028,18 +5036,20 @@ OrderedNode *OpDispatchBuilder::LoadSource_WithOpSize(FEXCore::IR::RegisterClass } } - if (AddrSize < GPRSize) { - // If AddrSize == 16 then we need to clear the upper bits - // GPRSize will be 32 in this case - Src = _Bfe(AddrSize, AddrSize * 8, 0, Src); - } - LoadableType = true; } else { LOGMAN_MSG_A_FMT("Unknown Src Type: {}\n", Operand.Type); } + if (LoadableType && AddrSize < GPRSize) { + // For 64-bit AddrSize can be 32-bit or 64-bit + // For 32-bit AddrSize can be 32-bit or 16-bit + // + // If the AddrSize is not the GPRSize then we need to clear the upper bits. + Src = _Bfe(GPRSize, AddrSize * 8, 0, Src); + } + if ((LoadableType && LoadData) || ForceLoad) { Src = AppendSegmentOffset(Src, Flags); @@ -5058,6 +5068,49 @@ OrderedNode *OpDispatchBuilder::GetRelocatedPC(FEXCore::X86Tables::DecodedOp con return _EntrypointOffset(Op->PC + Op->InstSize + Offset - Entry, GPRSize); } +OrderedNode *OpDispatchBuilder::LoadGPRRegister(uint32_t GPR, int8_t Size, uint8_t Offset) { + const uint8_t GPRSize = CTX->GetGPRSize(); + OrderedNode *Reg = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize); + + if (Size != -1 || Offset != 0) { + // Extract the subregister if requested. + Reg = _Bfe(Size, Size * 8, Offset, Reg); + } + return Reg; +} + +OrderedNode *OpDispatchBuilder::LoadXMMRegister(uint32_t XMM) { + const auto VectorSize = CTX->HostFeatures.SupportsAVX ? 32 : 16; + const auto VectorOffset = CTX->HostFeatures.SupportsAVX ? + offsetof(Core::CPUState, xmm.avx.data[XMM][0]) : + offsetof(Core::CPUState, xmm.sse.data[XMM][0]); + + OrderedNode *Reg = _LoadRegister(false, VectorOffset, FPRClass, FPRFixedClass, VectorSize); + return Reg; +} + +void OpDispatchBuilder::StoreGPRRegister(uint32_t GPR, OrderedNode *const Src, int8_t Size, uint8_t Offset) { + const uint8_t GPRSize = CTX->GetGPRSize(); + if (Size != -1 || Offset != 0) { + // Need to do an insert if not automatic size or zero offset. + OrderedNode *Reg = LoadGPRRegister(GPR); + Reg = _Bfi(GPRSize, Size * 8, Offset, Reg, Src); + _StoreRegister(Reg, false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize); + } + else { + _StoreRegister(Src, false, offsetof(FEXCore::Core::CPUState, gregs[GPR]), GPRClass, GPRFixedClass, GPRSize); + } +} + +void OpDispatchBuilder::StoreXMMRegister(uint32_t XMM, OrderedNode *const Src) { + const auto VectorSize = CTX->HostFeatures.SupportsAVX ? 32 : 16; + const auto VectorOffset = CTX->HostFeatures.SupportsAVX ? + offsetof(Core::CPUState, xmm.avx.data[XMM][0]) : + offsetof(Core::CPUState, xmm.sse.data[XMM][0]); + + _StoreRegister(Src, false, VectorOffset, FPRClass, FPRFixedClass, VectorSize); +} + OrderedNode *OpDispatchBuilder::LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData, bool ForceLoad, MemoryAccessType AccessType) { const uint8_t OpSize = GetSrcSize(Op); return LoadSource_WithOpSize(Class, Op, Operand, OpSize, Flags, Align, LoadData, ForceLoad, AccessType); @@ -5092,36 +5145,74 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl const auto gprIndex = gpr - X86State::REG_XMM_0; const auto highIndex = Operand.Data.GPR.HighBits ? 1 : 0; - if (CTX->HostFeatures.SupportsAVX) { - _StoreContext(OpSize, Class, Src, offsetof(Core::CPUState, xmm.avx.data[gprIndex][highIndex])); - } else { - _StoreContext(OpSize, Class, Src, offsetof(Core::CPUState, xmm.sse.data[gprIndex][highIndex])); + const auto VectorSize = CTX->HostFeatures.SupportsAVX ? 32 : 16; + const auto VectorOffset = CTX->HostFeatures.SupportsAVX ? + offsetof(Core::CPUState, xmm.avx.data[gprIndex][highIndex]) : + offsetof(Core::CPUState, xmm.sse.data[gprIndex][highIndex]); + + if (highIndex || OpSize != VectorSize) { + auto InsertResult = Src; + // Partial writes can come from GPR or FPR. + // TODO: Fix the instructions doing partial writes rather than dealing with it here. + auto SrcVector = _LoadRegister(false, VectorOffset, FPRClass, FPRFixedClass, OpSize); + + if (Class == IR::GPRClass) { + InsertResult = _VInsGPR(VectorSize, OpSize, highIndex, SrcVector, Src); + } + else { + // OpSize of 16 is special in that it is expected to zero the upper bits of the 256-bit operation. + // TODO: Longer term we should enforce the difference between zero and insert. + if (VectorSize == Core::CPUState::XMM_AVX_REG_SIZE && OpSize == Core::CPUState::XMM_SSE_REG_SIZE) { + InsertResult = _VMov(OpSize, Src); + } + else { + InsertResult = _VInsElement(VectorSize, OpSize, highIndex, 0, SrcVector, Src); + } + } + + _StoreRegister(InsertResult, false, VectorOffset, FPRClass, FPRFixedClass, VectorSize); + } + else { + _StoreRegister(Src, false, VectorOffset, FPRClass, FPRFixedClass, VectorSize); } } else { if (GPRSize == 8 && OpSize == 4) { // If the Source IR op is 64 bits, we need to zext the upper bits // For all other sizes, the upper bits are guaranteed to already be zero - OrderedNode *Value = GetOpSize(Src) == 8 ? _Bfe(4, 32, 0, Src) : Src; + OrderedNode *Value = GetOpSize(Src) == 8 ? _Bfe(4, 32, 0, Src) : Src; + _StoreRegister(Value, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize); LOGMAN_THROW_AA_FMT(!Operand.Data.GPR.HighBits, "Can't handle 32bit store to high 8bit register"); - _StoreContext(GPRSize, Class, Value, offsetof(FEXCore::Core::CPUState, gregs[gpr])); } else { LOGMAN_THROW_AA_FMT(!(GPRSize == 4 && OpSize > 4), "Oops had a {} GPR load", OpSize); - _StoreContext(std::min(GPRSize, OpSize), Class, Src, offsetof(FEXCore::Core::CPUState, gregs[gpr]) + (Operand.Data.GPR.HighBits ? 1 : 0)); + + if (GPRSize != OpSize) { + // if the GPR isn't the full size then we need to insert. + // eg: + // mov al, 2 ; Move in to lower 8-bits. + // mov ah, 2 ; Move in to upper 8-bits of 16-bit reg. + // mov ax, 2 ; Move in to lower 16-bits of reg. + auto RegDest = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize); + auto Result = _Bfi(GPRSize, OpSize * 8, Operand.Data.GPR.HighBits * 8, RegDest, Src); + _StoreRegister(Result, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, GPRSize); + } + else { + _StoreRegister(Src, false, offsetof(FEXCore::Core::CPUState, gregs[gpr]), GPRClass, GPRFixedClass, std::min(GPRSize, OpSize)); + } } } } else if (Operand.IsGPRDirect()) { - MemStoreDst = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR])); + MemStoreDst = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPR.GPR]), GPRClass, GPRFixedClass, GPRSize); MemStore = true; if (Operand.Data.GPR.GPR == FEXCore::X86State::REG_RSP && AccessType == MemoryAccessType::ACCESS_DEFAULT) { AccessType = MemoryAccessType::ACCESS_NONTSO; } } else if (Operand.IsGPRIndirect()) { - auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR])); + auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.GPRIndirect.GPR]), GPRClass, GPRFixedClass, GPRSize); auto Constant = _Constant(GPRSize * 8, Operand.Data.GPRIndirect.Displacement); MemStoreDst = _Add(GPR, Constant); @@ -5143,7 +5234,7 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl else if (Operand.IsSIB()) { OrderedNode *Tmp {}; if (Operand.Data.SIB.Index != FEXCore::X86State::REG_INVALID) { - Tmp = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index])); + Tmp = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Index]), GPRClass, GPRFixedClass, GPRSize); if (Operand.Data.SIB.Scale != 1) { auto Constant = _Constant(GPRSize * 8, Operand.Data.SIB.Scale); @@ -5152,7 +5243,7 @@ void OpDispatchBuilder::StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Cl } if (Operand.Data.SIB.Base != FEXCore::X86State::REG_INVALID) { - auto GPR = _LoadContext(AddrSize, GPRClass, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base])); + auto GPR = _LoadRegister(false, offsetof(FEXCore::Core::CPUState, gregs[Operand.Data.SIB.Base]), GPRClass, GPRFixedClass, GPRSize); if (Tmp != nullptr) { Tmp = _Add(Tmp, GPR); @@ -5525,8 +5616,6 @@ void OpDispatchBuilder::CLZeroOp(OpcodeArgs) { } void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) { - const uint8_t GPRSize = CTX->GetGPRSize(); - // RDTSCP is slightly different than RDTSC // IA32_TSC_AUX is returned in RCX // All previous loads are globally visible @@ -5540,9 +5629,9 @@ void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) { auto CounterLow = _Bfe(32, 0, Counter); auto CounterHigh = _Bfe(32, 32, Counter); auto ID = _ProcessorID(); - _StoreContext(GPRSize, GPRClass, CounterLow, GPROffset(X86State::REG_RAX)); - _StoreContext(GPRSize, GPRClass, ID, GPROffset(X86State::REG_RCX)); - _StoreContext(GPRSize, GPRClass, CounterHigh, GPROffset(X86State::REG_RDX)); + StoreGPRRegister(X86State::REG_RAX, CounterLow); + StoreGPRRegister(X86State::REG_RCX, ID); + StoreGPRRegister(X86State::REG_RDX, CounterHigh); } void OpDispatchBuilder::CRC32(OpcodeArgs) { @@ -6654,7 +6743,7 @@ constexpr uint16_t PF_F2 = 3; {OPD(0xDF, 0xE8), 8, &OpDispatchBuilder::FCOMIF64<80, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>}, {OPD(0xDF, 0xF0), 8, &OpDispatchBuilder::FCOMIF64<80, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>}, }; - + constexpr std::tuple X87OpTable[] = { {OPDReg(0xD8, 0) | 0x00, 8, &OpDispatchBuilder::FADD<32, false, OpDispatchBuilder::OpResult::RES_ST0>}, diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 21842733d..2de4cf7c7 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -76,10 +76,10 @@ public: OrderedNode* flagsOpSrcSigned{}; FEXCore::Context::Context *CTX{}; - + // Used during new op bringup bool ShouldDump {false}; - + struct JumpTargetInfo { OrderedNode* BlockEntry; bool HaveEmitted; @@ -532,7 +532,7 @@ public: void X87FRSTORF64(OpcodeArgs); void X87FXAMF64(OpcodeArgs); void X87LDENVF64(OpcodeArgs); - + template void FCOMIF64(OpcodeArgs); @@ -675,6 +675,11 @@ private: // Non-temporal streaming ACCESS_STREAM, }; + OrderedNode *LoadGPRRegister(uint32_t GPR, int8_t Size = -1, uint8_t Offset = 0); + OrderedNode *LoadXMMRegister(uint32_t XMM); + void StoreGPRRegister(uint32_t GPR, OrderedNode *const Src, int8_t Size = -1, uint8_t Offset = 0); + void StoreXMMRegister(uint32_t XMM, OrderedNode *const Src); + OrderedNode *GetRelocatedPC(FEXCore::X86Tables::DecodedOp const& Op, int64_t Offset = 0); OrderedNode *LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); OrderedNode *LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT); diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp index 96410b5cd..f8b1f252b 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp @@ -221,7 +221,8 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) { OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1); OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); - OrderedNode *XMM0 = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, xmm.avx.data[0])); + // Hardcoded to XMM0 + auto XMM0 = LoadXMMRegister(0); auto A0 = _VExtractToGPR(16, 4, Src, 3); auto B0 = _VExtractToGPR(16, 4, Src, 2); diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp index af04a2f9a..94af650be 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp @@ -115,9 +115,10 @@ void OpDispatchBuilder::MOVLPOp(OpcodeArgs) { StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Result, 16, 16); } else { - OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, 8, 16); + auto DstSize = GetDstSize(Op); + OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, DstSize, Op->Flags, -1); auto Result = _VInsElement(16, 8, 0, 0, Dest, Src); - StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Result, 8, 16); + StoreResult(FPRClass, Op, Result, -1); } } else { @@ -396,8 +397,9 @@ void OpDispatchBuilder::VectorALUROp(OpcodeArgs); template void OpDispatchBuilder::VectorScalarALUOp(OpcodeArgs) { - auto Size = GetSrcSize(Op); - OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1); + auto DstSize = GetDstSize(Op); + + OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, DstSize, Op->Flags, -1); OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); // If OpSize == ElementSize then it only does the lower scalar op @@ -407,9 +409,9 @@ void OpDispatchBuilder::VectorScalarALUOp(OpcodeArgs) { OrderedNode* Result = ALUOp; - if (Size != ElementSize) { + if (DstSize != ElementSize) { // Insert the lower bits - Result = _VInsElement(Size, ElementSize, 0, 0, Dest, Result); + Result = _VInsElement(DstSize, ElementSize, 0, 0, Dest, ALUOp); } StoreResult(FPRClass, Op, Result, -1); @@ -443,11 +445,12 @@ void OpDispatchBuilder::VectorScalarALUOp(OpcodeArgs); template void OpDispatchBuilder::VectorUnaryOp(OpcodeArgs) { auto Size = GetSrcSize(Op); + auto DstSize = GetDstSize(Op); if constexpr (Scalar) { Size = ElementSize; } OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); - OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1); + OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, DstSize, Op->Flags, -1); auto ALUOp = _VFSqrt(Size, ElementSize, Src); // Overwrite our IR's op type @@ -455,7 +458,7 @@ void OpDispatchBuilder::VectorUnaryOp(OpcodeArgs) { if constexpr (Scalar) { // Insert the lower bits - auto Result = _VInsElement(GetSrcSize(Op), ElementSize, 0, 0, Dest, ALUOp); + auto Result = _VInsElement(DstSize, ElementSize, 0, 0, Dest, ALUOp); StoreResult(FPRClass, Op, Result, -1); } else { @@ -516,14 +519,8 @@ void OpDispatchBuilder::MOVQOp(OpcodeArgs) { const auto gpr = Op->Dest.Data.GPR.GPR; const auto gprIndex = gpr - X86State::REG_XMM_0; - const auto fprLowOffset = CTX->HostFeatures.SupportsAVX ? offsetof(Core::CPUState, xmm.avx.data[gprIndex][0]) - : offsetof(Core::CPUState, xmm.sse.data[gprIndex][0]); - const auto fprHighOffset = CTX->HostFeatures.SupportsAVX ? offsetof(Core::CPUState, xmm.avx.data[gprIndex][1]) - : offsetof(Core::CPUState, xmm.sse.data[gprIndex][1]); - - _StoreContext(8, FPRClass, Src, fprLowOffset); - auto Const = _Constant(0); - _StoreContext(8, GPRClass, Const, fprHighOffset); + auto Reg = _VMov(16, Src); + StoreXMMRegister(gprIndex, Reg); } else { // This is simple, just store the result @@ -1186,13 +1183,16 @@ void OpDispatchBuilder::Vector_CVT_Float_To_Int<8, true, false>(OpcodeArgs); template void OpDispatchBuilder::Scalar_CVT_Float_To_Float(OpcodeArgs) { - OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1); + const auto DstSize = GetDstSize(Op); + + OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, DstSize, Op->Flags, -1); OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); Src = _Float_FToF(DstElementSize, SrcElementSize, Src); Src = _VInsElement(16, DstElementSize, 0, 0, Dest, Src); - StoreResult(FPRClass, Op, Src, -1); + auto Result = _VInsElement(DstSize, DstElementSize, 0, 0, Dest, Src); + StoreResult(FPRClass, Op, Result, -1); } template @@ -1202,8 +1202,9 @@ void OpDispatchBuilder::Scalar_CVT_Float_To_Float<8, 4>(OpcodeArgs); template void OpDispatchBuilder::Vector_CVT_Float_To_Float(OpcodeArgs) { + const auto Size = GetDstSize(Op); + OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); - size_t Size = GetDstSize(Op); if constexpr (DstElementSize > SrcElementSize) { Src = _Vector_FToF(Size, SrcElementSize << 1, Src, SrcElementSize); @@ -1280,13 +1281,12 @@ void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int<8, true>(OpcodeArgs); void OpDispatchBuilder::MASKMOVOp(OpcodeArgs) { // Until we get correct PHI nodes this is required to be a loop unroll - const auto GPRSize = CTX->GetGPRSize(); const auto Size = uint32_t{GetSrcSize(Op)} * 8; OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1); OrderedNode *Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1); - OrderedNode *MemDest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI)); + auto MemDest = LoadGPRRegister(X86State::REG_RDI); const size_t NumElements = Size / 64; for (size_t Element = 0; Element < NumElements; ++Element) { @@ -1497,16 +1497,9 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) { } const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U; - const auto GetXMMOffset = [this](size_t i) { - if (CTX->HostFeatures.SupportsAVX) { - return offsetof(Core::CPUState, xmm.avx.data[i]); - } else { - return offsetof(Core::CPUState, xmm.sse.data[i]); - } - }; for (unsigned i = 0; i < NumRegs; ++i) { - OrderedNode *XMMReg = _LoadContext(16, FPRClass, GetXMMOffset(i)); + OrderedNode *XMMReg = LoadXMMRegister(i); OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160)); _StoreMem(FPRClass, 16, MemLocation, XMMReg, 16); @@ -1554,18 +1547,11 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) { } const auto NumRegs = CTX->Config.Is64BitMode ? 16U : 8U; - const auto GetXMMOffset = [this](size_t i) { - if (CTX->HostFeatures.SupportsAVX) { - return offsetof(Core::CPUState, xmm.avx.data[i]); - } else { - return offsetof(Core::CPUState, xmm.sse.data[i]); - } - }; for (unsigned i = 0; i < NumRegs; ++i) { OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160)); auto XMMReg = _LoadMem(FPRClass, 16, MemLocation, 16); - _StoreContext(16, FPRClass, XMMReg, GetXMMOffset(i)); + StoreXMMRegister(i, XMMReg); } } @@ -1706,11 +1692,9 @@ void OpDispatchBuilder::MOVQ2DQ(OpcodeArgs) { // This instruction is a bit special in that if the source is MMX then it zexts to 128bit if constexpr (ToXMM) { const auto Index = Op->Dest.Data.GPR.GPR - FEXCore::X86State::REG_XMM_0; - const auto Offset = CTX->HostFeatures.SupportsAVX ? offsetof(FEXCore::Core::CPUState, xmm.avx.data[Index][0]) - : offsetof(FEXCore::Core::CPUState, xmm.sse.data[Index][0]); Src = _VMov(16, Src); - _StoreContext(16, FPRClass, Src, Offset); + StoreXMMRegister(Index, Src); } else { // This is simple, just store the result @@ -2470,7 +2454,8 @@ void OpDispatchBuilder::VectorVariableBlend(OpcodeArgs) { OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1); // The mask is hardcoded to be xmm0 in this instruction - OrderedNode *Mask = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, xmm.avx.data[0])); + auto Mask = LoadXMMRegister(0); + // Each element is selected by the high bit of that element size // Dest[ElementIdx] = Xmm0[ElementIndex][HighBit] ? Src : Dest; // diff --git a/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp b/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp index 0d7791435..446176071 100644 --- a/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp +++ b/External/FEXCore/Source/Interface/HLE/Thunks/Thunks.cpp @@ -199,7 +199,7 @@ namespace FEXCore { const uint8_t GPRSize = CTX->GetGPRSize(); - emit->_StoreContext(GPRSize, IR::GPRClass, emit->_Constant(Entrypoint), offsetof(Core::CPUState, gregs[X86State::REG_R11])); + emit->_StoreRegister(emit->_Constant(Entrypoint), false, offsetof(Core::CPUState, gregs[X86State::REG_R11]), IR::GPRClass, IR::GPRFixedClass, GPRSize); emit->_ExitFunction(emit->_Constant(GuestThunkEntrypoint)); }, CTX->ThunkHandler.get(), (void*)args->target_addr);