From 403e8f8702f18be486d14f1d4f0795ddea7a4503 Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Mon, 4 Nov 2024 17:10:50 +0100 Subject: [PATCH 1/4] CodeEmitter: Unify SingleUseForwardLabel and ForwardLabel --- CodeEmitter/CodeEmitter/ALUOps.inl | 15 ++--- CodeEmitter/CodeEmitter/BranchOps.inl | 48 ++++++---------- CodeEmitter/CodeEmitter/Emitter.h | 57 ++++++++++--------- CodeEmitter/CodeEmitter/LoadstoreOps.inl | 49 +++++----------- .../Interface/Core/Dispatcher/Dispatcher.cpp | 4 +- FEXCore/Source/Interface/Core/JIT/ALUOps.cpp | 26 ++++----- .../Source/Interface/Core/JIT/AtomicOps.cpp | 8 +-- .../Source/Interface/Core/JIT/BranchOps.cpp | 4 +- FEXCore/Source/Interface/Core/JIT/JIT.cpp | 8 +-- .../Source/Interface/Core/JIT/MemoryOps.cpp | 14 ++--- 10 files changed, 97 insertions(+), 136 deletions(-) diff --git a/CodeEmitter/CodeEmitter/ALUOps.inl b/CodeEmitter/CodeEmitter/ALUOps.inl index 777f3fa47..d6f9f45b6 100644 --- a/CodeEmitter/CodeEmitter/ALUOps.inl +++ b/CodeEmitter/CodeEmitter/ALUOps.inl @@ -43,10 +43,8 @@ public: constexpr uint32_t Op = 0b0001'0000 << 24; DataProcessing_PCRel_Imm(Op, rd, Imm); } - template - requires (std::is_same_v || std::is_same_v) - void adr(ARMEmitter::Register rd, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::ADR}); + void adr(ARMEmitter::Register rd, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::ADR}); constexpr uint32_t Op = 0b0001'0000 << 24; DataProcessing_PCRel_Imm(Op, rd, 0); } @@ -71,10 +69,8 @@ public: constexpr uint32_t Op = 0b1001'0000 << 24; DataProcessing_PCRel_Imm(Op, rd, Imm); } - template - requires (std::is_same_v || std::is_same_v) - void adrp(ARMEmitter::Register rd, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::ADRP}); + void adrp(ARMEmitter::Register rd, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::ADRP}); constexpr uint32_t Op = 0b1001'0000 << 24; DataProcessing_PCRel_Imm(Op, rd, 0); } @@ -112,8 +108,7 @@ public: } } void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) { - Label->Insts.emplace_back( - SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::LONG_ADDRESS_GEN}); + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN}); // Emit a register index and a nop. These will be backpatched. dc32(rd.Idx()); nop(); diff --git a/CodeEmitter/CodeEmitter/BranchOps.inl b/CodeEmitter/CodeEmitter/BranchOps.inl index 35f9a702a..a5975fb42 100644 --- a/CodeEmitter/CodeEmitter/BranchOps.inl +++ b/CodeEmitter/CodeEmitter/BranchOps.inl @@ -26,10 +26,8 @@ public: constexpr uint32_t Op = 0b0101'010 << 25; Branch_Conditional(Op, 0, 0, Cond, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void b(ARMEmitter::Condition Cond, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::BC}); + void b(ARMEmitter::Condition Cond, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC}); constexpr uint32_t Op = 0b0101'010 << 25; Branch_Conditional(Op, 0, 0, Cond, 0); } @@ -54,10 +52,8 @@ public: Branch_Conditional(Op, 0, 1, Cond, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void bc(ARMEmitter::Condition Cond, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::BC}); + void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC}); constexpr uint32_t Op = 0b0101'010 << 25; Branch_Conditional(Op, 0, 1, Cond, 0); } @@ -109,10 +105,8 @@ public: UnconditionalBranch(Op, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void b(LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::B}); + void b(ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::B}); constexpr uint32_t Op = 0b0001'01 << 26; UnconditionalBranch(Op, 0); @@ -139,10 +133,8 @@ public: UnconditionalBranch(Op, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void bl(LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::B}); + void bl(ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::B}); constexpr uint32_t Op = 0b1001'01 << 26; UnconditionalBranch(Op, 0); @@ -172,10 +164,8 @@ public: CompareAndBranch(Op, s, rt, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::BC}); + void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC}); constexpr uint32_t Op = 0b0011'0100 << 24; @@ -205,10 +195,8 @@ public: CompareAndBranch(Op, s, rt, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, LabelType* Label) { - AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::BC}); + void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::BC}); constexpr uint32_t Op = 0b0011'0101 << 24; @@ -238,11 +226,9 @@ public: TestAndBranch(Op, rt, Bit, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void tbz(ARMEmitter::Register rt, uint32_t Bit, LabelType* Label) { + void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) { AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::TEST_BRANCH}); + Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::TEST_BRANCH}); constexpr uint32_t Op = 0b0011'0110 << 24; @@ -271,11 +257,9 @@ public: TestAndBranch(Op, rt, Bit, Imm >> 2); } - template - requires (std::is_same_v || std::is_same_v) - void tbnz(ARMEmitter::Register rt, uint32_t Bit, LabelType* Label) { + void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) { AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::TEST_BRANCH}); + Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::TEST_BRANCH}); constexpr uint32_t Op = 0b0011'0111 << 24; TestAndBranch(Op, rt, Bit, 0); diff --git a/CodeEmitter/CodeEmitter/Emitter.h b/CodeEmitter/CodeEmitter/Emitter.h index 721e5b451..51381b6d7 100644 --- a/CodeEmitter/CodeEmitter/Emitter.h +++ b/CodeEmitter/CodeEmitter/Emitter.h @@ -513,7 +513,7 @@ enum class SVEFMaxMinImm : uint32_t { _1_0, }; -/* This `BackwardLabel` struct used for retaining a location for PC-Relative instructions. +/* This `BackwardLabel` struct is used for retaining a location for PC-Relative instructions. * This is specifically a label for a target that is logically `below` an instruction that uses it. * Which means that a branch would jump backwards. */ @@ -521,13 +521,11 @@ struct BackwardLabel { uint8_t* Location {}; }; -/* This `SingleUseForwardLabel` struct used for retaining a location for PC-Relative instructions. +/* This `ForwardLabel` struct is used for retaining a location for PC-Relative instructions. * This is specifically a label for a target that is logically `above` an instruction that uses it. * Which means that a branch would jump forwards. - * - * The `ForwardLabel` struct can be bound to multiple instructions, so it needs a vector for each bind instruction type. */ -struct SingleUseForwardLabel { +struct ForwardLabel { enum class InstType { UNKNOWN, ADR, @@ -538,12 +536,16 @@ struct SingleUseForwardLabel { RELATIVE_LOAD, LONG_ADDRESS_GEN, }; - uint8_t* Location {}; - InstType Type = InstType::UNKNOWN; -}; -struct ForwardLabel { - fextl::vector Insts {}; + struct Reference { + uint8_t* Location {}; + InstType Type = InstType::UNKNOWN; + }; + + // The first element is stored separately to avoid allocations for simple cases + Reference FirstInst; + + fextl::vector Insts; }; /* This `BiDirectionalLabel` struct used for retaining a location for PC-Relative instructions. @@ -555,14 +557,12 @@ struct BiDirectionalLabel { ForwardLabel Forward; }; -static inline void AddLocationToLabel(SingleUseForwardLabel* Label, SingleUseForwardLabel&& Location) { - LOGMAN_THROW_A_FMT(Label->Type == SingleUseForwardLabel::InstType::UNKNOWN, "Trying to bind a SingleUseForwardLabel to multiple " - "locations. Use ForwardLabel instead."); - *Label = std::move(Location); -} - -static inline void AddLocationToLabel(ForwardLabel* Label, SingleUseForwardLabel&& Location) { - Label->Insts.emplace_back(std::move(Location)); +static inline void AddLocationToLabel(ForwardLabel* Label, ForwardLabel::Reference&& Location) { + if (Label->FirstInst.Location == nullptr) { + Label->FirstInst = Location; + } else { + Label->Insts.push_back(Location); + } } // Some FCMA ASIMD instructions support a rotation argument. @@ -635,11 +635,11 @@ public: Label->Location = GetCursorAddress(); } - void Bind(const SingleUseForwardLabel* Label) { + void Bind(const ForwardLabel::Reference* Label) { uint8_t* CurrentAddress = GetCursorAddress(); // Patch up the instructions switch (Label->Type) { - case SingleUseForwardLabel::InstType::ADR: { + case ForwardLabel::InstType::ADR: { uint32_t* Instruction = reinterpret_cast(Label->Location); int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction); LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large"); @@ -651,7 +651,7 @@ public: *Instruction = Inst; break; } - case SingleUseForwardLabel::InstType::ADRP: { + case ForwardLabel::InstType::ADRP: { uint32_t* Instruction = reinterpret_cast(Label->Location); int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction); LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large"); @@ -665,7 +665,7 @@ public: break; } - case SingleUseForwardLabel::InstType::B: { + case ForwardLabel::InstType::B: { uint32_t* Instruction = reinterpret_cast(Label->Location); int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction); LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large"); @@ -679,7 +679,7 @@ public: break; } - case SingleUseForwardLabel::InstType::TEST_BRANCH: { + case ForwardLabel::InstType::TEST_BRANCH: { uint32_t* Instruction = reinterpret_cast(Label->Location); int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction); LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large"); @@ -692,8 +692,8 @@ public: break; } - case SingleUseForwardLabel::InstType::BC: - case SingleUseForwardLabel::InstType::RELATIVE_LOAD: { + case ForwardLabel::InstType::BC: + case ForwardLabel::InstType::RELATIVE_LOAD: { uint32_t* Instruction = reinterpret_cast(Label->Location); int64_t Imm = reinterpret_cast(CurrentAddress) - reinterpret_cast(Instruction); LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large"); @@ -705,7 +705,7 @@ public: *Instruction = Inst; break; } - case SingleUseForwardLabel::InstType::LONG_ADDRESS_GEN: { + case ForwardLabel::InstType::LONG_ADDRESS_GEN: { uint32_t* Instructions = reinterpret_cast(Label->Location); int64_t ImmInstOne = reinterpret_cast(CurrentAddress) - reinterpret_cast(&Instructions[0]); int64_t ImmInstTwo = reinterpret_cast(CurrentAddress) - reinterpret_cast(&Instructions[1]); @@ -753,7 +753,10 @@ public: template void Bind(ForwardLabel* Label) { if constexpr (WarnAboutEmpty) { - LOGMAN_THROW_A_FMT(Label->Insts.empty() == false, "Binding forward label that didn't have any instructions using it"); + LOGMAN_THROW_A_FMT(Label->FirstInst.Location != nullptr, "Binding forward label that didn't have any instructions using it"); + } + if (Label->FirstInst.Location) { + Bind(&Label->FirstInst); } for (auto& Inst : Label->Insts) { Bind(&Inst); diff --git a/CodeEmitter/CodeEmitter/LoadstoreOps.inl b/CodeEmitter/CodeEmitter/LoadstoreOps.inl index 3bd742e5e..49b4b9f83 100644 --- a/CodeEmitter/CodeEmitter/LoadstoreOps.inl +++ b/CodeEmitter/CodeEmitter/LoadstoreOps.inl @@ -2035,65 +2035,44 @@ public: LoadStoreLiteral(Op, prfop, static_cast(Imm >> 2) & 0x7'FFFF); } - template - requires (std::is_same_v || std::is_same_v) - void ldr(ARMEmitter::WRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldr(ARMEmitter::WRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b0001'1000 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void ldr(ARMEmitter::SRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldr(ARMEmitter::SRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b0001'1100 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void ldr(ARMEmitter::XRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldr(ARMEmitter::XRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b0101'1000 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void ldr(ARMEmitter::DRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldr(ARMEmitter::DRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b0101'1100 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void ldrsw(ARMEmitter::XRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldrsw(ARMEmitter::XRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b1001'1000 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void ldr(ARMEmitter::QRegister rt, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void ldr(ARMEmitter::QRegister rt, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b1001'1100 << 24; LoadStoreLiteral(Op, rt, 0); } - template - requires (std::is_same_v || std::is_same_v) - void prfm(ARMEmitter::Prefetch prfop, LabelType* Label) { - AddLocationToLabel( - Label, SingleUseForwardLabel {.Location = GetCursorAddress(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD}); + void prfm(ARMEmitter::Prefetch prfop, ForwardLabel* Label) { + AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress(), .Type = ForwardLabel::InstType::RELATIVE_LOAD}); constexpr uint32_t Op = 0b1101'1000 << 24; LoadStoreLiteral(Op, prfop, 0); } diff --git a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp index 2e0d2bfb2..60d250b9e 100644 --- a/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp +++ b/FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp @@ -63,7 +63,7 @@ void Dispatcher::EmitDispatcher() { // } ARMEmitter::ForwardLabel l_CTX; - ARMEmitter::SingleUseForwardLabel l_Sleep; + ARMEmitter::ForwardLabel l_Sleep; ARMEmitter::ForwardLabel l_CompileBlock; ARMEmitter::ForwardLabel l_CompileSingleStep; @@ -274,7 +274,7 @@ void Dispatcher::EmitDispatcher() { // Clobbers TMP1/2 auto EmitECExitCheck = [&]() { // Check the EC code bitmap incase we need to exit the JIT to call into native code. - ARMEmitter::SingleUseForwardLabel l_NotECCode; + ARMEmitter::ForwardLabel l_NotECCode; ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET); ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET); diff --git a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp index 2a96c992e..5b175c3d2 100644 --- a/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/ALUOps.cpp @@ -692,7 +692,7 @@ DEF_OP(ShiftFlags) { // updates for Src2=0 but anything that masks to zero. and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f); - ARMEmitter::SingleUseForwardLabel Done; + ARMEmitter::ForwardLabel Done; cbz(EmitSize, TMP1, &Done); { // PF/SF/ZF/OF @@ -773,7 +773,7 @@ DEF_OP(RotateFlags) { const auto EmitSize = Op->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit; // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz. - ARMEmitter::SingleUseForwardLabel Done; + ARMEmitter::ForwardLabel Done; cbz(EmitSize, Shift, &Done); { // Extract the last bit shifted in to CF @@ -862,7 +862,7 @@ DEF_OP(PDep) { const auto T1 = TMP4.R(); ARMEmitter::BackwardLabel NextBit; - ARMEmitter::SingleUseForwardLabel Done; + ARMEmitter::ForwardLabel Done; // First, copy the input/mask, since we'll be clobbering. Copy as 64-bit to // make this 0-uop on Firestorm. @@ -922,9 +922,9 @@ DEF_OP(PExt) { const auto BitReg = TMP2; const auto ValueReg = TMP3; - ARMEmitter::SingleUseForwardLabel EarlyExit; + ARMEmitter::ForwardLabel EarlyExit; ARMEmitter::BackwardLabel NextBit; - ARMEmitter::SingleUseForwardLabel Done; + ARMEmitter::ForwardLabel Done; cbz(EmitSize, Mask, &EarlyExit); mov(EmitSize, MaskReg, Mask); @@ -979,8 +979,8 @@ DEF_OP(LDiv) { break; } case IR::OpSize::i64Bit: { - ARMEmitter::SingleUseForwardLabel Only64Bit {}; - ARMEmitter::SingleUseForwardLabel LongDIVRet {}; + ARMEmitter::ForwardLabel Only64Bit {}; + ARMEmitter::ForwardLabel LongDIVRet {}; // Check if the upper bits match the top bit of the lower 64-bits // Sign extend the top bit of lower bits @@ -1047,8 +1047,8 @@ DEF_OP(LUDiv) { break; } case IR::OpSize::i64Bit: { - ARMEmitter::SingleUseForwardLabel Only64Bit {}; - ARMEmitter::SingleUseForwardLabel LongDIVRet {}; + ARMEmitter::ForwardLabel Only64Bit {}; + ARMEmitter::ForwardLabel LongDIVRet {}; // Check the upper bits for zero // If the upper bits are zero then we can do a 64-bit divide @@ -1115,8 +1115,8 @@ DEF_OP(LRem) { break; } case IR::OpSize::i64Bit: { - ARMEmitter::SingleUseForwardLabel Only64Bit {}; - ARMEmitter::SingleUseForwardLabel LongDIVRet {}; + ARMEmitter::ForwardLabel Only64Bit {}; + ARMEmitter::ForwardLabel LongDIVRet {}; // Check if the upper bits match the top bit of the lower 64-bits // Sign extend the top bit of lower bits @@ -1187,8 +1187,8 @@ DEF_OP(LURem) { break; } case IR::OpSize::i64Bit: { - ARMEmitter::SingleUseForwardLabel Only64Bit {}; - ARMEmitter::SingleUseForwardLabel LongDIVRet {}; + ARMEmitter::ForwardLabel Only64Bit {}; + ARMEmitter::ForwardLabel LongDIVRet {}; // Check the upper bits for zero // If the upper bits are zero then we can do a 64-bit divide diff --git a/FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp b/FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp index 79183935e..f50864d43 100644 --- a/FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp @@ -61,8 +61,8 @@ DEF_OP(CASPair) { mrs(TMP1, ARMEmitter::SystemRegister::NZCV); ARMEmitter::BackwardLabel LoopTop; - ARMEmitter::SingleUseForwardLabel LoopNotExpected; - ARMEmitter::SingleUseForwardLabel LoopExpected; + ARMEmitter::ForwardLabel LoopNotExpected; + ARMEmitter::ForwardLabel LoopExpected; Bind(&LoopTop); // This instruction sequence must be synced with HandleCASPAL_Armv8. @@ -108,8 +108,8 @@ DEF_OP(CAS) { mov(EmitSize, GetReg(Node), TMP2.R()); } else { ARMEmitter::BackwardLabel LoopTop; - ARMEmitter::SingleUseForwardLabel LoopNotExpected; - ARMEmitter::SingleUseForwardLabel LoopExpected; + ARMEmitter::ForwardLabel LoopNotExpected; + ARMEmitter::ForwardLabel LoopExpected; Bind(&LoopTop); ldaxr(SubEmitSize, TMP2, MemSrc); if (IROp->Size == IR::OpSize::i8Bit) { diff --git a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp index 56ab18729..2f9d61505 100644 --- a/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/BranchOps.cpp @@ -60,7 +60,7 @@ DEF_OP(ExitFunction) { br(TMP2); } else { #endif - ARMEmitter::SingleUseForwardLabel l_BranchHost; + ARMEmitter::ForwardLabel l_BranchHost; ldr(TMP1, &l_BranchHost); blr(TMP1); @@ -72,7 +72,7 @@ DEF_OP(ExitFunction) { #endif } else { - ARMEmitter::SingleUseForwardLabel FullLookup; + ARMEmitter::ForwardLabel FullLookup; auto RipReg = GetReg(Op->NewRIP.ID()); // L1 Cache diff --git a/FEXCore/Source/Interface/Core/JIT/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/JIT.cpp index e9de9e36b..6cd71bb46 100644 --- a/FEXCore/Source/Interface/Core/JIT/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/JIT.cpp @@ -470,7 +470,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker; uintptr_t branch = (uintptr_t)(Record)-8; ARMEmitter::Emitter emit((uint8_t*)(branch), 8); - ARMEmitter::SingleUseForwardLabel l_BranchHost; + ARMEmitter::ForwardLabel l_BranchHost; emit.ldr(TMP1, &l_BranchHost); emit.blr(TMP1); emit.Bind(&l_BranchHost); @@ -663,8 +663,8 @@ bool Arm64JITCore::IsGPR(IR::NodeID Node) const { void Arm64JITCore::EmitInterruptChecks(bool CheckTF) { if (CheckTF) { - ARMEmitter::SingleUseForwardLabel l_TFUnset; - ARMEmitter::SingleUseForwardLabel l_TFBlocked; + ARMEmitter::ForwardLabel l_TFUnset; + ARMEmitter::ForwardLabel l_TFBlocked; // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction. ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC])); @@ -714,7 +714,7 @@ void Arm64JITCore::EmitInterruptChecks(bool CheckTF) { static constexpr uint16_t SuspendMagic {0xCAFE}; ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell)); - ARMEmitter::SingleUseForwardLabel l_NoSuspend; + ARMEmitter::ForwardLabel l_NoSuspend; cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend); brk(SuspendMagic); Bind(&l_NoSuspend); diff --git a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp index f56c6bf88..b819aeecb 100644 --- a/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp @@ -899,7 +899,7 @@ DEF_OP(VLoadVectorMasked) { PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i); // If the sign bit is zero then skip the load - ARMEmitter::SingleUseForwardLabel Skip {}; + ARMEmitter::ForwardLabel Skip {}; tbz(WorkingReg, ElementSizeInBits - 1, &Skip); // Do the gather load for this element into the destination switch (IROp->ElementSize) { @@ -991,7 +991,7 @@ DEF_OP(VStoreVectorMasked) { PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i); // If the sign bit is zero then skip the load - ARMEmitter::SingleUseForwardLabel Skip {}; + ARMEmitter::ForwardLabel Skip {}; tbz(WorkingReg, ElementSizeInBits - 1, &Skip); // Do the gather load for this element into the destination switch (IROp->ElementSize) { @@ -1075,7 +1075,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, } for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) { - ARMEmitter::SingleUseForwardLabel Skip {}; + ARMEmitter::ForwardLabel Skip {}; // Extract mask element PerformMove(ElementSize, WorkingReg, MaskReg, i); @@ -1788,8 +1788,8 @@ DEF_OP(MemSet) { // // Counter is decremented regardless. - ARMEmitter::SingleUseForwardLabel BackwardImpl {}; - ARMEmitter::SingleUseForwardLabel Done {}; + ARMEmitter::ForwardLabel BackwardImpl {}; + ARMEmitter::ForwardLabel Done {}; mov(TMP1, Length.X()); if (Op->Prefix.IsInvalid()) { @@ -1980,8 +1980,8 @@ DEF_OP(MemCpy) { // // Counter is decremented regardless. - ARMEmitter::SingleUseForwardLabel BackwardImpl {}; - ARMEmitter::SingleUseForwardLabel Done {}; + ARMEmitter::ForwardLabel BackwardImpl {}; + ARMEmitter::ForwardLabel Done {}; mov(TMP1, Length.X()); mov(TMP2, MemRegDest.X()); From 9cdd759fc14b446caa020cf74515652d24c6766a Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Mon, 4 Nov 2024 17:16:56 +0100 Subject: [PATCH 2/4] CodeEmitter: Convert template specialization into function overload --- CodeEmitter/CodeEmitter/Emitter.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/CodeEmitter/CodeEmitter/Emitter.h b/CodeEmitter/CodeEmitter/Emitter.h index 51381b6d7..564f0bd00 100644 --- a/CodeEmitter/CodeEmitter/Emitter.h +++ b/CodeEmitter/CodeEmitter/Emitter.h @@ -804,7 +804,6 @@ protected: uint32_t Encode_rt2(T Reg) const { return Reg.Idx() << 10; } - template<> uint32_t Encode_rt2(uint32_t Reg) const { return Reg << 10; } @@ -840,7 +839,6 @@ protected: uint32_t Encode_rt(T Reg) const { return Reg.Idx(); } - template<> uint32_t Encode_rt(Prefetch Reg) const { return FEXCore::ToUnderlying(Reg); } From 4282f96d352d9556afd0ffea16b60ca1e88fd106 Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Mon, 4 Nov 2024 17:24:29 +0100 Subject: [PATCH 3/4] CodeEmitter: Use inline constexpr constants over constexpr functions --- CodeEmitter/CodeEmitter/Emitter.h | 116 ++++++++++++++---------------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/CodeEmitter/CodeEmitter/Emitter.h b/CodeEmitter/CodeEmitter/Emitter.h index 564f0bd00..84636c598 100644 --- a/CodeEmitter/CodeEmitter/Emitter.h +++ b/CodeEmitter/CodeEmitter/Emitter.h @@ -341,94 +341,88 @@ public: }; template -constexpr uint32_t GenSystemReg() { - return op0 << 19 | op1 << 16 | CRn << 12 | CRm << 8 | op2 << 5; -}; +inline constexpr uint32_t GenSystemReg = op0 << 19 | op1 << 16 | CRn << 12 | CRm << 8 | op2 << 5; // This `SystemRegister` enum is used for the mrs/msr instructions. enum class SystemRegister : uint32_t { - CTR_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b001>(), - DCZID_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b111>(), - TPIDR_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b010>(), - RNDR = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b000>(), - RNDRRS = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b001>(), - NZCV = GenSystemReg<0b11, 0b011, 0b0100, 0b0010, 0b000>(), - FPCR = GenSystemReg<0b11, 0b011, 0b0100, 0b0100, 0b000>(), - TPIDRRO_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b011>(), - CNTFRQ_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b000>(), - CNTVCT_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b010>(), + CTR_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b001>, + DCZID_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b111>, + TPIDR_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b010>, + RNDR = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b000>, + RNDRRS = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b001>, + NZCV = GenSystemReg<0b11, 0b011, 0b0100, 0b0010, 0b000>, + FPCR = GenSystemReg<0b11, 0b011, 0b0100, 0b0100, 0b000>, + TPIDRRO_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b011>, + CNTFRQ_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b000>, + CNTVCT_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b010>, }; template -constexpr uint32_t GenDCReg() { - return op1 << 16 | CRm << 8 | op2 << 5; -}; +inline constexpr uint32_t GenDCReg = op1 << 16 | CRm << 8 | op2 << 5; // This `DataCacheOperation` enum is used for the dc instruction. enum class DataCacheOperation : uint32_t { - IVAC = GenDCReg<0b000, 0b0110, 0b001>(), - ISW = GenDCReg<0b000, 0b0110, 0b010>(), - CSW = GenDCReg<0b000, 0b1010, 0b010>(), - CISW = GenDCReg<0b000, 0b1110, 0b010>(), - ZVA = GenDCReg<0b011, 0b0100, 0b001>(), - CVAC = GenDCReg<0b011, 0b1010, 0b001>(), - CVAU = GenDCReg<0b011, 0b1011, 0b001>(), - CIVAC = GenDCReg<0b011, 0b1110, 0b001>(), + IVAC = GenDCReg<0b000, 0b0110, 0b001>, + ISW = GenDCReg<0b000, 0b0110, 0b010>, + CSW = GenDCReg<0b000, 0b1010, 0b010>, + CISW = GenDCReg<0b000, 0b1110, 0b010>, + ZVA = GenDCReg<0b011, 0b0100, 0b001>, + CVAC = GenDCReg<0b011, 0b1010, 0b001>, + CVAU = GenDCReg<0b011, 0b1011, 0b001>, + CIVAC = GenDCReg<0b011, 0b1110, 0b001>, // MTE2 - IGVAC = GenDCReg<0b000, 0b0110, 0b011>(), - IGSW = GenDCReg<0b000, 0b0110, 0b100>(), - IGDVAC = GenDCReg<0b000, 0b0110, 0b101>(), - IGDSW = GenDCReg<0b000, 0b0110, 0b110>(), - CGSW = GenDCReg<0b000, 0b1010, 0b100>(), - CGDSW = GenDCReg<0b000, 0b1010, 0b110>(), - CIGSW = GenDCReg<0b000, 0b1110, 0b100>(), - CIGDSW = GenDCReg<0b000, 0b1110, 0b110>(), + IGVAC = GenDCReg<0b000, 0b0110, 0b011>, + IGSW = GenDCReg<0b000, 0b0110, 0b100>, + IGDVAC = GenDCReg<0b000, 0b0110, 0b101>, + IGDSW = GenDCReg<0b000, 0b0110, 0b110>, + CGSW = GenDCReg<0b000, 0b1010, 0b100>, + CGDSW = GenDCReg<0b000, 0b1010, 0b110>, + CIGSW = GenDCReg<0b000, 0b1110, 0b100>, + CIGDSW = GenDCReg<0b000, 0b1110, 0b110>, // MTE - GVA = GenDCReg<0b011, 0b0100, 0b011>(), - GZVA = GenDCReg<0b011, 0b0100, 0b100>(), - CGVAC = GenDCReg<0b011, 0b1010, 0b011>(), - CGDVAC = GenDCReg<0b011, 0b1010, 0b101>(), - CGVAP = GenDCReg<0b011, 0b1100, 0b011>(), - CGDVAP = GenDCReg<0b011, 0b1100, 0b101>(), - CGVADP = GenDCReg<0b011, 0b1101, 0b011>(), - CGDVADP = GenDCReg<0b011, 0b1101, 0b101>(), - CIGVAC = GenDCReg<0b011, 0b1110, 0b011>(), - CIGDVAC = GenDCReg<0b011, 0b1110, 0b101>(), + GVA = GenDCReg<0b011, 0b0100, 0b011>, + GZVA = GenDCReg<0b011, 0b0100, 0b100>, + CGVAC = GenDCReg<0b011, 0b1010, 0b011>, + CGDVAC = GenDCReg<0b011, 0b1010, 0b101>, + CGVAP = GenDCReg<0b011, 0b1100, 0b011>, + CGDVAP = GenDCReg<0b011, 0b1100, 0b101>, + CGVADP = GenDCReg<0b011, 0b1101, 0b011>, + CGDVADP = GenDCReg<0b011, 0b1101, 0b101>, + CIGVAC = GenDCReg<0b011, 0b1110, 0b011>, + CIGDVAC = GenDCReg<0b011, 0b1110, 0b101>, // DPB - CVAP = GenDCReg<0b011, 0b1100, 0b001>(), + CVAP = GenDCReg<0b011, 0b1100, 0b001>, // DPB2 - CVADP = GenDCReg<0b011, 0b1101, 0b001>(), + CVADP = GenDCReg<0b011, 0b1101, 0b001>, }; template -constexpr uint32_t GenHintBarrierReg() { - return CRm << 8 | op2 << 5; -} +inline constexpr uint32_t GenHintBarrierReg = CRm << 8 | op2 << 5; // This `HintRegister` enum is used for the hint instruction. enum class HintRegister : uint32_t { - NOP = GenHintBarrierReg<0b0000, 0b000>(), - YIELD = GenHintBarrierReg<0b0000, 0b001>(), - WFE = GenHintBarrierReg<0b0000, 0b010>(), - WFI = GenHintBarrierReg<0b0000, 0b011>(), - SEV = GenHintBarrierReg<0b0000, 0b100>(), - SEVL = GenHintBarrierReg<0b0000, 0b101>(), - DGH = GenHintBarrierReg<0b0000, 0b110>(), - CSDB = GenHintBarrierReg<0b0010, 0b100>(), + NOP = GenHintBarrierReg<0b0000, 0b000>, + YIELD = GenHintBarrierReg<0b0000, 0b001>, + WFE = GenHintBarrierReg<0b0000, 0b010>, + WFI = GenHintBarrierReg<0b0000, 0b011>, + SEV = GenHintBarrierReg<0b0000, 0b100>, + SEVL = GenHintBarrierReg<0b0000, 0b101>, + DGH = GenHintBarrierReg<0b0000, 0b110>, + CSDB = GenHintBarrierReg<0b0010, 0b100>, }; // This `BarrierRegister` enum is used for the various barrier instructions. enum class BarrierRegister : uint32_t { - CLREX = GenHintBarrierReg<0b0000, 0b010>(), - TCOMMIT = GenHintBarrierReg<0b0000, 0b011>(), - DSB = GenHintBarrierReg<0b0000, 0b100>(), - DMB = GenHintBarrierReg<0b0000, 0b101>(), - ISB = GenHintBarrierReg<0b0000, 0b110>(), - SB = GenHintBarrierReg<0b0000, 0b111>(), + CLREX = GenHintBarrierReg<0b0000, 0b010>, + TCOMMIT = GenHintBarrierReg<0b0000, 0b011>, + DSB = GenHintBarrierReg<0b0000, 0b100>, + DMB = GenHintBarrierReg<0b0000, 0b101>, + ISB = GenHintBarrierReg<0b0000, 0b110>, + SB = GenHintBarrierReg<0b0000, 0b111>, }; // This `BarrierScope` enum is used for the dsb/dmb instructions. From 264f3be8b4ce335c7847452f1a37001e86f2fe8a Mon Sep 17 00:00:00 2001 From: Tony Wasserka Date: Tue, 21 Jan 2025 11:08:45 +0100 Subject: [PATCH 4/4] CodeEmitter: Remove unused Bind validation logic --- CodeEmitter/CodeEmitter/Emitter.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CodeEmitter/CodeEmitter/Emitter.h b/CodeEmitter/CodeEmitter/Emitter.h index 84636c598..9b5466f85 100644 --- a/CodeEmitter/CodeEmitter/Emitter.h +++ b/CodeEmitter/CodeEmitter/Emitter.h @@ -744,11 +744,7 @@ public: // Bind a forward label to a location. // This walks all the instructions in the label's vector. // Then backpatching all instructions that have used the label. - template void Bind(ForwardLabel* Label) { - if constexpr (WarnAboutEmpty) { - LOGMAN_THROW_A_FMT(Label->FirstInst.Location != nullptr, "Binding forward label that didn't have any instructions using it"); - } if (Label->FirstInst.Location) { Bind(&Label->FirstInst); } @@ -763,7 +759,7 @@ public: if (!Label->Backward.Location) { Bind(&Label->Backward); } - Bind(&Label->Forward); + Bind(&Label->Forward); } #include