Merge pull request #4294 from neobrain/refactor_codeemitter_cleanups

CodeEmitter: Various cleanups

CodeEmitter/CodeEmitter/ALUOps.inl

@@ -43,10 +43,8 @@ public:
     constexpr uint32_t Op = 0b0001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void adr(ARMEmitter::Register rd, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::ADR});
+  void adr(ARMEmitter::Register rd, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADR});
     constexpr uint32_t Op = 0b0001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
   }
@@ -71,10 +69,8 @@ public:
     constexpr uint32_t Op = 0b1001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, Imm);
   }
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void adrp(ARMEmitter::Register rd, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::ADRP});
+  void adrp(ARMEmitter::Register rd, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::ADRP});
     constexpr uint32_t Op = 0b1001'0000 << 24;
     DataProcessing_PCRel_Imm(Op, rd, 0);
   }
@@ -112,8 +108,7 @@ public:
     }
   }
   void LongAddressGen(ARMEmitter::Register rd, ForwardLabel* Label) {
-    Label->Insts.emplace_back(
-      SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::LONG_ADDRESS_GEN});
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::LONG_ADDRESS_GEN});
     // Emit a register index and a nop. These will be backpatched.
     dc32(rd.Idx());
     nop();

CodeEmitter/CodeEmitter/BranchOps.inl

@@ -26,10 +26,8 @@ public:
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, Imm >> 2);
   }
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void b(ARMEmitter::Condition Cond, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::BC});
+  void b(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 0, Cond, 0);
   }
@@ -54,10 +52,8 @@ public:
     Branch_Conditional(Op, 0, 1, Cond, Imm >> 2);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void bc(ARMEmitter::Condition Cond, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::BC});
+  void bc(ARMEmitter::Condition Cond, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
     constexpr uint32_t Op = 0b0101'010 << 25;
     Branch_Conditional(Op, 0, 1, Cond, 0);
   }
@@ -109,10 +105,8 @@ public:
 
     UnconditionalBranch(Op, Imm >> 2);
   }
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void b(LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::B});
+  void b(ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b0001'01 << 26;
 
     UnconditionalBranch(Op, 0);
@@ -139,10 +133,8 @@ public:
 
     UnconditionalBranch(Op, Imm >> 2);
   }
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void bl(LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::B});
+  void bl(ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::B});
     constexpr uint32_t Op = 0b1001'01 << 26;
 
     UnconditionalBranch(Op, 0);
@@ -172,10 +164,8 @@ public:
     CompareAndBranch(Op, s, rt, Imm >> 2);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::BC});
+  void cbz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0100 << 24;
 
@@ -205,10 +195,8 @@ public:
     CompareAndBranch(Op, s, rt, Imm >> 2);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, LabelType* Label) {
-    AddLocationToLabel(Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::BC});
+  void cbnz(ARMEmitter::Size s, ARMEmitter::Register rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::BC});
 
     constexpr uint32_t Op = 0b0011'0101 << 24;
 
@@ -238,11 +226,9 @@ public:
     TestAndBranch(Op, rt, Bit, Imm >> 2);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void tbz(ARMEmitter::Register rt, uint32_t Bit, LabelType* Label) {
+  void tbz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::TEST_BRANCH});
+      Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
 
     constexpr uint32_t Op = 0b0011'0110 << 24;
 
@@ -271,11 +257,9 @@ public:
     TestAndBranch(Op, rt, Bit, Imm >> 2);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void tbnz(ARMEmitter::Register rt, uint32_t Bit, LabelType* Label) {
+  void tbnz(ARMEmitter::Register rt, uint32_t Bit, ForwardLabel* Label) {
     AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::TEST_BRANCH});
+      Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::TEST_BRANCH});
     constexpr uint32_t Op = 0b0011'0111 << 24;
 
     TestAndBranch(Op, rt, Bit, 0);

CodeEmitter/CodeEmitter/Emitter.h

@@ -341,94 +341,88 @@ public:
 };
 
 template<uint32_t op0, uint32_t op1, uint32_t CRn, uint32_t CRm, uint32_t op2>
-constexpr uint32_t GenSystemReg() {
-  return op0 << 19 | op1 << 16 | CRn << 12 | CRm << 8 | op2 << 5;
-};
+inline constexpr uint32_t GenSystemReg = op0 << 19 | op1 << 16 | CRn << 12 | CRm << 8 | op2 << 5;
 
 // This `SystemRegister` enum is used for the mrs/msr instructions.
 enum class SystemRegister : uint32_t {
-  CTR_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b001>(),
-  DCZID_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b111>(),
-  TPIDR_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b010>(),
-  RNDR = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b000>(),
-  RNDRRS = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b001>(),
-  NZCV = GenSystemReg<0b11, 0b011, 0b0100, 0b0010, 0b000>(),
-  FPCR = GenSystemReg<0b11, 0b011, 0b0100, 0b0100, 0b000>(),
-  TPIDRRO_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b011>(),
-  CNTFRQ_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b000>(),
-  CNTVCT_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b010>(),
+  CTR_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b001>,
+  DCZID_EL0 = GenSystemReg<0b11, 0b011, 0b0000, 0b0000, 0b111>,
+  TPIDR_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b010>,
+  RNDR = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b000>,
+  RNDRRS = GenSystemReg<0b11, 0b011, 0b0010, 0b0100, 0b001>,
+  NZCV = GenSystemReg<0b11, 0b011, 0b0100, 0b0010, 0b000>,
+  FPCR = GenSystemReg<0b11, 0b011, 0b0100, 0b0100, 0b000>,
+  TPIDRRO_EL0 = GenSystemReg<0b11, 0b011, 0b1101, 0b0000, 0b011>,
+  CNTFRQ_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b000>,
+  CNTVCT_EL0 = GenSystemReg<0b11, 0b011, 0b1110, 0b0000, 0b010>,
 };
 
 template<uint32_t op1, uint32_t CRm, uint32_t op2>
-constexpr uint32_t GenDCReg() {
-  return op1 << 16 | CRm << 8 | op2 << 5;
-};
+inline constexpr uint32_t GenDCReg = op1 << 16 | CRm << 8 | op2 << 5;
 
 // This `DataCacheOperation` enum is used for the dc instruction.
 enum class DataCacheOperation : uint32_t {
-  IVAC = GenDCReg<0b000, 0b0110, 0b001>(),
-  ISW = GenDCReg<0b000, 0b0110, 0b010>(),
-  CSW = GenDCReg<0b000, 0b1010, 0b010>(),
-  CISW = GenDCReg<0b000, 0b1110, 0b010>(),
-  ZVA = GenDCReg<0b011, 0b0100, 0b001>(),
-  CVAC = GenDCReg<0b011, 0b1010, 0b001>(),
-  CVAU = GenDCReg<0b011, 0b1011, 0b001>(),
-  CIVAC = GenDCReg<0b011, 0b1110, 0b001>(),
+  IVAC = GenDCReg<0b000, 0b0110, 0b001>,
+  ISW = GenDCReg<0b000, 0b0110, 0b010>,
+  CSW = GenDCReg<0b000, 0b1010, 0b010>,
+  CISW = GenDCReg<0b000, 0b1110, 0b010>,
+  ZVA = GenDCReg<0b011, 0b0100, 0b001>,
+  CVAC = GenDCReg<0b011, 0b1010, 0b001>,
+  CVAU = GenDCReg<0b011, 0b1011, 0b001>,
+  CIVAC = GenDCReg<0b011, 0b1110, 0b001>,
 
   // MTE2
-  IGVAC = GenDCReg<0b000, 0b0110, 0b011>(),
-  IGSW = GenDCReg<0b000, 0b0110, 0b100>(),
-  IGDVAC = GenDCReg<0b000, 0b0110, 0b101>(),
-  IGDSW = GenDCReg<0b000, 0b0110, 0b110>(),
-  CGSW = GenDCReg<0b000, 0b1010, 0b100>(),
-  CGDSW = GenDCReg<0b000, 0b1010, 0b110>(),
-  CIGSW = GenDCReg<0b000, 0b1110, 0b100>(),
-  CIGDSW = GenDCReg<0b000, 0b1110, 0b110>(),
+  IGVAC = GenDCReg<0b000, 0b0110, 0b011>,
+  IGSW = GenDCReg<0b000, 0b0110, 0b100>,
+  IGDVAC = GenDCReg<0b000, 0b0110, 0b101>,
+  IGDSW = GenDCReg<0b000, 0b0110, 0b110>,
+  CGSW = GenDCReg<0b000, 0b1010, 0b100>,
+  CGDSW = GenDCReg<0b000, 0b1010, 0b110>,
+  CIGSW = GenDCReg<0b000, 0b1110, 0b100>,
+  CIGDSW = GenDCReg<0b000, 0b1110, 0b110>,
 
   // MTE
-  GVA = GenDCReg<0b011, 0b0100, 0b011>(),
-  GZVA = GenDCReg<0b011, 0b0100, 0b100>(),
-  CGVAC = GenDCReg<0b011, 0b1010, 0b011>(),
-  CGDVAC = GenDCReg<0b011, 0b1010, 0b101>(),
-  CGVAP = GenDCReg<0b011, 0b1100, 0b011>(),
-  CGDVAP = GenDCReg<0b011, 0b1100, 0b101>(),
-  CGVADP = GenDCReg<0b011, 0b1101, 0b011>(),
-  CGDVADP = GenDCReg<0b011, 0b1101, 0b101>(),
-  CIGVAC = GenDCReg<0b011, 0b1110, 0b011>(),
-  CIGDVAC = GenDCReg<0b011, 0b1110, 0b101>(),
+  GVA = GenDCReg<0b011, 0b0100, 0b011>,
+  GZVA = GenDCReg<0b011, 0b0100, 0b100>,
+  CGVAC = GenDCReg<0b011, 0b1010, 0b011>,
+  CGDVAC = GenDCReg<0b011, 0b1010, 0b101>,
+  CGVAP = GenDCReg<0b011, 0b1100, 0b011>,
+  CGDVAP = GenDCReg<0b011, 0b1100, 0b101>,
+  CGVADP = GenDCReg<0b011, 0b1101, 0b011>,
+  CGDVADP = GenDCReg<0b011, 0b1101, 0b101>,
+  CIGVAC = GenDCReg<0b011, 0b1110, 0b011>,
+  CIGDVAC = GenDCReg<0b011, 0b1110, 0b101>,
 
   // DPB
-  CVAP = GenDCReg<0b011, 0b1100, 0b001>(),
+  CVAP = GenDCReg<0b011, 0b1100, 0b001>,
 
   // DPB2
-  CVADP = GenDCReg<0b011, 0b1101, 0b001>(),
+  CVADP = GenDCReg<0b011, 0b1101, 0b001>,
 };
 
 template<uint32_t CRm, uint32_t op2>
-constexpr uint32_t GenHintBarrierReg() {
-  return CRm << 8 | op2 << 5;
-}
+inline constexpr uint32_t GenHintBarrierReg = CRm << 8 | op2 << 5;
 
 // This `HintRegister` enum is used for the hint instruction.
 enum class HintRegister : uint32_t {
-  NOP = GenHintBarrierReg<0b0000, 0b000>(),
-  YIELD = GenHintBarrierReg<0b0000, 0b001>(),
-  WFE = GenHintBarrierReg<0b0000, 0b010>(),
-  WFI = GenHintBarrierReg<0b0000, 0b011>(),
-  SEV = GenHintBarrierReg<0b0000, 0b100>(),
-  SEVL = GenHintBarrierReg<0b0000, 0b101>(),
-  DGH = GenHintBarrierReg<0b0000, 0b110>(),
-  CSDB = GenHintBarrierReg<0b0010, 0b100>(),
+  NOP = GenHintBarrierReg<0b0000, 0b000>,
+  YIELD = GenHintBarrierReg<0b0000, 0b001>,
+  WFE = GenHintBarrierReg<0b0000, 0b010>,
+  WFI = GenHintBarrierReg<0b0000, 0b011>,
+  SEV = GenHintBarrierReg<0b0000, 0b100>,
+  SEVL = GenHintBarrierReg<0b0000, 0b101>,
+  DGH = GenHintBarrierReg<0b0000, 0b110>,
+  CSDB = GenHintBarrierReg<0b0010, 0b100>,
 };
 
 // This `BarrierRegister` enum is used for the various barrier instructions.
 enum class BarrierRegister : uint32_t {
-  CLREX = GenHintBarrierReg<0b0000, 0b010>(),
-  TCOMMIT = GenHintBarrierReg<0b0000, 0b011>(),
-  DSB = GenHintBarrierReg<0b0000, 0b100>(),
-  DMB = GenHintBarrierReg<0b0000, 0b101>(),
-  ISB = GenHintBarrierReg<0b0000, 0b110>(),
-  SB = GenHintBarrierReg<0b0000, 0b111>(),
+  CLREX = GenHintBarrierReg<0b0000, 0b010>,
+  TCOMMIT = GenHintBarrierReg<0b0000, 0b011>,
+  DSB = GenHintBarrierReg<0b0000, 0b100>,
+  DMB = GenHintBarrierReg<0b0000, 0b101>,
+  ISB = GenHintBarrierReg<0b0000, 0b110>,
+  SB = GenHintBarrierReg<0b0000, 0b111>,
 };
 
 // This `BarrierScope` enum is used for the dsb/dmb instructions.
@@ -513,7 +507,7 @@ enum class SVEFMaxMinImm : uint32_t {
   _1_0,
 };
 
-/* This `BackwardLabel` struct used for retaining a location for PC-Relative instructions.
+/* This `BackwardLabel` struct is used for retaining a location for PC-Relative instructions.
  * This is specifically a label for a target that is logically `below` an instruction that uses it.
  * Which means that a branch would jump backwards.
  */
@@ -521,13 +515,11 @@ struct BackwardLabel {
   uint8_t* Location {};
 };
 
-/* This `SingleUseForwardLabel` struct used for retaining a location for PC-Relative instructions.
+/* This `ForwardLabel` struct is used for retaining a location for PC-Relative instructions.
  * This is specifically a label for a target that is logically `above` an instruction that uses it.
  * Which means that a branch would jump forwards.
- *
- * The `ForwardLabel` struct can be bound to multiple instructions, so it needs a vector for each bind instruction type.
  */
-struct SingleUseForwardLabel {
+struct ForwardLabel {
   enum class InstType {
     UNKNOWN,
     ADR,
@@ -538,12 +530,16 @@ struct SingleUseForwardLabel {
     RELATIVE_LOAD,
     LONG_ADDRESS_GEN,
   };
-  uint8_t* Location {};
-  InstType Type = InstType::UNKNOWN;
-};
 
-struct ForwardLabel {
-  fextl::vector<SingleUseForwardLabel> Insts {};
+  struct Reference {
+    uint8_t* Location {};
+    InstType Type = InstType::UNKNOWN;
+  };
+
+  // The first element is stored separately to avoid allocations for simple cases
+  Reference FirstInst;
+
+  fextl::vector<Reference> Insts;
 };
 
 /* This `BiDirectionalLabel` struct used for retaining a location for PC-Relative instructions.
@@ -555,14 +551,12 @@ struct BiDirectionalLabel {
   ForwardLabel Forward;
 };
 
-static inline void AddLocationToLabel(SingleUseForwardLabel* Label, SingleUseForwardLabel&& Location) {
-  LOGMAN_THROW_A_FMT(Label->Type == SingleUseForwardLabel::InstType::UNKNOWN, "Trying to bind a SingleUseForwardLabel to multiple "
-                                                                              "locations. Use ForwardLabel instead.");
-  *Label = std::move(Location);
-}
-
-static inline void AddLocationToLabel(ForwardLabel* Label, SingleUseForwardLabel&& Location) {
-  Label->Insts.emplace_back(std::move(Location));
+static inline void AddLocationToLabel(ForwardLabel* Label, ForwardLabel::Reference&& Location) {
+  if (Label->FirstInst.Location == nullptr) {
+    Label->FirstInst = Location;
+  } else {
+    Label->Insts.push_back(Location);
+  }
 }
 
 // Some FCMA ASIMD instructions support a rotation argument.
@@ -635,11 +629,11 @@ public:
     Label->Location = GetCursorAddress<uint8_t*>();
   }
 
-  void Bind(const SingleUseForwardLabel* Label) {
+  void Bind(const ForwardLabel::Reference* Label) {
     uint8_t* CurrentAddress = GetCursorAddress<uint8_t*>();
     // Patch up the instructions
     switch (Label->Type) {
-    case SingleUseForwardLabel::InstType::ADR: {
+    case ForwardLabel::InstType::ADR: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
       LOGMAN_THROW_A_FMT(IsADRRange(Imm), "Unscaled offset too large");
@@ -651,7 +645,7 @@ public:
       *Instruction = Inst;
       break;
     }
-    case SingleUseForwardLabel::InstType::ADRP: {
+    case ForwardLabel::InstType::ADRP: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
       LOGMAN_THROW_A_FMT(IsADRPRange(Imm) && IsADRPAligned(Imm), "Unscaled offset too large");
@@ -665,7 +659,7 @@ public:
       break;
     }
 
-    case SingleUseForwardLabel::InstType::B: {
+    case ForwardLabel::InstType::B: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
       LOGMAN_THROW_A_FMT(Imm >= -134217728 && Imm <= 134217724 && ((Imm & 0b11) == 0), "Unscaled offset too large");
@@ -679,7 +673,7 @@ public:
       break;
     }
 
-    case SingleUseForwardLabel::InstType::TEST_BRANCH: {
+    case ForwardLabel::InstType::TEST_BRANCH: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
       LOGMAN_THROW_A_FMT(Imm >= -32768 && Imm <= 32764 && ((Imm & 0b11) == 0), "Unscaled offset too large");
@@ -692,8 +686,8 @@ public:
 
       break;
     }
-    case SingleUseForwardLabel::InstType::BC:
-    case SingleUseForwardLabel::InstType::RELATIVE_LOAD: {
+    case ForwardLabel::InstType::BC:
+    case ForwardLabel::InstType::RELATIVE_LOAD: {
       uint32_t* Instruction = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t Imm = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(Instruction);
       LOGMAN_THROW_A_FMT(Imm >= -1048576 && Imm <= 1048575 && ((Imm & 0b11) == 0), "Unscaled offset too large");
@@ -705,7 +699,7 @@ public:
       *Instruction = Inst;
       break;
     }
-    case SingleUseForwardLabel::InstType::LONG_ADDRESS_GEN: {
+    case ForwardLabel::InstType::LONG_ADDRESS_GEN: {
       uint32_t* Instructions = reinterpret_cast<uint32_t*>(Label->Location);
       int64_t ImmInstOne = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[0]);
       int64_t ImmInstTwo = reinterpret_cast<int64_t>(CurrentAddress) - reinterpret_cast<int64_t>(&Instructions[1]);
@@ -750,10 +744,9 @@ public:
   // Bind a forward label to a location.
   // This walks all the instructions in the label's vector.
   // Then backpatching all instructions that have used the label.
-  template<bool WarnAboutEmpty = false>
   void Bind(ForwardLabel* Label) {
-    if constexpr (WarnAboutEmpty) {
-      LOGMAN_THROW_A_FMT(Label->Insts.empty() == false, "Binding forward label that didn't have any instructions using it");
+    if (Label->FirstInst.Location) {
+      Bind(&Label->FirstInst);
     }
     for (auto& Inst : Label->Insts) {
       Bind(&Inst);
@@ -766,7 +759,7 @@ public:
     if (!Label->Backward.Location) {
       Bind(&Label->Backward);
     }
-    Bind<false>(&Label->Forward);
+    Bind(&Label->Forward);
   }
 
 #include <CodeEmitter/VixlUtils.inl>
@@ -801,7 +794,6 @@ protected:
   uint32_t Encode_rt2(T Reg) const {
     return Reg.Idx() << 10;
   }
-  template<>
   uint32_t Encode_rt2(uint32_t Reg) const {
     return Reg << 10;
   }
@@ -837,7 +829,6 @@ protected:
   uint32_t Encode_rt(T Reg) const {
     return Reg.Idx();
   }
-  template<>
   uint32_t Encode_rt(Prefetch Reg) const {
     return FEXCore::ToUnderlying(Reg);
   }

CodeEmitter/CodeEmitter/LoadstoreOps.inl

@@ -2035,65 +2035,44 @@ public:
     LoadStoreLiteral(Op, prfop, static_cast<uint32_t>(Imm >> 2) & 0x7'FFFF);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldr(ARMEmitter::WRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldr(ARMEmitter::WRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b0001'1000 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldr(ARMEmitter::SRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldr(ARMEmitter::SRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b0001'1100 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldr(ARMEmitter::XRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldr(ARMEmitter::XRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b0101'1000 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldr(ARMEmitter::DRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldr(ARMEmitter::DRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b0101'1100 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldrsw(ARMEmitter::XRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldrsw(ARMEmitter::XRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b1001'1000 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void ldr(ARMEmitter::QRegister rt, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void ldr(ARMEmitter::QRegister rt, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b1001'1100 << 24;
     LoadStoreLiteral(Op, rt, 0);
   }
 
-  template<typename LabelType>
-  requires (std::is_same_v<LabelType, ForwardLabel> || std::is_same_v<LabelType, SingleUseForwardLabel>)
-  void prfm(ARMEmitter::Prefetch prfop, LabelType* Label) {
-    AddLocationToLabel(
-      Label, SingleUseForwardLabel {.Location = GetCursorAddress<uint8_t*>(), .Type = SingleUseForwardLabel::InstType::RELATIVE_LOAD});
+  void prfm(ARMEmitter::Prefetch prfop, ForwardLabel* Label) {
+    AddLocationToLabel(Label, ForwardLabel::Reference {.Location = GetCursorAddress<uint8_t*>(), .Type = ForwardLabel::InstType::RELATIVE_LOAD});
     constexpr uint32_t Op = 0b1101'1000 << 24;
     LoadStoreLiteral(Op, prfop, 0);
   }

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -63,7 +63,7 @@ void Dispatcher::EmitDispatcher() {
   // }
 
   ARMEmitter::ForwardLabel l_CTX;
-  ARMEmitter::SingleUseForwardLabel l_Sleep;
+  ARMEmitter::ForwardLabel l_Sleep;
   ARMEmitter::ForwardLabel l_CompileBlock;
   ARMEmitter::ForwardLabel l_CompileSingleStep;
 
@@ -274,7 +274,7 @@ void Dispatcher::EmitDispatcher() {
   // Clobbers TMP1/2
   auto EmitECExitCheck = [&]() {
     // Check the EC code bitmap incase we need to exit the JIT to call into native code.
-    ARMEmitter::SingleUseForwardLabel l_NotECCode;
+    ARMEmitter::ForwardLabel l_NotECCode;
     ldr(TMP1, ARMEmitter::XReg::x18, TEB_PEB_OFFSET);
     ldr(TMP1, TMP1, PEB_EC_CODE_BITMAP_OFFSET);
 

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -692,7 +692,7 @@ DEF_OP(ShiftFlags) {
   // updates for Src2=0 but anything that masks to zero.
   and_(ARMEmitter::Size::i32Bit, TMP1, Src2, OpSize == IR::OpSize::i64Bit ? 0x3f : 0x1f);
 
-  ARMEmitter::SingleUseForwardLabel Done;
+  ARMEmitter::ForwardLabel Done;
   cbz(EmitSize, TMP1, &Done);
   {
     // PF/SF/ZF/OF
@@ -773,7 +773,7 @@ DEF_OP(RotateFlags) {
   const auto EmitSize = Op->Size == IR::OpSize::i64Bit ? ARMEmitter::Size::i64Bit : ARMEmitter::Size::i32Bit;
 
   // If shift=0, flags are unaffected. Wrap the whole implementation in a cbz.
-  ARMEmitter::SingleUseForwardLabel Done;
+  ARMEmitter::ForwardLabel Done;
   cbz(EmitSize, Shift, &Done);
   {
     // Extract the last bit shifted in to CF
@@ -862,7 +862,7 @@ DEF_OP(PDep) {
     const auto T1 = TMP4.R();
 
     ARMEmitter::BackwardLabel NextBit;
-    ARMEmitter::SingleUseForwardLabel Done;
+    ARMEmitter::ForwardLabel Done;
 
     // First, copy the input/mask, since we'll be clobbering. Copy as 64-bit to
     // make this 0-uop on Firestorm.
@@ -922,9 +922,9 @@ DEF_OP(PExt) {
     const auto BitReg = TMP2;
     const auto ValueReg = TMP3;
 
-    ARMEmitter::SingleUseForwardLabel EarlyExit;
+    ARMEmitter::ForwardLabel EarlyExit;
     ARMEmitter::BackwardLabel NextBit;
-    ARMEmitter::SingleUseForwardLabel Done;
+    ARMEmitter::ForwardLabel Done;
 
     cbz(EmitSize, Mask, &EarlyExit);
     mov(EmitSize, MaskReg, Mask);
@@ -979,8 +979,8 @@ DEF_OP(LDiv) {
     break;
   }
   case IR::OpSize::i64Bit: {
-    ARMEmitter::SingleUseForwardLabel Only64Bit {};
-    ARMEmitter::SingleUseForwardLabel LongDIVRet {};
+    ARMEmitter::ForwardLabel Only64Bit {};
+    ARMEmitter::ForwardLabel LongDIVRet {};
 
     // Check if the upper bits match the top bit of the lower 64-bits
     // Sign extend the top bit of lower bits
@@ -1047,8 +1047,8 @@ DEF_OP(LUDiv) {
     break;
   }
   case IR::OpSize::i64Bit: {
-    ARMEmitter::SingleUseForwardLabel Only64Bit {};
-    ARMEmitter::SingleUseForwardLabel LongDIVRet {};
+    ARMEmitter::ForwardLabel Only64Bit {};
+    ARMEmitter::ForwardLabel LongDIVRet {};
 
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
@@ -1115,8 +1115,8 @@ DEF_OP(LRem) {
     break;
   }
   case IR::OpSize::i64Bit: {
-    ARMEmitter::SingleUseForwardLabel Only64Bit {};
-    ARMEmitter::SingleUseForwardLabel LongDIVRet {};
+    ARMEmitter::ForwardLabel Only64Bit {};
+    ARMEmitter::ForwardLabel LongDIVRet {};
 
     // Check if the upper bits match the top bit of the lower 64-bits
     // Sign extend the top bit of lower bits
@@ -1187,8 +1187,8 @@ DEF_OP(LURem) {
     break;
   }
   case IR::OpSize::i64Bit: {
-    ARMEmitter::SingleUseForwardLabel Only64Bit {};
-    ARMEmitter::SingleUseForwardLabel LongDIVRet {};
+    ARMEmitter::ForwardLabel Only64Bit {};
+    ARMEmitter::ForwardLabel LongDIVRet {};
 
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide

FEXCore/Source/Interface/Core/JIT/AtomicOps.cpp

@@ -61,8 +61,8 @@ DEF_OP(CASPair) {
     mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
 
     ARMEmitter::BackwardLabel LoopTop;
-    ARMEmitter::SingleUseForwardLabel LoopNotExpected;
-    ARMEmitter::SingleUseForwardLabel LoopExpected;
+    ARMEmitter::ForwardLabel LoopNotExpected;
+    ARMEmitter::ForwardLabel LoopExpected;
     Bind(&LoopTop);
 
     // This instruction sequence must be synced with HandleCASPAL_Armv8.
@@ -108,8 +108,8 @@ DEF_OP(CAS) {
     mov(EmitSize, GetReg(Node), TMP2.R());
   } else {
     ARMEmitter::BackwardLabel LoopTop;
-    ARMEmitter::SingleUseForwardLabel LoopNotExpected;
-    ARMEmitter::SingleUseForwardLabel LoopExpected;
+    ARMEmitter::ForwardLabel LoopNotExpected;
+    ARMEmitter::ForwardLabel LoopExpected;
     Bind(&LoopTop);
     ldaxr(SubEmitSize, TMP2, MemSrc);
     if (IROp->Size == IR::OpSize::i8Bit) {

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -60,7 +60,7 @@ DEF_OP(ExitFunction) {
       br(TMP2);
     } else {
 #endif
-      ARMEmitter::SingleUseForwardLabel l_BranchHost;
+      ARMEmitter::ForwardLabel l_BranchHost;
       ldr(TMP1, &l_BranchHost);
       blr(TMP1);
 
@@ -72,7 +72,7 @@ DEF_OP(ExitFunction) {
 #endif
   } else {
 
-    ARMEmitter::SingleUseForwardLabel FullLookup;
+    ARMEmitter::ForwardLabel FullLookup;
     auto RipReg = GetReg(Op->NewRIP.ID());
 
     // L1 Cache

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -470,7 +470,7 @@ static void DirectBlockDelinker(FEXCore::Core::CpuStateFrame* Frame, FEXCore::Co
   auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
   uintptr_t branch = (uintptr_t)(Record)-8;
   ARMEmitter::Emitter emit((uint8_t*)(branch), 8);
-  ARMEmitter::SingleUseForwardLabel l_BranchHost;
+  ARMEmitter::ForwardLabel l_BranchHost;
   emit.ldr(TMP1, &l_BranchHost);
   emit.blr(TMP1);
   emit.Bind(&l_BranchHost);
@@ -663,8 +663,8 @@ bool Arm64JITCore::IsGPR(IR::NodeID Node) const {
 
 void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
   if (CheckTF) {
-    ARMEmitter::SingleUseForwardLabel l_TFUnset;
-    ARMEmitter::SingleUseForwardLabel l_TFBlocked;
+    ARMEmitter::ForwardLabel l_TFUnset;
+    ARMEmitter::ForwardLabel l_TFBlocked;
 
     // Note that this needs to be before the below suspend checks, as X86 checks this flag immediately after executing an instruction.
     ldrb(TMP1, STATE_PTR(CpuStateFrame, State.flags[X86State::RFLAG_TF_RAW_LOC]));
@@ -714,7 +714,7 @@ void Arm64JITCore::EmitInterruptChecks(bool CheckTF) {
   static constexpr uint16_t SuspendMagic {0xCAFE};
 
   ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));
-  ARMEmitter::SingleUseForwardLabel l_NoSuspend;
+  ARMEmitter::ForwardLabel l_NoSuspend;
   cbz(ARMEmitter::Size::i32Bit, TMP2, &l_NoSuspend);
   brk(SuspendMagic);
   Bind(&l_NoSuspend);

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -899,7 +899,7 @@ DEF_OP(VLoadVectorMasked) {
       PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i);
 
       // If the sign bit is zero then skip the load
-      ARMEmitter::SingleUseForwardLabel Skip {};
+      ARMEmitter::ForwardLabel Skip {};
       tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
@@ -991,7 +991,7 @@ DEF_OP(VStoreVectorMasked) {
       PerformMove(IROp->ElementSize, WorkingReg, MaskReg, i);
 
       // If the sign bit is zero then skip the load
-      ARMEmitter::SingleUseForwardLabel Skip {};
+      ARMEmitter::ForwardLabel Skip {};
       tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
       // Do the gather load for this element into the destination
       switch (IROp->ElementSize) {
@@ -1075,7 +1075,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
   }
 
   for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) {
-    ARMEmitter::SingleUseForwardLabel Skip {};
+    ARMEmitter::ForwardLabel Skip {};
     // Extract mask element
     PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
@@ -1788,8 +1788,8 @@ DEF_OP(MemSet) {
   //
   // Counter is decremented regardless.
 
-  ARMEmitter::SingleUseForwardLabel BackwardImpl {};
-  ARMEmitter::SingleUseForwardLabel Done {};
+  ARMEmitter::ForwardLabel BackwardImpl {};
+  ARMEmitter::ForwardLabel Done {};
 
   mov(TMP1, Length.X());
   if (Op->Prefix.IsInvalid()) {
@@ -1980,8 +1980,8 @@ DEF_OP(MemCpy) {
   //
   // Counter is decremented regardless.
 
-  ARMEmitter::SingleUseForwardLabel BackwardImpl {};
-  ARMEmitter::SingleUseForwardLabel Done {};
+  ARMEmitter::ForwardLabel BackwardImpl {};
+  ARMEmitter::ForwardLabel Done {};
 
   mov(TMP1, Length.X());
   mov(TMP2, MemRegDest.X());