diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp index abba04703..2b4abb2ba 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/AtomicOps.cpp @@ -69,8 +69,8 @@ DEF_OP(CASPair) { DEF_OP(CAS) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); // DataSrc = *Src1 // if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc // This will write to memory! Careful! @@ -79,13 +79,6 @@ DEF_OP(CAS) { auto Desired = GetReg(Op->Desired.ID()); auto MemSrc = GetReg(Op->Addr.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { mov(EmitSize, TMP2, Expected); casal(SubEmitSize, TMP2, Desired, MemSrc); @@ -96,9 +89,9 @@ DEF_OP(CAS) { ARMEmitter::SingleUseForwardLabel LoopExpected; Bind(&LoopTop); ldaxr(SubEmitSize, TMP2, MemSrc); - if (OpSize == 1) { + if (IROp->Size == 1) { cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTB, 0); - } else if (OpSize == 2) { + } else if (IROp->Size == 2) { cmp(EmitSize, TMP2, Expected, ARMEmitter::ExtendedType::UXTH, 0); } else { cmp(EmitSize, TMP2, Expected); @@ -120,19 +113,12 @@ DEF_OP(CAS) { DEF_OP(AtomicAdd) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { staddl(SubEmitSize, Src, MemSrc); } else { @@ -147,19 +133,12 @@ DEF_OP(AtomicAdd) { DEF_OP(AtomicSub) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { neg(EmitSize, TMP2, Src); staddl(SubEmitSize, TMP2, MemSrc); @@ -175,19 +154,12 @@ DEF_OP(AtomicSub) { DEF_OP(AtomicAnd) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { mvn(EmitSize, TMP2, Src); stclrl(SubEmitSize, TMP2, MemSrc); @@ -203,19 +175,12 @@ DEF_OP(AtomicAnd) { DEF_OP(AtomicCLR) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { stclrl(SubEmitSize, Src, MemSrc); } else { @@ -230,19 +195,12 @@ DEF_OP(AtomicCLR) { DEF_OP(AtomicOr) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { stsetl(SubEmitSize, Src, MemSrc); } else { @@ -257,19 +215,12 @@ DEF_OP(AtomicOr) { DEF_OP(AtomicXor) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { steorl(SubEmitSize, Src, MemSrc); } else { @@ -284,18 +235,11 @@ DEF_OP(AtomicXor) { DEF_OP(AtomicNeg) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - ARMEmitter::BackwardLabel LoopTop; Bind(&LoopTop); ldaxr(SubEmitSize, TMP2, MemSrc); @@ -333,19 +277,12 @@ DEF_OP(AtomicSwap) { DEF_OP(AtomicFetchAdd) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { ldaddal(SubEmitSize, Src, GetReg(Node), MemSrc); } else { @@ -361,19 +298,12 @@ DEF_OP(AtomicFetchAdd) { DEF_OP(AtomicFetchSub) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { neg(EmitSize, TMP2, Src); ldaddal(SubEmitSize, TMP2, GetReg(Node), MemSrc); @@ -390,19 +320,12 @@ DEF_OP(AtomicFetchSub) { DEF_OP(AtomicFetchAnd) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { mvn(EmitSize, TMP2, Src); ldclral(SubEmitSize, TMP2, GetReg(Node), MemSrc); @@ -419,19 +342,12 @@ DEF_OP(AtomicFetchAnd) { DEF_OP(AtomicFetchCLR) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { ldclral(SubEmitSize, Src, GetReg(Node), MemSrc); } else { @@ -447,19 +363,12 @@ DEF_OP(AtomicFetchCLR) { DEF_OP(AtomicFetchOr) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { ldsetal(SubEmitSize, Src, GetReg(Node), MemSrc); } else { @@ -475,19 +384,12 @@ DEF_OP(AtomicFetchOr) { DEF_OP(AtomicFetchXor) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); auto Src = GetReg(Op->Value.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - if (CTX->HostFeatures.SupportsAtomics) { ldeoral(SubEmitSize, Src, GetReg(Node), MemSrc); } else { @@ -503,18 +405,11 @@ DEF_OP(AtomicFetchXor) { DEF_OP(AtomicFetchNeg) { auto Op = IROp->C(); - uint8_t OpSize = IROp->Size; - LOGMAN_THROW_AA_FMT(OpSize == 8 || OpSize == 4 || OpSize == 2 || OpSize == 1, "Unexpected CAS size"); + const auto EmitSize = ConvertSize(IROp); + const auto SubEmitSize = ConvertSubRegSize8(IROp->Size); auto MemSrc = GetReg(Op->Addr.ID()); - const auto EmitSize = ConvertSize(IROp); - const auto SubEmitSize = OpSize == 8 ? ARMEmitter::SubRegSize::i64Bit : - OpSize == 4 ? ARMEmitter::SubRegSize::i32Bit : - OpSize == 2 ? ARMEmitter::SubRegSize::i16Bit : - OpSize == 1 ? ARMEmitter::SubRegSize::i8Bit : - ARMEmitter::SubRegSize::i8Bit; - ARMEmitter::BackwardLabel LoopTop; Bind(&LoopTop); ldaxr(SubEmitSize, TMP2, MemSrc); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index a0d37829b..886d70cb4 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -166,9 +166,7 @@ private: } [[nodiscard]] - ARMEmitter::SubRegSize ConvertSubRegSize16(const IR::IROp_Header* Op) { - const auto ElementSize = Op->ElementSize; - + ARMEmitter::SubRegSize ConvertSubRegSize16(uint8_t ElementSize) { LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16, "Invalid size"); return ElementSize == 1 ? ARMEmitter::SubRegSize::i8Bit : ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit : @@ -177,10 +175,20 @@ private: ARMEmitter::SubRegSize::i128Bit; } + [[nodiscard]] + ARMEmitter::SubRegSize ConvertSubRegSize16(const IR::IROp_Header* Op) { + return ConvertSubRegSize16(Op->ElementSize); + } + + [[nodiscard]] + ARMEmitter::SubRegSize ConvertSubRegSize8(uint8_t ElementSize) { + LOGMAN_THROW_AA_FMT(ElementSize != 16, "Invalid size"); + return ConvertSubRegSize16(ElementSize); + } + [[nodiscard]] ARMEmitter::SubRegSize ConvertSubRegSize8(const IR::IROp_Header* Op) { - LOGMAN_THROW_AA_FMT(Op->ElementSize != 16, "Invalid size"); - return ConvertSubRegSize16(Op); + return ConvertSubRegSize8(Op->ElementSize); } [[nodiscard]]