From 0bb4091e34e218330355b8fbbe24b0e55cb5ded1 Mon Sep 17 00:00:00 2001 From: lioncash Date: Tue, 4 Oct 2022 18:24:39 +0000 Subject: [PATCH] IR: Handle 256-bit VFCMPEQ Extends VFCMPEQ to handle 256-bit vectors. --- .../Interface/Core/Interpreter/VectorOps.cpp | 23 ++-- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 100 +++++++++++++----- .../Interface/Core/JIT/x86_64/VectorOps.cpp | 45 +++++--- 3 files changed, 119 insertions(+), 49 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 57a915eb3..ddac9ab32 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -1073,7 +1073,7 @@ DEF_OP(VCMPLTZ) { } DEF_OP(VFCMPEQ) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src1 = GetSrc(Data->SSAData, Op->Vector1); @@ -1081,21 +1081,28 @@ DEF_OP(VFCMPEQ) { const auto Func = [](auto a, auto b) { return a == b ? ~0ULL : 0; }; - uint8_t Tmp[16]; - const uint8_t Elements = OpSize / Op->Header.ElementSize; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; - if (Op->Header.ElementSize == OpSize) { - switch (Op->Header.ElementSize) { + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / ElementSize; + const auto IsScalar = ElementSize == OpSize; + + if (IsScalar) { + switch (ElementSize) { DO_SCALAR_COMPARE_OP(4, float, uint32_t, Func); DO_SCALAR_COMPARE_OP(8, double, uint64_t, Func); - default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize); + default: + LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", ElementSize); + break; } } else { - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_COMPARE_OP(4, float, uint32_t, Func); DO_VECTOR_COMPARE_OP(8, double, uint64_t, Func); - default: LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", Op->Header.ElementSize); + default: + LOGMAN_MSG_A_FMT("Unsupported elementSize: {}", ElementSize); + break; } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 0a0866a93..d2fd07423 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -2482,38 +2482,86 @@ DEF_OP(VCMPLTZ) { } DEF_OP(VFCMPEQ) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - if (Op->Header.ElementSize == OpSize) { - // Scalar - switch (Op->Header.ElementSize) { - case 4: { - fcmeq(GetDst(Node).S(), GetSrc(Op->Vector1.ID()).S(), GetSrc(Op->Vector2.ID()).S()); - break; - } - case 8: { - fcmeq(GetDst(Node).D(), GetSrc(Op->Vector1.ID()).D(), GetSrc(Op->Vector2.ID()).D()); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - } - else { - // Vector - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto IsScalar = ElementSize == OpSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto Vector1 = GetSrc(Op->Vector1.ID()); + const auto Vector2 = GetSrc(Op->Vector2.ID()); + + if (HostSupportsSVE && Is256Bit && !IsScalar) { + const auto Mask = PRED_TMP_32B.Zeroing(); + const auto ComparePred = p0; + + // Ensure we have no junk in the temporary. + eor(VTMP1.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD()); + + switch (ElementSize) { case 2: { - fcmeq(GetDst(Node).V8H(), GetSrc(Op->Vector1.ID()).V8H(), GetSrc(Op->Vector2.ID()).V8H()); - break; + fcmeq(ComparePred.VnH(), Mask, Vector1.Z().VnH(), Vector2.Z().VnH()); + not_(VTMP1.Z().VnH(), ComparePred.Merging(), Vector1.Z().VnH()); + orr(VTMP1.Z().VnH(), ComparePred.Merging(), VTMP1.Z().VnH(), Vector1.Z().VnH()); + break; } case 4: { - fcmeq(GetDst(Node).V4S(), GetSrc(Op->Vector1.ID()).V4S(), GetSrc(Op->Vector2.ID()).V4S()); - break; + fcmeq(ComparePred.VnS(), Mask, Vector1.Z().VnS(), Vector2.Z().VnS()); + not_(VTMP1.Z().VnS(), ComparePred.Merging(), Vector1.Z().VnS()); + orr(VTMP1.Z().VnS(), ComparePred.Merging(), VTMP1.Z().VnS(), Vector1.Z().VnS()); + break; } case 8: { - fcmeq(GetDst(Node).V2D(), GetSrc(Op->Vector1.ID()).V2D(), GetSrc(Op->Vector2.ID()).V2D()); - break; + fcmeq(ComparePred.VnD(), Mask, Vector1.Z().VnD(), Vector2.Z().VnD()); + not_(VTMP1.Z().VnD(), ComparePred.Merging(), Vector1.Z().VnD()); + orr(VTMP1.Z().VnD(), ComparePred.Merging(), VTMP1.Z().VnD(), Vector1.Z().VnD()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; + } + + mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + } else { + if (IsScalar) { + switch (ElementSize) { + case 2: { + fcmeq(Dst.H(), Vector1.H(), Vector2.H()); + break; + } + case 4: { + fcmeq(Dst.S(), Vector1.S(), Vector2.S()); + break; + } + case 8: { + fcmeq(Dst.D(), Vector1.D(), Vector2.D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 2: { + fcmeq(Dst.V8H(), Vector1.V8H(), Vector2.V8H()); + break; + } + case 4: { + fcmeq(Dst.V4S(), Vector1.V4S(), Vector2.V4S()); + break; + } + case 8: { + fcmeq(Dst.V2D(), Vector1.V2D(), Vector2.V2D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index 58ce303ed..b8acf7810 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -1520,29 +1520,44 @@ DEF_OP(VCMPLTZ) { } DEF_OP(VFCMPEQ) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - if (Op->Header.ElementSize == OpSize) { - switch (Op->Header.ElementSize) { + const auto ElementSize = Op->Header.ElementSize; + const auto IsScalar = ElementSize == OpSize; + + const auto Dst = GetDst(Node); + const auto Vector1 = GetSrc(Op->Vector1.ID()); + const auto Vector2 = GetSrc(Op->Vector2.ID()); + + if (IsScalar) { + switch (ElementSize) { case 4: - vcmpss(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 0); - break; + vcmpss(Dst, Vector1, Vector2, 0); + break; case 8: - vcmpsd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 0); - break; - default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize); + vcmpsd(Dst, Vector1, Vector2, 0); + break; + default: + LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); + break; } } else { - switch (Op->Header.ElementSize) { + const auto DstYMM = ToYMM(Dst); + const auto Vector1YMM = ToYMM(Vector1); + const auto Vector2YMM = ToYMM(Vector2); + + switch (ElementSize) { case 4: - vcmpps(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 0); - break; + vcmpps(DstYMM, Vector1YMM, Vector2YMM, 0); + break; case 8: - vcmppd(GetDst(Node), GetSrc(Op->Vector1.ID()), GetSrc(Op->Vector2.ID()), 0); - break; - default: LOGMAN_MSG_A_FMT("Unsupported element size: {}", Op->Header.ElementSize); + vcmppd(DstYMM, Vector1YMM, Vector2YMM, 0); + break; + default: + LOGMAN_MSG_A_FMT("Unsupported element size: {}", ElementSize); + break; } } }