From 6ddca804d17d69fbc697c1bc17629284219f8cb3 Mon Sep 17 00:00:00 2001 From: lioncash Date: Thu, 20 Oct 2022 18:04:19 +0000 Subject: [PATCH] IR: Handle 256-bit VUShrNI2 Extends VUShrNI2 to handle 256-bit vectors. --- .../Interface/Core/Interpreter/VectorOps.cpp | 15 ++-- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 81 ++++++++++++++----- .../Interface/Core/JIT/x86_64/VectorOps.cpp | 52 +++++++++--- 3 files changed, 115 insertions(+), 33 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 4c55244f7..a2bb202c0 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -1720,24 +1720,27 @@ DEF_OP(VUShrNI) { } DEF_OP(VUShrNI2) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src1 = GetSrc(Data->SSAData, Op->VectorLower); void *Src2 = GetSrc(Data->SSAData, Op->VectorUpper); - uint8_t BitShift = Op->BitShift; - uint8_t Tmp[16]; + const uint8_t BitShift = Op->BitShift; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; - const uint8_t Elements = OpSize / (Op->Header.ElementSize << 1); + const uint8_t ElementSize = Op->Header.ElementSize; + const uint8_t Elements = OpSize / (ElementSize << 1); const auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; }; - switch (Op->Header.ElementSize) { + switch (ElementSize) { DO_VECTOR_1SRC_2TYPE_OP_TOP(1, uint8_t, uint16_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint16_t, uint32_t, Func, 0, 0) DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint32_t, uint64_t, Func, 0, 0) - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 2df5a3c91..a9e240fea 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -4023,25 +4023,70 @@ DEF_OP(VUShrNI) { } DEF_OP(VUShrNI2) { - auto Op = IROp->C(); - mov(VTMP1, GetSrc(Op->VectorLower.ID())); - switch (Op->Header.ElementSize) { - case 1: { - shrn2(VTMP1.V16B(), GetSrc(Op->VectorUpper.ID()).V8H(), Op->BitShift); - break; - } - case 2: { - shrn2(VTMP1.V8H(), GetSrc(Op->VectorUpper.ID()).V4S(), Op->BitShift); - break; - } - case 4: { - shrn2(VTMP1.V4S(), GetSrc(Op->VectorUpper.ID()).V2D(), Op->BitShift); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - mov(GetDst(Node), VTMP1); + const auto BitShift = Op->BitShift; + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + if (HostSupportsSVE && Is256Bit) { + mov(VTMP1.Z().VnD(), VectorLower.Z().VnD()); + + const auto Mask = PRED_TMP_16B; + + switch (ElementSize) { + case 1: { + shrnb(VTMP2.Z().VnB(), VectorUpper.Z().VnH(), BitShift); + uzp1(VTMP2.Z().VnB(), VTMP2.Z().VnB(), VTMP2.Z().VnB()); + splice(VTMP1.Z().VnB(), Mask, VTMP1.Z().VnB(), VTMP2.Z().VnB()); + break; + } + case 2: { + shrnb(VTMP2.Z().VnH(), VectorUpper.Z().VnS(), BitShift); + uzp1(VTMP2.Z().VnH(), VTMP2.Z().VnH(), VTMP2.Z().VnH()); + splice(VTMP1.Z().VnH(), Mask, VTMP1.Z().VnH(), VTMP2.Z().VnH()); + break; + } + case 4: { + shrnb(VTMP2.Z().VnS(), VectorUpper.Z().VnD(), BitShift); + uzp1(VTMP2.Z().VnS(), VTMP2.Z().VnS(), VTMP2.Z().VnS()); + splice(VTMP1.Z().VnS(), Mask, VTMP1.Z().VnS(), VTMP2.Z().VnS()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; + } + + mov(Dst.Z().VnD(), VTMP1.Z().VnD()); + } else { + mov(VTMP1, VectorLower); + + switch (ElementSize) { + case 1: { + shrn2(VTMP1.V16B(), VectorUpper.V8H(), BitShift); + break; + } + case 2: { + shrn2(VTMP1.V8H(), VectorUpper.V4S(), BitShift); + break; + } + case 4: { + shrn2(VTMP1.V4S(), VectorUpper.V2D(), BitShift); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; + } + + mov(Dst, VTMP1); + } } DEF_OP(VSXTL) { diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index c1d469ca4..7ea1da35a 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -2265,38 +2265,72 @@ DEF_OP(VUShrNI) { DEF_OP(VUShrNI2) { // Src1 = Lower results // Src2 = Upper Results - auto Op = IROp->C(); - movapd(xmm13, GetSrc(Op->VectorUpper.ID())); - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto BitShift = Op->BitShift; + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + vmovapd(ymm13, ToYMM(VectorUpper)); + switch (ElementSize) { case 1: { - psrlw(xmm13, Op->BitShift); + if (Is256Bit) { + vpsrlw(ymm13, ymm13, BitShift); + } else { + vpsrlw(xmm13, xmm13, BitShift); + } // <8 x i16> -> <8 x i8> mov(rax, 0x80'80'80'80'80'80'80'80); // Lower mov(rcx, 0x0E'0C'0A'08'06'04'02'00); // Upper break; } case 2: { - psrld(xmm13, Op->BitShift); + if (Is256Bit) { + vpsrld(ymm13, ymm13, BitShift); + } else { + vpsrld(xmm13, xmm13, BitShift); + } // <4 x i32> -> <4 x i16> mov(rax, 0x80'80'80'80'80'80'80'80); // Lower mov(rcx, 0x0D'0C'09'08'05'04'01'00); // Upper break; } case 4: { - psrlq(xmm13, Op->BitShift); + if (Is256Bit) { + vpsrlq(ymm13, ymm13, BitShift); + } else { + vpsrlq(xmm13, xmm13, BitShift); + } // <2 x i64> -> <2 x i32> mov(rax, 0x80'80'80'80'80'80'80'80); // Lower mov(rcx, 0x0B'0A'09'08'03'02'01'00); // Upper break; } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; } vmovq(xmm15, rax); vmovq(xmm14, rcx); punpcklqdq(xmm15, xmm14); - vpshufb(xmm14, xmm13, xmm15); - vpor(GetDst(Node), xmm14, GetSrc(Op->VectorLower.ID())); + if (Is256Bit) { + vinserti128(ymm15, ymm15, xmm15, 1); + vpshufb(ymm14, ymm13, ymm15); + vextracti128(xmm12, ymm14, 1); + punpckhqdq(xmm14, xmm12); + + vmovapd(Dst, VectorLower); + vinserti128(ToYMM(Dst), ToYMM(Dst), xmm14, 1); + } else { + vpshufb(xmm14, xmm13, xmm15); + vpor(Dst, xmm14, VectorLower); + } } DEF_OP(VSXTL) {