From 5baa15de03b3b34f604877e55acd9a5715b863e6 Mon Sep 17 00:00:00 2001 From: lioncash Date: Thu, 20 Oct 2022 18:49:48 +0000 Subject: [PATCH 1/2] IR: Handle 256-bit VZip Extends VZip to handle 256-bit vectors. --- .../Interface/Core/Interpreter/VectorOps.cpp | 15 +-- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 95 +++++++++++++------ .../Interface/Core/JIT/x86_64/VectorOps.cpp | 77 +++++++++++---- 3 files changed, 132 insertions(+), 55 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 3567fd8f7..a4a52cf35 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -838,17 +838,18 @@ DEF_OP(VSMax) { } DEF_OP(VZip) { - auto Op = IROp->C(); + const auto Op = IROp->C(); const uint8_t OpSize = IROp->Size; void *Src1 = GetSrc(Data->SSAData, Op->VectorLower); void *Src2 = GetSrc(Data->SSAData, Op->VectorUpper); - uint8_t Tmp[16]; - uint8_t Elements = OpSize / Op->Header.ElementSize; - uint8_t BaseOffset = IROp->Op == IR::OP_VZIP2 ? (Elements / 2) : 0; + uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE]; + const uint8_t ElementSize = Op->Header.ElementSize; + uint8_t Elements = OpSize / ElementSize; + const uint8_t BaseOffset = IROp->Op == IR::OP_VZIP2 ? (Elements / 2) : 0; Elements >>= 1; - switch (Op->Header.ElementSize) { + switch (ElementSize) { case 1: { auto *Dst_d = reinterpret_cast(Tmp); auto *Src1_d = reinterpret_cast(Src1); @@ -889,7 +890,9 @@ DEF_OP(VZip) { } break; } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } memcpy(GDP, Tmp, OpSize); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 22f431437..07fc2f2e5 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -2091,44 +2091,79 @@ DEF_OP(VSMax) { } DEF_OP(VZip) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - if (OpSize == 8) { - switch (Op->Header.ElementSize) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + if (HostSupportsSVE && Is256Bit) { + switch (ElementSize) { case 1: { - zip1(GetDst(Node).V8B(), GetSrc(Op->VectorLower.ID()).V8B(), GetSrc(Op->VectorUpper.ID()).V8B()); - break; + zip1(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); + break; } case 2: { - zip1(GetDst(Node).V4H(), GetSrc(Op->VectorLower.ID()).V4H(), GetSrc(Op->VectorUpper.ID()).V4H()); - break; + zip1(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); + break; } case 4: { - zip1(GetDst(Node).V2S(), GetSrc(Op->VectorLower.ID()).V2S(), GetSrc(Op->VectorUpper.ID()).V2S()); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - } - else { - switch (Op->Header.ElementSize) { - case 1: { - zip1(GetDst(Node).V16B(), GetSrc(Op->VectorLower.ID()).V16B(), GetSrc(Op->VectorUpper.ID()).V16B()); - break; - } - case 2: { - zip1(GetDst(Node).V8H(), GetSrc(Op->VectorLower.ID()).V8H(), GetSrc(Op->VectorUpper.ID()).V8H()); - break; - } - case 4: { - zip1(GetDst(Node).V4S(), GetSrc(Op->VectorLower.ID()).V4S(), GetSrc(Op->VectorUpper.ID()).V4S()); - break; + zip1(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); + break; } case 8: { - zip1(GetDst(Node).V2D(), GetSrc(Op->VectorLower.ID()).V2D(), GetSrc(Op->VectorUpper.ID()).V2D()); - break; + zip1(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + if (OpSize == 8) { + switch (ElementSize) { + case 1: { + zip1(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); + break; + } + case 2: { + zip1(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); + break; + } + case 4: { + zip1(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 1: { + zip1(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); + break; + } + case 2: { + zip1(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); + break; + } + case 4: { + zip1(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); + break; + } + case 8: { + zip1(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index bd2269073..5c418f654 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -1230,29 +1230,68 @@ DEF_OP(VSMax) { } DEF_OP(VZip) { - auto Op = IROp->C(); - movapd(xmm15, GetSrc(Op->VectorLower.ID())); + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - switch (Op->Header.ElementSize) { - case 1: { - punpcklbw(xmm15, GetSrc(Op->VectorUpper.ID())); - break; + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + if (Is256Bit) { + switch (ElementSize) { + case 1: { + vpunpcklbw(xmm15, VectorLower, VectorUpper); + vpunpckhbw(xmm14, VectorLower, VectorUpper); + break; + } + case 2: { + vpunpcklwd(xmm15, VectorLower, VectorUpper); + vpunpckhwd(xmm14, VectorLower, VectorUpper); + break; + } + case 4: { + vpunpckldq(xmm15, VectorLower, VectorUpper); + vpunpckhdq(xmm14, VectorLower, VectorUpper); + break; + } + case 8: { + vpunpcklqdq(xmm15, VectorLower, VectorUpper); + vpunpckhqdq(xmm14, VectorLower, VectorUpper); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; } - case 2: { - punpcklwd(xmm15, GetSrc(Op->VectorUpper.ID())); - break; + + vinserti128(ymm15, ymm15, xmm14, 1); + vmovapd(ToYMM(Dst), ymm15); + } else { + switch (ElementSize) { + case 1: { + vpunpcklbw(Dst, VectorLower, VectorUpper); + break; + } + case 2: { + vpunpcklwd(Dst, VectorLower, VectorUpper); + break; + } + case 4: { + vpunpckldq(Dst, VectorLower, VectorUpper); + break; + } + case 8: { + vpunpcklqdq(Dst, VectorLower, VectorUpper); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } - case 4: { - punpckldq(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - case 8: { - punpcklqdq(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; } - movapd(GetDst(Node), xmm15); } DEF_OP(VZip2) { From 340b2ca6247cd9abd8d1c825a9747f9931b5b40f Mon Sep 17 00:00:00 2001 From: lioncash Date: Thu, 20 Oct 2022 19:28:03 +0000 Subject: [PATCH 2/2] IR: Handle 256-bit VZip2 Extends VZip2 to handle 256-bit vectors. --- .../Interface/Core/JIT/Arm64/VectorOps.cpp | 109 +++++++++------ .../Interface/Core/JIT/x86_64/VectorOps.cpp | 124 ++++++++++++------ 2 files changed, 154 insertions(+), 79 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp index 07fc2f2e5..04b277374 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp @@ -2169,44 +2169,79 @@ DEF_OP(VZip) { } DEF_OP(VZip2) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; - if (OpSize == 8) { - switch (Op->Header.ElementSize) { - case 1: { - zip2(GetDst(Node).V8B(), GetSrc(Op->VectorLower.ID()).V8B(), GetSrc(Op->VectorUpper.ID()).V8B()); - break; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + if (HostSupportsSVE && Is256Bit) { + switch (ElementSize) { + case 1: { + zip2(Dst.Z().VnB(), VectorLower.Z().VnB(), VectorUpper.Z().VnB()); + break; + } + case 2: { + zip2(Dst.Z().VnH(), VectorLower.Z().VnH(), VectorUpper.Z().VnH()); + break; + } + case 4: { + zip2(Dst.Z().VnS(), VectorLower.Z().VnS(), VectorUpper.Z().VnS()); + break; + } + case 8: { + zip2(Dst.Z().VnD(), VectorLower.Z().VnD(), VectorUpper.Z().VnD()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; } - case 2: { - zip2(GetDst(Node).V4H(), GetSrc(Op->VectorLower.ID()).V4H(), GetSrc(Op->VectorUpper.ID()).V4H()); - break; - } - case 4: { - zip2(GetDst(Node).V2S(), GetSrc(Op->VectorLower.ID()).V2S(), GetSrc(Op->VectorUpper.ID()).V2S()); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - } - else { - switch (Op->Header.ElementSize) { - case 1: { - zip2(GetDst(Node).V16B(), GetSrc(Op->VectorLower.ID()).V16B(), GetSrc(Op->VectorUpper.ID()).V16B()); - break; - } - case 2: { - zip2(GetDst(Node).V8H(), GetSrc(Op->VectorLower.ID()).V8H(), GetSrc(Op->VectorUpper.ID()).V8H()); - break; - } - case 4: { - zip2(GetDst(Node).V4S(), GetSrc(Op->VectorLower.ID()).V4S(), GetSrc(Op->VectorUpper.ID()).V4S()); - break; - } - case 8: { - zip2(GetDst(Node).V2D(), GetSrc(Op->VectorLower.ID()).V2D(), GetSrc(Op->VectorUpper.ID()).V2D()); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; + } else { + if (OpSize == 8) { + switch (ElementSize) { + case 1: { + zip2(Dst.V8B(), VectorLower.V8B(), VectorUpper.V8B()); + break; + } + case 2: { + zip2(Dst.V4H(), VectorLower.V4H(), VectorUpper.V4H()); + break; + } + case 4: { + zip2(Dst.V2S(), VectorLower.V2S(), VectorUpper.V2S()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 1: { + zip2(Dst.V16B(), VectorLower.V16B(), VectorUpper.V16B()); + break; + } + case 2: { + zip2(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H()); + break; + } + case 4: { + zip2(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S()); + break; + } + case 8: { + zip2(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D()); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp index 5c418f654..fba09013f 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp @@ -1295,51 +1295,91 @@ DEF_OP(VZip) { } DEF_OP(VZip2) { - auto Op = IROp->C(); - const uint8_t OpSize = IROp->Size; + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; - movapd(xmm15, GetSrc(Op->VectorLower.ID())); + const auto ElementSize = Op->Header.ElementSize; + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; - if (OpSize == 8) { - vpslldq(xmm15, GetSrc(Op->VectorLower.ID()), 4); - vpslldq(xmm14, GetSrc(Op->VectorUpper.ID()), 4); - switch (Op->Header.ElementSize) { - case 1: { - vpunpckhbw(GetDst(Node), xmm15, xmm14); - break; + const auto Dst = GetDst(Node); + const auto VectorLower = GetSrc(Op->VectorLower.ID()); + const auto VectorUpper = GetSrc(Op->VectorUpper.ID()); + + if (Is256Bit) { + vextracti128(xmm15, ToYMM(VectorLower), 1); + vextracti128(xmm14, ToYMM(VectorUpper), 1); + + switch (ElementSize) { + case 1: { + vpunpcklbw(Dst, xmm15, xmm14); + vpunpckhbw(xmm13, xmm15, xmm14); + break; + } + case 2: { + vpunpcklwd(Dst, xmm15, xmm14); + vpunpckhwd(xmm13, xmm15, xmm14); + break; + } + case 4: { + vpunpckldq(Dst, xmm15, xmm14); + vpunpckhdq(xmm13, xmm15, xmm14); + break; + } + case 8: { + vpunpcklqdq(Dst, xmm15, xmm14); + vpunpckhqdq(xmm13, xmm15, xmm14); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + return; + } + + vinserti128(ToYMM(Dst), ToYMM(Dst), xmm13, 1); + } else { + if (OpSize == 8) { + vpslldq(xmm15, VectorLower, 4); + vpslldq(xmm14, VectorUpper, 4); + switch (ElementSize) { + case 1: { + vpunpckhbw(Dst, xmm15, xmm14); + break; + } + case 2: { + vpunpckhwd(Dst, xmm15, xmm14); + break; + } + case 4: { + vpunpckhdq(Dst, xmm15, xmm14); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } + } else { + switch (ElementSize) { + case 1: { + vpunpckhbw(Dst, VectorLower, VectorUpper); + break; + } + case 2: { + vpunpckhwd(Dst, VectorLower, VectorUpper); + break; + } + case 4: { + vpunpckhdq(Dst, VectorLower, VectorUpper); + break; + } + case 8: { + vpunpckhqdq(Dst, VectorLower, VectorUpper); + break; + } + default: + LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); + break; + } } - case 2: { - vpunpckhwd(GetDst(Node), xmm15, xmm14); - break; - } - case 4: { - vpunpckhdq(GetDst(Node), xmm15, xmm14); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - } - else { - switch (Op->Header.ElementSize) { - case 1: { - punpckhbw(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - case 2: { - punpckhwd(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - case 4: { - punpckhdq(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - case 8: { - punpckhqdq(xmm15, GetSrc(Op->VectorUpper.ID())); - break; - } - default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break; - } - movapd(GetDst(Node), xmm15); } }