From a8d4d0d3c37ab26fbc60c1981e4b9ee9af1b4ed3 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 27 Mar 2020 18:14:44 -0700 Subject: [PATCH 1/5] Adds new IR ops vector pairwise add vector unsigned shift right with narrowing vector unsigned and signed extend with lengthening from upper bits vector unsigned and signed multiply with lengthening from upper bits --- .../Source/Interface/Core/OpcodeDispatcher.h | 39 ++++++++- External/FEXCore/Source/Interface/IR/IR.json | 82 +++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index de6aceb2a..2906301ae 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -316,7 +316,16 @@ public: Op.first->Header.HasDest = true; return Op; } - + IRPair _VBitcast(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + auto Op = AllocateOp(); + Op.first->Header.Size = RegisterSize / 8; + Op.first->Header.Elements = RegisterSize / ElementSize; + Op.first->Header.NumArgs = 1; + Op.first->Header.HasDest = true; + Op.first->Header.Args[0] = ssa0->Wrapped(ListData.Begin()); + ssa0->AddUse(); + return Op; + } IRPair _Bfe(uint8_t Width, uint8_t lsb, OrderedNode *ssa0) { return _Bfe(ssa0, Width, lsb); } @@ -371,6 +380,9 @@ public: IRPair _VSQSub(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSQSub(ssa0, ssa1, RegisterSize, ElementSize); } + IRPair _VAddP(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VAddP(ssa0, ssa1, RegisterSize, ElementSize); + } IRPair _VUMin(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUMin(ssa0, ssa1, RegisterSize, ElementSize); } @@ -443,6 +455,12 @@ public: IRPair _VSShrI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { return _VSShrI(ssa0, RegisterSize, ElementSize, BitShift); } + IRPair _VUShrNI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { + return _VUShrNI(ssa0, RegisterSize, ElementSize, BitShift); + } + IRPair _VUShrNI2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t BitShift) { + return _VUShrNI2(ssa0, ssa1, RegisterSize, ElementSize, BitShift); + } IRPair _VShlI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { return _VShlI(ssa0, RegisterSize, ElementSize, BitShift); } @@ -491,7 +509,24 @@ public: IRPair _VSMull(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSMull(ssa0, ssa1, RegisterSize, ElementSize); } - + IRPair _VUMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VUMull2(ssa0, ssa1, RegisterSize, ElementSize); + } + IRPair _VSMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VSMull2(ssa0, ssa1, RegisterSize, ElementSize); + } + IRPair _VSXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VSXTL(ssa0, RegisterSize, ElementSize); + } + IRPair _VSXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VSXTL2(ssa0, RegisterSize, ElementSize); + } + IRPair _VUXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VUXTL(ssa0, RegisterSize, ElementSize); + } + IRPair _VUXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VUXTL2(ssa0, RegisterSize, ElementSize); + } IRPair _Jump() { return _Jump(InvalidNode); } diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index 951092d24..f9ac5b840 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -708,6 +708,16 @@ ] }, + "VAddP": { + "Desc": "Does a horizontal pairwise add of elements across the two source vectors", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + "VFAdd": { "HasDest": true, "SSAArgs": "2", @@ -1080,13 +1090,52 @@ ] }, + "VUShrNI": { + "Desc": "Unsigned shifts right each element and then narrows to the next lower element size", + "HasDest": true, + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize", + "uint8_t", "BitShift" + ] + }, + + "VUShrNI2": { + "Desc": ["Unsigned shifts right each element and then narrows to the next lower element size", + "Inserts results in to the high elements of the first argument" + ], + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize", + "uint8_t", "BitShift" + ] + }, + "VBitcast": { "HasDest": true, "SSAArgs": "1" }, "VSXTL": { + "Desc": "Sign extends elements from the source element size to the next size up", "HasDest": true, + "DestSize": "RegisterSize", + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VSXTL2": { + "Desc": ["Sign extends elements from the source element size to the next size up", + "Source elements come from the upper 64bits of the register" + ], + "HasDest": true, + "DestSize": "RegisterSize", "SSAArgs": "1", "Args": [ "uint8_t", "RegisterSize", @@ -1095,6 +1144,19 @@ }, "VUXTL": { + "Desc": "Zero extends elements from the source element size to the next size up", + "HasDest": true, + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VUXTL2": { + "Desc": ["Zero extends elements from the source element size to the next size up", + "Source elements come from the upper 64bits of the register" + ], "HasDest": true, "SSAArgs": "1", "Args": [ @@ -1263,6 +1325,26 @@ ] }, + "VUMull2": { + "Dest": "Multiplies the high elements with size extension", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VSMull2": { + "Dest": "Multiplies the high elements with size extension", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + "GetHostFlag": { "HasDest": true, "SSAArgs": "1", From 3cd1e0ee8f52ca856cb33dee92dce840ae625d83 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 27 Mar 2020 18:17:25 -0700 Subject: [PATCH 2/5] Adds new vector ops to the interpreter --- .../Core/Interpreter/InterpreterCore.cpp | 298 +++++++++++++++--- 1 file changed, 260 insertions(+), 38 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp index b6c195365..d89295e1b 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp @@ -1999,6 +1999,17 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } \ break; \ } + #define DO_VECTOR_PAIR_OP(size, type, func) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src1_d = reinterpret_cast(Src1); \ + auto *Src2_d = reinterpret_cast(Src2); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = func(Src1_d[i*2], Src1_d[i*2 + 1]); \ + Dst_d[i+Elements] = func(Src2_d[i*2], Src2_d[i*2 + 1]); \ + } \ + break; \ + } #define DO_VECTOR_SCALAR_OP(size, type, func)\ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ @@ -2201,6 +2212,35 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VSQSUB: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto b) { + __int128_t res = a - b; + if (res < std::numeric_limits::min()) + return std::numeric_limits::min(); + + if (res > std::numeric_limits::max()) + return std::numeric_limits::max(); + return (decltype(a))res; + }; + + switch (Op->ElementSize) { + DO_VECTOR_OP(1, int8_t, Func) + DO_VECTOR_OP(2, int16_t, Func) + DO_VECTOR_OP(4, int32_t, Func) + DO_VECTOR_OP(8, int64_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VFADD: { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); @@ -2235,6 +2275,25 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto b) { return a + b; }; + switch (Op->ElementSize) { + DO_VECTOR_PAIR_OP(1, uint8_t, Func) + DO_VECTOR_PAIR_OP(2, uint16_t, Func) + DO_VECTOR_PAIR_OP(4, uint32_t, Func) + DO_VECTOR_PAIR_OP(8, uint64_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } case IR::OP_VFMUL: { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); @@ -2351,22 +2410,32 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } - #define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, max) \ + #define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, min, max) \ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ auto *Src_d = reinterpret_cast(Src); \ for (uint8_t i = 0; i < Elements; ++i) { \ - Dst_d[i] = (type)func(Src_d[i], max); \ + Dst_d[i] = (type)func(Src_d[i], min, max); \ } \ break; \ } - #define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, max) \ + #define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, min, max) \ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ auto *Src_d = reinterpret_cast(Src2); \ memcpy(Dst_d, Src1, Elements * sizeof(type2));\ for (uint8_t i = 0; i < Elements; ++i) { \ - Dst_d[i+Elements] = (type)func(Src_d[i], max); \ + Dst_d[i+Elements] = (type)func(Src_d[i], min, max); \ + } \ + break; \ + } + + #define DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(size, type, type2, func, min, max) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src_d = reinterpret_cast(Src); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = (type)func(Src_d[i+Elements], min, max); \ } \ break; \ } @@ -2380,18 +2449,98 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } \ break; \ } + #define DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(size, type, type2, func) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src1_d = reinterpret_cast(Src1); \ + auto *Src2_d = reinterpret_cast(Src2); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = (type)func((type)Src1_d[i+Elements], (type)Src2_d[i+Elements]); \ + } \ + break; \ + } - case IR::OP_VSQXTUN: { - auto Op = IROp->C(); + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); + uint8_t BitShift = Op->BitShift; + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, uint32_t, uint64_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t BitShift = Op->BitShift; uint8_t Tmp[16]; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); }; + auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, (1 << 8) - 1) - DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, (1 << 16) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP(8, uint32_t, uint64_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, int8_t, int16_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + DO_VECTOR_1SRC_2TYPE_OP(4, int16_t, int32_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, int8_t, int16_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, int16_t, int32_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTUN: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1) + DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2401,14 +2550,14 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); void *Src2 = GetSrc(Op->Header.Args[1]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); }; + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, (1 << 8) - 1) - DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, (1 << 16) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2421,10 +2570,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2437,10 +2586,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2453,10 +2602,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2469,10 +2618,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2554,18 +2703,73 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto b) { return a * b; }; + switch (Op->ElementSize) { + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto b) { return a * b; }; + switch (Op->ElementSize) { + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } case IR::OP_VSXTL: { auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2574,15 +2778,33 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { case IR::OP_VUXTL: { auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -3221,19 +3443,19 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Conv) { case 0x0804: { // Double <- float uint8_t Elements = Op->RegisterSize / Op->SrcElementSize; switch (Op->SrcElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0, 0) } break; } case 0x0408: { // Float <- Double uint8_t Elements = Op->RegisterSize / Op->SrcElementSize; switch (Op->SrcElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0, 0) } break; From 7c4b5c3be821f20d2aefbed51adac2d3acd040db Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 27 Mar 2020 18:17:43 -0700 Subject: [PATCH 3/5] Adds to IR ops to the AArch64 JIT --- .../Source/Interface/Core/JIT/Arm64/JIT.cpp | 405 +++++++++++++++--- 1 file changed, 357 insertions(+), 48 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index c393cea4d..5e29b7755 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -2197,6 +2197,63 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + smull(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + smull(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + smull(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + umull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + umull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + umull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + smull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + smull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + smull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VSQADD: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2243,6 +2300,75 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + uqadd(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + uqadd(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + uqadd(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + uqadd(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + uqsub(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + uqsub(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + uqsub(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + uqsub(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VADDP: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + addp(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + addp(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + addp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + addp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VFADD: { auto Op = IROp->C(); if (Op->ElementSize == Op->RegisterSize) { @@ -2543,6 +2669,59 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + sqxtn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + sqxtn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + case 8: + sqxtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 2: + sqxtn(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 4: + sqxtn(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 8: + sqxtn(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + else { + switch (Op->ElementSize) { + case 2: + sqxtn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + case 4: + sqxtn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + case 8: + sqxtn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + mov(GetDst(Node), VTMP1); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2561,19 +2740,39 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VSQXTUN2: { auto Op = IROp->C(); - mov(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); - switch (Op->ElementSize) { - case 2: - sqxtun2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - case 4: - sqxtun2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - case 8: - sqxtun2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 2: + sqxtun(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 4: + sqxtun(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 8: + sqxtun(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } } + else { + switch (Op->ElementSize) { + case 2: + sqxtun2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + case 4: + sqxtun2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + case 8: + sqxtun2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + mov(GetDst(Node), VTMP1); break; } case IR::OP_VSXTL: { @@ -2608,6 +2807,38 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: + sxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B()); + break; + case 2: + sxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + sxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: + uxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B()); + break; + case 2: + uxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + uxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_FLOAT_FTOF: { auto Op = IROp->C(); uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize; @@ -2993,49 +3224,85 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VZIP: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); - switch (Op->ElementSize) { - case 1: { - zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); - break; + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 1: { + zip1(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + zip1(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + zip1(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - } - case 4: { - zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - } - case 8: { - zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + else { + switch (Op->ElementSize) { + case 1: { + zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } break; } case IR::OP_VZIP2: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); - switch (Op->ElementSize) { - case 1: { - zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); - break; + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 1: { + zip2(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + zip2(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + zip2(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - } - case 4: { - zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - } - case 8: { - zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + else { + switch (Op->ElementSize) { + case 1: { + zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } break; } @@ -3163,6 +3430,48 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + + switch (Op->ElementSize) { + case 2: { + shrn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H(), Op->BitShift); + break; + } + case 4: { + shrn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S(), Op->BitShift); + break; + } + case 8: { + shrn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D(), Op->BitShift); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + switch (Op->ElementSize) { + case 2: { + shrn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H(), Op->BitShift); + break; + } + case 4: { + shrn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S(), Op->BitShift); + break; + } + case 8: { + shrn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D(), Op->BitShift); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + mov(GetDst(Node), VTMP1); + break; + } case IR::OP_VSLI: { auto Op = IROp->C(); uint8_t BitShift = Op->ByteShift * 8; From 6f14607a8298400abcec865b2e940d8de4ef616a Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 27 Mar 2020 18:18:17 -0700 Subject: [PATCH 4/5] Adds the new IR ops to the x86 JIT --- .../Source/Interface/Core/JIT/x86_64/JIT.cpp | 373 +++++++++++++++++- 1 file changed, 352 insertions(+), 21 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 4ea2d74cc..75b74777c 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -2119,6 +2119,36 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + vpaddusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + vpaddusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + vpsubusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + vpsubusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VSUB: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2142,6 +2172,19 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + vphaddw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + case 4: + vphaddd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VUMUL: case IR::OP_VSMUL: { auto Op = IROp->C(); @@ -2159,8 +2202,22 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const break; } case IR::OP_VUMULL: { - auto Op = IROp->C(); + auto Op = IROp->C(); switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[15:00] * src2[15:00] + // [63:32 ] = src1[31:16] * src2[31:16] + // [95:64 ] = src1[47:32] * src2[47:32] + // [127:96] = src1[63:48] * src2[63:48] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } case 4: { // We need to shuffle the data for this one // x86 PMULUDQ wants the 32bit values in [31:0] and [95:64] @@ -2175,6 +2232,117 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[15:00] * src2[15:00] + // [63:32 ] = src1[31:16] * src2[31:16] + // [95:64 ] = src1[47:32] * src2[47:32] + // [127:96] = src1[63:48] * src2[63:48] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + pslld(xmm15, 16); + pslld(xmm14, 16); + psrad(xmm15, 16); + psrad(xmm14, 16); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // We need to shuffle the data for this one + // x86 PMULDQ wants the 32bit values in [31:0] and [95:64] + // Which then extends out to [63:0] and [127:64] + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b10'10'00'00); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b10'10'00'00); + + vpmuldq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[79:64 ] * src2[79:64 ] + // [63:32 ] = src1[95:80 ] * src2[95:80 ] + // [95:64 ] = src1[111:96 ] * src2[111:96 ] + // [127:96] = src1[127:112] * src2[127:112] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // IR operation: + // [63:00 ] = src1[95:64 ] * src2[95:64 ] + // [127:64] = src1[127:96] * src2[127:96] + // + // x86 vpmuludq + // [63:00 ] = src1[31:0 ] * src2[31:0 ] + // [127:64] = src1[95:64] * src2[95:64] + + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10); + + vpmuludq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[79:64 ] * src2[79:64 ] + // [63:32 ] = src1[95:80 ] * src2[95:80 ] + // [95:64 ] = src1[111:96 ] * src2[111:96 ] + // [127:96] = src1[127:112] * src2[127:112] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + pslld(xmm15, 16); + pslld(xmm14, 16); + psrad(xmm15, 16); + psrad(xmm14, 16); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // IR operation: + // [63:00 ] = src1[95:64 ] * src2[95:64 ] + // [127:64] = src1[127:96] * src2[127:96] + // + // x86 vpmuludq + // [63:00 ] = src1[31:0 ] * src2[31:0 ] + // [127:64] = src1[95:64] * src2[95:64] + + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10); + + vpmuldq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VAND: { auto Op = IROp->C(); vpand(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); @@ -2507,6 +2675,41 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + packsswb(xmm15, GetSrc(Op->Header.Args[0].ID())); + break; + case 4: + packssdw(xmm15, GetSrc(Op->Header.Args[0].ID())); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + psrldq(xmm15, 8); + movaps(GetDst(Node), xmm15); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + // Zero the lower bits + vpxor(xmm15, xmm15, xmm15); + switch (Op->ElementSize) { + case 2: + packsswb(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + case 4: + packssdw(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + + if (Op->RegisterSize == 8) { + psrldq(xmm15, Op->RegisterSize / 2); + } + vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2535,6 +2738,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const break; default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); } + if (Op->RegisterSize == 8) { + psrldq(xmm15, Op->RegisterSize / 2); + } vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15); break; @@ -2555,6 +2761,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + vpsrldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2); + switch (Op->ElementSize) { + case 1: + pmovsxbw(GetDst(Node), GetDst(Node)); + break; + case 2: + pmovsxwd(GetDst(Node), GetDst(Node)); + break; + case 4: + pmovsxdq(GetDst(Node), GetDst(Node)); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_VUXTL: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2571,6 +2794,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + vpslldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2); + switch (Op->ElementSize) { + case 1: + pmovzxbw(GetDst(Node), GetDst(Node)); + break; + case 2: + pmovzxwd(GetDst(Node), GetDst(Node)); + break; + case 4: + pmovzxdq(GetDst(Node), GetDst(Node)); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_VECTOR_STOF: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2651,7 +2891,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VCMPEQ: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); switch (Op->ElementSize) { case 1: @@ -2672,7 +2911,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VCMPGT: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); switch (Op->ElementSize) { case 1: @@ -2896,26 +3134,47 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const auto Op = IROp->C(); movapd(xmm15, GetSrc(Op->Header.Args[0].ID())); - switch (Op->ElementSize) { - case 1: { - punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; + if (Op->RegisterSize == 8) { + vpslldq(xmm15, GetSrc(Op->Header.Args[0].ID()), 4); + vpslldq(xmm14, GetSrc(Op->Header.Args[1].ID()), 4); + switch (Op->ElementSize) { + case 1: { + vpunpckhbw(GetDst(Node), xmm15, xmm14); + break; + } + case 2: { + vpunpckhwd(GetDst(Node), xmm15, xmm14); + break; + } + case 4: { + vpunpckhdq(GetDst(Node), xmm15, xmm14); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; + else { + switch (Op->ElementSize) { + case 1: { + punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 4: { + punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 8: { + punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + movapd(GetDst(Node), xmm15); } - case 4: { - punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; - } - case 8: { - punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; - } - movapd(GetDst(Node), xmm15); break; } case IR::OP_VUSHLS: { @@ -3048,6 +3307,78 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + movapd(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + vpxor(xmm15, xmm15, xmm15); + switch (Op->ElementSize) { + case 2: { + psrlw(GetDst(Node), Op->BitShift); + // <8 x i16> -> <8 x i8> + mov(rax, 0x0E'0C'0A'08'06'04'02'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + case 4: { + psrld(GetDst(Node), Op->BitShift); + // <4 x i32> -> <4 x i16> + mov(rax, 0x0D'0C'09'08'05'04'01'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + case 8: { + psrlq(GetDst(Node), Op->BitShift); + // <2 x i64> -> <2 x i32> + mov(rax, 0x0B'0A'09'08'03'02'01'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + vmovq(xmm15, rax); + vmovq(xmm14, rcx); + punpcklqdq(xmm15, xmm14); + pshufb(GetDst(Node), xmm15); + break; + } + case IR::OP_VUSHRNI2: { + // Src1 = Lower results + // Src2 = Upper Results + auto Op = IROp->C(); + movapd(xmm13, GetSrc(Op->Header.Args[1].ID())); + switch (Op->ElementSize) { + case 2: { + psrlw(xmm13, Op->BitShift); + // <8 x i16> -> <8 x i8> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0E'0C'0A'08'06'04'02'00); // Upper + break; + } + case 4: { + psrld(xmm13, Op->BitShift); + // <4 x i32> -> <4 x i16> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0D'0C'09'08'05'04'01'00); // Upper + break; + } + case 8: { + psrlq(xmm13, Op->BitShift); + // <2 x i64> -> <2 x i32> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0B'0A'09'08'03'02'01'00); // Upper + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + vmovq(xmm15, rax); + vmovq(xmm14, rcx); + punpcklqdq(xmm15, xmm14); + vpshufb(xmm14, xmm13, xmm15); + vpor(GetDst(Node), xmm14, GetSrc(Op->Header.Args[0].ID())); + break; + } case IR::OP_VEXTR: { auto Op = IROp->C(); vpalignr(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()), Op->Index); From 5d55f73cd2d7765dcc8a4c6b99815be4f6a6f346 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 27 Mar 2020 18:19:06 -0700 Subject: [PATCH 5/5] Adds new IR ops to the LLVM JIT --- .../Interface/Core/LLVMJIT/LLVMCore.cpp | 736 +++++++++++++++++- 1 file changed, 724 insertions(+), 12 deletions(-) diff --git a/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp b/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp index 899301660..b14826d18 100644 --- a/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp +++ b/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp @@ -28,6 +28,10 @@ using DestMapType = std::unordered_map; using DestMapType = std::vector; #endif +#if defined(_M_ARM_64) && defined(_M_X86_64) +#define AARCH64_ON_X86 +#endif + namespace FEXCore::CPU { static void CPUIDRun_Thunk(CPUIDEmu::FunctionResults *Results, FEXCore::CPUIDEmu *Class, uint32_t Function) { @@ -38,7 +42,7 @@ static void SetExitState_Thunk(FEXCore::Core::InternalThreadState *Thread) { Thread->State.RunningEvents.ShouldStop = true; } -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) static uint64_t AArch64ReadCycleCounter() { uint64_t res{}; asm ("mrs %0, CNTVCT_EL0" @@ -94,7 +98,7 @@ private: llvm::Function *CPUIDFunction; llvm::Function *ExitVMFunction; llvm::Function *ValuePrinter; -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) llvm::Function *AArch64ReadCycleCounterFunction; #endif @@ -154,6 +158,332 @@ private: return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ctlz, ArgTypes, Args); } + + llvm::Value *VSQXTN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { + llvm::Value *Result{}; +#ifdef _M_X86_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + std::vector VectorMaskConstant; + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, // First 8 bytes from high result of source + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, // First 4 16bits from high result of source + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + + std::vector ArgTypes = { + }; + + std::vector Args = { + Arg, + llvm::UndefValue::get(Arg->getType()), + }; + + switch (ElementSize) { + case 2: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args); + break; + case 4: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); +#elif _M_ARM_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args); + +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { + std::vector VectorMaskConstant; + +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1); + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#else + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 8, 9, 10, 11, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 4, 5, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#endif + + // Convert our upper args + ArgUpper = VSQXTN_64(ArgUpper, RegisterSize, ElementSize); + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + + llvm::Value *VSQXTUN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { + llvm::Value *Result{}; +#ifdef _M_X86_64 + // Incoming source is a 64bit register + // We want to still pass it through SSE and NEON's 128bit implementation and ignore the upper 64bits + // Cast to the type we want + llvm::Value *UndefVector = llvm::UndefValue::get(Arg->getType()); + + std::vector LargeVectorMask; + for (uint32_t i = 0; i < (RegisterSize / ElementSize * 2); ++i) { + LargeVectorMask.emplace_back(i); + } + Arg = JITState.IRBuilder->CreateShuffleVector(Arg, UndefVector, LargeVectorMask); + + RegisterSize <<= 1; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + std::vector VectorMaskConstant; + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, // First 8 bytes from high result of source + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, // First 4 16bits from high result of source + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTUN size: %d", ElementSize); break; + } + + std::vector ArgTypes = { + }; + + std::vector Args = { + Arg, + llvm::UndefValue::get(Arg->getType()), + }; + + switch (ElementSize) { + case 2: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packuswb_128, ArgTypes, Args); + break; + case 4: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse41_packusdw, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); +#elif _M_ARM_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtun, ArgTypes, Args); + +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTUN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { + std::vector VectorMaskConstant; + +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1); + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#else + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 8, 9, 10, 11, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 4, 5, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#endif + // Convert our upper args + ArgUpper = VSQXTUN_64(ArgUpper, RegisterSize, ElementSize); + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + + llvm::Value *VSQXTN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + std::vector ArgTypes = { + }; + + std::vector Args = { + llvm::UndefValue::get(Arg->getType()), + Arg, + }; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + llvm::Value *Result{}; + std::vector VectorMaskConstant; + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 8, 9, 10, 11, 12, 13, 14, 15, // First 8 bytes from high result of source + 16, 17, 18, 19, 20, 21, 22, 23, // Second 8 bytes from the zero vector + }; + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args); + break; + case 4: + VectorMaskConstant = { + 4, 5, 6, 7, // First 4 16bits from high result of source + 8, 9, 10, 11, // Second 8 16bits from the zero vector + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); + +#elif _M_ARM_64 + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args); +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); +#endif + // Convert our upper args + ArgUpper = VSQXTN(ArgUpper, RegisterSize, ElementSize); + + std::vector VectorMaskConstant; + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + 16, 17, 18, 19, 20, 21, 22, 23 + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + 8, 9, 10, 11 + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + llvm::Value *VSQXTUN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { #ifdef _M_X86_64 std::vector ArgTypes = { @@ -212,6 +542,7 @@ private: return Result; } + llvm::Value *VSQXTUN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { #ifdef _M_X86_64 // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up @@ -243,6 +574,47 @@ private: return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); } + llvm::Value *VADDP(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + // Will generate VPHADD + uint8_t NumElements = RegisterSize / ElementSize; + std::vector Values; + + for (size_t i = 0; i < NumElements; ++i) { + Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgLower, i));; + } + + for (size_t i = 0; i < NumElements; ++i) { + Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgUpper, i));; + } + + for (size_t i = 0; i < NumElements; ++i) { + Values[i] = JITState.IRBuilder->CreateAdd(Values[i*2], Values[i*2+1]); + } + // Cast to the type we want + llvm::Value *Result = llvm::UndefValue::get(ArgLower->getType()); + + for (size_t i = 0; i < NumElements; ++i) { + Result = JITState.IRBuilder->CreateInsertElement(Result, Values[i], i); + } +#elif _M_ARM_64 + // AArch64 doesn't seem to generate addp through the normal method + std::vector ArgTypes = { + ArgLower->getType(), + }; + + std::vector Args = { + ArgLower, ArgUpper, + }; + + auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_addp, ArgTypes, Args); +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + llvm::CallInst *FSHL(llvm::Value *Val, llvm::Value *Val2, llvm::Value *Amt) { std::vector ArgTypes = { Val->getType(), @@ -316,6 +688,29 @@ private: return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ssub_sat, ArgTypes, Args); } + llvm::CallInst *UQADD(llvm::Value *Arg1, llvm::Value *Arg2) { + std::vector ArgTypes = { + Arg1->getType(), + }; + std::vector Args = { + Arg1, + Arg2, + }; + + return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::uadd_sat, ArgTypes, Args); + } + llvm::CallInst *UQSUB(llvm::Value *Arg1, llvm::Value *Arg2) { + std::vector ArgTypes = { + Arg1->getType(), + }; + std::vector Args = { + Arg1, + Arg2, + }; + + return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::usub_sat, ArgTypes, Args); + } + void CreateDebugPrint(llvm::Value *Val) { std::vector Args; Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast(this))); @@ -524,7 +919,7 @@ void LLVMJITCore::CreateGlobalVariables(llvm::ExecutionEngine *Engine, llvm::Mod Engine->addGlobalMapping(JITCurrentState.CPUIDFunction, Ptr.Data); } -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) // AArch64ReadCycleCounter Function { auto FuncType = FunctionType::get(i64, @@ -913,6 +1308,7 @@ llvm::Value *LLVMJITCore::CreateIndexedContextPtr(llvm::Value *Index, uint64_t O } llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) { + uint8_t DestSizeInBits = RegisterSize * ElementSize * 8; uint8_t NumElements = RegisterSize / ElementSize; llvm::Type *ElementType; if (Integer) { @@ -928,15 +1324,28 @@ llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8 } llvm::Type *VectorType = llvm::VectorType::get(ElementType, NumElements); + auto SrcType = Arg->getType(); - // This happens frequently - // If the source argument isn't of vector type then BitCast fails moving from Scalar->Vector domains - // Need to create a vector and insert elements in to that vector from the scalar type instead - if (!Arg->getType()->isVectorTy()) { + // If the source type is not vector(frequent with opaque types) and it matches the destination size then we can just bitcast it + if (!SrcType->isVectorTy() && SrcType->getPrimitiveSizeInBits() == DestSizeInBits) { return JITState.IRBuilder->CreateBitCast(Arg, VectorType); - } + } - return JITState.IRBuilder->CreateBitCast(Arg, VectorType); + uint8_t NumSrcElements = SrcType->getPrimitiveSizeInBits() / (ElementSize * 8); + llvm::Type *SrcVectorType = llvm::VectorType::get(ElementType, NumSrcElements); + + // First thing we need to cast the argument type to a vector size that is supported by the source type + Arg = JITState.IRBuilder->CreateBitCast(Arg, SrcVectorType); + + llvm::Value *Undef = llvm::UndefValue::get(SrcVectorType); + std::vector Mask; + // We then need to shuffle the elements to match the size we want + // If the destination size is smaller then we can just shuffle the low elements + // If our destination is more than the number of source elements then we need to pull in some undef + for (uint32_t i = 0; i < NumElements; ++i) + Mask.emplace_back(i); + + return JITState.IRBuilder->CreateShuffleVector(Arg, Undef, Mask); } llvm::Value *LLVMJITCore::CastScalarToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) { @@ -2003,6 +2412,20 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = VADDP(Src1, Src2, Op->RegisterSize, Op->ElementSize); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUMUL: case IR::OP_VSMUL: { auto Op = IROp->C(); @@ -2046,6 +2469,34 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = UQADD(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = UQSUB(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUMULL: { auto Op = IROp->C(); auto Src1 = GetSrc(Op->Header.Args[0]); @@ -2091,6 +2542,142 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {0, 1, 2, 3, 4, 5, 6, 7, 8}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {0, 1, 2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {0, 2}; // XXX: Why is this 0 and 2? It should be 0 and 1 but llvm "optimizes" the code to 0 and 4 in that case. How does this work? + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {9, 10, 11, 12, 13, 14, 15}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {4, 5, 6, 7}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateZExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateZExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {9, 10, 11, 12, 13, 14, 15}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {4, 5, 6, 7}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VNOT: { auto Op = IROp->C(); auto Src1 = GetSrc(Op->Header.Args[0]); @@ -2227,6 +2814,44 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + auto Src = GetSrc(Op->Header.Args[0]); + + // Cast to the type we want + Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); + + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTN_64(Src, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTN(Src, Op->RegisterSize, Op->ElementSize); + } + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]); @@ -2234,7 +2859,13 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Cast to the type we want Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); - auto Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize); + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTUN_64(Src, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize); + } SetDest(*WrapperOp, Result); break; @@ -2245,10 +2876,16 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap auto Src2 = GetSrc(Op->Header.Args[1]); // Cast to the type we want - Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1); + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1); Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); - auto Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTUN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } SetDest(*WrapperOp, Result); break; @@ -2282,6 +2919,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Now we will do a lshr -> auto Result = JITState.IRBuilder->CreateShl(Src1, Src2); + // Ensure we shift out to zero if the shift is > element size + auto Cmp = JITState.IRBuilder->CreateICmpUGT(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8 - 1))); + Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result); + SetDest(*WrapperOp, Result); break; } @@ -2314,6 +2955,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Now we will do a lshr -> auto Result = JITState.IRBuilder->CreateLShr(Src1, Src2); + // Ensure we shift out to zero if the shift is > element size + auto Cmp = JITState.IRBuilder->CreateICmpUGE(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8))); + Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result); + SetDest(*WrapperOp, Result); break; } @@ -2394,6 +3039,52 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + + uint32_t NumElements = Op->RegisterSize / Op->ElementSize; + uint8_t ElementSize = Op->ElementSize * 8; + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + + auto Src2 = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift))); + + // Now we will do a lshr -> + llvm::Value *Result = JITState.IRBuilder->CreateLShr(Src1, Src2); + Result = JITState.IRBuilder->CreateTrunc(Result, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements)); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + uint32_t NumElements = Op->RegisterSize / Op->ElementSize; + uint8_t ElementSize = Op->ElementSize * 8; + + // Low source needs to already be in the size we want (Typically from VUSHRNI) + Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1); + // Cast to the type we want + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Shift = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift))); + + // Now we will do a lshr -> + llvm::Value *ResultHigh = JITState.IRBuilder->CreateLShr(Src2, Shift); + ResultHigh = JITState.IRBuilder->CreateTrunc(ResultHigh, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements)); + + std::vector Mask; + for (uint32_t i = 0; i < (NumElements * 2); ++i) { + Mask.emplace_back(i); + } + auto Result = JITState.IRBuilder->CreateShuffleVector(Src1, ResultHigh, Mask, "Shuffle1"); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VSLI: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]); @@ -2451,6 +3142,27 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + auto Src = GetSrc(Op->Header.Args[0]); + + // Cast to the type we want + Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); + + uint8_t SrcNumElements = Op->RegisterSize / Op->ElementSize; + uint8_t DstNumElements = SrcNumElements / 2; + llvm::Type *TargetType = VectorType::get(Type::getIntNTy(*Con, Op->ElementSize * 16), DstNumElements); + + std::vector Mask; + for (uint32_t i = DstNumElements; i < (SrcNumElements); ++i) { + Mask.emplace_back(i); + } + // Shuffle out which elements we actually want + Src = JITState.IRBuilder->CreateShuffleVector(Src, Src, Mask); + auto Result = JITState.IRBuilder->CreateSExt(Src, TargetType); + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUXTL: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]);