diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp index b6c195365..d89295e1b 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp @@ -1999,6 +1999,17 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } \ break; \ } + #define DO_VECTOR_PAIR_OP(size, type, func) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src1_d = reinterpret_cast(Src1); \ + auto *Src2_d = reinterpret_cast(Src2); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = func(Src1_d[i*2], Src1_d[i*2 + 1]); \ + Dst_d[i+Elements] = func(Src2_d[i*2], Src2_d[i*2 + 1]); \ + } \ + break; \ + } #define DO_VECTOR_SCALAR_OP(size, type, func)\ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ @@ -2201,6 +2212,35 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VSQSUB: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto b) { + __int128_t res = a - b; + if (res < std::numeric_limits::min()) + return std::numeric_limits::min(); + + if (res > std::numeric_limits::max()) + return std::numeric_limits::max(); + return (decltype(a))res; + }; + + switch (Op->ElementSize) { + DO_VECTOR_OP(1, int8_t, Func) + DO_VECTOR_OP(2, int16_t, Func) + DO_VECTOR_OP(4, int32_t, Func) + DO_VECTOR_OP(8, int64_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VFADD: { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); @@ -2235,6 +2275,25 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto b) { return a + b; }; + switch (Op->ElementSize) { + DO_VECTOR_PAIR_OP(1, uint8_t, Func) + DO_VECTOR_PAIR_OP(2, uint16_t, Func) + DO_VECTOR_PAIR_OP(4, uint32_t, Func) + DO_VECTOR_PAIR_OP(8, uint64_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } case IR::OP_VFMUL: { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); @@ -2351,22 +2410,32 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } - #define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, max) \ + #define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, min, max) \ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ auto *Src_d = reinterpret_cast(Src); \ for (uint8_t i = 0; i < Elements; ++i) { \ - Dst_d[i] = (type)func(Src_d[i], max); \ + Dst_d[i] = (type)func(Src_d[i], min, max); \ } \ break; \ } - #define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, max) \ + #define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, min, max) \ case size: { \ auto *Dst_d = reinterpret_cast(Tmp); \ auto *Src_d = reinterpret_cast(Src2); \ memcpy(Dst_d, Src1, Elements * sizeof(type2));\ for (uint8_t i = 0; i < Elements; ++i) { \ - Dst_d[i+Elements] = (type)func(Src_d[i], max); \ + Dst_d[i+Elements] = (type)func(Src_d[i], min, max); \ + } \ + break; \ + } + + #define DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(size, type, type2, func, min, max) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src_d = reinterpret_cast(Src); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = (type)func(Src_d[i+Elements], min, max); \ } \ break; \ } @@ -2380,18 +2449,98 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } \ break; \ } + #define DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(size, type, type2, func) \ + case size: { \ + auto *Dst_d = reinterpret_cast(Tmp); \ + auto *Src1_d = reinterpret_cast(Src1); \ + auto *Src2_d = reinterpret_cast(Src2); \ + for (uint8_t i = 0; i < Elements; ++i) { \ + Dst_d[i] = (type)func((type)Src1_d[i+Elements], (type)Src2_d[i+Elements]); \ + } \ + break; \ + } - case IR::OP_VSQXTUN: { - auto Op = IROp->C(); + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); + uint8_t BitShift = Op->BitShift; + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, uint32_t, uint64_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t BitShift = Op->BitShift; uint8_t Tmp[16]; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); }; + auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, (1 << 8) - 1) - DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, (1 << 16) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP(8, uint32_t, uint64_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, int8_t, int16_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + DO_VECTOR_1SRC_2TYPE_OP(4, int16_t, int32_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, int8_t, int16_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, int16_t, int32_t, Func, std::numeric_limits::min(), std::numeric_limits::max()) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSQXTUN: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]{}; + + uint8_t Elements = Op->RegisterSize / Op->ElementSize; + + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1) + DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2401,14 +2550,14 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { auto Op = IROp->C(); void *Src1 = GetSrc(Op->Header.Args[0]); void *Src2 = GetSrc(Op->Header.Args[1]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); }; + auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, (1 << 8) - 1) - DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, (1 << 16) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1) + DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2421,10 +2570,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2437,10 +2586,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2453,10 +2602,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2469,10 +2618,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2554,18 +2703,73 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, Op->RegisterSize); break; } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto b) { return a * b; }; + switch (Op->ElementSize) { + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + void *Src1 = GetSrc(Op->Header.Args[0]); + void *Src2 = GetSrc(Op->Header.Args[1]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto b) { return a * b; }; + switch (Op->ElementSize) { + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func) + DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } case IR::OP_VSXTL: { auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -2574,15 +2778,33 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { case IR::OP_VUXTL: { auto Op = IROp->C(); void *Src = GetSrc(Op->Header.Args[0]); - uint8_t Tmp[16]; + uint8_t Tmp[16]{}; uint8_t Elements = Op->RegisterSize / Op->ElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Op->ElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0) - DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + memcpy(GDP, Tmp, Op->RegisterSize); + break; + } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + void *Src = GetSrc(Op->Header.Args[0]); + + uint8_t Tmp[16]; + + uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1); + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func, 0, 0) default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; } memcpy(GDP, Tmp, Op->RegisterSize); @@ -3221,19 +3443,19 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize; - auto Func = [](auto a, auto max) { return a; }; + auto Func = [](auto a, auto min, auto max) { return a; }; switch (Conv) { case 0x0804: { // Double <- float uint8_t Elements = Op->RegisterSize / Op->SrcElementSize; switch (Op->SrcElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0, 0) } break; } case 0x0408: { // Float <- Double uint8_t Elements = Op->RegisterSize / Op->SrcElementSize; switch (Op->SrcElementSize) { - DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0, 0) } break; diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index c393cea4d..5e29b7755 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -2197,6 +2197,63 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + smull(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + smull(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + smull(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + umull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + umull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + umull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + smull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + smull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + smull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VSQADD: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2243,6 +2300,75 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + uqadd(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + uqadd(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + uqadd(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + uqadd(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + uqsub(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + uqsub(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + uqsub(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + uqsub(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VADDP: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + addp(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + addp(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + addp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + addp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VFADD: { auto Op = IROp->C(); if (Op->ElementSize == Op->RegisterSize) { @@ -2543,6 +2669,59 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + sqxtn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + sqxtn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + case 8: + sqxtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 2: + sqxtn(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 4: + sqxtn(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 8: + sqxtn(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + else { + switch (Op->ElementSize) { + case 2: + sqxtn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + case 4: + sqxtn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + case 8: + sqxtn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + mov(GetDst(Node), VTMP1); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2561,19 +2740,39 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VSQXTUN2: { auto Op = IROp->C(); - mov(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); - switch (Op->ElementSize) { - case 2: - sqxtun2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - case 4: - sqxtun2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - case 8: - sqxtun2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 2: + sqxtun(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 4: + sqxtun(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + case 8: + sqxtun(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } } + else { + switch (Op->ElementSize) { + case 2: + sqxtun2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + case 4: + sqxtun2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + case 8: + sqxtun2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + } + mov(GetDst(Node), VTMP1); break; } case IR::OP_VSXTL: { @@ -2608,6 +2807,38 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: + sxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B()); + break; + case 2: + sxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + sxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: + uxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B()); + break; + case 2: + uxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H()); + break; + case 4: + uxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S()); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_FLOAT_FTOF: { auto Op = IROp->C(); uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize; @@ -2993,49 +3224,85 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VZIP: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); - switch (Op->ElementSize) { - case 1: { - zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); - break; + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 1: { + zip1(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + zip1(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + zip1(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - } - case 4: { - zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - } - case 8: { - zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + else { + switch (Op->ElementSize) { + case 1: { + zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } break; } case IR::OP_VZIP2: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); - switch (Op->ElementSize) { - case 1: { - zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); - break; + if (Op->RegisterSize == 8) { + switch (Op->ElementSize) { + case 1: { + zip2(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B()); + break; + } + case 2: { + zip2(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H()); + break; + } + case 4: { + zip2(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); - break; - } - case 4: { - zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); - break; - } - case 8: { - zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + else { + switch (Op->ElementSize) { + case 1: { + zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B()); + break; + } + case 2: { + zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H()); + break; + } + case 4: { + zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S()); + break; + } + case 8: { + zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D()); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } break; } @@ -3163,6 +3430,48 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + + switch (Op->ElementSize) { + case 2: { + shrn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H(), Op->BitShift); + break; + } + case 4: { + shrn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S(), Op->BitShift); + break; + } + case 8: { + shrn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D(), Op->BitShift); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + mov(VTMP1, GetSrc(Op->Header.Args[0].ID())); + switch (Op->ElementSize) { + case 2: { + shrn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H(), Op->BitShift); + break; + } + case 4: { + shrn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S(), Op->BitShift); + break; + } + case 8: { + shrn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D(), Op->BitShift); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + mov(GetDst(Node), VTMP1); + break; + } case IR::OP_VSLI: { auto Op = IROp->C(); uint8_t BitShift = Op->ByteShift * 8; diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 4ea2d74cc..75b74777c 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -2119,6 +2119,36 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + vpaddusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + vpaddusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 1: { + vpsubusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + vpsubusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VSUB: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2142,6 +2172,19 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + vphaddw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + case 4: + vphaddd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); + break; + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VUMUL: case IR::OP_VSMUL: { auto Op = IROp->C(); @@ -2159,8 +2202,22 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const break; } case IR::OP_VUMULL: { - auto Op = IROp->C(); + auto Op = IROp->C(); switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[15:00] * src2[15:00] + // [63:32 ] = src1[31:16] * src2[31:16] + // [95:64 ] = src1[47:32] * src2[47:32] + // [127:96] = src1[63:48] * src2[63:48] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } case 4: { // We need to shuffle the data for this one // x86 PMULUDQ wants the 32bit values in [31:0] and [95:64] @@ -2175,6 +2232,117 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[15:00] * src2[15:00] + // [63:32 ] = src1[31:16] * src2[31:16] + // [95:64 ] = src1[47:32] * src2[47:32] + // [127:96] = src1[63:48] * src2[63:48] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + pslld(xmm15, 16); + pslld(xmm14, 16); + psrad(xmm15, 16); + psrad(xmm14, 16); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // We need to shuffle the data for this one + // x86 PMULDQ wants the 32bit values in [31:0] and [95:64] + // Which then extends out to [63:0] and [127:64] + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b10'10'00'00); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b10'10'00'00); + + vpmuldq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[79:64 ] * src2[79:64 ] + // [63:32 ] = src1[95:80 ] * src2[95:80 ] + // [95:64 ] = src1[111:96 ] * src2[111:96 ] + // [127:96] = src1[127:112] * src2[127:112] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // IR operation: + // [63:00 ] = src1[95:64 ] * src2[95:64 ] + // [127:64] = src1[127:96] * src2[127:96] + // + // x86 vpmuludq + // [63:00 ] = src1[31:0 ] * src2[31:0 ] + // [127:64] = src1[95:64] * src2[95:64] + + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10); + + vpmuludq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: { + // IR operation: + // [31:00 ] = src1[79:64 ] * src2[79:64 ] + // [63:32 ] = src1[95:80 ] * src2[95:80 ] + // [95:64 ] = src1[111:96 ] * src2[111:96 ] + // [127:96] = src1[127:112] * src2[127:112] + // + vpxor(xmm15, xmm15, xmm15); + vpxor(xmm14, xmm14, xmm14); + vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15); + vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14); + pslld(xmm15, 16); + pslld(xmm14, 16); + psrad(xmm15, 16); + psrad(xmm14, 16); + vpmulld(GetDst(Node), xmm14, xmm15); + break; + } + case 4: { + // IR operation: + // [63:00 ] = src1[95:64 ] * src2[95:64 ] + // [127:64] = src1[127:96] * src2[127:96] + // + // x86 vpmuludq + // [63:00 ] = src1[31:0 ] * src2[31:0 ] + // [127:64] = src1[95:64] * src2[95:64] + + vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10); + vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10); + + vpmuldq(GetDst(Node), xmm14, xmm15); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + break; + } case IR::OP_VAND: { auto Op = IROp->C(); vpand(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID())); @@ -2507,6 +2675,41 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + switch (Op->ElementSize) { + case 2: + packsswb(xmm15, GetSrc(Op->Header.Args[0].ID())); + break; + case 4: + packssdw(xmm15, GetSrc(Op->Header.Args[0].ID())); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + psrldq(xmm15, 8); + movaps(GetDst(Node), xmm15); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + // Zero the lower bits + vpxor(xmm15, xmm15, xmm15); + switch (Op->ElementSize) { + case 2: + packsswb(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + case 4: + packssdw(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + + if (Op->RegisterSize == 8) { + psrldq(xmm15, Op->RegisterSize / 2); + } + vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2535,6 +2738,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const break; default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); } + if (Op->RegisterSize == 8) { + psrldq(xmm15, Op->RegisterSize / 2); + } vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15); break; @@ -2555,6 +2761,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + vpsrldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2); + switch (Op->ElementSize) { + case 1: + pmovsxbw(GetDst(Node), GetDst(Node)); + break; + case 2: + pmovsxwd(GetDst(Node), GetDst(Node)); + break; + case 4: + pmovsxdq(GetDst(Node), GetDst(Node)); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_VUXTL: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2571,6 +2794,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUXTL2: { + auto Op = IROp->C(); + vpslldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2); + switch (Op->ElementSize) { + case 1: + pmovzxbw(GetDst(Node), GetDst(Node)); + break; + case 2: + pmovzxwd(GetDst(Node), GetDst(Node)); + break; + case 4: + pmovzxdq(GetDst(Node), GetDst(Node)); + break; + default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize); + } + break; + } case IR::OP_VECTOR_STOF: { auto Op = IROp->C(); switch (Op->ElementSize) { @@ -2651,7 +2891,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VCMPEQ: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); switch (Op->ElementSize) { case 1: @@ -2672,7 +2911,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } case IR::OP_VCMPGT: { auto Op = IROp->C(); - LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize); switch (Op->ElementSize) { case 1: @@ -2896,26 +3134,47 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const auto Op = IROp->C(); movapd(xmm15, GetSrc(Op->Header.Args[0].ID())); - switch (Op->ElementSize) { - case 1: { - punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; + if (Op->RegisterSize == 8) { + vpslldq(xmm15, GetSrc(Op->Header.Args[0].ID()), 4); + vpslldq(xmm14, GetSrc(Op->Header.Args[1].ID()), 4); + switch (Op->ElementSize) { + case 1: { + vpunpckhbw(GetDst(Node), xmm15, xmm14); + break; + } + case 2: { + vpunpckhwd(GetDst(Node), xmm15, xmm14); + break; + } + case 4: { + vpunpckhdq(GetDst(Node), xmm15, xmm14); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } } - case 2: { - punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; + else { + switch (Op->ElementSize) { + case 1: { + punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 2: { + punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 4: { + punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + case 8: { + punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID())); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + movapd(GetDst(Node), xmm15); } - case 4: { - punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; - } - case 8: { - punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID())); - break; - } - default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; - } - movapd(GetDst(Node), xmm15); break; } case IR::OP_VUSHLS: { @@ -3048,6 +3307,78 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + movapd(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + vpxor(xmm15, xmm15, xmm15); + switch (Op->ElementSize) { + case 2: { + psrlw(GetDst(Node), Op->BitShift); + // <8 x i16> -> <8 x i8> + mov(rax, 0x0E'0C'0A'08'06'04'02'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + case 4: { + psrld(GetDst(Node), Op->BitShift); + // <4 x i32> -> <4 x i16> + mov(rax, 0x0D'0C'09'08'05'04'01'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + case 8: { + psrlq(GetDst(Node), Op->BitShift); + // <2 x i64> -> <2 x i32> + mov(rax, 0x0B'0A'09'08'03'02'01'00); // Lower + mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + vmovq(xmm15, rax); + vmovq(xmm14, rcx); + punpcklqdq(xmm15, xmm14); + pshufb(GetDst(Node), xmm15); + break; + } + case IR::OP_VUSHRNI2: { + // Src1 = Lower results + // Src2 = Upper Results + auto Op = IROp->C(); + movapd(xmm13, GetSrc(Op->Header.Args[1].ID())); + switch (Op->ElementSize) { + case 2: { + psrlw(xmm13, Op->BitShift); + // <8 x i16> -> <8 x i8> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0E'0C'0A'08'06'04'02'00); // Upper + break; + } + case 4: { + psrld(xmm13, Op->BitShift); + // <4 x i32> -> <4 x i16> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0D'0C'09'08'05'04'01'00); // Upper + break; + } + case 8: { + psrlq(xmm13, Op->BitShift); + // <2 x i64> -> <2 x i32> + mov(rax, 0x80'80'80'80'80'80'80'80); // Lower + mov(rcx, 0x0B'0A'09'08'03'02'01'00); // Upper + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + vmovq(xmm15, rax); + vmovq(xmm14, rcx); + punpcklqdq(xmm15, xmm14); + vpshufb(xmm14, xmm13, xmm15); + vpor(GetDst(Node), xmm14, GetSrc(Op->Header.Args[0].ID())); + break; + } case IR::OP_VEXTR: { auto Op = IROp->C(); vpalignr(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()), Op->Index); diff --git a/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp b/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp index 899301660..b14826d18 100644 --- a/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp +++ b/External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp @@ -28,6 +28,10 @@ using DestMapType = std::unordered_map; using DestMapType = std::vector; #endif +#if defined(_M_ARM_64) && defined(_M_X86_64) +#define AARCH64_ON_X86 +#endif + namespace FEXCore::CPU { static void CPUIDRun_Thunk(CPUIDEmu::FunctionResults *Results, FEXCore::CPUIDEmu *Class, uint32_t Function) { @@ -38,7 +42,7 @@ static void SetExitState_Thunk(FEXCore::Core::InternalThreadState *Thread) { Thread->State.RunningEvents.ShouldStop = true; } -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) static uint64_t AArch64ReadCycleCounter() { uint64_t res{}; asm ("mrs %0, CNTVCT_EL0" @@ -94,7 +98,7 @@ private: llvm::Function *CPUIDFunction; llvm::Function *ExitVMFunction; llvm::Function *ValuePrinter; -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) llvm::Function *AArch64ReadCycleCounterFunction; #endif @@ -154,6 +158,332 @@ private: return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ctlz, ArgTypes, Args); } + + llvm::Value *VSQXTN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { + llvm::Value *Result{}; +#ifdef _M_X86_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + std::vector VectorMaskConstant; + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, // First 8 bytes from high result of source + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, // First 4 16bits from high result of source + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + + std::vector ArgTypes = { + }; + + std::vector Args = { + Arg, + llvm::UndefValue::get(Arg->getType()), + }; + + switch (ElementSize) { + case 2: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args); + break; + case 4: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); +#elif _M_ARM_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args); + +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { + std::vector VectorMaskConstant; + +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1); + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#else + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 8, 9, 10, 11, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 4, 5, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#endif + + // Convert our upper args + ArgUpper = VSQXTN_64(ArgUpper, RegisterSize, ElementSize); + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + + llvm::Value *VSQXTUN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { + llvm::Value *Result{}; +#ifdef _M_X86_64 + // Incoming source is a 64bit register + // We want to still pass it through SSE and NEON's 128bit implementation and ignore the upper 64bits + // Cast to the type we want + llvm::Value *UndefVector = llvm::UndefValue::get(Arg->getType()); + + std::vector LargeVectorMask; + for (uint32_t i = 0; i < (RegisterSize / ElementSize * 2); ++i) { + LargeVectorMask.emplace_back(i); + } + Arg = JITState.IRBuilder->CreateShuffleVector(Arg, UndefVector, LargeVectorMask); + + RegisterSize <<= 1; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + std::vector VectorMaskConstant; + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, // First 8 bytes from high result of source + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, // First 4 16bits from high result of source + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTUN size: %d", ElementSize); break; + } + + std::vector ArgTypes = { + }; + + std::vector Args = { + Arg, + llvm::UndefValue::get(Arg->getType()), + }; + + switch (ElementSize) { + case 2: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packuswb_128, ArgTypes, Args); + break; + case 4: + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse41_packusdw, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); +#elif _M_ARM_64 + Arg = CastVectorToType(Arg, true, 16, ElementSize); + RegisterSize <<= 1; + + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtun, ArgTypes, Args); + +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTUN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { + std::vector VectorMaskConstant; + +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1); + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#else + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 8, 9, 10, 11, + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 4, 5, + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } +#endif + // Convert our upper args + ArgUpper = VSQXTUN_64(ArgUpper, RegisterSize, ElementSize); + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + + llvm::Value *VSQXTN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + std::vector ArgTypes = { + }; + + std::vector Args = { + llvm::UndefValue::get(Arg->getType()), + Arg, + }; + + uint8_t DestNumElements = RegisterSize / (ElementSize >> 1); + uint8_t DestElementSize = ElementSize >> 1; + + llvm::Value *Result{}; + std::vector VectorMaskConstant; + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 8, 9, 10, 11, 12, 13, 14, 15, // First 8 bytes from high result of source + 16, 17, 18, 19, 20, 21, 22, 23, // Second 8 bytes from the zero vector + }; + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args); + break; + case 4: + VectorMaskConstant = { + 4, 5, 6, 7, // First 4 16bits from high result of source + 8, 9, 10, 11, // Second 8 16bits from the zero vector + }; + + Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args); + break; + default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break; + } + + // We now need to shuffle the vectors + // We want the top 64bits to be zero + // The lower 64bits being the results we desire + auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0)); + + Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant); + +#elif _M_ARM_64 + std::vector ArgTypes = { + llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize), + }; + + std::vector Args = { + Arg, + }; + + auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args); +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + + llvm::Value *VSQXTN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up + // We don't care about the upper bits anyway + ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1); +#endif + // Convert our upper args + ArgUpper = VSQXTN(ArgUpper, RegisterSize, ElementSize); + + std::vector VectorMaskConstant; + + switch (ElementSize) { + case 2: + VectorMaskConstant = { + 0, 1, 2, 3, 4, 5, 6, 7, + 16, 17, 18, 19, 20, 21, 22, 23 + }; + break; + case 4: + VectorMaskConstant = { + 0, 1, 2, 3, + 8, 9, 10, 11 + }; + break; + default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break; + } + + // This will convert to VSQXTN2 in AArch64 + return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); + } + llvm::Value *VSQXTUN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) { #ifdef _M_X86_64 std::vector ArgTypes = { @@ -212,6 +542,7 @@ private: return Result; } + llvm::Value *VSQXTUN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { #ifdef _M_X86_64 // x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up @@ -243,6 +574,47 @@ private: return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant); } + llvm::Value *VADDP(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) { +#ifdef _M_X86_64 + // Will generate VPHADD + uint8_t NumElements = RegisterSize / ElementSize; + std::vector Values; + + for (size_t i = 0; i < NumElements; ++i) { + Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgLower, i));; + } + + for (size_t i = 0; i < NumElements; ++i) { + Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgUpper, i));; + } + + for (size_t i = 0; i < NumElements; ++i) { + Values[i] = JITState.IRBuilder->CreateAdd(Values[i*2], Values[i*2+1]); + } + // Cast to the type we want + llvm::Value *Result = llvm::UndefValue::get(ArgLower->getType()); + + for (size_t i = 0; i < NumElements; ++i) { + Result = JITState.IRBuilder->CreateInsertElement(Result, Values[i], i); + } +#elif _M_ARM_64 + // AArch64 doesn't seem to generate addp through the normal method + std::vector ArgTypes = { + ArgLower->getType(), + }; + + std::vector Args = { + ArgLower, ArgUpper, + }; + + auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_addp, ArgTypes, Args); +#else + static_assert(false, "Unhandle intrinsic"); +#endif + + return Result; + } + llvm::CallInst *FSHL(llvm::Value *Val, llvm::Value *Val2, llvm::Value *Amt) { std::vector ArgTypes = { Val->getType(), @@ -316,6 +688,29 @@ private: return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ssub_sat, ArgTypes, Args); } + llvm::CallInst *UQADD(llvm::Value *Arg1, llvm::Value *Arg2) { + std::vector ArgTypes = { + Arg1->getType(), + }; + std::vector Args = { + Arg1, + Arg2, + }; + + return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::uadd_sat, ArgTypes, Args); + } + llvm::CallInst *UQSUB(llvm::Value *Arg1, llvm::Value *Arg2) { + std::vector ArgTypes = { + Arg1->getType(), + }; + std::vector Args = { + Arg1, + Arg2, + }; + + return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::usub_sat, ArgTypes, Args); + } + void CreateDebugPrint(llvm::Value *Val) { std::vector Args; Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast(this))); @@ -524,7 +919,7 @@ void LLVMJITCore::CreateGlobalVariables(llvm::ExecutionEngine *Engine, llvm::Mod Engine->addGlobalMapping(JITCurrentState.CPUIDFunction, Ptr.Data); } -#ifdef _M_ARM_64 +#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86) // AArch64ReadCycleCounter Function { auto FuncType = FunctionType::get(i64, @@ -913,6 +1308,7 @@ llvm::Value *LLVMJITCore::CreateIndexedContextPtr(llvm::Value *Index, uint64_t O } llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) { + uint8_t DestSizeInBits = RegisterSize * ElementSize * 8; uint8_t NumElements = RegisterSize / ElementSize; llvm::Type *ElementType; if (Integer) { @@ -928,15 +1324,28 @@ llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8 } llvm::Type *VectorType = llvm::VectorType::get(ElementType, NumElements); + auto SrcType = Arg->getType(); - // This happens frequently - // If the source argument isn't of vector type then BitCast fails moving from Scalar->Vector domains - // Need to create a vector and insert elements in to that vector from the scalar type instead - if (!Arg->getType()->isVectorTy()) { + // If the source type is not vector(frequent with opaque types) and it matches the destination size then we can just bitcast it + if (!SrcType->isVectorTy() && SrcType->getPrimitiveSizeInBits() == DestSizeInBits) { return JITState.IRBuilder->CreateBitCast(Arg, VectorType); - } + } - return JITState.IRBuilder->CreateBitCast(Arg, VectorType); + uint8_t NumSrcElements = SrcType->getPrimitiveSizeInBits() / (ElementSize * 8); + llvm::Type *SrcVectorType = llvm::VectorType::get(ElementType, NumSrcElements); + + // First thing we need to cast the argument type to a vector size that is supported by the source type + Arg = JITState.IRBuilder->CreateBitCast(Arg, SrcVectorType); + + llvm::Value *Undef = llvm::UndefValue::get(SrcVectorType); + std::vector Mask; + // We then need to shuffle the elements to match the size we want + // If the destination size is smaller then we can just shuffle the low elements + // If our destination is more than the number of source elements then we need to pull in some undef + for (uint32_t i = 0; i < NumElements; ++i) + Mask.emplace_back(i); + + return JITState.IRBuilder->CreateShuffleVector(Arg, Undef, Mask); } llvm::Value *LLVMJITCore::CastScalarToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) { @@ -2003,6 +2412,20 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VADDP: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = VADDP(Src1, Src2, Op->RegisterSize, Op->ElementSize); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUMUL: case IR::OP_VSMUL: { auto Op = IROp->C(); @@ -2046,6 +2469,34 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VUQADD: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = UQADD(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUQSUB: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Result = UQSUB(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUMULL: { auto Op = IROp->C(); auto Src1 = GetSrc(Op->Header.Args[0]); @@ -2091,6 +2542,142 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSMULL: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {0, 1, 2, 3, 4, 5, 6, 7, 8}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {0, 1, 2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {0, 2}; // XXX: Why is this 0 and 2? It should be 0 and 1 but llvm "optimizes" the code to 0 and 4 in that case. How does this work? + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUMULL2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {9, 10, 11, 12, 13, 14, 15}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {4, 5, 6, 7}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateZExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateZExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VSMULL2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + switch (Op->ElementSize) { + case 1: { + std::vector Mask {9, 10, 11, 12, 13, 14, 15}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 2: { + std::vector Mask {4, 5, 6, 7}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask); + break; + } + case 4: { + std::vector Mask {2, 3}; + Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4)); + Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1"); + Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2"); + break; + } + default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break; + } + + auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2); + llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2); + + Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType); + Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType); + + auto Result = JITState.IRBuilder->CreateMul(Src1, Src2); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VNOT: { auto Op = IROp->C(); auto Src1 = GetSrc(Op->Header.Args[0]); @@ -2227,6 +2814,44 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSQXTN: { + auto Op = IROp->C(); + auto Src = GetSrc(Op->Header.Args[0]); + + // Cast to the type we want + Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); + + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTN_64(Src, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTN(Src, Op->RegisterSize, Op->ElementSize); + } + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VSQXTN2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1); + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VSQXTUN: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]); @@ -2234,7 +2859,13 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Cast to the type we want Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); - auto Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize); + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTUN_64(Src, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize); + } SetDest(*WrapperOp, Result); break; @@ -2245,10 +2876,16 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap auto Src2 = GetSrc(Op->Header.Args[1]); // Cast to the type we want - Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1); + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1); Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); - auto Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + llvm::Value *Result{}; + if (Op->RegisterSize == 8) { + Result = VSQXTUN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } + else { + Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize); + } SetDest(*WrapperOp, Result); break; @@ -2282,6 +2919,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Now we will do a lshr -> auto Result = JITState.IRBuilder->CreateShl(Src1, Src2); + // Ensure we shift out to zero if the shift is > element size + auto Cmp = JITState.IRBuilder->CreateICmpUGT(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8 - 1))); + Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result); + SetDest(*WrapperOp, Result); break; } @@ -2314,6 +2955,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap // Now we will do a lshr -> auto Result = JITState.IRBuilder->CreateLShr(Src1, Src2); + // Ensure we shift out to zero if the shift is > element size + auto Cmp = JITState.IRBuilder->CreateICmpUGE(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8))); + Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result); + SetDest(*WrapperOp, Result); break; } @@ -2394,6 +3039,52 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VUSHRNI: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + + uint32_t NumElements = Op->RegisterSize / Op->ElementSize; + uint8_t ElementSize = Op->ElementSize * 8; + // Cast to the type we want + Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize); + + auto Src2 = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift))); + + // Now we will do a lshr -> + llvm::Value *Result = JITState.IRBuilder->CreateLShr(Src1, Src2); + Result = JITState.IRBuilder->CreateTrunc(Result, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements)); + + SetDest(*WrapperOp, Result); + break; + } + case IR::OP_VUSHRNI2: { + auto Op = IROp->C(); + auto Src1 = GetSrc(Op->Header.Args[0]); + auto Src2 = GetSrc(Op->Header.Args[1]); + + uint32_t NumElements = Op->RegisterSize / Op->ElementSize; + uint8_t ElementSize = Op->ElementSize * 8; + + // Low source needs to already be in the size we want (Typically from VUSHRNI) + Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1); + // Cast to the type we want + Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize); + + auto Shift = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift))); + + // Now we will do a lshr -> + llvm::Value *ResultHigh = JITState.IRBuilder->CreateLShr(Src2, Shift); + ResultHigh = JITState.IRBuilder->CreateTrunc(ResultHigh, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements)); + + std::vector Mask; + for (uint32_t i = 0; i < (NumElements * 2); ++i) { + Mask.emplace_back(i); + } + auto Result = JITState.IRBuilder->CreateShuffleVector(Src1, ResultHigh, Mask, "Shuffle1"); + + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VSLI: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]); @@ -2451,6 +3142,27 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView const *IR, IR::NodeWrap SetDest(*WrapperOp, Result); break; } + case IR::OP_VSXTL2: { + auto Op = IROp->C(); + auto Src = GetSrc(Op->Header.Args[0]); + + // Cast to the type we want + Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize); + + uint8_t SrcNumElements = Op->RegisterSize / Op->ElementSize; + uint8_t DstNumElements = SrcNumElements / 2; + llvm::Type *TargetType = VectorType::get(Type::getIntNTy(*Con, Op->ElementSize * 16), DstNumElements); + + std::vector Mask; + for (uint32_t i = DstNumElements; i < (SrcNumElements); ++i) { + Mask.emplace_back(i); + } + // Shuffle out which elements we actually want + Src = JITState.IRBuilder->CreateShuffleVector(Src, Src, Mask); + auto Result = JITState.IRBuilder->CreateSExt(Src, TargetType); + SetDest(*WrapperOp, Result); + break; + } case IR::OP_VUXTL: { auto Op = IROp->C(); auto Src = GetSrc(Op->Header.Args[0]); diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index de6aceb2a..2906301ae 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -316,7 +316,16 @@ public: Op.first->Header.HasDest = true; return Op; } - + IRPair _VBitcast(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + auto Op = AllocateOp(); + Op.first->Header.Size = RegisterSize / 8; + Op.first->Header.Elements = RegisterSize / ElementSize; + Op.first->Header.NumArgs = 1; + Op.first->Header.HasDest = true; + Op.first->Header.Args[0] = ssa0->Wrapped(ListData.Begin()); + ssa0->AddUse(); + return Op; + } IRPair _Bfe(uint8_t Width, uint8_t lsb, OrderedNode *ssa0) { return _Bfe(ssa0, Width, lsb); } @@ -371,6 +380,9 @@ public: IRPair _VSQSub(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSQSub(ssa0, ssa1, RegisterSize, ElementSize); } + IRPair _VAddP(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VAddP(ssa0, ssa1, RegisterSize, ElementSize); + } IRPair _VUMin(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VUMin(ssa0, ssa1, RegisterSize, ElementSize); } @@ -443,6 +455,12 @@ public: IRPair _VSShrI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { return _VSShrI(ssa0, RegisterSize, ElementSize, BitShift); } + IRPair _VUShrNI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { + return _VUShrNI(ssa0, RegisterSize, ElementSize, BitShift); + } + IRPair _VUShrNI2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t BitShift) { + return _VUShrNI2(ssa0, ssa1, RegisterSize, ElementSize, BitShift); + } IRPair _VShlI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) { return _VShlI(ssa0, RegisterSize, ElementSize, BitShift); } @@ -491,7 +509,24 @@ public: IRPair _VSMull(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { return _VSMull(ssa0, ssa1, RegisterSize, ElementSize); } - + IRPair _VUMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VUMull2(ssa0, ssa1, RegisterSize, ElementSize); + } + IRPair _VSMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) { + return _VSMull2(ssa0, ssa1, RegisterSize, ElementSize); + } + IRPair _VSXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VSXTL(ssa0, RegisterSize, ElementSize); + } + IRPair _VSXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VSXTL2(ssa0, RegisterSize, ElementSize); + } + IRPair _VUXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VUXTL(ssa0, RegisterSize, ElementSize); + } + IRPair _VUXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) { + return _VUXTL2(ssa0, RegisterSize, ElementSize); + } IRPair _Jump() { return _Jump(InvalidNode); } diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index 951092d24..f9ac5b840 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -708,6 +708,16 @@ ] }, + "VAddP": { + "Desc": "Does a horizontal pairwise add of elements across the two source vectors", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + "VFAdd": { "HasDest": true, "SSAArgs": "2", @@ -1080,13 +1090,52 @@ ] }, + "VUShrNI": { + "Desc": "Unsigned shifts right each element and then narrows to the next lower element size", + "HasDest": true, + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize", + "uint8_t", "BitShift" + ] + }, + + "VUShrNI2": { + "Desc": ["Unsigned shifts right each element and then narrows to the next lower element size", + "Inserts results in to the high elements of the first argument" + ], + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize", + "uint8_t", "BitShift" + ] + }, + "VBitcast": { "HasDest": true, "SSAArgs": "1" }, "VSXTL": { + "Desc": "Sign extends elements from the source element size to the next size up", "HasDest": true, + "DestSize": "RegisterSize", + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VSXTL2": { + "Desc": ["Sign extends elements from the source element size to the next size up", + "Source elements come from the upper 64bits of the register" + ], + "HasDest": true, + "DestSize": "RegisterSize", "SSAArgs": "1", "Args": [ "uint8_t", "RegisterSize", @@ -1095,6 +1144,19 @@ }, "VUXTL": { + "Desc": "Zero extends elements from the source element size to the next size up", + "HasDest": true, + "SSAArgs": "1", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VUXTL2": { + "Desc": ["Zero extends elements from the source element size to the next size up", + "Source elements come from the upper 64bits of the register" + ], "HasDest": true, "SSAArgs": "1", "Args": [ @@ -1263,6 +1325,26 @@ ] }, + "VUMull2": { + "Dest": "Multiplies the high elements with size extension", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "VSMull2": { + "Dest": "Multiplies the high elements with size extension", + "HasDest": true, + "SSAArgs": "2", + "Args": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + "GetHostFlag": { "HasDest": true, "SSAArgs": "1",