steam-frame/FEX-Emu--FEX
1
0 Fork 0
mirror of https://github.com/FEX-Emu/FEX.git synced 2026-10-29 04:01:27 +01:00
Code Issues Packages Projects Releases 67 Wiki Activity

Merge pull request #72 from Sonicadvance1/new_ir_ops

Browse Source 
Adds new IR ops
This commit is contained in:
Stefanos Kornilios Mitsis Poiitidis authored and GitHub committed 2020-03-29 14:15:13 +03:00
2 parents 3fe256c370 5d55f73cd2
commit 66924cc37f
6 files changed
+1812 -121
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp
Vendored
+260 -38
View File
@@ -1999,6 +1999,17 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
} \
break; \
}
#define DO_VECTOR_PAIR_OP(size, type, func) \
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
auto *Src1_d = reinterpret_cast<type*>(Src1); \
auto *Src2_d = reinterpret_cast<type*>(Src2); \
for (uint8_t i = 0; i < Elements; ++i) { \
Dst_d[i] = func(Src1_d[i*2], Src1_d[i*2 + 1]); \
Dst_d[i+Elements] = func(Src2_d[i*2], Src2_d[i*2 + 1]); \
} \
break; \
}
#define DO_VECTOR_SCALAR_OP(size, type, func)\
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
@@ -2201,6 +2212,35 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSQSUB: {
auto Op = IROp->C<IR::IROp_VSQSub>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto b) {
__int128_t res = a - b;
if (res < std::numeric_limits<decltype(a)>::min())
return std::numeric_limits<decltype(a)>::min();
if (res > std::numeric_limits<decltype(a)>::max())
return std::numeric_limits<decltype(a)>::max();
return (decltype(a))res;
};
switch (Op->ElementSize) {
DO_VECTOR_OP(1, int8_t, Func)
DO_VECTOR_OP(2, int16_t, Func)
DO_VECTOR_OP(4, int32_t, Func)
DO_VECTOR_OP(8, int64_t, Func)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VFADD: {
auto Op = IROp->C<IR::IROp_VFAdd>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
@@ -2235,6 +2275,25 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VADDP: {
auto Op = IROp->C<IR::IROp_VAddP>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto b) { return a + b; };
switch (Op->ElementSize) {
DO_VECTOR_PAIR_OP(1, uint8_t, Func)
DO_VECTOR_PAIR_OP(2, uint16_t, Func)
DO_VECTOR_PAIR_OP(4, uint32_t, Func)
DO_VECTOR_PAIR_OP(8, uint64_t, Func)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VFMUL: {
auto Op = IROp->C<IR::IROp_VFMul>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
@@ -2351,22 +2410,32 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
#define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, max) \
#define DO_VECTOR_1SRC_2TYPE_OP(size, type, type2, func, min, max) \
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
auto *Src_d = reinterpret_cast<type2*>(Src); \
for (uint8_t i = 0; i < Elements; ++i) { \
Dst_d[i] = (type)func(Src_d[i], max); \
Dst_d[i] = (type)func(Src_d[i], min, max); \
} \
break; \
}
#define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, max) \
#define DO_VECTOR_1SRC_2TYPE_OP_TOP(size, type, type2, func, min, max) \
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
auto *Src_d = reinterpret_cast<type2*>(Src2); \
memcpy(Dst_d, Src1, Elements * sizeof(type2));\
for (uint8_t i = 0; i < Elements; ++i) { \
Dst_d[i+Elements] = (type)func(Src_d[i], max); \
Dst_d[i+Elements] = (type)func(Src_d[i], min, max); \
} \
break; \
}
#define DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(size, type, type2, func, min, max) \
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
auto *Src_d = reinterpret_cast<type2*>(Src); \
for (uint8_t i = 0; i < Elements; ++i) { \
Dst_d[i] = (type)func(Src_d[i+Elements], min, max); \
} \
break; \
}
@@ -2380,18 +2449,98 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
} \
break; \
}
#define DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(size, type, type2, func) \
case size: { \
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
auto *Src1_d = reinterpret_cast<type2*>(Src1); \
auto *Src2_d = reinterpret_cast<type2*>(Src2); \
for (uint8_t i = 0; i < Elements; ++i) { \
Dst_d[i] = (type)func((type)Src1_d[i+Elements], (type)Src2_d[i+Elements]); \
} \
break; \
}
case IR::OP_VSQXTUN: {
auto Op = IROp->C<IR::IROp_VSQXTUN>();
case IR::OP_VUSHRNI: {
auto Op = IROp->C<IR::IROp_VUShrNI>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t BitShift = Op->BitShift;
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, uint16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, uint32_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, uint32_t, uint64_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VUSHRNI2: {
auto Op = IROp->C<IR::IROp_VUShrNI2>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t BitShift = Op->BitShift;
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); };
auto Func = [BitShift](auto a, auto min, auto max) { return BitShift >= (sizeof(a) * 8) ? 0 : a >> BitShift; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, (1 << 8) - 1)
DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, (1 << 16) - 1)
DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, uint16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, uint32_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP(8, uint32_t, uint64_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSQXTN: {
auto Op = IROp->C<IR::IROp_VSQXTN>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(2, int8_t, int16_t, Func, std::numeric_limits<int8_t>::min(), std::numeric_limits<int8_t>::max())
DO_VECTOR_1SRC_2TYPE_OP(4, int16_t, int32_t, Func, std::numeric_limits<int16_t>::min(), std::numeric_limits<int16_t>::max())
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSQXTN2: {
auto Op = IROp->C<IR::IROp_VSQXTN2>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP_TOP(2, int8_t, int16_t, Func, std::numeric_limits<int8_t>::min(), std::numeric_limits<int8_t>::max())
DO_VECTOR_1SRC_2TYPE_OP_TOP(4, int16_t, int32_t, Func, std::numeric_limits<int16_t>::min(), std::numeric_limits<int16_t>::max())
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSQXTUN: {
auto Op = IROp->C<IR::IROp_VSQXTUN>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1)
DO_VECTOR_1SRC_2TYPE_OP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2401,14 +2550,14 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
auto Op = IROp->C<IR::IROp_VSQXTUN2>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16];
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))0); };
auto Func = [](auto a, auto min, auto max) { return std::max(std::min(a, (decltype(a))max), (decltype(a))min); };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, (1 << 8) - 1)
DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, (1 << 16) - 1)
DO_VECTOR_1SRC_2TYPE_OP_TOP(2, uint8_t, int16_t, Func, 0, (1 << 8) - 1)
DO_VECTOR_1SRC_2TYPE_OP_TOP(4, uint16_t, int32_t, Func, 0, (1 << 16) - 1)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2421,10 +2570,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, float, uint32_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, double, uint64_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2437,10 +2586,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, float, int32_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, double, int64_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2453,10 +2602,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2469,10 +2618,10 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2554,18 +2703,73 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VUMULL2: {
auto Op = IROp->C<IR::IROp_VUMull2>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1);
auto Func = [](auto a, auto b) { return a * b; };
switch (Op->ElementSize) {
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func)
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func)
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSMULL2: {
auto Op = IROp->C<IR::IROp_VSMull2>();
void *Src1 = GetSrc<void*>(Op->Header.Args[0]);
void *Src2 = GetSrc<void*>(Op->Header.Args[1]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1);
auto Func = [](auto a, auto b) { return a * b; };
switch (Op->ElementSize) {
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func)
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func)
DO_VECTOR_2SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSXTL: {
auto Op = IROp->C<IR::IROp_VSXTL>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16];
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(1, int16_t, int8_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(2, int32_t, int16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, int64_t, int32_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VSXTL2: {
auto Op = IROp->C<IR::IROp_VSXTL2>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1);
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, int16_t, int8_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, int32_t, int16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, int64_t, int32_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -2574,15 +2778,33 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
case IR::OP_VUXTL: {
auto Op = IROp->C<IR::IROp_VUXTL>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16];
uint8_t Tmp[16]{};
uint8_t Elements = Op->RegisterSize / Op->ElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(1, uint16_t, uint8_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(2, uint32_t, uint16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, uint64_t, uint32_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
break;
}
case IR::OP_VUXTL2: {
auto Op = IROp->C<IR::IROp_VUXTL2>();
void *Src = GetSrc<void*>(Op->Header.Args[0]);
uint8_t Tmp[16];
uint8_t Elements = Op->RegisterSize / (Op->ElementSize << 1);
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Op->ElementSize) {
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(1, uint16_t, uint8_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(2, uint32_t, uint16_t, Func, 0, 0)
DO_VECTOR_1SRC_2TYPE_OP_TOP_SRC(4, uint64_t, uint32_t, Func, 0, 0)
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
memcpy(GDP, Tmp, Op->RegisterSize);
@@ -3221,19 +3443,19 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) {
uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize;
auto Func = [](auto a, auto max) { return a; };
auto Func = [](auto a, auto min, auto max) { return a; };
switch (Conv) {
case 0x0804: { // Double <- float
uint8_t Elements = Op->RegisterSize / Op->SrcElementSize;
switch (Op->SrcElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(4, double, float, Func, 0, 0)
}
break;
}
case 0x0408: { // Float <- Double
uint8_t Elements = Op->RegisterSize / Op->SrcElementSize;
switch (Op->SrcElementSize) {
DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0)
DO_VECTOR_1SRC_2TYPE_OP(8, float, double, Func, 0, 0)
}
break;
External/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp
Vendored
+357 -48
View File
@@ -2197,6 +2197,63 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSMULL: {
auto Op = IROp->C<IR::IROp_VSMull>();
switch (Op->ElementSize) {
case 1: {
smull(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B());
break;
}
case 2: {
smull(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H());
break;
}
case 4: {
smull(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUMULL2: {
auto Op = IROp->C<IR::IROp_VUMull2>();
switch (Op->ElementSize) {
case 1: {
umull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
umull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
umull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VSMULL2: {
auto Op = IROp->C<IR::IROp_VSMull2>();
switch (Op->ElementSize) {
case 1: {
smull2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
smull2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
smull2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VSQADD: {
auto Op = IROp->C<IR::IROp_VSQAdd>();
switch (Op->ElementSize) {
@@ -2243,6 +2300,75 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUQADD: {
auto Op = IROp->C<IR::IROp_VUQAdd>();
switch (Op->ElementSize) {
case 1: {
uqadd(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
uqadd(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
uqadd(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
uqadd(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUQSUB: {
auto Op = IROp->C<IR::IROp_VUQSub>();
switch (Op->ElementSize) {
case 1: {
uqsub(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
uqsub(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
uqsub(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
uqsub(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VADDP: {
auto Op = IROp->C<IR::IROp_VAddP>();
switch (Op->ElementSize) {
case 1: {
addp(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
addp(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
addp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
addp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VFADD: {
auto Op = IROp->C<IR::IROp_VFAdd>();
if (Op->ElementSize == Op->RegisterSize) {
@@ -2543,6 +2669,59 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSQXTN: {
auto Op = IROp->C<IR::IROp_VSQXTN>();
switch (Op->ElementSize) {
case 2:
sqxtn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H());
break;
case 4:
sqxtn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
case 8:
sqxtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
break;
}
case IR::OP_VSQXTN2: {
auto Op = IROp->C<IR::IROp_VSQXTN2>();
mov(VTMP1, GetSrc(Op->Header.Args[0].ID()));
if (Op->RegisterSize == 8) {
switch (Op->ElementSize) {
case 2:
sqxtn(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
case 4:
sqxtn(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
case 8:
sqxtn(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
}
else {
switch (Op->ElementSize) {
case 2:
sqxtn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
case 4:
sqxtn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
case 8:
sqxtn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
}
mov(GetDst(Node), VTMP1);
break;
}
case IR::OP_VSQXTUN: {
auto Op = IROp->C<IR::IROp_VSQXTUN>();
switch (Op->ElementSize) {
@@ -2561,19 +2740,39 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
case IR::OP_VSQXTUN2: {
auto Op = IROp->C<IR::IROp_VSQXTUN2>();
mov(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
switch (Op->ElementSize) {
case 2:
sqxtun2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
case 4:
sqxtun2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
case 8:
sqxtun2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
mov(VTMP1, GetSrc(Op->Header.Args[0].ID()));
if (Op->RegisterSize == 8) {
switch (Op->ElementSize) {
case 2:
sqxtun(VTMP2.V8B(), GetSrc(Op->Header.Args[1].ID()).V8H());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
case 4:
sqxtun(VTMP2.V4H(), GetSrc(Op->Header.Args[1].ID()).V4S());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
case 8:
sqxtun(VTMP2.V2S(), GetSrc(Op->Header.Args[1].ID()).V2D());
ins(VTMP1.V4S(), 1, VTMP2.V4S(), 0);
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
}
else {
switch (Op->ElementSize) {
case 2:
sqxtun2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
case 4:
sqxtun2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
case 8:
sqxtun2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
}
mov(GetDst(Node), VTMP1);
break;
}
case IR::OP_VSXTL: {
@@ -2608,6 +2807,38 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSXTL2: {
auto Op = IROp->C<IR::IROp_VSXTL2>();
switch (Op->ElementSize) {
case 1:
sxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B());
break;
case 2:
sxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H());
break;
case 4:
sxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
break;
}
case IR::OP_VUXTL2: {
auto Op = IROp->C<IR::IROp_VUXTL2>();
switch (Op->ElementSize) {
case 1:
uxtl2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V16B());
break;
case 2:
uxtl2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V8H());
break;
case 4:
uxtl2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V4S());
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
break;
}
case IR::OP_FLOAT_FTOF: {
auto Op = IROp->C<IR::IROp_Float_FToF>();
uint16_t Conv = (Op->DstElementSize << 8) | Op->SrcElementSize;
@@ -2993,49 +3224,85 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
case IR::OP_VZIP: {
auto Op = IROp->C<IR::IROp_VZip>();
LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize);
switch (Op->ElementSize) {
case 1: {
zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
if (Op->RegisterSize == 8) {
switch (Op->ElementSize) {
case 1: {
zip1(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B());
break;
}
case 2: {
zip1(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H());
break;
}
case 4: {
zip1(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
}
case 2: {
zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
else {
switch (Op->ElementSize) {
case 1: {
zip1(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
zip1(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
zip1(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
zip1(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
}
break;
}
case IR::OP_VZIP2: {
auto Op = IROp->C<IR::IROp_VZip2>();
LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize);
switch (Op->ElementSize) {
case 1: {
zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
if (Op->RegisterSize == 8) {
switch (Op->ElementSize) {
case 1: {
zip2(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8B(), GetSrc(Op->Header.Args[1].ID()).V8B());
break;
}
case 2: {
zip2(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4H(), GetSrc(Op->Header.Args[1].ID()).V4H());
break;
}
case 4: {
zip2(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2S(), GetSrc(Op->Header.Args[1].ID()).V2S());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
}
case 2: {
zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
else {
switch (Op->ElementSize) {
case 1: {
zip2(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
break;
}
case 2: {
zip2(GetDst(Node).V8H(), GetSrc(Op->Header.Args[0].ID()).V8H(), GetSrc(Op->Header.Args[1].ID()).V8H());
break;
}
case 4: {
zip2(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S(), GetSrc(Op->Header.Args[1].ID()).V4S());
break;
}
case 8: {
zip2(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D(), GetSrc(Op->Header.Args[1].ID()).V2D());
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
}
break;
}
@@ -3163,6 +3430,48 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUSHRNI: {
auto Op = IROp->C<IR::IROp_VUShrNI>();
switch (Op->ElementSize) {
case 2: {
shrn(GetDst(Node).V8B(), GetSrc(Op->Header.Args[0].ID()).V8H(), Op->BitShift);
break;
}
case 4: {
shrn(GetDst(Node).V4H(), GetSrc(Op->Header.Args[0].ID()).V4S(), Op->BitShift);
break;
}
case 8: {
shrn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D(), Op->BitShift);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUSHRNI2: {
auto Op = IROp->C<IR::IROp_VUShrNI2>();
mov(VTMP1, GetSrc(Op->Header.Args[0].ID()));
switch (Op->ElementSize) {
case 2: {
shrn2(VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V8H(), Op->BitShift);
break;
}
case 4: {
shrn2(VTMP1.V8H(), GetSrc(Op->Header.Args[1].ID()).V4S(), Op->BitShift);
break;
}
case 8: {
shrn2(VTMP1.V4S(), GetSrc(Op->Header.Args[1].ID()).V2D(), Op->BitShift);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
mov(GetDst(Node), VTMP1);
break;
}
case IR::OP_VSLI: {
auto Op = IROp->C<IR::IROp_VSLI>();
uint8_t BitShift = Op->ByteShift * 8;
External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp
Vendored
+352 -21
View File
@@ -2119,6 +2119,36 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUQADD: {
auto Op = IROp->C<IR::IROp_VUQAdd>();
switch (Op->ElementSize) {
case 1: {
vpaddusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
}
case 2: {
vpaddusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUQSUB: {
auto Op = IROp->C<IR::IROp_VUQSub>();
switch (Op->ElementSize) {
case 1: {
vpsubusb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
}
case 2: {
vpsubusw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VSUB: {
auto Op = IROp->C<IR::IROp_VSub>();
switch (Op->ElementSize) {
@@ -2142,6 +2172,19 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VADDP: {
auto Op = IROp->C<IR::IROp_VAddP>();
switch (Op->ElementSize) {
case 2:
vphaddw(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
case 4:
vphaddd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
break;
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUMUL:
case IR::OP_VSMUL: {
auto Op = IROp->C<IR::IROp_VUMul>();
@@ -2159,8 +2202,22 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
break;
}
case IR::OP_VUMULL: {
auto Op = IROp->C<IR::IROp_VSub>();
auto Op = IROp->C<IR::IROp_VUMull>();
switch (Op->ElementSize) {
case 2: {
// IR operation:
// [31:00 ] = src1[15:00] * src2[15:00]
// [63:32 ] = src1[31:16] * src2[31:16]
// [95:64 ] = src1[47:32] * src2[47:32]
// [127:96] = src1[63:48] * src2[63:48]
//
vpxor(xmm15, xmm15, xmm15);
vpxor(xmm14, xmm14, xmm14);
vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15);
vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14);
vpmulld(GetDst(Node), xmm14, xmm15);
break;
}
case 4: {
// We need to shuffle the data for this one
// x86 PMULUDQ wants the 32bit values in [31:0] and [95:64]
@@ -2175,6 +2232,117 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSMULL: {
auto Op = IROp->C<IR::IROp_VSMull>();
switch (Op->ElementSize) {
case 2: {
// IR operation:
// [31:00 ] = src1[15:00] * src2[15:00]
// [63:32 ] = src1[31:16] * src2[31:16]
// [95:64 ] = src1[47:32] * src2[47:32]
// [127:96] = src1[63:48] * src2[63:48]
//
vpxor(xmm15, xmm15, xmm15);
vpxor(xmm14, xmm14, xmm14);
vpunpcklwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15);
vpunpcklwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14);
pslld(xmm15, 16);
pslld(xmm14, 16);
psrad(xmm15, 16);
psrad(xmm14, 16);
vpmulld(GetDst(Node), xmm14, xmm15);
break;
}
case 4: {
// We need to shuffle the data for this one
// x86 PMULDQ wants the 32bit values in [31:0] and [95:64]
// Which then extends out to [63:0] and [127:64]
vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b10'10'00'00);
vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b10'10'00'00);
vpmuldq(GetDst(Node), xmm14, xmm15);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VUMULL2: {
auto Op = IROp->C<IR::IROp_VUMull2>();
switch (Op->ElementSize) {
case 2: {
// IR operation:
// [31:00 ] = src1[79:64 ] * src2[79:64 ]
// [63:32 ] = src1[95:80 ] * src2[95:80 ]
// [95:64 ] = src1[111:96 ] * src2[111:96 ]
// [127:96] = src1[127:112] * src2[127:112]
//
vpxor(xmm15, xmm15, xmm15);
vpxor(xmm14, xmm14, xmm14);
vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15);
vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14);
vpmulld(GetDst(Node), xmm14, xmm15);
break;
}
case 4: {
// IR operation:
// [63:00 ] = src1[95:64 ] * src2[95:64 ]
// [127:64] = src1[127:96] * src2[127:96]
//
// x86 vpmuludq
// [63:00 ] = src1[31:0 ] * src2[31:0 ]
// [127:64] = src1[95:64] * src2[95:64]
vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10);
vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10);
vpmuludq(GetDst(Node), xmm14, xmm15);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VSMULL2: {
auto Op = IROp->C<IR::IROp_VSMull2>();
switch (Op->ElementSize) {
case 2: {
// IR operation:
// [31:00 ] = src1[79:64 ] * src2[79:64 ]
// [63:32 ] = src1[95:80 ] * src2[95:80 ]
// [95:64 ] = src1[111:96 ] * src2[111:96 ]
// [127:96] = src1[127:112] * src2[127:112]
//
vpxor(xmm15, xmm15, xmm15);
vpxor(xmm14, xmm14, xmm14);
vpunpckhwd(xmm15, GetSrc(Op->Header.Args[0].ID()), xmm15);
vpunpckhwd(xmm14, GetSrc(Op->Header.Args[1].ID()), xmm14);
pslld(xmm15, 16);
pslld(xmm14, 16);
psrad(xmm15, 16);
psrad(xmm14, 16);
vpmulld(GetDst(Node), xmm14, xmm15);
break;
}
case 4: {
// IR operation:
// [63:00 ] = src1[95:64 ] * src2[95:64 ]
// [127:64] = src1[127:96] * src2[127:96]
//
// x86 vpmuludq
// [63:00 ] = src1[31:0 ] * src2[31:0 ]
// [127:64] = src1[95:64] * src2[95:64]
vpshufd(xmm14, GetSrc(Op->Header.Args[0].ID()), 0b11'11'10'10);
vpshufd(xmm15, GetSrc(Op->Header.Args[1].ID()), 0b11'11'10'10);
vpmuldq(GetDst(Node), xmm14, xmm15);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
break;
}
case IR::OP_VAND: {
auto Op = IROp->C<IR::IROp_VAnd>();
vpand(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()));
@@ -2507,6 +2675,41 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSQXTN: {
auto Op = IROp->C<IR::IROp_VSQXTN>();
switch (Op->ElementSize) {
case 2:
packsswb(xmm15, GetSrc(Op->Header.Args[0].ID()));
break;
case 4:
packssdw(xmm15, GetSrc(Op->Header.Args[0].ID()));
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
psrldq(xmm15, 8);
movaps(GetDst(Node), xmm15);
break;
}
case IR::OP_VSQXTN2: {
auto Op = IROp->C<IR::IROp_VSQXTN2>();
// Zero the lower bits
vpxor(xmm15, xmm15, xmm15);
switch (Op->ElementSize) {
case 2:
packsswb(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
case 4:
packssdw(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
if (Op->RegisterSize == 8) {
psrldq(xmm15, Op->RegisterSize / 2);
}
vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15);
break;
}
case IR::OP_VSQXTUN: {
auto Op = IROp->C<IR::IROp_VSQXTUN>();
switch (Op->ElementSize) {
@@ -2535,6 +2738,9 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
if (Op->RegisterSize == 8) {
psrldq(xmm15, Op->RegisterSize / 2);
}
vpor(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15);
break;
@@ -2555,6 +2761,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VSXTL2: {
auto Op = IROp->C<IR::IROp_VSXTL2>();
vpsrldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2);
switch (Op->ElementSize) {
case 1:
pmovsxbw(GetDst(Node), GetDst(Node));
break;
case 2:
pmovsxwd(GetDst(Node), GetDst(Node));
break;
case 4:
pmovsxdq(GetDst(Node), GetDst(Node));
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
break;
}
case IR::OP_VUXTL: {
auto Op = IROp->C<IR::IROp_VUXTL>();
switch (Op->ElementSize) {
@@ -2571,6 +2794,23 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUXTL2: {
auto Op = IROp->C<IR::IROp_VUXTL2>();
vpslldq(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), Op->RegisterSize / 2);
switch (Op->ElementSize) {
case 1:
pmovzxbw(GetDst(Node), GetDst(Node));
break;
case 2:
pmovzxwd(GetDst(Node), GetDst(Node));
break;
case 4:
pmovzxdq(GetDst(Node), GetDst(Node));
break;
default: LogMan::Msg::A("Unknown element size: %d", Op->ElementSize);
}
break;
}
case IR::OP_VECTOR_STOF: {
auto Op = IROp->C<IR::IROp_Vector_SToF>();
switch (Op->ElementSize) {
@@ -2651,7 +2891,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
case IR::OP_VCMPEQ: {
auto Op = IROp->C<IR::IROp_VCMPEQ>();
LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize);
switch (Op->ElementSize) {
case 1:
@@ -2672,7 +2911,6 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
case IR::OP_VCMPGT: {
auto Op = IROp->C<IR::IROp_VCMPGT>();
LogMan::Throw::A(Op->RegisterSize == 16, "Can't handle register size of: %d", Op->RegisterSize);
switch (Op->ElementSize) {
case 1:
@@ -2896,26 +3134,47 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
auto Op = IROp->C<IR::IROp_VZip2>();
movapd(xmm15, GetSrc(Op->Header.Args[0].ID()));
switch (Op->ElementSize) {
case 1: {
punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
if (Op->RegisterSize == 8) {
vpslldq(xmm15, GetSrc(Op->Header.Args[0].ID()), 4);
vpslldq(xmm14, GetSrc(Op->Header.Args[1].ID()), 4);
switch (Op->ElementSize) {
case 1: {
vpunpckhbw(GetDst(Node), xmm15, xmm14);
break;
}
case 2: {
vpunpckhwd(GetDst(Node), xmm15, xmm14);
break;
}
case 4: {
vpunpckhdq(GetDst(Node), xmm15, xmm14);
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
}
case 2: {
punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
else {
switch (Op->ElementSize) {
case 1: {
punpckhbw(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
case 2: {
punpckhwd(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
case 4: {
punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
case 8: {
punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
movapd(GetDst(Node), xmm15);
}
case 4: {
punpckhdq(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
case 8: {
punpckhqdq(xmm15, GetSrc(Op->Header.Args[1].ID()));
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
movapd(GetDst(Node), xmm15);
break;
}
case IR::OP_VUSHLS: {
@@ -3048,6 +3307,78 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView<true> const
}
break;
}
case IR::OP_VUSHRNI: {
auto Op = IROp->C<IR::IROp_VUShrNI>();
movapd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
vpxor(xmm15, xmm15, xmm15);
switch (Op->ElementSize) {
case 2: {
psrlw(GetDst(Node), Op->BitShift);
// <8 x i16> -> <8 x i8>
mov(rax, 0x0E'0C'0A'08'06'04'02'00); // Lower
mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper
break;
}
case 4: {
psrld(GetDst(Node), Op->BitShift);
// <4 x i32> -> <4 x i16>
mov(rax, 0x0D'0C'09'08'05'04'01'00); // Lower
mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper
break;
}
case 8: {
psrlq(GetDst(Node), Op->BitShift);
// <2 x i64> -> <2 x i32>
mov(rax, 0x0B'0A'09'08'03'02'01'00); // Lower
mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
vmovq(xmm15, rax);
vmovq(xmm14, rcx);
punpcklqdq(xmm15, xmm14);
pshufb(GetDst(Node), xmm15);
break;
}
case IR::OP_VUSHRNI2: {
// Src1 = Lower results
// Src2 = Upper Results
auto Op = IROp->C<IR::IROp_VUShrNI2>();
movapd(xmm13, GetSrc(Op->Header.Args[1].ID()));
switch (Op->ElementSize) {
case 2: {
psrlw(xmm13, Op->BitShift);
// <8 x i16> -> <8 x i8>
mov(rax, 0x80'80'80'80'80'80'80'80); // Lower
mov(rcx, 0x0E'0C'0A'08'06'04'02'00); // Upper
break;
}
case 4: {
psrld(xmm13, Op->BitShift);
// <4 x i32> -> <4 x i16>
mov(rax, 0x80'80'80'80'80'80'80'80); // Lower
mov(rcx, 0x0D'0C'09'08'05'04'01'00); // Upper
break;
}
case 8: {
psrlq(xmm13, Op->BitShift);
// <2 x i64> -> <2 x i32>
mov(rax, 0x80'80'80'80'80'80'80'80); // Lower
mov(rcx, 0x0B'0A'09'08'03'02'01'00); // Upper
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
vmovq(xmm15, rax);
vmovq(xmm14, rcx);
punpcklqdq(xmm15, xmm14);
vpshufb(xmm14, xmm13, xmm15);
vpor(GetDst(Node), xmm14, GetSrc(Op->Header.Args[0].ID()));
break;
}
case IR::OP_VEXTR: {
auto Op = IROp->C<IR::IROp_VExtr>();
vpalignr(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), GetSrc(Op->Header.Args[1].ID()), Op->Index);
External/FEXCore/Source/Interface/Core/LLVMJIT/LLVMCore.cpp
Vendored
+724 -12
View File
@@ -28,6 +28,10 @@ using DestMapType = std::unordered_map<uint64_t, llvm::Value*>;
using DestMapType = std::vector<llvm::Value*>;
#endif
#if defined(_M_ARM_64) && defined(_M_X86_64)
#define AARCH64_ON_X86
#endif
namespace FEXCore::CPU {
static void CPUIDRun_Thunk(CPUIDEmu::FunctionResults *Results, FEXCore::CPUIDEmu *Class, uint32_t Function) {
@@ -38,7 +42,7 @@ static void SetExitState_Thunk(FEXCore::Core::InternalThreadState *Thread) {
Thread->State.RunningEvents.ShouldStop = true;
}
#ifdef _M_ARM_64
#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86)
static uint64_t AArch64ReadCycleCounter() {
uint64_t res{};
asm ("mrs %0, CNTVCT_EL0"
@@ -94,7 +98,7 @@ private:
llvm::Function *CPUIDFunction;
llvm::Function *ExitVMFunction;
llvm::Function *ValuePrinter;
#ifdef _M_ARM_64
#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86)
llvm::Function *AArch64ReadCycleCounterFunction;
#endif
@@ -154,6 +158,332 @@ private:
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ctlz, ArgTypes, Args);
}
llvm::Value *VSQXTN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) {
llvm::Value *Result{};
#ifdef _M_X86_64
Arg = CastVectorToType(Arg, true, 16, ElementSize);
RegisterSize <<= 1;
uint8_t DestNumElements = RegisterSize / (ElementSize >> 1);
uint8_t DestElementSize = ElementSize >> 1;
std::vector<uint32_t> VectorMaskConstant;
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, // First 8 bytes from high result of source
};
break;
case 4:
VectorMaskConstant = {
0, 1, // First 4 16bits from high result of source
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break;
}
std::vector<llvm::Type*> ArgTypes = {
};
std::vector<llvm::Value*> Args = {
Arg,
llvm::UndefValue::get(Arg->getType()),
};
switch (ElementSize) {
case 2:
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args);
break;
case 4:
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args);
break;
default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break;
}
// We now need to shuffle the vectors
// We want the top 64bits to be zero
// The lower 64bits being the results we desire
auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0));
Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant);
#elif _M_ARM_64
Arg = CastVectorToType(Arg, true, 16, ElementSize);
RegisterSize <<= 1;
std::vector<llvm::Type*> ArgTypes = {
llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize),
};
std::vector<llvm::Value*> Args = {
Arg,
};
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args);
#else
static_assert(false, "Unhandle intrinsic");
#endif
return Result;
}
llvm::Value *VSQXTN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) {
std::vector<uint32_t> VectorMaskConstant;
#ifdef _M_X86_64
// x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up
// We don't care about the upper bits anyway
ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1);
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, 4, 5, 6, 7,
};
break;
case 4:
VectorMaskConstant = {
0, 1, 2, 3,
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break;
}
#else
ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1);
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, 8, 9, 10, 11,
};
break;
case 4:
VectorMaskConstant = {
0, 1, 4, 5,
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break;
}
#endif
// Convert our upper args
ArgUpper = VSQXTN_64(ArgUpper, RegisterSize, ElementSize);
// This will convert to VSQXTN2 in AArch64
return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant);
}
llvm::Value *VSQXTUN_64(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) {
llvm::Value *Result{};
#ifdef _M_X86_64
// Incoming source is a 64bit register
// We want to still pass it through SSE and NEON's 128bit implementation and ignore the upper 64bits
// Cast to the type we want
llvm::Value *UndefVector = llvm::UndefValue::get(Arg->getType());
std::vector<uint32_t> LargeVectorMask;
for (uint32_t i = 0; i < (RegisterSize / ElementSize * 2); ++i) {
LargeVectorMask.emplace_back(i);
}
Arg = JITState.IRBuilder->CreateShuffleVector(Arg, UndefVector, LargeVectorMask);
RegisterSize <<= 1;
uint8_t DestNumElements = RegisterSize / (ElementSize >> 1);
uint8_t DestElementSize = ElementSize >> 1;
std::vector<uint32_t> VectorMaskConstant;
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, // First 8 bytes from high result of source
};
break;
case 4:
VectorMaskConstant = {
0, 1, // First 4 16bits from high result of source
};
break;
default: LogMan::Msg::A("Unhandled VSQXTUN size: %d", ElementSize); break;
}
std::vector<llvm::Type*> ArgTypes = {
};
std::vector<llvm::Value*> Args = {
Arg,
llvm::UndefValue::get(Arg->getType()),
};
switch (ElementSize) {
case 2:
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packuswb_128, ArgTypes, Args);
break;
case 4:
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse41_packusdw, ArgTypes, Args);
break;
default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break;
}
// We now need to shuffle the vectors
// We want the top 64bits to be zero
// The lower 64bits being the results we desire
auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0));
Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant);
#elif _M_ARM_64
Arg = CastVectorToType(Arg, true, 16, ElementSize);
RegisterSize <<= 1;
std::vector<llvm::Type*> ArgTypes = {
llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize),
};
std::vector<llvm::Value*> Args = {
Arg,
};
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtun, ArgTypes, Args);
#else
static_assert(false, "Unhandle intrinsic");
#endif
return Result;
}
llvm::Value *VSQXTUN2_64(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) {
std::vector<uint32_t> VectorMaskConstant;
#ifdef _M_X86_64
// x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up
// We don't care about the upper bits anyway
ArgLower = CastVectorToType(ArgLower, true, RegisterSize >> 1, ElementSize >> 1);
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, 4, 5, 6, 7,
};
break;
case 4:
VectorMaskConstant = {
0, 1, 2, 3,
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break;
}
#else
ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1);
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, 8, 9, 10, 11,
};
break;
case 4:
VectorMaskConstant = {
0, 1, 4, 5,
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break;
}
#endif
// Convert our upper args
ArgUpper = VSQXTUN_64(ArgUpper, RegisterSize, ElementSize);
// This will convert to VSQXTN2 in AArch64
return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant);
}
llvm::Value *VSQXTN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) {
#ifdef _M_X86_64
std::vector<llvm::Type*> ArgTypes = {
};
std::vector<llvm::Value*> Args = {
llvm::UndefValue::get(Arg->getType()),
Arg,
};
uint8_t DestNumElements = RegisterSize / (ElementSize >> 1);
uint8_t DestElementSize = ElementSize >> 1;
llvm::Value *Result{};
std::vector<uint32_t> VectorMaskConstant;
switch (ElementSize) {
case 2:
VectorMaskConstant = {
8, 9, 10, 11, 12, 13, 14, 15, // First 8 bytes from high result of source
16, 17, 18, 19, 20, 21, 22, 23, // Second 8 bytes from the zero vector
};
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packsswb_128, ArgTypes, Args);
break;
case 4:
VectorMaskConstant = {
4, 5, 6, 7, // First 4 16bits from high result of source
8, 9, 10, 11, // Second 8 16bits from the zero vector
};
Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::x86_sse2_packssdw_128, ArgTypes, Args);
break;
default: LogMan::Msg::A("Unhandled VSQXTN size: %d", ElementSize); break;
}
// We now need to shuffle the vectors
// We want the top 64bits to be zero
// The lower 64bits being the results we desire
auto ZeroVector = JITState.IRBuilder->CreateVectorSplat(DestNumElements, JITState.IRBuilder->getIntN(DestElementSize * 8, 0));
Result = JITState.IRBuilder->CreateShuffleVector(Result, ZeroVector, VectorMaskConstant);
#elif _M_ARM_64
std::vector<llvm::Type*> ArgTypes = {
llvm::VectorType::get(llvm::Type::getIntNTy(*Con, (ElementSize >> 1) * 8), RegisterSize / ElementSize),
};
std::vector<llvm::Value*> Args = {
Arg,
};
auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_sqxtn, ArgTypes, Args);
#else
static_assert(false, "Unhandle intrinsic");
#endif
return Result;
}
llvm::Value *VSQXTN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) {
#ifdef _M_X86_64
// x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up
// We don't care about the upper bits anyway
ArgLower = CastVectorToType(ArgLower, true, RegisterSize, ElementSize >> 1);
#endif
// Convert our upper args
ArgUpper = VSQXTN(ArgUpper, RegisterSize, ElementSize);
std::vector<uint32_t> VectorMaskConstant;
switch (ElementSize) {
case 2:
VectorMaskConstant = {
0, 1, 2, 3, 4, 5, 6, 7,
16, 17, 18, 19, 20, 21, 22, 23
};
break;
case 4:
VectorMaskConstant = {
0, 1, 2, 3,
8, 9, 10, 11
};
break;
default: LogMan::Msg::A("Unhandled VSQXTN2 size: %d", ElementSize); break;
}
// This will convert to VSQXTN2 in AArch64
return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant);
}
llvm::Value *VSQXTUN(llvm::Value *Arg, uint8_t RegisterSize, uint8_t ElementSize) {
#ifdef _M_X86_64
std::vector<llvm::Type*> ArgTypes = {
@@ -212,6 +542,7 @@ private:
return Result;
}
llvm::Value *VSQXTUN2(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) {
#ifdef _M_X86_64
// x86 handles the incoming lower register as a full 128bit, cast it back to ensure LLVM doesn't throw up
@@ -243,6 +574,47 @@ private:
return JITState.IRBuilder->CreateShuffleVector(ArgLower, ArgUpper, VectorMaskConstant);
}
llvm::Value *VADDP(llvm::Value *ArgLower, llvm::Value *ArgUpper, uint8_t RegisterSize, uint8_t ElementSize) {
#ifdef _M_X86_64
// Will generate VPHADD
uint8_t NumElements = RegisterSize / ElementSize;
std::vector<llvm::Value*> Values;
for (size_t i = 0; i < NumElements; ++i) {
Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgLower, i));;
}
for (size_t i = 0; i < NumElements; ++i) {
Values.emplace_back(JITState.IRBuilder->CreateExtractElement(ArgUpper, i));;
}
for (size_t i = 0; i < NumElements; ++i) {
Values[i] = JITState.IRBuilder->CreateAdd(Values[i*2], Values[i*2+1]);
}
// Cast to the type we want
llvm::Value *Result = llvm::UndefValue::get(ArgLower->getType());
for (size_t i = 0; i < NumElements; ++i) {
Result = JITState.IRBuilder->CreateInsertElement(Result, Values[i], i);
}
#elif _M_ARM_64
// AArch64 doesn't seem to generate addp through the normal method
std::vector<llvm::Type*> ArgTypes = {
ArgLower->getType(),
};
std::vector<llvm::Value*> Args = {
ArgLower, ArgUpper,
};
auto Result = JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::aarch64_neon_addp, ArgTypes, Args);
#else
static_assert(false, "Unhandle intrinsic");
#endif
return Result;
}
llvm::CallInst *FSHL(llvm::Value *Val, llvm::Value *Val2, llvm::Value *Amt) {
std::vector<llvm::Type*> ArgTypes = {
Val->getType(),
@@ -316,6 +688,29 @@ private:
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::ssub_sat, ArgTypes, Args);
}
llvm::CallInst *UQADD(llvm::Value *Arg1, llvm::Value *Arg2) {
std::vector<llvm::Type*> ArgTypes = {
Arg1->getType(),
};
std::vector<llvm::Value*> Args = {
Arg1,
Arg2,
};
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::uadd_sat, ArgTypes, Args);
}
llvm::CallInst *UQSUB(llvm::Value *Arg1, llvm::Value *Arg2) {
std::vector<llvm::Type*> ArgTypes = {
Arg1->getType(),
};
std::vector<llvm::Value*> Args = {
Arg1,
Arg2,
};
return JITState.IRBuilder->CreateIntrinsic(llvm::Intrinsic::usub_sat, ArgTypes, Args);
}
void CreateDebugPrint(llvm::Value *Val) {
std::vector<llvm::Value*> Args;
Args.emplace_back(JITState.IRBuilder->getInt64(reinterpret_cast<uint64_t>(this)));
@@ -524,7 +919,7 @@ void LLVMJITCore::CreateGlobalVariables(llvm::ExecutionEngine *Engine, llvm::Mod
Engine->addGlobalMapping(JITCurrentState.CPUIDFunction, Ptr.Data);
}
#ifdef _M_ARM_64
#if defined(_M_ARM_64) && !defined(AARCH64_ON_X86)
// AArch64ReadCycleCounter Function
{
auto FuncType = FunctionType::get(i64,
@@ -913,6 +1308,7 @@ llvm::Value *LLVMJITCore::CreateIndexedContextPtr(llvm::Value *Index, uint64_t O
}
llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) {
uint8_t DestSizeInBits = RegisterSize * ElementSize * 8;
uint8_t NumElements = RegisterSize / ElementSize;
llvm::Type *ElementType;
if (Integer) {
@@ -928,15 +1324,28 @@ llvm::Value *LLVMJITCore::CastVectorToType(llvm::Value *Arg, bool Integer, uint8
}
llvm::Type *VectorType = llvm::VectorType::get(ElementType, NumElements);
auto SrcType = Arg->getType();
// This happens frequently
// If the source argument isn't of vector type then BitCast fails moving from Scalar->Vector domains
// Need to create a vector and insert elements in to that vector from the scalar type instead
if (!Arg->getType()->isVectorTy()) {
// If the source type is not vector(frequent with opaque types) and it matches the destination size then we can just bitcast it
if (!SrcType->isVectorTy() && SrcType->getPrimitiveSizeInBits() == DestSizeInBits) {
return JITState.IRBuilder->CreateBitCast(Arg, VectorType);
}
}
return JITState.IRBuilder->CreateBitCast(Arg, VectorType);
uint8_t NumSrcElements = SrcType->getPrimitiveSizeInBits() / (ElementSize * 8);
llvm::Type *SrcVectorType = llvm::VectorType::get(ElementType, NumSrcElements);
// First thing we need to cast the argument type to a vector size that is supported by the source type
Arg = JITState.IRBuilder->CreateBitCast(Arg, SrcVectorType);
llvm::Value *Undef = llvm::UndefValue::get(SrcVectorType);
std::vector<uint32_t> Mask;
// We then need to shuffle the elements to match the size we want
// If the destination size is smaller then we can just shuffle the low elements
// If our destination is more than the number of source elements then we need to pull in some undef
for (uint32_t i = 0; i < NumElements; ++i)
Mask.emplace_back(i);
return JITState.IRBuilder->CreateShuffleVector(Arg, Undef, Mask);
}
llvm::Value *LLVMJITCore::CastScalarToType(llvm::Value *Arg, bool Integer, uint8_t RegisterSize, uint8_t ElementSize) {
@@ -2003,6 +2412,20 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VADDP: {
auto Op = IROp->C<IR::IROp_VAddP>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
auto Result = VADDP(Src1, Src2, Op->RegisterSize, Op->ElementSize);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUMUL:
case IR::OP_VSMUL: {
auto Op = IROp->C<IR::IROp_VUMul>();
@@ -2046,6 +2469,34 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUQADD: {
auto Op = IROp->C<IR::IROp_VUQAdd>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
auto Result = UQADD(Src1, Src2);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUQSUB: {
auto Op = IROp->C<IR::IROp_VUQSub>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
auto Result = UQSUB(Src1, Src2);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUMULL: {
auto Op = IROp->C<IR::IROp_VUMull>();
auto Src1 = GetSrc(Op->Header.Args[0]);
@@ -2091,6 +2542,142 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSMULL: {
auto Op = IROp->C<IR::IROp_VSMull>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
switch (Op->ElementSize) {
case 1: {
std::vector<uint32_t> Mask {0, 1, 2, 3, 4, 5, 6, 7, 8};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 2: {
std::vector<uint32_t> Mask {0, 1, 2, 3};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 4: {
std::vector<uint32_t> Mask {0, 2}; // XXX: Why is this 0 and 2? It should be 0 and 1 but llvm "optimizes" the code to 0 and 4 in that case. How does this work?
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1");
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2");
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2);
llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2);
Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType);
Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType);
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUMULL2: {
auto Op = IROp->C<IR::IROp_VUMull2>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
switch (Op->ElementSize) {
case 1: {
std::vector<uint32_t> Mask {9, 10, 11, 12, 13, 14, 15};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 2: {
std::vector<uint32_t> Mask {4, 5, 6, 7};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 4: {
std::vector<uint32_t> Mask {2, 3};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1");
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2");
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2);
llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2);
Src1 = JITState.IRBuilder->CreateZExt(Src1, VectorType);
Src2 = JITState.IRBuilder->CreateZExt(Src2, VectorType);
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSMULL2: {
auto Op = IROp->C<IR::IROp_VSMull2>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
switch (Op->ElementSize) {
case 1: {
std::vector<uint32_t> Mask {9, 10, 11, 12, 13, 14, 15};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 16));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 2: {
std::vector<uint32_t> Mask {4, 5, 6, 7};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 8));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask);
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask);
break;
}
case 4: {
std::vector<uint32_t> Mask {2, 3};
Value *Undef = UndefValue::get(VectorType::get(Src1->getType()->getVectorElementType(), 4));
Src1 = JITState.IRBuilder->CreateShuffleVector(Src1, Undef, Mask, "Shuffle1");
Src2 = JITState.IRBuilder->CreateShuffleVector(Src2, Undef, Mask, "Shuffle2");
break;
}
default: LogMan::Msg::A("Unknown Element Size: %d", Op->ElementSize); break;
}
auto ElementType = llvm::Type::getIntNTy(*Con, Op->ElementSize * 8 * 2);
llvm::Type *VectorType = llvm::VectorType::get(ElementType, Op->RegisterSize / Op->ElementSize / 2);
Src1 = JITState.IRBuilder->CreateSExt(Src1, VectorType);
Src2 = JITState.IRBuilder->CreateSExt(Src2, VectorType);
auto Result = JITState.IRBuilder->CreateMul(Src1, Src2);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VNOT: {
auto Op = IROp->C<IR::IROp_VNot>();
auto Src1 = GetSrc(Op->Header.Args[0]);
@@ -2227,6 +2814,44 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSQXTN: {
auto Op = IROp->C<IR::IROp_VSQXTN>();
auto Src = GetSrc(Op->Header.Args[0]);
// Cast to the type we want
Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize);
llvm::Value *Result{};
if (Op->RegisterSize == 8) {
Result = VSQXTN_64(Src, Op->RegisterSize, Op->ElementSize);
}
else {
Result = VSQXTN(Src, Op->RegisterSize, Op->ElementSize);
}
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSQXTN2: {
auto Op = IROp->C<IR::IROp_VSQXTN2>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
llvm::Value *Result{};
if (Op->RegisterSize == 8) {
Result = VSQXTN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize);
}
else {
Result = VSQXTN2(Src1, Src2, Op->RegisterSize, Op->ElementSize);
}
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSQXTUN: {
auto Op = IROp->C<IR::IROp_VSQXTUN>();
auto Src = GetSrc(Op->Header.Args[0]);
@@ -2234,7 +2859,13 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
// Cast to the type we want
Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize);
auto Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize);
llvm::Value *Result{};
if (Op->RegisterSize == 8) {
Result = VSQXTUN_64(Src, Op->RegisterSize, Op->ElementSize);
}
else {
Result = VSQXTUN(Src, Op->RegisterSize, Op->ElementSize);
}
SetDest(*WrapperOp, Result);
break;
@@ -2245,10 +2876,16 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
auto Src2 = GetSrc(Op->Header.Args[1]);
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1);
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize >> 1);
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
auto Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize);
llvm::Value *Result{};
if (Op->RegisterSize == 8) {
Result = VSQXTUN2_64(Src1, Src2, Op->RegisterSize, Op->ElementSize);
}
else {
Result = VSQXTUN2(Src1, Src2, Op->RegisterSize, Op->ElementSize);
}
SetDest(*WrapperOp, Result);
break;
@@ -2282,6 +2919,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
auto Result = JITState.IRBuilder->CreateShl(Src1, Src2);
// Ensure we shift out to zero if the shift is > element size
auto Cmp = JITState.IRBuilder->CreateICmpUGT(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8 - 1)));
Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result);
SetDest(*WrapperOp, Result);
break;
}
@@ -2314,6 +2955,10 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
auto Result = JITState.IRBuilder->CreateLShr(Src1, Src2);
// Ensure we shift out to zero if the shift is > element size
auto Cmp = JITState.IRBuilder->CreateICmpUGE(Src2, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, Op->ElementSize * 8)));
Result = JITState.IRBuilder->CreateSelect(Cmp, JITState.IRBuilder->CreateVectorSplat(Op->RegisterSize / Op->ElementSize, JITState.IRBuilder->getIntN(Op->ElementSize * 8, 0)), Result);
SetDest(*WrapperOp, Result);
break;
}
@@ -2394,6 +3039,52 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUSHRNI: {
auto Op = IROp->C<IR::IROp_VUShrNI>();
auto Src1 = GetSrc(Op->Header.Args[0]);
uint32_t NumElements = Op->RegisterSize / Op->ElementSize;
uint8_t ElementSize = Op->ElementSize * 8;
// Cast to the type we want
Src1 = CastVectorToType(Src1, true, Op->RegisterSize, Op->ElementSize);
auto Src2 = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift)));
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
llvm::Value *Result = JITState.IRBuilder->CreateLShr(Src1, Src2);
Result = JITState.IRBuilder->CreateTrunc(Result, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements));
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUSHRNI2: {
auto Op = IROp->C<IR::IROp_VUShrNI2>();
auto Src1 = GetSrc(Op->Header.Args[0]);
auto Src2 = GetSrc(Op->Header.Args[1]);
uint32_t NumElements = Op->RegisterSize / Op->ElementSize;
uint8_t ElementSize = Op->ElementSize * 8;
// Low source needs to already be in the size we want (Typically from VUSHRNI)
Src1 = CastVectorToType(Src1, true, Op->RegisterSize >> 1, Op->ElementSize >> 1);
// Cast to the type we want
Src2 = CastVectorToType(Src2, true, Op->RegisterSize, Op->ElementSize);
auto Shift = JITState.IRBuilder->CreateVectorSplat(NumElements, JITState.IRBuilder->getIntN(ElementSize, std::min((uint8_t)(ElementSize - 1), Op->BitShift)));
// Now we will do a lshr <NumElements x i1> -> <NumElements x ElementSize>
llvm::Value *ResultHigh = JITState.IRBuilder->CreateLShr(Src2, Shift);
ResultHigh = JITState.IRBuilder->CreateTrunc(ResultHigh, llvm::VectorType::get(llvm::Type::getIntNTy(*Con, ElementSize >> 1), NumElements));
std::vector<uint32_t> Mask;
for (uint32_t i = 0; i < (NumElements * 2); ++i) {
Mask.emplace_back(i);
}
auto Result = JITState.IRBuilder->CreateShuffleVector(Src1, ResultHigh, Mask, "Shuffle1");
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSLI: {
auto Op = IROp->C<IR::IROp_VSLI>();
auto Src = GetSrc(Op->Header.Args[0]);
@@ -2451,6 +3142,27 @@ void LLVMJITCore::HandleIR(FEXCore::IR::IRListView<true> const *IR, IR::NodeWrap
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VSXTL2: {
auto Op = IROp->C<IR::IROp_VSXTL2>();
auto Src = GetSrc(Op->Header.Args[0]);
// Cast to the type we want
Src = CastVectorToType(Src, true, Op->RegisterSize, Op->ElementSize);
uint8_t SrcNumElements = Op->RegisterSize / Op->ElementSize;
uint8_t DstNumElements = SrcNumElements / 2;
llvm::Type *TargetType = VectorType::get(Type::getIntNTy(*Con, Op->ElementSize * 16), DstNumElements);
std::vector<uint32_t> Mask;
for (uint32_t i = DstNumElements; i < (SrcNumElements); ++i) {
Mask.emplace_back(i);
}
// Shuffle out which elements we actually want
Src = JITState.IRBuilder->CreateShuffleVector(Src, Src, Mask);
auto Result = JITState.IRBuilder->CreateSExt(Src, TargetType);
SetDest(*WrapperOp, Result);
break;
}
case IR::OP_VUXTL: {
auto Op = IROp->C<IR::IROp_VUXTL>();
auto Src = GetSrc(Op->Header.Args[0]);
External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
Vendored
+37 -2
View File
@@ -316,7 +316,16 @@ public:
Op.first->Header.HasDest = true;
return Op;
}
IRPair<IROp_VBitcast> _VBitcast(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) {
auto Op = AllocateOp<IROp_VBitcast, IROps::OP_VBITCAST>();
Op.first->Header.Size = RegisterSize / 8;
Op.first->Header.Elements = RegisterSize / ElementSize;
Op.first->Header.NumArgs = 1;
Op.first->Header.HasDest = true;
Op.first->Header.Args[0] = ssa0->Wrapped(ListData.Begin());
ssa0->AddUse();
return Op;
}
IRPair<IROp_Bfe> _Bfe(uint8_t Width, uint8_t lsb, OrderedNode *ssa0) {
return _Bfe(ssa0, Width, lsb);
}
@@ -371,6 +380,9 @@ public:
IRPair<IROp_VSQSub> _VSQSub(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VSQSub(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VAddP> _VAddP(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VAddP(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VUMin> _VUMin(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VUMin(ssa0, ssa1, RegisterSize, ElementSize);
}
@@ -443,6 +455,12 @@ public:
IRPair<IROp_VSShrI> _VSShrI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) {
return _VSShrI(ssa0, RegisterSize, ElementSize, BitShift);
}
IRPair<IROp_VUShrNI> _VUShrNI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) {
return _VUShrNI(ssa0, RegisterSize, ElementSize, BitShift);
}
IRPair<IROp_VUShrNI2> _VUShrNI2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t BitShift) {
return _VUShrNI2(ssa0, ssa1, RegisterSize, ElementSize, BitShift);
}
IRPair<IROp_VShlI> _VShlI(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, uint8_t BitShift) {
return _VShlI(ssa0, RegisterSize, ElementSize, BitShift);
}
@@ -491,7 +509,24 @@ public:
IRPair<IROp_VSMull> _VSMull(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VSMull(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VUMull2> _VUMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VUMull2(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VSMull2> _VSMull2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0, OrderedNode *ssa1) {
return _VSMull2(ssa0, ssa1, RegisterSize, ElementSize);
}
IRPair<IROp_VSXTL> _VSXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) {
return _VSXTL(ssa0, RegisterSize, ElementSize);
}
IRPair<IROp_VSXTL2> _VSXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) {
return _VSXTL2(ssa0, RegisterSize, ElementSize);
}
IRPair<IROp_VUXTL> _VUXTL(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) {
return _VUXTL(ssa0, RegisterSize, ElementSize);
}
IRPair<IROp_VUXTL2> _VUXTL2(uint8_t RegisterSize, uint8_t ElementSize, OrderedNode *ssa0) {
return _VUXTL2(ssa0, RegisterSize, ElementSize);
}
IRPair<IROp_Jump> _Jump() {
return _Jump(InvalidNode);
}
External/FEXCore/Source/Interface/IR/IR.json
Vendored
+82
View File
@@ -708,6 +708,16 @@
]
},
"VAddP": {
"Desc": "Does a horizontal pairwise add of elements across the two source vectors",
"HasDest": true,
"SSAArgs": "2",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VFAdd": {
"HasDest": true,
"SSAArgs": "2",
@@ -1080,13 +1090,52 @@
]
},
"VUShrNI": {
"Desc": "Unsigned shifts right each element and then narrows to the next lower element size",
"HasDest": true,
"SSAArgs": "1",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize",
"uint8_t", "BitShift"
]
},
"VUShrNI2": {
"Desc": ["Unsigned shifts right each element and then narrows to the next lower element size",
"Inserts results in to the high elements of the first argument"
],
"HasDest": true,
"SSAArgs": "2",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize",
"uint8_t", "BitShift"
]
},
"VBitcast": {
"HasDest": true,
"SSAArgs": "1"
},
"VSXTL": {
"Desc": "Sign extends elements from the source element size to the next size up",
"HasDest": true,
"DestSize": "RegisterSize",
"SSAArgs": "1",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VSXTL2": {
"Desc": ["Sign extends elements from the source element size to the next size up",
"Source elements come from the upper 64bits of the register"
],
"HasDest": true,
"DestSize": "RegisterSize",
"SSAArgs": "1",
"Args": [
"uint8_t", "RegisterSize",
@@ -1095,6 +1144,19 @@
},
"VUXTL": {
"Desc": "Zero extends elements from the source element size to the next size up",
"HasDest": true,
"SSAArgs": "1",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VUXTL2": {
"Desc": ["Zero extends elements from the source element size to the next size up",
"Source elements come from the upper 64bits of the register"
],
"HasDest": true,
"SSAArgs": "1",
"Args": [
@@ -1263,6 +1325,26 @@
]
},
"VUMull2": {
"Dest": "Multiplies the high elements with size extension",
"HasDest": true,
"SSAArgs": "2",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"VSMull2": {
"Dest": "Multiplies the high elements with size extension",
"HasDest": true,
"SSAArgs": "2",
"Args": [
"uint8_t", "RegisterSize",
"uint8_t", "ElementSize"
]
},
"GetHostFlag": {
"HasDest": true,
"SSAArgs": "1",