From e4182108f6ef06e54651dc2e26aa68d32f14c81c Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 22 Sep 2020 18:42:23 -0700 Subject: [PATCH 1/4] Adds new IR ops for supporting host rounding mode conversion --- External/FEXCore/Source/Interface/IR/IR.json | 104 ++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index 76f79c68b..a0261a325 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -40,7 +40,13 @@ "static constexpr FEXCore::IR::FenceType Fence_Load {0}", "static constexpr FEXCore::IR::FenceType Fence_Store {1}", - "static constexpr FEXCore::IR::FenceType Fence_LoadStore {2}" + "static constexpr FEXCore::IR::FenceType Fence_LoadStore {2}", + + "constexpr static uint8_t ROUND_MODE_NEAREST = 0", + "constexpr static uint8_t ROUND_MODE_NEGATIVE_INFINITY = 1", + "constexpr static uint8_t ROUND_MODE_POSITIVE_INFINITY = 2", + "constexpr static uint8_t ROUND_MODE_TOWARDS_ZERO = 3", + "constexpr static uint8_t ROUND_MODE_FLUSH_TO_ZERO = 1 << 2" ], "Ops": { @@ -261,6 +267,28 @@ ] }, + "GetRoundingMode": { + "Desc": ["Gets the current rounding mode options" + ], + "OpClass": "Misc", + "HasDest": true, + "DestClass": "GPR", + "FixedDestSize": "4" + }, + + "SetRoundingMode": { + "Desc": ["Sets the current rounding mode options for the thread" + ], + "HasSideEffects": true, + "OpClass": "Misc", + "HasDest": false, + "DestClass": "GPR", + "SSAArgs": "1", + "SSANames": [ + "RoundMode" + ] + }, + "Constant": { "Desc": ["Generates a 64bit constant inside of a GPR", "Unsupported to create a constant in FPR" @@ -1323,6 +1351,42 @@ ] }, + "Float_ToGPR_U": { + "Desc": ["Moves the scalar element to a GPR with conversion", + "Converts the 32bit or 64bit float to an unsigned integer", + "Rounding mode determined by host flag's rounding mode" + ], + "OpClass": "ALU", + "HasDest": true, + "DestClass": "GPR", + "DestSize": "ElementSize", + "SSAArgs": "1", + "SSANames": [ + "Scalar" + ], + "Args": [ + "uint8_t", "ElementSize" + ] + }, + + "Float_ToGPR_S": { + "Desc": ["Moves the scalar element to a GPR with conversion", + "Converts the 32bit or 64bit float to an signed integer", + "Rounding mode determined by host flag's rounding mode" + ], + "OpClass": "ALU", + "HasDest": true, + "DestClass": "GPR", + "DestSize": "ElementSize", + "SSAArgs": "1", + "SSANames": [ + "Scalar" + ], + "Args": [ + "uint8_t", "ElementSize" + ] + }, + "Float_ToGPR_ZU": { "Desc": ["Moves the scalar element to a GPR with conversion", "Converts the 32bit or 64bit float to an unsigned integer rounding towards zero (Truncating)" @@ -2778,6 +2842,44 @@ ] }, + "Vector_FToU": { + "OpClass": "Conv", + "Desc": ["Vector op: Converts float to unsigned integer", + "Rounding mode determined by host rounding mode" + ], + "HasDest": true, + "DestClass": "FPR", + "DestSize": "RegisterSize", + "NumElements": "RegisterSize / ElementSize", + "SSAArgs": "1", + "SSANames": [ + "Vector" + ], + "HelperArgs": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + + "Vector_FToS": { + "OpClass": "Conv", + "Desc": ["Vector op: Converts float to signed integer, rounding towards zero", + "Rounding mode determined by host rounding mode" + ], + "HasDest": true, + "DestClass": "FPR", + "DestSize": "RegisterSize", + "NumElements": "RegisterSize / ElementSize", + "SSAArgs": "1", + "SSANames": [ + "Vector" + ], + "HelperArgs": [ + "uint8_t", "RegisterSize", + "uint8_t", "ElementSize" + ] + }, + "Vector_FToZU": { "OpClass": "Conv", "Desc": "Vector op: Converts float to unsigned integer, rounding towards zero", From e84e65d246301876aee05a49bae4f9c07a2c535b Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 22 Sep 2020 18:43:40 -0700 Subject: [PATCH 2/4] Implements new IR ops in the interpreter --- .../Core/Interpreter/InterpreterCore.cpp | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp index c0261be57..fc7e67cd5 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterCore.cpp @@ -19,6 +19,9 @@ #include #include #include +#ifdef _M_X86_64 +#include +#endif namespace FEXCore::CPU { @@ -2893,6 +2896,38 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { memcpy(GDP, Tmp, OpSize); break; } + case IR::OP_VECTOR_FTOU: { + auto Op = IROp->C(); + void *Src = GetSrc(SSAData, Op->Header.Args[0]); + uint8_t Tmp[16]; + + uint8_t Elements = OpSize / Op->Header.ElementSize; + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->Header.ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(4, uint32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, uint64_t, double, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break; + } + memcpy(GDP, Tmp, OpSize); + break; + } + case IR::OP_VECTOR_FTOS: { + auto Op = IROp->C(); + void *Src = GetSrc(SSAData, Op->Header.Args[0]); + uint8_t Tmp[16]; + + uint8_t Elements = OpSize / Op->Header.ElementSize; + + auto Func = [](auto a, auto min, auto max) { return a; }; + switch (Op->Header.ElementSize) { + DO_VECTOR_1SRC_2TYPE_OP(4, int32_t, float, Func, 0, 0) + DO_VECTOR_1SRC_2TYPE_OP(8, int64_t, double, Func, 0, 0) + default: LogMan::Msg::A("Unknown Element Size: %d", Op->Header.ElementSize); break; + } + memcpy(GDP, Tmp, OpSize); + break; + } case IR::OP_VUMUL: { auto Op = IROp->C(); void *Src1 = GetSrc(SSAData, Op->Header.Args[0]); @@ -3698,6 +3733,30 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { } break; } + case IR::OP_FLOAT_TOGPR_S: { + auto Op = IROp->C(); + if (Op->Header.ElementSize == 8) { + int64_t Dst = (int64_t)*GetSrc(SSAData, Op->Header.Args[0]); + memcpy(GDP, &Dst, Op->Header.ElementSize); + } + else { + int32_t Dst = (int32_t)*GetSrc(SSAData, Op->Header.Args[0]); + memcpy(GDP, &Dst, Op->Header.ElementSize); + } + break; + } + case IR::OP_FLOAT_TOGPR_U: { + auto Op = IROp->C(); + if (Op->Header.ElementSize == 8) { + uint64_t Dst = (uint64_t)*GetSrc(SSAData, Op->Header.Args[0]); + memcpy(GDP, &Dst, Op->Header.ElementSize); + } + else { + uint32_t Dst = (uint32_t)*GetSrc(SSAData, Op->Header.Args[0]); + memcpy(GDP, &Dst, Op->Header.ElementSize); + } + break; + } case IR::OP_FLOAT_FTOF: { auto Op = IROp->C(); uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; @@ -4248,6 +4307,77 @@ void InterpreterCore::ExecuteCode(FEXCore::Core::InternalThreadState *Thread) { GD = ResultFlags; break; } + case IR::OP_GETROUNDINGMODE: { + uint32_t GuestRounding{}; +#ifdef _M_ARM_64 + uint64_t Tmp{}; + __asm(R"( + mrs %[Tmp], FPCR; + )" + : [Tmp] "=r" (Tmp)); + // Extract the rounding + // On ARM the ordering is different than on x86 + GuestRounding |= ((Tmp >> 24) & 1) ? ROUND_MODE_FLUSH_TO_ZERO : 0; + uint8_t RoundingMode = (Tmp >> 22) & 0b11; + if (RoundingMode == 0) + GuestRounding |= ROUND_MODE_NEAREST; + else if (RoundingMode == 1) + GuestRounding |= ROUND_MODE_POSITIVE_INFINITY; + else if (RoundingMode == 2) + GuestRounding |= ROUND_MODE_NEGATIVE_INFINITY; + else if (RoundingMode == 3) + GuestRounding |= ROUND_MODE_TOWARDS_ZERO; +#else + GuestRounding = _mm_getcsr(); + + // Extract the rounding + GuestRounding = (GuestRounding >> 13) & 0b111; +#endif + memcpy(GDP, &GuestRounding, sizeof(GuestRounding)); + break; + } + + case IR::OP_SETROUNDINGMODE: { + auto Op = IROp->C(); + uint8_t GuestRounding = *GetSrc(SSAData, Op->Header.Args[0]); +#ifdef _M_ARM_64 + uint64_t HostRounding{}; + __asm volatile(R"( + mrs %[Tmp], FPCR; + )" + : [Tmp] "=r" (HostRounding)); + // Mask out the rounding + HostRounding &= ~(0b111 << 22); + + HostRounding |= (GuestRounding & ROUND_MODE_FLUSH_TO_ZERO) ? (1U << 24) : 0; + + uint8_t RoundingMode = GuestRounding & 0b11; + if (RoundingMode == ROUND_MODE_NEAREST) + HostRounding |= (0b00U << 22); + else if (RoundingMode == ROUND_MODE_POSITIVE_INFINITY) + HostRounding |= (0b01U << 22); + else if (RoundingMode == ROUND_MODE_NEGATIVE_INFINITY) + HostRounding |= (0b10U << 22); + else if (RoundingMode == ROUND_MODE_TOWARDS_ZERO) + HostRounding |= (0b11U << 22); + + __asm volatile(R"( + msr FPCR, %[Tmp]; + )" + :: [Tmp] "r" (HostRounding)); +#else + uint32_t HostRounding = _mm_getcsr(); + + // Cut out the host rounding mode + HostRounding &= ~(0b111 << 13); + + // Insert our new rounding mode + HostRounding |= GuestRounding << 13; + _mm_setcsr(HostRounding); +#endif + + break; + } default: LogMan::Msg::A("Unknown IR Op: %d(%s)", IROp->Op, FEXCore::IR::GetName(IROp->Op).data()); break; From 6da5071433a4cd66d5d670d40e1fd5c494180296 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 22 Sep 2020 18:45:16 -0700 Subject: [PATCH 3/4] Implements the new IR ops in the Arm64 JIT --- .../Interface/Core/JIT/Arm64/ALUOps.cpp | 18 ++++++ .../Core/JIT/Arm64/ConversionOps.cpp | 32 ++++++++++ .../Interface/Core/JIT/Arm64/JITClass.h | 6 ++ .../Interface/Core/JIT/Arm64/MiscOps.cpp | 61 +++++++++++++++++++ 4 files changed, 117 insertions(+) diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp index 31fa5fc67..40cef05a4 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ALUOps.cpp @@ -1032,6 +1032,22 @@ DEF_OP(Float_ToGPR_ZS) { } } +DEF_OP(Float_ToGPR_U) { + LogMan::Msg::D("Unimplemented"); +} + +DEF_OP(Float_ToGPR_S) { + auto Op = IROp->C(); + if (Op->Header.ElementSize == 8) { + frinti(VTMP1.D(), GetSrc(Op->Header.Args[0].ID()).D()); + fcvtzs(GetReg(Node), VTMP1.D()); + } + else { + frinti(VTMP1.S(), GetSrc(Op->Header.Args[0].ID()).S()); + fcvtzs(GetReg(Node), VTMP1.S()); + } +} + DEF_OP(FCmp) { auto Op = IROp->C(); @@ -1110,6 +1126,8 @@ void JITCore::RegisterALUHandlers() { REGISTER_OP(VEXTRACTTOGPR, VExtractToGPR); REGISTER_OP(FLOAT_TOGPR_ZU, Float_ToGPR_ZU); REGISTER_OP(FLOAT_TOGPR_ZS, Float_ToGPR_ZS); + REGISTER_OP(FLOAT_TOGPR_U, Float_ToGPR_U); + REGISTER_OP(FLOAT_TOGPR_S, Float_ToGPR_S); REGISTER_OP(FCMP, FCmp); REGISTER_OP(F80CMP, F80Cmp); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp index e5b5493c4..7d5a0b8e2 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/ConversionOps.cpp @@ -132,6 +132,36 @@ DEF_OP(Vector_FToZS) { } } +DEF_OP(Vector_FToU) { + auto Op = IROp->C(); + switch (Op->Header.ElementSize) { + case 4: + frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S()); + fcvtzu(GetDst(Node).V4S(), GetDst(Node).V4S()); + break; + case 8: + frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D()); + fcvtzu(GetDst(Node).V2D(), GetDst(Node).V2D()); + break; + default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize); + } +} + +DEF_OP(Vector_FToS) { + auto Op = IROp->C(); + switch (Op->Header.ElementSize) { + case 4: + frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S()); + fcvtzs(GetDst(Node).V4S(), GetDst(Node).V4S()); + break; + case 8: + frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D()); + fcvtzs(GetDst(Node).V2D(), GetDst(Node).V2D()); + break; + default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize); + } +} + DEF_OP(Vector_FToF) { auto Op = IROp->C(); uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; @@ -161,6 +191,8 @@ void JITCore::RegisterConversionHandlers() { REGISTER_OP(VECTOR_STOF, Vector_SToF); REGISTER_OP(VECTOR_FTOZU, Vector_FToZU); REGISTER_OP(VECTOR_FTOZS, Vector_FToZS); + REGISTER_OP(VECTOR_FTOU, Vector_FToU); + REGISTER_OP(VECTOR_FTOS, Vector_FToS); REGISTER_OP(VECTOR_FTOF, Vector_FToF); #undef REGISTER_OP } diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index 8160234b7..104b1e5e8 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -280,6 +280,8 @@ private: DEF_OP(VExtractToGPR); DEF_OP(Float_ToGPR_ZU); DEF_OP(Float_ToGPR_ZS); + DEF_OP(Float_ToGPR_U); + DEF_OP(Float_ToGPR_S); DEF_OP(FCmp); DEF_OP(F80Cmp); @@ -323,6 +325,8 @@ private: DEF_OP(Vector_SToF); DEF_OP(Vector_FToZU); DEF_OP(Vector_FToZS); + DEF_OP(Vector_FToU); + DEF_OP(Vector_FToS); DEF_OP(Vector_FToF); ///< Flag ops @@ -353,6 +357,8 @@ private: DEF_OP(Phi); DEF_OP(PhiValue); DEF_OP(Print); + DEF_OP(GetRoundingMode); + DEF_OP(SetRoundingMode); ///< Move ops DEF_OP(ExtractElementPair); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp index ae0022195..2eda9945f 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MiscOps.cpp @@ -52,6 +52,65 @@ DEF_OP(Break) { } } +DEF_OP(GetRoundingMode) { + auto Op = IROp->C(); + auto Dst = GetReg(Node); + mrs(Dst, FPCR); + lsr(Dst, Dst, 22); + + // FTZ is already in the correct location + // Rounding mode is different + and_(TMP1, Dst, 0b11); + + cmp(TMP1, 1); + LoadConstant(TMP3, IR::ROUND_MODE_POSITIVE_INFINITY); + csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq); + + cmp(TMP1, 2); + LoadConstant(TMP3, IR::ROUND_MODE_NEGATIVE_INFINITY); + csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + + cmp(TMP1, 3); + LoadConstant(TMP3, IR::ROUND_MODE_TOWARDS_ZERO); + csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + + orr(Dst, Dst, TMP2); + + bfi(Dst, TMP2, 0, 2); +} + +DEF_OP(SetRoundingMode) { + auto Op = IROp->C(); + auto Src = GetReg(Op->Header.Args[0].ID()); + + // Setup the rounding flags correctly + and_(TMP1, Src, 0b11); + + cmp(TMP1, IR::ROUND_MODE_POSITIVE_INFINITY); + LoadConstant(TMP3, 1); + csel(TMP2, TMP3, xzr, vixl::aarch64::Condition::eq); + + cmp(TMP1, IR::ROUND_MODE_NEGATIVE_INFINITY); + LoadConstant(TMP3, 2); + csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + + cmp(TMP1, IR::ROUND_MODE_TOWARDS_ZERO); + LoadConstant(TMP3, 3); + csel(TMP2, TMP3, TMP2, vixl::aarch64::Condition::eq); + + mrs(TMP1, FPCR); + + // Insert the rounding flags + bfi(TMP1, TMP2, 22, 2); + + // Insert the FTZ flag + lsr(TMP2, Src, 2); + bfi(TMP1, TMP2, 24, 1); + + // Now save the new FPCR + msr(FPCR, TMP1); +} + #undef DEF_OP void JITCore::RegisterMiscHandlers() { #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &JITCore::Op_##x @@ -65,6 +124,8 @@ void JITCore::RegisterMiscHandlers() { REGISTER_OP(PHI, NoOp); REGISTER_OP(PHIVALUE, NoOp); REGISTER_OP(PRINT, Unhandled); + REGISTER_OP(GETROUNDINGMODE, GetRoundingMode); + REGISTER_OP(SETROUNDINGMODE, SetRoundingMode); #undef REGISTER_OP } } From 394f20d2c807b9b09d4d13a0bee914b3c88eec6c Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Tue, 22 Sep 2020 18:46:04 -0700 Subject: [PATCH 4/4] Implements the new IR ops in the x86 JIT --- .../Source/Interface/Core/JIT/x86_64/JIT.cpp | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp index 1735b77be..c0c4a8161 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JIT.cpp @@ -3455,6 +3455,19 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const } break; } + case IR::OP_VECTOR_FTOS: { + auto Op = IROp->C(); + switch (Op->Header.ElementSize) { + case 4: + cvtps2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + break; + case 8: + cvtpd2dq(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + break; + default: LogMan::Msg::A("Unknown castGPR element size: %d", Op->Header.ElementSize); + } + break; + } case IR::OP_VECTOR_FTOF: { auto Op = IROp->C(); uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; @@ -4145,6 +4158,15 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const cvttss2si(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); break; } + case IR::OP_FLOAT_TOGPR_S: { + auto Op = IROp->C(); + if (Op->Header.ElementSize == 8) { + cvtsd2si(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + } + else + cvtss2si(GetDst(Node), GetSrc(Op->Header.Args[0].ID())); + break; + } case IR::OP_FLOAT_FTOF: { auto Op = IROp->C(); uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize; @@ -4910,6 +4932,39 @@ void *JITCore::CompileCode([[maybe_unused]] FEXCore::IR::IRListView const break; } + case IR::OP_GETROUNDINGMODE: { + auto Dst = GetDst(Node); + sub(rsp, 4); + // Only stores to memory + stmxcsr(dword [rsp]); + mov(Dst, dword [rsp]); + add(rsp, 4); + shr(Dst, 13); + break; + } + case IR::OP_SETROUNDINGMODE: { + auto Op = IROp->C(); + auto Src = GetSrc(Op->Header.Args[0].ID()); + + // Load old mxcsr + // Only stores to memory + sub(rsp, 4); + stmxcsr(dword [rsp]); + mov(TMP1.cvt32(), dword [rsp]); + + // Insert the new rounding mode + and(TMP1.cvt32(), ~(0b111 << 13)); + mov(TMP2.cvt32(), Src); + shl(TMP2.cvt32(), 13); + or(TMP1.cvt32(), TMP2.cvt32()); + + // Store it to mxcsr + // Only loads from memory + mov(dword [rsp], TMP1.cvt32()); + ldmxcsr(dword [rsp]); + add(rsp, 4); + break; + } case IR::OP_DUMMY: case IR::OP_BEGINBLOCK: case IR::OP_ENDBLOCK: