From b1df63f425682ec74e62854f306fe92bbe504088 Mon Sep 17 00:00:00 2001 From: Lioncache Date: Mon, 17 Apr 2023 22:04:46 -0400 Subject: [PATCH 1/4] Interpreter: Move fallbacks into new directory Will be used to store fallbacks and separate the definition struct from the F80 fallbacks --- External/FEXCore/Source/CMakeLists.txt | 2 +- .../Core/Interpreter/{ => Fallbacks}/InterpreterFallbacks.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) rename External/FEXCore/Source/Interface/Core/Interpreter/{ => Fallbacks}/InterpreterFallbacks.cpp (99%) diff --git a/External/FEXCore/Source/CMakeLists.txt b/External/FEXCore/Source/CMakeLists.txt index d534350c6..558967ca2 100644 --- a/External/FEXCore/Source/CMakeLists.txt +++ b/External/FEXCore/Source/CMakeLists.txt @@ -107,7 +107,7 @@ set (SRCS Interface/Core/Dispatcher/Dispatcher.cpp Interface/Core/Dispatcher/X86Dispatcher.cpp Interface/Core/Dispatcher/Arm64Dispatcher.cpp - Interface/Core/Interpreter/InterpreterFallbacks.cpp + Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp Interface/Core/X86Tables/BaseTables.cpp Interface/Core/X86Tables/DDDTables.cpp Interface/Core/X86Tables/EVEXTables.cpp diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterFallbacks.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp similarity index 99% rename from External/FEXCore/Source/Interface/Core/Interpreter/InterpreterFallbacks.cpp rename to External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index a43230911..31a39da54 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterFallbacks.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -1,4 +1,5 @@ -#include "FEXCore/Core/CoreState.h" +#include + #include "Interface/Core/Interpreter/InterpreterOps.h" #include "Interface/Core/Interpreter/F80Ops.h" From 8481aeccb58e51eb45137ab4b1018c5144b4dcbf Mon Sep 17 00:00:00 2001 From: Lioncache Date: Mon, 17 Apr 2023 22:08:39 -0400 Subject: [PATCH 2/4] Interpreter: Move F80Ops.h into Fallback directory We can also rename it to F80Fallbacks.h to make the file purpose a little more explicit. --- External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp | 2 +- .../Core/Interpreter/{F80Ops.h => Fallbacks/F80Fallbacks.h} | 0 .../Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp | 2 +- .../Source/Interface/Core/Interpreter/InterpreterOps.cpp | 1 - 4 files changed, 2 insertions(+), 3 deletions(-) rename External/FEXCore/Source/Interface/Core/Interpreter/{F80Ops.h => Fallbacks/F80Fallbacks.h} (100%) diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp index d0d9247b0..e2ed5e786 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp @@ -8,7 +8,7 @@ $end_info$ #include "Interface/Core/Interpreter/InterpreterOps.h" #include "Interface/Core/Interpreter/InterpreterDefines.h" -#include "F80Ops.h" +#include "Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h" #include diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.h b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h similarity index 100% rename from External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.h rename to External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index 31a39da54..5f3440270 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -1,7 +1,7 @@ #include #include "Interface/Core/Interpreter/InterpreterOps.h" -#include "Interface/Core/Interpreter/F80Ops.h" +#include "Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h" #include #include diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp index 4030cadec..61a3f4cc8 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp @@ -2,7 +2,6 @@ #include "Interface/Core/CPUID.h" #include "InterpreterDefines.h" #include "InterpreterOps.h" -#include "F80Ops.h" #ifdef _M_ARM_64 #include "Interface/Core/ArchHelpers/Arm64.h" From 98fbc4a46d4b1327ad3f8faefd22ec1fb5dc6313 Mon Sep 17 00:00:00 2001 From: Lioncache Date: Mon, 17 Apr 2023 22:37:30 -0400 Subject: [PATCH 3/4] Interpreter: Move OpHandler struct into its own header We can also provide a general rundown for hooking up interpreter fallbacks here for the uninitiated. --- .../Core/Interpreter/Fallbacks/F80Fallbacks.h | 7 +- .../Interpreter/Fallbacks/FallbackOpHandler.h | 75 +++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index 8971575d8..575599d27 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -4,12 +4,9 @@ #include +#include "Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h" + namespace FEXCore::CPU { -template -struct OpHandlers { - -}; - template<> struct OpHandlers { static X80SoftFloat handle4(float src) { diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h new file mode 100644 index 000000000..a35b732e8 --- /dev/null +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h @@ -0,0 +1,75 @@ +#pragma once + +#include + +namespace FEXCore::IR { +enum IROps : uint8_t; +} + +namespace FEXCore::CPU { + +// Base template for fallback handling. +// +// Registering and hooking up fallback is currently like so: +// +// 1. Go to InterpreterFallbacks.cpp and create a template specialization of +// the GetFallbackInfo member function. +// +// This member function should reasonably define what the fallback you're +// going to create will take as parameters and return as a result. For example: +// +// template<> +// FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(double), Core::FallbackHandlerIndex Index) { +// return {FABI_F80_F64, (void*)fn, Index}; +// } +// +// Defines info about a fallback that takes a double as an argument and +// returns a X80SoftFloat instance. +// +// You will also want to define a new FallbackHandlerIndex enum member and use it +// to set up the new info handler into the Info array in FillFallbackIndexPointers. +// +// 1.1. (potentially optional). Define a new ABI element in the FallbackAPI enum. +// This ABI enum value will be used to tell the JITs how to handle the fallback +// properly. These enum values specify the return type followed by its argument types. +// +// So, FABI_I64_F80_F80, for example indicates that the function will behave like a +// function as if were defined as: +// +// uint64_t fn(X80SoftFloat, X80SoftFloat) +// +// 1.2. (potentially optional). If you needed to define a new enum ABI type like in 1.1, then +// you need to add the handling for it in the JITs, which can be found in the respective +// JIT's JIT.cpp file in a function called Op_Unhandled +// +// You need to add a new case to the ABI switch statement using the new ABI type +// and do the necessary moving of data from register-allocated JIT parameters +// into that platform's registers that respects the calling convention. After this is +// done, most of the necessary background boilerplate is finished. +// +// 2. Now, make a specialization of this class with a member function named 'handle()' +// that takes the same parameters as the ones described in the fallback info function +// specialization. +// +// For example, if you have the fallback info from the example in step 1, it would be: +// +// template <> +// struct OpHandlers { +// static X80SoftFloat handle(double src) { +// return ...; +// } +// }; +// +// 3. Fill out the behavior of the OpHandler specialization to perform what you would like +// the fallback to do. +// +// 4. Add an implementation of the IR op to the Interpreter that passes through to the +// OpHandler implementation. +// +// 5. Done. +// +template +struct OpHandlers { +}; + +} // namespace FEXCore::CPU From 8e391e7a61f317e6ce25d418eed01e963512f39f Mon Sep 17 00:00:00 2001 From: Lioncache Date: Mon, 17 Apr 2023 22:41:25 -0400 Subject: [PATCH 4/4] Interpreter: Move PCMPESTRX fallback to VectorFallbacks Now that OpHandlers isn't coupled to the F80 ops anymore, we can move this over to its own file dedicated to vector fallbacks. --- .../Interface/Core/Interpreter/F80Ops.cpp | 18 - .../Core/Interpreter/Fallbacks/F80Fallbacks.h | 346 +---------------- .../Fallbacks/InterpreterFallbacks.cpp | 1 + .../Interpreter/Fallbacks/VectorFallbacks.h | 359 ++++++++++++++++++ .../Interface/Core/Interpreter/VectorOps.cpp | 20 + 5 files changed, 381 insertions(+), 363 deletions(-) create mode 100644 External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp index e2ed5e786..0569d465a 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/F80Ops.cpp @@ -417,24 +417,6 @@ DEF_OP(F64SCALE) { memcpy(GDP, &Tmp, sizeof(double)); } -DEF_OP(VPCMPESTRX) { - const auto Op = IROp->C(); - const auto Is64Bit = Op->GPRSize == 8; - - const auto RAX = *GetSrc(Data->SSAData, Op->RAX); - const auto RDX = *GetSrc(Data->SSAData, Op->RDX);; - const auto LHS = *GetSrc<__uint128_t*>(Data->SSAData, Op->LHS); - const auto RHS = *GetSrc<__uint128_t*>(Data->SSAData, Op->RHS); - - // We can be cheeky and encode the size at bit 8 to save a parameter - const auto Control = Op->Control | (uint16_t(Is64Bit) << 8); - - const auto Result = OpHandlers::handle(RAX, RDX, LHS, RHS, Control); - - memset(GDP, 0, sizeof(uint64_t)); - memcpy(GDP, &Result, sizeof(Result)); -} - #undef DEF_OP } // namespace FEXCore::CPU diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h index 575599d27..efea1a24c 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h @@ -392,348 +392,4 @@ struct OpHandlers { } }; -template<> -struct OpHandlers { - enum class AggregationOp { - EqualAny = 0b00, - Ranges = 0b01, - EqualEach = 0b10, - EqualOrdered = 0b11, - }; - - enum class SourceData { - U8, - U16, - S8, - S16, - }; - - enum class Polarity { - Positive, - Negative, - PositiveMasked, - NegativeMasked, - }; - - static uint32_t handle(uint64_t RAX, uint64_t RDX, __uint128_t lhs, __uint128_t rhs, uint16_t control) { - // Subtract by 1 in order to make validity limits 0-based - const auto valid_lhs = GetExplicitLength(RAX, control) - 1; - const auto valid_rhs = GetExplicitLength(RDX, control) - 1; - - return MainBody(lhs, valid_lhs, rhs, valid_rhs, control); - } - - // Main PCMPXSTRX algorithm body. Allows for reuse with both implicit and explicit length variants. - static uint32_t MainBody(const __uint128_t& lhs, int valid_lhs, const __uint128_t& rhs, int valid_rhs, uint16_t control) { - const uint32_t aggregation = PerformAggregation(lhs, valid_lhs, rhs, valid_rhs, control); - const uint32_t upper_limit = (16U >> (control & 1)) - 1; - - // Bits are arranged as: - // Bit #: 3 2 1 0 - // [OF | CF | SF | ZF] - uint32_t flags = 0; - flags |= (valid_rhs < upper_limit) ? 0b01 : 0b00; - flags |= (valid_lhs < upper_limit) ? 0b10 : 0b00; - - const uint32_t result = HandlePolarity(aggregation, control, upper_limit, valid_rhs); - if (result != 0) { - flags |= 0b0100; - } - if ((result & 1) != 0) { - flags |= 0b1000; - } - - // We tack the flags on top of the result to avoid needing to handle - // multiple return values in the JITs. - return result | (flags << 16); - } - - static int32_t GetExplicitLength(uint64_t reg, uint16_t control) { - // Bit 8 controls whether or not the reg value is 64-bit or 32-bit. - int64_t value = 0; - if (((control >> 8) & 1) != 0) { - value = static_cast(reg); - } else { - // We need a sign extend in this case. - value = static_cast(reg); - } - - // If control[0] is set, then we're dealing with words instead of bytes - const int64_t limit = (control & 1) != 0 ? 8 : 16; - - // Length needs to saturate to 16 (if bytes) or 8 (if words) - // when the length value is greater than 16 (if bytes)/8 (if words) - // or if the length value is less than -16 (if bytes)/-8 (if words). - if (value < -limit || value > limit) { - return limit; - } - - return std::abs(static_cast(value)); - } - - static int32_t GetElement(const __uint128_t& vec, int32_t index, uint16_t control) { - const auto* vec_ptr = reinterpret_cast(&vec); - - // Control bits [1:0] define the data type being dealt with. - switch (static_cast(control & 0b11)) { - case SourceData::U8: - return static_cast(vec_ptr[index]); - case SourceData::U16: { - uint16_t value{}; - std::memcpy(&value, vec_ptr + (sizeof(uint16_t) * static_cast(index)), sizeof(value)); - return value; - } - case SourceData::S8: - return static_cast(vec_ptr[index]); - case SourceData::S16: - default: { - int16_t value{}; - std::memcpy(&value, vec_ptr + (sizeof(int16_t) * static_cast(index)), sizeof(value)); - return value; - } - } - } - - static uint32_t PerformAggregation(const __uint128_t& lhs, int32_t valid_lhs, - const __uint128_t& rhs, int32_t valid_rhs, - uint16_t control) { - switch (static_cast((control >> 2) & 0b11)) { - case AggregationOp::EqualAny: - return HandleEqualAny(lhs, valid_lhs, rhs, valid_rhs, control); - case AggregationOp::Ranges: - return HandleRanges(lhs, valid_lhs, rhs, valid_rhs, control); - case AggregationOp::EqualEach: - return HandleEqualEach(lhs, valid_lhs, rhs, valid_rhs, control); - case AggregationOp::EqualOrdered: - default: - return HandleEqualOrdered(lhs, valid_lhs, rhs, valid_rhs, control); - } - } - - static uint32_t HandlePolarity(uint32_t value, uint16_t control, int upper_limit, int valid_rhs) { - switch (static_cast((control >> 4) & 0b11)) { - case Polarity::Negative: - return value ^ ((2U << upper_limit) - 1); - case Polarity::NegativeMasked: - return value ^ ((1U << (valid_rhs + 1)) - 1); - case Polarity::Positive: - case Polarity::PositiveMasked: - default: - // Both positive masking and positive polarity are documented - // as both being equivalent to "IntRes2 = IntRes1", where IntRes1 - // is our 'value' parameter, so we don't need to do anything in - // these cases except return the same value. - return value; - } - } - - // Finds characters from an overall character set. - // - // Scans through RHS trying to find any characters contained in LHS. - // Think of this as a sort of vectorized version of strspn (kind of). - // - // e.g. Assume operating on two character vectors as unsigned words - // - // 0 1 2 3 4 5 6 7 - // LHS -> [a, b, c, d, e, f, g, n] - // RHS -> [z, k, v, c, d, o, p, n] - // - // With both explicit lengths for each string being 8 (the max length for words), - // this would result in an intermediate result like: - // - // 0b1001'1000 - // │ │ │ - // 'n' match ───┘ │ │ - // │ │ - // 'd' match ──────┘ │ - // │ - // 'c' match ────────┘ - // - static uint32_t HandleEqualAny(const __uint128_t& lhs, int32_t valid_lhs, - const __uint128_t& rhs, int32_t valid_rhs, - uint16_t control) { - uint32_t result = 0; - - for (int j = valid_rhs; j >= 0; j--) { - result <<= 1; - - const int rhs_value = GetElement(rhs, j, control); - for (int i = valid_lhs; i >= 0; i--) { - const int lhs_value = GetElement(lhs, i, control); - result |= static_cast(rhs_value == lhs_value); - } - } - - return result; - } - - // Determines if a character falls within a limited range - // - // Scans through rhs using a range denoted by two elements - // in lhs and determines if the respective character in rhs - // falls within its range. - // - // i.e. - // lhs_upper_bound >= rhs_value && lhs_lower_bound <= rhs_value - // - // e.g. Assume operating on two character vectors as unsigned words - // - // 0 1 2 3 4 5 6 7 - // LHS -> [a, z, A, Z, 0, 0, 0, 0] - // RHS -> [z, k, ., C, M, ;, \, '] - // - // With LHS's length being 4 and RHS's lenth being 8, - // this would result in an intermediate result like: - // - // 0b0001'1011 - // │ │ ││ - // 'z' >= 'M' && 'a' <= 'M' ─────┘ │ ││ - // │ ││ - // 'z' >= 'C' && 'a' <= 'C' ───────┘ ││ - // ││ - // 'Z' >= 'k' && 'A' <= 'k' ─────────┘│ - // │ - // 'Z' >= 'z' && 'A' <= 'z' ──────────┘ - // - static uint32_t HandleRanges(const __uint128_t& lhs, int32_t valid_lhs, - const __uint128_t& rhs, int32_t valid_rhs, - uint16_t control) { - uint32_t result = 0; - - for (int j = valid_rhs; j >= 0; j--) { - result <<= 1; - - const int element = GetElement(rhs, j, control); - for (int i = (valid_lhs - 1) | 1; i >= 0; i -= 2) { - const int upper_bound = GetElement(lhs, i - 0, control); - const int lower_bound = GetElement(lhs, i - 1, control); - - const bool ge = upper_bound >= element; - const bool le = lower_bound <= element; - - result |= static_cast(ge && le); - } - } - - return result; - } - - // Determines if each character is equal to one another (string compare) - // - // Essentially the PCMPXSTRX variant of memcmp/strcmp. Sets the bit of the - // resulting mask if both elements are equal to one another. Otherwise - // sets it to false. - // - // e.g. Assume operating on two character vectors as unsigned words - // - // 0 1 2 3 4 5 6 7 - // LHS -> [a, b, c, d, e, f, g, n] - // RHS -> [a, b, c, d, e, f, e, x] - // - // With both explicit lengths for each string being 8 (the max length for words), - // this would result in an intermediate result like: - // - // 0b0011'1111 - // ││ ││││ - // 'f' == 'f' ────┘│ ││││ - // │ ││││ - // 'e' == 'e' ─────┘ ││││ - // ││││ - // 'd' == 'd' ───────┘│││ - // │││ - // 'c' == 'c' ────────┘││ - // ││ - // 'b' == 'b' ─────────┘│ - // │ - // 'a' == 'a' ──────────┘ - // - static uint32_t HandleEqualEach(const __uint128_t& lhs, int32_t valid_lhs, - const __uint128_t& rhs, int32_t valid_rhs, - uint16_t control) { - const auto upper_limit = (16 >> (control & 1)) - 1; - const auto max_valid = std::max(valid_lhs, valid_rhs); - const auto min_valid = std::min(valid_lhs, valid_rhs); - - // All values past the end of string must be forced to true. - // (See 4.1.6 Valid/Invalid Override of Comparisons in the Intel Software Development Manual) - // So we can calculate this part of the mask ahead of time and set all those to-be bits to true - // and then progressively shift them into place over the course of execution. - uint32_t result = (1U << (upper_limit - max_valid)) - 1; - result <<= (max_valid - min_valid); - - for (int i = min_valid; i >= 0; i--) { - const int lhs_element = GetElement(lhs, i, control); - const int rhs_element = GetElement(rhs, i, control); - - result <<= 1; - result |= static_cast(lhs_element == rhs_element); - } - - return result; - } - - // Determines if a substring exists within an overall string - // - // Somewhat equivalent to the behavior of strstr. - // - // Sets the corresponding index in the result where a substring is found. - // - // e.g. Assume operating on two character vectors as unsigned words - // - // 0 1 2 3 4 5 6 7 - // LHS -> [b, a, x, z, y, v, o, m] - // RHS -> [b, a, d, b, a, n, k, s] - // - // With the length of LHS being 2 and the length of RHS being 8, we have a composition like: - // - // Substring to look for - // ┌──┴──┐ - // LHS -> [b, a, x, z, y, v, o, m] - // RHS -> [b, a, d, b, a, n, k, s] - // └───────────┬────────────┘ - // Entire string to search - // - // And we end up with a result like: - // - // 0b0000'1001 - // │ │ - // At index 3 ───────┘ │ - // │ - // At index 0 ──────────┘ - // - static uint32_t HandleEqualOrdered(const __uint128_t& lhs, int32_t valid_lhs, - const __uint128_t& rhs, int32_t valid_rhs, - uint16_t control) { - const auto upper_limit = (16 >> (control & 1)) - 1; - - // Edge case! - // If we have *no* valid characters in our inner string, then - // we need to return the intermediate result as - // 0xFF (if operating on words) or 0xFFFF (if operating on bytes) - if (valid_lhs == -1) { - return (2U << upper_limit) - 1; - } - - uint32_t result = 0; - const int initial = valid_rhs == upper_limit ? valid_rhs - : valid_rhs - valid_lhs; - for (int j = initial; j >= 0; j--) { - result <<= 1; - - uint32_t value = 1; - const int start = std::min(valid_rhs - j, valid_lhs); - for (int i = start; i >= 0; i--) { - const int lhs_value = GetElement(lhs, i + 0, control); - const int rhs_value = GetElement(rhs, i + j, control); - - value &= static_cast(lhs_value == rhs_value); - } - - result |= value; - } - - return result; - } -}; - -} +} // namespace FEXCore::CPU diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp index 5f3440270..2eace11dc 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp @@ -2,6 +2,7 @@ #include "Interface/Core/Interpreter/InterpreterOps.h" #include "Interface/Core/Interpreter/Fallbacks/F80Fallbacks.h" +#include "Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h" #include #include diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h new file mode 100644 index 000000000..6e2a976f2 --- /dev/null +++ b/External/FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h @@ -0,0 +1,359 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include "Interface/Core/Interpreter/Fallbacks/FallbackOpHandler.h" + +namespace FEXCore::CPU { + +template<> +struct OpHandlers { + enum class AggregationOp { + EqualAny = 0b00, + Ranges = 0b01, + EqualEach = 0b10, + EqualOrdered = 0b11, + }; + + enum class SourceData { + U8, + U16, + S8, + S16, + }; + + enum class Polarity { + Positive, + Negative, + PositiveMasked, + NegativeMasked, + }; + + static uint32_t handle(uint64_t RAX, uint64_t RDX, __uint128_t lhs, __uint128_t rhs, uint16_t control) { + // Subtract by 1 in order to make validity limits 0-based + const auto valid_lhs = GetExplicitLength(RAX, control) - 1; + const auto valid_rhs = GetExplicitLength(RDX, control) - 1; + + return MainBody(lhs, valid_lhs, rhs, valid_rhs, control); + } + + // Main PCMPXSTRX algorithm body. Allows for reuse with both implicit and explicit length variants. + static uint32_t MainBody(const __uint128_t& lhs, int valid_lhs, const __uint128_t& rhs, int valid_rhs, uint16_t control) { + const uint32_t aggregation = PerformAggregation(lhs, valid_lhs, rhs, valid_rhs, control); + const uint32_t upper_limit = (16U >> (control & 1)) - 1; + + // Bits are arranged as: + // Bit #: 3 2 1 0 + // [OF | CF | SF | ZF] + uint32_t flags = 0; + flags |= (valid_rhs < upper_limit) ? 0b01 : 0b00; + flags |= (valid_lhs < upper_limit) ? 0b10 : 0b00; + + const uint32_t result = HandlePolarity(aggregation, control, upper_limit, valid_rhs); + if (result != 0) { + flags |= 0b0100; + } + if ((result & 1) != 0) { + flags |= 0b1000; + } + + // We tack the flags on top of the result to avoid needing to handle + // multiple return values in the JITs. + return result | (flags << 16); + } + + static int32_t GetExplicitLength(uint64_t reg, uint16_t control) { + // Bit 8 controls whether or not the reg value is 64-bit or 32-bit. + int64_t value = 0; + if (((control >> 8) & 1) != 0) { + value = static_cast(reg); + } else { + // We need a sign extend in this case. + value = static_cast(reg); + } + + // If control[0] is set, then we're dealing with words instead of bytes + const int64_t limit = (control & 1) != 0 ? 8 : 16; + + // Length needs to saturate to 16 (if bytes) or 8 (if words) + // when the length value is greater than 16 (if bytes)/8 (if words) + // or if the length value is less than -16 (if bytes)/-8 (if words). + if (value < -limit || value > limit) { + return limit; + } + + return std::abs(static_cast(value)); + } + + static int32_t GetElement(const __uint128_t& vec, int32_t index, uint16_t control) { + const auto* vec_ptr = reinterpret_cast(&vec); + + // Control bits [1:0] define the data type being dealt with. + switch (static_cast(control & 0b11)) { + case SourceData::U8: + return static_cast(vec_ptr[index]); + case SourceData::U16: { + uint16_t value{}; + std::memcpy(&value, vec_ptr + (sizeof(uint16_t) * static_cast(index)), sizeof(value)); + return value; + } + case SourceData::S8: + return static_cast(vec_ptr[index]); + case SourceData::S16: + default: { + int16_t value{}; + std::memcpy(&value, vec_ptr + (sizeof(int16_t) * static_cast(index)), sizeof(value)); + return value; + } + } + } + + static uint32_t PerformAggregation(const __uint128_t& lhs, int32_t valid_lhs, + const __uint128_t& rhs, int32_t valid_rhs, + uint16_t control) { + switch (static_cast((control >> 2) & 0b11)) { + case AggregationOp::EqualAny: + return HandleEqualAny(lhs, valid_lhs, rhs, valid_rhs, control); + case AggregationOp::Ranges: + return HandleRanges(lhs, valid_lhs, rhs, valid_rhs, control); + case AggregationOp::EqualEach: + return HandleEqualEach(lhs, valid_lhs, rhs, valid_rhs, control); + case AggregationOp::EqualOrdered: + default: + return HandleEqualOrdered(lhs, valid_lhs, rhs, valid_rhs, control); + } + } + + static uint32_t HandlePolarity(uint32_t value, uint16_t control, int upper_limit, int valid_rhs) { + switch (static_cast((control >> 4) & 0b11)) { + case Polarity::Negative: + return value ^ ((2U << upper_limit) - 1); + case Polarity::NegativeMasked: + return value ^ ((1U << (valid_rhs + 1)) - 1); + case Polarity::Positive: + case Polarity::PositiveMasked: + default: + // Both positive masking and positive polarity are documented + // as both being equivalent to "IntRes2 = IntRes1", where IntRes1 + // is our 'value' parameter, so we don't need to do anything in + // these cases except return the same value. + return value; + } + } + + // Finds characters from an overall character set. + // + // Scans through RHS trying to find any characters contained in LHS. + // Think of this as a sort of vectorized version of strspn (kind of). + // + // e.g. Assume operating on two character vectors as unsigned words + // + // 0 1 2 3 4 5 6 7 + // LHS -> [a, b, c, d, e, f, g, n] + // RHS -> [z, k, v, c, d, o, p, n] + // + // With both explicit lengths for each string being 8 (the max length for words), + // this would result in an intermediate result like: + // + // 0b1001'1000 + // │ │ │ + // 'n' match ───┘ │ │ + // │ │ + // 'd' match ──────┘ │ + // │ + // 'c' match ────────┘ + // + static uint32_t HandleEqualAny(const __uint128_t& lhs, int32_t valid_lhs, + const __uint128_t& rhs, int32_t valid_rhs, + uint16_t control) { + uint32_t result = 0; + + for (int j = valid_rhs; j >= 0; j--) { + result <<= 1; + + const int rhs_value = GetElement(rhs, j, control); + for (int i = valid_lhs; i >= 0; i--) { + const int lhs_value = GetElement(lhs, i, control); + result |= static_cast(rhs_value == lhs_value); + } + } + + return result; + } + + // Determines if a character falls within a limited range + // + // Scans through rhs using a range denoted by two elements + // in lhs and determines if the respective character in rhs + // falls within its range. + // + // i.e. + // lhs_upper_bound >= rhs_value && lhs_lower_bound <= rhs_value + // + // e.g. Assume operating on two character vectors as unsigned words + // + // 0 1 2 3 4 5 6 7 + // LHS -> [a, z, A, Z, 0, 0, 0, 0] + // RHS -> [z, k, ., C, M, ;, \, '] + // + // With LHS's length being 4 and RHS's lenth being 8, + // this would result in an intermediate result like: + // + // 0b0001'1011 + // │ │ ││ + // 'z' >= 'M' && 'a' <= 'M' ─────┘ │ ││ + // │ ││ + // 'z' >= 'C' && 'a' <= 'C' ───────┘ ││ + // ││ + // 'Z' >= 'k' && 'A' <= 'k' ─────────┘│ + // │ + // 'Z' >= 'z' && 'A' <= 'z' ──────────┘ + // + static uint32_t HandleRanges(const __uint128_t& lhs, int32_t valid_lhs, + const __uint128_t& rhs, int32_t valid_rhs, + uint16_t control) { + uint32_t result = 0; + + for (int j = valid_rhs; j >= 0; j--) { + result <<= 1; + + const int element = GetElement(rhs, j, control); + for (int i = (valid_lhs - 1) | 1; i >= 0; i -= 2) { + const int upper_bound = GetElement(lhs, i - 0, control); + const int lower_bound = GetElement(lhs, i - 1, control); + + const bool ge = upper_bound >= element; + const bool le = lower_bound <= element; + + result |= static_cast(ge && le); + } + } + + return result; + } + + // Determines if each character is equal to one another (string compare) + // + // Essentially the PCMPXSTRX variant of memcmp/strcmp. Sets the bit of the + // resulting mask if both elements are equal to one another. Otherwise + // sets it to false. + // + // e.g. Assume operating on two character vectors as unsigned words + // + // 0 1 2 3 4 5 6 7 + // LHS -> [a, b, c, d, e, f, g, n] + // RHS -> [a, b, c, d, e, f, e, x] + // + // With both explicit lengths for each string being 8 (the max length for words), + // this would result in an intermediate result like: + // + // 0b0011'1111 + // ││ ││││ + // 'f' == 'f' ────┘│ ││││ + // │ ││││ + // 'e' == 'e' ─────┘ ││││ + // ││││ + // 'd' == 'd' ───────┘│││ + // │││ + // 'c' == 'c' ────────┘││ + // ││ + // 'b' == 'b' ─────────┘│ + // │ + // 'a' == 'a' ──────────┘ + // + static uint32_t HandleEqualEach(const __uint128_t& lhs, int32_t valid_lhs, + const __uint128_t& rhs, int32_t valid_rhs, + uint16_t control) { + const auto upper_limit = (16 >> (control & 1)) - 1; + const auto max_valid = std::max(valid_lhs, valid_rhs); + const auto min_valid = std::min(valid_lhs, valid_rhs); + + // All values past the end of string must be forced to true. + // (See 4.1.6 Valid/Invalid Override of Comparisons in the Intel Software Development Manual) + // So we can calculate this part of the mask ahead of time and set all those to-be bits to true + // and then progressively shift them into place over the course of execution. + uint32_t result = (1U << (upper_limit - max_valid)) - 1; + result <<= (max_valid - min_valid); + + for (int i = min_valid; i >= 0; i--) { + const int lhs_element = GetElement(lhs, i, control); + const int rhs_element = GetElement(rhs, i, control); + + result <<= 1; + result |= static_cast(lhs_element == rhs_element); + } + + return result; + } + + // Determines if a substring exists within an overall string + // + // Somewhat equivalent to the behavior of strstr. + // + // Sets the corresponding index in the result where a substring is found. + // + // e.g. Assume operating on two character vectors as unsigned words + // + // 0 1 2 3 4 5 6 7 + // LHS -> [b, a, x, z, y, v, o, m] + // RHS -> [b, a, d, b, a, n, k, s] + // + // With the length of LHS being 2 and the length of RHS being 8, we have a composition like: + // + // Substring to look for + // ┌──┴──┐ + // LHS -> [b, a, x, z, y, v, o, m] + // RHS -> [b, a, d, b, a, n, k, s] + // └───────────┬────────────┘ + // Entire string to search + // + // And we end up with a result like: + // + // 0b0000'1001 + // │ │ + // At index 3 ───────┘ │ + // │ + // At index 0 ──────────┘ + // + static uint32_t HandleEqualOrdered(const __uint128_t& lhs, int32_t valid_lhs, + const __uint128_t& rhs, int32_t valid_rhs, + uint16_t control) { + const auto upper_limit = (16 >> (control & 1)) - 1; + + // Edge case! + // If we have *no* valid characters in our inner string, then + // we need to return the intermediate result as + // 0xFF (if operating on words) or 0xFFFF (if operating on bytes) + if (valid_lhs == -1) { + return (2U << upper_limit) - 1; + } + + uint32_t result = 0; + const int initial = valid_rhs == upper_limit ? valid_rhs + : valid_rhs - valid_lhs; + for (int j = initial; j >= 0; j--) { + result <<= 1; + + uint32_t value = 1; + const int start = std::min(valid_rhs - j, valid_lhs); + for (int i = start; i >= 0; i--) { + const int lhs_value = GetElement(lhs, i + 0, control); + const int rhs_value = GetElement(rhs, i + j, control); + + value &= static_cast(lhs_value == rhs_value); + } + + result |= value; + } + + return result; + } +}; + +} // namespace FEXCore::CPU diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp index 3c8728b1f..66ce688d6 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp @@ -8,6 +8,8 @@ $end_info$ #include "Interface/Core/Interpreter/InterpreterOps.h" #include "Interface/Core/Interpreter/InterpreterDefines.h" +#include "Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h" + #include #include @@ -2276,6 +2278,24 @@ DEF_OP(VRev64) { memcpy(GDP, Tmp, OpSize); } +DEF_OP(VPCMPESTRX) { + const auto Op = IROp->C(); + const auto Is64Bit = Op->GPRSize == 8; + + const auto RAX = *GetSrc(Data->SSAData, Op->RAX); + const auto RDX = *GetSrc(Data->SSAData, Op->RDX);; + const auto LHS = *GetSrc<__uint128_t*>(Data->SSAData, Op->LHS); + const auto RHS = *GetSrc<__uint128_t*>(Data->SSAData, Op->RHS); + + // We can be cheeky and encode the size at bit 8 to save a parameter + const auto Control = Op->Control | (uint16_t(Is64Bit) << 8); + + const auto Result = OpHandlers::handle(RAX, RDX, LHS, RHS, Control); + + memset(GDP, 0, sizeof(uint64_t)); + memcpy(GDP, &Result, sizeof(Result)); +} + #undef DEF_OP } // namespace FEXCore::CPU