Merge pull request #2998 from Sonicadvance1/optimize_addsub_fcma

OpcodeDispatcher: Optimize addsubp{s,d} using fcadd

FEXCore/Source/Interface/Config/Config.json.in

@@ -74,7 +74,9 @@
           "ENABLECLZERO": "enableclzero",
           "DISABLECLZERO": "disableclzero",
           "ENABLEATOMICS": "enableatomics",
-          "DISABLEATOMICS": "disableatomics"
+          "DISABLEATOMICS": "disableatomics",
+          "ENABLEFCMA": "enablefcma",
+          "DISABLEFCMA": "disablefcma"
         },
         "Desc": [
           "Allows controlling of the CPU features in the JIT.",
@@ -88,7 +90,8 @@
           "\t{enable,disable}pmull128: Will force enable or disable pmull128 even if the host doesn't support it",
           "\t{enable,disable}rng: Will force enable or disable rng even if the host doesn't support it",
           "\t{enable,disable}clzero: Will force enable or disable clzero even if the host doesn't support it",
-          "\t{enable,disable}atomics: Will force enable or disable ARMv8.1 LSE atomics even if the host doesn't support it"
+          "\t{enable,disable}atomics: Will force enable or disable ARMv8.1 LSE atomics even if the host doesn't support it",
+          "\t{enable,disable}fcma: Will force enable or disable fcma even if the host doesn't support it"
         ]
       }
     },

FEXCore/Source/Interface/Core/HostFeatures.cpp

@@ -96,6 +96,10 @@ static void OverrideFeatures(HostFeatures *Features) {
   const bool EnableAtomics = HostFeatures() & FEXCore::Config::HostFeatures::ENABLEATOMICS;
   LogMan::Throw::AFmt(!(DisableAtomics && EnableAtomics), "Disabling and Enabling CPU features are mutually exclusive");
 
+  const bool DisableFCMA = HostFeatures() & FEXCore::Config::HostFeatures::DISABLEFCMA;
+  const bool EnableFCMA = HostFeatures() & FEXCore::Config::HostFeatures::ENABLEFCMA;
+  LogMan::Throw::AFmt(!(DisableFCMA && EnableFCMA), "Disabling and Enabling CPU features are mutually exclusive");
+
   if (EnableAVX) {
     Features->SupportsAVX = true;
   }
@@ -156,6 +160,12 @@ static void OverrideFeatures(HostFeatures *Features) {
   else if (DisableAtomics) {
     Features->SupportsAtomics = false;
   }
+  if (EnableFCMA) {
+    Features->SupportsFCMA = true;
+  }
+  else if (DisableFCMA) {
+    Features->SupportsFCMA = false;
+  }
 }
 
 HostFeatures::HostFeatures() {
@@ -181,6 +191,7 @@ HostFeatures::HostFeatures() {
   SupportsTSOImm9 = Features.Has(vixl::CPUFeatures::Feature::kRCpcImm);
   SupportsPMULL_128Bit = Features.Has(vixl::CPUFeatures::Feature::kPmull1Q);
   SupportsCSSC = Features.Has(vixl::CPUFeatures::Feature::kCSSC);
+  SupportsFCMA = Features.Has(vixl::CPUFeatures::Feature::kFcma);
 
   Supports3DNow = true;
   SupportsSSE4A = true;

FEXCore/Source/Interface/Core/Interpreter/InterpreterDefines.h

@@ -65,6 +65,17 @@
   }                                                         \
   break;                                                    \
   }
+#define DO_VECTOR_FCADD_PAIR_OP(size, type, func)           \
+  case size: {                                              \
+  auto *Dst_d  = reinterpret_cast<type*>(std::data(Tmp));   \
+  auto *Src1_d = reinterpret_cast<const type*>(Src1);             \
+  auto *Src2_d = reinterpret_cast<const type*>(Src2);             \
+  for (uint8_t i = 0; i < Elements; i += 2) {               \
+    func(&Dst_d[i], &Src1_d[i], &Src2_d[i]);                \
+  }                                                         \
+  break;                                                    \
+  }
+
 #define DO_VECTOR_SCALAR_OP(size, type, func)              \
   case size: {                                             \
   auto *Dst_d  = reinterpret_cast<type*>(std::data(Tmp));  \

FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -286,6 +286,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(VREV64,                 VRev64);
   REGISTER_OP(VPCMPESTRX,             VPCMPESTRX);
   REGISTER_OP(VPCMPISTRX,             VPCMPISTRX);
+  REGISTER_OP(VFCADD,                 VFCADD);
 
   // Encryption ops
   REGISTER_OP(VAESIMC,                AESImc);

FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -314,6 +314,7 @@ namespace FEXCore::CPU {
   DEF_OP(VRev64);
   DEF_OP(VPCMPESTRX);
   DEF_OP(VPCMPISTRX);
+  DEF_OP(VFCADD);
 
   ///< Encryption ops
   DEF_OP(AESImc);

FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp

@@ -2561,6 +2561,44 @@ DEF_OP(VPCMPISTRX) {
   memcpy(GDP, &Result, sizeof(Result));
 }
 
+DEF_OP(VFCADD) {
+  const auto Op = IROp->C<IR::IROp_VFCADD>();
+  const uint8_t OpSize = IROp->Size;
+
+  const auto *Src1 = GetSrc<uint8_t*>(Data->SSAData, Op->Vector1);
+  const auto *Src2 = GetSrc<uint8_t*>(Data->SSAData, Op->Vector2);
+  const auto Rotate = Op->Rotate;
+  LOGMAN_THROW_A_FMT(Rotate == 90 || Rotate == 270, "Invalid rotate!");
+
+  TempVectorDataArray Tmp;
+
+  const uint8_t ElementSize = Op->Header.ElementSize;
+  const uint8_t Elements = OpSize / ElementSize;
+
+  const auto Func = [Rotate](auto dst, auto src1, auto src2) {
+    auto Element1 = src2[1];
+    auto Element3 = src2[0];
+    if (Rotate == 90) {
+      Element1 = -Element1;
+    }
+    else {
+      Element3 = -Element3;
+    }
+    dst[0] = src1[0] + Element1;
+    dst[1] = src1[1] + Element3;
+  };
+
+  switch (ElementSize) {
+    //DO_VECTOR_FCADD_PAIR_OP(2, float16_t, Func)
+    DO_VECTOR_FCADD_PAIR_OP(4, float, Func)
+    DO_VECTOR_FCADD_PAIR_OP(8, double, Func)
+    default:
+      LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
+      break;
+  }
+  memcpy(GDP, Tmp.data(), OpSize);
+}
+
 #undef DEF_OP
 
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -1104,6 +1104,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
         REGISTER_OP(VTBL1,             VTBL1);
         REGISTER_OP(VREV32,            VRev32);
         REGISTER_OP(VREV64,            VRev64);
+        REGISTER_OP(VFCADD,            VFCADD);
 #undef REGISTER_OP
 
         default:

FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h

@@ -470,6 +470,7 @@ private:
   DEF_OP(VTBL1);
   DEF_OP(VRev32);
   DEF_OP(VRev64);
+  DEF_OP(VFCADD);
 
   ///< Encryption ops
   DEF_OP(AESImc);

FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp

@@ -3825,6 +3825,50 @@ DEF_OP(VRev64) {
   }
 }
 
+DEF_OP(VFCADD) {
+  const auto Op = IROp->C<IR::IROp_VFCADD>();
+  const auto OpSize = IROp->Size;
+
+  const auto ElementSize = Op->Header.ElementSize;
+  const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
+
+  const auto Dst = GetVReg(Node);
+  const auto Vector1 = GetVReg(Op->Vector1.ID());
+  const auto Vector2 = GetVReg(Op->Vector2.ID());
+
+  LOGMAN_THROW_AA_FMT(ElementSize == 2 || ElementSize == 4 || ElementSize == 8, "Invalid size");
+  LOGMAN_THROW_A_FMT(Op->Rotate == 90 || Op->Rotate == 270, "Invalidate Rotate");
+  const auto SubRegSize =
+    ElementSize == 2 ? ARMEmitter::SubRegSize::i16Bit :
+    ElementSize == 4 ? ARMEmitter::SubRegSize::i32Bit : ARMEmitter::SubRegSize::i64Bit;
+  const auto Rotate =
+    Op->Rotate == 90 ? ARMEmitter::Rotation::ROTATE_90 : ARMEmitter::Rotation::ROTATE_270;
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    const auto Mask = PRED_TMP_32B.Merging();
+
+    if (Dst == Vector1) {
+      // Trivial case where we already have first vector in the destination
+      // register. We can just do the operation in place.
+      fcadd(SubRegSize, Dst.Z(), Mask, Vector1.Z(), Vector2.Z(), Rotate);
+    }
+    else {
+      // SVE FCADD is a destructive operation, so we need
+      // a temporary for performing operations.
+      movprfx(VTMP1.Z(), Vector1.Z());
+      fcadd(SubRegSize, VTMP1.Z(), Mask, VTMP1.Z(), Vector2.Z(), Rotate);
+      mov(Dst.Z(), VTMP1.Z());
+    }
+  } else {
+    if (OpSize == 8) {
+      fcadd(SubRegSize, Dst.D(), Vector1.D(), Vector2.D(), Rotate);
+    }
+    else {
+      fcadd(SubRegSize, Dst.Q(), Vector1.Q(), Vector2.Q(), Rotate);
+    }
+  }
+}
+
 #undef DEF_OP
 }
 

FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -2962,9 +2962,21 @@ OrderedNode* OpDispatchBuilder::ADDSUBPOpImpl(OpcodeArgs, size_t ElementSize,
                                               OrderedNode *Src1, OrderedNode *Src2) {
   const auto Size = GetSrcSize(Op);
 
-  auto ConstantEOR = LoadAndCacheNamedVectorConstant(Size, ElementSize == 4 ? NAMED_VECTOR_PADDSUBPS_INVERT : NAMED_VECTOR_PADDSUBPD_INVERT);
-  auto InvertedSource = _VXor(Size, ElementSize, Src2, ConstantEOR);
-  return _VFAdd(Size, ElementSize, Src1, InvertedSource);
+  if (CTX->HostFeatures.SupportsFCMA) {
+    if (ElementSize == 4) {
+      auto Swizzle = _VRev64(Size, 4, Src2);
+      return _VFCADD(Size, ElementSize, Src1, Swizzle, 90);
+    }
+    else {
+      auto Swizzle = _VExtr(Size, 1, Src2, Src2, 8);
+      return _VFCADD(Size, ElementSize, Src1, Swizzle, 90);
+    }
+  }
+  else {
+    auto ConstantEOR = LoadAndCacheNamedVectorConstant(Size, ElementSize == 4 ? NAMED_VECTOR_PADDSUBPS_INVERT : NAMED_VECTOR_PADDSUBPD_INVERT);
+    auto InvertedSource = _VXor(Size, ElementSize, Src2, ConstantEOR);
+    return _VFAdd(Size, ElementSize, Src1, InvertedSource);
+  }
 }
 
 template<size_t ElementSize>

FEXCore/Source/Interface/IR/IR.json

@@ -1558,6 +1558,10 @@
                  "course of creating the intermediate result"
                 ],
         "DestSize": "4"
+      },
+      "FPR = VFCADD u8:#RegisterSize, u8:#ElementSize, FPR:$Vector1, FPR:$Vector2, u16:$Rotate": {
+        "DestSize": "RegisterSize",
+        "NumElements": "RegisterSize / ElementSize"
       }
     },
     "Conv": {

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -31,6 +31,7 @@ class HostFeatures final {
     bool SupportsCLWB{};
     bool SupportsPMULL_128Bit{};
     bool SupportsCSSC{};
+    bool SupportsFCMA{};
 
     // Float exception behaviour
     bool SupportsFlushInputsToZero{};

Scripts/InstructionCountParser.py

@@ -46,12 +46,14 @@ class HostFeatures(Flag) :
     FEATURE_SVE256 = (1 << 1)
     FEATURE_CLZERO = (1 << 2)
     FEATURE_RNG    = (1 << 3)
+    FEATURE_FCMA   = (1 << 4)
 
 HostFeaturesLookup = {
     "SVE128"  : HostFeatures.FEATURE_SVE128,
     "SVE256"  : HostFeatures.FEATURE_SVE256,
     "CLZERO"  : HostFeatures.FEATURE_CLZERO,
     "RNG"     : HostFeatures.FEATURE_RNG,
+    "FCMA"    : HostFeatures.FEATURE_FCMA,
 }
 
 def GetHostFeatures(data):

Source/Tools/CodeSizeValidation/Main.cpp

@@ -384,6 +384,7 @@ int main(int argc, char **argv, char **const envp) {
     FEATURE_SVE256 = (1U << 1),
     FEATURE_CLZERO = (1U << 2),
     FEATURE_RNG    = (1U << 3),
+    FEATURE_FCMA   = (1U << 4),
   };
 
   uint64_t SVEWidth = 0;
@@ -402,6 +403,9 @@ int main(int argc, char **argv, char **const envp) {
   if (TestHeaderData->EnabledHostFeatures & FEATURE_RNG) {
     HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLERNG);
   }
+  if (TestHeaderData->EnabledHostFeatures & FEATURE_FCMA) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLEFCMA);
+  }
 
   // Always enable ARMv8.1 LSE atomics.
   HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLEATOMICS);
@@ -418,6 +422,9 @@ int main(int argc, char **argv, char **const envp) {
   if (TestHeaderData->DisabledHostFeatures & FEATURE_RNG) {
     HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::DISABLERNG);
   }
+  if (TestHeaderData->DisabledHostFeatures & FEATURE_FCMA) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::DISABLEFCMA);
+  }
   FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_HOSTFEATURES, fextl::fmt::format("{}", HostFeatureControl));
   FEXCore::Config::EraseSet(FEXCore::Config::CONFIG_FORCESVEWIDTH, fextl::fmt::format("{}", SVEWidth));
 

unittests/InstructionCountCI/Secondary_OpSize.json

@@ -4,7 +4,8 @@
     "EnabledHostFeatures": [],
     "DisabledHostFeatures": [
       "SVE128",
-      "SVE256"
+      "SVE256",
+      "FCMA"
     ]
   },
   "Instructions": {

unittests/InstructionCountCI/Secondary_OpSize_FCMA.json

@@ -0,0 +1,23 @@
+{
+  "Features": {
+    "Bitness": 64,
+    "EnabledHostFeatures": [
+      "FCMA"
+    ],
+    "DisabledHostFeatures": [
+      "SVE128",
+      "SVE256"
+    ]
+  },
+  "Instructions": {
+    "addsubpd xmm0, xmm1": {
+      "ExpectedInstructionCount": 2,
+      "Optimal": "Yes",
+      "Comment": "0x66 0x0f 0xd0",
+      "ExpectedArm64ASM": [
+        "ext v4.16b, v17.16b, v17.16b, #8",
+        "fcadd v16.2d, v16.2d, v4.2d, #90"
+      ]
+    }
+  }
+}

unittests/InstructionCountCI/Secondary_REPNE.json

@@ -4,7 +4,8 @@
     "EnabledHostFeatures": [],
     "DisabledHostFeatures": [
       "SVE128",
-      "SVE256"
+      "SVE256",
+      "FCMA"
     ]
   },
   "Instructions": {

unittests/InstructionCountCI/Secondary_REPNE_FCMA.json

@@ -0,0 +1,23 @@
+{
+  "Features": {
+    "Bitness": 64,
+    "EnabledHostFeatures": [
+      "FCMA"
+    ],
+    "DisabledHostFeatures": [
+      "SVE128",
+      "SVE256"
+    ]
+  },
+  "Instructions": {
+    "addsubps xmm0, xmm1": {
+      "ExpectedInstructionCount": 2,
+      "Optimal": "Yes",
+      "Comment": "0xf2 0x0f 0xd0",
+      "ExpectedArm64ASM": [
+        "rev64 v4.4s, v17.4s",
+        "fcadd v16.4s, v16.4s, v4.4s, #90"
+      ]
+    }
+  }
+}

unittests/InstructionCountCI/VEX_map1.json

@@ -5,7 +5,9 @@
       "SVE128",
       "SVE256"
     ],
-    "DisabledHostFeatures": []
+    "DisabledHostFeatures": [
+      "FCMA"
+    ]
   },
   "Instructions": {
     "vmovups xmm0, [rax]": {

unittests/InstructionCountCI/VEX_map1_FCMA.json

@@ -0,0 +1,71 @@
+{
+  "Features": {
+    "Bitness": 64,
+    "EnabledHostFeatures": [
+      "SVE128",
+      "SVE256",
+      "FCMA"
+    ],
+    "DisabledHostFeatures": []
+  },
+  "Instructions": {
+    "vaddsubpd xmm0, xmm1, xmm2": {
+      "ExpectedInstructionCount": 6,
+      "Optimal": "No",
+      "Comment": [
+        "Map 1 0b01 0xd0 128-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "mov z4.d, p7/m, z17.d",
+        "mov z5.d, p7/m, z18.d",
+        "ext v5.16b, v5.16b, v5.16b, #8",
+        "fcadd v4.2d, v4.2d, v5.2d, #90",
+        "mov v4.16b, v4.16b",
+        "mov z16.d, p7/m, z4.d"
+      ]
+    },
+    "vaddsubpd ymm0, ymm1, ymm2": {
+      "ExpectedInstructionCount": 5,
+      "Optimal": "No",
+      "Comment": [
+        "Map 1 0b01 0xd0 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "mov z4.d, p7/m, z17.d",
+        "mov z5.d, p7/m, z18.d",
+        "ext z5.b, z5.b, z5.b, #8",
+        "fcadd z4.d, p7/m, z4.d, z5.d, #90",
+        "mov z16.d, p7/m, z4.d"
+      ]
+    },
+    "vaddsubps xmm0, xmm1, xmm2": {
+      "ExpectedInstructionCount": 6,
+      "Optimal": "No",
+      "Comment": [
+        "Map 1 0b11 0xd0 128-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "mov z4.d, p7/m, z17.d",
+        "mov z5.d, p7/m, z18.d",
+        "rev64 v5.4s, v5.4s",
+        "fcadd v4.4s, v4.4s, v5.4s, #90",
+        "mov v4.16b, v4.16b",
+        "mov z16.d, p7/m, z4.d"
+      ]
+    },
+    "vaddsubps ymm0, ymm1, ymm2": {
+      "ExpectedInstructionCount": 5,
+      "Optimal": "No",
+      "Comment": [
+        "Map 1 0b11 0xd0 256-bit"
+      ],
+      "ExpectedArm64ASM": [
+        "mov z4.d, p7/m, z17.d",
+        "mov z5.d, p7/m, z18.d",
+        "revw z5.d, p7/m, z5.d",
+        "fcadd z4.s, p7/m, z4.s, z5.s, #90",
+        "mov z16.d, p7/m, z4.d"
+      ]
+    }
+  }
+}