Merge pull request #2025 from lioncash/vfaddp

IR: Handle 256-bit VFAddP

External/FEXCore/Source/Interface/Core/Interpreter/VectorOps.cpp

@@ -479,20 +479,23 @@ DEF_OP(VFAdd) {
 }
 
 DEF_OP(VFAddP) {
-  auto Op = IROp->C<IR::IROp_VFAddP>();
+  const auto Op = IROp->C<IR::IROp_VFAddP>();
   const uint8_t OpSize = IROp->Size;
 
   void *Src1 = GetSrc<void*>(Data->SSAData, Op->VectorLower);
   void *Src2 = GetSrc<void*>(Data->SSAData, Op->VectorUpper);
-  uint8_t Tmp[16];
+  uint8_t Tmp[Core::CPUState::XMM_AVX_REG_SIZE];
 
-  const uint8_t Elements = (OpSize / Op->Header.ElementSize) / 2;
+  const uint8_t ElementSize = Op->Header.ElementSize;
+  const uint8_t Elements = (OpSize / ElementSize) / 2;
 
   const auto Func = [](auto a, auto b) { return a + b; };
-  switch (Op->Header.ElementSize) {
+  switch (ElementSize) {
     DO_VECTOR_PAIR_OP(4, float, Func)
     DO_VECTOR_PAIR_OP(8, double, Func)
-    default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
+    default:
+      LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
+      break;
   }
   memcpy(GDP, Tmp, OpSize);
 }

External/FEXCore/Source/Interface/Core/JIT/Arm64/VectorOps.cpp

@@ -878,17 +878,77 @@ DEF_OP(VFAdd) {
 }
 
 DEF_OP(VFAddP) {
-  auto Op = IROp->C<IR::IROp_VFAddP>();
-  switch (Op->Header.ElementSize) {
-    case 4: {
-      faddp(GetDst(Node).V4S(), GetSrc(Op->VectorLower.ID()).V4S(), GetSrc(Op->VectorUpper.ID()).V4S());
-    break;
+  const auto Op = IROp->C<IR::IROp_VFAddP>();
+  const auto OpSize = IROp->Size;
+
+  const auto ElementSize = Op->Header.ElementSize;
+
+  const auto Dst = GetDst(Node);
+  const auto VectorLower = GetSrc(Op->VectorLower.ID());
+  const auto VectorUpper = GetSrc(Op->VectorUpper.ID());
+
+  const bool Is256Bit = OpSize == 32;
+
+  if (HostSupportsSVE && Is256Bit) {
+    const auto Pred = PRED_TMP_32B.Merging();
+
+    // SVE FADDP is a destructive operation, so we need a temporary
+    mov(VTMP1.Z().VnD(), VectorLower.Z().VnD());
+
+    // Unlike Adv. SIMD's version of FADDP, which acts like it concats the
+    // upper vector onto the end of the lower vector and then performs
+    // pairwise addition, the SVE version actually interleaves the
+    // results of the pairwise addition (gross!), so we need to undo that.
+    switch (ElementSize) {
+      case 2: {
+        faddp(VTMP1.Z().VnH(), Pred, VectorLower.Z().VnH(), VectorUpper.Z().VnH());
+        uzp1(VTMP2.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH());
+        uzp2(VTMP3.Z().VnH(), VTMP1.Z().VnH(), VTMP1.Z().VnH());
+        break;
+      }
+      case 4: {
+        faddp(VTMP1.Z().VnS(), Pred, VTMP1.Z().VnS(), VectorUpper.Z().VnS());
+        uzp1(VTMP2.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS());
+        uzp2(VTMP3.Z().VnS(), VTMP1.Z().VnS(), VTMP1.Z().VnS());
+        break;
+      }
+      case 8: {
+        faddp(VTMP1.Z().VnD(), Pred, VTMP1.Z().VnD(), VectorUpper.Z().VnD());
+        uzp1(VTMP2.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD());
+        uzp2(VTMP3.Z().VnD(), VTMP1.Z().VnD(), VTMP1.Z().VnD());
+        break;
+      }
+      default:
+        LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
+        return;
     }
-    case 8: {
-      faddp(GetDst(Node).V2D(), GetSrc(Op->VectorLower.ID()).V2D(), GetSrc(Op->VectorUpper.ID()).V2D());
-    break;
+
+    // Shift the entire vector over by 64 bits (128-bit vector case)
+    // or by 128 bits (256-bit vector case).
+    mov(TMP1, 0);
+    insr(VTMP3.Z().VnD(), TMP1);
+    insr(VTMP3.Z().VnD(), TMP1);
+
+    // Now combine the lower and upper halves.
+    orr(Dst.Z().VnD(), VTMP2.Z().VnD(), VTMP3.Z().VnD());
+  } else {
+    switch (ElementSize) {
+      case 2: {
+        faddp(Dst.V8H(), VectorLower.V8H(), VectorUpper.V8H());
+        break;
+      }
+      case 4: {
+        faddp(Dst.V4S(), VectorLower.V4S(), VectorUpper.V4S());
+        break;
+      }
+      case 8: {
+        faddp(Dst.V2D(), VectorLower.V2D(), VectorUpper.V2D());
+        break;
+      }
+      default:
+        LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
+        break;
     }
-    default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
   }
 }
 

External/FEXCore/Source/Interface/Core/JIT/x86_64/VectorOps.cpp

@@ -613,15 +613,24 @@ DEF_OP(VFAdd) {
 }
 
 DEF_OP(VFAddP) {
-  auto Op = IROp->C<IR::IROp_VFAddP>();
-  switch (Op->Header.ElementSize) {
+  const auto Op = IROp->C<IR::IROp_VFAddP>();
+
+  const auto ElementSize = Op->Header.ElementSize;
+
+  const auto Dst = ToYMM(GetDst(Node));
+  const auto VectorLower = ToYMM(GetSrc(Op->VectorLower.ID()));
+  const auto VectorUpper = ToYMM(GetSrc(Op->VectorUpper.ID()));
+
+  switch (ElementSize) {
     case 4:
-      vhaddps(GetDst(Node), GetSrc(Op->VectorLower.ID()), GetSrc(Op->VectorUpper.ID()));
+      vhaddps(Dst, VectorLower, VectorUpper);
       break;
     case 8:
-      vhaddpd(GetDst(Node), GetSrc(Op->VectorLower.ID()), GetSrc(Op->VectorUpper.ID()));
+      vhaddpd(Dst, VectorLower, VectorUpper);
+      break;
+    default:
+      LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize);
       break;
-    default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
   }
 }