IR: Implement support for vector load element

Matches Arm64 LD1 semantics.

FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -154,6 +154,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(STOREMEMTSO,            StoreMem);
   REGISTER_OP(VLOADVECTORMASKED,      VLoadVectorMasked);
   REGISTER_OP(VSTOREVECTORMASKED,     VStoreVectorMasked);
+  REGISTER_OP(VLOADVECTORELEMENT,     VLoadVectorElement);
   REGISTER_OP(VBROADCASTFROMMEM,      VBroadcastFromMem);
   REGISTER_OP(PUSH,                   Push);
   REGISTER_OP(MEMSET,                 MemSet);

FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -190,6 +190,7 @@ namespace FEXCore::CPU {
   DEF_OP(StoreMem);
   DEF_OP(VLoadVectorMasked);
   DEF_OP(VStoreVectorMasked);
+  DEF_OP(VLoadVectorElement);
   DEF_OP(VBroadcastFromMem);
   DEF_OP(Push);
   DEF_OP(MemSet);

FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp

@@ -418,6 +418,53 @@ DEF_OP(VStoreVectorMasked) {
   }
 }
 
+DEF_OP(VLoadVectorElement) {
+  const auto Op = IROp->C<IR::IROp_VLoadVectorElement>();
+  const auto OpSize = IROp->Size;
+
+  const auto ElementSize = IROp->ElementSize;
+
+  auto *Mem = *GetSrc<uint8_t**>(Data->SSAData, Op->Addr);
+  const auto *DstSrc = GetSrc<uint8_t const*>(Data->SSAData, Op->DstSrc);
+
+  const auto SetElements = []<typename T>(void* Dst, const T* MemPtr, const auto Index) {
+    std::memcpy(static_cast<uint8_t*>(Dst) + (Index * sizeof(T)), MemPtr, sizeof(T));
+  };
+
+  // Copy the source data first.
+  memcpy(GDP, DstSrc, OpSize);
+
+  switch (ElementSize) {
+    case 1: {
+      SetElements(GDP,
+                  Mem,
+                  Op->Index);
+      return;
+    }
+    case 2: {
+      SetElements(GDP,
+                  reinterpret_cast<const uint16_t*>(Mem),
+                  Op->Index);
+      return;
+    }
+    case 4: {
+      SetElements(GDP,
+                  reinterpret_cast<const uint32_t*>(Mem),
+                  Op->Index);
+      return;
+    }
+    case 8: {
+      SetElements(GDP,
+                  reinterpret_cast<const uint64_t*>(Mem),
+                  Op->Index);
+      return;
+    }
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled {} element size: {}", __func__, ElementSize);
+      return;
+  }
+}
+
 DEF_OP(VBroadcastFromMem) {
   const auto Op = IROp->C<IR::IROp_VBroadcastFromMem>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp

@@ -972,6 +972,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry,
         REGISTER_OP_RT(STOREMEMTSO,      StoreMemTSO);
         REGISTER_OP(VLOADVECTORMASKED,   VLoadVectorMasked);
         REGISTER_OP(VSTOREVECTORMASKED,  VStoreVectorMasked);
+        REGISTER_OP(VLOADVECTORELEMENT,  VLoadVectorElement);
         REGISTER_OP(VBROADCASTFROMMEM,   VBroadcastFromMem);
         REGISTER_OP(PUSH,                Push);
         REGISTER_OP(MEMSET,              MemSet);

FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h

@@ -343,6 +343,7 @@ private:
   DEF_OP(StoreMemTSO);
   DEF_OP(VLoadVectorMasked);
   DEF_OP(VStoreVectorMasked);
+  DEF_OP(VLoadVectorElement);
   DEF_OP(VBroadcastFromMem);
   DEF_OP(Push);
   DEF_OP(MemSet);

FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp

@@ -1429,6 +1429,55 @@ DEF_OP(VStoreVectorMasked) {
   }
 }
 
+DEF_OP(VLoadVectorElement) {
+  const auto Op = IROp->C<IR::IROp_VLoadVectorElement>();
+  const auto OpSize = IROp->Size;
+
+  const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
+  const auto ElementSize = IROp->ElementSize;
+
+  const auto Dst = GetVReg(Node);
+  const auto DstSrc = GetVReg(Op->DstSrc.ID());
+  const auto MemReg = GetReg(Op->Addr.ID());
+
+  LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 ||
+                      ElementSize == 4 || ElementSize == 8 ||
+                      ElementSize == 16, "Invalid element size");
+
+  if (Is256Bit) {
+    LOGMAN_MSG_A_FMT("Unsupported 256-bit VLoadVectorElement");
+  } else {
+    if (Dst != DstSrc && ElementSize != 16) {
+      mov(Dst.Q(), DstSrc.Q());
+    }
+    switch (ElementSize) {
+    case 1:
+      ld1<ARMEmitter::SubRegSize::i8Bit>(Dst.Q(), Op->Index, MemReg);
+      break;
+    case 2:
+      ld1<ARMEmitter::SubRegSize::i16Bit>(Dst.Q(), Op->Index, MemReg);
+      break;
+    case 4:
+      ld1<ARMEmitter::SubRegSize::i32Bit>(Dst.Q(), Op->Index, MemReg);
+      break;
+    case 8:
+      ld1<ARMEmitter::SubRegSize::i64Bit>(Dst.Q(), Op->Index, MemReg);
+      break;
+    case 16:
+      ldr(Dst.Q(), MemReg);
+      break;
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize);
+      return;
+    }
+  }
+
+  // Emit a half-barrier if TSO is enabled.
+  if (CTX->IsAtomicTSOEnabled()) {
+    dmb(ARMEmitter::BarrierScope::ISHLD);
+  }
+}
+
 DEF_OP(VBroadcastFromMem) {
   const auto Op = IROp->C<IR::IROp_VBroadcastFromMem>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h

@@ -349,6 +349,7 @@ private:
   DEF_OP(StoreMem);
   DEF_OP(VLoadVectorMasked);
   DEF_OP(VStoreVectorMasked);
+  DEF_OP(VLoadVectorElement);
   DEF_OP(VBroadcastFromMem);
   DEF_OP(Push);
   DEF_OP(MemSet);

FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp

@@ -843,6 +843,43 @@ DEF_OP(VStoreVectorMasked) {
       return;
   }
 }
+DEF_OP(VLoadVectorElement) {
+  const auto Op = IROp->C<IR::IROp_VLoadVectorElement>();
+  const auto OpSize = IROp->Size;
+
+  const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE;
+  const auto ElementSize = IROp->ElementSize;
+
+  const auto Dst = GetDst(Node);
+  const auto DstSrc = GetSrc(Op->DstSrc.ID());
+  const Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
+
+  if (Is256Bit) {
+    LOGMAN_MSG_A_FMT("Unsupported 256-bit VLoadVectorElement");
+  } else {
+    switch (ElementSize) {
+    case 1:
+      vpinsrb(Dst, DstSrc, byte [MemReg], Op->Index);
+      break;
+    case 2:
+      vpinsrw(Dst, DstSrc, word [MemReg], Op->Index);
+      break;
+    case 4:
+      vpinsrd(Dst, DstSrc, dword [MemReg], Op->Index);
+      break;
+    case 8:
+      vpinsrq(Dst, DstSrc, qword [MemReg], Op->Index);
+      break;
+    case 16:
+      // Normal load
+      vmovups(Dst, xword [MemReg]);
+      break;
+    default:
+      LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize);
+      return;
+    }
+  }
+}
 
 DEF_OP(VBroadcastFromMem) {
   const auto Op = IROp->C<IR::IROp_VBroadcastFromMem>();
@@ -1212,6 +1249,7 @@ void X86JITCore::RegisterMemoryHandlers() {
   REGISTER_OP(STOREMEMTSO,         StoreMem);
   REGISTER_OP(VLOADVECTORMASKED,   VLoadVectorMasked);
   REGISTER_OP(VSTOREVECTORMASKED,  VStoreVectorMasked);
+  REGISTER_OP(VLOADVECTORELEMENT,  VLoadVectorElement);
   REGISTER_OP(VBROADCASTFROMMEM,   VBroadcastFromMem);
   REGISTER_OP(PUSH,                Push);
   REGISTER_OP(MEMSET,              MemSet);

FEXCore/Source/Interface/IR/IR.json

@@ -491,7 +491,6 @@
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
       },
-
       "VStoreVectorMasked u8:#RegisterSize, u8:#ElementSize, FPR:$Mask, FPR:$Data, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": {
         "Desc": ["Does a masked store similar to VPMASKMOV/VMASKMOV where the upper bit of each element",
                  "determines whether or not that element will be stored to memory"],
@@ -499,7 +498,13 @@
         "DestSize": "RegisterSize",
         "NumElements": "RegisterSize / ElementSize"
       },
-
+      "FPR = VLoadVectorElement u8:#RegisterSize, u8:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr": {
+        "Desc": ["Does a memory load to a single element of a vector.",
+                 "Leaves the rest of the vector's data intact.",
+                 "Matches arm64 ld1 semantics"],
+        "DestSize": "RegisterSize",
+        "NumElements": "RegisterSize / ElementSize"
+      },
       "FPR = VBroadcastFromMem u8:#RegisterSize, u8:#ElementSize, GPR:$Address": {
         "Desc": ["Broadcasts an ElementSize value from memory into each element of a vector."],
         "DestSize": "RegisterSize",