diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp index 8d487a848..4be69a20e 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp @@ -154,6 +154,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] { REGISTER_OP(STOREMEMTSO, StoreMem); REGISTER_OP(VLOADVECTORMASKED, VLoadVectorMasked); REGISTER_OP(VSTOREVECTORMASKED, VStoreVectorMasked); + REGISTER_OP(VLOADVECTORELEMENT, VLoadVectorElement); REGISTER_OP(VBROADCASTFROMMEM, VBroadcastFromMem); REGISTER_OP(PUSH, Push); REGISTER_OP(MEMSET, MemSet); diff --git a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index be3714e8d..448362f63 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -190,6 +190,7 @@ namespace FEXCore::CPU { DEF_OP(StoreMem); DEF_OP(VLoadVectorMasked); DEF_OP(VStoreVectorMasked); + DEF_OP(VLoadVectorElement); DEF_OP(VBroadcastFromMem); DEF_OP(Push); DEF_OP(MemSet); diff --git a/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp b/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp index 895abfc1d..a10dc7d74 100644 --- a/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp @@ -418,6 +418,53 @@ DEF_OP(VStoreVectorMasked) { } } +DEF_OP(VLoadVectorElement) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto ElementSize = IROp->ElementSize; + + auto *Mem = *GetSrc(Data->SSAData, Op->Addr); + const auto *DstSrc = GetSrc(Data->SSAData, Op->DstSrc); + + const auto SetElements = [](void* Dst, const T* MemPtr, const auto Index) { + std::memcpy(static_cast(Dst) + (Index * sizeof(T)), MemPtr, sizeof(T)); + }; + + // Copy the source data first. + memcpy(GDP, DstSrc, OpSize); + + switch (ElementSize) { + case 1: { + SetElements(GDP, + Mem, + Op->Index); + return; + } + case 2: { + SetElements(GDP, + reinterpret_cast(Mem), + Op->Index); + return; + } + case 4: { + SetElements(GDP, + reinterpret_cast(Mem), + Op->Index); + return; + } + case 8: { + SetElements(GDP, + reinterpret_cast(Mem), + Op->Index); + return; + } + default: + LOGMAN_MSG_A_FMT("Unhandled {} element size: {}", __func__, ElementSize); + return; + } +} + DEF_OP(VBroadcastFromMem) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp index 8a265a6f8..e3d7c4b91 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JIT.cpp @@ -972,6 +972,7 @@ CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, REGISTER_OP_RT(STOREMEMTSO, StoreMemTSO); REGISTER_OP(VLOADVECTORMASKED, VLoadVectorMasked); REGISTER_OP(VSTOREVECTORMASKED, VStoreVectorMasked); + REGISTER_OP(VLOADVECTORELEMENT, VLoadVectorElement); REGISTER_OP(VBROADCASTFROMMEM, VBroadcastFromMem); REGISTER_OP(PUSH, Push); REGISTER_OP(MEMSET, MemSet); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index d8d4cc84b..2de124ac2 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -343,6 +343,7 @@ private: DEF_OP(StoreMemTSO); DEF_OP(VLoadVectorMasked); DEF_OP(VStoreVectorMasked); + DEF_OP(VLoadVectorElement); DEF_OP(VBroadcastFromMem); DEF_OP(Push); DEF_OP(MemSet); diff --git a/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index 293176813..8284e34b9 100644 --- a/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -1429,6 +1429,55 @@ DEF_OP(VStoreVectorMasked) { } } +DEF_OP(VLoadVectorElement) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + const auto ElementSize = IROp->ElementSize; + + const auto Dst = GetVReg(Node); + const auto DstSrc = GetVReg(Op->DstSrc.ID()); + const auto MemReg = GetReg(Op->Addr.ID()); + + LOGMAN_THROW_AA_FMT(ElementSize == 1 || ElementSize == 2 || + ElementSize == 4 || ElementSize == 8 || + ElementSize == 16, "Invalid element size"); + + if (Is256Bit) { + LOGMAN_MSG_A_FMT("Unsupported 256-bit VLoadVectorElement"); + } else { + if (Dst != DstSrc && ElementSize != 16) { + mov(Dst.Q(), DstSrc.Q()); + } + switch (ElementSize) { + case 1: + ld1(Dst.Q(), Op->Index, MemReg); + break; + case 2: + ld1(Dst.Q(), Op->Index, MemReg); + break; + case 4: + ld1(Dst.Q(), Op->Index, MemReg); + break; + case 8: + ld1(Dst.Q(), Op->Index, MemReg); + break; + case 16: + ldr(Dst.Q(), MemReg); + break; + default: + LOGMAN_MSG_A_FMT("Unhandled {} size: {}", __func__, ElementSize); + return; + } + } + + // Emit a half-barrier if TSO is enabled. + if (CTX->IsAtomicTSOEnabled()) { + dmb(ARMEmitter::BarrierScope::ISHLD); + } +} + DEF_OP(VBroadcastFromMem) { const auto Op = IROp->C(); const auto OpSize = IROp->Size; diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index e2a47c7e4..6eb9a32d3 100644 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -349,6 +349,7 @@ private: DEF_OP(StoreMem); DEF_OP(VLoadVectorMasked); DEF_OP(VStoreVectorMasked); + DEF_OP(VLoadVectorElement); DEF_OP(VBroadcastFromMem); DEF_OP(Push); DEF_OP(MemSet); diff --git a/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp index cd047416a..9cd22d23d 100644 --- a/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp +++ b/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp @@ -843,6 +843,43 @@ DEF_OP(VStoreVectorMasked) { return; } } +DEF_OP(VLoadVectorElement) { + const auto Op = IROp->C(); + const auto OpSize = IROp->Size; + + const auto Is256Bit = OpSize == Core::CPUState::XMM_AVX_REG_SIZE; + const auto ElementSize = IROp->ElementSize; + + const auto Dst = GetDst(Node); + const auto DstSrc = GetSrc(Op->DstSrc.ID()); + const Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); + + if (Is256Bit) { + LOGMAN_MSG_A_FMT("Unsupported 256-bit VLoadVectorElement"); + } else { + switch (ElementSize) { + case 1: + vpinsrb(Dst, DstSrc, byte [MemReg], Op->Index); + break; + case 2: + vpinsrw(Dst, DstSrc, word [MemReg], Op->Index); + break; + case 4: + vpinsrd(Dst, DstSrc, dword [MemReg], Op->Index); + break; + case 8: + vpinsrq(Dst, DstSrc, qword [MemReg], Op->Index); + break; + case 16: + // Normal load + vmovups(Dst, xword [MemReg]); + break; + default: + LOGMAN_MSG_A_FMT("Unhandled VBroadcastFromMem size: {}", ElementSize); + return; + } + } +} DEF_OP(VBroadcastFromMem) { const auto Op = IROp->C(); @@ -1212,6 +1249,7 @@ void X86JITCore::RegisterMemoryHandlers() { REGISTER_OP(STOREMEMTSO, StoreMem); REGISTER_OP(VLOADVECTORMASKED, VLoadVectorMasked); REGISTER_OP(VSTOREVECTORMASKED, VStoreVectorMasked); + REGISTER_OP(VLOADVECTORELEMENT, VLoadVectorElement); REGISTER_OP(VBROADCASTFROMMEM, VBroadcastFromMem); REGISTER_OP(PUSH, Push); REGISTER_OP(MEMSET, MemSet); diff --git a/FEXCore/Source/Interface/IR/IR.json b/FEXCore/Source/Interface/IR/IR.json index 7db1c7188..f3d7675e8 100644 --- a/FEXCore/Source/Interface/IR/IR.json +++ b/FEXCore/Source/Interface/IR/IR.json @@ -491,7 +491,6 @@ "DestSize": "RegisterSize", "NumElements": "RegisterSize / ElementSize" }, - "VStoreVectorMasked u8:#RegisterSize, u8:#ElementSize, FPR:$Mask, FPR:$Data, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": { "Desc": ["Does a masked store similar to VPMASKMOV/VMASKMOV where the upper bit of each element", "determines whether or not that element will be stored to memory"], @@ -499,7 +498,13 @@ "DestSize": "RegisterSize", "NumElements": "RegisterSize / ElementSize" }, - + "FPR = VLoadVectorElement u8:#RegisterSize, u8:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr": { + "Desc": ["Does a memory load to a single element of a vector.", + "Leaves the rest of the vector's data intact.", + "Matches arm64 ld1 semantics"], + "DestSize": "RegisterSize", + "NumElements": "RegisterSize / ElementSize" + }, "FPR = VBroadcastFromMem u8:#RegisterSize, u8:#ElementSize, GPR:$Address": { "Desc": ["Broadcasts an ElementSize value from memory into each element of a vector."], "DestSize": "RegisterSize",