diff --git a/External/FEXCore/Source/Interface/Core/CPUID.cpp b/External/FEXCore/Source/Interface/Core/CPUID.cpp index 5560949ca..cedbb73cd 100644 --- a/External/FEXCore/Source/Interface/Core/CPUID.cpp +++ b/External/FEXCore/Source/Interface/Core/CPUID.cpp @@ -757,7 +757,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) { Res.ebx = (0 << 2) | // XSaveErPtr: Saving and restoring error pointers (0 << 1) | // IRPerf: Instructions retired count support - (0 << 0); // CLZERO support + (CTX->HostFeatures.SupportsCLZERO << 0); // CLZERO support uint32_t CoreCount = Cores() - 1; Res.ecx = diff --git a/External/FEXCore/Source/Interface/Core/CPUID.h b/External/FEXCore/Source/Interface/Core/CPUID.h index 8d4e8f989..be1a93a65 100644 --- a/External/FEXCore/Source/Interface/Core/CPUID.h +++ b/External/FEXCore/Source/Interface/Core/CPUID.h @@ -23,6 +23,10 @@ private: constexpr static uint32_t CPUID_VENDOR_AMD3 = 0x444D4163; // "cAMD" public: + // X86 cacheline size effectively has to be hardcoded to 64 + // if we report anything differently then applications are likely to break + constexpr static uint64_t CACHELINE_SIZE = 64; + void Init(FEXCore::Context::Context *ctx); FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) { diff --git a/External/FEXCore/Source/Interface/Core/HostFeatures.cpp b/External/FEXCore/Source/Interface/Core/HostFeatures.cpp index 4ee4cb53b..2d2eee4fc 100644 --- a/External/FEXCore/Source/Interface/Core/HostFeatures.cpp +++ b/External/FEXCore/Source/Interface/Core/HostFeatures.cpp @@ -1,3 +1,4 @@ +#include "Interface/Core/CPUID.h" #include "Interface/Core/HostFeatures.h" #ifdef _M_ARM_64 @@ -13,6 +14,29 @@ namespace FEXCore { +// Data Zero Prohibited flag +// 0b0 = ZVA/GVA/GZVA permitted +// 0b1 = ZVA/GVA/GZVA prohibited +constexpr uint32_t DCZID_DZP_MASK = 0b1'0000; +// Log2 of the blocksize in 32-bit words +constexpr uint32_t DCZID_BS_MASK = 0b0'1111; + +#ifdef _M_ARM_64 +static uint32_t GetDCZID() { + uint64_t Result{}; + __asm("mrs %[Res], DCZID_EL0" + : [Res] "=r" (Result)); + return Result; +} + +#else +static uint32_t GetDCZID() { + // Return unsupported + return DCZID_DZP_MASK; +} +#endif + + HostFeatures::HostFeatures() { #ifdef _M_ARM_64 auto Features = vixl::CPUFeatures::InferFromOS(); @@ -22,5 +46,15 @@ HostFeatures::HostFeatures() { Xbyak::util::Cpu Features{}; SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI); #endif + + // Check if we can support cacheline clears + uint32_t DCZID = GetDCZID(); + if ((DCZID & DCZID_DZP_MASK) == 0) { + uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK; + uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t); + // If the DC ZVA size matches the emulated cache line size + // This means we can use the instruction + SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE; + } } } diff --git a/External/FEXCore/Source/Interface/Core/HostFeatures.h b/External/FEXCore/Source/Interface/Core/HostFeatures.h index cdbe264de..93cd81f52 100644 --- a/External/FEXCore/Source/Interface/Core/HostFeatures.h +++ b/External/FEXCore/Source/Interface/Core/HostFeatures.h @@ -5,5 +5,6 @@ class HostFeatures final { public: HostFeatures(); bool SupportsAES{}; + bool SupportsCLZERO{}; }; } diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp index 249fc0c59..04d865132 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp @@ -157,6 +157,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); // Misc ops REGISTER_OP(DUMMY, NoOp); diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h index 940114083..662df2b51 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h +++ b/External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h @@ -180,6 +180,7 @@ namespace FEXCore::CPU { DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp index 3f4ea7df8..ed6c0cb03 100644 --- a/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp @@ -264,5 +264,22 @@ DEF_OP(CacheLineClear) { CacheLineFlush(MemData); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + uintptr_t MemData = *GetSrc(Data->SSAData, Op->Addr); + + // Force cacheline alignment + MemData = MemData & ~(CPUIDEmu::CACHELINE_SIZE - 1); + + using DataType = uint64_t; + DataType *MemData64 = reinterpret_cast(MemData); + + // 64-byte cache line zero + for (size_t i = 0; i < (CPUIDEmu::CACHELINE_SIZE / sizeof(DataType)); ++i) { + MemData64[i] = 0; + } +} + #undef DEF_OP } // namespace FEXCore::CPU diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h index b269c74a2..253903380 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h @@ -321,6 +321,7 @@ private: DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp index 125f09f45..35a9031ec 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp @@ -946,6 +946,28 @@ DEF_OP(CacheLineClear) { dsb(InnerShareable, BarrierAll); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + auto MemReg = GetReg(Op->Header.Args[0].ID()); + + if (CTX->HostFeatures.SupportsCLZERO) { + // We can use this instruction directly + dc(DataCacheOp::ZVA, MemReg); + } + else { + // We must walk the cacheline ourselves + // Force cacheline alignment + and_(TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1)); + // This will end up being four STPs + // Depending on uarch it could be slightly more efficient in instructions emitted + // and uops to use vector pair STP, but we want the non-temporal bit specifically here + for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += 16) { + stnp(xzr, xzr, MemOperand(TMP1, i, Offset)); + } + } +} + #undef DEF_OP void Arm64JITCore::RegisterMemoryHandlers() { #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x @@ -972,6 +994,7 @@ void Arm64JITCore::RegisterMemoryHandlers() { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); #undef REGISTER_OP } } diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h index 55c9f4c46..d26b469d6 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h @@ -315,6 +315,7 @@ private: DEF_OP(VLoadMemElement); DEF_OP(VStoreMemElement); DEF_OP(CacheLineClear); + DEF_OP(CacheLineZero); ///< Misc ops DEF_OP(EndBlock); diff --git a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp index 52e9bc8f5..f8be6a794 100644 --- a/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp +++ b/External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp @@ -595,6 +595,23 @@ DEF_OP(CacheLineClear) { clflush(ptr [MemReg]); } +DEF_OP(CacheLineZero) { + auto Op = IROp->C(); + + Xbyak::Reg MemReg = GetSrc(Op->Addr.ID()); + + // Align by cacheline + mov (TMP1, CPUIDEmu::CACHELINE_SIZE - 1); + andn(TMP1, TMP1, MemReg.cvt64()); + xor_(TMP2, TMP2); + + using DataType = uint64_t; + // 64-byte cache line zero + for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += sizeof(DataType)) { + mov (qword [TMP1 + i], TMP2); + } +} + #undef DEF_OP void X86JITCore::RegisterMemoryHandlers() { #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x @@ -615,6 +632,7 @@ void X86JITCore::RegisterMemoryHandlers() { REGISTER_OP(VLOADMEMELEMENT, VLoadMemElement); REGISTER_OP(VSTOREMEMELEMENT, VStoreMemElement); REGISTER_OP(CACHELINECLEAR, CacheLineClear); + REGISTER_OP(CACHELINEZERO, CacheLineZero); #undef REGISTER_OP } } diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp index ae9025f79..91f6d56e6 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp @@ -5145,6 +5145,11 @@ void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) { } } +void OpDispatchBuilder::CLZeroOp(OpcodeArgs) { + OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1, false); + _CacheLineZero(DestMem); +} + void OpDispatchBuilder::UnimplementedOp(OpcodeArgs) { const uint8_t GPRSize = CTX->GetGPRSize(); @@ -5791,6 +5796,10 @@ constexpr uint16_t PF_F2 = 3; const std::vector> SecondaryModRMExtensionOpTable = { // REG /2 {((1 << 3) | 0), 1, &OpDispatchBuilder::UnimplementedOp}, + + // REG /7 + {((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp}, + }; // Top bit indicating if it needs to be repeated with {0x40, 0x80} or'd in // All OPDReg versions need it diff --git a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h index 6e728d1b1..40fc60c88 100644 --- a/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h +++ b/External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h @@ -469,6 +469,7 @@ public: void FenceOp(OpcodeArgs); void StoreFenceOrCLFlush(OpcodeArgs); + void CLZeroOp(OpcodeArgs); void PSADBW(OpcodeArgs); diff --git a/External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp b/External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp index 9dcc228a0..cb0cfe64f 100644 --- a/External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp +++ b/External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp @@ -50,7 +50,7 @@ void InitializeSecondaryModRMTables() { {((3 << 3) | 1), 1, X86InstInfo{"RDTSCP", TYPE_PRIV, FLAGS_NONE, 0, nullptr}}, {((3 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV, FLAGS_NONE, 0, nullptr}}, {((3 << 3) | 3), 1, X86InstInfo{"MWAITX", TYPE_PRIV, FLAGS_NONE, 0, nullptr}}, - {((3 << 3) | 4), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}}, + {((3 << 3) | 4), 1, X86InstInfo{"CLZERO", TYPE_INST, FLAGS_SF_SRC_RAX, 0, nullptr}}, {((3 << 3) | 5), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}}, {((3 << 3) | 6), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}}, {((3 << 3) | 7), 1, X86InstInfo{"", TYPE_INVALID, FLAGS_NONE, 0, nullptr}}, diff --git a/External/FEXCore/Source/Interface/IR/IR.json b/External/FEXCore/Source/Interface/IR/IR.json index 8c3dd834b..4cfeb4aff 100644 --- a/External/FEXCore/Source/Interface/IR/IR.json +++ b/External/FEXCore/Source/Interface/IR/IR.json @@ -858,7 +858,22 @@ }, "CacheLineClear": { - "Desc": ["Does a 64 byte cacheline clear at the address specified" + "Desc": ["Does a 64 byte cacheline clear at the address specified", + "Only clears the data cachelines. Doesn't do any zeroing" + ], + "HasSideEffects": true, + "OpClass": "Memory", + "SSAArgs": "1", + "SSANames": [ + "Addr" + ] + }, + + "CacheLineZero": { + "Desc": ["Does a 64 byte zero at the address specified", + "Writing zeroes to memory", + "It is specifically non-temporal and weakly ordered", + "This matches CLZero behaviour" ], "HasSideEffects": true, "OpClass": "Memory", diff --git a/Source/Tests/HarnessHelpers.h b/Source/Tests/HarnessHelpers.h index e61b6a537..573aabac4 100644 --- a/Source/Tests/HarnessHelpers.h +++ b/Source/Tests/HarnessHelpers.h @@ -234,7 +234,7 @@ namespace FEX::HarnessHelper { return; } - fmt::format("{}: 0x{:016x} {} 0x{:016x} (Expected)\n", Name, A, A==B ? "==" : "!=", B); + fmt::print("{}: 0x{:016x} {} 0x{:016x} (Expected)\n", Name, A, A==B ? "==" : "!=", B); }; const auto CheckGPRs = [&Matches, DumpGPRs](const std::string& Name, uint64_t A, uint64_t B) { diff --git a/unittests/ASM/Disabled_Tests_host b/unittests/ASM/Disabled_Tests_host index b2fce52b0..3bc616a4b 100644 --- a/unittests/ASM/Disabled_Tests_host +++ b/unittests/ASM/Disabled_Tests_host @@ -7,3 +7,7 @@ Test_REPNE/F2_2B.asm # 3DNow! no longer exists on AMD Zen CPUs Test_TwoByte/0F_0E.asm + +# Intel doesn't support CLZero, which the CI uses +Test_SecondaryModRM/Reg_7_4.asm +Test_SecondaryModRM/Reg_7_4_2.asm diff --git a/unittests/ASM/SecondaryModRM/Reg_7_4.asm b/unittests/ASM/SecondaryModRM/Reg_7_4.asm new file mode 100644 index 000000000..f342062f5 --- /dev/null +++ b/unittests/ASM/SecondaryModRM/Reg_7_4.asm @@ -0,0 +1,36 @@ +%ifdef CONFIG +{ + "RegData": { + "RBX": "0x0" + } +} +%endif + +; Starting address to store to +mov rax, 0xe8000000 + +; Set up the cacheline with garbage +mov rbx, 0x4142434445464748 +mov [rax + 8 * 0], rbx +mov [rax + 8 * 1], rbx +mov [rax + 8 * 2], rbx +mov [rax + 8 * 3], rbx +mov [rax + 8 * 4], rbx +mov [rax + 8 * 5], rbx +mov [rax + 8 * 6], rbx +mov [rax + 8 * 7], rbx + +clzero + +mov rbx, 0 + +add rbx, [rax + 8 * 0] +add rbx, [rax + 8 * 1] +add rbx, [rax + 8 * 2] +add rbx, [rax + 8 * 3] +add rbx, [rax + 8 * 4] +add rbx, [rax + 8 * 5] +add rbx, [rax + 8 * 6] +add rbx, [rax + 8 * 7] + +hlt diff --git a/unittests/ASM/SecondaryModRM/Reg_7_4_2.asm b/unittests/ASM/SecondaryModRM/Reg_7_4_2.asm new file mode 100644 index 000000000..990a1b2aa --- /dev/null +++ b/unittests/ASM/SecondaryModRM/Reg_7_4_2.asm @@ -0,0 +1,91 @@ +%ifdef CONFIG +{ + "RegData": { + "RBX": "0x0", + "RCX": "0x000000020A121A20", + "RDX": "0x0A121A2000000000" + } +} +%endif + +; Starting address to store to +mov rax, 0xe8000000 + +; Set up the cachelines with garbage + +; Cacheline 0 +mov rbx, 0x0000000041424344 +mov [rax + 8 * 0], rbx +mov [rax + 8 * 1], rbx +mov [rax + 8 * 2], rbx +mov [rax + 8 * 3], rbx +mov [rax + 8 * 4], rbx +mov [rax + 8 * 5], rbx +mov [rax + 8 * 6], rbx +mov [rax + 8 * 7], rbx + +; Cacheline 1 +mov rbx, 0x5152535455565758 +mov [rax + 8 * 8], rbx +mov [rax + 8 * 9], rbx +mov [rax + 8 * 10], rbx +mov [rax + 8 * 11], rbx ; clzero here +mov [rax + 8 * 12], rbx +mov [rax + 8 * 13], rbx +mov [rax + 8 * 14], rbx +mov [rax + 8 * 15], rbx + +; Cacheline 2 +mov rbx, 0x4142434400000000 +mov [rax + 8 * 16], rbx +mov [rax + 8 * 17], rbx +mov [rax + 8 * 18], rbx +mov [rax + 8 * 19], rbx +mov [rax + 8 * 20], rbx +mov [rax + 8 * 21], rbx +mov [rax + 8 * 22], rbx +mov [rax + 8 * 23], rbx + +; Set RAX to the middle of cacheline 1 to ensure alignment +lea rax, [rax + 8 * 11] + +clzero + +; Set rax back to the start +mov rax, 0xe8000000 + +mov rbx, 0 +mov rcx, 0 +mov rdx, 0 + +; Cacheline 0 should be unmodified +add rcx, [rax + 8 * 0] +add rcx, [rax + 8 * 1] +add rcx, [rax + 8 * 2] +add rcx, [rax + 8 * 3] +add rcx, [rax + 8 * 4] +add rcx, [rax + 8 * 5] +add rcx, [rax + 8 * 6] +add rcx, [rax + 8 * 7] + +; Cacheline 1 Should be zero +add rbx, [rax + 8 * 8] +add rbx, [rax + 8 * 9] +add rbx, [rax + 8 * 10] +add rbx, [rax + 8 * 11] +add rbx, [rax + 8 * 12] +add rbx, [rax + 8 * 13] +add rbx, [rax + 8 * 14] +add rbx, [rax + 8 * 15] + +; Cacheline 2 should be unmodified +add rdx, [rax + 8 * 16] +add rdx, [rax + 8 * 17] +add rdx, [rax + 8 * 18] +add rdx, [rax + 8 * 19] +add rdx, [rax + 8 * 20] +add rdx, [rax + 8 * 21] +add rdx, [rax + 8 * 22] +add rdx, [rax + 8 * 23] + +hlt