Merge pull request #1422 from Sonicadvance1/implement_clzero

Implements CLZero instruction

External/FEXCore/Source/Interface/Core/CPUID.cpp

@@ -757,7 +757,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0008h(uint32_t Leaf) {
   Res.ebx =
     (0 << 2) | // XSaveErPtr: Saving and restoring error pointers
     (0 << 1) | // IRPerf: Instructions retired count support
-    (0 << 0);  // CLZERO support
+    (CTX->HostFeatures.SupportsCLZERO << 0);  // CLZERO support
 
   uint32_t CoreCount = Cores() - 1;
   Res.ecx =

External/FEXCore/Source/Interface/Core/CPUID.h

@@ -23,6 +23,10 @@ private:
   constexpr static uint32_t CPUID_VENDOR_AMD3 = 0x444D4163; // "cAMD"
 
 public:
+  // X86 cacheline size effectively has to be hardcoded to 64
+  // if we report anything differently then applications are likely to break
+  constexpr static uint64_t CACHELINE_SIZE = 64;
+
   void Init(FEXCore::Context::Context *ctx);
 
   FEXCore::CPUID::FunctionResults RunFunction(uint32_t Function, uint32_t Leaf) {

External/FEXCore/Source/Interface/Core/HostFeatures.cpp

@@ -1,3 +1,4 @@
+#include "Interface/Core/CPUID.h"
 #include "Interface/Core/HostFeatures.h"
 
 #ifdef _M_ARM_64
@@ -13,6 +14,29 @@
 
 namespace FEXCore {
 
+// Data Zero Prohibited flag
+// 0b0 = ZVA/GVA/GZVA permitted
+// 0b1 = ZVA/GVA/GZVA prohibited
+constexpr uint32_t DCZID_DZP_MASK = 0b1'0000;
+// Log2 of the blocksize in 32-bit words
+constexpr uint32_t DCZID_BS_MASK = 0b0'1111;
+
+#ifdef _M_ARM_64
+static uint32_t GetDCZID() {
+  uint64_t Result{};
+  __asm("mrs %[Res], DCZID_EL0"
+      : [Res] "=r" (Result));
+  return Result;
+}
+
+#else
+static uint32_t GetDCZID() {
+  // Return unsupported
+  return DCZID_DZP_MASK;
+}
+#endif
+
+
 HostFeatures::HostFeatures() {
 #ifdef _M_ARM_64
   auto Features = vixl::CPUFeatures::InferFromOS();
@@ -22,5 +46,15 @@ HostFeatures::HostFeatures() {
   Xbyak::util::Cpu Features{};
   SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI);
 #endif
+
+  // Check if we can support cacheline clears
+  uint32_t DCZID = GetDCZID();
+  if ((DCZID & DCZID_DZP_MASK) == 0) {
+    uint32_t DCZID_Log2 = DCZID & DCZID_BS_MASK;
+    uint32_t DCZID_Bytes = (1 << DCZID_Log2) * sizeof(uint32_t);
+    // If the DC ZVA size matches the emulated cache line size
+    // This means we can use the instruction
+    SupportsCLZERO = DCZID_Bytes == CPUIDEmu::CACHELINE_SIZE;
+  }
 }
 }

External/FEXCore/Source/Interface/Core/HostFeatures.h

@@ -5,5 +5,6 @@ class HostFeatures final {
   public:
     HostFeatures();
     bool SupportsAES{};
+    bool SupportsCLZERO{};
 };
 }

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.cpp

@@ -157,6 +157,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
   REGISTER_OP(VLOADMEMELEMENT,        VLoadMemElement);
   REGISTER_OP(VSTOREMEMELEMENT,       VStoreMemElement);
   REGISTER_OP(CACHELINECLEAR,         CacheLineClear);
+  REGISTER_OP(CACHELINEZERO,          CacheLineZero);
 
   // Misc ops
   REGISTER_OP(DUMMY,                  NoOp);

External/FEXCore/Source/Interface/Core/Interpreter/InterpreterOps.h

@@ -180,6 +180,7 @@ namespace FEXCore::CPU {
   DEF_OP(VLoadMemElement);
   DEF_OP(VStoreMemElement);
   DEF_OP(CacheLineClear);
+  DEF_OP(CacheLineZero);
 
   ///< Misc ops
   DEF_OP(EndBlock);

External/FEXCore/Source/Interface/Core/Interpreter/MemoryOps.cpp

@@ -264,5 +264,22 @@ DEF_OP(CacheLineClear) {
   CacheLineFlush(MemData);
 }
 
+DEF_OP(CacheLineZero) {
+  auto Op = IROp->C<IR::IROp_CacheLineZero>();
+
+  uintptr_t MemData = *GetSrc<uintptr_t*>(Data->SSAData, Op->Addr);
+
+  // Force cacheline alignment
+  MemData = MemData & ~(CPUIDEmu::CACHELINE_SIZE - 1);
+
+  using DataType = uint64_t;
+  DataType *MemData64 = reinterpret_cast<DataType*>(MemData);
+
+  // 64-byte cache line zero
+  for (size_t i = 0; i < (CPUIDEmu::CACHELINE_SIZE / sizeof(DataType)); ++i) {
+    MemData64[i] = 0;
+  }
+}
+
 #undef DEF_OP
 } // namespace FEXCore::CPU

External/FEXCore/Source/Interface/Core/JIT/Arm64/JITClass.h

@@ -321,6 +321,7 @@ private:
   DEF_OP(VLoadMemElement);
   DEF_OP(VStoreMemElement);
   DEF_OP(CacheLineClear);
+  DEF_OP(CacheLineZero);
 
   ///< Misc ops
   DEF_OP(EndBlock);

External/FEXCore/Source/Interface/Core/JIT/Arm64/MemoryOps.cpp

@@ -946,6 +946,28 @@ DEF_OP(CacheLineClear) {
   dsb(InnerShareable, BarrierAll);
 }
 
+DEF_OP(CacheLineZero) {
+  auto Op = IROp->C<IR::IROp_CacheLineZero>();
+
+  auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
+
+  if (CTX->HostFeatures.SupportsCLZERO) {
+    // We can use this instruction directly
+    dc(DataCacheOp::ZVA, MemReg);
+  }
+  else {
+    // We must walk the cacheline ourselves
+    // Force cacheline alignment
+    and_(TMP1, MemReg, ~(CPUIDEmu::CACHELINE_SIZE - 1));
+    // This will end up being four STPs
+    // Depending on uarch it could be slightly more efficient in instructions emitted
+    // and uops to use vector pair STP, but we want the non-temporal bit specifically here
+    for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += 16) {
+      stnp(xzr, xzr, MemOperand(TMP1, i, Offset));
+    }
+  }
+}
+
 #undef DEF_OP
 void Arm64JITCore::RegisterMemoryHandlers() {
 #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
@@ -972,6 +994,7 @@ void Arm64JITCore::RegisterMemoryHandlers() {
   REGISTER_OP(VLOADMEMELEMENT,     VLoadMemElement);
   REGISTER_OP(VSTOREMEMELEMENT,    VStoreMemElement);
   REGISTER_OP(CACHELINECLEAR,      CacheLineClear);
+  REGISTER_OP(CACHELINEZERO,       CacheLineZero);
 #undef REGISTER_OP
 }
 }

External/FEXCore/Source/Interface/Core/JIT/x86_64/JITClass.h

@@ -315,6 +315,7 @@ private:
   DEF_OP(VLoadMemElement);
   DEF_OP(VStoreMemElement);
   DEF_OP(CacheLineClear);
+  DEF_OP(CacheLineZero);
 
   ///< Misc ops
   DEF_OP(EndBlock);

External/FEXCore/Source/Interface/Core/JIT/x86_64/MemoryOps.cpp

@@ -595,6 +595,23 @@ DEF_OP(CacheLineClear) {
   clflush(ptr [MemReg]);
 }
 
+DEF_OP(CacheLineZero) {
+  auto Op = IROp->C<IR::IROp_CacheLineZero>();
+
+  Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
+
+  // Align by cacheline
+  mov (TMP1, CPUIDEmu::CACHELINE_SIZE - 1);
+  andn(TMP1, TMP1, MemReg.cvt64());
+  xor_(TMP2, TMP2);
+
+  using DataType = uint64_t;
+  // 64-byte cache line zero
+  for (size_t i = 0; i < CPUIDEmu::CACHELINE_SIZE; i += sizeof(DataType)) {
+    mov (qword [TMP1 + i], TMP2);
+  }
+}
+
 #undef DEF_OP
 void X86JITCore::RegisterMemoryHandlers() {
 #define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
@@ -615,6 +632,7 @@ void X86JITCore::RegisterMemoryHandlers() {
   REGISTER_OP(VLOADMEMELEMENT,     VLoadMemElement);
   REGISTER_OP(VSTOREMEMELEMENT,    VStoreMemElement);
   REGISTER_OP(CACHELINECLEAR,      CacheLineClear);
+  REGISTER_OP(CACHELINEZERO,       CacheLineZero);
 #undef REGISTER_OP
 }
 }

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -5145,6 +5145,11 @@ void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
   }
 }
 
+void OpDispatchBuilder::CLZeroOp(OpcodeArgs) {
+  OrderedNode *DestMem = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1, false);
+  _CacheLineZero(DestMem);
+}
+
 void OpDispatchBuilder::UnimplementedOp(OpcodeArgs) {
   const uint8_t GPRSize = CTX->GetGPRSize();
 
@@ -5791,6 +5796,10 @@ constexpr uint16_t PF_F2 = 3;
   const std::vector<std::tuple<uint8_t, uint8_t, FEXCore::X86Tables::OpDispatchPtr>> SecondaryModRMExtensionOpTable = {
     // REG /2
     {((1 << 3) | 0), 1, &OpDispatchBuilder::UnimplementedOp},
+
+    // REG /7
+    {((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
+
   };
 // Top bit indicating if it needs to be repeated with {0x40, 0x80} or'd in
 // All OPDReg versions need it

External/FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -469,6 +469,7 @@ public:
   void FenceOp(OpcodeArgs);
 
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void CLZeroOp(OpcodeArgs);
 
   void PSADBW(OpcodeArgs);
 

External/FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp

@@ -50,7 +50,7 @@ void InitializeSecondaryModRMTables() {
     {((3 << 3) | 1), 1, X86InstInfo{"RDTSCP",   TYPE_PRIV,    FLAGS_NONE, 0, nullptr}},
     {((3 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV,    FLAGS_NONE, 0, nullptr}},
     {((3 << 3) | 3), 1, X86InstInfo{"MWAITX",   TYPE_PRIV,    FLAGS_NONE, 0, nullptr}},
-    {((3 << 3) | 4), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
+    {((3 << 3) | 4), 1, X86InstInfo{"CLZERO",   TYPE_INST,    FLAGS_SF_SRC_RAX, 0, nullptr}},
     {((3 << 3) | 5), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
     {((3 << 3) | 6), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0, nullptr}},
     {((3 << 3) | 7), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0, nullptr}},

External/FEXCore/Source/Interface/IR/IR.json

@@ -858,7 +858,22 @@
     },
 
     "CacheLineClear": {
-      "Desc": ["Does a 64 byte cacheline clear at the address specified"
+      "Desc": ["Does a 64 byte cacheline clear at the address specified",
+               "Only clears the data cachelines. Doesn't do any zeroing"
+              ],
+      "HasSideEffects": true,
+      "OpClass": "Memory",
+      "SSAArgs": "1",
+      "SSANames": [
+        "Addr"
+      ]
+    },
+
+    "CacheLineZero": {
+      "Desc": ["Does a 64 byte zero at the address specified",
+               "Writing zeroes to memory",
+               "It is specifically non-temporal and weakly ordered",
+               "This matches CLZero behaviour"
               ],
       "HasSideEffects": true,
       "OpClass": "Memory",

Source/Tests/HarnessHelpers.h

@@ -234,7 +234,7 @@ namespace FEX::HarnessHelper {
               return;
             }
 
-            fmt::format("{}: 0x{:016x} {} 0x{:016x} (Expected)\n", Name, A, A==B ? "==" : "!=", B);
+            fmt::print("{}: 0x{:016x} {} 0x{:016x} (Expected)\n", Name, A, A==B ? "==" : "!=", B);
           };
 
           const auto CheckGPRs = [&Matches, DumpGPRs](const std::string& Name, uint64_t A, uint64_t B) {

unittests/ASM/Disabled_Tests_host

@@ -7,3 +7,7 @@ Test_REPNE/F2_2B.asm
 
 # 3DNow! no longer exists on AMD Zen CPUs
 Test_TwoByte/0F_0E.asm
+
+# Intel doesn't support CLZero, which the CI uses
+Test_SecondaryModRM/Reg_7_4.asm
+Test_SecondaryModRM/Reg_7_4_2.asm

unittests/ASM/SecondaryModRM/Reg_7_4.asm

@@ -0,0 +1,36 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RBX": "0x0"
+  }
+}
+%endif
+
+; Starting address to store to
+mov rax, 0xe8000000
+
+; Set up the cacheline with garbage
+mov rbx, 0x4142434445464748
+mov [rax + 8 * 0], rbx
+mov [rax + 8 * 1], rbx
+mov [rax + 8 * 2], rbx
+mov [rax + 8 * 3], rbx
+mov [rax + 8 * 4], rbx
+mov [rax + 8 * 5], rbx
+mov [rax + 8 * 6], rbx
+mov [rax + 8 * 7], rbx
+
+clzero
+
+mov rbx, 0
+
+add rbx, [rax + 8 * 0]
+add rbx, [rax + 8 * 1]
+add rbx, [rax + 8 * 2]
+add rbx, [rax + 8 * 3]
+add rbx, [rax + 8 * 4]
+add rbx, [rax + 8 * 5]
+add rbx, [rax + 8 * 6]
+add rbx, [rax + 8 * 7]
+
+hlt

unittests/ASM/SecondaryModRM/Reg_7_4_2.asm

@@ -0,0 +1,91 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RBX": "0x0",
+    "RCX": "0x000000020A121A20",
+    "RDX": "0x0A121A2000000000"
+  }
+}
+%endif
+
+; Starting address to store to
+mov rax, 0xe8000000
+
+; Set up the cachelines with garbage
+
+; Cacheline 0
+mov rbx, 0x0000000041424344
+mov [rax + 8 * 0], rbx
+mov [rax + 8 * 1], rbx
+mov [rax + 8 * 2], rbx
+mov [rax + 8 * 3], rbx
+mov [rax + 8 * 4], rbx
+mov [rax + 8 * 5], rbx
+mov [rax + 8 * 6], rbx
+mov [rax + 8 * 7], rbx
+
+; Cacheline 1
+mov rbx, 0x5152535455565758
+mov [rax + 8 * 8], rbx
+mov [rax + 8 * 9], rbx
+mov [rax + 8 * 10], rbx
+mov [rax + 8 * 11], rbx ; clzero here
+mov [rax + 8 * 12], rbx
+mov [rax + 8 * 13], rbx
+mov [rax + 8 * 14], rbx
+mov [rax + 8 * 15], rbx
+
+; Cacheline 2
+mov rbx, 0x4142434400000000
+mov [rax + 8 * 16], rbx
+mov [rax + 8 * 17], rbx
+mov [rax + 8 * 18], rbx
+mov [rax + 8 * 19], rbx
+mov [rax + 8 * 20], rbx
+mov [rax + 8 * 21], rbx
+mov [rax + 8 * 22], rbx
+mov [rax + 8 * 23], rbx
+
+; Set RAX to the middle of cacheline 1 to ensure alignment
+lea rax, [rax + 8 * 11]
+
+clzero
+
+; Set rax back to the start
+mov rax, 0xe8000000
+
+mov rbx, 0
+mov rcx, 0
+mov rdx, 0
+
+; Cacheline 0 should be unmodified
+add rcx, [rax + 8 * 0]
+add rcx, [rax + 8 * 1]
+add rcx, [rax + 8 * 2]
+add rcx, [rax + 8 * 3]
+add rcx, [rax + 8 * 4]
+add rcx, [rax + 8 * 5]
+add rcx, [rax + 8 * 6]
+add rcx, [rax + 8 * 7]
+
+; Cacheline 1 Should be zero
+add rbx, [rax + 8 * 8]
+add rbx, [rax + 8 * 9]
+add rbx, [rax + 8 * 10]
+add rbx, [rax + 8 * 11]
+add rbx, [rax + 8 * 12]
+add rbx, [rax + 8 * 13]
+add rbx, [rax + 8 * 14]
+add rbx, [rax + 8 * 15]
+
+; Cacheline 2 should be unmodified
+add rdx, [rax + 8 * 16]
+add rdx, [rax + 8 * 17]
+add rdx, [rax + 8 * 18]
+add rdx, [rax + 8 * 19]
+add rdx, [rax + 8 * 20]
+add rdx, [rax + 8 * 21]
+add rdx, [rax + 8 * 22]
+add rdx, [rax + 8 * 23]
+
+hlt