FEXCore: Fixes bug with 32-bit adcx

When a 32-bit adcx instruction was encountered, it was getting treated
as a 16-bit adcx instruction instead. This is because of the 0x66 prefix
required to handle this instruction.

Adds a unit test to ensure it doesn't break again.

External/FEXCore/Source/Interface/Core/Frontend.cpp

@@ -820,6 +820,15 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
           }
 
           uint16_t LocalOp = (Prefix << 8) | ReadByte();
+
+          bool NoOverlay66 = (FEXCore::X86Tables::H0F38TableOps[LocalOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
+          if (DecodeInst->LastEscapePrefix == 0x66 && NoOverlay66) { // Operand Size
+            // Remove prefix so it doesn't effect calculations.
+            // This is only an escape prefix rather than modifier now
+            DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
+            DecodeFlags::PopOpAddrIf(&DecodeInst->Flags, DecodeFlags::FLAG_OPERAND_SIZE_LAST);
+          }
+
           return NormalOpHeader(&FEXCore::X86Tables::H0F38TableOps[LocalOp], LocalOp);
         break;
         }

External/FEXCore/Source/Interface/Core/X86Tables/H0F38Tables.cpp

@@ -111,7 +111,7 @@ void InitializeH0F38Tables() {
     {OPD(PF_38_66 | PF_38_F2,   0xF0), 1, X86InstInfo{"CRC32",      TYPE_INST, GenFlagsSizes(SIZE_DEF, SIZE_8BIT) | FLAGS_MODRM, 0, nullptr}},
     {OPD(PF_38_66 | PF_38_F2,   0xF1), 1, X86InstInfo{"CRC32",      TYPE_INST, FLAGS_MODRM, 0, nullptr}},
 
-    {OPD(PF_38_66,   0xF6), 1, X86InstInfo{"ADCX",       TYPE_INST, FLAGS_MODRM, 0, nullptr}},
+    {OPD(PF_38_66,   0xF6), 1, X86InstInfo{"ADCX",       TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66, 0, nullptr}},
     {OPD(PF_38_F3,   0xF6), 1, X86InstInfo{"ADOX",       TYPE_INST, FLAGS_MODRM, 0, nullptr}},
   };
 #undef OPD

unittests/ASM/FEX_bugs/adcx_size.asm

@@ -0,0 +1,41 @@
+%ifdef CONFIG
+{
+  "RegData": {
+    "RAX": "0x000000007ee544ac",
+    "RBX": "0x0fb22768a2cf00bb",
+    "RCX": "0x00000000e19be77f",
+    "RDX": "0x06726399b9f09d2f",
+    "RSI": "0xe544b42838dd404d",
+    "RDI": "0x6d78590ca1418bd1",
+    "RSP": "0x20bfe50ddcfce881",
+    "RBP": "0x56c870e2dcbf6522"
+  }
+}
+%endif
+
+mov rax, 0x6B11A609DC1643F1
+mov rbx, 0x0FB22768A2CF00BB
+mov rcx, 0x48E1BB8327AB4A4F
+mov rdx, 0x06726399B9F09D2F
+mov rsi, 0x77CC5B1B979BB47C
+mov rdi, 0x6D78590CA1418BD1
+mov rsp, 0xC9F7742B003D835E
+mov rbp, 0x56C870E2DCBF6522
+
+; 32-bit clc
+clc
+adcx eax, ebx
+
+; 32-bit stc
+stc
+adcx ecx, edx
+
+; 64-bit clc
+clc
+adcx rsi, rdi
+
+; 64-bit stc
+stc
+adcx rsp, rbp
+
+hlt