FEXCore/Frontend: Ensure multiple prefix bytes work

Only the last prefix byte is retained when multiple are set. We were
accidentally generating a mask.

Additionally with 64-bit code, the legacy segment prefixes don't
overwrite if FS or GS have been set. So no weird behaviour where FS/GS
is set, a legacy prefix is used for padding, and then it "ignores" a bad
prefix by ignoring only the latest one.
This commit is contained in:
Ryan Houdek committed 2025-07-30 12:30:13 -07:00
1 parent 6724673482
commit d4dcbfa90b
4 files changed
+22 -23

No files matched your search

+6 -10
View File
@@ -918,28 +918,24 @@ bool Decoder::DecodeInstructionImpl(uint64_t PC) {
break;
case 0x26: // ES legacy prefix
if (!BlockInfo.Is64BitMode) {
DecodeInst->Flags |= DecodeFlags::FLAG_ES_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_ES_PREFIX;
}
break;
case 0x2E: // CS legacy prefix
if (!BlockInfo.Is64BitMode) {
DecodeInst->Flags |= DecodeFlags::FLAG_CS_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_CS_PREFIX;
}
break;
case 0x36: // SS legacy prefix
if (!BlockInfo.Is64BitMode) {
DecodeInst->Flags |= DecodeFlags::FLAG_SS_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_SS_PREFIX;
}
break;
case 0x3E: // DS legacy prefix
// Annoyingly GCC generates NOP ops with these prefixes
// Just ignore them for now
// eg. 66 2e 0f 1f 84 00 00 00 00 00 nop WORD PTR cs:[rax+rax*1+0x0]
if (!BlockInfo.Is64BitMode) {
DecodeInst->Flags |= DecodeFlags::FLAG_DS_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_DS_PREFIX;
}
break;
break;
case 0xF0: // LOCK prefix
DecodeInst->Flags |= DecodeFlags::FLAG_LOCK;
break;
@@ -952,10 +948,10 @@ bool Decoder::DecodeInstructionImpl(uint64_t PC) {
DecodeInst->LastEscapePrefix = Op;
break;
case 0x64: // FS prefix
DecodeInst->Flags |= DecodeFlags::FLAG_FS_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_FS_PREFIX;
break;
case 0x65: // GS prefix
DecodeInst->Flags |= DecodeFlags::FLAG_GS_PREFIX;
DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_GS_PREFIX;
break;
default:
[[likely]] { // Default base table
@@ -3916,17 +3916,17 @@ uint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {
Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
const auto GPRSize = GetGPROpSize();
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
if (Is64BitMode) {
if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
} else if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
} else if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
}
// If there was any other segment in 64bit then it is ignored
} else {
uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
if (!Prefix || Override) {
if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX || Override) {
// If there was no prefix then use the default one if available
// Or the argument only uses a specific prefix (with override set)
Prefix = DefaultPrefix;
@@ -3934,6 +3934,7 @@ Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool O
// With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
Ref SegmentResult {};
switch (Prefix) {
[[likely]] case FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX: return nullptr;
case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
break;
@@ -3952,7 +3953,7 @@ Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool O
case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
break;
default: return nullptr;
default: FEX_UNREACHABLE;
}
CheckLegacySegmentRead(SegmentResult, Prefix);
@@ -1513,7 +1513,7 @@ private:
#undef OpcodeArgs
Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);
void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);
@@ -32,13 +32,15 @@ constexpr uint32_t FLAG_REX_WIDENING = (1 << 7);
constexpr uint32_t FLAG_REX_XGPR_B = (1 << 8);
constexpr uint32_t FLAG_REX_XGPR_X = (1 << 9);
constexpr uint32_t FLAG_REX_XGPR_R = (1 << 10);
constexpr uint32_t FLAG_ES_PREFIX = (1 << 11);
constexpr uint32_t FLAG_CS_PREFIX = (1 << 12);
constexpr uint32_t FLAG_SS_PREFIX = (1 << 13);
constexpr uint32_t FLAG_DS_PREFIX = (1 << 14);
constexpr uint32_t FLAG_FS_PREFIX = (1 << 15);
constexpr uint32_t FLAG_GS_PREFIX = (1 << 16);
constexpr uint32_t FLAG_SEGMENTS = (0b11'1111 << 11);
constexpr uint32_t FLAG_NO_PREFIX = (0b000 << 11);
constexpr uint32_t FLAG_ES_PREFIX = (0b001 << 11);
constexpr uint32_t FLAG_CS_PREFIX = (0b010 << 11);
constexpr uint32_t FLAG_SS_PREFIX = (0b011 << 11);
constexpr uint32_t FLAG_DS_PREFIX = (0b100 << 11);
constexpr uint32_t FLAG_FS_PREFIX = (0b101 << 11);
constexpr uint32_t FLAG_GS_PREFIX = (0b110 << 11);
constexpr uint32_t FLAG_SEGMENTS = (0b111 << 11);
// Bits 14, 15, 16 - Unused
constexpr uint32_t FLAG_REP_PREFIX = (1 << 17);
constexpr uint32_t FLAG_REPNE_PREFIX = (1 << 18);