FEXCore: Accurately store segment descriptors

Previously we were only storing the 32-bit base address which isn't
actually how segment descriptors work.

In reality segment descriptors are 64-bit descriptors that are laid out
in a particular layout depending on the 4-bit type value. In reality we
only care about code and data segment layouts since the rest are
bonkers.

Describe these descriptors correctly and setup a default code descriptor
for the operating mode that FEX is starting in.

FEXCore/Source/Interface/Core/Core.cpp

@@ -440,6 +440,22 @@ ContextImpl::CreateThread(uint64_t InitialRIP, uint64_t StackPointer, const FEXC
   Thread->CurrentFrame->State.gregs[X86State::REG_RSP] = StackPointer;
   Thread->CurrentFrame->State.rip = InitialRIP;
 
+  // Set up default code segment.
+  // Default code segment indexes match the numbers that the Linux kernel uses.
+  Thread->CurrentFrame->State.cs_idx = 6 << 3;
+  auto &GDT = Thread->CurrentFrame->State.gdt[Thread->CurrentFrame->State.cs_idx >> 3];
+  Thread->CurrentFrame->State.SetGDTBase(&GDT, 0);
+  Thread->CurrentFrame->State.SetGDTLimit(&GDT, 0xF'FFFFU);
+
+  if (Config.Is64BitMode) {
+    GDT.L = 1; // L = Long Mode = 64-bit
+    GDT.D = 0; // D = Default Operand SIze = Reserved
+  }
+  else {
+    GDT.L = 0; // L = Long Mode = 32-bit
+    GDT.D = 1; // D = Default Operand Size = 32-bit
+  }
+
   // Copy over the new thread state to the new object
   if (NewThreadState) {
     memcpy(&Thread->CurrentFrame->State, NewThreadState, sizeof(FEXCore::Core::CPUState));

FEXCore/include/FEXCore/Core/CoreState.h

@@ -121,9 +121,48 @@ struct CPUState {
   uint64_t mm[8][2] {};
 
   // 32bit x86 state
-  struct {
-    uint32_t base;
+  struct gdt_segment {
+    uint16_t Limit0;
+    uint16_t Base0;
+    uint16_t Base1  : 8;
+    uint16_t Type   : 4;
+    uint16_t S      : 1;
+    uint16_t DPL    : 2;
+    uint16_t P      : 1;
+    uint16_t Limit1 : 4;
+    uint16_t AVL    : 1;
+    uint16_t L      : 1;
+    uint16_t D      : 1;
+    uint16_t G      : 1;
+    uint16_t Base2  : 8;
   } gdt[32] {};
+
+  static uint32_t CalculateGDTBase(gdt_segment GDT) {
+    uint32_t Base{};
+    Base |= GDT.Base2 << 24;
+    Base |= GDT.Base1 << 16;
+    Base |= GDT.Base0;
+    return Base;
+  }
+
+  static uint32_t CalculateGDTLimit(gdt_segment GDT) {
+    uint32_t Limit{};
+    Limit |= GDT.Limit1 << 16;
+    Limit |= GDT.Limit0;
+    return Limit;
+  }
+
+  static void SetGDTBase(gdt_segment *GDT, uint32_t Base) {
+    GDT->Base0 = Base;
+    GDT->Base1 = Base >> 16;
+    GDT->Base2 = Base >> 24;
+  }
+
+  static void SetGDTLimit(gdt_segment *GDT, uint32_t Limit) {
+    GDT->Limit0 = Limit;
+    GDT->Limit1 = Limit >> 16;
+  }
+
   uint16_t FCW {0x37F};
   uint8_t AbridgedFTW {};
 
@@ -135,6 +174,7 @@ struct CPUState {
 
   static constexpr size_t FLAG_SIZE = sizeof(flags[0]);
   static constexpr size_t GDT_SIZE = sizeof(gdt[0]);
+  static_assert(GDT_SIZE == sizeof(uint64_t), "Segments required to be 8-byte in size.");
   static constexpr size_t GPR_REG_SIZE = sizeof(gregs[0]);
   static constexpr size_t XMM_AVX_REG_SIZE = sizeof(xmm.avx.data[0]);
   static constexpr size_t XMM_SSE_REG_SIZE = XMM_AVX_REG_SIZE / 2;

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator/GuestFramesManagement.cpp

@@ -199,12 +199,12 @@ void SignalDelegator::RestoreFrame_ia32(FEXCore::Core::InternalThreadState* Thre
     Frame->State.gs_idx = guest_uctx->sc.gs;
     Frame->State.ss_idx = guest_uctx->sc.ss;
 
-    Frame->State.cs_cached = Frame->State.gdt[Frame->State.cs_idx >> 3].base;
-    Frame->State.ds_cached = Frame->State.gdt[Frame->State.ds_idx >> 3].base;
-    Frame->State.es_cached = Frame->State.gdt[Frame->State.es_idx >> 3].base;
-    Frame->State.fs_cached = Frame->State.gdt[Frame->State.fs_idx >> 3].base;
-    Frame->State.gs_cached = Frame->State.gdt[Frame->State.gs_idx >> 3].base;
-    Frame->State.ss_cached = Frame->State.gdt[Frame->State.ss_idx >> 3].base;
+    Frame->State.cs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.cs_idx >> 3]);
+    Frame->State.ds_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.ds_idx >> 3]);
+    Frame->State.es_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.es_idx >> 3]);
+    Frame->State.fs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.fs_idx >> 3]);
+    Frame->State.gs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.gs_idx >> 3]);
+    Frame->State.ss_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.ss_idx >> 3]);
 
 #define COPY_REG(x, y) Frame->State.gregs[FEXCore::X86State::REG_##x] = guest_uctx->sc.y;
     COPY_REG(RDI, di);
@@ -272,12 +272,12 @@ void SignalDelegator::RestoreRTFrame_ia32(FEXCore::Core::InternalThreadState* Th
     Frame->State.gs_idx = guest_uctx->uc.uc_mcontext.gregs[FEXCore::x86::FEX_REG_GS];
     Frame->State.ss_idx = guest_uctx->uc.uc_mcontext.gregs[FEXCore::x86::FEX_REG_SS];
 
-    Frame->State.cs_cached = Frame->State.gdt[Frame->State.cs_idx >> 3].base;
-    Frame->State.ds_cached = Frame->State.gdt[Frame->State.ds_idx >> 3].base;
-    Frame->State.es_cached = Frame->State.gdt[Frame->State.es_idx >> 3].base;
-    Frame->State.fs_cached = Frame->State.gdt[Frame->State.fs_idx >> 3].base;
-    Frame->State.gs_cached = Frame->State.gdt[Frame->State.gs_idx >> 3].base;
-    Frame->State.ss_cached = Frame->State.gdt[Frame->State.ss_idx >> 3].base;
+    Frame->State.cs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.cs_idx >> 3]);
+    Frame->State.ds_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.ds_idx >> 3]);
+    Frame->State.es_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.es_idx >> 3]);
+    Frame->State.fs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.fs_idx >> 3]);
+    Frame->State.gs_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.gs_idx >> 3]);
+    Frame->State.ss_cached = Frame->State.CalculateGDTBase(Frame->State.gdt[Frame->State.ss_idx >> 3]);
 
 #define COPY_REG(x) Frame->State.gregs[FEXCore::X86State::REG_##x] = guest_uctx->uc.uc_mcontext.gregs[FEXCore::x86::FEX_REG_##x];
     COPY_REG(RDI);

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/Thread.cpp

@@ -54,8 +54,8 @@ uint64_t SetThreadArea(FEXCore::Core::CpuStateFrame* Frame, void* tls) {
   if (u_info->entry_number == -1) {
     for (uint32_t i = TLS_NextEntry; i < TLS_MaxEntry; ++i) {
       auto GDT = &Frame->State.gdt[i];
-      if (GDT->base == 0) {
-        // If the base is zero then it isn't present with our setup
+      if (Frame->State.CalculateGDTLimit(*GDT) == 0) {
+        // If the limit is zero then it isn't present with our setup
         u_info->entry_number = i;
         break;
       }
@@ -69,29 +69,30 @@ uint64_t SetThreadArea(FEXCore::Core::CpuStateFrame* Frame, void* tls) {
 
   // Now we need to update the thread's GDT to handle this change
   auto GDT = &Frame->State.gdt[u_info->entry_number];
-  GDT->base = u_info->base_addr;
+  Frame->State.SetGDTBase(GDT, u_info->base_addr);
+  Frame->State.SetGDTLimit(GDT, 0xF'FFFFU);
 
   // With the segment register optimization we need to check all of the segment registers and update.
   const auto GetEntry = [](auto value) {
     return value >> 3;
   };
   if (GetEntry(Frame->State.cs_idx) == u_info->entry_number) {
-    Frame->State.cs_cached = GDT->base;
+    Frame->State.cs_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   if (GetEntry(Frame->State.ds_idx) == u_info->entry_number) {
-    Frame->State.ds_cached = GDT->base;
+    Frame->State.ds_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   if (GetEntry(Frame->State.es_idx) == u_info->entry_number) {
-    Frame->State.es_cached = GDT->base;
+    Frame->State.es_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   if (GetEntry(Frame->State.fs_idx) == u_info->entry_number) {
-    Frame->State.fs_cached = GDT->base;
+    Frame->State.fs_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   if (GetEntry(Frame->State.gs_idx) == u_info->entry_number) {
-    Frame->State.gs_cached = GDT->base;
+    Frame->State.gs_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   if (GetEntry(Frame->State.ss_idx) == u_info->entry_number) {
-    Frame->State.ss_cached = GDT->base;
+    Frame->State.ss_cached = Frame->State.CalculateGDTBase(*GDT);
   }
   return 0;
 }
@@ -164,7 +165,7 @@ void RegisterThread(FEX::HLE::SyscallHandler* Handler) {
     memset(u_info, 0, sizeof(*u_info));
 
     // FEX only stores base instead of the full GDT
-    u_info->base_addr = GDT->base;
+    u_info->base_addr = Frame->State.CalculateGDTBase(*GDT);
 
     // Fill the rest of the structure with expected data (even if wrong at the moment)
     if (u_info->base_addr) {