Hook more glibc hooks and allow clearing hooks in Allocator

Due to a disjoint mechanism inside of glibc we need to override both the glibc
publicly visible allocation functions AND the hooks.

We had broken the overriding when we changed the default visibility. So first fix that.
Then only override to FEX's allocators once we are ready for it.
Then also replace the hooks.

Additionally have a way to clear the hooks back to default.

External/FEXCore/Source/Utils/Allocator.cpp

@@ -3,6 +3,7 @@
 #include <sys/mman.h>
 #include <jemalloc/jemalloc.h>
 #include <memory>
+#include <malloc.h>
 
 extern "C" {
   extern void *__libc_malloc(size_t size);
@@ -16,6 +17,15 @@ extern "C" {
 
   extern mmap_hook_type __mmap_hook;
   extern munmap_hook_type __munmap_hook;
+
+  static FEXCore::Allocator::MALLOC_Hook global_malloc {::__libc_malloc};
+  static FEXCore::Allocator::REALLOC_Hook global_realloc {::__libc_realloc};
+  static FEXCore::Allocator::FREE_Hook global_free {::__libc_free};
+
+  // Override the global functions
+  FEX_DEFAULT_VISIBILITY void *malloc(size_t size) { return global_malloc(size); }
+  FEX_DEFAULT_VISIBILITY void *realloc(void *ptr, size_t size) { return global_realloc(ptr, size); }
+  FEX_DEFAULT_VISIBILITY void free(void *ptr) { return global_free(ptr); }
 }
 
 namespace FEXCore::Allocator {
@@ -25,6 +35,10 @@ namespace FEXCore::Allocator {
   REALLOC_Hook realloc {::__libc_realloc};
   FREE_Hook free {::__libc_free};
 
+  using GLIBC_MALLOC_Hook = void*(*)(size_t, const void *caller);
+  using GLIBC_REALLOC_Hook = void*(*)(void*, size_t, const void *caller);
+  using GLIBC_FREE_Hook = void(*)(void*, const void *caller);
+
   std::unique_ptr<Alloc::HostAllocator> Alloc64{};
 
   void *FEX_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
@@ -57,6 +71,8 @@ namespace FEXCore::Allocator {
     return ::je_free(ptr);
   }
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
   void SetupHooks() {
     Alloc64.reset(Alloc::OSAllocator::Create64BitAllocator());
     __mmap_hook   = FEX_mmap;
@@ -66,12 +82,37 @@ namespace FEXCore::Allocator {
     FEXCore::Allocator::malloc = ::je_malloc;
     FEXCore::Allocator::realloc = ::je_realloc;
     FEXCore::Allocator::free = ::je_free;
+
+    global_malloc = ::je_malloc;
+    global_realloc = ::je_realloc;
+    global_free = ::je_free;
+
+    __malloc_hook = FEXCore::Allocator::FEX_malloc_hook;
+    __realloc_hook = FEXCore::Allocator::FEX_realloc_hook;
+    __free_hook = FEXCore::Allocator::FEX_free_hook;
   }
+
+  void ClearHooks() {
+    __mmap_hook   = ::mmap;
+    __munmap_hook = ::munmap;
+    FEXCore::Allocator::mmap = ::mmap;
+    FEXCore::Allocator::munmap = ::munmap;
+    FEXCore::Allocator::malloc = ::__libc_malloc;
+    FEXCore::Allocator::realloc = ::__libc_realloc;
+    FEXCore::Allocator::free = ::__libc_free;
+
+    global_malloc = ::__libc_malloc;
+    global_realloc = ::__libc_realloc;
+    global_free = ::__libc_free;
+
+    // Reset's glibc hooks
+    __malloc_hook = 0;
+    __realloc_hook = 0;
+    __free_hook = 0;
+  }
+#pragma GCC diagnostic pop
+
 }
 
 extern "C" {
-  // Override the global functions
-  void *malloc(size_t size) { return FEXCore::Allocator::malloc(size); }
-  void *realloc(void *ptr, size_t size) { return FEXCore::Allocator::realloc(ptr, size); }
-  void free(void *ptr) { return FEXCore::Allocator::free(ptr); }
 }

External/FEXCore/include/FEXCore/Utils/Allocator.h

@@ -20,4 +20,5 @@ namespace FEXCore::Allocator {
   FEX_DEFAULT_VISIBILITY extern FREE_Hook free;
 
   void SetupHooks();
+  void ClearHooks();
 }