From b6417153d74cadb95c6640dd2ef5eaeb98526abf Mon Sep 17 00:00:00 2001
From: Ryan Houdek <Sonicadvance1@gmail.com>
Date: Wed, 9 Jun 2021 18:03:27 -0700
Subject: [PATCH] Hook more glibc hooks and allow clearing hooks in Allocator

Due to a disjoint mechanism inside of glibc we need to override both the glibc
publicly visible allocation functions AND the hooks.

We had broken the overriding when we changed the default visibility. So first fix that.
Then only override to FEX's allocators once we are ready for it.
Then also replace the hooks.

Additionally have a way to clear the hooks back to default.
---
 External/FEXCore/Source/Utils/Allocator.cpp   | 49 +++++++++++++++++--
 .../FEXCore/include/FEXCore/Utils/Allocator.h |  1 +
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/External/FEXCore/Source/Utils/Allocator.cpp b/External/FEXCore/Source/Utils/Allocator.cpp
index 713fd367d..d870f2220 100644
--- a/External/FEXCore/Source/Utils/Allocator.cpp
+++ b/External/FEXCore/Source/Utils/Allocator.cpp
@@ -3,6 +3,7 @@
 #include <sys/mman.h>
 #include <jemalloc/jemalloc.h>
 #include <memory>
+#include <malloc.h>
 
 extern "C" {
   extern void *__libc_malloc(size_t size);
@@ -16,6 +17,15 @@ extern "C" {
 
   extern mmap_hook_type __mmap_hook;
   extern munmap_hook_type __munmap_hook;
+
+  static FEXCore::Allocator::MALLOC_Hook global_malloc {::__libc_malloc};
+  static FEXCore::Allocator::REALLOC_Hook global_realloc {::__libc_realloc};
+  static FEXCore::Allocator::FREE_Hook global_free {::__libc_free};
+
+  // Override the global functions
+  FEX_DEFAULT_VISIBILITY void *malloc(size_t size) { return global_malloc(size); }
+  FEX_DEFAULT_VISIBILITY void *realloc(void *ptr, size_t size) { return global_realloc(ptr, size); }
+  FEX_DEFAULT_VISIBILITY void free(void *ptr) { return global_free(ptr); }
 }
 
 namespace FEXCore::Allocator {
@@ -25,6 +35,10 @@ namespace FEXCore::Allocator {
   REALLOC_Hook realloc {::__libc_realloc};
   FREE_Hook free {::__libc_free};
 
+  using GLIBC_MALLOC_Hook = void*(*)(size_t, const void *caller);
+  using GLIBC_REALLOC_Hook = void*(*)(void*, size_t, const void *caller);
+  using GLIBC_FREE_Hook = void(*)(void*, const void *caller);
+
   std::unique_ptr<Alloc::HostAllocator> Alloc64{};
 
   void *FEX_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) {
@@ -57,6 +71,8 @@ namespace FEXCore::Allocator {
     return ::je_free(ptr);
   }
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
   void SetupHooks() {
     Alloc64.reset(Alloc::OSAllocator::Create64BitAllocator());
     __mmap_hook   = FEX_mmap;
@@ -66,12 +82,37 @@ namespace FEXCore::Allocator {
     FEXCore::Allocator::malloc = ::je_malloc;
     FEXCore::Allocator::realloc = ::je_realloc;
     FEXCore::Allocator::free = ::je_free;
+
+    global_malloc = ::je_malloc;
+    global_realloc = ::je_realloc;
+    global_free = ::je_free;
+
+    __malloc_hook = FEXCore::Allocator::FEX_malloc_hook;
+    __realloc_hook = FEXCore::Allocator::FEX_realloc_hook;
+    __free_hook = FEXCore::Allocator::FEX_free_hook;
   }
+
+  void ClearHooks() {
+    __mmap_hook   = ::mmap;
+    __munmap_hook = ::munmap;
+    FEXCore::Allocator::mmap = ::mmap;
+    FEXCore::Allocator::munmap = ::munmap;
+    FEXCore::Allocator::malloc = ::__libc_malloc;
+    FEXCore::Allocator::realloc = ::__libc_realloc;
+    FEXCore::Allocator::free = ::__libc_free;
+
+    global_malloc = ::__libc_malloc;
+    global_realloc = ::__libc_realloc;
+    global_free = ::__libc_free;
+
+    // Reset's glibc hooks
+    __malloc_hook = 0;
+    __realloc_hook = 0;
+    __free_hook = 0;
+  }
+#pragma GCC diagnostic pop
+
 }
 
 extern "C" {
-  // Override the global functions
-  void *malloc(size_t size) { return FEXCore::Allocator::malloc(size); }
-  void *realloc(void *ptr, size_t size) { return FEXCore::Allocator::realloc(ptr, size); }
-  void free(void *ptr) { return FEXCore::Allocator::free(ptr); }
 }
diff --git a/External/FEXCore/include/FEXCore/Utils/Allocator.h b/External/FEXCore/include/FEXCore/Utils/Allocator.h
index 6f0fa42f8..ab8c1e1fe 100644
--- a/External/FEXCore/include/FEXCore/Utils/Allocator.h
+++ b/External/FEXCore/include/FEXCore/Utils/Allocator.h
@@ -20,4 +20,5 @@ namespace FEXCore::Allocator {
   FEX_DEFAULT_VISIBILITY extern FREE_Hook free;
 
   void SetupHooks();
+  void ClearHooks();
 }