mirror of
https://github.com/Dingo-Shenanigans/ReSkate.git
synced 2026-10-26 06:56:45 +01:00
Descriptor heaps: dense custom maps no longer overflow the GPU's descriptor heap and crash it
The renderer gives each frame 131,072 shader-visible descriptors past a persistent 120,000, in heaps of 333,333. Dense custom maps (the Los Santos City sublevels, San Andreas) draw more than that in a frame, and the renderer wrote past the end of the heap: shaders read garbage descriptors, which faulted or hung the GPU (page faults at nonsense addresses, in draws, texture fetches and ray tracing alike). ReSkate now patches the device setup at boot so each heap is D3D12's 1,000,000 and a frame may use 524,288; heaps are 644,288 descriptors. LS-B2, LS-B3 and San Andreas (210,000 a frame at its peak) now hold. Also in this change: - Memory: perf memory owners/heap/gpu/report, a ledger of who commits the game's RAM; the render-resource pool no longer allocates its 115 MB twice; the EadpErrorsData arena is capped at 32 MiB (EA's error reporting is blocked anyway). - Crash reports: tagged with the map, destination, graphics card and driver version (protocol v3), so Sentry can group GPU hangs by map and card. - GPU diagnostics (--gpu-diagnostics): a fuller DRED report, the device-removal and video memory watches, a large-draw watch, a record of which game code made and freed each GPU heap and resource, checks of every GPU address bound, and a meter of each descriptor heap's use. - World pools: peak use of entity pages, render instances and spatial blocks in the log. - dingosdk_mod_mesh_check: checks every MeshSet a mod ships against its geometry; the MeshSet reader now knows static meshes' layout.
This commit is contained in:
AyeZeeBB committed
1 parent
67d84d5bd7
commit
164fdffbd0
34 files changed
+2968
-216
No files matched your search
@@ -229,6 +229,11 @@ void crash_check(const std::filesystem::path& root, bool native) {
|
||||
(native ? "game-native" : "unhandled-exception"), "Wrong capture source");
|
||||
check(Json::parse(read(dump.wstring() + L".json")).at("attachment.log").string() == "complete" &&
|
||||
read(log_attachment_path(dump)) == log, "Crash did not snapshot its session log");
|
||||
const auto tags = Json::parse(read(dump.wstring() + L".json"));
|
||||
check(tags.at("game.map").string() == "dingolevel_reskate_lsc_b2" &&
|
||||
tags.at("game.destination").string() == "Levels/Game/DingoLevel_Root/DingoLevel_Root + levels/game/dingolevel_reskate_lsc_b2/dingolevel_reskate_lsc_b2 @ DefaultStartPoint" &&
|
||||
tags.at("gpu.name").string() == "Fixture GPU" && tags.at("gpu.driver").string() == "32.0.21025.1",
|
||||
"Crash was not tagged with its level and graphics card");
|
||||
if (native) {
|
||||
const auto source = root / L"local" / L"ReSkate" / L"Game" / L"Skate" / L"CrashDumps";
|
||||
const auto pid = GetProcessId(child.value), created = process_creation_seconds(child.value);
|
||||
@@ -298,6 +303,8 @@ int wmain(int argc, wchar_t** argv) {
|
||||
logging::Options options; options.directory = argv[2]; options.truncate_file = true;
|
||||
check(logging::initialize(options), "Fixture logging failed");
|
||||
if (native) SetUnhandledExceptionFilter(native_child_handler); // Reproduce the game's later handler registration.
|
||||
set_destination(L"Levels/Game/DingoLevel_Root/DingoLevel_Root + levels/game/dingolevel_reskate_lsc_b2/dingolevel_reskate_lsc_b2 @ DefaultStartPoint");
|
||||
set_gpu(L"Fixture GPU", L"32.0.21025.1");
|
||||
RaiseException(test_exception, EXCEPTION_NONCONTINUABLE, 0, nullptr);
|
||||
return 3;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,11 @@ template<std::size_t N> bool copy(std::array<wchar_t, N>& target, const std::wst
|
||||
target[value.size()] = 0;
|
||||
return true;
|
||||
}
|
||||
template<std::size_t N> void copy_cut(std::array<wchar_t, N>& target, std::wstring_view value) noexcept {
|
||||
const auto size = std::min(value.size(), N - 1);
|
||||
std::copy_n(value.begin(), size, target.begin());
|
||||
target[size] = 0;
|
||||
}
|
||||
}
|
||||
StartResult start(const std::filesystem::path& log_directory) noexcept {
|
||||
try {
|
||||
@@ -109,6 +114,15 @@ void capture(EXCEPTION_POINTERS* exception) noexcept {
|
||||
WaitForMultipleObjects(2, wait_for, FALSE, capture_timeout_ms);
|
||||
}
|
||||
}
|
||||
void set_destination(std::wstring_view destination) noexcept {
|
||||
if (auto* const shared = report) copy_cut(shared->destination, destination);
|
||||
}
|
||||
void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept {
|
||||
if (auto* const shared = report) {
|
||||
copy_cut(shared->gpu_name, name);
|
||||
copy_cut(shared->gpu_driver, driver);
|
||||
}
|
||||
}
|
||||
void stop() noexcept {
|
||||
// An exception already using these handles owns them until process exit.
|
||||
// Never unmap its report or close its events from a concurrent shutdown.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
#include <Windows.h>
|
||||
#include <filesystem>
|
||||
#include <string_view>
|
||||
|
||||
namespace dingosdk::crash_report {
|
||||
enum class StartResult { disabled, ready, failed };
|
||||
@@ -9,4 +10,9 @@ StartResult start(const std::filesystem::path& log_directory) noexcept;
|
||||
void stop() noexcept;
|
||||
// Last-chance path: only signals the prestarted helper and waits at most 15 seconds.
|
||||
void capture(EXCEPTION_POINTERS* exception) noexcept;
|
||||
// The level being loaded ("root + sublevel @ start point") and the graphics card the game renders
|
||||
// on, for a report's tags: game.map, game.destination, gpu.name and gpu.driver. Nothing while
|
||||
// reporting is off; too long a value is cut short.
|
||||
void set_destination(std::wstring_view destination) noexcept;
|
||||
void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept;
|
||||
}
|
||||
@@ -4,7 +4,7 @@
|
||||
#include <string>
|
||||
|
||||
namespace dingosdk::crash_report {
|
||||
inline constexpr DWORD protocol_version = 2;
|
||||
inline constexpr DWORD protocol_version = 3;
|
||||
inline constexpr DWORD capture_timeout_ms = 15000;
|
||||
struct SharedReport {
|
||||
DWORD version{protocol_version};
|
||||
@@ -14,6 +14,12 @@ struct SharedReport {
|
||||
std::array<wchar_t, 32768> directory{};
|
||||
std::array<wchar_t, 260> application{};
|
||||
std::array<wchar_t, 32768> native_directory{};
|
||||
// What the game was doing, for the report's tags: the level it loaded last, and the graphics
|
||||
// card it renders on with its driver's version. Written by the game as they change, read by
|
||||
// the helper after a crash (a torn read only spoils a tag).
|
||||
std::array<wchar_t, 512> destination{};
|
||||
std::array<wchar_t, 128> gpu_name{};
|
||||
std::array<wchar_t, 32> gpu_driver{};
|
||||
};
|
||||
// Explicitly inherited unnamed handles; no token or paths on the command line.
|
||||
enum HandleIndex : std::size_t { mapping, requested, captured, stopping, ready, parent, handle_count };
|
||||
|
||||
@@ -33,6 +33,33 @@ inline bool peek_bytes(std::uintptr_t address, void* destination, std::size_t si
|
||||
template<class T> requires std::is_trivially_copyable_v<T>
|
||||
bool peek(std::uintptr_t address, T& value) noexcept { return peek_bytes(address, &value, sizeof(value)); }
|
||||
|
||||
// Fresh read/write memory within reach of a rel32 jump from an image at base: the first free
|
||||
// spot past its end, for a code stub that a patched instruction jumps to. Null when nothing is
|
||||
// free within 1.75 GB of the image.
|
||||
inline void* allocate_near_image(std::uintptr_t base, std::size_t image_size, std::size_t bytes) noexcept {
|
||||
SYSTEM_INFO information{};
|
||||
GetSystemInfo(&information);
|
||||
const auto granularity = static_cast<std::uintptr_t>(information.dwAllocationGranularity);
|
||||
const auto align_up = [granularity](std::uintptr_t value) { return (value + granularity - 1) & ~(granularity - 1); };
|
||||
auto cursor = align_up(base + image_size);
|
||||
const auto limit = base + 0x70000000ULL;
|
||||
while (cursor < limit) {
|
||||
MEMORY_BASIC_INFORMATION region{};
|
||||
if (!VirtualQuery(reinterpret_cast<const void*>(cursor), ®ion, sizeof(region))) break;
|
||||
const auto region_base = reinterpret_cast<std::uintptr_t>(region.BaseAddress);
|
||||
const auto region_end = region_base + region.RegionSize;
|
||||
if (region.State == MEM_FREE) {
|
||||
const auto candidate = align_up(cursor > region_base ? cursor : region_base);
|
||||
if (candidate <= region_end && region_end - candidate >= bytes)
|
||||
if (auto* const result = VirtualAlloc(reinterpret_cast<void*>(candidate), bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE))
|
||||
return result;
|
||||
}
|
||||
if (region_end <= cursor) break;
|
||||
cursor = align_up(region_end);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// A NUL-terminated string copied the same way: the length (terminator written at destination[length]),
|
||||
// or -1 if it is unreadable or has no terminator within capacity bytes. Reads byte by byte, so it never
|
||||
// touches memory past the terminator.
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// Reviewed native contract for supported_build::game_sha256 (September 29 2026).
|
||||
// The size of the renderer's shader-visible CBV/SRV/UAV descriptor heaps: a persistent part
|
||||
// (120,000) and, in each frame's heap, a part for that frame's descriptors: what is left of a
|
||||
// total of 333,333 (a third of D3D12's 1,000,000), capped at 131,072.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
|
||||
namespace dingosdk::game::build::v20260929::descriptor_heap_size {
|
||||
// In the renderer's device setup (0x0410e3f0), where the total each heap may have (333,333, from
|
||||
// the device or from the platform's own description, PQXD; both paths join here, at the first
|
||||
// instruction) is capped at 1,000,000 unless the device's resource binding is tier 3:
|
||||
// cmp dword ptr [rsi + 0x7a4], 3
|
||||
// je +0x0a ; past the cap
|
||||
// mov edx, 1000000 / cmp ecx, edx / cmova ecx, edx (left as they are)
|
||||
inline constexpr std::uintptr_t total_cap_rva = 0x0410eaa0;
|
||||
inline constexpr std::array<std::uint8_t, 9> total_cap{0x83, 0xbe, 0xa4, 0x07, 0x00, 0x00, 0x03, 0x74, 0x0a};
|
||||
// In the renderer's device setup (0x0410e3f0), the per-frame part: what the device allows past
|
||||
// the persistent part, divided among the heaps, then capped by the setting at desc+8
|
||||
// (0xffffffff: no cap):
|
||||
// mov ecx, dword ptr [r14 + 8] ; the cap
|
||||
// div r9d ; eax = room / heaps
|
||||
// mov edi, eax
|
||||
// mov eax, 0xffffffff
|
||||
// cmp ecx, eax / je / cmp edi, ecx / cmovb ecx, edi / mov edi, ecx (left as they are)
|
||||
// No branch lands inside these four instructions.
|
||||
inline constexpr std::uintptr_t cap_load_rva = 0x0410eaee;
|
||||
inline constexpr std::array<std::uint8_t, 14> cap_load{
|
||||
0x41, 0x8b, 0x4e, 0x08, 0x41, 0xf7, 0xf1, 0x8b, 0xf8, 0xb8, 0xff, 0xff, 0xff, 0xff};
|
||||
} // namespace dingosdk::game::build::v20260929::descriptor_heap_size
|
||||
@@ -0,0 +1,21 @@
|
||||
// Reviewed native contract for supported_build::game_sha256 (September 29 2026).
|
||||
// The arena EA's error reporting (EadpErrorsData) allocates from: 80 MiB, made
|
||||
// about seven seconds into startup, before the named settings can be changed.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
|
||||
namespace dingosdk::game::build::v20260929::eadp_errors_arena {
|
||||
// In the arena's factory (0x02318660): the load of its size from the settings
|
||||
// object (EadpErrorsData.EadpErrorsDataPlatformSpecificMemPoolSize) and the
|
||||
// vtable load after it, the two complete instructions moved into the stub.
|
||||
// mov eax, dword ptr [rax + 0x7c]
|
||||
// lea rcx, [rip + 0x03fffd17] ; 0x063183d8
|
||||
// No branch in the factory lands inside them; the factory goes on at resume_rva.
|
||||
inline constexpr std::uintptr_t size_load_rva = 0x023186b7;
|
||||
inline constexpr std::array<std::uint8_t, 10> size_load{
|
||||
0x8b, 0x40, 0x7c, 0x48, 0x8d, 0x0d, 0x17, 0xfd, 0xff, 0x03};
|
||||
inline constexpr std::uintptr_t vtable_rva = 0x063183d8;
|
||||
inline constexpr std::uintptr_t resume_rva = 0x023186c1;
|
||||
} // namespace dingosdk::game::build::v20260929::eadp_errors_arena
|
||||
@@ -90,6 +90,11 @@ inline constexpr std::ptrdiff_t resource_state_vector_offset = 0x2da920;
|
||||
inline constexpr std::ptrdiff_t relation_mask_vector_offset = 0x114ab60;
|
||||
|
||||
inline constexpr std::uintptr_t constructor_rva = 0x03e7f300;
|
||||
// The constructor is the base of two renderers. Only the one constructed here
|
||||
// (vtable 0x64f2c10) has the resource-state and relation-mask vectors: every
|
||||
// read of owner+0x2da920 and owner+0x114ab60 is in its code, and the other
|
||||
// (constructed at 0x030a0a46, vtable 0x6414168) keeps fields of its own there.
|
||||
inline constexpr std::uintptr_t state_owner_call_rva = 0x03f33a46;
|
||||
inline constexpr std::uintptr_t index_table_constructor_rva = 0x03e7e400;
|
||||
inline constexpr std::uintptr_t resource_flags_priority_rva = 0x03f56073;
|
||||
inline constexpr std::uintptr_t resource_flags_visibility_rva = 0x03f5789a;
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
// Reviewed native contract for supported_build::game_sha256 (September 29 2026).
|
||||
// Where the three world pools ReSkate enlarges (entity_pages.h, spatial_buckets.h)
|
||||
// can be read for how full they are: for telling whether a level needs more than
|
||||
// the game's own sizes.
|
||||
#pragma once
|
||||
|
||||
#include <array>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
|
||||
namespace dingosdk::game::build::v20260929::world_pool_usage {
|
||||
// Entity snapshots: the page allocator, (owner, allocator, pages). It fills the
|
||||
// snapshot's 128 page pointers at pages[0..127], each page zeroed when handed out,
|
||||
// so the highest non-zero entry is the highest entity slot in use.
|
||||
inline constexpr std::uintptr_t entity_page_allocator_rva = 0x048bd000;
|
||||
inline constexpr std::array<std::uint8_t, 10> entity_page_allocator_entry{
|
||||
0x40, 0x53, 0x55, 0x56, 0x57, 0x41, 0x56, 0x48, 0x83, 0xec};
|
||||
// The game's own entity slots (128 pages of 1,024).
|
||||
inline constexpr std::size_t entity_slots_stock = 131'072;
|
||||
|
||||
// Render instances: the pool is [[global] + 0x4a0] + 0x1300 (as the one caller of
|
||||
// its allocator loads it), and its next-index counter, taken with lock xadd, is
|
||||
// the most it has ever handed out.
|
||||
inline constexpr std::uintptr_t render_instance_load_rva = 0x050e59ac;
|
||||
inline constexpr std::array<std::uint8_t, 27> render_instance_load{
|
||||
0x48, 0x8b, 0x05, 0x25, 0xcf, 0x7c, 0x02, 0x48, 0x8b, 0xd9, 0x48, 0x8b, 0xd1, 0x48,
|
||||
0x8b, 0xa8, 0xa0, 0x04, 0x00, 0x00, 0x48, 0x8d, 0x8d, 0x00, 0x13, 0x00, 0x00};
|
||||
inline constexpr std::uintptr_t render_instance_owner_global = 0x078b28d8;
|
||||
inline constexpr std::ptrdiff_t render_instance_manager_offset = 0x4a0;
|
||||
inline constexpr std::ptrdiff_t render_instance_pool_offset = 0x1300;
|
||||
inline constexpr std::uintptr_t render_instance_counter_rva = 0x050e23ec;
|
||||
inline constexpr std::array<std::uint8_t, 8> render_instance_counter{
|
||||
0xf0, 0x0f, 0xc1, 0xb7, 0x08, 0x10, 0x00, 0x00}; // lock xadd [rdi+0x1008], esi
|
||||
inline constexpr std::ptrdiff_t render_instance_counter_offset = 0x1008;
|
||||
// The game's own render instances (512 pages of 512).
|
||||
inline constexpr std::size_t render_instances_stock = 262'144;
|
||||
|
||||
// Spatial buckets: a query, (root, ...), four register arguments; the root's node
|
||||
// count and each node's block count are at spatial_buckets.h's offsets.
|
||||
inline constexpr std::uintptr_t spatial_query_rva = 0x01967420;
|
||||
inline constexpr std::array<std::uint8_t, 15> spatial_query_entry{
|
||||
0x4c, 0x8b, 0xdc, 0x4d, 0x89, 0x4b, 0x20, 0x41, 0x55, 0x41, 0x56, 0x48, 0x83, 0xec, 0x68};
|
||||
inline constexpr std::ptrdiff_t spatial_nodes_offset = 0x130;
|
||||
// The game's own blocks per bucket (its root bucket overflowed at 2,049).
|
||||
inline constexpr std::size_t spatial_blocks_stock = 2'048;
|
||||
} // namespace dingosdk::game::build::v20260929::world_pool_usage
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "mesh_set.h"
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <stdexcept>
|
||||
@@ -131,6 +132,118 @@ std::vector<std::array<float, 3>> read_mesh_positions(std::span<const std::byte>
|
||||
return result;
|
||||
}
|
||||
|
||||
namespace {
|
||||
// The two layouts check_mesh_lod reads: the skinned one above, and the static one world meshes
|
||||
// have (192, 176, 384, 0), whose levels of detail are laid out the same. A static mesh's sections
|
||||
// are 384 or 416 bytes, and nothing found yet says which: check_mesh_lod takes the size under
|
||||
// which every section reads as one.
|
||||
struct Layout {
|
||||
bool skinned{};
|
||||
};
|
||||
Layout layout_of(const Reader& r) {
|
||||
const auto first = r.at<std::uint32_t>(0), second = r.at<std::uint32_t>(4), third = r.at<std::uint32_t>(8);
|
||||
if (first == 208 && second == 188 && third == 384) return {true};
|
||||
if (first == 192 && second == 176 && third == 384) return {false};
|
||||
require(false, "its layout is not one this reader knows");
|
||||
return {};
|
||||
}
|
||||
// Whether every section, `stride` bytes apart, reads as one: a triangle list whose indices are in
|
||||
// the index buffer and whose vertices start in the vertex buffer.
|
||||
bool sections_read(const Reader& r, std::size_t table, std::uint32_t sections, std::size_t stride, std::size_t index_size,
|
||||
std::size_t vertex_size, std::size_t index_total) {
|
||||
for (std::uint32_t section = 0; section < sections; ++section) {
|
||||
const auto s = table + section * stride;
|
||||
if (s + stride > r.data.size()) return false;
|
||||
const std::size_t triangles = r.at<std::uint32_t>(s + triangle_count);
|
||||
if (!triangles) continue;
|
||||
if (r.at<std::uint8_t>(s + primitive) != triangle_list || r.at<std::uint32_t>(s + vertex_start) >= vertex_size ||
|
||||
std::size_t{r.at<std::uint32_t>(s + first_index)} * index_size + triangles * 3 * index_size > index_total)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> check_mesh_lod(std::span<const std::byte> resource, std::size_t lod,
|
||||
std::span<const std::byte> geometry, std::uint32_t* format_out) {
|
||||
const Reader r{resource};
|
||||
const auto shape = layout_of(r);
|
||||
require(lod < r.at<std::uint16_t>(lod_count), "it has no such level of detail");
|
||||
const auto at = r.pointer(lod_table + lod * 8);
|
||||
const std::size_t vertex_size = r.at<std::uint32_t>(at + vertex_bytes), index_size_total = r.at<std::uint32_t>(at + index_bytes);
|
||||
const auto format = r.at<std::uint32_t>(at + index_format);
|
||||
if (format_out) *format_out = format;
|
||||
const std::size_t index_size = format == sixteen_bit_indices ? 2 : 4;
|
||||
std::vector<std::string> problems;
|
||||
const auto say = [&](std::string text) { if (problems.size() < 8) problems.push_back(std::move(text)); };
|
||||
if (geometry.size() < vertex_size + index_size_total)
|
||||
say("the geometry chunk holds " + std::to_string(geometry.size()) + " bytes of the " +
|
||||
std::to_string(vertex_size + index_size_total) + " its buffers need");
|
||||
const auto sections = r.at<std::uint32_t>(at + section_count);
|
||||
require(sections <= 4096, "it has too many sections");
|
||||
if (!sections) return problems;
|
||||
const auto table = r.pointer(at + section_table);
|
||||
std::size_t stride = section_size;
|
||||
if (!shape.skinned) {
|
||||
if (sections_read(r, table, sections, 384, index_size, vertex_size, index_size_total)) stride = 384;
|
||||
else if (sections_read(r, table, sections, 416, index_size, vertex_size, index_size_total)) stride = 416;
|
||||
else say("its sections read as neither 384 nor 416 bytes apart: one of them is out of its buffers");
|
||||
}
|
||||
for (std::uint32_t section = 0; section < sections; ++section) {
|
||||
const auto s = table + section * stride;
|
||||
const auto name = "section " + std::to_string(section);
|
||||
const auto type = r.at<std::uint8_t>(s + primitive);
|
||||
const std::size_t count = r.at<std::uint32_t>(s + vertex_count), triangles = r.at<std::uint32_t>(s + triangle_count);
|
||||
if (!triangles) continue;
|
||||
if (type != triangle_list) { say(name + " is primitive type " + std::to_string(type) + ", not a triangle list"); continue; }
|
||||
if (!count) { say(name + " has " + std::to_string(triangles) + " triangles and no vertices"); continue; }
|
||||
const std::size_t first = r.at<std::uint32_t>(s + vertex_start);
|
||||
if (first >= vertex_size) say(name + "'s vertices start at byte " + std::to_string(first) + " of a " + std::to_string(vertex_size) + "-byte vertex buffer");
|
||||
if (shape.skinned) {
|
||||
// (Where its vertices end: each stream's follow the one before.)
|
||||
const auto d = s + declaration;
|
||||
const auto streams = r.at<std::uint8_t>(d + stream_total);
|
||||
std::size_t end = first;
|
||||
for (std::size_t stream = 0; stream < streams && stream < 16; ++stream)
|
||||
end += std::size_t{r.at<std::uint8_t>(d + stream_table + stream * 2)} * count;
|
||||
if (end > vertex_size)
|
||||
say(name + "'s vertices end at byte " + std::to_string(end) + " of a " + std::to_string(vertex_size) + "-byte vertex buffer");
|
||||
}
|
||||
const std::size_t start = std::size_t{r.at<std::uint32_t>(s + first_index)} * index_size;
|
||||
const std::size_t length = triangles * 3 * index_size;
|
||||
if (start + length > index_size_total) {
|
||||
say(name + "'s indices end at byte " + std::to_string(start + length) + " of a " + std::to_string(index_size_total) + "-byte index buffer");
|
||||
continue;
|
||||
}
|
||||
const auto base = vertex_size + start;
|
||||
if (base + length > geometry.size()) continue;
|
||||
std::size_t outside{}, largest{};
|
||||
for (std::size_t i = 0; i < triangles * 3; ++i) {
|
||||
std::size_t index{};
|
||||
std::memcpy(&index, geometry.data() + base + i * index_size, index_size);
|
||||
if (index >= count) ++outside, largest = std::max(largest, index);
|
||||
}
|
||||
if (outside)
|
||||
say(name + ": " + std::to_string(outside) + " of " + std::to_string(triangles * 3) + " indices name a vertex outside its " +
|
||||
std::to_string(count) + " (the largest " + std::to_string(largest) + ")");
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
MeshLod any_mesh_lod_geometry(std::span<const std::byte> resource, std::size_t lod, std::size_t* lods) {
|
||||
const Reader r{resource};
|
||||
(void)layout_of(r);
|
||||
const std::size_t total = r.at<std::uint16_t>(lod_count);
|
||||
if (lods) *lods = total;
|
||||
require(lod < total, "it has no such level of detail");
|
||||
const auto at = r.pointer(lod_table + lod * 8);
|
||||
MeshLod result;
|
||||
result.chunk = r.at<Guid>(at + geometry_chunk);
|
||||
result.vertex_bytes = r.at<std::uint32_t>(at + vertex_bytes);
|
||||
result.index_bytes = r.at<std::uint32_t>(at + index_bytes);
|
||||
return result;
|
||||
}
|
||||
|
||||
SkinnedMesh read_skinned_mesh(std::span<const std::byte> resource, std::size_t lod, std::span<const std::byte> geometry) {
|
||||
const auto layout = read_mesh_lod(resource, lod);
|
||||
const Reader r{resource}, g{geometry};
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace dingosdk::frostbite {
|
||||
@@ -28,6 +29,15 @@ struct MeshLod {
|
||||
[[nodiscard]] std::vector<std::array<float, 3>> read_mesh_positions(std::span<const std::byte> resource, std::size_t lod,
|
||||
std::span<const std::byte> geometry);
|
||||
|
||||
// A level of detail's geometry chunk and buffer sizes, of either layout check_mesh_lod reads (the
|
||||
// skinned one, and world meshes' static one); `lods` gets how many levels there are.
|
||||
[[nodiscard]] MeshLod any_mesh_lod_geometry(std::span<const std::byte> resource, std::size_t lod, std::size_t* lods = nullptr);
|
||||
// What is wrong with a level of detail's geometry as the GPU would read it, a skinned or a static
|
||||
// MeshSet: a section whose vertices or indices run past their buffer, or a triangle that
|
||||
// names a vertex outside its section. Empty when it is sound. `index_format` gets the level's
|
||||
// index format (33: 16-bit; anything else is read as 32-bit). Throws for another layout.
|
||||
[[nodiscard]] std::vector<std::string> check_mesh_lod(std::span<const std::byte> resource, std::size_t lod,
|
||||
std::span<const std::byte> geometry, std::uint32_t* index_format = nullptr);
|
||||
struct SkinnedVertex {
|
||||
std::array<float, 3> position{}; // model space
|
||||
std::array<std::uint16_t, 8> bones{}; // skeleton bones
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
// The meshes a mod ships, checked as the GPU would read them (frostbite::check_mesh_lod): a
|
||||
// section whose vertices or indices run past their buffer, or a triangle naming a vertex outside
|
||||
// its section, sends a draw, or a ray traced against the mesh, to memory that is not the mesh's.
|
||||
// For finding what in a map hangs or faults the GPU when it loads.
|
||||
//
|
||||
// Arguments: <Skate folder> <mod folder> [<bundle filter>]. Every MeshSet the mod's own files
|
||||
// hold is read, each level of detail checked; what is wrong is listed by bundle and mesh, then
|
||||
// how many meshes each bundle has and how many are wrong.
|
||||
#include "Engine/Resource/binary_bundle.h"
|
||||
#include "Engine/Resource/cas_codec.h"
|
||||
#include "Engine/Resource/mesh_set.h"
|
||||
#include "Engine/Resource/toc.h"
|
||||
#include "Engine/Vfs/mod_merge_internal.h"
|
||||
#include <Windows.h>
|
||||
#include <algorithm>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
namespace fb = dingosdk::frostbite;
|
||||
using namespace dingosdk::mods::detail;
|
||||
|
||||
namespace {
|
||||
constexpr std::uint32_t mesh_set_type = 0x49b156d4;
|
||||
|
||||
struct Tally {
|
||||
std::size_t meshes{}, wrong{}, unreadable{};
|
||||
};
|
||||
}
|
||||
|
||||
int wmain(int argc, wchar_t** argv) {
|
||||
if (argc < 3) {
|
||||
std::cerr << "Usage: dingosdk_mod_mesh_check <Skate folder> <mod folder> [<bundle filter>]\n";
|
||||
return 2;
|
||||
}
|
||||
const fs::path game_root = argv[1], mod = argv[2];
|
||||
std::string filter;
|
||||
if (argc > 3) for (const wchar_t* c = argv[3]; *c; ++c) filter.push_back(static_cast<char>(*c));
|
||||
const auto base_root = game_root / L"Data";
|
||||
try {
|
||||
const auto layout = dingosdk::vfs::read_layout(base_root / L"layout.toc");
|
||||
// (Nothing is written: the store is only read through.)
|
||||
const auto scratch = fs::temp_directory_path() / L"reskate-mesh-check";
|
||||
CasStore store(base_root, scratch, layout.root);
|
||||
std::map<std::string, Tally> tallies;
|
||||
std::map<std::uint32_t, std::size_t> formats;
|
||||
for (const auto& relative : scan(mod).tocs) {
|
||||
fb::TocDocument own;
|
||||
try {
|
||||
own = fb::read_toc(read_file(mod / fs::path(relative)));
|
||||
} catch (const std::exception& error) {
|
||||
std::cout << "unreadable toc " << fs::path(relative).string() << ": " << error.what() << "\n";
|
||||
continue;
|
||||
}
|
||||
for (const auto& bundle : own.bundles) {
|
||||
if (!filter.empty() && bundle.name.find(filter) == std::string::npos) continue;
|
||||
std::optional<Listing> listing;
|
||||
try {
|
||||
listing = list_bundle(store, mod, base_root, bundle, game_root);
|
||||
} catch (const std::exception& error) {
|
||||
std::cout << "unreadable bundle " << bundle.name << ": " << error.what() << "\n";
|
||||
continue;
|
||||
}
|
||||
if (!listing) continue;
|
||||
const auto& manifest = listing->manifest;
|
||||
auto& tally = tallies[bundle.name];
|
||||
for (std::size_t index = 0; index < manifest.resources.size(); ++index) {
|
||||
const auto file = manifest.ebx.size() + index;
|
||||
const auto at = listing->first + file;
|
||||
if (manifest.resources[index].resourceType != mesh_set_type || at >= listing->files.size() ||
|
||||
!listing->files[at].location.patch)
|
||||
continue; // (the game's own meshes are not the mod's to check)
|
||||
const auto& name = manifest.resources[index].name;
|
||||
++tally.meshes;
|
||||
std::vector<std::byte> resource;
|
||||
std::size_t lods{};
|
||||
try {
|
||||
resource = read_asset(store, mod, base_root, *listing, file, game_root);
|
||||
(void)fb::any_mesh_lod_geometry(resource, 0, &lods);
|
||||
} catch (const std::exception& error) {
|
||||
++tally.unreadable;
|
||||
// (Each layout this reader does not know, once: its first words and size.)
|
||||
static std::map<std::string, bool> shown;
|
||||
std::string head;
|
||||
for (std::size_t word = 0; word < 16 && (word + 1) * 4 <= resource.size(); ++word) {
|
||||
std::uint32_t value{};
|
||||
std::memcpy(&value, resource.data() + word * 4, 4);
|
||||
head += std::to_string(value) + " ";
|
||||
}
|
||||
if (!shown[head.substr(0, 48)]) {
|
||||
shown[head.substr(0, 48)] = true;
|
||||
std::cout << "layout " << head << "(" << resource.size() << " bytes) e.g. " << name << ": " << error.what() << "\n";
|
||||
// (RESKATE_MESH_DUMP=1: the first one whole, eight bytes a line, for working out its layout.)
|
||||
static bool dumped{};
|
||||
if (!dumped && GetEnvironmentVariableW(L"RESKATE_MESH_DUMP", nullptr, 0)) {
|
||||
dumped = true;
|
||||
for (std::size_t offset = 0; offset + 8 <= resource.size(); offset += 8) {
|
||||
std::uint32_t low{}, high{};
|
||||
std::memcpy(&low, resource.data() + offset, 4);
|
||||
std::memcpy(&high, resource.data() + offset + 4, 4);
|
||||
std::cout << " +" << at << ": " << low << " " << high << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// (RESKATE_MESH_DUMP_NAME=<part of a name>: that mesh's resource, offset and 32-bit word a line.)
|
||||
if (wchar_t wanted_name[256]{}; GetEnvironmentVariableW(L"RESKATE_MESH_DUMP_NAME", wanted_name, 256)) {
|
||||
std::string part;
|
||||
for (const wchar_t* c = wanted_name; *c; ++c) part.push_back(static_cast<char>(*c));
|
||||
if (name.find(part) != std::string::npos)
|
||||
for (std::size_t offset = 0; offset + 4 <= resource.size(); offset += 4) {
|
||||
std::uint32_t value{};
|
||||
std::memcpy(&value, resource.data() + offset, 4);
|
||||
std::cout << "dump " << offset << " " << value << "\n";
|
||||
}
|
||||
}
|
||||
bool wrong{};
|
||||
for (std::size_t lod = 0; lod < lods; ++lod) {
|
||||
try {
|
||||
const auto wanted = fb::any_mesh_lod_geometry(resource, lod).chunk;
|
||||
std::vector<std::byte> geometry;
|
||||
for (std::size_t chunk = 0; chunk < manifest.chunks.size() && geometry.empty(); ++chunk)
|
||||
if (manifest.chunks[chunk].guid == wanted)
|
||||
geometry = read_asset(store, mod, base_root, *listing,
|
||||
manifest.ebx.size() + manifest.resources.size() + chunk, game_root);
|
||||
for (auto chunk = own.chunks.begin(); chunk != own.chunks.end() && geometry.empty(); ++chunk)
|
||||
if (chunk->guid == wanted && !chunk->removed)
|
||||
geometry = fb::decode_cas(store.read(chunk->location.patch ? mod : base_root, chunk->location,
|
||||
chunk->offset, chunk->size), {game_root});
|
||||
if (geometry.empty()) {
|
||||
std::cout << bundle.name << " | " << name << " | lod " << lod << " | its geometry chunk is in neither the bundle nor its superbundle\n";
|
||||
wrong = true;
|
||||
continue;
|
||||
}
|
||||
std::uint32_t format{};
|
||||
const auto problems = fb::check_mesh_lod(resource, lod, geometry, &format);
|
||||
// (RESKATE_MESH_STATS=1: each level's size, for comparing a map's meshes with another's:
|
||||
// sections, buffer bytes, and the most vertices and triangles a section has.)
|
||||
if (GetEnvironmentVariableW(L"RESKATE_MESH_STATS", nullptr, 0)) {
|
||||
const auto word = [&](std::size_t offset) {
|
||||
std::uint64_t value{};
|
||||
std::memcpy(&value, resource.data() + offset, 8);
|
||||
return value;
|
||||
};
|
||||
const auto lod_at = static_cast<std::size_t>(word(48 + lod * 8)) + 16;
|
||||
const auto sections = static_cast<std::uint32_t>(word(lod_at + 8));
|
||||
const auto table = static_cast<std::size_t>(word(lod_at + 12)) + 16;
|
||||
const auto sizes = fb::any_mesh_lod_geometry(resource, lod);
|
||||
// (Static meshes' sections are 384 or 416 bytes; the larger section counts read by either.)
|
||||
std::uint64_t most_vertices{}, most_triangles{};
|
||||
for (std::uint32_t section = 0; section < sections; ++section)
|
||||
for (const std::size_t stride : {384u, 416u}) {
|
||||
const auto at_section = table + section * stride;
|
||||
if (at_section + 48 > resource.size()) continue;
|
||||
std::uint32_t triangles{}, vertices{};
|
||||
std::memcpy(&triangles, resource.data() + at_section + 32, 4);
|
||||
std::memcpy(&vertices, resource.data() + at_section + 44, 4);
|
||||
if (vertices <= sizes.vertex_bytes && triangles * 6ull <= sizes.index_bytes + 6ull) {
|
||||
most_vertices = std::max<std::uint64_t>(most_vertices, vertices);
|
||||
most_triangles = std::max<std::uint64_t>(most_triangles, triangles);
|
||||
}
|
||||
}
|
||||
std::cout << "stats " << bundle.name << " " << name << " " << lod << " " << sections << " " << sizes.vertex_bytes
|
||||
<< " " << sizes.index_bytes << " " << most_vertices << " " << most_triangles << "\n";
|
||||
}
|
||||
++formats[format];
|
||||
for (const auto& problem : problems) std::cout << bundle.name << " | " << name << " | lod " << lod << " | " << problem << "\n";
|
||||
wrong = wrong || !problems.empty();
|
||||
} catch (const std::exception& error) {
|
||||
std::cout << bundle.name << " | " << name << " | lod " << lod << " | unreadable: " << error.what() << "\n";
|
||||
}
|
||||
}
|
||||
tally.wrong += wrong;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::cout << "\nBundle | meshes | wrong | unreadable\n";
|
||||
for (const auto& [name, tally] : tallies)
|
||||
if (tally.meshes) std::cout << name << " | " << tally.meshes << " | " << tally.wrong << " | " << tally.unreadable << "\n";
|
||||
std::cout << "Index formats seen:";
|
||||
for (const auto& [format, count] : formats) std::cout << " " << format << " (" << count << " levels)";
|
||||
std::cout << "\n";
|
||||
} catch (const std::exception& error) {
|
||||
std::cerr << error.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
#include "native_render_resource_pool.h"
|
||||
|
||||
#include "Engine/Core/Hooks/hooks.h"
|
||||
#include "Engine/Core/Log/logging.h"
|
||||
#include "Engine/Core/Platform/memory.h"
|
||||
#include "Engine/Game/Build/supported_build.h"
|
||||
#include "Engine/Game/Build/addresses.h"
|
||||
#include "Engine/Game/Build/20260929/render_resource_pool.h"
|
||||
|
||||
#include <Windows.h>
|
||||
#include <intrin.h>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
@@ -13,6 +16,7 @@
|
||||
#include <limits>
|
||||
#include <mutex>
|
||||
#include <span>
|
||||
#include <utility>
|
||||
|
||||
namespace dingosdk {
|
||||
namespace {
|
||||
@@ -61,6 +65,10 @@ static_assert(relation_mask_bytes == 0x00300000);
|
||||
using NativeConstructor = void* (__fastcall *)(void*, void*);
|
||||
using IndexTableConstructor = void* (__fastcall *)(void*, void*, void*);
|
||||
|
||||
// Records and metadata every renderer built on the constructor uses; the
|
||||
// resource states and relation masks only the state owner has
|
||||
// (contract::state_owner_call_rva). 115.5 MB of states went to the other
|
||||
// renderer as well, which never touches them.
|
||||
struct Buffers {
|
||||
std::byte* records{};
|
||||
std::uint32_t* metadata{};
|
||||
@@ -68,9 +76,8 @@ struct Buffers {
|
||||
std::byte* resource_states{};
|
||||
std::uint64_t* relation_masks{};
|
||||
|
||||
explicit operator bool() const noexcept {
|
||||
return records && metadata && resource_flags && resource_states && relation_masks;
|
||||
}
|
||||
explicit operator bool() const noexcept { return records && metadata && resource_flags; }
|
||||
bool has_states() const noexcept { return resource_states && relation_masks; }
|
||||
};
|
||||
|
||||
struct IndexTableBuffers {
|
||||
@@ -89,6 +96,10 @@ std::array<IndexTableBuffers, 2> reserved_index_tables{};
|
||||
std::size_t next_reserved_index_table{};
|
||||
std::byte* resource_flag_stubs{};
|
||||
std::uintptr_t installed_base{};
|
||||
// Where the state owner's constructor call returns to in the game.
|
||||
std::atomic<std::uintptr_t> state_owner_return{};
|
||||
// The renderer that has resource states, once constructed.
|
||||
std::atomic<std::uintptr_t> state_owner{};
|
||||
|
||||
bool read_memory(const void* source, void* destination, std::size_t size) noexcept {
|
||||
SIZE_T count{};
|
||||
@@ -122,13 +133,22 @@ bool validate_contract(std::uintptr_t base) noexcept {
|
||||
contract::resource_flags_read_priority) ||
|
||||
!matches(base + contract::resource_flags_visibility_rva,
|
||||
contract::resource_flags_read_visibility)) return false;
|
||||
// The state owner's call to the constructor: an E8 call whose target is it.
|
||||
std::array<std::uint8_t, 5> call{};
|
||||
std::int32_t displacement{};
|
||||
if (!read_memory(reinterpret_cast<const void*>(base + contract::state_owner_call_rva), call.data(), call.size()) ||
|
||||
call[0] != 0xe8) return false;
|
||||
std::memcpy(&displacement, call.data() + 1, sizeof(displacement));
|
||||
if (contract::state_owner_call_rva + call.size() + displacement != contract::constructor_rva) return false;
|
||||
for (const auto& patch : contract::patches)
|
||||
if (patch.expected.size() != patch.replacement.size() ||
|
||||
!matches(base + patch.rva, patch.expected)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
Buffers allocate_buffers() noexcept {
|
||||
void free_buffers(Buffers& buffers) noexcept;
|
||||
|
||||
Buffers allocate_buffers(bool states) noexcept {
|
||||
Buffers result{};
|
||||
result.records = static_cast<std::byte*>(VirtualAlloc(
|
||||
nullptr, record_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
@@ -140,19 +160,13 @@ Buffers allocate_buffers() noexcept {
|
||||
return {};
|
||||
}
|
||||
result.resource_flags = reinterpret_cast<std::uint8_t*>(result.metadata) + metadata_bytes;
|
||||
if (!states) return result;
|
||||
result.resource_states = static_cast<std::byte*>(VirtualAlloc(
|
||||
nullptr, resource_state_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
if (!result.resource_states) {
|
||||
VirtualFree(result.metadata, 0, MEM_RELEASE);
|
||||
VirtualFree(result.records, 0, MEM_RELEASE);
|
||||
return {};
|
||||
}
|
||||
result.relation_masks = static_cast<std::uint64_t*>(VirtualAlloc(
|
||||
result.relation_masks = !result.resource_states ? nullptr : static_cast<std::uint64_t*>(VirtualAlloc(
|
||||
nullptr, relation_mask_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE));
|
||||
if (!result.relation_masks) {
|
||||
VirtualFree(result.resource_states, 0, MEM_RELEASE);
|
||||
VirtualFree(result.metadata, 0, MEM_RELEASE);
|
||||
VirtualFree(result.records, 0, MEM_RELEASE);
|
||||
if (!result.has_states()) {
|
||||
free_buffers(result);
|
||||
return {};
|
||||
}
|
||||
return result;
|
||||
@@ -195,16 +209,34 @@ void free_reserved_allocations() noexcept {
|
||||
next_reserved_index_table = 0;
|
||||
}
|
||||
|
||||
Buffers take_buffers() noexcept {
|
||||
Buffers take_buffers(bool states) noexcept {
|
||||
std::lock_guard lock(allocation_mutex);
|
||||
// Two renderers are built on the constructor: the first takes the reserved
|
||||
// records and metadata, the state owner the reserved states, whichever
|
||||
// comes first. Another instance costs an isolated allocation of its own
|
||||
// and avoids aliasing records.
|
||||
Buffers result{};
|
||||
if (reserved_buffers) {
|
||||
const auto result = reserved_buffers;
|
||||
reserved_buffers = {};
|
||||
result.records = std::exchange(reserved_buffers.records, nullptr);
|
||||
result.metadata = std::exchange(reserved_buffers.metadata, nullptr);
|
||||
result.resource_flags = std::exchange(reserved_buffers.resource_flags, nullptr);
|
||||
} else {
|
||||
result = allocate_buffers(false);
|
||||
if (!result) return {};
|
||||
}
|
||||
if (!states) return result;
|
||||
if (reserved_buffers.has_states()) {
|
||||
result.resource_states = std::exchange(reserved_buffers.resource_states, nullptr);
|
||||
result.relation_masks = std::exchange(reserved_buffers.relation_masks, nullptr);
|
||||
return result;
|
||||
}
|
||||
// The renderer is normally a singleton. Supporting a subsequent native
|
||||
// instance costs another isolated allocation and avoids aliasing records.
|
||||
return allocate_buffers();
|
||||
const auto fresh = allocate_buffers(true);
|
||||
if (!fresh) { free_buffers(result); return {}; }
|
||||
result.resource_states = fresh.resource_states;
|
||||
result.relation_masks = fresh.relation_masks;
|
||||
VirtualFree(fresh.metadata, 0, MEM_RELEASE);
|
||||
VirtualFree(fresh.records, 0, MEM_RELEASE);
|
||||
return result;
|
||||
}
|
||||
|
||||
IndexTableBuffers take_index_table_buffers() noexcept {
|
||||
@@ -259,9 +291,15 @@ void* __fastcall construct_index_table(void* self, void* first, void* second) {
|
||||
|
||||
void* __fastcall construct_native_renderer(void* self, void* configuration) {
|
||||
const auto incoming_error = GetLastError();
|
||||
const auto buffers = take_buffers();
|
||||
// (The hook is jumped to, so this returns where the game's call does.)
|
||||
const auto caller = reinterpret_cast<std::uintptr_t>(_ReturnAddress());
|
||||
const bool states = caller == state_owner_return.load(std::memory_order_acquire);
|
||||
const auto buffers = take_buffers(states);
|
||||
const auto original = original_constructor.load(std::memory_order_acquire);
|
||||
if (!self || !buffers || !original) allocation_failure();
|
||||
if (!self || !buffers || (states && !buffers.has_states()) || !original) allocation_failure();
|
||||
logging::log(logging::Level::info, logging::Channel::graphics,
|
||||
"Render-resource pool: renderer built from Skate.exe+{:#x}, {}.", caller - installed_base,
|
||||
states ? "with the enlarged resource states" : "without resource states (it has none)");
|
||||
|
||||
auto* const record_end = buffers.records + record_bytes;
|
||||
auto* const metadata_end = buffers.metadata + native_render_resource_capacity;
|
||||
@@ -272,8 +310,10 @@ void* __fastcall construct_native_renderer(void* self, void* configuration) {
|
||||
store_field(self, vector_begin_offset, buffers.records);
|
||||
store_field(self, metadata_begin_offset, buffers.metadata);
|
||||
store_field(self, metadata_end_offset, metadata_end);
|
||||
store_field(self, resource_state_vector_offset, buffers.resource_states);
|
||||
store_field(self, relation_mask_vector_offset, relation_mask_begin);
|
||||
if (states) {
|
||||
store_field(self, resource_state_vector_offset, buffers.resource_states);
|
||||
store_field(self, relation_mask_vector_offset, relation_mask_begin);
|
||||
}
|
||||
SetLastError(incoming_error);
|
||||
auto* const result = original(self, configuration);
|
||||
|
||||
@@ -281,6 +321,7 @@ void* __fastcall construct_native_renderer(void* self, void* configuration) {
|
||||
// stock build. Keep the external values authoritative across construction.
|
||||
store_field(self, metadata_begin_offset, buffers.metadata);
|
||||
store_field(self, metadata_end_offset, metadata_end);
|
||||
if (states) state_owner.store(reinterpret_cast<std::uintptr_t>(self), std::memory_order_release);
|
||||
if (load_field<std::byte*>(self, vector_begin_offset) != buffers.records ||
|
||||
load_field<std::byte*>(self, vector_capacity_offset) != record_end ||
|
||||
load_field<std::byte*>(self, vector_end_offset) < buffers.records ||
|
||||
@@ -314,34 +355,6 @@ bool relative_jump(std::uintptr_t instruction, std::uintptr_t destination,
|
||||
return true;
|
||||
}
|
||||
|
||||
std::byte* allocate_near_game_image(std::uintptr_t base) noexcept {
|
||||
SYSTEM_INFO information{};
|
||||
GetSystemInfo(&information);
|
||||
const auto granularity = static_cast<std::uintptr_t>(information.dwAllocationGranularity);
|
||||
const auto align_up = [granularity](std::uintptr_t value) {
|
||||
return (value + granularity - 1) & ~(granularity - 1);
|
||||
};
|
||||
auto cursor = align_up(base + supported_build::game_image_size);
|
||||
const auto limit = base + 0x70000000ULL;
|
||||
while (cursor < limit) {
|
||||
MEMORY_BASIC_INFORMATION region{};
|
||||
if (!VirtualQuery(reinterpret_cast<const void*>(cursor), ®ion, sizeof(region))) break;
|
||||
const auto region_base = reinterpret_cast<std::uintptr_t>(region.BaseAddress);
|
||||
const auto region_end = region_base + region.RegionSize;
|
||||
if (region.State == MEM_FREE) {
|
||||
const auto candidate = align_up(cursor > region_base ? cursor : region_base);
|
||||
if (candidate <= region_end && region_end - candidate >= resource_flag_stub_page_bytes) {
|
||||
if (auto* const result = static_cast<std::byte*>(VirtualAlloc(
|
||||
reinterpret_cast<void*>(candidate), resource_flag_stub_page_bytes,
|
||||
MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE))) return result;
|
||||
}
|
||||
}
|
||||
if (region_end <= cursor) break;
|
||||
cursor = align_up(region_end);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template<std::size_t N>
|
||||
bool write_stub(std::byte* destination, std::span<const std::uint8_t, N> prefix,
|
||||
std::uintptr_t return_address) noexcept {
|
||||
@@ -357,7 +370,8 @@ bool write_stub(std::byte* destination, std::span<const std::uint8_t, N> prefix,
|
||||
}
|
||||
|
||||
bool prepare_resource_flag_stubs(std::uintptr_t base) noexcept {
|
||||
resource_flag_stubs = allocate_near_game_image(base);
|
||||
resource_flag_stubs = static_cast<std::byte*>(
|
||||
memory::allocate_near_image(base, supported_build::game_image_size, resource_flag_stub_page_bytes));
|
||||
if (!resource_flag_stubs) return false;
|
||||
std::memset(resource_flag_stubs, 0xcc, resource_flag_stub_page_bytes);
|
||||
if (!write_stub(resource_flag_stubs + priority_stub_offset,
|
||||
@@ -431,6 +445,33 @@ void restore_patches(std::uintptr_t base, std::size_t count) noexcept {
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void report_native_render_resources() noexcept {
|
||||
const auto owner = state_owner.load(std::memory_order_acquire);
|
||||
std::uintptr_t begin{}, end{};
|
||||
if (!owner || !memory::peek(owner + vector_begin_offset, begin) || !memory::peek(owner + vector_end_offset, end) ||
|
||||
end < begin || end > begin + record_bytes)
|
||||
return;
|
||||
const auto used = (end - begin) / native_render_resource_record_size;
|
||||
static std::size_t said{};
|
||||
static bool warned{}, full{};
|
||||
// (A new high, in steps of 1/32: how close a level came is worth having in a log.)
|
||||
if (used < said + native_render_resource_capacity / 32 && !(used >= native_render_resource_capacity && !full)) return;
|
||||
said = used;
|
||||
const auto percent = used * 100 / native_render_resource_capacity;
|
||||
if (used >= native_render_resource_capacity) {
|
||||
full = true;
|
||||
logging::log(logging::Level::error, logging::Channel::graphics,
|
||||
"Render-resource pool: all {} records are taken; the level needs more than the game can address.", used);
|
||||
} else if (percent >= 90 && !warned) {
|
||||
warned = true;
|
||||
logging::log(logging::Level::warning, logging::Channel::graphics, "Render-resource pool: {} of {} records taken ({}%).", used,
|
||||
native_render_resource_capacity, percent);
|
||||
} else {
|
||||
logging::log(logging::Level::info, logging::Channel::graphics, "Render-resource pool: {} of {} records taken ({}%).", used,
|
||||
native_render_resource_capacity, percent);
|
||||
}
|
||||
}
|
||||
|
||||
bool start_native_render_resource_pool(std::uintptr_t base, std::string& error) {
|
||||
std::lock_guard lock(install_mutex);
|
||||
error.clear();
|
||||
@@ -446,7 +487,7 @@ bool start_native_render_resource_pool(std::uintptr_t base, std::string& error)
|
||||
|
||||
{
|
||||
std::lock_guard allocation_lock(allocation_mutex);
|
||||
reserved_buffers = allocate_buffers();
|
||||
reserved_buffers = allocate_buffers(true);
|
||||
for (auto& table : reserved_index_tables) table = allocate_index_table_buffers();
|
||||
next_reserved_index_table = 0;
|
||||
if (!reserved_buffers || !reserved_index_tables[0] || !reserved_index_tables[1]) {
|
||||
@@ -520,6 +561,7 @@ bool start_native_render_resource_pool(std::uintptr_t base, std::string& error)
|
||||
return false;
|
||||
}
|
||||
|
||||
state_owner_return.store(base + contract::state_owner_call_rva + 5, std::memory_order_release);
|
||||
original_constructor.store(original, std::memory_order_release);
|
||||
original_index_table_constructor.store(original_index, std::memory_order_release);
|
||||
const auto queued_index = hook_queue_enable(index_target);
|
||||
|
||||
@@ -14,4 +14,9 @@ inline constexpr std::size_t native_render_resource_record_size = 0xf0;
|
||||
// constructing its records normally, but its backing storage and all parallel
|
||||
// per-resource arrays are redirected to process-lifetime allocations.
|
||||
bool start_native_render_resource_pool(std::uintptr_t base, std::string& error);
|
||||
|
||||
// How many of the pool's records the renderer that has resource states has taken: a level that
|
||||
// needs more than native_render_resource_capacity cannot be drawn. Said in the log as it grows,
|
||||
// checked every frame so the last count before a crash is there. Game thread.
|
||||
void report_native_render_resources() noexcept;
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,61 @@
|
||||
#include "descriptor_heap_size.h"
|
||||
|
||||
#include "Engine/Core/Platform/memory.h"
|
||||
#include "Engine/Game/Build/addresses.h"
|
||||
#include "Engine/Game/Build/supported_build.h"
|
||||
#include "Engine/Game/Build/20260929/descriptor_heap_size.h"
|
||||
|
||||
#include <Windows.h>
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
|
||||
namespace dingosdk {
|
||||
namespace {
|
||||
namespace contract = addr::descriptor_heap_size;
|
||||
|
||||
bool supported_image(std::uintptr_t base) noexcept {
|
||||
IMAGE_DOS_HEADER dos{};
|
||||
IMAGE_NT_HEADERS64 nt{};
|
||||
return base && memory::read(base, dos) && dos.e_magic == IMAGE_DOS_SIGNATURE && dos.e_lfanew > 0 &&
|
||||
dos.e_lfanew <= 0x1000 && memory::read(base + dos.e_lfanew, nt) && nt.Signature == IMAGE_NT_SIGNATURE &&
|
||||
nt.OptionalHeader.SizeOfImage == supported_build::game_image_size;
|
||||
}
|
||||
|
||||
bool write_code(std::uintptr_t address, const void* bytes, std::size_t size) noexcept {
|
||||
auto* const destination = reinterpret_cast<void*>(address);
|
||||
DWORD previous{};
|
||||
if (!VirtualProtect(destination, size, PAGE_EXECUTE_READWRITE, &previous)) return false;
|
||||
std::memcpy(destination, bytes, size);
|
||||
const bool flushed = FlushInstructionCache(GetCurrentProcess(), destination, size) != FALSE;
|
||||
DWORD discarded{};
|
||||
return VirtualProtect(destination, size, previous, &discarded) != FALSE && flushed;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool start_descriptor_heap_size(std::uintptr_t base, std::string& error) {
|
||||
std::array<std::uint8_t, contract::cap_load.size()> actual{};
|
||||
std::array<std::uint8_t, contract::total_cap.size()> total{};
|
||||
if (!supported_image(base) || !memory::read_bytes(base + contract::cap_load_rva, actual.data(), actual.size()) ||
|
||||
actual != contract::cap_load || !memory::read_bytes(base + contract::total_cap_rva, total.data(), total.size()) ||
|
||||
total != contract::total_cap) {
|
||||
error = "the renderer's descriptor heap setup does not match the supported game";
|
||||
return false;
|
||||
}
|
||||
// The same divide, with the cap a constant instead of the setting (which is never set):
|
||||
// div r9d / mov edi, eax / mov ecx, cap / or eax, -1 / nop
|
||||
std::array<std::uint8_t, contract::cap_load.size()> code{
|
||||
0x41, 0xf7, 0xf1, 0x8b, 0xf8, 0xb9, 0, 0, 0, 0, 0x83, 0xc8, 0xff, 0x90};
|
||||
std::memcpy(code.data() + 6, &frame_descriptor_cap, 4);
|
||||
// The total, a constant, then capped at 1,000,000 as before: mov ecx, total / nop (4 bytes).
|
||||
std::array<std::uint8_t, contract::total_cap.size()> load{0xb9, 0, 0, 0, 0, 0x0f, 0x1f, 0x40, 0x00};
|
||||
std::memcpy(load.data() + 1, &heap_descriptor_total, 4);
|
||||
if (!write_code(base + contract::cap_load_rva, code.data(), code.size()) ||
|
||||
!write_code(base + contract::total_cap_rva, load.data(), load.size())) {
|
||||
(void)write_code(base + contract::cap_load_rva, contract::cap_load.data(), contract::cap_load.size());
|
||||
(void)write_code(base + contract::total_cap_rva, contract::total_cap.data(), contract::total_cap.size());
|
||||
error = "the heap sizes could not be patched";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace dingosdk {
|
||||
// Each frame's share of the renderer's shader-visible descriptor heap is capped at 131,072
|
||||
// descriptors, in a heap of 333,333. A dense level draws more than that in a frame (the Los Santos
|
||||
// City sublevels with more than about 65,000 mesh sections), and the renderer then writes past
|
||||
// the end of the heap: shaders read descriptors that are not there and fault or hang the GPU.
|
||||
// Each heap is allowed D3D12's 1,000,000 instead, and a frame's share is capped at this, which
|
||||
// makes each frame's heap 644,288 descriptors.
|
||||
inline constexpr std::uint32_t heap_descriptor_total = 1'000'000;
|
||||
inline constexpr std::uint32_t frame_descriptor_cap = 524'288;
|
||||
|
||||
// Installs before executable entry, while the game is suspended.
|
||||
bool start_descriptor_heap_size(std::uintptr_t base, std::string& error);
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,93 @@
|
||||
#include "eadp_errors_arena.h"
|
||||
|
||||
#include "Engine/Core/Platform/memory.h"
|
||||
#include "Engine/Game/Build/addresses.h"
|
||||
#include "Engine/Game/Build/supported_build.h"
|
||||
#include "Engine/Game/Build/20260929/eadp_errors_arena.h"
|
||||
|
||||
#include <Windows.h>
|
||||
#include <array>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
|
||||
namespace dingosdk {
|
||||
namespace {
|
||||
namespace contract = addr::eadp_errors_arena;
|
||||
|
||||
constexpr std::size_t stub_page_bytes = 0x1000;
|
||||
|
||||
bool supported_image(std::uintptr_t base) noexcept {
|
||||
IMAGE_DOS_HEADER dos{};
|
||||
IMAGE_NT_HEADERS64 nt{};
|
||||
return base && memory::read(base, dos) && dos.e_magic == IMAGE_DOS_SIGNATURE && dos.e_lfanew > 0 &&
|
||||
dos.e_lfanew <= 0x1000 && memory::read(base + dos.e_lfanew, nt) && nt.Signature == IMAGE_NT_SIGNATURE &&
|
||||
nt.OptionalHeader.SizeOfImage == supported_build::game_image_size;
|
||||
}
|
||||
|
||||
bool rel32(std::uintptr_t next_instruction, std::uintptr_t target, std::int32_t& displacement) noexcept {
|
||||
const auto delta = static_cast<std::int64_t>(target) - static_cast<std::int64_t>(next_instruction);
|
||||
if (delta < std::numeric_limits<std::int32_t>::min() || delta > std::numeric_limits<std::int32_t>::max()) return false;
|
||||
displacement = static_cast<std::int32_t>(delta);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool write_code(std::uintptr_t address, const void* bytes, std::size_t size) noexcept {
|
||||
auto* const destination = reinterpret_cast<void*>(address);
|
||||
DWORD previous{};
|
||||
if (!VirtualProtect(destination, size, PAGE_EXECUTE_READWRITE, &previous)) return false;
|
||||
std::memcpy(destination, bytes, size);
|
||||
const bool flushed = FlushInstructionCache(GetCurrentProcess(), destination, size) != FALSE;
|
||||
DWORD discarded{};
|
||||
return VirtualProtect(destination, size, previous, &discarded) != FALSE && flushed;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool start_eadp_errors_arena(std::uintptr_t base, std::string& error) {
|
||||
std::array<std::uint8_t, contract::size_load.size()> actual{};
|
||||
if (!supported_image(base) || !memory::read_bytes(base + contract::size_load_rva, actual.data(), actual.size()) ||
|
||||
actual != contract::size_load) {
|
||||
error = "the arena's factory does not match the supported game";
|
||||
return false;
|
||||
}
|
||||
auto* const stub = static_cast<std::uint8_t*>(memory::allocate_near_image(base, supported_build::game_image_size, stub_page_bytes));
|
||||
if (!stub) {
|
||||
error = "no room for its stub near the game";
|
||||
return false;
|
||||
}
|
||||
const auto at = reinterpret_cast<std::uintptr_t>(stub);
|
||||
// The two moved instructions, with the size capped between them:
|
||||
// mov eax, [rax+0x7c] / cmp eax, cap / jbe +5 / mov eax, cap / lea rcx, [vtable] / jmp resume
|
||||
std::array<std::uint8_t, 27> code{0x8b, 0x40, 0x7c, 0x3d, 0, 0, 0, 0, 0x76, 0x05, 0xb8, 0, 0, 0, 0,
|
||||
0x48, 0x8d, 0x0d, 0, 0, 0, 0, 0xe9, 0, 0, 0, 0};
|
||||
std::memcpy(code.data() + 4, &eadp_errors_arena_bytes, 4);
|
||||
std::memcpy(code.data() + 11, &eadp_errors_arena_bytes, 4);
|
||||
std::int32_t vtable{}, resume{}, into{};
|
||||
if (!rel32(at + 22, base + contract::vtable_rva, vtable) || !rel32(at + 27, base + contract::resume_rva, resume) ||
|
||||
!rel32(base + contract::size_load_rva + 5, at, into)) {
|
||||
VirtualFree(stub, 0, MEM_RELEASE);
|
||||
error = "its stub is out of reach of the game";
|
||||
return false;
|
||||
}
|
||||
std::memcpy(code.data() + 18, &vtable, 4);
|
||||
std::memcpy(code.data() + 23, &resume, 4);
|
||||
std::memset(stub, 0xcc, stub_page_bytes);
|
||||
std::memcpy(stub, code.data(), code.size());
|
||||
DWORD previous{};
|
||||
if (!VirtualProtect(stub, stub_page_bytes, PAGE_EXECUTE_READ, &previous) ||
|
||||
!FlushInstructionCache(GetCurrentProcess(), stub, stub_page_bytes)) {
|
||||
VirtualFree(stub, 0, MEM_RELEASE);
|
||||
error = "its stub could not be made executable";
|
||||
return false;
|
||||
}
|
||||
std::array<std::uint8_t, contract::size_load.size()> jump{0xe9};
|
||||
std::memcpy(jump.data() + 1, &into, 4);
|
||||
std::memset(jump.data() + 5, 0x90, jump.size() - 5);
|
||||
if (!write_code(base + contract::size_load_rva, jump.data(), jump.size())) {
|
||||
(void)write_code(base + contract::size_load_rva, contract::size_load.data(), contract::size_load.size());
|
||||
VirtualFree(stub, 0, MEM_RELEASE);
|
||||
error = "the size load could not be redirected";
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,17 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace dingosdk {
|
||||
// The arena EA's error reporting allocates from is 80 MiB, made at startup and kept in RAM, of
|
||||
// which about 10 MB was in use: reporting is off (ea_service_block, and the EadpErrorsData
|
||||
// settings in client_tick.cpp), so its reports are never built. It is made before those settings
|
||||
// can be changed, so its size is capped where the game reads it instead. A smaller configured
|
||||
// size is left as it is.
|
||||
inline constexpr std::uint32_t eadp_errors_arena_bytes = 32u << 20;
|
||||
|
||||
// Installs before executable entry, while the game is suspended.
|
||||
bool start_eadp_errors_arena(std::uintptr_t base, std::string& error);
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,812 @@
|
||||
#include "Extension/Console/memory_ledger.h"
|
||||
#include "Engine/Core/Hooks/hooks.h"
|
||||
#include "Engine/Core/Log/logging.h"
|
||||
#include <Windows.h>
|
||||
#include <Psapi.h>
|
||||
#include <dxgi1_4.h>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <format>
|
||||
#include <fstream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace dingosdk::memory_ledger {
|
||||
namespace {
|
||||
constexpr std::size_t depth = 14;
|
||||
constexpr std::size_t caller_slots = 8192;
|
||||
constexpr unsigned block_bits = 18;
|
||||
constexpr std::size_t block_slots = std::size_t{1} << block_bits;
|
||||
constexpr std::uint32_t gone = 0xffffffff;
|
||||
|
||||
struct Caller {
|
||||
std::uint64_t hash{};
|
||||
std::array<void*, depth> frames{};
|
||||
unsigned frame_count{};
|
||||
// What was asked for while `perf memory trace` was on: reservations and commits of the
|
||||
// traced sizes. smallest/largest: the sizes of those.
|
||||
std::uint64_t asked_count{}, asked_bytes{}, smallest{}, largest{};
|
||||
};
|
||||
// A reservation the process still holds and who made it. caller: an index into callers plus
|
||||
// one; 0 is an empty slot, `gone` one whose reservation was given back (kept so a search
|
||||
// for one further along still finds it).
|
||||
struct Block {
|
||||
std::uintptr_t base{};
|
||||
std::uint32_t caller{};
|
||||
std::uint32_t made{}; // seconds after the ledger started
|
||||
};
|
||||
|
||||
// Never held while allocating: the heap may grow and come back through the hook on the same
|
||||
// thread. What is copied out under it goes into storage allocated before.
|
||||
// (None of this is ever destroyed: the hooks stay on while the process exits, and what other
|
||||
// modules give back then still passes them.)
|
||||
std::mutex& mutex = *new std::mutex;
|
||||
std::vector<Caller>& callers = *new std::vector<Caller>(caller_slots);
|
||||
std::vector<Block>& blocks = *new std::vector<Block>(block_slots);
|
||||
std::size_t blocks_live{}, blocks_gone{};
|
||||
std::uint64_t untracked{}; // reservations the tables had no room for
|
||||
std::atomic<bool> running{}, counting{};
|
||||
std::atomic<std::uint64_t> least{1}, most{UINT64_MAX};
|
||||
std::atomic<std::uint64_t> began{};
|
||||
std::mutex& start_mutex = *new std::mutex;
|
||||
bool prepared{};
|
||||
|
||||
using Allocate = LONG (NTAPI*)(HANDLE, PVOID*, ULONG_PTR, PSIZE_T, ULONG, ULONG);
|
||||
using AllocateEx = LONG (NTAPI*)(HANDLE, PVOID*, PSIZE_T, ULONG, ULONG, void*, ULONG);
|
||||
using Free = LONG (NTAPI*)(HANDLE, PVOID*, PSIZE_T, ULONG);
|
||||
Allocate original{};
|
||||
AllocateEx original_ex{};
|
||||
Free original_free{};
|
||||
|
||||
std::size_t block_slot(std::uintptr_t base) noexcept {
|
||||
return static_cast<std::size_t>(((base >> 16) * 0x9e3779b97f4a7c15ull) >> (64 - block_bits));
|
||||
}
|
||||
// Under the lock. The search is bounded: a hook cannot take long.
|
||||
void remember(std::uintptr_t base, std::uint32_t caller, std::uint32_t made) noexcept {
|
||||
std::size_t free_at = block_slots;
|
||||
for (std::size_t probe = 0; probe < 2048; ++probe) {
|
||||
const auto at = (block_slot(base) + probe) & (block_slots - 1);
|
||||
auto& block = blocks[at];
|
||||
if (!block.caller) { if (free_at == block_slots) free_at = at; break; }
|
||||
if (block.caller == gone) { if (free_at == block_slots) free_at = at; continue; }
|
||||
// (A release that was not seen: the same base is made again.)
|
||||
if (block.base == base) { block.caller = caller; block.made = made; return; }
|
||||
}
|
||||
if (free_at == block_slots || (!blocks[free_at].caller && (blocks_live + blocks_gone) * 8 >= block_slots * 7)) {
|
||||
++untracked;
|
||||
return;
|
||||
}
|
||||
if (blocks[free_at].caller == gone) --blocks_gone;
|
||||
blocks[free_at] = {base, caller, made};
|
||||
++blocks_live;
|
||||
}
|
||||
void forget(std::uintptr_t base) noexcept {
|
||||
for (std::size_t probe = 0; probe < 2048; ++probe) {
|
||||
auto& block = blocks[(block_slot(base) + probe) & (block_slots - 1)];
|
||||
if (!block.caller) return;
|
||||
if (block.caller != gone && block.base == base) {
|
||||
block.caller = gone;
|
||||
--blocks_live;
|
||||
++blocks_gone;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
const Block* find(const std::vector<Block>& table, std::uintptr_t base) noexcept {
|
||||
for (std::size_t probe = 0; probe < 2048; ++probe) {
|
||||
const auto& block = table[(block_slot(base) + probe) & (block_slots - 1)];
|
||||
if (!block.caller) return nullptr;
|
||||
if (block.caller != gone && block.base == base) return █
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
// Under the lock: the caller's index plus one, made if new; 0 when the table is full.
|
||||
std::uint32_t caller_of(std::uint64_t hash, const std::array<void*, depth>& frames, unsigned count) noexcept {
|
||||
for (std::size_t probe = 0; probe < 128; ++probe) {
|
||||
const auto at = (hash + probe) % caller_slots;
|
||||
auto& caller = callers[at];
|
||||
if (caller.hash && caller.hash != hash) continue;
|
||||
if (!caller.hash) {
|
||||
caller = {};
|
||||
caller.hash = hash;
|
||||
caller.frames = frames;
|
||||
caller.frame_count = count;
|
||||
}
|
||||
return static_cast<std::uint32_t>(at + 1);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
// This process, by its pseudo-handle or a real handle to it (which some code opens for itself).
|
||||
bool own(HANDLE process) noexcept {
|
||||
return process == GetCurrentProcess() || (process && GetProcessId(process) == GetCurrentProcessId());
|
||||
}
|
||||
|
||||
// (Not inlined: the stack is captured from a known depth, two frames under the caller of
|
||||
// NtAllocateVirtualMemory.)
|
||||
__declspec(noinline) void note(HANDLE process, void* base, SIZE_T asked, ULONG type) noexcept {
|
||||
if (!running.load(std::memory_order_relaxed) || !(type & (MEM_COMMIT | MEM_RESERVE)) || !own(process))
|
||||
return;
|
||||
const bool reserve = (type & MEM_RESERVE) != 0;
|
||||
const bool counted = counting.load(std::memory_order_relaxed) && asked >= least.load(std::memory_order_relaxed) &&
|
||||
asked <= most.load(std::memory_order_relaxed);
|
||||
if (!reserve && !counted) return; // (a commit inside a reservation that is already noted)
|
||||
std::array<void*, depth> frames{};
|
||||
const unsigned count = CaptureStackBackTrace(2, static_cast<DWORD>(depth), frames.data(), nullptr);
|
||||
std::uint64_t hash = 1469598103934665603ull;
|
||||
for (unsigned i = 0; i < count; ++i) hash = hash * 1099511628211ull ^ reinterpret_cast<std::uintptr_t>(frames[i]);
|
||||
if (!hash) hash = 1;
|
||||
const auto made = static_cast<std::uint32_t>((GetTickCount64() - began.load(std::memory_order_relaxed)) / 1000);
|
||||
std::lock_guard lock(mutex);
|
||||
const auto index = caller_of(hash, frames, count);
|
||||
if (!index) { if (reserve) ++untracked; return; }
|
||||
if (counted) {
|
||||
auto& caller = callers[index - 1];
|
||||
caller.smallest = caller.asked_count ? std::min<std::uint64_t>(caller.smallest, asked) : asked;
|
||||
caller.largest = std::max<std::uint64_t>(caller.largest, asked);
|
||||
++caller.asked_count;
|
||||
caller.asked_bytes += asked;
|
||||
}
|
||||
if (reserve && base) remember(reinterpret_cast<std::uintptr_t>(base), index, made);
|
||||
}
|
||||
LONG NTAPI hooked(HANDLE process, PVOID* base, ULONG_PTR zero_bits, PSIZE_T size, ULONG type, ULONG protect) {
|
||||
const auto asked = size ? *size : 0;
|
||||
const auto status = original(process, base, zero_bits, size, type, protect);
|
||||
if (status >= 0) note(process, base ? *base : nullptr, asked, type);
|
||||
return status;
|
||||
}
|
||||
// The newer call for the same (VirtualAlloc2, which graphics drivers and the D3D12 runtime use).
|
||||
LONG NTAPI hooked_ex(HANDLE process, PVOID* base, PSIZE_T size, ULONG type, ULONG protect, void* parameters, ULONG parameter_count) {
|
||||
const auto asked = size ? *size : 0;
|
||||
const auto status = original_ex(process, base, size, type, protect, parameters, parameter_count);
|
||||
if (status >= 0) note(process, base ? *base : nullptr, asked, type);
|
||||
return status;
|
||||
}
|
||||
LONG NTAPI hooked_free(HANDLE process, PVOID* base, PSIZE_T size, ULONG type) {
|
||||
const auto at = base ? reinterpret_cast<std::uintptr_t>(*base) : 0;
|
||||
const auto status = original_free(process, base, size, type);
|
||||
// (A placeholder split or joined is still reserved: MEM_COALESCE/PRESERVE_PLACEHOLDER.)
|
||||
if (status >= 0 && at && (type & MEM_RELEASE) && !(type & 0x3) && running.load(std::memory_order_relaxed) &&
|
||||
own(process)) {
|
||||
std::lock_guard lock(mutex);
|
||||
forget(at);
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
void* ntdll_function(const char* name) {
|
||||
return reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), name));
|
||||
}
|
||||
|
||||
std::string megabytes(std::uint64_t bytes) {
|
||||
return bytes >= (1ull << 30) ? std::format("{:.2f} GB", static_cast<double>(bytes) / (1ull << 30))
|
||||
: std::format("{:.1f} MB", static_cast<double>(bytes) / (1ull << 20));
|
||||
}
|
||||
std::string age(std::uint32_t seconds) {
|
||||
return seconds < 120 ? std::format("{} s", seconds) : seconds < 7200 ? std::format("{} min", seconds / 60)
|
||||
: std::format("{:.1f} h", seconds / 3600.0);
|
||||
}
|
||||
|
||||
// ---- reading what the process holds ----
|
||||
struct Range {
|
||||
std::uintptr_t base{};
|
||||
std::uint64_t bytes{};
|
||||
DWORD protect{};
|
||||
};
|
||||
struct Allocation {
|
||||
std::uint64_t committed{}, reserved{}; // reserved: the whole reservation, committed or not
|
||||
DWORD reserved_as{}; // the protection it was reserved with
|
||||
DWORD committed_as{}; // every protection its committed parts have, together
|
||||
std::vector<Range> ranges; // what of it is committed (kept only when asked for)
|
||||
};
|
||||
struct Held {
|
||||
std::map<std::uintptr_t, Allocation> private_memory;
|
||||
std::uint64_t committed{}, mapped{}, image{};
|
||||
};
|
||||
Held read_held(bool keep_ranges) {
|
||||
Held held;
|
||||
SYSTEM_INFO system{};
|
||||
GetSystemInfo(&system);
|
||||
auto at = reinterpret_cast<std::uintptr_t>(system.lpMinimumApplicationAddress);
|
||||
const auto end = reinterpret_cast<std::uintptr_t>(system.lpMaximumApplicationAddress);
|
||||
MEMORY_BASIC_INFORMATION info{};
|
||||
while (at < end && VirtualQuery(reinterpret_cast<void*>(at), &info, sizeof(info)) == sizeof(info)) {
|
||||
if (info.State != MEM_FREE) {
|
||||
if (info.Type == MEM_PRIVATE) {
|
||||
auto& allocation = held.private_memory[reinterpret_cast<std::uintptr_t>(info.AllocationBase)];
|
||||
allocation.reserved += info.RegionSize;
|
||||
allocation.reserved_as = info.AllocationProtect;
|
||||
if (info.State == MEM_COMMIT) {
|
||||
allocation.committed += info.RegionSize;
|
||||
allocation.committed_as |= info.Protect;
|
||||
held.committed += info.RegionSize;
|
||||
if (keep_ranges)
|
||||
allocation.ranges.push_back({reinterpret_cast<std::uintptr_t>(info.BaseAddress), info.RegionSize, info.Protect});
|
||||
}
|
||||
} else if (info.State == MEM_COMMIT) {
|
||||
(info.Type == MEM_IMAGE ? held.image : held.mapped) += info.RegionSize;
|
||||
}
|
||||
}
|
||||
at = reinterpret_cast<std::uintptr_t>(info.BaseAddress) + info.RegionSize;
|
||||
}
|
||||
return held;
|
||||
}
|
||||
|
||||
struct Copy {
|
||||
std::vector<Caller> callers;
|
||||
std::vector<Block> blocks;
|
||||
std::uint64_t untracked{};
|
||||
std::size_t live{};
|
||||
};
|
||||
Copy copy_tables() {
|
||||
Copy copy;
|
||||
copy.callers.resize(caller_slots);
|
||||
copy.blocks.resize(block_slots);
|
||||
std::lock_guard lock(mutex);
|
||||
std::copy(callers.begin(), callers.end(), copy.callers.begin());
|
||||
std::copy(blocks.begin(), blocks.end(), copy.blocks.begin());
|
||||
copy.untracked = untracked;
|
||||
copy.live = blocks_live;
|
||||
return copy;
|
||||
}
|
||||
|
||||
// The module an address is in, by name; empty when it is in none.
|
||||
struct Module {
|
||||
std::string name;
|
||||
std::uintptr_t base{};
|
||||
};
|
||||
Module module_at(void* address) {
|
||||
HMODULE module{};
|
||||
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
||||
reinterpret_cast<LPCWSTR>(address), &module) || !module)
|
||||
return {};
|
||||
wchar_t path[MAX_PATH]{};
|
||||
GetModuleFileNameW(module, path, MAX_PATH);
|
||||
const wchar_t* name = path;
|
||||
for (const wchar_t* c = path; *c; ++c) if (*c == L'\\') name = c + 1;
|
||||
Module result{{}, reinterpret_cast<std::uintptr_t>(module)};
|
||||
for (; *name; ++name) result.name += static_cast<char>(*name < 128 ? *name : '?');
|
||||
return result;
|
||||
}
|
||||
std::string place(void* address) {
|
||||
const auto module = module_at(address);
|
||||
if (module.name.empty()) return std::format("{:#x}", reinterpret_cast<std::uintptr_t>(address));
|
||||
return std::format("{}+{:#x}", module.name, reinterpret_cast<std::uintptr_t>(address) - module.base);
|
||||
}
|
||||
std::string lower(std::string text) {
|
||||
for (auto& c : text) c = static_cast<char>(c >= 'A' && c <= 'Z' ? c + 32 : c);
|
||||
return text;
|
||||
}
|
||||
// Who a caller's memory belongs to: the first module on its stack that is not Windows' or the
|
||||
// C runtime's way to memory. Through a heap when Windows' heap asked (the heap grew for
|
||||
// whoever was allocating when it ran out).
|
||||
std::string owner_of(const Caller& caller) {
|
||||
static const std::array<std::string_view, 7> passing{"ntdll.dll", "kernelbase.dll", "kernel32.dll", "ucrtbase.dll",
|
||||
"vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"};
|
||||
bool heap{};
|
||||
for (unsigned i = 0; i < caller.frame_count; ++i) {
|
||||
const auto module = module_at(caller.frames[i]);
|
||||
const auto name = lower(module.name);
|
||||
if (i == 0 && name == "ntdll.dll") heap = true;
|
||||
if (name.empty() || std::ranges::find(passing, name) != passing.end()) continue;
|
||||
return heap ? module.name + " (through a heap)" : module.name;
|
||||
}
|
||||
return heap ? "Windows (a heap)" : "Windows";
|
||||
}
|
||||
// "read/write, write-combined": what a page protection allows, in words.
|
||||
std::string protection(DWORD protect) {
|
||||
std::string text;
|
||||
const auto add = [&](std::string_view word) { text += (text.empty() ? "" : "+") + std::string(word); };
|
||||
constexpr std::pair<DWORD, std::string_view> kinds[] = {{PAGE_NOACCESS, "no access"}, {PAGE_READONLY, "read"},
|
||||
{PAGE_READWRITE, "read/write"}, {PAGE_WRITECOPY, "write-copy"}, {PAGE_EXECUTE, "execute"},
|
||||
{PAGE_EXECUTE_READ, "execute/read"}, {PAGE_EXECUTE_READWRITE, "execute/read/write"},
|
||||
{PAGE_EXECUTE_WRITECOPY, "execute/write-copy"}};
|
||||
for (const auto& [bit, word] : kinds) if (protect & bit) add(word);
|
||||
if (protect & PAGE_GUARD) text += ", guard";
|
||||
if (protect & PAGE_NOCACHE) text += ", uncached";
|
||||
if (protect & PAGE_WRITECOMBINE) text += ", write-combined";
|
||||
return text.empty() ? std::format("{:#x}", protect) : text;
|
||||
}
|
||||
|
||||
std::string stack_of(const Caller& caller) {
|
||||
std::string line;
|
||||
for (unsigned i = 0; i < caller.frame_count; ++i) line += " < " + place(caller.frames[i]);
|
||||
return line;
|
||||
}
|
||||
|
||||
constexpr std::string_view unseen = "not seen being made (before the ledger started, or by Windows itself: thread stacks)";
|
||||
|
||||
// How much of an allocation is in RAM, and of a few of those pages, how many hold only zeros.
|
||||
struct Use {
|
||||
std::uint64_t resident{};
|
||||
unsigned sampled{}, zero{};
|
||||
};
|
||||
Use use_of(const Allocation& allocation, std::vector<PSAPI_WORKING_SET_EX_INFORMATION>& pages) {
|
||||
Use use;
|
||||
const auto total_pages = allocation.committed / 4096;
|
||||
const std::uint64_t stride = std::max<std::uint64_t>(1, total_pages / 8);
|
||||
std::uint64_t page_number{};
|
||||
std::array<unsigned char, 4096> bytes{};
|
||||
for (const auto& range : allocation.ranges) {
|
||||
// (Pages that would be spoiled or slow to read are not read: a guard page would lose
|
||||
// its guard, and write-combined or uncached memory is the GPU's.)
|
||||
const bool readable = !(range.protect & (PAGE_GUARD | PAGE_NOACCESS | PAGE_WRITECOMBINE | PAGE_NOCACHE));
|
||||
for (std::uint64_t done = 0; done < range.bytes;) {
|
||||
const auto count = static_cast<std::size_t>(std::min<std::uint64_t>(pages.size(), (range.bytes - done) / 4096));
|
||||
if (!count) break;
|
||||
for (std::size_t i = 0; i < count; ++i) pages[i].VirtualAddress = reinterpret_cast<void*>(range.base + done + i * 4096);
|
||||
if (!QueryWorkingSetEx(GetCurrentProcess(), pages.data(), static_cast<DWORD>(count * sizeof(pages[0])))) return use;
|
||||
for (std::size_t i = 0; i < count; ++i, ++page_number) {
|
||||
if (!pages[i].VirtualAttributes.Valid) continue;
|
||||
use.resident += 4096;
|
||||
if (!readable || page_number % stride || use.sampled >= 8) continue;
|
||||
SIZE_T got{};
|
||||
if (!ReadProcessMemory(GetCurrentProcess(), pages[i].VirtualAddress, bytes.data(), bytes.size(), &got) ||
|
||||
got != bytes.size())
|
||||
continue;
|
||||
++use.sampled;
|
||||
use.zero += std::all_of(bytes.begin(), bytes.end(), [](unsigned char byte) { return byte == 0; });
|
||||
}
|
||||
done += count * 4096ull;
|
||||
}
|
||||
}
|
||||
return use;
|
||||
}
|
||||
|
||||
struct Totals {
|
||||
std::uint64_t committed{}, reserved{}, resident{}, zero{}, blocks{};
|
||||
std::uint32_t first_made{UINT32_MAX}, last_made{}; // seconds after the ledger started
|
||||
std::uint64_t smallest{UINT64_MAX}, largest{};
|
||||
void add(const Allocation& allocation, const Use& use, std::uint32_t made, bool known) {
|
||||
committed += allocation.committed;
|
||||
reserved += allocation.reserved;
|
||||
resident += use.resident;
|
||||
// (An estimate: the share of the pages looked at that were all zero.)
|
||||
if (use.sampled) zero += use.resident * use.zero / use.sampled;
|
||||
++blocks;
|
||||
smallest = std::min(smallest, allocation.committed);
|
||||
largest = std::max(largest, allocation.committed);
|
||||
if (known) first_made = std::min(first_made, made), last_made = std::max(last_made, made);
|
||||
}
|
||||
};
|
||||
std::string totals_text(const Totals& t) {
|
||||
return std::format("{} committed, {} in RAM, {} never used or moved out, ~{} in RAM but all zero; {} reserved in {} block(s)",
|
||||
megabytes(t.committed), megabytes(t.resident), megabytes(t.committed - std::min(t.committed, t.resident)), megabytes(t.zero),
|
||||
megabytes(t.reserved), t.blocks);
|
||||
}
|
||||
|
||||
std::uint32_t seconds_running() noexcept {
|
||||
return static_cast<std::uint32_t>((GetTickCount64() - began.load()) / 1000);
|
||||
}
|
||||
|
||||
// ---- the leak watcher ----
|
||||
std::mutex& watch_mutex = *new std::mutex;
|
||||
std::string& watched_owners = *new std::string;
|
||||
struct History {
|
||||
std::array<std::uint64_t, 16> committed{};
|
||||
unsigned count{};
|
||||
std::uint64_t said_at{};
|
||||
};
|
||||
std::unordered_map<std::uint64_t, History>& histories = *new std::unordered_map<std::uint64_t, History>; // by caller hash; the watcher's own
|
||||
std::unordered_map<std::uint64_t, std::string>& owner_names = *new std::unordered_map<std::uint64_t, std::string>;
|
||||
|
||||
void rebuild_if_worn() {
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
if (blocks_gone * 8 < block_slots) return;
|
||||
}
|
||||
std::vector<Block> fresh(block_slots);
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
fresh.swap(blocks);
|
||||
blocks_live = blocks_gone = 0;
|
||||
for (const auto& block : fresh)
|
||||
if (block.caller && block.caller != gone) remember(block.base, block.caller, block.made);
|
||||
}
|
||||
}
|
||||
|
||||
void watch_once() {
|
||||
rebuild_if_worn();
|
||||
const auto held = read_held(false);
|
||||
const auto copy = copy_tables();
|
||||
std::unordered_map<std::uint32_t, std::uint64_t> by_caller;
|
||||
std::map<std::string, std::uint64_t> by_owner;
|
||||
std::uint64_t unseen_bytes{};
|
||||
for (const auto& [base, allocation] : held.private_memory) {
|
||||
if (!allocation.committed) continue;
|
||||
if (const auto* block = find(copy.blocks, base)) by_caller[block->caller] += allocation.committed;
|
||||
else unseen_bytes += allocation.committed;
|
||||
}
|
||||
const auto now = GetTickCount64();
|
||||
for (const auto& [index, committed] : by_caller) {
|
||||
const auto& caller = copy.callers[index - 1];
|
||||
auto& name = owner_names[caller.hash];
|
||||
if (name.empty()) name = owner_of(caller);
|
||||
by_owner[name] += committed;
|
||||
auto& history = histories[caller.hash];
|
||||
std::rotate(history.committed.begin(), history.committed.begin() + 1, history.committed.end());
|
||||
history.committed.back() = committed;
|
||||
history.count = std::min<unsigned>(history.count + 1, static_cast<unsigned>(history.committed.size()));
|
||||
// Growing: over the last ten minutes, by 64 MB or more, and larger at nearly every look.
|
||||
constexpr unsigned span = 10;
|
||||
if (history.count <= span || now - history.said_at < 600000) continue;
|
||||
const auto* last = history.committed.data() + history.committed.size() - span - 1;
|
||||
unsigned rises{}, falls{};
|
||||
for (unsigned i = 1; i <= span; ++i) rises += last[i] > last[i - 1], falls += last[i] < last[i - 1];
|
||||
if (last[span] < last[0] + (64ull << 20) || rises < span * 7 / 10 || falls > 1) continue;
|
||||
history.said_at = now;
|
||||
logging::log(logging::Level::warning, logging::Channel::runtime,
|
||||
"Memory keeps growing: {} has {} committed, {} more than {} min ago; made by{}", name, megabytes(last[span]),
|
||||
megabytes(last[span] - last[0]), span, stack_of(caller));
|
||||
}
|
||||
// (The callers no longer holding anything are dropped from the history.)
|
||||
std::unordered_set<std::uint64_t> holding;
|
||||
for (const auto& [index, committed] : by_caller) holding.insert(copy.callers[index - 1].hash);
|
||||
std::erase_if(histories, [&](const auto& entry) { return !holding.contains(entry.first); });
|
||||
std::vector<std::pair<std::uint64_t, std::string>> order;
|
||||
for (const auto& [name, committed] : by_owner) order.emplace_back(committed, name);
|
||||
order.emplace_back(unseen_bytes, "not seen made");
|
||||
std::ranges::sort(order, std::greater{});
|
||||
std::string line = "Committed by owner:";
|
||||
for (std::size_t i = 0; i < order.size() && i < 5; ++i)
|
||||
line += std::format("{} {} {}", i ? "," : "", order[i].second, megabytes(order[i].first));
|
||||
std::lock_guard lock(watch_mutex);
|
||||
watched_owners = std::move(line);
|
||||
}
|
||||
|
||||
void watch() {
|
||||
SetThreadDescription(GetCurrentThread(), L"ReSkate memory ledger");
|
||||
for (;;) {
|
||||
Sleep(60000);
|
||||
try { watch_once(); } catch (...) {}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- heaps ----
|
||||
struct SizeCount {
|
||||
std::uint64_t size{}, count{};
|
||||
};
|
||||
// Block sizes go into storage made before a heap is locked: allocating from a heap while
|
||||
// walking it would change what is walked.
|
||||
struct SizeTable {
|
||||
static constexpr std::size_t slots = std::size_t{1} << 16;
|
||||
std::vector<SizeCount> counts = std::vector<SizeCount>(slots);
|
||||
std::uint64_t other_count{}, other_bytes{};
|
||||
void add(std::uint64_t size) noexcept {
|
||||
for (std::size_t probe = 0; probe < 64; ++probe) {
|
||||
auto& slot = counts[((size * 0x9e3779b97f4a7c15ull >> 48) + probe) & (slots - 1)];
|
||||
if (slot.count && slot.size != size) continue;
|
||||
slot.size = size;
|
||||
++slot.count;
|
||||
return;
|
||||
}
|
||||
++other_count;
|
||||
other_bytes += size;
|
||||
}
|
||||
};
|
||||
struct HeapSeen {
|
||||
HANDLE heap{};
|
||||
std::uint64_t committed{}, busy{}, free{}, largest_free{}, busy_blocks{};
|
||||
bool whole{};
|
||||
};
|
||||
// (A fault in the walk is caught: a heap made without its own lock may change under it.)
|
||||
void walk_heap(HeapSeen& heap, SizeTable& sizes) noexcept {
|
||||
if (!HeapLock(heap.heap)) return;
|
||||
heap.whole = true;
|
||||
__try {
|
||||
PROCESS_HEAP_ENTRY entry{};
|
||||
while (HeapWalk(heap.heap, &entry)) {
|
||||
if (entry.wFlags & PROCESS_HEAP_REGION) {
|
||||
heap.committed += entry.Region.dwCommittedSize;
|
||||
} else if (entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) {
|
||||
heap.busy += entry.cbData;
|
||||
++heap.busy_blocks;
|
||||
sizes.add(entry.cbData);
|
||||
} else if (!(entry.wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE)) {
|
||||
heap.free += entry.cbData;
|
||||
heap.largest_free = std::max<std::uint64_t>(heap.largest_free, entry.cbData);
|
||||
}
|
||||
}
|
||||
} __except (EXCEPTION_EXECUTE_HANDLER) {
|
||||
heap.whole = false;
|
||||
}
|
||||
HeapUnlock(heap.heap);
|
||||
}
|
||||
std::mutex heap_mutex;
|
||||
std::map<std::uint64_t, std::uint64_t> heap_mark; // block size -> count
|
||||
bool heap_marked{};
|
||||
} // namespace
|
||||
|
||||
bool start(std::string& error) {
|
||||
std::lock_guard start_lock(start_mutex);
|
||||
if (running.load()) return true;
|
||||
if (!prepared) {
|
||||
const auto allocate = ntdll_function("NtAllocateVirtualMemory");
|
||||
const auto release = ntdll_function("NtFreeVirtualMemory");
|
||||
if (!allocate || !release || hook_prepare(allocate, reinterpret_cast<void*>(&hooked), reinterpret_cast<void**>(&original)) != HookOk) {
|
||||
error = "the allocation hook could not be prepared";
|
||||
return false;
|
||||
}
|
||||
if (hook_prepare(release, reinterpret_cast<void*>(&hooked_free), reinterpret_cast<void**>(&original_free)) != HookOk) {
|
||||
hook_remove(allocate);
|
||||
error = "the release hook could not be prepared";
|
||||
return false;
|
||||
}
|
||||
// (Missing on an older Windows: then only the first is followed.)
|
||||
if (const auto ex = ntdll_function("NtAllocateVirtualMemoryEx");
|
||||
ex && hook_prepare(ex, reinterpret_cast<void*>(&hooked_ex), reinterpret_cast<void**>(&original_ex)) != HookOk)
|
||||
original_ex = nullptr;
|
||||
prepared = true;
|
||||
}
|
||||
began.store(GetTickCount64());
|
||||
running.store(true);
|
||||
// (The release hook first: a block noted and given back before it is on would stay noted.)
|
||||
if (hook_enable(ntdll_function("NtFreeVirtualMemory")) != HookOk || hook_enable(ntdll_function("NtAllocateVirtualMemory")) != HookOk) {
|
||||
running.store(false);
|
||||
hook_disable(ntdll_function("NtFreeVirtualMemory"));
|
||||
error = "the allocation hooks could not be attached";
|
||||
return false;
|
||||
}
|
||||
if (original_ex) hook_enable(ntdll_function("NtAllocateVirtualMemoryEx"));
|
||||
std::thread(watch).detach();
|
||||
logging::log(logging::Level::info, logging::Channel::runtime,
|
||||
"Memory ledger on: who makes each reservation of memory is noted (`perf memory owners`), and an owner whose memory keeps growing is said here.");
|
||||
return true;
|
||||
}
|
||||
bool on() noexcept { return running.load(); }
|
||||
|
||||
bool start_asking(std::uint64_t smallest, std::uint64_t largest, std::string& error) {
|
||||
if (!start(error)) return false;
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
for (auto& caller : callers) caller.asked_count = caller.asked_bytes = caller.smallest = caller.largest = 0;
|
||||
}
|
||||
least.store(smallest);
|
||||
most.store(largest);
|
||||
counting.store(true);
|
||||
return true;
|
||||
}
|
||||
void stop_asking() noexcept { counting.store(false); }
|
||||
bool asking() noexcept { return counting.load(); }
|
||||
|
||||
void asked_report(const Line& out) {
|
||||
const auto copy = copy_tables();
|
||||
std::vector<const Caller*> seen;
|
||||
for (const auto& caller : copy.callers) if (caller.asked_count) seen.push_back(&caller);
|
||||
std::ranges::sort(seen, [](const Caller* a, const Caller* b) { return a->asked_bytes > b->asked_bytes; });
|
||||
out(std::format("Memory asked for while tracing (blocks of {} and up), by who asked ({} callers), the most first:",
|
||||
megabytes(least.load()), seen.size()));
|
||||
for (std::size_t i = 0; i < seen.size() && i < 16; ++i) {
|
||||
out(seen[i]->smallest == seen[i]->largest
|
||||
? std::format(" {} x {} bytes = {}", seen[i]->asked_count, seen[i]->smallest, megabytes(seen[i]->asked_bytes))
|
||||
: std::format(" {} blocks of {} to {} bytes = {}", seen[i]->asked_count, seen[i]->smallest, seen[i]->largest,
|
||||
megabytes(seen[i]->asked_bytes)));
|
||||
out(" " + stack_of(*seen[i]));
|
||||
}
|
||||
}
|
||||
|
||||
void owners_report(const Line& out, std::size_t most_callers) {
|
||||
if (std::string error; !start(error)) { out("error: " + error); return; }
|
||||
const auto copy = copy_tables();
|
||||
const auto held = read_held(true);
|
||||
std::vector<PSAPI_WORKING_SET_EX_INFORMATION> pages(16384);
|
||||
std::unordered_map<std::uint32_t, Totals> by_caller;
|
||||
std::map<std::string, Totals> by_owner;
|
||||
Totals all, unseen_totals;
|
||||
struct Unseen { std::uintptr_t base; const Allocation* allocation; Use use; };
|
||||
std::vector<Unseen> unseen_blocks;
|
||||
for (const auto& [base, allocation] : held.private_memory) {
|
||||
if (!allocation.committed) continue;
|
||||
const auto use = use_of(allocation, pages);
|
||||
const auto* block = find(copy.blocks, base);
|
||||
all.add(allocation, use, 0, false);
|
||||
if (!block) {
|
||||
unseen_totals.add(allocation, use, 0, false);
|
||||
unseen_blocks.push_back({base, &allocation, use});
|
||||
continue;
|
||||
}
|
||||
by_caller[block->caller].add(allocation, use, block->made, true);
|
||||
}
|
||||
std::unordered_map<std::uint32_t, std::string> names;
|
||||
for (const auto& [index, totals] : by_caller) {
|
||||
auto& name = names[index] = owner_of(copy.callers[index - 1]);
|
||||
auto& owner = by_owner[name];
|
||||
owner.committed += totals.committed;
|
||||
owner.reserved += totals.reserved;
|
||||
owner.resident += totals.resident;
|
||||
owner.zero += totals.zero;
|
||||
owner.blocks += totals.blocks;
|
||||
}
|
||||
by_owner[std::string(unseen)] = unseen_totals;
|
||||
|
||||
out(std::format("Memory: {} committed, {} of it in RAM; {} of files mapped in, {} of programs.", megabytes(held.committed),
|
||||
megabytes(all.resident), megabytes(held.mapped), megabytes(held.image)));
|
||||
out(std::format("Of what is committed: {} never used or moved out of RAM by Windows, and about {} in RAM but all zero.",
|
||||
megabytes(all.committed - std::min(all.committed, all.resident)), megabytes(all.zero)));
|
||||
gpu_report(out);
|
||||
out(std::format("Ledger: on for {}, {} reservations held by {} callers{}.", age(seconds_running()), copy.live,
|
||||
std::ranges::count_if(copy.callers, [](const Caller& c) { return c.hash != 0; }),
|
||||
copy.untracked ? std::format(", {} not noted (its tables were full)", copy.untracked) : std::string()));
|
||||
out("By owner, the most committed first:");
|
||||
std::vector<std::pair<std::string, const Totals*>> owners;
|
||||
for (const auto& [name, totals] : by_owner) owners.emplace_back(name, &totals);
|
||||
std::ranges::sort(owners, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; });
|
||||
for (const auto& [name, totals] : owners)
|
||||
if (totals->committed) out(std::format(" {}: {}", name, totals_text(*totals)));
|
||||
|
||||
// What the memory no hook saw being made is: memory Windows or a driver maps in from the
|
||||
// kernel (a GPU's memory is write-combined or uncached) never passes NtAllocateVirtualMemory.
|
||||
std::map<std::string, Totals> unseen_by_kind;
|
||||
for (const auto& block : unseen_blocks)
|
||||
unseen_by_kind[std::format("committed {}; reserved {}", protection(block.allocation->committed_as),
|
||||
protection(block.allocation->reserved_as))].add(*block.allocation, block.use, 0, false);
|
||||
out("Not seen being made, by protection:");
|
||||
std::vector<std::pair<std::string, const Totals*>> kinds;
|
||||
for (const auto& [kind, totals] : unseen_by_kind) kinds.emplace_back(kind, &totals);
|
||||
std::ranges::sort(kinds, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; });
|
||||
for (const auto& [kind, totals] : kinds) out(std::format(" {}: {}", kind, totals_text(*totals)));
|
||||
std::ranges::sort(unseen_blocks, [](const Unseen& a, const Unseen& b) { return a.allocation->committed > b.allocation->committed; });
|
||||
out("The largest of them:");
|
||||
for (std::size_t i = 0; i < unseen_blocks.size() && i < 16; ++i) {
|
||||
const auto& block = unseen_blocks[i];
|
||||
out(std::format(" {:#x}: {} committed of {} reserved, {} in RAM{}; committed {}, reserved {}, in {} part(s)", block.base,
|
||||
megabytes(block.allocation->committed), megabytes(block.allocation->reserved), megabytes(block.use.resident),
|
||||
block.use.sampled ? std::format(" ({} of {} pages looked at all zero)", block.use.zero, block.use.sampled) : std::string(),
|
||||
protection(block.allocation->committed_as), protection(block.allocation->reserved_as), block.allocation->ranges.size()));
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::uint32_t, const Totals*>> order;
|
||||
for (const auto& [index, totals] : by_caller) order.emplace_back(index, &totals);
|
||||
std::ranges::sort(order, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; });
|
||||
out(std::format("By caller ({}), the most committed first:", order.size()));
|
||||
for (std::size_t i = 0; i < order.size() && i < most_callers; ++i) {
|
||||
const auto& totals = *order[i].second;
|
||||
out(std::format(" {}: {}", names[order[i].first], totals_text(totals)));
|
||||
out(std::format(" blocks of {} to {}; the newest made {} ago, the oldest {} ago", megabytes(totals.smallest),
|
||||
megabytes(totals.largest), age(seconds_running() - std::min(seconds_running(), totals.last_made)),
|
||||
age(seconds_running() - std::min(seconds_running(), totals.first_made))));
|
||||
out(" " + stack_of(copy.callers[order[i].first - 1]));
|
||||
}
|
||||
}
|
||||
|
||||
void heap_report(const Line& out, bool mark, std::size_t most_sizes) {
|
||||
std::lock_guard report_lock(heap_mutex);
|
||||
struct Summary { DWORD size; SIZE_T allocated, committed, reserved, most; };
|
||||
static const auto heap_summary = reinterpret_cast<BOOL (WINAPI*)(HANDLE, DWORD, Summary*)>(
|
||||
GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "HeapSummary"));
|
||||
std::array<HANDLE, 256> heaps{};
|
||||
const auto heap_count = std::min<DWORD>(GetProcessHeaps(static_cast<DWORD>(heaps.size()), heaps.data()), static_cast<DWORD>(heaps.size()));
|
||||
SizeTable sizes;
|
||||
std::vector<HeapSeen> seen(heap_count);
|
||||
std::uint64_t total_committed{}, total_busy{}, total_free{};
|
||||
std::size_t partial{};
|
||||
for (DWORD i = 0; i < heap_count; ++i) {
|
||||
auto& heap = seen[i];
|
||||
heap.heap = heaps[i];
|
||||
walk_heap(heap, sizes);
|
||||
partial += !heap.whole;
|
||||
// (A heap whose segments the walk does not show, such as one of the segment heap's: its
|
||||
// summary says what it holds.)
|
||||
if (Summary summary{sizeof(Summary)}; heap_summary && heap_summary(heaps[i], 0, &summary) && summary.committed > heap.committed)
|
||||
heap.committed = summary.committed;
|
||||
total_committed += heap.committed;
|
||||
total_busy += heap.busy;
|
||||
total_free += heap.free;
|
||||
}
|
||||
out(std::format("Heaps: {} committed in {} heap(s), {} of it in use, {} free inside them{}.", megabytes(total_committed),
|
||||
heap_count, megabytes(total_busy), megabytes(total_free),
|
||||
partial ? std::format(" ({} could not be walked whole)", partial) : std::string()));
|
||||
std::ranges::sort(seen, [](const HeapSeen& a, const HeapSeen& b) { return a.committed > b.committed; });
|
||||
for (std::size_t i = 0; i < seen.size() && i < 8; ++i) {
|
||||
if (!seen[i].committed) continue;
|
||||
out(std::format(" heap {:#x}{}: {} committed, {} in use in {} blocks, {} free (the largest free piece {})",
|
||||
reinterpret_cast<std::uintptr_t>(seen[i].heap), seen[i].heap == GetProcessHeap() ? " (the process heap)" : "",
|
||||
megabytes(seen[i].committed), megabytes(seen[i].busy), seen[i].busy_blocks, megabytes(seen[i].free),
|
||||
megabytes(seen[i].largest_free)));
|
||||
}
|
||||
std::map<std::uint64_t, std::uint64_t> now;
|
||||
for (const auto& slot : sizes.counts) if (slot.count) now[slot.size] = slot.count;
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> order(now.begin(), now.end()); // size, count
|
||||
std::ranges::sort(order, [](const auto& a, const auto& b) { return a.first * a.second > b.first * b.second; });
|
||||
out("Blocks in use by size, the sizes that add up to most first:");
|
||||
for (std::size_t i = 0; i < order.size() && i < most_sizes; ++i)
|
||||
out(std::format(" {} x {} bytes = {}", order[i].second, order[i].first, megabytes(order[i].first * order[i].second)));
|
||||
if (sizes.other_count) out(std::format(" (and {} blocks of other sizes, {})", sizes.other_count, megabytes(sizes.other_bytes)));
|
||||
if (mark) {
|
||||
heap_mark = std::move(now);
|
||||
heap_marked = true;
|
||||
out("Marked. Do what you want to measure, then run `perf memory heap` again.");
|
||||
return;
|
||||
}
|
||||
if (!heap_marked) return;
|
||||
// What was added since the mark, by size: a leak shows as a size whose count only climbs.
|
||||
std::vector<std::tuple<std::uint64_t, std::int64_t>> added; // size, count added
|
||||
for (const auto& [size, count] : now) {
|
||||
const auto before = heap_mark.find(size);
|
||||
const auto was = before == heap_mark.end() ? 0 : before->second;
|
||||
if (count > was) added.emplace_back(size, static_cast<std::int64_t>(count - was));
|
||||
}
|
||||
std::ranges::sort(added, [](const auto& a, const auto& b) {
|
||||
return std::get<0>(a) * std::get<1>(a) > std::get<0>(b) * std::get<1>(b);
|
||||
});
|
||||
out("Added since the mark, by size:");
|
||||
for (std::size_t i = 0; i < added.size() && i < most_sizes; ++i)
|
||||
out(std::format(" +{} x {} bytes = {}", std::get<1>(added[i]), std::get<0>(added[i]),
|
||||
megabytes(std::get<0>(added[i]) * static_cast<std::uint64_t>(std::get<1>(added[i])))));
|
||||
}
|
||||
|
||||
namespace {
|
||||
struct Gpu {
|
||||
std::string name;
|
||||
std::uint64_t local{}, local_budget{}, shared{}, shared_budget{};
|
||||
};
|
||||
std::vector<Gpu> read_gpus() {
|
||||
std::vector<Gpu> gpus;
|
||||
IDXGIFactory4* factory{};
|
||||
if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory4), reinterpret_cast<void**>(&factory))) || !factory) return gpus;
|
||||
IDXGIAdapter1* adapter{};
|
||||
for (UINT i = 0; factory->EnumAdapters1(i, &adapter) != DXGI_ERROR_NOT_FOUND; ++i) {
|
||||
DXGI_ADAPTER_DESC1 description{};
|
||||
IDXGIAdapter3* adapter3{};
|
||||
if (SUCCEEDED(adapter->GetDesc1(&description)) && !(description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) &&
|
||||
SUCCEEDED(adapter->QueryInterface(__uuidof(IDXGIAdapter3), reinterpret_cast<void**>(&adapter3)))) {
|
||||
DXGI_QUERY_VIDEO_MEMORY_INFO local{}, shared{};
|
||||
adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &local);
|
||||
adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL, &shared);
|
||||
// (Only an adapter this process uses.)
|
||||
if (local.CurrentUsage || shared.CurrentUsage) {
|
||||
Gpu gpu{{}, local.CurrentUsage, local.Budget, shared.CurrentUsage, shared.Budget};
|
||||
for (const wchar_t* c = description.Description; *c; ++c) gpu.name += static_cast<char>(*c < 128 ? *c : '?');
|
||||
gpus.push_back(std::move(gpu));
|
||||
}
|
||||
adapter3->Release();
|
||||
}
|
||||
adapter->Release();
|
||||
}
|
||||
factory->Release();
|
||||
return gpus;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void gpu_report(const Line& out) {
|
||||
for (const auto& gpu : read_gpus())
|
||||
out(std::format("GPU {}: {} of video memory (budget {}), {} of system memory (budget {}).", gpu.name,
|
||||
megabytes(gpu.local), megabytes(gpu.local_budget), megabytes(gpu.shared), megabytes(gpu.shared_budget)));
|
||||
}
|
||||
|
||||
std::string owners_line() {
|
||||
if (!running.load()) return {};
|
||||
std::lock_guard lock(watch_mutex);
|
||||
return watched_owners;
|
||||
}
|
||||
std::string gpu_line() {
|
||||
std::string line;
|
||||
for (const auto& gpu : read_gpus())
|
||||
line += std::format("{}GPU: {} video memory, {} system memory", line.empty() ? "" : "; ", megabytes(gpu.local), megabytes(gpu.shared));
|
||||
return line;
|
||||
}
|
||||
|
||||
std::filesystem::path write_report() {
|
||||
SYSTEMTIME now{};
|
||||
GetLocalTime(&now);
|
||||
const auto path = logging::status().directory / std::format("memory-{:04}{:02}{:02}-{:02}{:02}{:02}.txt", now.wYear, now.wMonth,
|
||||
now.wDay, now.wHour, now.wMinute, now.wSecond);
|
||||
std::string text;
|
||||
const Line add = [&](const std::string& line) { text += line + "\n"; };
|
||||
owners_report(add, 60);
|
||||
add("");
|
||||
heap_report(add, false, 60);
|
||||
if (asking()) {
|
||||
add("");
|
||||
asked_report(add);
|
||||
}
|
||||
std::ofstream file(path, std::ios::binary);
|
||||
file << text;
|
||||
return file ? path : std::filesystem::path{};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
// Who owns the memory the process holds, and how much of it is used (`perf memory owners`).
|
||||
// While it is on, every reservation of memory from Windows is noted with the code that asked
|
||||
// for it, and forgotten when it is given back; a report then joins what the process holds now
|
||||
// with who made it: what is committed, what of that is in RAM, and what is in RAM but all zero.
|
||||
// A minute at a time it also looks for an owner whose memory only grows, and says so in the log.
|
||||
// It is a research aid and off until asked for: every allocation in the process passes it.
|
||||
namespace dingosdk::memory_ledger {
|
||||
using Line = std::function<void(const std::string&)>;
|
||||
|
||||
// Starts noting reservations. What was made before is reported as not seen being made.
|
||||
bool start(std::string& error);
|
||||
bool on() noexcept;
|
||||
// `perf memory trace`: also counts what is asked for (commits too) of sizes in [least, most]
|
||||
// by who asked, from now on. Starts the ledger if it is off.
|
||||
bool start_asking(std::uint64_t least, std::uint64_t most, std::string& error);
|
||||
void stop_asking() noexcept;
|
||||
bool asking() noexcept;
|
||||
void asked_report(const Line& out);
|
||||
|
||||
// What the process holds by owner and by caller (the ledger is started if it is off).
|
||||
// `most` callers are listed, each with its stack.
|
||||
void owners_report(const Line& out, std::size_t most = 12);
|
||||
// The process's heaps: what each holds and has free inside it, and its blocks by size; with
|
||||
// `mark`, the sizes are remembered and the next report says what was added since.
|
||||
void heap_report(const Line& out, bool mark, std::size_t most = 20);
|
||||
// Video memory and the system memory the GPU uses for this process, one line per adapter.
|
||||
void gpu_report(const Line& out);
|
||||
// Everything above at length, into memory-<time>.txt beside ReSkate.log.
|
||||
std::filesystem::path write_report();
|
||||
|
||||
// For the periodic "Memory:" log line: the largest owners, as the leak watcher last saw them
|
||||
// (empty while the ledger is off), and the GPU's use.
|
||||
std::string owners_line();
|
||||
std::string gpu_line();
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "Extension/Console/commands.h"
|
||||
#include "Engine/Core/Profiling/profiler.h"
|
||||
#include "Extension/Console/memory_ledger.h"
|
||||
#include "Extension/Settings/engine_tweaks.h"
|
||||
#include "Extension/Settings/job_spin.h"
|
||||
#include "Extension/UI/Overlay/overlay.h"
|
||||
@@ -139,127 +140,10 @@ void print_by_size(const std::vector<std::uint64_t>& sizes, const Output& out, s
|
||||
out(std::format(" {} x {} = {}", order[i].second.first, megabytes(order[i].first), megabytes(order[i].second.second)));
|
||||
}
|
||||
|
||||
// `perf memory trace`: who asks Windows for the middle-sized blocks (2 to 64 MB) while it is on,
|
||||
// by the code that called for each: where a block of a size `perf memory` showed many of comes
|
||||
// from. Every commit of memory in the process passes the hook while it is on, so it is a
|
||||
// research aid and off until asked for.
|
||||
namespace trace {
|
||||
constexpr std::size_t callers = 8192, depth = 14;
|
||||
struct Caller {
|
||||
std::uint64_t hash{}, count{}, bytes{}, size{}, largest{}; // size: the smallest such block; largest: the largest
|
||||
std::array<void*, depth> frames{};
|
||||
unsigned frame_count{};
|
||||
};
|
||||
std::mutex mutex;
|
||||
std::vector<Caller> table(callers); // found by hash: a commit of any size can be traced
|
||||
std::atomic<bool> on{};
|
||||
// The sizes traced: 2 to 64 MB from the console; the largest blocks alone from the start of
|
||||
// the game (RESKATE_MEMORY_TRACE), where nearly everything is being allocated.
|
||||
std::atomic<std::uint64_t> least{2ull << 20}, most{64ull << 20};
|
||||
bool prepared{};
|
||||
using Allocate = LONG (NTAPI*)(HANDLE, PVOID*, ULONG_PTR, PSIZE_T, ULONG, ULONG);
|
||||
Allocate original{};
|
||||
|
||||
// The newer call for the same (VirtualAlloc2, which graphics drivers and the D3D12 runtime use).
|
||||
using AllocateEx = LONG (NTAPI*)(HANDLE, PVOID*, PSIZE_T, ULONG, ULONG, void*, ULONG);
|
||||
AllocateEx original_ex{};
|
||||
void note(HANDLE process, SIZE_T asked, ULONG type) noexcept;
|
||||
|
||||
LONG NTAPI hooked(HANDLE process, PVOID* base, ULONG_PTR zero_bits, PSIZE_T size, ULONG type, ULONG protect) {
|
||||
const auto asked = size ? *size : 0;
|
||||
const auto status = original(process, base, zero_bits, size, type, protect);
|
||||
if (status >= 0) note(process, asked, type);
|
||||
return status;
|
||||
}
|
||||
LONG NTAPI hooked_ex(HANDLE process, PVOID* base, PSIZE_T size, ULONG type, ULONG protect, void* parameters, ULONG parameter_count) {
|
||||
const auto asked = size ? *size : 0;
|
||||
const auto status = original_ex(process, base, size, type, protect, parameters, parameter_count);
|
||||
if (status >= 0) note(process, asked, type);
|
||||
return status;
|
||||
}
|
||||
void note(HANDLE process, SIZE_T asked, ULONG type) noexcept {
|
||||
if (!on.load(std::memory_order_relaxed) || !(type & (MEM_COMMIT | MEM_RESERVE)) || process != GetCurrentProcess() ||
|
||||
asked < least.load(std::memory_order_relaxed) || asked > most.load(std::memory_order_relaxed)) return;
|
||||
// (Reservations count too: a block reserved whole and committed a piece at a time is only
|
||||
// this large when it is reserved.)
|
||||
Caller seen;
|
||||
seen.frame_count = CaptureStackBackTrace(2, static_cast<DWORD>(depth), seen.frames.data(), nullptr);
|
||||
// By who asked alone: one caller's blocks of every size add up together.
|
||||
std::uint64_t hash = 1469598103934665603ull;
|
||||
for (unsigned i = 0; i < seen.frame_count; ++i) hash = hash * 1099511628211ull ^ reinterpret_cast<std::uintptr_t>(seen.frames[i]);
|
||||
if (!hash) hash = 1;
|
||||
std::lock_guard lock(mutex);
|
||||
for (std::size_t probe = 0; probe < 128; ++probe) {
|
||||
auto& caller = table[(hash + probe) % callers];
|
||||
if (caller.count && caller.hash != hash) continue;
|
||||
if (!caller.count) { caller = seen; caller.hash = hash; caller.size = asked; }
|
||||
++caller.count;
|
||||
caller.bytes += asked;
|
||||
caller.size = std::min<std::uint64_t>(caller.size, asked);
|
||||
caller.largest = std::max<std::uint64_t>(caller.largest, asked);
|
||||
break;
|
||||
}
|
||||
}
|
||||
bool start(std::string& error) {
|
||||
if (!prepared) {
|
||||
const auto target = reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemory"));
|
||||
if (!target || hook_prepare(target, reinterpret_cast<void*>(&hooked), reinterpret_cast<void**>(&original)) != HookOk) {
|
||||
error = "the allocation hook could not be prepared";
|
||||
return false;
|
||||
}
|
||||
// (Missing on an older Windows: then only the first is traced.)
|
||||
if (const auto ex = reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemoryEx"));
|
||||
ex && hook_prepare(ex, reinterpret_cast<void*>(&hooked_ex), reinterpret_cast<void**>(&original_ex)) != HookOk)
|
||||
original_ex = nullptr;
|
||||
prepared = true;
|
||||
}
|
||||
{ std::lock_guard lock(mutex); std::fill(table.begin(), table.end(), Caller{}); }
|
||||
const auto target = reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemory"));
|
||||
if (!on.load() && hook_enable(target) != HookOk) { error = "the allocation hook could not be attached"; return false; }
|
||||
if (!on.load() && original_ex) hook_enable(reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemoryEx")));
|
||||
on.store(true);
|
||||
return true;
|
||||
}
|
||||
void stop() {
|
||||
if (!on.exchange(false)) return;
|
||||
hook_disable(reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemory")));
|
||||
if (original_ex) hook_disable(reinterpret_cast<void*>(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtAllocateVirtualMemoryEx")));
|
||||
}
|
||||
std::string place(void* address) {
|
||||
HMODULE module{};
|
||||
if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT,
|
||||
reinterpret_cast<LPCWSTR>(address), &module) || !module)
|
||||
return std::format("{:#x}", reinterpret_cast<std::uintptr_t>(address));
|
||||
wchar_t path[MAX_PATH]{};
|
||||
GetModuleFileNameW(module, path, MAX_PATH);
|
||||
const wchar_t* name = path;
|
||||
for (const wchar_t* c = path; *c; ++c) if (*c == L'\\') name = c + 1;
|
||||
std::string text;
|
||||
for (; *name; ++name) text += static_cast<char>(*name);
|
||||
return std::format("{}+{:#x}", text, reinterpret_cast<std::uintptr_t>(address) - reinterpret_cast<std::uintptr_t>(module));
|
||||
}
|
||||
void report(const Output& out) {
|
||||
std::vector<Caller> seen;
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
for (const auto& caller : table) if (caller.count) seen.push_back(caller);
|
||||
}
|
||||
std::sort(seen.begin(), seen.end(), [](const Caller& a, const Caller& b) { return a.bytes > b.bytes; });
|
||||
out(std::format("Memory asked for while tracing (blocks of {} and up), by who asked ({} callers), the most first:",
|
||||
megabytes(least.load()), seen.size()));
|
||||
for (std::size_t i = 0; i < seen.size() && i < 16; ++i) {
|
||||
out(seen[i].size == seen[i].largest
|
||||
? std::format(" {} x {} bytes = {}", seen[i].count, seen[i].size, megabytes(seen[i].bytes))
|
||||
: std::format(" {} blocks of {} to {} bytes = {}", seen[i].count, seen[i].size, seen[i].largest, megabytes(seen[i].bytes)));
|
||||
std::string line = " ";
|
||||
for (unsigned frame = 0; frame < seen[i].frame_count; ++frame) line += " < " + place(seen[i].frames[frame]);
|
||||
out(line);
|
||||
}
|
||||
}
|
||||
} // namespace trace
|
||||
|
||||
// RESKATE_MEMORY_TRACE=<megabytes>: from the start of the game, who asks for each block of at
|
||||
// least that size (`perf memory` then lists them). A research aid for what a map's load commits.
|
||||
// least that size (`perf memory` then lists them), and the memory ledger from then on, so
|
||||
// `perf memory owners` can say who made nearly everything. A research aid for what a map's
|
||||
// load commits.
|
||||
void start_memory_trace_from_environment() noexcept {
|
||||
wchar_t value[16]{};
|
||||
const auto length = GetEnvironmentVariableW(L"RESKATE_MEMORY_TRACE", value, 16);
|
||||
@@ -269,10 +153,8 @@ void start_memory_trace_from_environment() noexcept {
|
||||
const auto megabytes_least = std::wcstoul(value, nullptr, 10);
|
||||
if (!megabytes_least && !all) return;
|
||||
try {
|
||||
trace::least.store(all ? 1 : static_cast<std::uint64_t>(megabytes_least) << 20);
|
||||
trace::most.store(UINT64_MAX);
|
||||
std::string error;
|
||||
(void)trace::start(error);
|
||||
(void)memory_ledger::start_asking(all ? 1 : static_cast<std::uint64_t>(megabytes_least) << 20, UINT64_MAX, error);
|
||||
} catch (...) {}
|
||||
}
|
||||
|
||||
@@ -531,15 +413,32 @@ void register_perf_commands(Commands& registry) {
|
||||
registry.add(std::move(tweaks));
|
||||
|
||||
auto what = argument("mark", Type::text, true);
|
||||
what.complete = [](const Model&, auto) { return std::vector<std::string>{"mark", "trace", "stop", "big"}; };
|
||||
what.complete = [](const Model&, auto) {
|
||||
return std::vector<std::string>{"mark", "trace", "stop", "big", "owners", "heap", "gpu", "report"};
|
||||
};
|
||||
auto heap_mark = argument("mark", Type::text, true);
|
||||
heap_mark.complete = [](const Model&, auto) { return std::vector<std::string>{"mark"}; };
|
||||
auto memory = action("perf memory", "The game's memory by allocation; `perf memory mark` remembers it and the next `perf memory` says what was added since; "
|
||||
"`perf memory trace` then also says who asked for the middle-sized blocks, until `perf memory stop`; `perf memory big` lists the largest blocks and how much of each is in use",
|
||||
Group::console, {std::move(what)});
|
||||
"`perf memory trace` then also says who asked for the middle-sized blocks, until `perf memory stop`; `perf memory big` lists the largest blocks and how much of each is in use; "
|
||||
"`perf memory owners` says who made what is held and how much of it is used (it starts the memory ledger, which then also says in the log what keeps growing); "
|
||||
"`perf memory heap [mark]` lists the heaps' blocks by size and what was added since a mark; `perf memory gpu` the video memory; "
|
||||
"`perf memory report` writes all of it to a file beside ReSkate.log",
|
||||
Group::console, {std::move(what), std::move(heap_mark)});
|
||||
memory.execution = Execution::local;
|
||||
memory.run = [](const Model&, const Values& args, const Output& out) {
|
||||
static std::mutex mutex;
|
||||
static MemoryMap marked;
|
||||
static bool has_mark{};
|
||||
const memory_ledger::Line line = [&](const std::string& text) { out(text); };
|
||||
const auto word = args.empty() ? std::string() : std::get<std::string>(args[0]);
|
||||
if (equal(word, "owners")) return memory_ledger::owners_report(line);
|
||||
if (equal(word, "heap")) return memory_ledger::heap_report(line, args.size() > 1 && equal(std::get<std::string>(args[1]), "mark"));
|
||||
if (equal(word, "gpu")) return memory_ledger::gpu_report(line);
|
||||
if (equal(word, "report")) {
|
||||
const auto path = memory_ledger::write_report();
|
||||
out(path.empty() ? std::string("error: the report could not be written") : "Memory report: " + narrow(path));
|
||||
return;
|
||||
}
|
||||
std::lock_guard lock(mutex);
|
||||
const auto now = read_memory_map();
|
||||
out(std::format("Memory: {} committed in {} allocations, {} of files mapped in, {} of programs.", megabytes(now.committed),
|
||||
@@ -592,16 +491,15 @@ void register_perf_commands(Commands& registry) {
|
||||
}
|
||||
const bool tracing = !args.empty() && equal(std::get<std::string>(args[0]), "trace");
|
||||
if (!args.empty() && equal(std::get<std::string>(args[0]), "stop")) {
|
||||
trace::stop();
|
||||
// (The memory ledger stays on: what it missed being given back would be wrong.)
|
||||
memory_ledger::stop_asking();
|
||||
out("Tracing stopped.");
|
||||
return;
|
||||
}
|
||||
if (tracing) {
|
||||
std::string error;
|
||||
// Every size: a large block is often set aside whole and filled a page at a time.
|
||||
trace::least.store(1);
|
||||
trace::most.store(UINT64_MAX);
|
||||
if (!trace::start(error)) { out("error: " + error); return; }
|
||||
if (!memory_ledger::start_asking(1, UINT64_MAX, error)) { out("error: " + error); return; }
|
||||
}
|
||||
if (tracing || (!args.empty() && equal(std::get<std::string>(args[0]), "mark"))) {
|
||||
marked = now;
|
||||
@@ -615,7 +513,7 @@ void register_perf_commands(Commands& registry) {
|
||||
for (const auto& [base, size] : now.allocations) sizes.push_back(size);
|
||||
out("The allocation sizes that add up to most:");
|
||||
print_by_size(sizes, out, 20);
|
||||
if (trace::on.load()) trace::report(out);
|
||||
if (memory_ledger::asking()) memory_ledger::asked_report(line);
|
||||
return;
|
||||
}
|
||||
std::vector<std::uint64_t> added, grown;
|
||||
@@ -636,7 +534,7 @@ void register_perf_commands(Commands& registry) {
|
||||
out(std::format("Growth of allocations that were there: {} in {}. By how much each grew:", megabytes(grown_bytes), grown.size()));
|
||||
print_by_size(grown, out, 12);
|
||||
out(std::format("Given back: {}.", megabytes(freed_bytes)));
|
||||
if (trace::on.load()) trace::report(out);
|
||||
if (memory_ledger::asking()) memory_ledger::asked_report(line);
|
||||
};
|
||||
registry.add(std::move(memory));
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -2,10 +2,34 @@
|
||||
|
||||
#include <Windows.h>
|
||||
#include <d3d12.h>
|
||||
#include <cstdint>
|
||||
|
||||
namespace dingosdk::overlay {
|
||||
bool graphics_option_enabled(const wchar_t* name) noexcept;
|
||||
void initialize_graphics_diagnostics() noexcept;
|
||||
void record_device_failure(ID3D12Device* device, HRESULT result,
|
||||
const char* operation) noexcept;
|
||||
// With RESKATE_GPU_DIAGNOSTICS on: records the failure the moment the device is removed, from a
|
||||
// thread of its own, instead of when the overlay next draws. The game's own error dialog can
|
||||
// otherwise come first and end the process before then.
|
||||
void watch_device_removal(ID3D12Device* device) noexcept;
|
||||
// With RESKATE_GPU_DIAGNOSTICS on: every DrawInstanced and DrawIndexedInstanced of the process (the
|
||||
// runtime's, found through this command list) is checked, and one of more than ten million
|
||||
// vertices or indices over its instances is said in the log with the code that recorded it. A
|
||||
// count gone wrong makes a draw the GPU cannot finish. (Draws the GPU counts itself, through
|
||||
// ExecuteIndirect, are not seen.)
|
||||
void watch_draws(ID3D12GraphicsCommandList* list) noexcept;
|
||||
// With RESKATE_GPU_DIAGNOSTICS on, before the game makes its device: every heap and resource the
|
||||
// device makes is kept with the code that made it, and the code that freed it once it goes. A
|
||||
// page fault's report (record_device_failure) then says what was at, or freed from, the address.
|
||||
void watch_gpu_memory() noexcept;
|
||||
void describe_gpu_address(std::uint64_t address) noexcept;
|
||||
// With the GPU memory watch on: every GPU address the game binds for a draw or dispatch (vertex and
|
||||
// index buffers, root buffer views), puts in a constant-buffer or acceleration-structure view, or
|
||||
// builds a ray-tracing structure from is checked against the memory that exists. One in nothing
|
||||
// is said in the log once per calling code, with what was there before.
|
||||
void watch_addresses(ID3D12GraphicsCommandList* list) noexcept;
|
||||
// The graphics card the game renders on and its driver's version: said in the log once, and kept
|
||||
// for crash reports' tags (crash_report::set_gpu). Always on.
|
||||
void record_gpu_identity(ID3D12Device* device) noexcept;
|
||||
}
|
||||
@@ -391,6 +391,7 @@ bool start_overlay(const dingosdk::overlay::CallbacksV3* callbacks) {
|
||||
reinterpret_cast<LPCWSTR>(&start_overlay), &pinned)) return false;
|
||||
const auto status = dingosdk::hook_initialize();
|
||||
if (status != dingosdk::HookOk && status != dingosdk::HookAlreadyInitialized) return false;
|
||||
dingosdk::overlay::watch_gpu_memory();
|
||||
const auto user32 = GetModuleHandleW(L"user32.dll");
|
||||
if (!user32 || !install(s.clip_cursor, reinterpret_cast<void*>(GetProcAddress(user32, "ClipCursor")),
|
||||
reinterpret_cast<void*>(clip_cursor)) ||
|
||||
|
||||
@@ -224,6 +224,8 @@ bool setup_graphics() {
|
||||
GetWindowThreadProcessId(description.OutputWindow, &pid);
|
||||
if (pid != GetCurrentProcessId()) return false;
|
||||
if (FAILED(s.swapchain->GetDevice(IID_PPV_ARGS(&s.device)))) return false;
|
||||
dingosdk::overlay::record_gpu_identity(s.device.Get());
|
||||
dingosdk::overlay::watch_device_removal(s.device.Get());
|
||||
const auto adapter = s.device->GetAdapterLuid();
|
||||
dingosdk::logging::printf(dingosdk::logging::Level::debug, dingosdk::logging::Channel::graphics, "Device=%p adapter_luid=%08lx:%08lx node_count=%u",
|
||||
s.device.Get(), static_cast<unsigned long>(adapter.HighPart),
|
||||
@@ -345,6 +347,8 @@ bool setup_graphics() {
|
||||
return false;
|
||||
}
|
||||
s.commands->SetName(L"ReSkate overlay draw");
|
||||
dingosdk::overlay::watch_draws(s.commands.Get());
|
||||
dingosdk::overlay::watch_addresses(s.commands.Get());
|
||||
s.fence->SetName(L"ReSkate overlay fence");
|
||||
s.fence_event = CreateEventW(nullptr, FALSE, FALSE, nullptr);
|
||||
if (!s.fence_event) return false;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
#include "Engine/Core/Log/types.h"
|
||||
#include "Engine/Core/Debug/crash_report.h"
|
||||
#include "Engine/Game/World/client_state.h"
|
||||
#include <cstdint>
|
||||
#include <format>
|
||||
@@ -63,6 +64,8 @@ struct Progress {
|
||||
if (((s.state >= 8 && s.state <= 13) || (s.state >= 16 && s.state <= 21)) &&
|
||||
!s.destination.empty() && s.destination != destination) {
|
||||
destination = s.destination;
|
||||
// (A crash report from here on is tagged with this level: crash_report.h.)
|
||||
dingosdk::crash_report::set_destination(std::wstring(destination.begin(), destination.end()));
|
||||
if (destination_lines++ < 4)
|
||||
emit(Level::info, std::format("Level destination: {}.", destination));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
#include "world_pool_usage.h"
|
||||
|
||||
#include "Engine/Core/Hooks/hooks.h"
|
||||
#include "Engine/Core/Log/logging.h"
|
||||
#include "Engine/Core/Platform/memory.h"
|
||||
#include "Engine/Game/Build/supported_build.h"
|
||||
#include "Engine/Game/Build/addresses.h"
|
||||
#include "Engine/Game/Build/20260929/entity_pages.h"
|
||||
#include "Engine/Game/Build/20260929/spatial_buckets.h"
|
||||
#include "Engine/Game/Build/20260929/world_pool_usage.h"
|
||||
|
||||
#include <Windows.h>
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace dingosdk {
|
||||
namespace {
|
||||
namespace contract = addr::world_pool_usage;
|
||||
namespace pages = addr::entity_pages;
|
||||
namespace spatial = addr::spatial_buckets;
|
||||
|
||||
// What the hooks have seen: snapshot page arrays and spatial roots, kept for the watcher to read.
|
||||
// A snapshot or root that is freed later stays listed; its reads then fail or find it zeroed.
|
||||
template<std::size_t N>
|
||||
struct Seen {
|
||||
std::array<std::atomic<std::uintptr_t>, N> items{};
|
||||
void add(std::uintptr_t value) noexcept {
|
||||
if (!value) return;
|
||||
for (auto& item : items) {
|
||||
auto current = item.load(std::memory_order_relaxed);
|
||||
if (current == value) return;
|
||||
if (!current && item.compare_exchange_strong(current, value)) return;
|
||||
if (current == value) return;
|
||||
}
|
||||
}
|
||||
};
|
||||
Seen<64> snapshots;
|
||||
Seen<64> roots;
|
||||
|
||||
using PageAllocator = void (*)(void*, void*, std::uint64_t*);
|
||||
using SpatialQuery = void (*)(void*, void*, void*, void*);
|
||||
std::atomic<PageAllocator> original_allocator{};
|
||||
std::atomic<SpatialQuery> original_query{};
|
||||
|
||||
void hooked_allocator(void* owner, void* allocator, std::uint64_t* page_array) {
|
||||
original_allocator.load(std::memory_order_relaxed)(owner, allocator, page_array);
|
||||
snapshots.add(reinterpret_cast<std::uintptr_t>(page_array));
|
||||
}
|
||||
// (Both callers walk the roots and drop the result, so none is passed back.)
|
||||
void hooked_query(void* root, void* a, void* b, void* c) {
|
||||
roots.add(reinterpret_cast<std::uintptr_t>(root));
|
||||
original_query.load(std::memory_order_relaxed)(root, a, b, c);
|
||||
}
|
||||
|
||||
bool matches(std::uintptr_t address, const auto& expected) noexcept {
|
||||
std::array<std::uint8_t, 32> actual{};
|
||||
return expected.size() <= actual.size() && memory::read_bytes(address, actual.data(), expected.size()) &&
|
||||
std::equal(expected.begin(), expected.end(), actual.begin());
|
||||
}
|
||||
|
||||
// The highest entity slot in use in one snapshot: its last page with anything in it, and the last
|
||||
// entry there that is not zero. Pages are read from the top down: the unused ones are zero.
|
||||
std::size_t entity_peak(std::uintptr_t page_array, std::vector<std::uint64_t>& page) noexcept {
|
||||
for (std::size_t index = pages::page_count; index-- > 0;) {
|
||||
std::uintptr_t address{};
|
||||
if (!memory::peek(page_array + index * sizeof(std::uint64_t), address) || !address) continue;
|
||||
if (!memory::peek_bytes(address, page.data(), pages::page_bytes)) continue;
|
||||
for (std::size_t entry = pages::page_entries; entry-- > 0;)
|
||||
if (page[entry]) return index * pages::page_entries + entry + 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::size_t spatial_peak(std::uintptr_t root) noexcept {
|
||||
std::uint32_t nodes{};
|
||||
if (!memory::peek(root + offsetof(spatial::RootLayout, node_count), nodes) || nodes > spatial::nodes_per_root) return 0;
|
||||
std::size_t most{};
|
||||
for (std::uint32_t node = 0; node < nodes; ++node) {
|
||||
std::uint32_t blocks{};
|
||||
const auto at = root + contract::spatial_nodes_offset + node * sizeof(spatial::BucketLayout) +
|
||||
offsetof(spatial::BucketLayout, block_count);
|
||||
if (memory::peek(at, blocks) && blocks <= spatial::blocks_per_bucket) most = std::max<std::size_t>(most, blocks);
|
||||
}
|
||||
return most;
|
||||
}
|
||||
|
||||
std::size_t render_instance_peak(std::uintptr_t base) noexcept {
|
||||
std::uintptr_t owner{}, manager{};
|
||||
std::uint32_t counter{};
|
||||
if (!memory::peek(base + contract::render_instance_owner_global, owner) || !owner ||
|
||||
!memory::peek(owner + contract::render_instance_manager_offset, manager) || !manager ||
|
||||
!memory::peek(manager + contract::render_instance_pool_offset + contract::render_instance_counter_offset, counter))
|
||||
return 0;
|
||||
return counter;
|
||||
}
|
||||
|
||||
// A new high is said when it has grown by a sixteenth of the game's own size since the last, and
|
||||
// whenever it first passes the game's own size.
|
||||
struct Peak {
|
||||
const char* name;
|
||||
std::size_t stock;
|
||||
std::size_t capacity;
|
||||
std::size_t said{};
|
||||
bool past{};
|
||||
void see(std::size_t value) {
|
||||
if (value <= said) return;
|
||||
const bool now_past = value > stock;
|
||||
if (value < said + stock / 16 && now_past == past) return;
|
||||
said = value;
|
||||
logging::log(now_past && !past ? logging::Level::warning : logging::Level::info, logging::Channel::world,
|
||||
"World pools: {} {} of ReSkate's {} ({:.0f}% of the game's own {}){}", name, value, capacity, 100.0 * value / stock, stock,
|
||||
now_past && !past ? ": past the game's own size" : "");
|
||||
past = now_past;
|
||||
}
|
||||
};
|
||||
|
||||
void watch(std::uintptr_t base) {
|
||||
SetThreadDescription(GetCurrentThread(), L"ReSkate world pool watch");
|
||||
std::vector<std::uint64_t> page(pages::page_entries);
|
||||
Peak entities{"entity slots", contract::entity_slots_stock, pages::capacity};
|
||||
Peak instances{"render instances", contract::render_instances_stock, std::size_t{512} << pages::render_instance_page_shift};
|
||||
Peak blocks{"spatial blocks in a bucket", contract::spatial_blocks_stock, spatial::blocks_per_bucket};
|
||||
for (;;) {
|
||||
Sleep(1000);
|
||||
try {
|
||||
std::size_t entity_most{}, block_most{};
|
||||
for (const auto& item : snapshots.items)
|
||||
if (const auto value = item.load(std::memory_order_relaxed)) entity_most = std::max(entity_most, entity_peak(value, page));
|
||||
for (const auto& item : roots.items)
|
||||
if (const auto value = item.load(std::memory_order_relaxed)) block_most = std::max(block_most, spatial_peak(value));
|
||||
entities.see(entity_most);
|
||||
instances.see(render_instance_peak(base));
|
||||
blocks.see(block_most);
|
||||
} catch (...) {}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool start_world_pool_usage(std::uintptr_t base, std::string& error) {
|
||||
wchar_t value[2]{};
|
||||
if (GetEnvironmentVariableW(L"RESKATE_GPU_DIAGNOSTICS", value, 2) != 1 || value[0] != L'1') return true;
|
||||
if (!matches(base + contract::entity_page_allocator_rva, contract::entity_page_allocator_entry) ||
|
||||
!matches(base + contract::spatial_query_rva, contract::spatial_query_entry) ||
|
||||
!matches(base + contract::render_instance_load_rva, contract::render_instance_load) ||
|
||||
!matches(base + contract::render_instance_counter_rva, contract::render_instance_counter)) {
|
||||
error = "the world pools do not match the supported game";
|
||||
return false;
|
||||
}
|
||||
auto* const allocator = reinterpret_cast<void*>(base + contract::entity_page_allocator_rva);
|
||||
auto* const query = reinterpret_cast<void*>(base + contract::spatial_query_rva);
|
||||
void* allocator_original{};
|
||||
void* query_original{};
|
||||
if (hook_prepare(allocator, reinterpret_cast<void*>(&hooked_allocator), &allocator_original) != HookOk ||
|
||||
hook_prepare(query, reinterpret_cast<void*>(&hooked_query), &query_original) != HookOk) {
|
||||
error = "its hooks could not be prepared";
|
||||
return false;
|
||||
}
|
||||
original_allocator.store(reinterpret_cast<PageAllocator>(allocator_original));
|
||||
original_query.store(reinterpret_cast<SpatialQuery>(query_original));
|
||||
if (hook_enable(allocator) != HookOk || hook_enable(query) != HookOk) {
|
||||
error = "its hooks could not be attached";
|
||||
return false;
|
||||
}
|
||||
std::thread(watch, base).detach();
|
||||
return true;
|
||||
}
|
||||
} // namespace dingosdk
|
||||
@@ -0,0 +1,13 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
|
||||
namespace dingosdk {
|
||||
// With RESKATE_GPU_DIAGNOSTICS on: how full the world pools ReSkate enlarges get (entity slots,
|
||||
// render instances, spatial blocks per bucket), against the game's own sizes, said in the log as
|
||||
// each grows. A level that needs more than the game's own size of one of them depends on
|
||||
// ReSkate's enlargement, and on whatever else in the game was sized to match. Read on a thread of
|
||||
// its own, once a second. Installs before executable entry; off without diagnostics.
|
||||
bool start_world_pool_usage(std::uintptr_t base, std::string& error);
|
||||
} // namespace dingosdk
|
||||
@@ -23,6 +23,17 @@ std::string utf8(const wchar_t* text) {
|
||||
result.pop_back();
|
||||
return result;
|
||||
}
|
||||
template<std::size_t N> std::string field(const std::array<wchar_t, N>& value) {
|
||||
return utf8(std::wstring(value.data(), wcsnlen(value.data(), N)).c_str());
|
||||
}
|
||||
// "Levels/Game/DingoLevel_Root/DingoLevel_Root + levels/game/x/x @ Start" -> "x": the sublevel when
|
||||
// there is one, else the level, by its last name.
|
||||
std::string map_name(std::string destination) {
|
||||
if (const auto at = destination.find(" @ "); at != std::string::npos) destination.resize(at);
|
||||
if (const auto plus = destination.find(" + "); plus != std::string::npos) destination.erase(0, plus + 3);
|
||||
if (const auto slash = destination.find_last_of('/'); slash != std::string::npos) destination.erase(0, slash + 1);
|
||||
return destination;
|
||||
}
|
||||
bool publish_dump(const std::filesystem::path& temporary, const SharedReport& report, const char* source) {
|
||||
auto dump = temporary; dump.replace_extension(L".dmp");
|
||||
Json attributes = Json::object();
|
||||
@@ -38,6 +49,13 @@ bool publish_dump(const std::filesystem::path& temporary, const SharedReport& re
|
||||
attributes["error.type"] = "crash";
|
||||
attributes["report.id"] = utf8(dump.stem().c_str());
|
||||
attributes["capture.source"] = source;
|
||||
// (Tags for grouping crashes by the level and the graphics card; absent when unknown.)
|
||||
if (const auto destination = field(report.destination); !destination.empty()) {
|
||||
attributes["game.destination"] = destination;
|
||||
attributes["game.map"] = map_name(destination);
|
||||
}
|
||||
if (const auto gpu = field(report.gpu_name); !gpu.empty()) attributes["gpu.name"] = gpu;
|
||||
if (const auto driver = field(report.gpu_driver); !driver.empty()) attributes["gpu.driver"] = driver;
|
||||
const auto log = snapshot_log(std::filesystem::path(report.directory.data()).parent_path() / L"ReSkate.log", dump);
|
||||
attributes["attachment.log"] = log == LogSnapshot::complete ? "complete" : log == LogSnapshot::tail ? "tail" : "unavailable";
|
||||
std::ofstream metadata(dump.wstring() + L".json", std::ios::binary);
|
||||
|
||||
@@ -3,10 +3,13 @@
|
||||
#include "Engine/Core/Hooks/hooks.h"
|
||||
#include "Engine/Core/Platform/launcher_support.h"
|
||||
#include "Extension/Boot/startup_interventions.h"
|
||||
#include "Extension/Console/commands.h"
|
||||
#include "Extension/Boot/steam_restart_guard.h"
|
||||
#include "Extension/Boot/user_data_redirect.h"
|
||||
#include "Extension/Boot/ea_app_block.h"
|
||||
#include "Extension/Boot/ea_service_block.h"
|
||||
#include "Extension/Boot/descriptor_heap_size.h"
|
||||
#include "Extension/Boot/eadp_errors_arena.h"
|
||||
#include "Extension/Assets/native_patch_support.h"
|
||||
#include "Extension/Assets/mod_layers.h"
|
||||
#include "Extension/Assets/loose_files.h"
|
||||
@@ -17,6 +20,7 @@
|
||||
#include "Extension/Rendering/display_startup.h"
|
||||
#include "Extension/World/native_route_lookahead.h"
|
||||
#include "Extension/World/native_entity_pages.h"
|
||||
#include "Extension/World/world_pool_usage.h"
|
||||
#include "Extension/World/physics_world_size.h"
|
||||
#include <string>
|
||||
|
||||
@@ -40,6 +44,9 @@ bool initialize_bootstrap(std::uintptr_t base) {
|
||||
return false;
|
||||
}
|
||||
logging::write(logging::Level::info, Channel::hooks, "Microsoft Detours hook service ready.");
|
||||
// The memory ledger, when RESKATE_MEMORY_TRACE asks for it: here, while the game is still
|
||||
// suspended and before the pools below are enlarged, so it sees who makes all of them.
|
||||
console::start_memory_trace_from_environment();
|
||||
error.clear(); ready = start_native_patch_support(base, error);
|
||||
if (!stage(Channel::assets, "Native InitFS/package support", ready, error)) return false;
|
||||
error.clear(); ready = start_mod_layers(base, error);
|
||||
@@ -57,6 +64,9 @@ bool initialize_bootstrap(std::uintptr_t base) {
|
||||
if (!stage(Channel::assets, "Native render-resource pools", ready, error)) return false;
|
||||
error.clear(); ready = start_native_entity_pages(base, error);
|
||||
if (!stage(Channel::world, "Native entity pages (524288 entity/update slots, 1048576 render slots, 16384 spatial blocks/bucket, 128 MiB arena minimum)", ready, error)) return false;
|
||||
// (Diagnostics only: how full those pools get. A mismatch leaves the game as it is.)
|
||||
error.clear(); ready = start_world_pool_usage(base, error);
|
||||
(void)stage(Channel::world, "World pool usage watch", ready, error);
|
||||
error.clear(); ready = start_physics_world_size(base, error);
|
||||
if (!stage(Channel::world, "Physics world pools (65000 static bodies)", ready, error)) return false;
|
||||
error.clear(); ready = start_native_route_lookahead(base, error);
|
||||
@@ -66,12 +76,19 @@ bool initialize_bootstrap(std::uintptr_t base) {
|
||||
// left registered it kept the level from ever finishing its unload (a map change that never
|
||||
// ends, which is worse than the crash). It goes back in once it can release what it skips.
|
||||
if (!stage(Channel::graphics, "Display startup settings", start_display_settings(base), {})) return false;
|
||||
// (Not needed for the game to run: a mismatch leaves the heaps at the game's size, and only
|
||||
// the densest levels outgrow them.)
|
||||
error.clear(); ready = start_descriptor_heap_size(base, error);
|
||||
(void)stage(Channel::graphics, "Descriptor heaps: 1,000,000, with 524,288 a frame", ready, error);
|
||||
error.clear(); ready = start_user_data_redirect(error);
|
||||
if (!stage(Channel::runtime, "Separate game user data", ready, error)) return false;
|
||||
error.clear(); ready = start_ea_app_block(error);
|
||||
if (!stage(Channel::runtime, "EA app launch block", ready, error)) return false;
|
||||
error.clear(); ready = start_ea_service_block(error);
|
||||
if (!stage(Channel::runtime, "EA online services block", ready, error)) return false;
|
||||
// (Not needed for the game to run: a mismatch leaves the arena at the game's size.)
|
||||
error.clear(); ready = start_eadp_errors_arena(base, error);
|
||||
(void)stage(Channel::runtime, "EA error-report arena capped at 32 MiB", ready, error);
|
||||
error.clear(); ready = start_steam_restart_guard(error);
|
||||
return stage(Channel::runtime, launcher::offline_mode() ? "Offline Steam" : "Steam restart guard", ready, error);
|
||||
}
|
||||
|
||||
+22
-2
@@ -8,8 +8,10 @@
|
||||
#include "Extension/Multiplayer/Hud/game_ui_state.h"
|
||||
#include "Extension/Music/music_inject.h"
|
||||
#include "Extension/World/texture_handle_limit.h"
|
||||
#include "Extension/Assets/native_render_resource_pool.h"
|
||||
#include "Engine/Core/Log/logging.h"
|
||||
#include "Engine/Core/Profiling/profiler.h"
|
||||
#include "Extension/Console/memory_ledger.h"
|
||||
#include "Extension/Settings/job_spin.h"
|
||||
#include "Extension/Settings/replay_drafts.h"
|
||||
#include "Engine/Game/World/client_state.h"
|
||||
@@ -201,8 +203,18 @@ Card largest_card() {
|
||||
factory->Release();
|
||||
return card;
|
||||
}
|
||||
// RESKATE_MESH_POOL_MB: the pool's size for this session in MB, whatever the card (520 to 3584);
|
||||
// for finding what size a map that fails needs.
|
||||
std::uint32_t override_kb() {
|
||||
wchar_t value[16]{};
|
||||
const auto length = GetEnvironmentVariableW(L"RESKATE_MESH_POOL_MB", value, 16);
|
||||
if (!length || length >= 16) return 0;
|
||||
const auto megabytes = std::wcstoul(value, nullptr, 10);
|
||||
return megabytes ? static_cast<std::uint32_t>(std::clamp<std::uint64_t>(std::uint64_t{megabytes} * 1024, stock_kb, most_kb)) : 0;
|
||||
}
|
||||
// KiB for the pool on this card, or 0 to keep the game's own value.
|
||||
std::uint32_t size_kb(std::uint64_t memory) {
|
||||
if (const auto chosen = override_kb()) return chosen;
|
||||
// Cards report a little under or over their nominal size.
|
||||
constexpr std::uint64_t six_gib = 6ull << 30, twelve_gib = 12ull << 30, slack = 256ull << 20;
|
||||
if (memory <= six_gib + slack) return 0;
|
||||
@@ -1257,6 +1269,7 @@ void tick(std::uintptr_t client, std::uintptr_t update) {
|
||||
dingosdk::map_download::tick();
|
||||
dingosdk::map_icons::tick();
|
||||
dingosdk::texture_handle_limit::tick();
|
||||
dingosdk::report_native_render_resources();
|
||||
// (Whether a menu of the game's is up, the music screen among them: a playlist changed then
|
||||
// is the player's own choice, and one changed in play is a speaker's music being grabbed.)
|
||||
{
|
||||
@@ -1326,12 +1339,19 @@ void tick(std::uintptr_t client, std::uintptr_t update) {
|
||||
if (entry.th32OwnerProcessID == self) ++threads;
|
||||
CloseHandle(snapshot);
|
||||
}
|
||||
// The GPU's share (its system memory is not in the committed figure), and while
|
||||
// the memory ledger is on, who holds the most (memory_ledger.h).
|
||||
std::string more;
|
||||
try {
|
||||
if (const auto gpu = dingosdk::memory_ledger::gpu_line(); !gpu.empty()) more += " " + gpu + ".";
|
||||
if (const auto owners = dingosdk::memory_ledger::owners_line(); !owners.empty()) more += " " + owners + ".";
|
||||
} catch (...) {}
|
||||
dingosdk::logging::printf(dingosdk::logging::Level::info, dingosdk::logging::Channel::runtime,
|
||||
"Memory: %.2f GB committed, %.2f GB in RAM; %d player(s) in the session, %d built as skaters (player distance %.0f m); map %s. "
|
||||
"Outside the game's own pools: %.2f GB in use of %.2f GB in %lu heap(s); %u thread(s), %lu handle(s).",
|
||||
"Outside the game's own pools: %.2f GB in use of %.2f GB in %lu heap(s); %u thread(s), %lu handle(s).%s",
|
||||
gigabytes(counters.PrivateUsage), gigabytes(counters.WorkingSetSize), players, skaters, mp.player_distance,
|
||||
(session && !mp.map.empty() ? mp.map : r.multiplayer_map).c_str(),
|
||||
gigabytes(heap_allocated), gigabytes(heap_committed), heap_count, threads, handles);
|
||||
gigabytes(heap_allocated), gigabytes(heap_committed), heap_count, threads, handles, more.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -340,7 +340,6 @@ extern "C" __declspec(dllexport) BOOL WINAPI DingoSDKDebugInitialize() {
|
||||
dingosdk::spotify_web::start();
|
||||
dingosdk::youtube_music::start(); // (the player's YouTube playlists, read before the music screen is built)
|
||||
dingosdk::spotify_dashboard::start(); // (the page in the browser that sets Spotify up) // (the player's playlists, fetched before the music screen is built)
|
||||
dingosdk::console::start_memory_trace_from_environment();
|
||||
const bool fixed_stop_provider = (global_offline_hook && progression_guard_armed) &&
|
||||
dingosdk::prepare_fixed_stop_entitlement_provider(r.base);
|
||||
record("{\"event\":\"fixed_stop_entitlement_provider_initialized\",\"prepared\":" +
|
||||
|
||||
@@ -120,6 +120,10 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32)
|
||||
target_link_libraries(dingosdk_store_copies_tests PRIVATE dingosdk_mods dingosdk_content_cache)
|
||||
target_include_directories(dingosdk_store_copies_tests PRIVATE "${PROJECT_SOURCE_DIR}")
|
||||
add_test(NAME store_copies COMMAND dingosdk_store_copies_tests "${DINGOSDK_TEST_GAME_ROOT}")
|
||||
# A tool, not a test: checks a mod's meshes as the GPU reads them (Engine/Vfs/Test/mod_mesh_check.cpp).
|
||||
add_executable(dingosdk_mod_mesh_check Engine/Vfs/Test/mod_mesh_check.cpp)
|
||||
target_link_libraries(dingosdk_mod_mesh_check PRIVATE dingosdk_mods dingosdk_frostbite dingosdk_game_archives)
|
||||
target_include_directories(dingosdk_mod_mesh_check PRIVATE "${PROJECT_SOURCE_DIR}")
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
|
||||
+5
-1
@@ -11,6 +11,7 @@ add_library(dingosdk_runtime SHARED
|
||||
Engine/Game/Abi/native_data.cpp
|
||||
Extension/Console/commands.cpp
|
||||
Extension/Console/perf_commands.cpp
|
||||
Extension/Console/memory_ledger.cpp
|
||||
Extension/Skater/console_commands.cpp
|
||||
Extension/Skater/drone_outfit.cpp
|
||||
Extension/Multiplayer/console_commands.cpp
|
||||
@@ -119,6 +120,7 @@ add_library(dingosdk_runtime SHARED
|
||||
Extension/Assets/morph_memory_pool.cpp
|
||||
Extension/Assets/native_render_resource_pool.cpp
|
||||
Extension/World/native_entity_pages.cpp
|
||||
Extension/World/world_pool_usage.cpp
|
||||
Extension/World/physics_world_size.cpp
|
||||
Extension/Rendering/display_startup.cpp
|
||||
Extension/Rendering/replay_export.cpp
|
||||
@@ -226,7 +228,9 @@ add_library(dingosdk_runtime SHARED
|
||||
Extension/Progression/neighborhood_unlock_override.cpp
|
||||
Extension/Boot/user_data_redirect.cpp
|
||||
Extension/Boot/ea_app_block.cpp
|
||||
Extension/Boot/ea_service_block.cpp)
|
||||
Extension/Boot/ea_service_block.cpp
|
||||
Extension/Boot/eadp_errors_arena.cpp
|
||||
Extension/Boot/descriptor_heap_size.cpp)
|
||||
|
||||
target_link_libraries(dingosdk_runtime PRIVATE dingosdk_logging dingosdk_profiler dingosdk_supported_build dingosdk_overlay dingosdk_console_core dingosdk_steam_restart_guard
|
||||
dingosdk_startup_interventions dingosdk_fast_travel_unlock dingosdk_local_profile dingosdk_initfs dingosdk_mods dingosdk_mod_list dingosdk_launcher_support dingosdk_miniz shell32 dingosdk_startup_window dingosdk_custom_scripts dingosdk_world_layer_scan bcrypt
|
||||
|
||||
Reference in new issue
Block a user