diff --git a/Engine/Core/Profiling/sampler.cpp b/Engine/Core/Profiling/sampler.cpp index fc62615..0f201cf 100644 --- a/Engine/Core/Profiling/sampler.cpp +++ b/Engine/Core/Profiling/sampler.cpp @@ -247,7 +247,7 @@ std::string sanitize(std::string text) { struct Sampled { HANDLE handle{}; std::uint32_t id{}; std::string label; }; // One sample: where its frames start in the pool, how many, and which thread it came from. -struct Record { std::uint32_t offset{}; std::uint16_t thread{}; std::uint8_t depth{}; }; +struct Record { std::uint32_t offset{}; std::uint16_t thread{}; std::uint8_t depth{}; std::uint32_t at_us{}; }; // The pool holds at most this many frames (64 MiB); a sample that no longer fits ends the run. constexpr std::size_t pool_limit = 8u << 20; @@ -305,7 +305,8 @@ void run(SampleRequest request, std::vector threads) { if (GetThreadContext(threads[t].handle, &context)) depth = walk(modules, context, pool.data() + used); ResumeThread(threads[t].handle); if (!depth) continue; - records.push_back({static_cast(used), static_cast(t), static_cast(depth)}); + records.push_back({static_cast(used), static_cast(t), static_cast(depth), + static_cast(std::chrono::duration_cast(now - start).count())}); used += depth; } if (!any && !full) { report.error = "The thread has ended."; break; } @@ -404,6 +405,83 @@ void run(SampleRequest request, std::vector threads) { return a.self != b.self ? a.self > b.self : a.total > b.total; }); + // ---- the slowest client frames: what a hitch is made of ---- + // An average hides a hitch: one 60 ms frame in thirty seconds is a fifth of a percent of + // the samples. A client frame is a run of samples with the client frame job on the + // stack; the longest few are listed with what they spent their time in, below that job. + std::string hitches; + if (request.target == Target::client && !records.empty()) { + const auto period_us = static_cast(1'000'000 / request.rate); + const auto stack_of = [&](const Record& record, std::vector& out) { + out.clear(); + const auto* sample = pool.data() + record.offset; + for (std::uint32_t f = 0; f < record.depth; ++f) { + const Module* module{}; + out.push_back(function_start(modules, f ? sample[f] - 1 : sample[f], &module)); + owners[out.back()] = module; + } + }; + // Where in a stack the client frame job is (counting from the leaf), or none. + const auto job_at = [&](const std::vector& frames) -> std::ptrdiff_t { + for (std::size_t f = 0; f < frames.size(); ++f) + if (name_of(frames[f], owners[frames[f]]).find("Client frame job") != std::string::npos) + return static_cast(f); + return -1; + }; + struct Frame { std::size_t first{}, last{}; std::uint32_t us{}; }; + std::vector frames; + std::vector walked; + bool inside{}; + for (std::size_t i = 0; i < records.size(); ++i) { + stack_of(records[i], walked); + const bool in_frame = job_at(walked) >= 0; + // (A gap in the samples ends a frame too: the sampler itself was held up.) + const bool continues = inside && in_frame && records[i].at_us - records[frames.back().last].at_us < period_us * 4; + if (continues) frames.back().last = i; + else if (in_frame) frames.push_back({i, i, 0}); + inside = in_frame; + } + for (auto& frame : frames) frame.us = records[frame.last].at_us - records[frame.first].at_us + period_us; + if (frames.size() >= 8) { + std::vector lengths; + for (const auto& frame : frames) lengths.push_back(frame.us); + std::ranges::sort(lengths); + const auto median = lengths[lengths.size() / 2]; + std::ranges::sort(frames, [](const Frame& a, const Frame& b) { return a.us > b.us; }); + hitches = std::format("Slowest client frames, of {} (the usual one takes {:.1f} ms of this thread). Under each: what it spent\n" + "its time in, as shares of that frame.\n", frames.size(), median / 1000.0); + for (std::size_t n = 0; n < frames.size() && n < 6; ++n) { + const auto& frame = frames[n]; + if (frame.us < median * 2 && n) break; + std::unordered_map inner; + std::uint32_t samples{}; + for (std::size_t i = frame.first; i <= frame.last; ++i) { + stack_of(records[i], walked); + const auto job = job_at(walked); + if (job < 0) continue; + ++samples; + for (std::ptrdiff_t f = 0; f < job; ++f) { + auto& counts = inner[walked[static_cast(f)]]; + counts.module = owners[walked[static_cast(f)]]; + if (!f) ++counts.self; + const auto here = walked.begin() + f; + if (std::find(walked.begin(), here, walked[static_cast(f)]) == here) ++counts.total; + } + } + hitches += std::format("\n {:.1f} ms, {:.2f} s into the sample ({} samples)\n", frame.us / 1000.0, + records[frame.first].at_us / 1e6, samples); + std::vector> order(inner.begin(), inner.end()); + // The work, not the way down to it: what nearly the whole frame passes through is left out. + std::erase_if(order, [&](const auto& entry) { return entry.second.total * 100 >= samples * 97; }); + std::ranges::sort(order, [](const auto& a, const auto& b) { return a.second.total > b.second.total; }); + for (std::size_t i = 0; i < order.size() && i < 14; ++i) + hitches += std::format(" {:5.1f}% (self {:4.1f}%) {}\n", 100.0 * order[i].second.total / samples, + 100.0 * order[i].second.self / samples, name_of(order[i].first, order[i].second.module)); + } + hitches += "\n"; + } + } + // ---- files next to ReSkate.log ---- const auto directory = logging::status().directory; SYSTEMTIME now{}; @@ -432,6 +510,7 @@ void run(SampleRequest request, std::vector threads) { out << "Skate.exe functions are Ghidra addresses (image base 0x140000000). Name them in\n" "ReSkate.labels.tsv next to Skate.exe (\"
\" per line) for the next report.\n" "Flame graph: open the .folded file at https://www.speedscope.app\n\n"; + out << hitches; out << " Self Total Function\n"; for (const auto& f : report.functions) { if (f.total * 1000.0 < per) continue; // under 0.1% diff --git a/Engine/Game/Build/20260929/media_catalog.h b/Engine/Game/Build/20260929/media_catalog.h new file mode 100644 index 0000000..b0b9273 --- /dev/null +++ b/Engine/Game/Build/20260929/media_catalog.h @@ -0,0 +1,38 @@ +#pragma once +#include "Engine/Game/Build/fingerprint.h" +#include + +// The game's media catalogue: thumbnails and labels of saved replay drafts (and the game's own +// park saves), kept as numbered folders with a manifest.json under / +// (Extension/Settings/replay_drafts.cpp; analysis/replay-editor.md, section 6). +namespace dingosdk::game::build::v20260929::media_catalog { +// The backend services object (engine.h backend_services) when it is the game's own. +inline constexpr std::uintptr_t services_vtable = 0x6070f98; +// lookup(services): the registered "catalog" container, or null. +inline constexpr Fingerprint lookup{0x90c420, { + 0x48,0x89,0x5c,0x24,0x10,0x57,0x48,0x83,0xec,0x30,0x48,0x8d,0x99,0x90,0x04,0x00, + 0x00,0x48,0x8b,0xf9,0x48,0x8b,0xcb,0xe8,0x44,0x39,0xc8,0x00,0x41,0xb9,0x07,0x00}}; +// add_container(services): makes the container and registers it. The services constructor +// calls it among its online registrations (0x282fa0), none of which run offline. +inline constexpr Fingerprint add_container{0x279720, { + 0x48,0x8b,0xc4,0x53,0x57,0x41,0x56,0x48,0x83,0xec,0x50,0x48,0x89,0x68,0x20,0x48, + 0x89,0x70,0xe0,0x4c,0x89,0x60,0xd8,0x4c,0x89,0x68,0xd0,0x4c,0x8b,0xe9,0x48,0x8b}}; +// user_added(listener, &user): the container's answer to a user signing in (its listener, at +// container + 0x60, is subscribed to "user_manager" when there is one). It makes the catalogue +// for the user's id (an engine string at user + 0x18) and keeps it at container + 0x68: the +// constructor (0x63be70) lists the folder and loads each entry (0x659450 -> 0x657480), then a +// posted task (0x659980) sets the catalogue's ready flag. +inline constexpr Fingerprint user_added{0x651bb0, { + 0x48,0x89,0x5c,0x24,0x20,0x56,0x48,0x83,0xec,0x30,0x48,0x8b,0xda,0x48,0x89,0x6c, + 0x24,0x48,0x48,0x8b,0x15,0x1f,0x6e,0xe3,0x06,0x48,0x8b,0xe9,0xb9,0x60,0x00,0x00}}; +inline constexpr std::uintptr_t container_vtable = 0x60cd120; +inline constexpr std::uintptr_t listener_vtable = 0x60cd140; +inline constexpr std::uintptr_t container_listener = 0x60; +inline constexpr std::uintptr_t container_catalogue = 0x68; +// In a catalogue: its folder (an engine string), its entries (0x130 bytes each) and whether +// its first load is done. Updating and deleting an entry need the flag. +inline constexpr std::uintptr_t catalogue_folder = 0x0; +inline constexpr std::uintptr_t catalogue_entries = 0x38; +inline constexpr std::uintptr_t catalogue_entry_size = 0x130; +inline constexpr std::uintptr_t catalogue_ready = 0x5c; +} diff --git a/Extension/Settings/replay_drafts.cpp b/Extension/Settings/replay_drafts.cpp new file mode 100644 index 0000000..c8d56c6 --- /dev/null +++ b/Extension/Settings/replay_drafts.cpp @@ -0,0 +1,187 @@ +#include "replay_drafts.h" +#include "Engine/Core/Json/json.h" +#include "Engine/Core/Log/logging.h" +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/20260929/engine.h" +#include "Engine/Game/Build/20260929/media_catalog.h" +#include "Extension/Boot/user_data_redirect.h" +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::replay_drafts { +namespace { +using Address = std::uintptr_t; +namespace mc = addr::media_catalog; +constexpr char user_id[] = "reskate"; +enum class Stage { waiting, loading, done, failed }; +Stage stage{}; +ULONGLONG signed_in_at{}; +Address watched{}; +std::size_t entries_seen{}; + +bool matches(Address base, const game::build::Fingerprint& fingerprint) noexcept { + std::array bytes{}; + return memory::peek(base + fingerprint.rva, bytes) && bytes == fingerprint.bytes; +} +Address pointer(Address at) noexcept { + Address value{}; + return memory::peek(at, value) ? value : 0; +} +// An engine string: sixteen bytes, the text inline unless the last byte's top bit says heap. +std::string engine_string(Address at) { + std::array raw{}; + if (!memory::peek(at, raw)) return {}; + if (!(raw[15] & 0x80)) return std::string(raw.data(), strnlen(raw.data(), 15)); + Address heap{}; + std::memcpy(&heap, raw.data(), sizeof(heap)); + std::array text{}; + return memory::peek_cstring(heap, text.data(), text.size()) > 0 ? std::string(text.data()) : std::string{}; +} +// What a signed-in user is to the catalogue: an id at + 0x18. +struct LocalUser { + std::array head{}; + std::array id{}; + std::array tail{}; +}; +// The game's calls, with a fault in them kept from taking the game down: drafts then stay +// as they were (unlisted). +__declspec(noinline) bool call_add_container(Address base, Address services) noexcept { + __try { + reinterpret_cast
(base + mc::add_container.rva)(services); + return true; + } __except (EXCEPTION_EXECUTE_HANDLER) { return false; } +} +__declspec(noinline) bool call_user_added(Address base, Address listener, LocalUser** user) noexcept { + __try { + reinterpret_cast(base + mc::user_added.rva)(listener, user); + return true; + } __except (EXCEPTION_EXECUTE_HANDLER) { return false; } +} +Address container_of(Address base, Address services) noexcept { + const auto container = reinterpret_cast
(base + mc::lookup.rva)(services); + return container && pointer(container) == base + mc::container_vtable && + pointer(container + mc::container_listener) == base + mc::listener_vtable ? container : 0; +} +void fail(const char* why) { + stage = Stage::failed; + logging::log(logging::Level::warning, logging::Channel::runtime, "Replay drafts: {}; drafts stay unavailable.", why); +} + +// A draft's footage is gone when the game closes, and the game lists drafts from what was +// saved this session, so the thumbnails of earlier sessions (about 2 MB each) only pile up. +// They are removed before the catalogue is made. Only a numbered folder whose manifest is a +// replay's (one source.png and a replay length) goes: anything else in the catalogue stays. +void remove_old_thumbnails() { + namespace fs = std::filesystem; + std::error_code error; + const auto folder = redirected_local_app_data() / L"Skate" / L"data" / L"catalog" / std::wstring(user_id, user_id + sizeof(user_id) - 1); + if (!fs::is_directory(folder, error)) return; + unsigned removed{}; + std::uintmax_t bytes{}; + for (const auto& item : fs::directory_iterator(folder, error)) { + const auto name = item.path().filename().wstring(); + if (!item.is_directory(error) || name.empty() || !std::all_of(name.begin(), name.end(), [](wchar_t c) { return c >= L'0' && c <= L'9'; })) + continue; + try { + std::ifstream file(item.path() / L"manifest.json", std::ios::binary); + const std::string text((std::istreambuf_iterator(file)), std::istreambuf_iterator()); + const auto manifest = Json::parse(text); + const auto& files = manifest.at("files"); + if (!files.is_object() || files.size() != 1 || !files.contains("source.png") || !manifest.at("context").contains("v_dur_i")) + continue; + } catch (...) { continue; } + std::uintmax_t size{}; + for (const auto& inside : fs::directory_iterator(item.path(), error)) + if (inside.is_regular_file(error)) size += inside.file_size(error); + if (fs::remove_all(item.path(), error) && !error) { ++removed; bytes += size; } + } + if (removed) + logging::log(logging::Level::info, logging::Channel::runtime, + "Replay drafts: {} thumbnail(s) of earlier sessions' drafts removed ({:.1f} MB).", removed, bytes / 1048576.0); +} + +void sign_in(Address base) { + if (!matches(base, mc::lookup) || !matches(base, mc::add_container) || !matches(base, mc::user_added)) + return fail("this build's media catalogue differs"); + const auto services = pointer(base + addr::engine::backend_services); + if (!services) return; // not made yet + if (pointer(services) != base + mc::services_vtable) return fail("the backend services are not the game's own"); + auto container = container_of(base, services); + if (!container) { + if (!call_add_container(base, services)) return fail("the game faulted registering its media catalogue"); + container = container_of(base, services); + if (!container) return fail("the game did not register its media catalogue"); + } + if (pointer(container + mc::container_catalogue)) { + // A user the game signed in itself: its catalogue is the one to use. + stage = Stage::done; + logging::write(logging::Level::info, logging::Channel::runtime, "Replay drafts: the game already has a media catalogue."); + return; + } + try { remove_old_thumbnails(); } catch (...) {} + static LocalUser user; + std::memcpy(user.id.data(), user_id, sizeof(user_id)); + user.id[15] = static_cast(15 - (sizeof(user_id) - 1)); + auto* held = &user; + if (!call_user_added(base, container + mc::container_listener, &held)) return fail("the game faulted making its media catalogue"); + if (!pointer(container + mc::container_catalogue)) return fail("the game did not make its media catalogue"); + stage = Stage::loading; + signed_in_at = GetTickCount64(); +} + +// The catalogue's first load ends in a task the services post to themselves. Nothing may be +// running those offline, so after a while the flag is set here: entries of earlier sessions +// are then not listed, which costs nothing (their footage went with that session). +void await_load(Address base) { + const auto services = pointer(base + addr::engine::backend_services); + const auto container = services ? container_of(base, services) : 0; + const auto catalogue = container ? pointer(container + mc::container_catalogue) : 0; + if (!catalogue) return fail("the media catalogue went away"); + std::uint8_t ready{}; + if (!memory::peek(catalogue + mc::catalogue_ready, ready)) return fail("the media catalogue cannot be read"); + const bool waited = GetTickCount64() - signed_in_at > 5000; + if (!ready && !waited) return; + if (!ready) *reinterpret_cast(catalogue + mc::catalogue_ready) = 1; + const auto first = pointer(catalogue + mc::catalogue_entries), last = pointer(catalogue + mc::catalogue_entries + 8); + stage = Stage::done; + watched = catalogue; + entries_seen = last >= first ? (last - first) / mc::catalogue_entry_size : 0; + logging::log(logging::Level::info, logging::Channel::runtime, + "Replay drafts: media catalogue ready at {} with {} entries{}.", engine_string(catalogue + mc::catalogue_folder), + last >= first ? (last - first) / mc::catalogue_entry_size : 0, + ready ? "" : " (its load never reported back; marked ready)"); +} +// Says when an entry is made or removed: a saved draft is one more, a deleted one is one fewer. +void watch_entries(Address base) { + const auto services = pointer(base + addr::engine::backend_services); + const auto container = services ? container_of(base, services) : 0; + if (!container || pointer(container + mc::container_catalogue) != watched) { watched = 0; return; } + const auto first = pointer(watched + mc::catalogue_entries), last = pointer(watched + mc::catalogue_entries + 8); + const std::size_t count = last >= first ? (last - first) / mc::catalogue_entry_size : 0; + if (count == entries_seen) return; + logging::log(logging::Level::info, logging::Channel::runtime, "Replay drafts: the media catalogue went from {} to {} entries.", + entries_seen, count); + entries_seen = count; +} +} // namespace + +void on_client_tick(std::uintptr_t base, std::uint32_t state) noexcept { + if (stage == Stage::done && watched && base) { + try { watch_entries(base); } catch (...) { watched = 0; } + return; + } + if (stage == Stage::done || stage == Stage::failed || !base) return; + // In a level (13) or its menu (21): the services are long made by then. + if (state != 13 && state != 21) return; + try { + if (stage == Stage::waiting) sign_in(base); + else await_load(base); + } catch (...) { stage = Stage::failed; } +} +} // namespace dingosdk::replay_drafts diff --git a/Extension/Settings/replay_drafts.h b/Extension/Settings/replay_drafts.h new file mode 100644 index 0000000..056a99f --- /dev/null +++ b/Extension/Settings/replay_drafts.h @@ -0,0 +1,14 @@ +#pragma once +#include + +// Replay drafts. The replay editor saves a draft as a pin on the recording, which is local, +// and a thumbnail and label in the game's media catalogue, which the Drafts screen lists from. +// The game makes that catalogue for the user signed in to its backend; with no backend there +// is neither the service nor a user, and a saved draft has nowhere to be listed. This gives +// the game the service and a local user, both with its own functions: the catalogue is then +// the one the game keeps on disk, and its Drafts screen works as it was built. +// A draft's footage stays in memory, as it does in the game: it does not outlive the session. +namespace dingosdk::replay_drafts { +// On the client thread, at the world model's pace, with the client's state. +void on_client_tick(std::uintptr_t base, std::uint32_t state) noexcept; +} diff --git a/Runtime/client_tick.cpp b/Runtime/client_tick.cpp index 09ca0c9..e6dc18f 100644 --- a/Runtime/client_tick.cpp +++ b/Runtime/client_tick.cpp @@ -6,6 +6,7 @@ #include "Engine/Core/Log/logging.h" #include "Engine/Core/Profiling/profiler.h" #include "Extension/Settings/job_spin.h" +#include "Extension/Settings/replay_drafts.h" #include "Engine/Game/World/client_state.h" #include "Extension/HallOfMeat/hall_of_meat.h" #include "Extension/RoadRash/road_rash.h" @@ -798,6 +799,7 @@ void update_model(std::uintptr_t client, TickState& frame) { const auto& current_level = description.lm_level.empty() ? description.level : description.lm_level; dingosdk::live_mods::set_current_level(current_level); dingosdk::hall_of_meat::set_level(current_level); + dingosdk::replay_drafts::on_client_tick(r.base, state); // A live mod apply hands over the destinations its mods declare. A map // being played that is no longer among them (its mod disabled or deleted) // cannot stay loaded: the player goes to San Van. diff --git a/cmake/Runtime.cmake b/cmake/Runtime.cmake index 16ab5d9..d95bfbf 100644 --- a/cmake/Runtime.cmake +++ b/cmake/Runtime.cmake @@ -70,6 +70,7 @@ add_library(dingosdk_runtime SHARED Extension/Settings/job_spin.cpp Extension/Settings/engine_tweaks.cpp Extension/Settings/replay_guard.cpp + Extension/Settings/replay_drafts.cpp Extension/UI/ui_pointer_skip.cpp Extension/Settings/named_settings.cpp Extension/World/console_commands.cpp