diff --git a/.gitignore b/.gitignore index 8e41f97d..70500ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,7 @@ analysis/raw/**/*.res /analysis/ /docs/ /release + +# Drone model sources (assets/models/fpv/CREDITS.md): large; only the prepared drone.glb is kept. +/assets/models/fpv/dji_avata2.glb +/assets/models/fpv/fpv_drone.glb diff --git a/BuiltinPacker/main.cpp b/BuiltinPacker/main.cpp new file mode 100644 index 00000000..e93853d2 --- /dev/null +++ b/BuiltinPacker/main.cpp @@ -0,0 +1,96 @@ +// Builds a built-in content pack (Engine/Vfs/builtin_packs.h) from its zip in assets/builtin: +// ReSkateBuiltinPacker +// The zip's files go into one plain archive (builtin_pack_format.h) with files of the same name +// next to each other (a Board Break deck pack carries much the same files once for every deck), +// which is then compressed whole with zstd and long-distance matching. Zip compresses each file +// on its own and finds none of that; this way each repeat costs next to nothing. +#include "Engine/Vfs/builtin_pack_format.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct File { + std::string name; + std::vector bytes; +}; +std::string base_name(const std::string& path) { + const auto slash = path.find_last_of('/'); + return slash == std::string::npos ? path : path.substr(slash + 1); +} +template void put(std::vector& out, T value) { + for (std::size_t i = 0; i < sizeof(T); ++i) out.push_back(static_cast(value >> (8 * i))); +} +int fail(const std::string& message) { + std::cerr << "ReSkateBuiltinPacker: " << message << '\n'; + return 1; +} +} + +int main(int argc, char** argv) { + if (argc != 3) return fail("usage: ReSkateBuiltinPacker "); + std::ifstream input(argv[1], std::ios::binary); + const std::vector zip_bytes{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + if (zip_bytes.empty()) return fail(std::string("cannot read ") + argv[1]); + mz_zip_archive zip{}; + if (!mz_zip_reader_init_mem(&zip, zip_bytes.data(), zip_bytes.size(), 0)) return fail("not a zip"); + std::vector files; + std::uint64_t total{}; + for (mz_uint index = 0; index < mz_zip_reader_get_num_files(&zip); ++index) { + mz_zip_archive_file_stat info{}; + if (!mz_zip_reader_file_stat(&zip, index, &info)) return fail("an unreadable entry"); + if (info.m_is_directory) continue; + File file; + file.name = info.m_filename; + if (!dingosdk::mods::builtin_pack::safe_name(file.name)) return fail("an entry the game may not have: " + file.name); + file.bytes.resize(static_cast(info.m_uncomp_size)); + if (!file.bytes.empty() && !mz_zip_reader_extract_to_mem(&zip, index, file.bytes.data(), file.bytes.size(), 0)) + return fail("cannot unpack " + file.name); + total += file.bytes.size(); + files.push_back(std::move(file)); + } + mz_zip_reader_end(&zip); + if (total > dingosdk::mods::builtin_pack::max_bytes) return fail("the pack is larger than a built-in pack may be"); + // Same names together (a deck pack's decks// for every deck), then by path. + std::stable_sort(files.begin(), files.end(), [](const File& a, const File& b) { + const auto ka = base_name(a.name), kb = base_name(b.name); + return ka != kb ? ka < kb : a.name < b.name; + }); + + std::vector plain; + plain.insert(plain.end(), dingosdk::mods::builtin_pack::magic.begin(), dingosdk::mods::builtin_pack::magic.end()); + put(plain, static_cast(files.size())); + for (const auto& file : files) { + put(plain, static_cast(file.name.size())); + plain.insert(plain.end(), file.name.begin(), file.name.end()); + put(plain, file.bytes.size()); + } + for (const auto& file : files) plain.insert(plain.end(), file.bytes.begin(), file.bytes.end()); + + ZSTD_CCtx* context = ZSTD_createCCtx(); + if (!context) return fail("out of memory"); + // A window to cover the whole pack (within what any decoder takes by default, 2^27), so a + // file repeated anywhere in it is found. + int window = 10; + while (window < dingosdk::mods::builtin_pack::max_window_log && (std::size_t{1} << window) < plain.size()) ++window; + ZSTD_CCtx_setParameter(context, ZSTD_c_compressionLevel, 19); + ZSTD_CCtx_setParameter(context, ZSTD_c_enableLongDistanceMatching, 1); + ZSTD_CCtx_setParameter(context, ZSTD_c_windowLog, window); + ZSTD_CCtx_setParameter(context, ZSTD_c_checksumFlag, 1); + std::vector packed(ZSTD_compressBound(plain.size())); + const auto size = ZSTD_compress2(context, packed.data(), packed.size(), plain.data(), plain.size()); + ZSTD_freeCCtx(context); + if (ZSTD_isError(size)) return fail(std::string("zstd: ") + ZSTD_getErrorName(size)); + packed.resize(size); + std::ofstream output(argv[2], std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(packed.data()), static_cast(packed.size())); + if (!output) return fail(std::string("cannot write ") + argv[2]); + std::cout << argv[1] << ": " << files.size() << " files, " << total << " bytes -> " << packed.size() << " bytes (" + << zip_bytes.size() << " as a zip)\n"; + return 0; +} diff --git a/CMakeLists.txt b/CMakeLists.txt index 007c40df..f4cf838a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,21 @@ endif() option(DINGOSDK_BUILD_MULTIPLAYER_TESTS "Build multiplayer protocol regression tests" OFF) if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) enable_testing() + add_executable(dingosdk_map_fetch_policy_tests Extension/Multiplayer/Test/map_fetch_policy_tests.cpp) + target_include_directories(dingosdk_map_fetch_policy_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME map_fetch_policy COMMAND dingosdk_map_fetch_policy_tests) if(WIN32) + add_executable(dingosdk_https_download_timeout_tests Engine/Vfs/Test/https_download_timeout_tests.cpp) + target_include_directories(dingosdk_https_download_timeout_tests PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_https_download_timeout_tests PRIVATE winhttp) + add_test(NAME https_download_timeout COMMAND dingosdk_https_download_timeout_tests + "${CMAKE_CURRENT_BINARY_DIR}/https-download-fixtures") + add_executable(dingosdk_map_download_timeout_tests Extension/Assets/Test/map_download_timeout_tests.cpp + Engine/Vfs/thunderstore_package.cpp) + target_include_directories(dingosdk_map_download_timeout_tests PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_map_download_timeout_tests PRIVATE dingosdk_json) + add_test(NAME map_download_timeout COMMAND dingosdk_map_download_timeout_tests + "${CMAKE_CURRENT_BINARY_DIR}/map-download-fixtures") add_executable(dingosdk_voice_tests Extension/Multiplayer/Test/voice_chat_tests.cpp Extension/Multiplayer/Net/protocol.cpp Extension/Multiplayer/Net/delta_codec.cpp Extension/Multiplayer/Net/wire_codec.cpp) target_link_libraries(dingosdk_voice_tests PRIVATE dingosdk_lz4 dingosdk_zstd) @@ -113,6 +127,24 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) dingosdk_lz4 dingosdk_zstd dingosdk_miniz dingosdk_word_filter OpenSSL::Crypto dl pthread) endif() add_test(NAME server_park_mods COMMAND dingosdk_server_park_tests "${PROJECT_SOURCE_DIR}/Server/Test/fixtures/park_mods") + # Tools locked for single players (lock, throwdown_locks), as each player's game reads its roster. + add_executable(dingosdk_server_lock_tests Server/Test/server_lock_tests.cpp Server/server_host.cpp Server/server_party.cpp + Extension/Multiplayer/Session/party_book.cpp Server/server_config.cpp Server/server_update.cpp Server/server_release.cpp + $<$:Launcher/updater.cpp> Server/global_bans.cpp Server/server_words.cpp Extension/Multiplayer/developer_identity.cpp + Extension/Multiplayer/word_lists.cpp Extension/Multiplayer/Net/protocol.cpp Extension/Multiplayer/Net/delta_codec.cpp + Extension/Multiplayer/Net/wire_codec.cpp Extension/Multiplayer/Remote/playback_buffers.cpp + Extension/Multiplayer/Session/password.cpp Server/server_activity.cpp Server/server_votes.cpp Server/server_commands.cpp + Server/server_parks.cpp Extension/Throwdowns/throwdown_wire.cpp) + target_include_directories(dingosdk_server_lock_tests PRIVATE "${PROJECT_SOURCE_DIR}") + target_include_directories(dingosdk_server_lock_tests SYSTEM PRIVATE "${PROJECT_SOURCE_DIR}/External/steam_networking") + if(WIN32) + target_link_libraries(dingosdk_server_lock_tests PRIVATE dingosdk_launcher_support dingosdk_world_layer_scan dingosdk_json + dingosdk_lz4 dingosdk_zstd dingosdk_logging dingosdk_miniz dingosdk_word_filter dingosdk_https winhttp bcrypt winmm) + else() + target_link_libraries(dingosdk_server_lock_tests PRIVATE dingosdk_launcher_support dingosdk_world_layer_scan dingosdk_json + dingosdk_lz4 dingosdk_zstd dingosdk_miniz dingosdk_word_filter OpenSSL::Crypto dl pthread) + endif() + add_test(NAME server_locks COMMAND dingosdk_server_lock_tests) add_executable(dingosdk_server_release_tests Server/Test/server_release_tests.cpp Server/server_release.cpp) target_include_directories(dingosdk_server_release_tests PRIVATE "${PROJECT_SOURCE_DIR}") target_link_libraries(dingosdk_server_release_tests PRIVATE dingosdk_json) @@ -165,6 +197,12 @@ endif() if(DINGOSDK_BUILD_MULTIPLAYER_TESTS AND WIN32) add_executable(dingosdk_velocity_boost_tests Engine/Game/Skater/Test/velocity_boost_tests.cpp) add_test(NAME velocity_boost COMMAND dingosdk_velocity_boost_tests) + add_executable(dingosdk_drone_radio_tests Extension/Skater/Test/drone_radio_tests.cpp) + target_include_directories(dingosdk_drone_radio_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME drone_radio COMMAND dingosdk_drone_radio_tests) + add_executable(dingosdk_drone_flight_tests Extension/Skater/Test/drone_flight_tests.cpp) + target_include_directories(dingosdk_drone_flight_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME drone_flight COMMAND dingosdk_drone_flight_tests) if(EXISTS "${PROJECT_SOURCE_DIR}/tests/first_person_spring_tests.cpp") add_executable(dingosdk_first_person_spring_tests tests/first_person_spring_tests.cpp) target_include_directories(dingosdk_first_person_spring_tests PRIVATE "${PROJECT_SOURCE_DIR}") @@ -316,6 +354,11 @@ if(DINGOSDK_BUILD_TRAINER_TESTS) add_executable(dingosdk_trainer_landing_tests Extension/Trainer/Test/landing_tests.cpp) target_include_directories(dingosdk_trainer_landing_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME trainer_landing COMMAND dingosdk_trainer_landing_tests) + # Built-in presets: Hardcore goes at least as far as Realistic, class names are spelt right, and + # the player's own presets never keep a built-in's name. + add_executable(dingosdk_trainer_presets_tests Extension/Trainer/Test/presets_tests.cpp Extension/Trainer/trainer_presets.cpp) + target_include_directories(dingosdk_trainer_presets_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME trainer_presets COMMAND dingosdk_trainer_presets_tests) add_executable(dingosdk_trainer_asset_list Extension/Trainer/Test/asset_list.cpp) target_include_directories(dingosdk_trainer_asset_list PRIVATE "${PROJECT_SOURCE_DIR}") target_link_libraries(dingosdk_trainer_asset_list PRIVATE dingosdk_game_archives) @@ -336,6 +379,16 @@ if(DINGOSDK_BUILD_BOARD_BREAK_TESTS) target_include_directories(dingosdk_board_break_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME board_break COMMAND dingosdk_board_break_tests) endif() +option(DINGOSDK_BUILD_REPLAY_KEYFRAMES_TESTS "Build replay keyframe smoothing regression tests" OFF) +if(DINGOSDK_BUILD_REPLAY_KEYFRAMES_TESTS) + enable_testing() + # The smoother curve: stock at 0, every key met, C2 at 1, and a framing segment the game's spline plays as + # meant; the saved settings read back as written. + add_executable(dingosdk_replay_keyframes_tests Extension/ReplayKeyframes/Test/replay_keyframes_tests.cpp + Extension/ReplayKeyframes/replay_curve.cpp Extension/ReplayKeyframes/replay_settings.cpp) + target_include_directories(dingosdk_replay_keyframes_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME replay_keyframes COMMAND dingosdk_replay_keyframes_tests) +endif() option(DINGOSDK_BUILD_HALL_OF_MEAT_TESTS "Build Hall of Meat regression tests" OFF) if(DINGOSDK_BUILD_HALL_OF_MEAT_TESTS) enable_testing() @@ -353,6 +406,11 @@ if(DINGOSDK_BUILD_HALL_OF_MEAT_TESTS) endif() get_property(project_targets DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) foreach(target IN LISTS project_targets) + # (A build step, such as packing the built-in content, compiles nothing.) + get_target_property(target_kind ${target} TYPE) + if(target_kind STREQUAL "UTILITY") + continue() + endif() if(NOT target MATCHES "^(dingosdk_imgui|dingosdk_detours|dingosdk_sqlite|dingosdk_lz4|dingosdk_zstd|dingosdk_miniz|dingosdk_spotify_device)$") dingosdk_configure_target(${target}) endif() diff --git a/Engine/Core/Console/command_registry.h b/Engine/Core/Console/command_registry.h index 841aa0bf..eb75aa01 100644 --- a/Engine/Core/Console/command_registry.h +++ b/Engine/Core/Console/command_registry.h @@ -139,6 +139,23 @@ template class Registry { const auto it = lookup_.find(lower(name)); return it == lookup_.end() ? nullptr : &entries_[it->second]; } + // The other entries whose name or an alias goes on from a name by more words: + // `perf hud` and `perf status` under `perf`. + std::vector under(std::string_view name) const { + const auto prefix = std::string(name) + ' '; + const auto *self = find(name); // (`log` is also `log status`) + std::vector result; + for (const auto &entry : entries_) { + if (&entry == self) + continue; + bool below = ascii_case_insensitive_starts_with(entry.name, prefix); + for (const auto &alias : entry.aliases) + below = below || ascii_case_insensitive_starts_with(alias, prefix); + if (below) + result.push_back(&entry); + } + return result; + } static std::string usage(const Entry &entry) { std::string result = entry.name; for (const auto &arg : entry.arguments) { @@ -259,7 +276,9 @@ template class Registry { values.push_back(std::move(*value)); } if (index != args.size()) { - output("error: Too many arguments. Usage: " + usage(*entry)); + // `perf hdu`: neither an argument of perf nor a command under it. + output("error: Too many arguments. Usage: " + usage(*entry) + + (under(entry->name).empty() ? "" : ". Use help " + entry->name + " for the commands under it.")); return false; } if (!entry->run) { diff --git a/Engine/Core/Debug/Test/crash_report_tests.cpp b/Engine/Core/Debug/Test/crash_report_tests.cpp index a77cb78b..2c41c795 100644 --- a/Engine/Core/Debug/Test/crash_report_tests.cpp +++ b/Engine/Core/Debug/Test/crash_report_tests.cpp @@ -3,15 +3,18 @@ #include "Engine/Core/Debug/crash_report.h" #include "Engine/Core/Debug/native_dump.h" #include "Engine/Core/Debug/upload.h" -#include "Engine/Core/Debug/multipart.h" +#include "Engine/Core/Debug/report_envelope.h" +#include "Engine/Core/Debug/report_details.h" #include "Engine/Core/Debug/protocol.h" #include "Engine/Core/Debug/trace.h" #include "crash_report_config.h" #include "Engine/Core/Log/logging.h" +#include #include #include #include #include +#include using namespace dingosdk; using namespace dingosdk::crash_report; @@ -30,6 +33,19 @@ LONG WINAPI native_child_handler(EXCEPTION_POINTERS* pointers) { CopyFileW(path.c_str(), (native_fixture_directory / L"duplicate.dmp").c_str(), TRUE); ExitProcess(42); } +// A heap block only the faulting stack points at: present in a dump only with the memory the +// stacks reference. Made at run time, so its bytes are nowhere else in the process. +// (Filled in place: a copy left on a stack would be in every dump.) +__declspec(noinline) void fill_marker(std::array& bytes) { + for (std::size_t i = 0; i < bytes.size(); ++i) bytes[i] = static_cast((i * 37 + 11) ^ 0xa5); +} +// The game's own fatal handler: ReSkate's import hook asks for its dump first, then the game +// writes its own (game_fatal_dump.cpp). +LONG WINAPI fatal_child_handler(EXCEPTION_POINTERS* pointers) { + capture_game_fatal(GetCurrentThreadId(), pointers); + return native_child_handler(pointers); +} +enum class Mode { direct, native, fatal }; std::string read(const std::filesystem::path& path) { std::ifstream file(path, std::ios::binary); return {std::istreambuf_iterator(file), std::istreambuf_iterator()}; @@ -49,6 +65,12 @@ void endpoint_checks() { check(accepted_response(200, "c350c4ee5e2b4b779b2e5e4a2b0e1b9d", id), "Unhyphenated receipt rejected"); for (const auto* response : {"", "\n", "OK", R"({"detail":"ok"})", "c350c4ee-5e2b-4b77-9b2e", "c350c4ee-5e2b-4b77-9b2e-5e4a2b0e1b9d-00"}) check(!accepted_response(200, response, id), "Unconfirmed submission accepted"); + check(accepted_response(200, R"({"id":"c350c4ee5e2b4b779b2e5e4a2b0e1b9d"})", id) && id == "c350c4ee5e2b4b779b2e5e4a2b0e1b9d", + "Envelope receipt rejected"); + check(!accepted_response(200, R"({"id":"short"})", id) && !accepted_response(200, R"({"id":)", id), "Broken envelope receipt accepted"); + check(envelope_endpoint(L"https://sentry.example.com/api/2/minidump/?sentry_key=0123abcd") == + L"https://sentry.example.com/api/2/envelope/?sentry_key=0123abcd" && envelope_endpoint(L"https://example.com/post").empty(), + "Crash envelope endpoint not derived from the minidump endpoint"); check(!accepted_response(429, receipt, id), "Rate limited submission accepted"); check(!accepted_response(500, receipt, id), "Failed submission accepted"); } @@ -96,54 +118,92 @@ void attachment_checks(const std::filesystem::path& root) { std::ofstream(log) << "a later game session"; check(read(log_attachment_path(dump)) == original_log, "Queued attachment changed with live log"); auto attributes = Json::object(); attributes["version"] = "fixture-build"; attributes["attachment.log"] = "complete"; + std::ofstream(event_details_path(dump), std::ios::binary) + << R"({"breadcrumbs":{"values":[{"message":"last crash message"}]},"user":{"id":"0123456789abcdef0123456789abcdef"},)" + R"("contexts":{"reskate":{"mods":["A 1.0"]}},"tags":{"version":"replaced"},"release":"replaced"})"; + const std::string event_id = "0123456789abcdef0123456789abcdef"; { - MultipartReport body(dump, attributes, "fixture-boundary"); + ReportEnvelope body(dump, attributes, event_id); check(body.error() == 0, "Cannot prepare attached report"); std::string wire; check(body.write([&](const void* data, DWORD bytes) { wire.append(static_cast(data), bytes); return true; }), "Cannot stream attached report"); - check(wire.size() == body.size(), "Multipart Content-Length does not match transmitted bytes"); - // Parse the emitted MIME parts independently to check names, boundaries and raw payloads. - std::map> parts; - const std::string delimiter = "--fixture-boundary"; - std::size_t cursor{}; - for (;;) { - check(wire.compare(cursor, delimiter.size(), delimiter) == 0, "Malformed MIME delimiter"); - cursor += delimiter.size(); - if (wire.compare(cursor, 4, "--\r\n") == 0) { check(cursor + 4 == wire.size(), "Trailing MIME data"); break; } - check(wire.compare(cursor, 2, "\r\n") == 0, "Missing MIME header separator"); cursor += 2; - const auto end = wire.find("\r\n\r\n", cursor), next = wire.find("\r\n" + delimiter, end + 4); - check(end != std::string::npos && next != std::string::npos, "Unterminated MIME part"); - const auto header = wire.substr(cursor, end - cursor); - const auto name = header.find("name=\""); - check(name != std::string::npos, "Missing MIME field name"); - const auto name_end = header.find('"', name + 6); - check(name_end != std::string::npos, "Invalid MIME field name"); - check(parts.emplace(header.substr(name + 6, name_end - name - 6), - std::make_pair(header, wire.substr(end + 4, next - end - 4))).second, "Duplicate MIME field"); - cursor = next + 2; + check(wire.size() == body.size(), "Envelope Content-Length does not match transmitted bytes"); + // Read the envelope back independently: a header line, then each item's header line and + // exactly `length` bytes of payload, each followed by a newline. + std::size_t cursor = wire.find('\n'); + check(cursor != std::string::npos, "Envelope has no header line"); + const auto header = Json::parse(wire.substr(0, cursor)); + check(header.at("event_id").string() == event_id, "Envelope header lacks the event ID"); + ++cursor; + std::vector> items; + while (cursor < wire.size()) { + const auto line_end = wire.find('\n', cursor); + check(line_end != std::string::npos, "Unterminated item header"); + auto item = Json::parse(wire.substr(cursor, line_end - cursor)); + const auto length = item.at("length").get(); + check(line_end + 1 + length <= wire.size(), "Item longer than the envelope"); + items.emplace_back(std::move(item), wire.substr(line_end + 1, length)); + cursor = line_end + 1 + length; + check(cursor == wire.size() || wire[cursor] == '\n', "Item payload not followed by a newline"); + if (cursor < wire.size()) ++cursor; } - const auto event = Json::parse(parts.at("sentry").second); - check(parts.size() == 4 && event.at("tags").at("version").string() == "fixture-build" && - event.at("release").string() == "reskate@fixture-build", "Attributes lost during multipart upload"); - check(parts.at("upload_file_minidump").second == dump_bytes, "Dump bytes corrupted"); - check(parts.at("attachment_ReSkate.log").second == original_log && - parts.at("attachment_ReSkate.log").first.find("filename=\"ReSkate.log\"") != std::string::npos, "Log is not a named file attachment"); - check(Json::parse(parts.at("attachment_report.json").second).at("version").string() == "fixture-build", "Report details attachment missing"); + check(items.size() == 4, "Envelope does not hold the event, dump, report and log"); + const auto& [event_item, event_payload] = items[0]; + const auto event = Json::parse(event_payload); + check(event_item.at("type").string() == "event" && event.at("event_id").string() == event_id && + event.at("platform").string() == "native" && event.at("level").string() == "fatal" && + event.at("timestamp").get() > 1.7e9, "Event item wrong"); + check(event.at("tags").at("version").string() == "fixture-build" && + event.at("release").string() == "reskate@fixture-build", "Attributes lost from the event"); + check(event.at("breadcrumbs").at("values")[0].at("message").string() == "last crash message" && + event.at("user").at("id").string() == "0123456789abcdef0123456789abcdef" && + event.at("contexts").at("reskate").at("mods")[0].string() == "A 1.0", "Event details did not reach the event"); + check(items[1].first.at("type").string() == "attachment" && items[1].first.at("attachment_type").string() == "event.minidump" && + items[1].second == dump_bytes, "Dump not the event's minidump, or its bytes corrupted"); + check(items[2].first.at("filename").string() == "report.json" && + Json::parse(items[2].second).at("version").string() == "fixture-build", "Report details attachment missing"); + check(items[3].first.at("filename").string() == "ReSkate.log" && items[3].second == original_log, "Log is not a named file attachment"); } + check(ReportEnvelope(dump, attributes, "not-an-id").error() != 0, "An envelope without a proper event ID"); { - MultipartReport body(dump, attributes, "fixture-boundary"); + ReportEnvelope body(dump, attributes, event_id); check(!body.write([](const void*, DWORD) { return false; }), "Stream failure ignored"); } std::filesystem::remove(log_attachment_path(dump)); - check(MultipartReport(dump, attributes, "fixture-boundary").error() != 0, "Missing queued log silently dropped"); + check(ReportEnvelope(dump, attributes, event_id).error() != 0, "Missing queued log silently dropped"); attributes.erase("attachment.log"); - check(MultipartReport(dump, attributes, "fixture-boundary").error() == 0, "Legacy report without logs blocked"); + check(ReportEnvelope(dump, attributes, event_id).error() == 0, "Legacy report without logs blocked"); const std::string tail(max_log_bytes, 'x'); std::ofstream(log, std::ios::binary) << "discard this old prefix" << tail; check(snapshot_log(log, dump) == LogSnapshot::tail && read(log_attachment_path(dump)) == tail, "Large log did not retain bounded tail"); check(std::filesystem::file_size(log) > max_log_bytes, "Original log was truncated"); check(snapshot_log(directory / L"missing.log", directory / L"missing.dmp") == LogSnapshot::unavailable, "Missing source log reported as attached"); } +void details_checks(const std::filesystem::path& root) { + const auto directory = root / L"details"; + std::filesystem::create_directories(directory); + const auto log = directory / L"ReSkate.log"; + std::ofstream(log, std::ios::binary) + << "[2026-10-16 16:56:01.000] [1:2] SDK: [Runtime] [DEBUG] old debug line\r\n" + << "[2026-10-16 16:56:02.000] [1:2] Native(C): [Level] [INFO] Level active [13]\r\n" + << "[2026-10-16 16:56:03.000] [1:2] Native(M): [Graphics] [WARNING] Descriptor heap nearly full\r\n" + << "[2026-10-16 16:56:04.000] [1:2] SDK: [Runtime] [DEBUG] recent debug line\r\n" + << "a continuation without a time \xff\r\n"; + const auto crumbs = log_breadcrumbs(log, 10, 2); + const auto& values = crumbs.at("values"); + check(values.size() == 4, "Breadcrumbs kept a debug line from before the last few, or lost one"); + check(values[0].at("message").string() == "Level active [13]" && values[0].at("level").string() == "info" && + values[0].at("category").string() == "Native(C)/Level", "Breadcrumb text, level or category wrong"); + check(values[1].at("level").string() == "warning" && values[2].at("level").string() == "debug", "Breadcrumb levels wrong"); + check(values[3].at("message").string() == "a continuation without a time ?" && + values[3].at("timestamp").get() == values[2].at("timestamp").get(), "A line without a time not kept with the one before"); + check(values[1].at("timestamp").get() - values[0].at("timestamp").get() == 1.0 && + values[0].at("timestamp").get() > 1.7e9, "Breadcrumb times wrong"); + check(log_breadcrumbs(directory / L"missing.log").is_null(), "Breadcrumbs from a missing log"); + const auto id = install_id(directory); + check(id.size() == 32 && install_id(directory) == id, "Install ID not made once and kept"); + check(crash_site(log).empty(), "A crash site read from something that is not a dump"); +} void queue_checks(const std::filesystem::path& root) { const auto queue = root / L"queue"; std::filesystem::create_directories(queue); @@ -165,6 +225,19 @@ void queue_checks(const std::filesystem::path& root) { CloseHandle(lock.release()); check(upload_pending(L"test", queue, success) == 1, "Confirmed upload not counted"); check(!std::filesystem::exists(dump) && !std::filesystem::exists(metadata) && !std::filesystem::exists(log_attachment_path(dump)), "Confirmed report or attachment not removed"); + const auto sent = queue / L"sent" / L"fixture.dmp"; + check(std::filesystem::exists(sent) && read(log_attachment_path(sent)) == "original-session-log" && + std::filesystem::exists(sent.wstring() + L".json"), "Confirmed report not kept in sent"); + for (std::size_t i = 0; i < kept_sent_reports + 3; ++i) { + const auto path = queue / std::format(L"sent-{}.dmp", i); + std::ofstream(path) << "synthetic test data"; + std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now() + std::chrono::seconds(i + 1)); + keep_sent_report(path); + } + std::size_t kept{}; + for (const auto& entry : std::filesystem::directory_iterator(queue / L"sent")) kept += entry.path().extension() == L".dmp"; + check(kept == kept_sent_reports && !std::filesystem::exists(sent) && std::filesystem::exists(queue / L"sent" / L"sent-12.dmp"), + "Sent reports not pruned to the newest"); check(queue_directory(root, L"https://a/token/minidump") != queue_directory(root, L"https://b/token/minidump"), "Different destinations share queue"); for (std::size_t i = 0; i < max_pending_reports + 2; ++i) { const auto path = queue / std::format(L"{}.dmp", i); @@ -190,7 +263,8 @@ void queue_checks(const std::filesystem::path& root) { upload_pending(L"test", queue, failure); check(!std::filesystem::exists(expired) && !std::filesystem::exists(log_attachment_path(expired)), "Expired report or attachment retained"); } -void crash_check(const std::filesystem::path& root, bool native) { +void crash_check(const std::filesystem::path& root, Mode mode) { + const bool native = mode != Mode::direct; std::filesystem::create_directories(root); std::array executable{}; GetModuleFileNameW(nullptr, executable.data(), static_cast(executable.size())); @@ -200,7 +274,8 @@ void crash_check(const std::filesystem::path& root, bool native) { std::filesystem::copy_file(executable.data(), child_path); std::filesystem::copy_file(std::filesystem::path(executable.data()).parent_path() / L"ReSkateLauncher.exe", root / L"ReSkateLauncher.exe"); } - auto command = std::format(L"\"{}\" {} \"{}\"", child_path.wstring(), native ? L"--native-crash" : L"--crash", root.wstring()); + auto command = std::format(L"\"{}\" {} \"{}\"", child_path.wstring(), + mode == Mode::fatal ? L"--fatal-crash" : native ? L"--native-crash" : L"--crash", root.wstring()); STARTUPINFOW startup{sizeof(startup)}; PROCESS_INFORMATION process{}; check(CreateProcessW(child_path.c_str(), command.data(), nullptr, nullptr, FALSE, CREATE_NO_WINDOW, nullptr, nullptr, &startup, &process) != FALSE, "Cannot start crash fixture"); @@ -226,10 +301,28 @@ void crash_check(const std::filesystem::path& root, bool native) { check(MiniDumpReadDumpStream(bytes.data(), ModuleListStream, &stream, &data, &size) != FALSE, "Minidump module list missing"); check(Json::parse(read(dump.wstring() + L".json")).at("application").string() == "ReSkate", "Crash attributes missing"); check(Json::parse(read(dump.wstring() + L".json")).at("capture.source").string() == - (native ? "game-native" : "unhandled-exception"), "Wrong capture source"); + (mode == Mode::fatal ? "game-fatal-handler" : native ? "game-native" : "unhandled-exception"), "Wrong capture source"); + if (mode != Mode::native) { + std::array marker{}; + fill_marker(marker); + check(bytes.find(std::string_view(marker.data(), marker.size())) != std::string::npos, + "Dump lacks the heap memory the faulting stack points at"); + } check(Json::parse(read(dump.wstring() + L".json")).at("attachment.log").string() == "complete" && read(log_attachment_path(dump)) == log, "Crash did not snapshot its session log"); - if (native) { + const auto tags = Json::parse(read(dump.wstring() + L".json")); + const auto site = tags.at("crash.site").string(); + // (The fixture raises its exception, so it happens in RaiseException.) + check(site.starts_with("KERNELBASE.dll+0x") && site.size() > 17, "Crash site not tagged"); + const auto details = Json::parse(read(event_details_path(dump))); + check(details.at("user").at("id").string().size() == 32 && details.at("breadcrumbs").at("values").size() > 0 && + details.at("contexts").at("reskate").at("mods").is_array() && tags.at("mods.count").string() == "0", + "Crash did not carry its event details"); + check(tags.at("game.map").string() == "dingolevel_reskate_lsc_b2" && + tags.at("game.destination").string() == "Levels/Game/DingoLevel_Root/DingoLevel_Root + levels/game/dingolevel_reskate_lsc_b2/dingolevel_reskate_lsc_b2 @ DefaultStartPoint" && + tags.at("gpu.name").string() == "Fixture GPU" && tags.at("gpu.driver").string() == "32.0.21025.1", + "Crash was not tagged with its level and graphics card"); + if (mode == Mode::native) { const auto source = root / L"local" / L"ReSkate" / L"Game" / L"Skate" / L"CrashDumps"; const auto pid = GetProcessId(child.value), created = process_creation_seconds(child.value); const auto selected = find_native_dump(source, pid, created); @@ -284,8 +377,10 @@ int wmain(int argc, wchar_t** argv) { check(result.accepted, "Sentry did not confirm test report"); return 0; } - if (argc == 3 && (std::wstring_view(argv[1]) == L"--crash" || std::wstring_view(argv[1]) == L"--native-crash")) { - const bool native = std::wstring_view(argv[1]) == L"--native-crash"; + if (argc == 3 && (std::wstring_view(argv[1]) == L"--crash" || std::wstring_view(argv[1]) == L"--native-crash" || + std::wstring_view(argv[1]) == L"--fatal-crash")) { + const bool fatal = std::wstring_view(argv[1]) == L"--fatal-crash"; + const bool native = fatal || std::wstring_view(argv[1]) == L"--native-crash"; if (native) { const auto local = std::filesystem::path(argv[2]) / L"local"; SetEnvironmentVariableW(L"LOCALAPPDATA", local.c_str()); @@ -297,8 +392,13 @@ int wmain(int argc, wchar_t** argv) { SetUnhandledExceptionFilter(finish_child); logging::Options options; options.directory = argv[2]; options.truncate_file = true; check(logging::initialize(options), "Fixture logging failed"); - if (native) SetUnhandledExceptionFilter(native_child_handler); // Reproduce the game's later handler registration. - RaiseException(test_exception, EXCEPTION_NONCONTINUABLE, 0, nullptr); + if (native) SetUnhandledExceptionFilter(fatal ? fatal_child_handler : native_child_handler); // Reproduce the game's later handler registration. + set_destination(L"Levels/Game/DingoLevel_Root/DingoLevel_Root + levels/game/dingolevel_reskate_lsc_b2/dingolevel_reskate_lsc_b2 @ DefaultStartPoint"); + set_gpu(L"Fixture GPU", L"32.0.21025.1"); + auto* const volatile marker = new std::array; + fill_marker(*marker); + const ULONG_PTR arguments[]{reinterpret_cast(marker)}; + RaiseException(test_exception, EXCEPTION_NONCONTINUABLE, 1, arguments); return 3; } check(argc == 2, "Pass a fixture root"); @@ -312,9 +412,11 @@ int wmain(int argc, wchar_t** argv) { check(start(root) == StartResult::failed, "Invalid endpoint enabled reporter"); endpoint_checks(); attachment_checks(root); + details_checks(root); queue_checks(root); - crash_check(root / L"direct", false); - crash_check(root / L"native", true); - std::cout << "Timing envelopes, endpoint, receipts, multipart attachments, log snapshots, retries, retention, opt-out, exception chaining, direct capture and native-handler fallback checks passed.\n"; + crash_check(root / L"direct", Mode::direct); + crash_check(root / L"native", Mode::native); + crash_check(root / L"fatal", Mode::fatal); + std::cout << "Timing envelopes, endpoint, receipts, multipart attachments, log snapshots, retries, retention, opt-out, exception chaining, direct capture, native-handler fallback and game fatal-handler capture checks passed.\n"; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; } } diff --git a/Engine/Core/Debug/client.cpp b/Engine/Core/Debug/client.cpp index cf7213d3..5f2d0c8d 100644 --- a/Engine/Core/Debug/client.cpp +++ b/Engine/Core/Debug/client.cpp @@ -22,6 +22,11 @@ template bool copy(std::array& target, const std::wst target[value.size()] = 0; return true; } +template void copy_cut(std::array& target, std::wstring_view value) noexcept { + const auto size = std::min(value.size(), N - 1); + std::copy_n(value.begin(), size, target.begin()); + target[size] = 0; +} } StartResult start(const std::filesystem::path& log_directory) noexcept { try { @@ -99,16 +104,34 @@ StartResult start(const std::filesystem::path& log_directory) noexcept { return StartResult::ready; } catch (...) { return StartResult::failed; } } -void capture(EXCEPTION_POINTERS* exception) noexcept { - if (!exception || InterlockedCompareExchange(&capturing, 1, 0) != 0) return; - report->thread_id = GetCurrentThreadId(); +namespace { +void capture_on(DWORD thread_id, EXCEPTION_POINTERS* exception, CaptureSource source) noexcept { + if (!exception || !thread_id || InterlockedCompareExchange(&capturing, 1, 0) != 0) return; + report->thread_id = thread_id; report->exception = exception; + report->source = source; MemoryBarrier(); if (SetEvent(handles[requested])) { const HANDLE wait_for[]{handles[captured], helper}; WaitForMultipleObjects(2, wait_for, FALSE, capture_timeout_ms); } } +} +void capture(EXCEPTION_POINTERS* exception) noexcept { + capture_on(GetCurrentThreadId(), exception, CaptureSource::unhandled_exception); +} +void capture_game_fatal(DWORD thread_id, EXCEPTION_POINTERS* exception) noexcept { + capture_on(thread_id, exception, CaptureSource::game_fatal_handler); +} +void set_destination(std::wstring_view destination) noexcept { + if (auto* const shared = report) copy_cut(shared->destination, destination); +} +void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept { + if (auto* const shared = report) { + copy_cut(shared->gpu_name, name); + copy_cut(shared->gpu_driver, driver); + } +} void stop() noexcept { // An exception already using these handles owns them until process exit. // Never unmap its report or close its events from a concurrent shutdown. diff --git a/Engine/Core/Debug/crash_report.h b/Engine/Core/Debug/crash_report.h index b36e46b1..231f4ca0 100644 --- a/Engine/Core/Debug/crash_report.h +++ b/Engine/Core/Debug/crash_report.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include namespace dingosdk::crash_report { enum class StartResult { disabled, ready, failed }; @@ -9,4 +10,12 @@ StartResult start(const std::filesystem::path& log_directory) noexcept; void stop() noexcept; // Last-chance path: only signals the prestarted helper and waits at most 15 seconds. void capture(EXCEPTION_POINTERS* exception) noexcept; +// The same, from the game's own fatal handler just before it writes its dump: thread_id is the +// thread that faulted, which need not be the calling one. The first capture of either kind wins. +void capture_game_fatal(DWORD thread_id, EXCEPTION_POINTERS* exception) noexcept; +// The level being loaded ("root + sublevel @ start point") and the graphics card the game renders +// on, for a report's tags: game.map, game.destination, gpu.name and gpu.driver. Nothing while +// reporting is off; too long a value is cut short. +void set_destination(std::wstring_view destination) noexcept; +void set_gpu(std::wstring_view name, std::wstring_view driver) noexcept; } diff --git a/Engine/Core/Debug/protocol.h b/Engine/Core/Debug/protocol.h index 9de6dc21..902d2a23 100644 --- a/Engine/Core/Debug/protocol.h +++ b/Engine/Core/Debug/protocol.h @@ -4,7 +4,10 @@ #include namespace dingosdk::crash_report { -inline constexpr DWORD protocol_version = 2; +inline constexpr DWORD protocol_version = 4; +// Who asked for the dump: ReSkate's last-chance filter, or the game's own fatal handler about to +// write its dump (which it does without ever reaching that filter). +enum class CaptureSource : DWORD { unhandled_exception, game_fatal_handler }; inline constexpr DWORD capture_timeout_ms = 15000; struct SharedReport { DWORD version{protocol_version}; @@ -14,6 +17,13 @@ struct SharedReport { std::array directory{}; std::array application{}; std::array native_directory{}; + // What the game was doing, for the report's tags: the level it loaded last, and the graphics + // card it renders on with its driver's version. Written by the game as they change, read by + // the helper after a crash (a torn read only spoils a tag). + std::array destination{}; + std::array gpu_name{}; + std::array gpu_driver{}; + CaptureSource source{}; }; // Explicitly inherited unnamed handles; no token or paths on the command line. enum HandleIndex : std::size_t { mapping, requested, captured, stopping, ready, parent, handle_count }; diff --git a/Engine/Core/Debug/report_details.cpp b/Engine/Core/Debug/report_details.cpp new file mode 100644 index 00000000..5e8fae52 --- /dev/null +++ b/Engine/Core/Debug/report_details.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include "report_details.h" +#include "report_envelope.h" +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::crash_report { +namespace { +constexpr std::size_t max_breadcrumb_text = 600; +// How much of the log's end is read for breadcrumbs: far more than 100 lines ever take. +constexpr std::uintmax_t breadcrumb_window = 1024 * 1024; + +// JSON text must be UTF-8: anything else in a line becomes '?'. +std::string valid_utf8(std::string_view text) { + std::string out; + out.reserve(text.size()); + for (std::size_t i = 0; i < text.size();) { + const auto c = static_cast(text[i]); + std::size_t length = c < 0x80 ? 1 : (c & 0xe0) == 0xc0 ? 2 : (c & 0xf0) == 0xe0 ? 3 : (c & 0xf8) == 0xf0 ? 4 : 0; + bool ok = length && i + length <= text.size() && !(length == 2 && c < 0xc2); + for (std::size_t k = 1; ok && k < length; ++k) ok = (static_cast(text[i + k]) & 0xc0) == 0x80; + if (!ok) { out.push_back('?'); ++i; continue; } + if (length == 1 && c < 0x20 && c != '\t') out.push_back(' '); + else out.append(text.substr(i, length)); + i += length; + } + return out; +} +// "[2026-10-16 16:56:20.354]" in local time -> seconds since 1970 (UTC), or a negative number. +double log_time(std::string_view stamp) { + int y{}, mo{}, d{}, h{}, mi{}, s{}, ms{}; + if (stamp.size() < 25 || sscanf_s(std::string(stamp.substr(1, 23)).c_str(), "%d-%d-%d %d:%d:%d.%d", &y, &mo, &d, &h, &mi, &s, &ms) != 7) + return -1.0; + SYSTEMTIME local{static_cast(y), static_cast(mo), 0, static_cast(d), static_cast(h), static_cast(mi), + static_cast(s), static_cast(ms)}; + SYSTEMTIME utc{}; + FILETIME file{}; + if (!TzSpecificLocalTimeToSystemTime(nullptr, &local, &utc) || !SystemTimeToFileTime(&utc, &file)) return -1.0; + ULARGE_INTEGER ticks{}; + ticks.LowPart = file.dwLowDateTime; + ticks.HighPart = file.dwHighDateTime; + return static_cast(ticks.QuadPart) / 1e7 - 11644473600.0; +} +// One log line: "[time] [pid:tid] Source: [Channel] [LEVEL] text". +struct Line { double time{-1}; std::string category, level, text; bool debug{}; }; +Line parse_line(std::string_view raw) { + Line line; + line.level = "info"; + line.time = log_time(raw); + auto rest = raw; + if (line.time >= 0.0) { + if (const auto close = rest.find("] [", 1); close != std::string_view::npos) rest.remove_prefix(close + 2); + if (const auto close = rest.find("] "); close != std::string_view::npos) rest.remove_prefix(close + 2); + } + std::string source; + if (const auto colon = rest.find(": ["); colon != std::string_view::npos && colon < 24) { + source = rest.substr(0, colon); + rest.remove_prefix(colon + 2); + } + std::string channel; + if (rest.starts_with('[')) { + if (const auto close = rest.find("] "); close != std::string_view::npos && close < 40) { + channel = rest.substr(1, close - 1); + rest.remove_prefix(close + 2); + } + } + if (rest.starts_with('[')) { + if (const auto close = rest.find("] "); close != std::string_view::npos && close < 12) { + const auto level = rest.substr(1, close - 1); + rest.remove_prefix(close + 2); + line.debug = level == "DEBUG" || level == "TRACE"; + line.level = level == "WARNING" ? "warning" : level == "ERROR" ? "error" : level == "FATAL" || level == "CRITICAL" ? "fatal" + : line.debug ? "debug" : "info"; + } + } + line.category = source.empty() ? channel : channel.empty() ? source : source + "/" + channel; + line.text = valid_utf8(rest.substr(0, max_breadcrumb_text)); + if (rest.size() > max_breadcrumb_text) line.text += "..."; + line.category = valid_utf8(line.category); + return line; +} +} + +std::filesystem::path event_details_path(const std::filesystem::path& dump) { return dump.wstring() + L".event.json"; } + +Json log_breadcrumbs(const std::filesystem::path& log, std::size_t count, std::size_t recent) noexcept { + try { + std::ifstream input(log, std::ios::binary); + if (!input) return {}; + input.seekg(0, std::ios::end); + const auto size = static_cast(input.tellg()); + const auto start = size > breadcrumb_window ? size - breadcrumb_window : 0; + input.seekg(static_cast(start)); + std::string text(static_cast(size - start), '\0'); + if (!input.read(text.data(), static_cast(text.size()))) return {}; + std::vector lines; + for (std::size_t at = 0; at < text.size();) { + auto end = text.find('\n', at); + if (end == std::string::npos) end = text.size(); + auto line = std::string_view(text).substr(at, end - at); + if (line.ends_with('\r')) line.remove_suffix(1); + if (!line.empty()) lines.push_back(line); + at = end + 1; + } + if (start && !lines.empty()) lines.erase(lines.begin()); // (the first one is cut short) + std::vector kept; + for (std::size_t back = 0; back < lines.size() && kept.size() < count; ++back) { + auto line = parse_line(lines[lines.size() - 1 - back]); + if (line.debug && back >= recent) continue; + kept.push_back(std::move(line)); + } + Json values = Json::array(); + double last_time = -1.0; + for (auto it = kept.rbegin(); it != kept.rend(); ++it) { + Json crumb = Json::object(); + // (A line without a time of its own, a continuation, takes the one before it.) + if (it->time >= 0.0) last_time = it->time; + if (last_time >= 0.0) crumb["timestamp"] = last_time; + if (!it->category.empty()) crumb["category"] = it->category; + crumb["level"] = it->level; + crumb["message"] = it->text; + values.push_back(std::move(crumb)); + } + Json result = Json::object(); + result["values"] = std::move(values); + return result; + } catch (...) { return {}; } +} + +std::string crash_site(const std::filesystem::path& dump) noexcept { + try { + std::ifstream input(dump, std::ios::binary); + if (!input) return {}; + std::vector bytes((std::istreambuf_iterator(input)), std::istreambuf_iterator()); + const std::span data(reinterpret_cast(bytes.data()), bytes.size()); + const auto read = [&](std::size_t offset, T& value) { + if (offset > data.size() || sizeof(T) > data.size() - offset) return false; + std::memcpy(&value, data.data() + offset, sizeof(T)); + return true; + }; + MINIDUMP_HEADER header{}; + if (!read(std::size_t{0}, header) || header.Signature != MINIDUMP_SIGNATURE || header.NumberOfStreams > 256) return {}; + MINIDUMP_EXCEPTION_STREAM exception{}; + MINIDUMP_LOCATION_DESCRIPTOR modules{}; + bool have_exception{}, have_modules{}; + for (ULONG32 i = 0; i < header.NumberOfStreams; ++i) { + MINIDUMP_DIRECTORY entry{}; + if (!read(static_cast(header.StreamDirectoryRva) + i * sizeof(entry), entry)) return {}; + if (entry.StreamType == ExceptionStream) have_exception = read(static_cast(entry.Location.Rva), exception); + if (entry.StreamType == ModuleListStream) { modules = entry.Location; have_modules = true; } + } + if (!have_exception || !have_modules) return {}; + const auto address = exception.ExceptionRecord.ExceptionAddress; + ULONG32 module_count{}; + if (!read(static_cast(modules.Rva), module_count) || module_count > 4096) return {}; + for (ULONG32 i = 0; i < module_count; ++i) { + MINIDUMP_MODULE module{}; + if (!read(static_cast(modules.Rva) + sizeof(ULONG32) + i * sizeof(MINIDUMP_MODULE), module)) return {}; + if (address < module.BaseOfImage || address - module.BaseOfImage >= module.SizeOfImage) continue; + ULONG32 name_bytes{}; + if (!read(static_cast(module.ModuleNameRva), name_bytes) || name_bytes > 2048 || name_bytes % 2) return {}; + const auto name_at = static_cast(module.ModuleNameRva) + sizeof(ULONG32); + if (name_at > data.size() || name_bytes > data.size() - name_at) return {}; + std::wstring name(name_bytes / 2, L'\0'); + std::memcpy(name.data(), data.data() + name_at, name_bytes); + const auto file = std::filesystem::path(name).filename().wstring(); + std::string ascii; + for (const auto c : file) ascii.push_back(c < 0x80 && c >= 0x20 ? static_cast(c) : '?'); + return std::format("{}+{:#x}", ascii, address - module.BaseOfImage); + } + return {}; + } catch (...) { return {}; } +} + +std::string install_id(const std::filesystem::path& directory) noexcept { + try { + const auto path = directory / L"install-id.txt"; + { + std::ifstream input(path, std::ios::binary); + std::string id; + if (input && std::getline(input, id) && id.size() == 32 && id.find_first_not_of("0123456789abcdef") == std::string::npos) + return id; + } + std::array random{}; + if (!BCRYPT_SUCCESS(BCryptGenRandom(nullptr, random.data(), static_cast(random.size()), BCRYPT_USE_SYSTEM_PREFERRED_RNG))) + return {}; + std::string id; + for (const auto byte : random) id += std::format("{:02x}", byte); + std::filesystem::create_directories(directory); + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output << id << '\n'; + return output ? id : std::string{}; + } catch (...) { return {}; } +} + +void remove_report_files(const std::filesystem::path& dump) noexcept { + std::error_code error; + std::filesystem::remove(dump, error); + if (error) return; + std::filesystem::remove(dump.wstring() + L".json", error); + std::filesystem::remove(log_attachment_path(dump), error); + std::filesystem::remove(event_details_path(dump), error); +} + +void keep_sent_report(const std::filesystem::path& dump, std::size_t keep) noexcept { + try { + const auto sent = dump.parent_path() / L"sent"; + std::error_code error; + std::filesystem::create_directories(sent, error); + // The dump last: a report in sent/ without it is pruned, never one in the queue twice. + for (const auto& file : {std::filesystem::path(dump.wstring() + L".json"), log_attachment_path(dump), event_details_path(dump), dump}) { + if (!std::filesystem::exists(file, error)) continue; + std::filesystem::rename(file, sent / file.filename(), error); + if (error) std::filesystem::remove(file, error); + } + std::vector kept; + for (const auto& entry : std::filesystem::directory_iterator(sent, error)) + if (entry.is_regular_file() && entry.path().extension() == L".dmp") kept.push_back(entry); + std::sort(kept.begin(), kept.end(), [](const auto& a, const auto& b) { return a.last_write_time() > b.last_write_time(); }); + for (std::size_t i = keep; i < kept.size(); ++i) remove_report_files(kept[i].path()); + } catch (...) { remove_report_files(dump); } +} +} diff --git a/Engine/Core/Debug/report_details.h b/Engine/Core/Debug/report_details.h new file mode 100644 index 00000000..7793611e --- /dev/null +++ b/Engine/Core/Debug/report_details.h @@ -0,0 +1,34 @@ +#pragma once +#include "Engine/Core/Json/json.h" +#include +#include +#include + +// What a crash report carries to Sentry besides its tags and attachments. Attachments (the log) +// are kept only where the server stores them; these go in the event itself, which it always does. +namespace dingosdk::crash_report { +// .event.json: the report's event fields other than tags and release (breadcrumbs, +// contexts, user), merged into the event when it is sent. +std::filesystem::path event_details_path(const std::filesystem::path& dump); +inline constexpr std::size_t max_event_details_bytes = 512 * 1024; + +// The log's last lines as Sentry breadcrumbs ({"values": [...]}, oldest first): up to `count` +// lines, the debug ones only among the last `recent` (what happened just before is worth all of +// it; further back only what was said above debug). Each line's time, source, level and text. +Json log_breadcrumbs(const std::filesystem::path& log, std::size_t count = 100, std::size_t recent = 25) noexcept; + +// Where the dump's exception happened, as "+0x", or empty when the dump +// does not say (no exception, or an address outside every module). +std::string crash_site(const std::filesystem::path& dump) noexcept; + +// An ID for this install, random and made once, kept in /install-id.txt: tells one +// player's crashes from another's without saying who they are. Empty when it cannot be kept. +std::string install_id(const std::filesystem::path& directory) noexcept; + +// An accepted report's files go to /sent, which keeps the newest `keep` reports: the +// player keeps their crashes after they are sent, whatever the server keeps. +inline constexpr std::size_t kept_sent_reports = 10; +void keep_sent_report(const std::filesystem::path& dump, std::size_t keep = kept_sent_reports) noexcept; +// Every file a queued report has (the dump, its tags, log and event details). +void remove_report_files(const std::filesystem::path& dump) noexcept; +} diff --git a/Engine/Core/Debug/multipart.cpp b/Engine/Core/Debug/report_envelope.cpp similarity index 52% rename from Engine/Core/Debug/multipart.cpp rename to Engine/Core/Debug/report_envelope.cpp index 18727f1a..ff43e7a2 100644 --- a/Engine/Core/Debug/multipart.cpp +++ b/Engine/Core/Debug/report_envelope.cpp @@ -1,7 +1,9 @@ -#include "multipart.h" +#include "report_envelope.h" #include "upload.h" +#include "report_details.h" #include #include +#include #include namespace dingosdk::crash_report { @@ -23,6 +25,37 @@ bool file_length(HANDLE file, std::uintmax_t maximum, DWORD& bytes) { bytes = static_cast(length.QuadPart); return true; } +// When the dump was written, as seconds since 1970: the crash's time, however late it is sent. +double written_at(HANDLE file) { + FILETIME created{}, accessed{}, written{}; + if (!GetFileTime(file, &created, &accessed, &written)) return 0.0; + ULARGE_INTEGER ticks{}; + ticks.LowPart = written.dwLowDateTime; + ticks.HighPart = written.dwHighDateTime; + return static_cast(ticks.QuadPart) / 1e7 - 11644473600.0; +} +// An envelope item's header line: its type, length and what else Sentry reads of it. +std::string item_header(Json header, std::uint64_t length) { + header["length"] = length; + return header.dump() + "\n"; +} +// The rest of the event the report made (report_details.h): the log's last lines, the mods, who and +// how long. A report queued without it, or with one that is unreadable, goes without. +void merge_details(const std::filesystem::path& dump, Json& event) { + const auto path = event_details_path(dump); + std::error_code missing; + const auto size = std::filesystem::file_size(path, missing); + if (missing || size > max_event_details_bytes) return; + std::ifstream input(path, std::ios::binary); + std::string text(static_cast(size), '\0'); + if (!input.read(text.data(), static_cast(text.size()))) return; + try { + const auto details = Json::parse(text, {max_event_details_bytes, 16, 65536}); + if (!details.is_object()) return; + for (const auto& [key, value] : details.items()) + if (key == "breadcrumbs" || key == "contexts" || key == "user" || key == "extra") event[key] = value; + } catch (...) {} +} } std::filesystem::path log_attachment_path(const std::filesystem::path& dump) { return dump.wstring() + L".log"; } LogSnapshot snapshot_log(const std::filesystem::path& source, const std::filesystem::path& dump) noexcept { @@ -48,46 +81,74 @@ LogSnapshot snapshot_log(const std::filesystem::path& source, const std::filesys return truncated ? LogSnapshot::tail : LogSnapshot::complete; } catch (...) { return LogSnapshot::unavailable; } } -MultipartReport::MultipartReport(const std::filesystem::path& dump, const Json& attributes, const std::string& boundary) { - if (!attributes.is_object() || boundary.empty() || boundary.size() > 70 || - boundary.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-") != std::string::npos) { +ReportEnvelope::ReportEnvelope(const std::filesystem::path& dump, const Json& attributes, const std::string& event_id) { + if (!attributes.is_object() || event_id.size() != 32 || event_id.find_first_not_of("0123456789abcdef") != std::string::npos) { error_ = ERROR_INVALID_PARAMETER; return; } dump_.value = CreateFileW(dump.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); if (!file_length(dump_.value, max_dump_bytes, dump_bytes_) || !dump_bytes_) { error_ = ERROR_FILE_INVALID; return; } for (const auto& [key, value] : attributes.items()) { if (key.empty() || key.size() > 128 || key.find_first_not_of("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-") != std::string::npos || - !value.is_string() || value.string().size() > 4096 || value.string().find(boundary) != std::string::npos) { + !value.is_string() || value.string().size() > 4096) { error_ = ERROR_INVALID_DATA; return; } } - // Sentry reads event attributes from the "sentry" field; attributes become searchable tags. + // The event: Sentry reads the crash from the minidump attached to it; the attributes are tags. Json event = Json::object(); + event["event_id"] = event_id; + event["platform"] = "native"; + event["level"] = "fatal"; + if (const auto time = written_at(dump_.value); time > 0.0) event["timestamp"] = time; event["tags"] = attributes; if (const auto version = attributes.value("version", ""); !version.empty()) event["release"] = "reskate@" + version; - const auto fields = event.dump(); - if (fields.size() > 65536 || fields.find(boundary) != std::string::npos) { error_ = ERROR_INVALID_DATA; return; } - prefix_ = std::format("--{}\r\nContent-Disposition: form-data; name=\"sentry\"\r\n\r\n{}\r\n", boundary, fields); - prefix_ += std::format("--{}\r\nContent-Disposition: form-data; name=\"upload_file_minidump\"; filename=\"crash.dmp\"\r\nContent-Type: application/octet-stream\r\n\r\n", boundary); + merge_details(dump, event); + const auto payload = event.dump(); + if (payload.size() > max_event_details_bytes + 65536) { error_ = ERROR_INVALID_DATA; return; } + Json header = Json::object(); + header["event_id"] = event_id; + prefix_ = header.dump() + "\n"; + Json event_item = Json::object(); + event_item["type"] = "event"; + event_item["content_type"] = "application/json"; + prefix_ += item_header(std::move(event_item), payload.size()) + payload + "\n"; + Json minidump = Json::object(); + minidump["type"] = "attachment"; + minidump["attachment_type"] = "event.minidump"; + minidump["filename"] = "crash.dmp"; + prefix_ += item_header(std::move(minidump), dump_bytes_); const auto metadata = attributes.dump(); if (metadata.size() > 65536) { error_ = ERROR_INVALID_DATA; return; } - attachments_ = std::format("\r\n--{}\r\nContent-Disposition: form-data; name=\"attachment_report.json\"; filename=\"report.json\"\r\nContent-Type: application/json\r\n\r\n{}", boundary, metadata); + Json report = Json::object(); + report["type"] = "attachment"; + report["filename"] = "report.json"; + report["content_type"] = "application/json"; + middle_ = "\n" + item_header(std::move(report), metadata.size()) + metadata + "\n"; const auto log_status = attributes.value("attachment.log", "unavailable"); if (log_status == "complete" || log_status == "tail") { log_.value = CreateFileW(log_attachment_path(dump).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); // If a queued attachment becomes unreadable, retain the report for retry // instead of confirming an upload that silently omitted its log. if (!file_length(log_.value, max_log_bytes, log_bytes_)) { error_ = ERROR_FILE_INVALID; return; } - attachments_ += std::format("\r\n--{}\r\nContent-Disposition: form-data; name=\"attachment_ReSkate.log\"; filename=\"ReSkate.log\"\r\nContent-Type: text/plain\r\n\r\n", boundary); + Json log = Json::object(); + log["type"] = "attachment"; + log["filename"] = "ReSkate.log"; + log["content_type"] = "text/plain"; + middle_ += item_header(std::move(log), log_bytes_); + suffix_ = "\n"; } - suffix_ = std::format("\r\n--{}--\r\n", boundary); - const auto total = prefix_.size() + static_cast(dump_bytes_) + attachments_.size() + log_bytes_ + suffix_.size(); + const auto total = prefix_.size() + static_cast(dump_bytes_) + middle_.size() + log_bytes_ + suffix_.size(); if (total > std::numeric_limits::max()) { error_ = ERROR_FILE_TOO_LARGE; return; } size_ = static_cast(total); } -bool MultipartReport::write(const std::function& sink) { +bool ReportEnvelope::write(const std::function& sink) { return !error_ && sink(prefix_.data(), static_cast(prefix_.size())) && stream_file(dump_.value, dump_bytes_, sink) && - sink(attachments_.data(), static_cast(attachments_.size())) && stream_file(log_.value, log_bytes_, sink) && - sink(suffix_.data(), static_cast(suffix_.size())); + sink(middle_.data(), static_cast(middle_.size())) && stream_file(log_.value, log_bytes_, sink) && + (suffix_.empty() || sink(suffix_.data(), static_cast(suffix_.size()))); +} +std::wstring envelope_endpoint(std::wstring_view minidump_url) { + constexpr std::wstring_view minidump = L"/minidump/"; + const auto at = minidump_url.find(minidump); + if (at == std::wstring_view::npos) return {}; + return std::wstring(minidump_url.substr(0, at)) + L"/envelope/" + std::wstring(minidump_url.substr(at + minidump.size())); } } diff --git a/Engine/Core/Debug/multipart.h b/Engine/Core/Debug/report_envelope.h similarity index 50% rename from Engine/Core/Debug/multipart.h rename to Engine/Core/Debug/report_envelope.h index 63afbe8a..b80b7956 100644 --- a/Engine/Core/Debug/multipart.h +++ b/Engine/Core/Debug/report_envelope.h @@ -11,16 +11,24 @@ std::filesystem::path log_attachment_path(const std::filesystem::path& dump); // Snapshot before publishing the dump; retrying must never read a later session's log. LogSnapshot snapshot_log(const std::filesystem::path& source, const std::filesystem::path& dump) noexcept; +// A queued report as one Sentry envelope, as sentry-native sends a crash: the event (tags, release, +// time and the details report_details.h made), the minidump, report.json and the log. Sentry's +// minidump endpoint streams attachments other than the dump to an object store, which a +// self-hosted Sentry without one drops ("upload_failed"); an envelope's are kept. // Keeps the queued files open and unchanged until the complete request is sent. -class MultipartReport { +class ReportEnvelope { public: - MultipartReport(const std::filesystem::path& dump, const Json& attributes, const std::string& boundary); + // event_id: 32 lowercase hexadecimal digits. + ReportEnvelope(const std::filesystem::path& dump, const Json& attributes, const std::string& event_id); DWORD error() const { return error_; } DWORD size() const { return size_; } bool write(const std::function& sink); private: Handle dump_, log_; DWORD dump_bytes_{}, log_bytes_{}, size_{}, error_{}; - std::string prefix_, attachments_, suffix_; + std::string prefix_, middle_, suffix_; }; +// The envelope endpoint of a Sentry minidump endpoint URL (".../api//minidump/?sentry_key=..." +// -> ".../api//envelope/?sentry_key=..."), or empty when it is not one. +std::wstring envelope_endpoint(std::wstring_view minidump_url); } diff --git a/Engine/Core/Debug/trace.cpp b/Engine/Core/Debug/trace.cpp index 56fc569a..a97216b2 100644 --- a/Engine/Core/Debug/trace.cpp +++ b/Engine/Core/Debug/trace.cpp @@ -78,7 +78,7 @@ void configure(State& s) { } // Set by start() and never destroyed: the worker thread may outlive static // destruction, and a DLL must not join a thread while the loader lock is held. -std::atomic active{}; +std::atomic started_state{}; constexpr std::uint64_t hitch_ns = 50ull * 1000 * 1000; struct Pacing { std::atomic last{}, frames{}, total{}, hitches{}, worst{}; @@ -165,17 +165,18 @@ std::string envelope(const Transaction& transaction) { } bool start() noexcept { try { - if (active.load()) return true; + if (started_state.load()) return true; auto s = std::make_unique(); configure(*s); if (!s->enabled) return false; State* expected{}; - if (active.compare_exchange_strong(expected, s.get())) (void)s.release(); + if (started_state.compare_exchange_strong(expected, s.get())) (void)s.release(); return true; } catch (...) { return false; } } +bool active() noexcept { return started_state.load(std::memory_order_relaxed) != nullptr; } void frame(std::uint64_t now_ns) noexcept { - if (!active.load(std::memory_order_relaxed)) return; + if (!started_state.load(std::memory_order_relaxed)) return; const auto previous = pacing.last.exchange(now_ns, std::memory_order_relaxed); if (!previous || now_ns <= previous) return; const auto took = now_ns - previous; @@ -191,7 +192,7 @@ FramePacing take_frame_pacing() noexcept { void record(std::string_view name, std::string_view operation, std::chrono::milliseconds duration, bool ok, Tags tags, Measurements measurements) noexcept { try { - auto* const started = active.load(); + auto* const started = started_state.load(); if (!started) return; auto& s = *started; Transaction transaction; @@ -218,7 +219,7 @@ void record(std::string_view name, std::string_view operation, std::chrono::mill } void flush(std::chrono::milliseconds timeout) noexcept { try { - auto* const started = active.load(); + auto* const started = started_state.load(); if (!started) return; auto& s = *started; std::unique_lock lock(s.mutex); diff --git a/Engine/Core/Debug/trace.h b/Engine/Core/Debug/trace.h index 497c93a1..f643e202 100644 --- a/Engine/Core/Debug/trace.h +++ b/Engine/Core/Debug/trace.h @@ -18,6 +18,8 @@ using Measurements = std::vector; // unless RESKATE_TRACING=1 (the launcher's "Send performance timings" setting), // and off whenever crash reporting is (RESKATE_CRASH_REPORTING=0). bool start() noexcept; +// Whether this launch sends timings (start() succeeded). +bool active() noexcept; // Queues one finished operation as a Sentry transaction. Never waits on the // network and never throws. A launch's launcher and game share one trace. void record(std::string_view name, std::string_view operation, std::chrono::milliseconds duration, diff --git a/Engine/Core/Debug/upload.cpp b/Engine/Core/Debug/upload.cpp index 24789d24..4fcfc5d0 100644 --- a/Engine/Core/Debug/upload.cpp +++ b/Engine/Core/Debug/upload.cpp @@ -3,7 +3,8 @@ #include #include "upload.h" #include "protocol.h" -#include "multipart.h" +#include "report_envelope.h" +#include "report_details.h" #include #include #include @@ -23,14 +24,7 @@ std::string utf8(std::wstring_view value) { WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value.data(), static_cast(value.size()), result.data(), size, nullptr, nullptr); return result; } -void remove_report(const std::filesystem::path& dump) { - std::error_code error; - std::filesystem::remove(dump, error); - if (!error) { - std::filesystem::remove(dump.wstring() + L".json", error); - std::filesystem::remove(log_attachment_path(dump), error); - } -} +void remove_report(const std::filesystem::path& dump) { remove_report_files(dump); } void status_file(const std::filesystem::path& directory, const std::string& text) { // Overwrite a bounded status file. Never include the endpoint, token, response body, or local paths. std::ofstream(directory / L"status.txt", std::ios::trunc) << text << '\n'; @@ -54,10 +48,16 @@ bool parse_endpoint(const std::wstring& url, Endpoint& endpoint) { bool accepted_response(DWORD status, std::string_view response, std::string& report_id) { report_id.clear(); if (status < 200 || status >= 300) return false; - // Sentry's minidump endpoint answers with the event ID as plain text. + // Sentry's envelope endpoint answers {"id":""}; its minidump endpoint the ID as plain text. const auto first = response.find_first_not_of(" \t\r\n"), last = response.find_last_not_of(" \t\r\n"); if (first == std::string_view::npos) return false; - const auto id = response.substr(first, last - first + 1); + auto id = std::string(response.substr(first, last - first + 1)); + if (id.starts_with('{')) { + try { + const auto receipt = Json::parse(id, {4096, 4, 64}); + id = receipt.is_object() && receipt.contains("id") && receipt.at("id").is_string() ? receipt.at("id").string() : std::string{}; + } catch (...) { return false; } + } if (id.size() > 36 || id.find_first_not_of("0123456789abcdefABCDEF-") != std::string_view::npos || std::count_if(id.begin(), id.end(), [](char c) { return c != '-'; }) != 32) return false; report_id = id; @@ -78,8 +78,17 @@ UploadResult upload_report(const std::wstring& url, const std::filesystem::path& UploadResult result; Endpoint endpoint; if (!parse_endpoint(url, endpoint) || !attributes.is_object()) { result.error = ERROR_INVALID_PARAMETER; return result; } - const auto boundary = std::format("ReSkate-{:x}-{:x}", GetCurrentProcessId(), GetTickCount64()); - MultipartReport body(dump, attributes, boundary); + // Sent as an envelope (report_envelope.h), to the envelope endpoint beside the minidump one. + Endpoint envelope; + if (!parse_endpoint(envelope_endpoint(url), envelope)) { result.error = ERROR_INVALID_PARAMETER; return result; } + endpoint = envelope; + std::array random{}; + if (!BCRYPT_SUCCESS(BCryptGenRandom(nullptr, random.data(), static_cast(random.size()), BCRYPT_USE_SYSTEM_PREFERRED_RNG))) { + result.error = ERROR_INVALID_DATA; return result; + } + std::string event_id; + for (const auto byte : random) event_id += std::format("{:02x}", byte); + ReportEnvelope body(dump, attributes, event_id); if (body.error()) { result.error = body.error(); return result; } Internet session{WinHttpOpen(L"ReSkate-CrashReporter/1", WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY, nullptr, nullptr, 0)}; if (!session.value) { result.error = GetLastError(); return result; } @@ -94,8 +103,7 @@ UploadResult upload_report(const std::wstring& url, const std::filesystem::path& !WinHttpSetOption(request.value, WINHTTP_OPTION_AUTOLOGON_POLICY, &logon, sizeof(logon))) { result.error = GetLastError(); return result; } - const std::wstring wide_boundary(boundary.begin(), boundary.end()); - const auto headers = L"Content-Type: multipart/form-data; boundary=" + wide_boundary + L"\r\n"; + const std::wstring headers = L"Content-Type: application/x-sentry-envelope\r\n"; if (!WinHttpSendRequest(request.value, headers.c_str(), static_cast(headers.size()), nullptr, 0, body.size(), 0)) { result.error = GetLastError(); return result; } @@ -156,7 +164,7 @@ std::size_t upload_pending(const std::wstring& url, const std::filesystem::path& break; } status_file(directory, "Accepted by Sentry: " + result.report_id); - remove_report(entry.path()); + keep_sent_report(entry.path()); if (++uploaded == 4) break; // Bound work for a single launch. } return uploaded; diff --git a/Engine/Core/Hooks/hooks.cpp b/Engine/Core/Hooks/hooks.cpp index aef3e25f..974fb1a1 100644 --- a/Engine/Core/Hooks/hooks.cpp +++ b/Engine/Core/Hooks/hooks.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace dingosdk { @@ -193,14 +194,27 @@ HookStatus WINAPI hook_prepare(void* target, void* replacement, void** original) return HookOk; }); } +namespace { +thread_local unsigned batch_depth{}, batch_queued{}; +} HookStatus WINAPI hook_enable(void* target) { return locked([&](Registry& r) { if (!r.initialized) return HookNotInitialized; auto* hook = find(r, target); if (!hook) return HookNotFound; if (hook->attached) return HookOk; + if (batch_depth) { + hook->queued = true; + ++batch_queued; + return HookOk; + } return transact({{hook, true}}); }); } +HookBatch::HookBatch() noexcept { ++batch_depth; } +HookBatch::~HookBatch() { + if (--batch_depth || !std::exchange(batch_queued, 0)) return; + (void)hook_apply_queued(); +} HookStatus WINAPI hook_disable(void* target) { return locked([&](Registry& r) { if (!r.initialized) return HookNotInitialized; diff --git a/Engine/Core/Hooks/hooks.h b/Engine/Core/Hooks/hooks.h index 408bda1f..1f831be9 100644 --- a/Engine/Core/Hooks/hooks.h +++ b/Engine/Core/Hooks/hooks.h @@ -30,6 +30,19 @@ HookStatus WINAPI hook_disable(void* target); HookStatus WINAPI hook_remove(void* target); HookStatus WINAPI hook_queue_enable(void* target); HookStatus WINAPI hook_apply_queued(); +// While one is alive on a thread, hook_enable on that thread only queues the hook (and answers +// HookOk); the outermost one, as it ends, attaches all of them in one transaction. Each +// transaction lists every thread on the system and suspends every one of the game's, a few +// milliseconds each, so installs made while the game runs are grouped: on the client thread, +// a frame paid for each of them (2026-10-15). A hook that then fails to attach stays detached, +// as a failed hook_enable leaves it. +class HookBatch { +public: + HookBatch() noexcept; + ~HookBatch(); + HookBatch(const HookBatch&) = delete; + HookBatch& operator=(const HookBatch&) = delete; +}; HookStatus WINAPI hook_shutdown(); const char* WINAPI hook_status_string(HookStatus status); // Captured on the calling thread; formatting/logging happens only after Detours diff --git a/Engine/Core/Platform/launcher_pe.cpp b/Engine/Core/Platform/launcher_pe.cpp index 64a72a4c..6fdd5be4 100644 --- a/Engine/Core/Platform/launcher_pe.cpp +++ b/Engine/Core/Platform/launcher_pe.cpp @@ -3,6 +3,8 @@ #include "path_text.h" #ifdef _WIN32 +#include +#include #include #include #else @@ -304,7 +306,8 @@ void validate_game_file(const fs::path& path) { detail::fail("Skate.exe SHA-256 does not match the supported build"); } -void validate_steam_api_file(const fs::path& path) { +namespace { +void check_steam_api_file(const fs::path& path) { std::error_code error; const auto size = fs::file_size(path, error); if (error || size != expected_steam_api_file_size) @@ -317,6 +320,35 @@ void validate_steam_api_file(const fs::path& path) { if (sha256_file(path) != expected_steam_api_sha256) detail::fail("steam_api64.dll is not the supported original Steam library (SHA-256 mismatch)"); } +} // namespace + +// Checked once a file for the process: the file is read and hashed, and the game's Steam +// features each ask on first use, some again every few seconds while Steam is not ready, on the +// client thread (2026-10-15). What it found, either way, is the answer after that. +void validate_steam_api_file(const fs::path& path) { + static std::mutex mutex; + static std::map checked; // path -> the failure, empty when it passed + const auto key = path.wstring(); + { + std::lock_guard lock(mutex); + if (const auto found = checked.find(key); found != checked.end()) { + if (!found->second.empty()) detail::fail(found->second.c_str()); + return; + } + } + std::string failure; + try { + check_steam_api_file(path); + } catch (const std::exception& error) { + failure = error.what(); + if (failure.empty()) failure = "steam_api64.dll could not be checked"; + } + { + std::lock_guard lock(mutex); + checked.emplace(key, failure); + } + if (!failure.empty()) detail::fail(failure.c_str()); +} #else // _WIN32 diff --git a/Engine/Core/Platform/memory.h b/Engine/Core/Platform/memory.h index 589416be..403a642c 100644 --- a/Engine/Core/Platform/memory.h +++ b/Engine/Core/Platform/memory.h @@ -33,6 +33,33 @@ inline bool peek_bytes(std::uintptr_t address, void* destination, std::size_t si template requires std::is_trivially_copyable_v bool peek(std::uintptr_t address, T& value) noexcept { return peek_bytes(address, &value, sizeof(value)); } +// Fresh read/write memory within reach of a rel32 jump from an image at base: the first free +// spot past its end, for a code stub that a patched instruction jumps to. Null when nothing is +// free within 1.75 GB of the image. +inline void* allocate_near_image(std::uintptr_t base, std::size_t image_size, std::size_t bytes) noexcept { + SYSTEM_INFO information{}; + GetSystemInfo(&information); + const auto granularity = static_cast(information.dwAllocationGranularity); + const auto align_up = [granularity](std::uintptr_t value) { return (value + granularity - 1) & ~(granularity - 1); }; + auto cursor = align_up(base + image_size); + const auto limit = base + 0x70000000ULL; + while (cursor < limit) { + MEMORY_BASIC_INFORMATION region{}; + if (!VirtualQuery(reinterpret_cast(cursor), ®ion, sizeof(region))) break; + const auto region_base = reinterpret_cast(region.BaseAddress); + const auto region_end = region_base + region.RegionSize; + if (region.State == MEM_FREE) { + const auto candidate = align_up(cursor > region_base ? cursor : region_base); + if (candidate <= region_end && region_end - candidate >= bytes) + if (auto* const result = VirtualAlloc(reinterpret_cast(candidate), bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)) + return result; + } + if (region_end <= cursor) break; + cursor = align_up(region_end); + } + return nullptr; +} + // A NUL-terminated string copied the same way: the length (terminator written at destination[length]), // or -1 if it is unreadable or has no terminator within capacity bytes. Reads byte by byte, so it never // touches memory past the terminator. diff --git a/Engine/Core/Profiling/profiler.cpp b/Engine/Core/Profiling/profiler.cpp index a58087be..f5138083 100644 --- a/Engine/Core/Profiling/profiler.cpp +++ b/Engine/Core/Profiling/profiler.cpp @@ -13,10 +13,15 @@ std::atomic zones_enabled{false}; namespace detail { std::atomic client_thread{0}, present_thread{0}; std::atomic sampling{false}; +std::atomic marking_presents{false}; +std::atomic present_mark_count{0}; +std::array, present_mark_capacity> present_marks{}; } namespace { std::atomic sites{nullptr}; std::atomic hud_visible{false}, window_visible{false}, monitor_running{false}; +std::atomic zones_kept{false}; +std::atomic own_latest{0}, present_count{0}; // Recent values for the HUD's graphs. One writer each (the client update thread, the present // thread); the overlay reads them racily, which at worst shows one value twice. @@ -241,7 +246,7 @@ void monitor() { std::lock_guard lock(summary_mutex); latest_summary = std::move(next); } - zones_enabled.store(false, std::memory_order_relaxed); + zones_enabled.store(zones_kept.load(std::memory_order_relaxed), std::memory_order_relaxed); monitor_running.store(false, std::memory_order_release); // Something may have asked again between the last check and the flag being cleared. if (wanted()) detail::wake_monitor(); @@ -270,6 +275,7 @@ Site::Site(const char* site_name) noexcept : name(site_name) { void record_client_update(std::uint64_t own_ns) noexcept { // Known even while off, so a sample can target it straight away. detail::client_thread.store(GetCurrentThreadId(), std::memory_order_relaxed); + own_latest.store(own_ns, std::memory_order_relaxed); if (!zones_enabled.load(std::memory_order_relaxed)) return; static std::uint64_t last{}; const auto now = now_ns(); @@ -290,6 +296,11 @@ void record_client_frame(std::uint64_t frame_ns) noexcept { void record_present() noexcept { detail::present_thread.store(GetCurrentThreadId(), std::memory_order_relaxed); + present_count.fetch_add(1, std::memory_order_relaxed); + if (detail::marking_presents.load(std::memory_order_relaxed)) { + const auto at = detail::present_mark_count.fetch_add(1, std::memory_order_relaxed); + if (at < detail::present_mark_capacity) detail::present_marks[at].store(now_ns(), std::memory_order_relaxed); + } if (!zones_enabled.load(std::memory_order_relaxed)) return; static std::uint64_t last{}; const auto now = now_ns(); @@ -315,6 +326,51 @@ void set_window(bool visible) noexcept { window_visible.store(visible, std::memo bool window() noexcept { return window_visible.load(std::memory_order_acquire); } bool active() noexcept { return wanted(); } +void keep_zones(bool on) noexcept { + zones_kept.store(on, std::memory_order_relaxed); + if (on) zones_enabled.store(true, std::memory_order_relaxed); + else if (!monitor_running.load(std::memory_order_acquire)) zones_enabled.store(false, std::memory_order_relaxed); +} +std::uint64_t last_client_own_ns() noexcept { return own_latest.load(std::memory_order_relaxed); } +std::uint64_t presents() noexcept { return present_count.load(std::memory_order_relaxed); } + +void mark_zones(ZoneMark& mark) noexcept { + try { + mark.totals.clear(); + for (const auto* site = sites.load(std::memory_order_acquire); site; site = site->next) + mark.totals.emplace_back(site, site->ticks.load(std::memory_order_relaxed)); + mark.tsc = __rdtsc(); + mark.ns = now_ns(); + } catch (...) { mark.totals.clear(); } +} + +std::vector zones_since(const ZoneMark& mark, std::size_t most) noexcept { + std::vector result; + try { + const auto ns = now_ns(), tsc = __rdtsc(); + if (ns <= mark.ns || tsc <= mark.tsc) return result; + // Ticks to milliseconds over this very stretch. + const double ms_per_tick = static_cast(ns - mark.ns) / 1e6 / static_cast(tsc - mark.tsc); + for (const auto* site = sites.load(std::memory_order_acquire); site; site = site->next) { + const auto now = site->ticks.load(std::memory_order_relaxed); + // (Sites only ever join the list's head, so one the mark lacks began at zero; the + // profiler window's summary sets them back to zero each second, and a total below + // the mark's started again since.) + const auto found = std::ranges::find(mark.totals, site, &std::pair::first); + const auto before = found == mark.totals.end() ? 0 : found->second; + const auto ticks = now >= before ? now - before : now; + if (!ticks) continue; + const auto ms = static_cast(ticks) * ms_per_tick; + const auto same = std::ranges::find(result, std::string_view(site->name), &ZoneTime::name); + if (same != result.end()) same->milliseconds += ms; + else result.push_back({site->name, ms}); + } + std::ranges::sort(result, [](const ZoneTime& a, const ZoneTime& b) { return a.milliseconds > b.milliseconds; }); + if (result.size() > most) result.resize(most); + } catch (...) { result.clear(); } + return result; +} + std::shared_ptr summary() noexcept { std::lock_guard lock(summary_mutex); return latest_summary; diff --git a/Engine/Core/Profiling/profiler.h b/Engine/Core/Profiling/profiler.h index 029738dd..d1e5d11d 100644 --- a/Engine/Core/Profiling/profiler.h +++ b/Engine/Core/Profiling/profiler.h @@ -71,6 +71,28 @@ enum class Series { client_interval, client_frame, client_own, present_interval // The most recent values in milliseconds, oldest first; returns how many were written. std::size_t history(Series series, std::span out) noexcept; +// ---- describing a freeze ---------------------------------------------------------------------- +// Kept on while this launch sends timings (Runtime/frame_timing.cpp): zones then time ReSkate's +// work whatever the profiler window shows, so a frame that froze can say what of ReSkate's ran in +// it. Off again leaves them to the window. +void keep_zones(bool on) noexcept; +// ReSkate's own share of the latest client update, and how many presents there have been: both +// counted always (a store and an add). +std::uint64_t last_client_own_ns() noexcept; +std::uint64_t presents() noexcept; +// Every zone's running total at one moment; zones_since then gives the zones that took the most +// time between that moment and now, longest first. A mark is reused from frame to frame. +struct ZoneMark { + std::vector> totals; + std::uint64_t tsc{}, ns{}; +}; +void mark_zones(ZoneMark& mark) noexcept; +struct ZoneTime { + std::string name; + double milliseconds{}; +}; +std::vector zones_since(const ZoneMark& mark, std::size_t most) noexcept; + // ---- what is on ------------------------------------------------------------------------------ // The HUD and window are the overlay's (both start off each launch); turning either on starts // the timing and thread monitor. diff --git a/Engine/Core/Profiling/profiler_internal.h b/Engine/Core/Profiling/profiler_internal.h index 4e7f9ddd..b366313f 100644 --- a/Engine/Core/Profiling/profiler_internal.h +++ b/Engine/Core/Profiling/profiler_internal.h @@ -1,6 +1,7 @@ #pragma once #include "profiler.h" #include +#include namespace dingosdk::profiler::detail { // Threads the summary and the sampler name: the one running the client update, and the one @@ -8,6 +9,12 @@ namespace dingosdk::profiler::detail { extern std::atomic client_thread, present_thread; // True while a sample runs; keeps the monitor (and so the zones) alive. extern std::atomic sampling; +// While a sample of the present thread runs: when each frame was presented (now_ns), so its report +// can give the slowest frames as the client one does. One writer, the present thread. +constexpr std::size_t present_mark_capacity = 1 << 14; +extern std::atomic marking_presents; +extern std::atomic present_mark_count; +extern std::array, present_mark_capacity> present_marks; // Starts the once-a-second monitor if anything wants it. void wake_monitor() noexcept; // What a thread is, for reports: its description, or what its start address belongs to. diff --git a/Engine/Core/Profiling/sampler.cpp b/Engine/Core/Profiling/sampler.cpp index 0258ed18..83e7ba84 100644 --- a/Engine/Core/Profiling/sampler.cpp +++ b/Engine/Core/Profiling/sampler.cpp @@ -286,6 +286,12 @@ void run(SampleRequest request, std::vector threads) { const HANDLE timer = CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); const auto period = std::chrono::nanoseconds(1'000'000'000 / request.rate); + // Sampling the present thread: when each frame is presented, for its slowest frames. + const bool presents = request.target == Target::present; + if (presents) { + detail::present_mark_count.store(0, std::memory_order_relaxed); + detail::marking_presents.store(true, std::memory_order_release); + } const auto start = std::chrono::steady_clock::now(); const auto finish = start + std::chrono::duration_cast(std::chrono::duration(request.seconds)); auto next = start; @@ -337,6 +343,18 @@ void run(SampleRequest request, std::vector threads) { } if (timer) CloseHandle(timer); close(); + // Each present, in microseconds into the sample as the samples are. + std::vector present_at; + if (presents) { + detail::marking_presents.store(false, std::memory_order_release); + const auto start_ns = static_cast(std::chrono::duration_cast(start.time_since_epoch()).count()); + const auto marked = std::min(detail::present_mark_count.load(std::memory_order_acquire), detail::present_mark_capacity); + for (std::size_t i = 0; i < marked; ++i) { + const auto at = detail::present_marks[i].load(std::memory_order_relaxed); + if (at > start_ns) present_at.push_back(static_cast((at - start_ns) / 1000)); + } + std::ranges::sort(present_at); + } report.seconds = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); report.progress = 1; @@ -487,6 +505,67 @@ void run(SampleRequest request, std::vector threads) { } } + // ---- the slowest presented frames: from one present to the next ---- + // A hitch shows here as one long gap between presents. What the present thread did in it: + // the shares of its samples in that gap, and how many of them were waiting (on the GPU, the + // driver or another thread) rather than working. + if (presents && present_at.size() >= 9 && !records.empty()) { + struct Gap { std::uint32_t from{}, to{}; }; + std::vector gaps; + for (std::size_t i = 1; i < present_at.size(); ++i) gaps.push_back({present_at[i - 1], present_at[i]}); + std::vector lengths; + for (const auto& gap : gaps) lengths.push_back(gap.to - gap.from); + std::ranges::sort(lengths); + const auto median = std::max(1, lengths[lengths.size() / 2]); + std::ranges::sort(gaps, [](const Gap& a, const Gap& b) { return a.to - a.from > b.to - b.from; }); + hitches += std::format("Slowest presented frames, of {} (the usual one takes {:.1f} ms from one present to the next).\n" + "Under each: what the present thread spent that time in, as shares of its samples then.\n", + gaps.size(), median / 1000.0); + std::vector walked; + for (std::size_t n = 0; n < gaps.size() && n < 6; ++n) { + const auto& gap = gaps[n]; + const auto length = gap.to - gap.from; + if (length < median * 2 && n) break; + std::unordered_map inner; + std::uint32_t samples{}, waited{}; + // (The samples are in time order.) + const auto first = std::ranges::lower_bound(records, gap.from, {}, &Record::at_us); + for (auto it = first; it != records.end() && it->at_us < gap.to; ++it) { + walked.clear(); + const auto* sample = pool.data() + it->offset; + for (std::uint32_t f = 0; f < it->depth; ++f) { + const Module* module{}; + walked.push_back(function_start(modules, f ? sample[f] - 1 : sample[f], &module)); + owners[walked.back()] = module; + } + if (walked.empty()) continue; + ++samples; + waited += waiting(name_of(walked.front(), owners[walked.front()])) ? 1u : 0u; + for (std::size_t f = 0; f < walked.size(); ++f) { + auto& counts = inner[walked[f]]; + counts.module = owners[walked[f]]; + if (!f) ++counts.self; + const auto here = walked.begin() + static_cast(f); + if (std::find(walked.begin(), here, walked[f]) == here) ++counts.total; + } + } + hitches += std::format("\n {:.1f} ms, {:.2f} s into the sample ({} samples, {:.0f}% of them waiting)\n", length / 1000.0, + gap.from / 1e6, samples, samples ? 100.0 * waited / samples : 0.0); + if (!samples) continue; + std::vector> order(inner.begin(), inner.end()); + // The work, not the way down to it: what nearly every sample passes through is left out. + std::erase_if(order, [&](const auto& entry) { return entry.second.total * 100 >= samples * 97; }); + std::ranges::sort(order, [](const auto& a, const auto& b) { return a.second.total > b.second.total; }); + for (std::size_t i = 0; i < order.size() && i < 16; ++i) + hitches += std::format(" {:5.1f}% (self {:4.1f}%) {}\n", 100.0 * order[i].second.total / samples, + 100.0 * order[i].second.self / samples, name_of(order[i].first, order[i].second.module)); + } + hitches += "\n"; + } else if (presents) { + hitches += std::format("No frame times: {} present(s) were seen while sampling (ReSkate's present hook notes them).\n\n", + present_at.size()); + } + // ---- files next to ReSkate.log ---- const auto directory = logging::status().directory; SYSTEMTIME now{}; diff --git a/Engine/Core/Storage/save_database.cpp b/Engine/Core/Storage/save_database.cpp index b2ce12d0..5f41476e 100644 --- a/Engine/Core/Storage/save_database.cpp +++ b/Engine/Core/Storage/save_database.cpp @@ -212,7 +212,12 @@ void apply_changes(sqlite3* db, SaveSchema schema, const Json* before, const Jso void configure_writer(sqlite3* db) { Statement mode(db, "PRAGMA journal_mode=WAL"); check(mode.row() && mode.text(0) == "wal" && !mode.row(), "Cannot enable save database WAL"); - execute(db, "PRAGMA foreign_keys=ON; PRAGMA synchronous=FULL; PRAGMA wal_autocheckpoint=256; PRAGMA journal_size_limit=1048576; PRAGMA max_page_count=16384;"); + // NORMAL, not FULL: in WAL mode a commit then no longer waits for the disk. A transaction is + // still kept when the game crashes (it is in the operating system's hands); only a power + // loss or a crash of Windows itself can take back the last commits, never corrupt the file. + // FULL made every commit wait for a flush, on the game's own threads: each play event the + // game saves, each option, under locks the game's hooks wait on (2026-10-15). + execute(db, "PRAGMA foreign_keys=ON; PRAGMA synchronous=NORMAL; PRAGMA wal_autocheckpoint=256; PRAGMA journal_size_limit=1048576; PRAGMA max_page_count=16384;"); } void check_size(const std::filesystem::path& path) { check(std::filesystem::file_size(path) <= max_bytes, "Save database exceeds size limit"); } void clear_staging(const std::filesystem::path& path) { diff --git a/Engine/Game/Build/20260929/descriptor_heap_size.h b/Engine/Game/Build/20260929/descriptor_heap_size.h new file mode 100644 index 00000000..6399a094 --- /dev/null +++ b/Engine/Game/Build/20260929/descriptor_heap_size.h @@ -0,0 +1,31 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// The size of the renderer's shader-visible CBV/SRV/UAV descriptor heaps: a persistent part +// (120,000) and, in each frame's heap, a part for that frame's descriptors: what is left of a +// total of 333,333 (a third of D3D12's 1,000,000), capped at 131,072. +#pragma once + +#include +#include + +namespace dingosdk::game::build::v20260929::descriptor_heap_size { +// In the renderer's device setup (0x0410e3f0), where the total each heap may have (333,333, from +// the device or from the platform's own description, PQXD; both paths join here, at the first +// instruction) is capped at 1,000,000 unless the device's resource binding is tier 3: +// cmp dword ptr [rsi + 0x7a4], 3 +// je +0x0a ; past the cap +// mov edx, 1000000 / cmp ecx, edx / cmova ecx, edx (left as they are) +inline constexpr std::uintptr_t total_cap_rva = 0x0410eaa0; +inline constexpr std::array total_cap{0x83, 0xbe, 0xa4, 0x07, 0x00, 0x00, 0x03, 0x74, 0x0a}; +// In the renderer's device setup (0x0410e3f0), the per-frame part: what the device allows past +// the persistent part, divided among the heaps, then capped by the setting at desc+8 +// (0xffffffff: no cap): +// mov ecx, dword ptr [r14 + 8] ; the cap +// div r9d ; eax = room / heaps +// mov edi, eax +// mov eax, 0xffffffff +// cmp ecx, eax / je / cmp edi, ecx / cmovb ecx, edi / mov edi, ecx (left as they are) +// No branch lands inside these four instructions. +inline constexpr std::uintptr_t cap_load_rva = 0x0410eaee; +inline constexpr std::array cap_load{ + 0x41, 0x8b, 0x4e, 0x08, 0x41, 0xf7, 0xf1, 0x8b, 0xf8, 0xb8, 0xff, 0xff, 0xff, 0xff}; +} // namespace dingosdk::game::build::v20260929::descriptor_heap_size diff --git a/Engine/Game/Build/20260929/eadp_errors_arena.h b/Engine/Game/Build/20260929/eadp_errors_arena.h new file mode 100644 index 00000000..38bdc224 --- /dev/null +++ b/Engine/Game/Build/20260929/eadp_errors_arena.h @@ -0,0 +1,21 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// The arena EA's error reporting (EadpErrorsData) allocates from: 80 MiB, made +// about seven seconds into startup, before the named settings can be changed. +#pragma once + +#include +#include + +namespace dingosdk::game::build::v20260929::eadp_errors_arena { +// In the arena's factory (0x02318660): the load of its size from the settings +// object (EadpErrorsData.EadpErrorsDataPlatformSpecificMemPoolSize) and the +// vtable load after it, the two complete instructions moved into the stub. +// mov eax, dword ptr [rax + 0x7c] +// lea rcx, [rip + 0x03fffd17] ; 0x063183d8 +// No branch in the factory lands inside them; the factory goes on at resume_rva. +inline constexpr std::uintptr_t size_load_rva = 0x023186b7; +inline constexpr std::array size_load{ + 0x8b, 0x40, 0x7c, 0x48, 0x8d, 0x0d, 0x17, 0xfd, 0xff, 0x03}; +inline constexpr std::uintptr_t vtable_rva = 0x063183d8; +inline constexpr std::uintptr_t resume_rva = 0x023186c1; +} // namespace dingosdk::game::build::v20260929::eadp_errors_arena diff --git a/Engine/Game/Build/20260929/game_fatal_dump.h b/Engine/Game/Build/20260929/game_fatal_dump.h new file mode 100644 index 00000000..ffcd9891 --- /dev/null +++ b/Engine/Game/Build/20260929/game_fatal_dump.h @@ -0,0 +1,15 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// The game's fatal handlers catch most crashes themselves (ReSkate's last-chance filter never +// sees them) and write their own minidump, of thread stacks only, in-process: three callers +// (0x18e3ea1 "Frostbite MiniDump", 0x2d28c9d, 0x2d2bfee) through one import thunk. +#pragma once +#include +#include + +namespace dingosdk::game::build::v20260929::game_fatal_dump { +// MiniDumpWriteDump's import slot (api-ms-win-core-debug-minidump-l1-1-0.dll). +inline constexpr std::uintptr_t import_slot = 0x05feb2e0; +// The thunk every caller goes through: jmp qword ptr [import_slot]. +inline constexpr std::uintptr_t thunk = 0x0546dbc6; +inline constexpr std::array thunk_bytes{0xff, 0x25, 0x14, 0xd7, 0xb7, 0x00}; +} // namespace dingosdk::game::build::v20260929::game_fatal_dump diff --git a/Engine/Game/Build/20260929/light_tables.h b/Engine/Game/Build/20260929/light_tables.h new file mode 100644 index 00000000..123a5c87 --- /dev/null +++ b/Engine/Game/Build/20260929/light_tables.h @@ -0,0 +1,34 @@ +#pragma once +#include "Engine/Game/Build/fingerprint.h" +#include +#include + +// The light renderer (Extension/World/light_table_guard.cpp). Each kind of light has a table of 64 +// pages of 128 and an ID allocator at manager+0xb8; nothing checks the IDs against the table. +namespace dingosdk::game::build::v20260929::light_tables { +inline constexpr std::uintptr_t renderer = 0x77ef140; // the light renderer, null until made +// The allocator: the free list head at +0xe8 (low 32 bits 0xffffffff when empty), fresh IDs given at +0xf0. +inline constexpr std::uint32_t allocator = 0xb8, free_head = 0xe8, issued = 0xf0; +inline constexpr std::uint32_t capacity = 64 * 128; +inline constexpr Fingerprint take_id{0x3e844f0, { + 0x40,0x53,0x55,0x56,0x57,0x41,0x56,0x48,0x83,0xec,0x20,0x48,0x8b,0xb9,0xe8,0x00, + 0x00,0x00,0x48,0x8b,0xf1,0x90,0x8b,0xdf,0x83,0xff,0xff,0x74,0x45,0x48,0xbd,0x00}}; + +// factory(creator, creation info): the entity, or null. The six begin alike. +inline constexpr std::array factory_head{ + 0x40,0x53,0x56,0x57,0x41,0x56,0x48,0x83,0xec,0x58,0x48,0x8b,0x9a,0x40,0x01,0x00, + 0x00,0x48,0x8b,0xfa,0x4c,0x8b,0xf1,0x48,0x8d,0x54,0x24,0x30,0x48,0x8b,0x4f,0x10}; +struct Kind { + const char* name; + std::uint32_t manager; // in the renderer + Fingerprint factory; +}; +inline constexpr Kind kinds[]{ + {"OldPointLight", 0x60, {0x52d3f50, factory_head}}, + {"OldSpotLight", 0x570, {0x52d4480, factory_head}}, + {"PbrSphereLight", 0xa80, {0x52d0e00, factory_head}}, + {"PbrSpotLight", 0xf90, {0x52d1740, factory_head}}, + {"PbrTubeLight", 0x14a0, {0x52d1d40, factory_head}}, + {"PbrRectangularLight", 0x19b0, {0x52d2680, factory_head}}, +}; +} diff --git a/Engine/Game/Build/20260929/local_music.h b/Engine/Game/Build/20260929/local_music.h index 25c07805..87cc5e4c 100644 --- a/Engine/Game/Build/20260929/local_music.h +++ b/Engine/Game/Build/20260929/local_music.h @@ -72,6 +72,11 @@ inline constexpr Fingerprint favorite_apply_contract{0x6f51e0, {0x48,0x89,0x5c,0 // Queue selector shared by UI Next and natural end-of-track selection. The queue // supplies its own channel and registered playlist asset; do not hard-code IDs. inline constexpr Fingerprint playback_select_next_contract{0x14f8ed0, {0x48,0x89,0x54,0x24,0x10,0x55,0x53,0x56,0x57,0x41,0x54,0x41,0x55,0x41,0x56,0x41,0x57,0x48,0x8d,0x6c,0x24,0xc8,0x48,0x81,0xec,0x38,0x01,0x00,0x00,0x4c,0x8b,0xe1}}; +// The callback the queue selector hands the game's song scorer for a playlist asset with a +// selector: (u32 place, u32 *out, context) -> *out, a number of the song at that place of the +// asset's songs (song + 0x7c), or 0 where the place holds none. The whole function (32 bytes): +// it does not look at how many songs the asset has. +inline constexpr Fingerprint playback_song_score_contract{0x14f23a0, {0x49,0x8b,0x00,0x44,0x8b,0xc9,0x48,0x8b,0x08,0x4a,0x8b,0x04,0xc9,0x48,0x83,0xe0,0xfb,0x74,0x06,0x8b,0x40,0x7c,0x89,0x02,0xc3,0xc7,0x02,0x00,0x00,0x00,0x00,0xc3}}; // Native playlist playback-mode setter (PlaylistPlaybackMode, not MusicPlaylistType). inline constexpr Fingerprint playback_mode_setter_contract{0x443aa0, {0x44,0x8b,0x91,0xa8,0x00,0x00,0x00,0x4c,0x8b,0x99,0xa0,0x00,0x00,0x00,0x44,0x8b,0xca,0x8b,0xc2,0x33,0xd2,0x49,0xf7,0xf2,0x8b,0xc2,0x49,0x8b,0x04,0xc3,0x48,0x85}}; // Native channel-to-playlist binding entrypoint, used by the generated UI/script thunk. diff --git a/Engine/Game/Build/20260929/mesh_settings.h b/Engine/Game/Build/20260929/mesh_settings.h new file mode 100644 index 00000000..f2ba12a7 --- /dev/null +++ b/Engine/Game/Build/20260929/mesh_settings.h @@ -0,0 +1,22 @@ +#pragma once +#include "Engine/Game/Build/fingerprint.h" +#include + +// A mesh's render settings (Extension/World/mesh_settings_guard.cpp). +namespace dingosdk::game::build::v20260929::mesh_settings { +// apply_settings(mesh, settings): as a level's meshes load (0x3f59610, for each mesh whose flag +// at + 0x3b0 is clear), it builds the settings object of the mesh's asset in place at mesh + 0x340 +// (0x70 bytes, a reflected instance of 0x76f26f0) when the asset has one; settings is that +// object's source, null when it has none. Its caller sets the flag either way. +inline constexpr Fingerprint apply_settings{0x3f5beb0, { + 0x48,0x89,0x5c,0x24,0x10,0x57,0x48,0x83,0xec,0x60,0x80,0xb9,0xb0,0x03,0x00,0x00, + 0x00,0x48,0x8b,0xfa,0x48,0x8b,0xd9,0x74,0x20,0x48,0x81,0xc1,0x40,0x03,0x00,0x00}}; +// The mesh (0x3c0 bytes, made at 0x3f33770) leaves + 0x340 to + 0x3af as the allocation had it. +inline constexpr std::ptrdiff_t settings_offset = 0x340; +// The settings object's string the mesh frees as it unloads (0x3f5fa20, through 0x1584c50, +// whenever the flag is set): with no settings object it is whatever the allocation left there. +inline constexpr std::ptrdiff_t unload_string_offset = 0x360; +inline constexpr std::ptrdiff_t applied_flag_offset = 0x3b0; +// The game's empty string: a string that holds it frees nothing (0x1584c50 compares with it). +inline constexpr std::uintptr_t empty_string = 0x72fe704; +} diff --git a/Engine/Game/Build/20260929/native_cosmetics.h b/Engine/Game/Build/20260929/native_cosmetics.h index c169c63b..5293a913 100644 --- a/Engine/Game/Build/20260929/native_cosmetics.h +++ b/Engine/Game/Build/20260929/native_cosmetics.h @@ -47,6 +47,13 @@ inline constexpr std::array add_shader_preset_prefix{ inline constexpr std::uintptr_t script_preset_reference = 0x336f060; inline constexpr std::array script_preset_reference_prefix{0x4c,0x8d,0x05,0x81,0xa2,0x0d,0x03,0xe9}; inline constexpr std::uintptr_t referenced_preset = 0x16fe9c0; +// An AppearanceShaderExpressionPreset as loaded (its EBX layout): ParamInfo, the array of its +// AppearanceExpressionParamInfo at +0x20, 40 bytes each: the value's boxed {TypeInfo*, data*} at +// +0, its Parameter at +0x10, its ParamKey at +0x18. Applying a preset to a material (0x141314340, +// under 0x141330210) type-checks every entry whose data is set, reading the TypeInfo unchecked. +inline constexpr std::size_t preset_parameters = 0x20; +inline constexpr std::size_t preset_parameter_size = 40; +inline constexpr std::size_t preset_parameter_key = 0x18; inline constexpr std::array referenced_preset_prefix{ 0x48,0x8b,0x09,0x48,0x85,0xc9,0x74,0x07,0x48,0x8b,0x01,0x48,0xff,0x60,0x18,0x33,0xc0}; } diff --git a/Engine/Game/Build/20260929/native_vfx.h b/Engine/Game/Build/20260929/native_vfx.h index b8577076..c8446bfe 100644 --- a/Engine/Game/Build/20260929/native_vfx.h +++ b/Engine/Game/Build/20260929/native_vfx.h @@ -34,6 +34,10 @@ inline constexpr std::size_t component_owner = 0x18; // (rig, joint id, 1): the joint's index in the rig's skeleton, or -1. Reads no pose. inline constexpr Function joint_index{ 0x12e08b0, {0x48,0x8b,0x89,0x38,0x01,0x00,0x00,0xe9,0x84,0x76,0x5b,0x03,0xcc,0xcc,0xcc,0xcc}}; +// joint_index looks the joint up in the skeleton's map (EASTL: a node's right child at +0, its +// left at +8, its joint id at +0x20; the root at skeleton +0x68), with no bound on the walk +// (0x4897f40). +inline constexpr std::size_t skeleton_joints_root = 0x68, joint_node_right = 0, joint_node_left = 8, joint_node_id = 0x20; // An effects component's rig: component -> its rig's owner -> the rig; the rig's skeleton. An // effect set's effects (an array of pointers), and the joint each sits on (-1: none). inline constexpr std::size_t component_rig_owner = 0x50, rig_owner_rig = 0x78, rig_skeleton = 0x138, set_items = 0x30, diff --git a/Engine/Game/Build/20260929/render_resource_pool.h b/Engine/Game/Build/20260929/render_resource_pool.h index df45bfc0..bf6feceb 100644 --- a/Engine/Game/Build/20260929/render_resource_pool.h +++ b/Engine/Game/Build/20260929/render_resource_pool.h @@ -29,6 +29,9 @@ inline constexpr auto constructor_entry = hex_bytes( inline constexpr auto index_table_constructor_entry = hex_bytes( "4053555657415641574883ec78c5f9efc033ed488bf948896c2428488d4c2440"); +// mov ecx, 0x2daa00 -> mov ecx, 0x11ab080 (the state owner's size, 0x3efd347). +inline constexpr auto other_renderer_size_before = hex_bytes("b900aa2d00"); +inline constexpr auto other_renderer_size_after = hex_bytes("b980b01a01"); inline constexpr auto vector_begin_before = hex_bytes("488d81b0000000"); inline constexpr auto vector_begin_after = hex_bytes("488b8190000000"); inline constexpr auto vector_bytes_before = hex_bytes("480500001e00"); @@ -90,10 +93,22 @@ inline constexpr std::ptrdiff_t resource_state_vector_offset = 0x2da920; inline constexpr std::ptrdiff_t relation_mask_vector_offset = 0x114ab60; inline constexpr std::uintptr_t constructor_rva = 0x03e7f300; +// The constructor is the base of two renderers. Only the one constructed here +// (vtable 0x64f2c10, 0x11ab080 bytes) has the resource-state and relation-mask +// vectors: every read of owner+0x2da920 and owner+0x114ab60 is in its code. The +// other (constructed at 0x030a0a46, vtable 0x6414168, kept at 0x75f2700) keeps +// fields of its own at +0x2da920. Both use the metadata: the patched view +// (0x030e6bff, called from 0x30a7e18 with the other renderer) and append +// (0x03e94fe6, called from 0x30d8ebb) read its pointers at +0x11aae28/+0x11aae30. +// The other renderer was allocated 0x2daa00 bytes, which ends before them, so it +// is allocated the state owner's 0x11ab080 instead (other_renderer_size, above). +inline constexpr std::uintptr_t state_owner_call_rva = 0x03f33a46; inline constexpr std::uintptr_t index_table_constructor_rva = 0x03e7e400; inline constexpr std::uintptr_t resource_flags_priority_rva = 0x03f56073; inline constexpr std::uintptr_t resource_flags_visibility_rva = 0x03f5789a; inline constexpr Patch patches[]{ + // The other renderer's allocation, large enough for the metadata pointers. + {0x030c64cc, other_renderer_size_before, other_renderer_size_after}, {0x03e7f33c, vector_begin_before, vector_begin_after}, {0x03e7f351, vector_bytes_before, vector_bytes_after}, {0x03e94e99, capacity_before, capacity_after}, diff --git a/Engine/Game/Build/20260929/replay_export.h b/Engine/Game/Build/20260929/replay_export.h index 560d936a..0e099704 100644 --- a/Engine/Game/Build/20260929/replay_export.h +++ b/Engine/Game/Build/20260929/replay_export.h @@ -11,6 +11,17 @@ inline constexpr std::uintptr_t submit_frame = 0x34f4610; inline constexpr std::array submit_frame_prefix{ 0x40,0x57,0x48,0x81,0xec,0x50,0x08,0x00,0x00,0x48,0x8b,0x05,0xa0,0xfd,0xcc,0x03, 0x48,0x33,0xc4,0x48,0x89,0x84,0x24,0x20}; +// Maps the export dialog's 8/12/18 Mbps selection to the H.264 encoder rate. +// Raise all three outputs so export cannot silently fall back to the low-bitrate +// preset. The replacement rate is 100 Mbps (100000 kbps). +inline constexpr std::uintptr_t bitrate_switch = 0x62c240; +inline constexpr std::array bitrate_switch_prefix{ + 0x2b,0x85,0xc9,0x74,0x18,0x83,0xe9,0x01,0x74,0x05,0x83,0xf9,0x01,0x74,0x07,0xb8, + 0xe0,0x2e,0x00,0x00,0xeb,0x0c,0xb8,0x50,0x46,0x00,0x00,0xeb,0x05,0xb8,0x40,0x1f, + 0x00,0x00}; +inline constexpr std::array bitrate_immediate_offsets{16, 23, 30}; +inline constexpr std::array stock_bitrates_kbps{12000, 18000, 8000}; +inline constexpr std::uint32_t replay_bitrate_kbps = 100000; // The encoder the replay editor exports with; the engine has others. inline constexpr std::uintptr_t replay_encoder_vtable = 0x608c0b0; // Encoder byte: a frame may be encoded. A world update sets it and an encoded diff --git a/Engine/Game/Build/20260929/replay_keyframes.h b/Engine/Game/Build/20260929/replay_keyframes.h new file mode 100644 index 00000000..3e019063 --- /dev/null +++ b/Engine/Game/Build/20260929/replay_keyframes.h @@ -0,0 +1,78 @@ +#pragma once +#include "Engine/Game/Build/fingerprint.h" +#include +#include + +// The replay editor's camera keyframes (Extension/ReplayKeyframes/replay_keyframes.h): framing keys +// (Behaviour, Subject, offsets, FOV), the free camera's among them, played as cubic Hermite segments +// between neighbouring keys. (The camera's other track, whole transforms played by 0x6222e0, isn't +// used by the free camera and is left alone.) +namespace dingosdk::game::build::v20260929::replay_keyframes { +// ---- Framing track: segments built when keys change ---- +// rebuild(manager, session): any key edit ends here. It builds a segment for every framing key of the +// session with segment_build and sends the lot, with the transform keys, to the camera object. The +// manager sends one package at a time (its handle is at manager + 0x40): the session it rebuilt last +// is the one the camera plays. +inline constexpr Fingerprint session_rebuild{0x62c500, { + 0x89,0x54,0x24,0x10,0x48,0x89,0x4c,0x24,0x08,0x55,0x53,0x57,0x41,0x57,0x48,0x8d, + 0xac,0x24,0x08,0xfe,0xff,0xff,0x48,0x81,0xec,0xf8,0x02,0x00,0x00,0x4c,0x8b,0xf9}}; +// Segment* segment_build(manager, Segment* out, Node* keys_end, Node** key): the segment from a key +// to the next. keys_end is the anchor of the session's key map (session node + 0x60: [1] is the first +// key); the session's id is the int at session node + 0x20. +inline constexpr Fingerprint segment_build{0x60c7a0, { + 0x40,0x53,0x56,0x57,0x41,0x54,0x41,0x55,0x41,0x56,0x41,0x57,0x48,0x81,0xec,0xf0, + 0x00,0x00,0x00,0x4c,0x8b,0xe1,0x4d,0x8b,0xf9,0x48,0x8b,0xca,0x4d,0x8b,0xe8,0x48}}; +inline constexpr std::size_t session_id_before_keys = 0x60 - 0x20; +// void slot_build(manager, const void* value, double time, int behaviour, int subject, Slot* out): +// one key as a segment slot, under a Behaviour and Subject (segment_build passes the segment's own). +inline constexpr Fingerprint slot_build{0x60ec20, { + 0x4c,0x8b,0xdc,0x49,0x89,0x73,0x10,0x57,0x48,0x81,0xec,0x60,0x01,0x00,0x00,0x48, + 0x8b,0x82,0xf0,0x00,0x00,0x00,0x48,0x8b,0xfa,0x48,0x8b,0xb4,0x24,0x98,0x01,0x00}}; +// void slot_unwrap(Slot* adjust, const Slot* reference, int behaviour): adjust's angles to within π +// of the reference's. +inline constexpr Fingerprint slot_unwrap{0x61abd0, { + 0x48,0x83,0xec,0x48,0x45,0x85,0xc0,0x75,0x6d,0xc5,0xfa,0x10,0x52,0x48,0xc5,0xfa, + 0x10,0x49,0x48,0xc5,0xf2,0x5c,0xc2,0xc5,0xf8,0x54,0x05,0xf1,0xb9,0xa5,0x05,0xc5}}; +// bool segment_eval(camera, double time, const Segment*, Out*): plays a framing segment (every frame; +// not hooked: what it plays is decided when the segment is built), every lane through the game's +// spline (p0, p1, p2, p3, s, k1, k2): +// m1 = (1 − k1)·0.5·(p2 − p0), m2 = (1 − k2)·0.5·(p3 − p1), Hermite from p1 to p2. +inline constexpr Fingerprint segment_eval{0x6220d0, { + 0x48,0x8b,0xc4,0x48,0x89,0x58,0x08,0x57,0x48,0x81,0xec,0x50,0x01,0x00,0x00,0xc5, + 0xfc,0x10,0x05,0xe9,0x15,0xdb,0x05,0xc5,0xfc,0x10,0x15,0x01,0x16,0xdb,0x05,0x41}}; +// A segment this short or shorter plays as its end (s = 1). +inline constexpr double tiny_gap = 1.0 / 65536.0; +// The manager singleton. +inline constexpr std::uintptr_t manager = 0x71f2130; +inline constexpr std::size_t manager_sessions = 0x10; // EASTL map: anchor; root at +0x20 + +// A key's node in a session's framing map: time at + 0x20, value at + 0x30. +inline constexpr std::size_t framing_key_time = 0x20; +inline constexpr std::size_t framing_key_value = 0x30; + +// The segment (0x1e8 bytes): four slots of 0x60 bytes, two blocks of twelve k, Behaviour, Subject. +inline constexpr std::size_t slot_size = 0x60; +inline constexpr std::size_t slot_floats = slot_size / sizeof(float); +inline constexpr std::size_t slot_time = 0x40; // double +inline constexpr std::size_t segment_next = 0x000; // p3, rescaled to one segment's time away +inline constexpr std::size_t segment_to = 0x060; // p2 +inline constexpr std::size_t segment_previous = 0x0c0; // p0, rescaled likewise +inline constexpr std::size_t segment_from = 0x120; // p1 +inline constexpr std::size_t segment_k_to = 0x180; // twelve floats: the k of the to key +inline constexpr std::size_t segment_k_from = 0x1b0; // and of the from key +inline constexpr std::size_t segment_k_count = 12; +inline constexpr std::size_t segment_behaviour = 0x1e0; +// The free camera's keys play under this Behaviour (with Subject 0): seen in game, 2026-10-16. +inline constexpr int free_camera_behaviour = 1; +inline constexpr std::size_t segment_subject = 0x1e4; +inline constexpr std::size_t segment_size = 0x1e8; + +// Which slot floats each Behaviour plays, and with which k (segment_eval's evaluators 0x609a50, +// 0x608e10 and 0x609fe0, plus the FOV lane segment_eval plays itself). +struct Lane { + unsigned char slot_float, k; +}; +inline constexpr Lane behaviour_0_lanes[]{{0, 0}, {1, 1}, {2, 2}, {8, 4}, {9, 5}, {10, 6}, {18, 8}, {19, 9}, {21, 10}}; +inline constexpr Lane behaviour_1_lanes[]{{12, 0}, {13, 1}, {14, 2}, {4, 4}, {5, 5}, {6, 6}, {21, 10}}; +inline constexpr Lane behaviour_2_lanes[]{{0, 0}, {1, 1}, {2, 2}, {4, 4}, {5, 5}, {6, 6}, {18, 8}, {19, 9}, {21, 10}}; +} diff --git a/Engine/Game/Build/20260929/shader_registry.h b/Engine/Game/Build/20260929/shader_registry.h new file mode 100644 index 00000000..28e9df70 --- /dev/null +++ b/Engine/Game/Build/20260929/shader_registry.h @@ -0,0 +1,34 @@ +#pragma once +#include "Engine/Game/Build/fingerprint.h" +#include +#include + +// The game's expression shader databases and their registry (Extension/World/shader_registry_guard.cpp). +// A level's `*_win32_shaderstate_db` resource (ExpressionShaderDatabase, handler 0x3f84cf0) holds a +// blob of 0xa8-byte records; its object keeps one 0x70-byte item per record in a vector +// (begin +0x38, end +0x40) and the lookup ("MemoryInBuffer", pointer +0x18, size +0x20): u32 key +// count, then per key u64 key, u32 n, u32 record index[n]. Registering (0x3f71ac0, under the +// registry's lock, from the shader batch job 0x3f8a390) first adds, per key, the items +// `items + index * 0x70` to the registry's interned item lists (0x3f72280, no bounds check), then +// flags every item (+0x48) and records the registry (+0x30). The destructor unregisters through +// +0x30, which removes the flagged items from every list (0x3f79e20 -> 0x3f7aa10 -> 0x3f7a050). +// An index past the records, or a database destroyed before its queued registration ran, leaves +// a pointer in a list that nothing removes; the next walk of the lists (0x3f7ce90, reading +// *(item + 0x40)) crashes on it (Skate.exe+0x3f7cf33 / +0x3f7ceed / +0x20960b, mostly on custom +// map changes, 2026-10-17). +namespace dingosdk::game::build::v20260929::shader_registry { +// ExpressionShaderDatabase(this, allocator, blob, lookup, lookup size) -> this. +inline constexpr Fingerprint construct{0x3f7ee10, { + 0x48,0x89,0x5c,0x24,0x18,0x48,0x89,0x54,0x24,0x10,0x55,0x56,0x57,0x41,0x54,0x41, + 0x55,0x41,0x56,0x41,0x57,0x48,0x83,0xec,0x30,0x45,0x33,0xd2,0x4c,0x89,0x49,0x18}}; +// ~ExpressionShaderDatabase(this, delete flags) -> this. +inline constexpr Fingerprint destroy{0x3f7fb50, { + 0x48,0x89,0x5c,0x24,0x08,0x57,0x48,0x83,0xec,0x20,0x48,0x8d,0x05,0x77,0xf9,0x57, + 0x02,0x48,0x8b,0xd9,0x48,0x89,0x01,0x8b,0xfa,0x48,0x8b,0x49,0x30,0x48,0x85,0xc9}}; +// register(registrar, database). +inline constexpr Fingerprint register_database{0x3f71ac0, { + 0x48,0x89,0x54,0x24,0x10,0x53,0x41,0x54,0x41,0x55,0x41,0x57,0x48,0x83,0xec,0x48, + 0x4c,0x8b,0xe9,0x4c,0x8b,0xe2,0x48,0x8b,0x49,0x08,0xe8,0xa1,0x07,0x00,0x00,0x49}}; +inline constexpr std::uintptr_t lookup_data = 0x18, lookup_size = 0x20, items_begin = 0x38, items_end = 0x40; +inline constexpr std::uintptr_t item_size = 0x70; +} diff --git a/Engine/Game/Build/20260929/spatial_cull_ids.h b/Engine/Game/Build/20260929/spatial_cull_ids.h new file mode 100644 index 00000000..90d7714b --- /dev/null +++ b/Engine/Game/Build/20260929/spatial_cull_ids.h @@ -0,0 +1,31 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// A spatial transaction queue (SpatialQueryManager's, and one in each world's client and render +// spatial managers) hands every object it tracks a cull ID from a free list. The list is made +// once, when the queue is, at SpatialQueryManagerSettings.InitialCullIdListSize entries (the +// game's config: 40,000; 80,000 in GameView.cfg), and never grows: an insert with the list empty +// reads entry 0xffffffff of it and crashes (0x489cfe9, loading a large custom map). +#pragma once +#include +#include +#include + +namespace dingosdk::game::build::v20260929::spatial_cull_ids { +// make_list(list, allocator): looks the settings up, sizes the list's two arrays (u32 next IDs, +// u64 per-ID entries) to InitialCullIdListSize and chains them into the free list. Its only +// callers are the two queue constructors (0x489b518, 0x489ba10). +inline constexpr std::uintptr_t make_list_rva = 0x0489b770; +inline constexpr std::array make_list_entry{ + 0x40, 0x53, 0x55, 0x56, 0x57, 0x41, 0x56, 0x48, 0x83, 0xec, 0x20, 0x45, 0x33, 0xf6, 0x48}; +// lea rdx, [settings type 0x77ae5e0] ... mov rcx, [settings manager 0x7482ce0]; +// call settings_lookup; test rax, rax; je; mov esi, [rax + 0x28]; jmp; mov esi, 1. +inline constexpr std::uintptr_t settings_type_load_rva = 0x0489b78d; +inline constexpr std::array settings_type_load{0x48, 0x8d, 0x15, 0x4c, 0x2e, 0xf1, 0x02}; +inline constexpr std::uintptr_t size_read_rva = 0x0489b7ae; +inline constexpr std::array size_read{ + 0x48, 0x8b, 0x0d, 0x2b, 0x75, 0xbe, 0x02, 0xe8, 0x36, 0xb7, 0x03, 0xfd, 0x48, 0x85, + 0xc0, 0x74, 0x05, 0x8b, 0x70, 0x28, 0xeb, 0x05, 0xbe, 0x01, 0x00, 0x00, 0x00}; +inline constexpr std::uintptr_t settings_manager = 0x07482ce0; +inline constexpr std::uintptr_t settings_lookup = 0x018d6ef0; +inline constexpr std::uintptr_t settings_type = 0x077ae5e0; +inline constexpr std::size_t initial_cull_id_list_size = 0x28; +} // namespace dingosdk::game::build::v20260929::spatial_cull_ids diff --git a/Engine/Game/Build/20260929/world_pool_usage.h b/Engine/Game/Build/20260929/world_pool_usage.h new file mode 100644 index 00000000..3f75fe3a --- /dev/null +++ b/Engine/Game/Build/20260929/world_pool_usage.h @@ -0,0 +1,46 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// Where the three world pools ReSkate enlarges (entity_pages.h, spatial_buckets.h) +// can be read for how full they are: for telling whether a level needs more than +// the game's own sizes. +#pragma once + +#include +#include +#include + +namespace dingosdk::game::build::v20260929::world_pool_usage { +// Entity snapshots: the page allocator, (owner, allocator, pages). It fills the +// snapshot's 128 page pointers at pages[0..127], each page zeroed when handed out, +// so the highest non-zero entry is the highest entity slot in use. +inline constexpr std::uintptr_t entity_page_allocator_rva = 0x048bd000; +inline constexpr std::array entity_page_allocator_entry{ + 0x40, 0x53, 0x55, 0x56, 0x57, 0x41, 0x56, 0x48, 0x83, 0xec}; +// The game's own entity slots (128 pages of 1,024). +inline constexpr std::size_t entity_slots_stock = 131'072; + +// Render instances: the pool is [[global] + 0x4a0] + 0x1300 (as the one caller of +// its allocator loads it), and its next-index counter, taken with lock xadd, is +// the most it has ever handed out. +inline constexpr std::uintptr_t render_instance_load_rva = 0x050e59ac; +inline constexpr std::array render_instance_load{ + 0x48, 0x8b, 0x05, 0x25, 0xcf, 0x7c, 0x02, 0x48, 0x8b, 0xd9, 0x48, 0x8b, 0xd1, 0x48, + 0x8b, 0xa8, 0xa0, 0x04, 0x00, 0x00, 0x48, 0x8d, 0x8d, 0x00, 0x13, 0x00, 0x00}; +inline constexpr std::uintptr_t render_instance_owner_global = 0x078b28d8; +inline constexpr std::ptrdiff_t render_instance_manager_offset = 0x4a0; +inline constexpr std::ptrdiff_t render_instance_pool_offset = 0x1300; +inline constexpr std::uintptr_t render_instance_counter_rva = 0x050e23ec; +inline constexpr std::array render_instance_counter{ + 0xf0, 0x0f, 0xc1, 0xb7, 0x08, 0x10, 0x00, 0x00}; // lock xadd [rdi+0x1008], esi +inline constexpr std::ptrdiff_t render_instance_counter_offset = 0x1008; +// The game's own render instances (512 pages of 512). +inline constexpr std::size_t render_instances_stock = 262'144; + +// Spatial buckets: a query, (root, ...), four register arguments; the root's node +// count and each node's block count are at spatial_buckets.h's offsets. +inline constexpr std::uintptr_t spatial_query_rva = 0x01967420; +inline constexpr std::array spatial_query_entry{ + 0x4c, 0x8b, 0xdc, 0x4d, 0x89, 0x4b, 0x20, 0x41, 0x55, 0x41, 0x56, 0x48, 0x83, 0xec, 0x68}; +inline constexpr std::ptrdiff_t spatial_nodes_offset = 0x130; +// The game's own blocks per bucket (its root bucket overflowed at 2,049). +inline constexpr std::size_t spatial_blocks_stock = 2'048; +} // namespace dingosdk::game::build::v20260929::world_pool_usage diff --git a/Engine/Game/Build/20260929/world_props.h b/Engine/Game/Build/20260929/world_props.h new file mode 100644 index 00000000..6ba173a9 --- /dev/null +++ b/Engine/Game/Build/20260929/world_props.h @@ -0,0 +1,16 @@ +#pragma once +#include + +// Props spawned by ReSkate itself (Extension/Multiplayer/Remote/native_prop.cpp): an object +// blueprint loaded on the map, created as a client-only entity the way the remote skateboard is +// (skater_entities.h, native_skater.h) and moved every frame by the plain entity transform setter +// (skater_entities::place_entity). Found in game on 2026-10-15 with the traffic cone +// (world/props/common/construction/cmn_prp_trafficcone_01/cmn_prp_trafficcone_01): the creation +// list made one entity, and only place_entity moved it smoothly (the blueprint transform did not). +namespace dingosdk::game::build::v20260929::world_props { +// An ObjectBlueprint's vtable (the cone's). The skater code's blueprint vtable is +// engine::blueprint_vtable; an object blueprint has its own. +inline constexpr std::uintptr_t object_blueprint_vtable = 0x638dfa8; +// The entity a static-model object blueprint creates on the client (the cone's). +inline constexpr std::uintptr_t static_model_entity_vtable = 0x6403720; +} // namespace dingosdk::game::build::v20260929::world_props diff --git a/Engine/Game/Build/supported_build.h b/Engine/Game/Build/supported_build.h index 88c658d2..56b15ba5 100644 --- a/Engine/Game/Build/supported_build.h +++ b/Engine/Game/Build/supported_build.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -27,6 +28,12 @@ inline constexpr std::string_view steam_build_id = "25414733"; // what mods made for it carry too. Data of another number beside a supported Skate.exe is a // game that was updated or copied over in part, and has to be verified. inline constexpr std::string_view game_data_changelist = "29544144"; +// How many world layers this build's levels have: the catalog players scan from their game and +// a dedicated server reads from world-layers.json. A session's layers are sent as one choice for +// each, in the catalog's order, so both ends must have the same catalog. (2.0.8's server was +// released with the catalog of a game Steam had updated, 263 layers, and no player could join a +// server that shared its layers: 2026-10-13.) +inline constexpr std::size_t world_layer_count = 261; // The live game's HTTP content cache for this build (catalogue chunks and CDN // images), published as a ReSkateCache release and installed by the launcher. inline constexpr std::wstring_view content_cache_url = diff --git a/Engine/Game/Input/controller_bindings.h b/Engine/Game/Input/controller_bindings.h index 639ed1e3..09067553 100644 --- a/Engine/Game/Input/controller_bindings.h +++ b/Engine/Game/Input/controller_bindings.h @@ -89,7 +89,8 @@ struct ControllerInput { struct ActionBind { std::string_view key, name, label, command; }; -inline constexpr std::array action_binds{{ +inline constexpr std::array action_binds{{ + {"reskate_menu", "reskatemenu", "Open the ReSkate menu", "menu"}, {"first_person", "firstperson", "First person", "firstperson toggle"}, {"hide_hud", "hidehud", "Hide HUD", "hideui toggle"}, {"voice_chat", "voicechat", "Voice chat on / off", "mp voice-chat toggle"}, diff --git a/Engine/Game/World/park_editor.h b/Engine/Game/World/park_editor.h index c45a6e54..6539b633 100644 --- a/Engine/Game/World/park_editor.h +++ b/Engine/Game/World/park_editor.h @@ -53,6 +53,24 @@ struct EditorPasteRequest { std::uint64_t generation{}, revision{}; std::vector objects; }; +// Removes several objects in one edit (one undo step), as a paste adds several. +struct EditorDeleteRequest { + std::uint64_t generation{}, revision{}; + std::vector objects; +}; +// A saved set of objects (/object_sets/.park.json), placed again as one paste on any +// map. Positions are relative to the set's anchor: the middle of its footprint, at its lowest origin. +struct EditorObjectSet { + std::string name, map; // map: where the set was saved, for display only + std::vector objects; +}; +enum class EditorObjectSetAction { save, remove }; +struct EditorObjectSetRequest { + EditorObjectSetAction action{}; + std::uint64_t generation{}; + std::string name; + std::vector objects; // save: the objects in world space +}; // A mod with a parks/ folder (Mods//parks/.park.json). struct EditorParkMod { std::string folder, title, author, version, description; @@ -71,5 +89,6 @@ struct ParkEditorModel { bool can_load_parks{}; // false for multiplayer guests: only the host loads parks EditorSurfaceHit surface; std::uint64_t preview_token{}; + std::shared_ptr> object_sets; }; } // namespace dingosdk diff --git a/Engine/Game/World/world_layers.h b/Engine/Game/World/world_layers.h index 88886f88..ab833a87 100644 --- a/Engine/Game/World/world_layers.h +++ b/Engine/Game/World/world_layers.h @@ -169,6 +169,29 @@ inline WorldLayerMapSelection select_world_layer_map( return result; } +// A TOD__ layer (or a lighting tier's _TOD__): one time of day among its siblings. +inline bool world_layer_time_of_day(std::size_t slot) { + const auto& bundle = world_layer_nodes()[slot].bundle; + auto leaf = bundle.substr(bundle.find_last_of('/') + 1); + for (auto& c : leaf) if (c >= 'A' && c <= 'Z') c = static_cast(c + ('a' - 'A')); + const auto mid = leaf.find("_tod_"); + const auto at = leaf.starts_with("tod_") ? 4 : mid == std::string::npos ? leaf.size() : mid + 5; + return at < leaf.size() && leaf[at] >= '1' && leaf[at] <= '9'; +} + +// A sibling time of day that is to be off but not fully unloaded yet, which `slot` must wait for; +// world_layer_nodes().size() when there is none. +template +std::size_t world_layer_old_time_of_day(std::size_t slot, const std::vector>& desired, FullyOff&& fully_off) { + const auto& nodes = world_layer_nodes(); + if (!world_layer_time_of_day(slot)) return nodes.size(); + for (std::size_t at = 0; at < nodes.size(); ++at) + if (at != slot && nodes[at].parent == nodes[slot].parent && nodes[at].map == nodes[slot].map && + desired[at] == false && world_layer_time_of_day(at) && !fully_off(at)) + return at; + return nodes.size(); +} + inline bool world_layer_supported( std::size_t choice, const std::vector& present) noexcept { diff --git a/Engine/Resource/Test/ebx_fixup_repair_tests.cpp b/Engine/Resource/Test/ebx_fixup_repair_tests.cpp new file mode 100644 index 00000000..87543378 --- /dev/null +++ b/Engine/Resource/Test/ebx_fixup_repair_tests.cpp @@ -0,0 +1,105 @@ +// The EBX type-reference repair: an entry on another fixup's slot, or listed twice, comes out of +// the list, the chunk and RIFF sizes follow, and a document without one is left alone. +#include "Engine/Resource/ebx_fixup_repair.h" +#include +#include +#include +#include + +namespace fb = dingosdk::frostbite; + +namespace { +int failures = 0; +void check(bool condition, const char* what) { + if (!condition) { std::cerr << "FAILED: " << what << "\n"; ++failures; } +} +void u32(std::vector& bytes, std::uint32_t value) { + const auto at = bytes.size(); + bytes.resize(at + 4); + std::memcpy(bytes.data() + at, &value, 4); +} +std::uint32_t read(const std::vector& bytes, std::size_t at) { + std::uint32_t value{}; + std::memcpy(&value, bytes.data() + at, 4); + return value; +} +void chunk(std::vector& riff, std::string_view id, const std::vector& data) { + for (const auto ch : id) riff.push_back(static_cast(ch)); + u32(riff, static_cast(data.size())); + riff.insert(riff.end(), data.begin(), data.end()); + if (data.size() & 1) riff.push_back(std::byte{}); +} +std::vector list_at(const std::vector& bytes, std::size_t at) { + std::vector out(read(bytes, at)); + for (std::size_t index = 0; index < out.size(); ++index) out[index] = read(bytes, at + 4 + 4 * index); + return out; +} + +// An EBX with one type, one instance at 0x10, a pointer at 0x40, one import relocated at 0x60, +// and the type references given. Returns where the type-reference count lies. +std::vector document(const std::vector& types, std::size_t& typeList) { + std::vector efix(16); // the file guid + u32(efix, 1); + efix.resize(efix.size() + 16); // its type + u32(efix, 0); // no signatures + u32(efix, 1); // exported + u32(efix, 1); + u32(efix, 0x10); // the instance + u32(efix, 1); + u32(efix, 0x40); // a pointer + u32(efix, 0); // no resource references + u32(efix, 1); + efix.resize(efix.size() + 32); // an import + u32(efix, 1); + u32(efix, 0x60); // its relocation + const auto list = efix.size(); + u32(efix, static_cast(types.size())); + for (const auto type : types) u32(efix, type); + u32(efix, 0x80); + u32(efix, 0x80); + u32(efix, 0x80); // the sections + std::vector riff; + for (const auto ch : std::string_view("RIFF")) riff.push_back(static_cast(ch)); + u32(riff, 0); + for (const auto ch : std::string_view("EBX\0", 4)) riff.push_back(static_cast(ch)); + chunk(riff, "EBXD", std::vector(0x90)); + typeList = riff.size() + 8 + list; + chunk(riff, "EFIX", efix); + chunk(riff, "EBXX", std::vector(8)); + chunk(riff, "REFL", std::vector(5)); + const auto size = static_cast(riff.size() - 8); + std::memcpy(riff.data() + 4, &size, 4); + return riff; +} +} // namespace + +int main() { + { + std::size_t list{}; + const auto clean = document({0x20, 0x28}, list); + check(!fb::ebx::repair_type_fixups(clean), "a document without collisions is left alone"); + } + { + // On the import's slot, inside the instance's header, and the same slot twice. + std::size_t list{}; + const auto broken = document({0x20, 0x60, 0x18, 0x28, 0x20}, list); + const auto repaired = fb::ebx::repair_type_fixups(broken); + check(repaired.has_value(), "a document with collisions is repaired"); + if (repaired) { + check(repaired->removed == 3, "three entries are removed"); + check(repaired->bytes.size() == broken.size() - 12, "the document shrinks by those entries"); + check(list_at(repaired->bytes, list) == std::vector{0x20, 0x28}, "the clean entries stay, in order"); + check(read(repaired->bytes, 4) == read(broken, 4) - 12, "the RIFF size follows"); + constexpr std::size_t efixSize = 12 + 8 + 0x90 + 4; // after the header and EBXD + check(read(repaired->bytes, efixSize) == read(broken, efixSize) - 12, "the EFIX size follows"); + // EBXX (8 + 8 bytes) and REFL (8 + 5, padded) end the document. + check(std::memcmp(repaired->bytes.data() + repaired->bytes.size() - 30, "EBXX", 4) == 0, + "the chunks after EFIX are carried along"); + check(!fb::ebx::repair_type_fixups(repaired->bytes), "the repaired document needs nothing more"); + } + } + check(!fb::ebx::repair_type_fixups(std::vector(4)), "something that is not an EBX is left alone"); + if (failures) return 1; + std::cout << "ebx fixup repair tests passed\n"; + return 0; +} diff --git a/Engine/Resource/Test/oodle_test_codec.cpp b/Engine/Resource/Test/oodle_test_codec.cpp index 9c794acb..a0957543 100644 --- a/Engine/Resource/Test/oodle_test_codec.cpp +++ b/Engine/Resource/Test/oodle_test_codec.cpp @@ -8,3 +8,8 @@ extern "C" __declspec(dllexport) std::int64_t __cdecl OodleLZ_Decompress( *static_cast(output) = *static_cast(input); return 1; } + +// The buffer a decode into `rawSize` bytes needs: as the real codec, a little more than asked. +extern "C" __declspec(dllexport) std::int64_t __cdecl OodleLZ_GetDecodeBufferSize(int, std::int64_t rawSize, int) { + return rawSize + 16; +} diff --git a/Engine/Resource/Test/shader_lookup_tests.cpp b/Engine/Resource/Test/shader_lookup_tests.cpp new file mode 100644 index 00000000..5b86d754 --- /dev/null +++ b/Engine/Resource/Test/shader_lookup_tests.cpp @@ -0,0 +1,87 @@ +// The shader program lookup's merge: the rows mods add are combined onto the base, and a row +// that points past the base's shader records is left out (the game reads such an index outside +// its shader database). +#include "Engine/Resource/shader_lookup.h" +#include +#include +#include +#include + +namespace fb = dingosdk::frostbite; + +namespace { +int failures = 0; +void check(bool condition, const char* what) { + if (!condition) { std::cerr << "FAILED: " << what << "\n"; ++failures; } +} +template void append(std::vector& bytes, Type value) { + const auto at = bytes.size(); + bytes.resize(at + sizeof(value)); + std::memcpy(bytes.data() + at, &value, sizeof(value)); +} +template Type read(const std::vector& bytes, std::size_t at) { + Type value{}; + std::memcpy(&value, bytes.data() + at, sizeof(value)); + return value; +} +// A program lookup: a blob of `records` records (only its count matters here), no relocations, +// and the rows (key -> record indices). +struct Lookup { + std::vector resource, meta; + fb::shader::Table table() const { return {resource, meta}; } +}; +Lookup lookup(std::uint32_t records, const std::map>& rows) { + Lookup out; + append(out.resource, records); + append(out.resource, std::uint32_t{0}); // the rest of the blob + const auto program = static_cast(out.resource.size()); + append(out.resource, static_cast(rows.size())); + for (const auto& [key, values] : rows) { + append(out.resource, key); + append(out.resource, static_cast(values.size())); + for (const auto value : values) append(out.resource, value); + } + append(out.meta, program); + append(out.meta, std::uint32_t{0}); + append(out.meta, static_cast(out.resource.size() - program)); + append(out.meta, std::uint32_t{0}); + return out; +} +// The merged rows, read back. +std::map> rows_of(const fb::shader::MergedTable& merged) { + std::size_t at = read(merged.resourceMeta, 0); + const auto count = read(merged.resource, at); + at += 4; + std::map> rows; + for (std::uint32_t row = 0; row < count; ++row) { + const auto key = read(merged.resource, at); + const auto values = read(merged.resource, at + 8); + at += 12; + auto& out = rows[key]; + for (std::uint32_t value = 0; value < values; ++value, at += 4) out.push_back(read(merged.resource, at)); + } + return rows; +} +} // namespace + +int main() { + const auto base = lookup(3, {{1, {0}}}); + // One mod adds a row onto a program the game has; another brings programs of its own and + // points rows at them (indices 3 and 4, which the base's three records do not have). + const auto good = lookup(3, {{1, {0}}, {2, {2}}}); + const auto own = lookup(5, {{1, {0}}, {3, {4}}, {4, {1, 3}}, {5, {1}}}); + const std::vector edits{good.table(), own.table()}; + const auto merged = fb::shader::merge_program_lookup(base.table(), edits); + const auto rows = rows_of(merged); + check(merged.added == 2 && rows.contains(2) && rows.contains(5), "Rows onto the base's programs are merged"); + check(merged.outside == 2 && !rows.contains(3) && !rows.contains(4), + "Rows pointing past the base's records are left out, also when only one of their indices does"); + check(rows.size() == 3 && rows.at(1) == std::vector{0}, "The base's own rows stay as they were"); + check(read(merged.resourceMeta, 8) == merged.resource.size() - read(merged.resourceMeta, 0), + "The lookup size in the metadata matches the merged rows"); + for (const auto& [key, values] : rows) + for (const auto value : values) check(value < 3, "No merged row points past the base's records"); + if (failures) return 1; + std::cout << "shader lookup tests passed\n"; + return 0; +} diff --git a/Engine/Resource/cas_codec.cpp b/Engine/Resource/cas_codec.cpp index 83b45709..a1ab5641 100644 --- a/Engine/Resource/cas_codec.cpp +++ b/Engine/Resource/cas_codec.cpp @@ -50,6 +50,10 @@ private: // Oodle is the game's own module, so it is resolved once and never unloaded. #ifdef _WIN32 +// What Oodle does when one of its checks fails on what it is given: its default stops the +// process at a breakpoint (the launcher's mod scan, "EXCEPTION_BREAKPOINT" in oo2core, +// 2026-10-15). Not stopping, the call fails and the block is reported as unreadable. +int __cdecl oodle_assertion(const char*, int, const char*, const char*) { return 0; } HMODULE oodle(const std::filesystem::path& gameRoot) { static std::mutex mutex; static HMODULE handle{}; @@ -74,6 +78,13 @@ HMODULE oodle(const std::filesystem::path& gameRoot) { LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32); if (!handle) throw std::runtime_error("Cannot load " + path_utf8(matches.back()) + " (Windows error " + std::to_string(GetLastError()) + ")"); + // (Only for a copy loaded here: the game's own, found above, keeps the game's handler.) + using Assertion = int(__cdecl*)(const char*, int, const char*, const char*); +#pragma warning(push) +#pragma warning(disable: 4191) + if (const auto set = reinterpret_cast(GetProcAddress(handle, "OodleCore_Plugins_SetAssertion"))) +#pragma warning(pop) + set(&oodle_assertion); return handle; } @@ -134,12 +145,24 @@ std::vector oodle_decode(std::span input, std::size_ #ifdef _WIN32 using Function = std::int64_t(__cdecl*)(const void*, std::int64_t, void*, std::int64_t, int, int, int, void*, std::int64_t, void*, void*, void*, std::int64_t, int); + using BufferSize = std::int64_t(__cdecl*)(int, std::int64_t, int); const auto decode = oodle_function(gameRoot, "OodleLZ_Decompress"); - std::vector output(size); + // A block can be corrupt, or not a block at all (read_manifest tries a payload as one when + // it does not read as a manifest), and its size is only what its header says. Oodle may + // then write past that size: the buffer it says such data needs (for any compressor) is + // given, and cut to the size afterwards. With exactly the size, it overwrote the heap + // (the launcher's mod scan crashing in malloc after an Oodle block, 2026-10-15). + constexpr int any_compressor = -1; // OodleLZ_Compressor_Invalid + constexpr std::int64_t block = 0x40000; // OODLELZ_BLOCK_LEN + const auto needed = oodle_function(gameRoot, "OodleLZ_GetDecodeBufferSize")( + any_compressor, static_cast(size), 1); + const auto padded = std::max(needed, static_cast(size) + block); + std::vector output(static_cast(padded)); if (decode(input.data(), static_cast(input.size()), output.data(), - static_cast(output.size()), 1, 0, 0, nullptr, 0, nullptr, nullptr, + static_cast(size), 1, 0, 0, nullptr, 0, nullptr, nullptr, nullptr, 0, 3) != static_cast(size)) throw std::runtime_error("Oodle CAS block failed to decode"); + output.resize(size); return output; #else (void)input; diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index 3f439739..c0960a9a 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -1,6 +1,7 @@ #include "ebx_document.h" #include +#include #include #include #include @@ -292,7 +293,14 @@ std::vector read_reflection(BinaryReader& reader, const auto groupCount = static_cast(nonnegative(reader, "reflection group")); ensure_records(reader, chunk.end, groupCount, 12, "reflection group"); - reader.skip(static_cast(groupCount * 12)); + document.reflectionGroups.reserve(groupCount); + for (std::size_t index = 0; index < groupCount; ++index) { + ReflectionGroup group; + group.hash = reader.u32(); + group.count = reader.u32(); + group.first = reader.u32(); + document.reflectionGroups.push_back(group); + } const auto mappingCount = static_cast(nonnegative(reader, "reflection mapping")); ensure_records(reader, chunk.end, mappingCount, 8, "reflection mapping"); reader.skip(static_cast(mappingCount * 8)); @@ -356,7 +364,9 @@ void map_instances(Document& document, class ValueReader final { public: ValueReader(const std::span bytes, Document& document) - : bytes_(bytes), reader_(bytes), document_(document) {} + : bytes_(bytes), reader_(bytes), document_(document) { + for (const auto& record : document.arrays) arrayHashes_.emplace(record.offset, record.hash); + } void read_instances() { for (auto& instance : document_.instances) { @@ -423,7 +433,19 @@ private: } const auto position = start + field.dataOffset; if (position > document_.dataEnd) throw std::runtime_error("EBX field offset exceeds EBXD"); - output.fields.push_back({descriptorIndex, field.name, read_field(field, position)}); + auto value = read_field(field, position); + if (field.category() == FieldCategory::array) { + const auto* values = std::get_if(&value.data); + if (values && !values->empty()) { + std::int32_t displacement{}; + std::memcpy(&displacement, bytes_.data() + position, sizeof(displacement)); + const auto target = static_cast(position) + displacement - + static_cast(document_.dataStart); + if (const auto found = arrayHashes_.find(target); found != arrayHashes_.end()) + value.arrayHash = found->second; + } + } + output.fields.push_back({descriptorIndex, field.name, std::move(value)}); } } @@ -552,6 +574,8 @@ private: } std::span bytes_; + std::unordered_map arrayHashes_; // record offset -> its hash + BinaryReader reader_; Document& document_; unsigned depth_{}; @@ -615,6 +639,8 @@ Document read_document(const std::span bytes) { result.hashedReflectionNames = reflection2 != nullptr; result.reflectionData.assign(bytes.begin() + static_cast(reflectionChunk.begin), bytes.begin() + static_cast(reflectionChunk.end)); + result.payload.assign(bytes.begin() + static_cast(result.dataStart), + bytes.begin() + static_cast(result.dataEnd)); map_instances(result, fixup, reflectionIds); ValueReader(bytes, result).read_instances(); std::ranges::sort(result.boxedValues, {}, &BoxedValueRecord::offset); diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index 4ae98098..e99b2f45 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,14 @@ struct ArrayRecord { std::uint16_t classRef{}; }; +// A 12-byte record of the RFL2 group table: one per array type the document uses, sorted by +// hash (first[i + 1] = first[i] + count[i]). Every array record's hash is one of these keys. +struct ReflectionGroup { + std::uint32_t hash{}; + std::uint32_t count{}; + std::uint32_t first{}; +}; + struct BoxedValueRecord { std::uint32_t offset{}; std::uint32_t count{}; @@ -149,6 +158,9 @@ struct Value { PointerReference, TypeReference, BoxedReference, std::shared_ptr, Array>; Storage data; + // An array's own record hash, kept when the document is written again; a new array has none + // and borrows one (ebx_writer.cpp, array_hash). + std::optional arrayHash; }; struct FieldValue { @@ -189,6 +201,10 @@ struct Document { std::vector importOffsets; std::vector typeInfoOffsets; std::vector boxedValueSlots; + // The EBXD bytes the document was read from (offsets as in the fixup tables): what boxed + // values point at (arrays, strings) is carried from here when the document is written. + std::vector payload; + std::vector reflectionGroups; [[nodiscard]] const InstanceRecord* root() const noexcept; }; diff --git a/Engine/Resource/ebx_fixup_repair.cpp b/Engine/Resource/ebx_fixup_repair.cpp new file mode 100644 index 00000000..5c9cef6b --- /dev/null +++ b/Engine/Resource/ebx_fixup_repair.cpp @@ -0,0 +1,137 @@ +#include "ebx_fixup_repair.h" + +#include +#include +#include +#include + +namespace dingosdk::frostbite::ebx { +namespace { + +std::uint32_t u32_at(std::span bytes, std::size_t at) { + if (at > bytes.size() || bytes.size() - at < 4) throw std::out_of_range("EBX read past its end"); + std::uint32_t value{}; + std::memcpy(&value, bytes.data() + at, 4); + return value; +} +void put_u32(std::vector& bytes, std::size_t at, std::uint32_t value) { std::memcpy(bytes.data() + at, &value, 4); } + +// The fixup lists of an EBX, read from its EFIX chunk alone: this runs on every EBX a mod +// ships, and the rest of the document has nothing to say about where the fixups go. +struct Fixups { + std::size_t efix{}, efixSize{}; // the chunk's data and its size + std::size_t typeList{}; // where the type-reference count is + std::vector instances, pointers, resources, imports, types, boxed; +}; + +std::optional read_fixups(std::span ebx) { + if (ebx.size() < 12 || std::memcmp(ebx.data(), "RIFF", 4) != 0) return std::nullopt; + Fixups out; + const auto riffEnd = std::size_t{u32_at(ebx, 4)} + 8; + if (riffEnd > ebx.size()) return std::nullopt; + for (std::size_t at = 12; at + 8 <= riffEnd;) { + const auto size = std::size_t{u32_at(ebx, at + 4)}; + if (std::memcmp(ebx.data() + at, "EFIX", 4) == 0) { + out.efix = at + 8; + out.efixSize = size; + break; + } + at += 8 + size + (size & 1); + } + const auto end = out.efix + out.efixSize; + if (!out.efix || end > riffEnd) return std::nullopt; + + std::size_t at = out.efix + 16; // the file guid + const auto count = [&](std::size_t width) { + const auto value = std::size_t{u32_at(ebx, at)}; + at += 4; + if (value > (end - at) / width) throw std::out_of_range("EFIX list past its chunk"); + return value; + }; + const auto list = [&](std::vector& into) { + into.resize(count(4)); + for (auto& value : into) { value = u32_at(ebx, at); at += 4; } + }; + at += 16 * count(16); // types + at += 4 * count(4); // signatures + at += 4; // exported instances + list(out.instances); + list(out.pointers); + list(out.resources); + at += 32 * count(32); // imports + list(out.imports); + out.typeList = at; + list(out.types); + if (end - at < 12) return std::nullopt; + at += 12; // the array, boxed-value and string sections + if (at < end) list(out.boxed); + if (at != end) return std::nullopt; + return out; +} + +// Which type-reference entries collide with another fixup's slot, or repeat. +std::set colliding(const Fixups& fixups) { + struct Range { std::uint64_t begin, end; }; + std::vector others; + const auto add = [&](std::uint64_t begin, std::uint64_t size) { others.push_back({begin, begin + size}); }; + for (const auto offset : fixups.pointers) add(offset, 8); + for (const auto offset : fixups.resources) add(offset, 8); + for (const auto offset : fixups.imports) add(offset, 8); + for (const auto offset : fixups.boxed) add(std::uint64_t{offset} + 8, 8); + for (const auto offset : fixups.instances) add(offset, 16); + std::set bad; + std::set seen; + for (std::size_t index = 0; index < fixups.types.size(); ++index) { + const std::uint64_t offset = fixups.types[index]; + if (!seen.insert(fixups.types[index]).second) { + bad.insert(index); + continue; + } + for (const auto& other : others) + if (offset < other.end && other.begin < offset + 8) { + bad.insert(index); + break; + } + } + return bad; +} + +} // namespace + +std::optional repair_type_fixups(std::span ebx) { + std::optional fixups; + try { + fixups = read_fixups(ebx); + } catch (const std::exception&) { + return std::nullopt; + } + if (!fixups) return std::nullopt; + const auto bad = colliding(*fixups); + if (bad.empty()) return std::nullopt; + + // The type-reference list is the only thing that changes length. EBXD comes ahead of EFIX, + // so the data's alignment and every offset into it stay as they are. + FixupRepair out; + out.removed = bad.size(); + const auto list = fixups->typeList; + out.bytes.assign(ebx.begin(), ebx.begin() + static_cast(list + 4)); + for (std::size_t index = 0; index < fixups->types.size(); ++index) { + if (bad.contains(index)) continue; + const auto at = out.bytes.size(); + out.bytes.resize(at + 4); + put_u32(out.bytes, at, fixups->types[index]); + } + const auto rest = list + 4 + 4 * fixups->types.size(); + out.bytes.insert(out.bytes.end(), ebx.begin() + static_cast(rest), ebx.end()); + const auto shrink = static_cast(4 * bad.size()); + put_u32(out.bytes, list, static_cast(fixups->types.size() - bad.size())); + put_u32(out.bytes, fixups->efix - 4, static_cast(fixups->efixSize) - shrink); // the chunk's size (its padding is unchanged) + put_u32(out.bytes, 4, u32_at(ebx, 4) - shrink); // the RIFF's + // What was written reads back, without the collisions. + const auto again = read_fixups(out.bytes); + if (!again || !colliding(*again).empty() || again->types.size() != fixups->types.size() - bad.size()) + throw std::runtime_error("EBX type-reference repair did not read back"); + return out; +} + +} // namespace dingosdk::frostbite::ebx diff --git a/Engine/Resource/ebx_fixup_repair.h b/Engine/Resource/ebx_fixup_repair.h new file mode 100644 index 00000000..2c3476a0 --- /dev/null +++ b/Engine/Resource/ebx_fixup_repair.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include +#include +#include + +// An EBX's type-reference fixups that would crash the game's load. The fast-path partition load +// (Skate.exe 0x3def1c0) applies the other fixups first and then reads each type-reference slot as +// a type index; a type reference listed on a slot another fixup also writes (an import, a pointer, +// a resource reference, an instance's header, a boxed reference's pointer half), or listed twice, +// reads a pointer as an index and the game crashes (Skate.exe+0x3def285 / +0x3def262: Master +// Chief and TASM2 costume presets, 2026-10-17). The game's own files never do this. Removing those +// entries from the type-reference list leaves the slot to the fixup that owns it. +namespace dingosdk::frostbite::ebx { + +struct FixupRepair { + std::vector bytes; // the document with those entries removed + std::size_t removed{}; +}; + +// The document repaired, or nothing when it needs none (or is not one this reads). +[[nodiscard]] std::optional repair_type_fixups(std::span ebx); + +} // namespace dingosdk::frostbite::ebx diff --git a/Engine/Resource/ebx_writer.cpp b/Engine/Resource/ebx_writer.cpp index a1d7bcf6..395b6455 100644 --- a/Engine/Resource/ebx_writer.cpp +++ b/Engine/Resource/ebx_writer.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -85,7 +87,8 @@ std::size_t scalar_size(const FieldType type) { class Builder final { public: - explicit Builder(const Document& document) : document_(document) {} + Builder(const Document& document, const bool refuseNewArrayHashes) + : document_(document), refuse_(refuseNewArrayHashes) {} std::vector run() { validate(); @@ -106,6 +109,8 @@ private: std::uint32_t slot{}; const Value::Array* values{}; std::size_t descriptor{}; + std::optional hash; + std::int32_t holder{-1}; // type of the object holding the field bool emitted{}; }; @@ -208,7 +213,8 @@ private: static const Value::Array empty; const auto* array = value ? std::get_if(&value->data) : nullptr; arraysPending_.push_back({checked_u32(position, "EBX array slot"), array ? array : &empty, - descriptorIndex}); + descriptorIndex, value ? value->arrayHash : std::nullopt, + object ? object->descriptor : static_cast(typeIndex)}); } else if (field.type() == FieldType::structure) { if (field.classRef >= document_.types.size()) throw std::runtime_error("EBX struct type is invalid"); const auto nestedStart = align_up(position, std::max(document_.types[field.classRef].alignment, 1)); @@ -352,6 +358,44 @@ private: return field.nameHash; } + // A new array takes the hash of an array the document read for the same field of the same type; + // failing that it borrows (array_hash), or is refused (see write_document). + std::uint32_t new_array_hash(const PendingArray& pending, const FieldDescriptor& field) { + if (!knownWalked_) { + knownWalked_ = true; + for (const auto& instance : document_.instances) + if (instance.object) collect_hashes(*instance.object); + } + const auto found = knownHashes_.find({pending.holder, pending.descriptor}); + if (refuse_) { + if (found == knownHashes_.end()) + throw std::runtime_error("EBX new array " + std::string(field.name) + + " has no array of its field to take a hash from"); + if (document_.hashedReflectionNames && + std::ranges::none_of(document_.reflectionGroups, + [&](const ReflectionGroup& group) { return group.hash == found->second; })) + throw std::runtime_error("EBX array hash of " + std::string(field.name) + + " is not in the reflection's group table"); + } + return found != knownHashes_.end() ? found->second : array_hash(field); + } + + void collect_hashes(const Object& object) { + for (const auto& field : object.fields) { + if (field.value.arrayHash) + knownHashes_.emplace(std::pair{object.descriptor, field.descriptor}, *field.value.arrayHash); + collect_value(field.value); + } + } + + void collect_value(const Value& value) { + if (const auto* nested = std::get_if>(&value.data)) { + if (*nested) collect_hashes(**nested); + } else if (const auto* array = std::get_if(&value.data)) { + for (const auto& element : *array) collect_value(element); + } + } + void write_arrays() { arraysOffset_ = checked_u32(align_up(payload_.size(), 16), "EBX array section"); grow(static_cast(arraysOffset_) + 32); @@ -359,6 +403,124 @@ private: for (std::size_t index = 0; index < arraysPending_.size(); ++index) { if (!arraysPending_[index].emitted) emit_array(index); } + carry_boxed_arrays(); + } + + // What a boxed value points at is not in the document's values: its bytes are carried as they + // were, so the arrays and strings its pointers reach are carried with them and the pointers + // moved to the new places. Without this a rewrite dropped them (seen on the game's own + // level roots: a boxed debug string and about 1 KB of arrays). + + enum class Target { array, string, instance, boxed, empty, unknown }; + + std::pair old_target(const std::uint32_t slot) const { + const auto& old = document_.payload; + std::int32_t displacement{}; + std::memcpy(&displacement, old.data() + slot, sizeof(displacement)); + const auto target = static_cast(static_cast(slot) + displacement); + if (oldArrays_.contains(target)) return {Target::array, target}; + if (document_.stringsOffset <= target && target < old.size()) return {Target::string, target}; + if (const auto found = oldInstances_.find(target); found != oldInstances_.end()) + return {Target::instance, found->second}; + if (document_.boxedValuesOffset <= target && target < document_.stringsOffset) return {Target::boxed, target}; + if (target >= 4 && target <= old.size()) { + std::uint32_t count{}; + std::memcpy(&count, old.data() + target - 4, sizeof(count)); + if (count == 0) return {Target::empty, target}; + } + return {Target::unknown, target}; + } + + static std::vector old_slots(const std::uint32_t start, const std::size_t length, + std::vector offsets) { + std::ranges::sort(offsets); + offsets.erase(std::unique(offsets.begin(), offsets.end()), offsets.end()); + std::vector out; + for (const auto value : offsets) + if (value >= start && value - start < length) out.push_back(value); + return out; + } + + void carry_boxed_arrays() { + if (document_.payload.empty()) return; + for (const auto& record : document_.arrays) oldArrays_.emplace(record.offset, record); + for (std::size_t index = 0; index < document_.instances.size(); ++index) + oldInstances_.emplace(document_.instances[index].dataOffset, static_cast(index)); + auto records = document_.boxedValues; + std::ranges::sort(records, {}, &BoxedValueRecord::offset); + for (const auto& source : records) + for (const auto slot : old_slots(source.offset, source.rawBytes.size(), document_.pointerOffsets)) { + const auto [kind, key] = old_target(slot); + if (kind == Target::array) carry_array(oldArrays_.at(key)); + } + } + + std::pair array_element(const ArrayRecord& record) const { + FieldDescriptor probe; + probe.encodedType = static_cast(record.encodedType >> 1); + if (probe.type() == FieldType::structure) { + const auto slot = record.classRef; + if (slot < document_.fixupTypeGuids.size() && slot < document_.fixupTypeSignatures.size()) + for (const auto& type : document_.types) + if (type.guid == document_.fixupTypeGuids[slot] && type.signature == document_.fixupTypeSignatures[slot]) + return {std::max(type.size, 1), std::max(type.alignment, 1)}; + throw std::runtime_error("EBX array record has an unknown struct type"); + } + const auto stride = scalar_size(probe.type()); + return {stride, std::max(std::min(stride, 8), 1)}; + } + + void carry_array(const ArrayRecord& record) { + if (carried_.contains(record.offset)) return; + const auto [stride, alignment] = array_element(record); + const auto length = stride * record.count; + if (record.offset + length > document_.payload.size()) throw std::runtime_error("EBX carried array exceeds EBXD"); + const auto data = align_up(payload_.size() + 4, alignment); + put_number(data - 4, record.count); + put(data, std::span(document_.payload).subspan(record.offset, length)); + const auto placed = checked_u32(data, "EBX array data"); + carried_[record.offset] = placed; + arrays_.push_back({placed, record.count, record.hash, record.encodedType, record.classRef}); + relocate(record.offset, length, data); + } + + void relocate(const std::uint32_t oldStart, const std::size_t length, const std::size_t newStart) { + const auto moved = [&](const std::uint32_t old) { + return checked_u32(newStart + (old - oldStart), "EBX carried relocation"); + }; + for (const auto old : old_slots(oldStart, length, document_.resourceRefOffsets)) resourceOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.importOffsets)) importOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.typeInfoOffsets)) typeInfoOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.pointerOffsets)) { + const auto slot = moved(old); + pointerOffsets_.push_back(slot); + if (document_.payload.empty()) continue; + const auto [kind, key] = old_target(old); + switch (kind) { + case Target::array: + carry_array(oldArrays_.at(key)); + put_number(slot, static_cast(carried_.at(key) - slot)); + break; + case Target::string: { + const auto& source = document_.payload; + auto end = static_cast(key); + while (end < source.size() && source[end] != std::byte{0}) ++end; + stringsPending_.push_back({slot, std::string(reinterpret_cast(source.data()) + key, end - key)}); + break; + } + case Target::instance: + put_number(slot, static_cast(instanceOffsets_.at(key) - slot)); + break; + case Target::empty: + put_number(slot, static_cast(emptyArrayTarget_ - slot)); + break; + case Target::boxed: + boxedTargets_.emplace_back(slot, key); + break; + case Target::unknown: + break; + } + } } void emit_array(const std::size_t index) { @@ -387,8 +549,11 @@ private: write_scalar(field, elementStart, &value); } } + // An array read from a record keeps that record's hash; a new one borrows (array_hash). + auto hash = arraysPending_[index].hash; + if (!hash) hash = new_array_hash(arraysPending_[index], field); arrays_.push_back({checked_u32(data, "EBX array data"), checked_u32(values->size(), "EBX array count"), - array_hash(field), static_cast(field.encodedType << 1), + *hash, static_cast(field.encodedType << 1), field.type() == FieldType::structure ? fixup_slot_for_descriptor(field.classRef) : static_cast(0xFFFF)}); @@ -407,23 +572,14 @@ private: if (source.offset < document_.boxedValuesOffset) throw std::runtime_error("EBX boxed record precedes its section"); const auto offset = static_cast(boxedOffset_) + source.offset - document_.boxedValuesOffset; put(offset, source.rawBytes); - const auto relocate = [&](const std::vector& sourceOffsets, - std::vector& destinationOffsets) { - for (const auto old : sourceOffsets) { - if (old < source.offset || old - source.offset >= source.rawBytes.size()) continue; - destinationOffsets.push_back(checked_u32(offset + old - source.offset, - "EBX boxed relocation")); - } - }; - relocate(document_.pointerOffsets, pointerOffsets_); - relocate(document_.resourceRefOffsets, resourceOffsets_); - relocate(document_.importOffsets, importOffsets_); - relocate(document_.typeInfoOffsets, typeInfoOffsets_); + relocate(source.offset, source.rawBytes.size(), offset); moved[source.offset] = checked_u32(offset, "EBX boxed value"); auto authored = source; authored.offset = checked_u32(offset, "EBX boxed value"); boxed_.push_back(std::move(authored)); } + for (const auto& [slot, target] : boxedTargets_) + put_number(slot, static_cast(boxedOffset_ + (target - document_.boxedValuesOffset) - slot)); for (const auto& pending : boxesPending_) { put_number(pending.slot, pending.value.encodedType); put_number(static_cast(pending.slot) + 4, std::uint32_t{}); @@ -588,6 +744,13 @@ private: std::uint32_t arraysOffset_{}; std::uint32_t boxedOffset_{}; std::uint32_t emptyArrayTarget_{}; + std::unordered_map oldArrays_; + std::unordered_map oldInstances_; + std::unordered_map carried_; + std::vector> boxedTargets_; + bool refuse_{}; + bool knownWalked_{}; + std::map, std::uint32_t> knownHashes_; // (holder, field) -> hash }; } @@ -611,8 +774,8 @@ std::uint16_t ensure_fixup_type(Document& document, const std::int32_t descripto return static_cast(count); } -std::vector write_document(const Document& document) { - return Builder(document).run(); +std::vector write_document(const Document& document, const bool refuseNewArrayHashes) { + return Builder(document, refuseNewArrayHashes).run(); } } diff --git a/Engine/Resource/ebx_writer.h b/Engine/Resource/ebx_writer.h index c042f681..96179a0d 100644 --- a/Engine/Resource/ebx_writer.h +++ b/Engine/Resource/ebx_writer.h @@ -9,6 +9,10 @@ namespace dingosdk::frostbite::ebx { [[nodiscard]] std::uint16_t ensure_fixup_type(Document& document, std::int32_t descriptor); -[[nodiscard]] std::vector write_document(const Document& document); +// A new array (one no record was read for) takes the hash of an array the document read for the +// same field of the same type. Without one it borrows another array's hash (the merge's way), or, +// with refuseNewArrayHashes (the map builder's), the write is refused: the RFL2 group table would +// need a new entry. A hash that is not a key of that table is refused the same way. +[[nodiscard]] std::vector write_document(const Document& document, bool refuseNewArrayHashes = false); } diff --git a/Engine/Resource/mesh_set.cpp b/Engine/Resource/mesh_set.cpp index 3e1fb9db..63511bce 100644 --- a/Engine/Resource/mesh_set.cpp +++ b/Engine/Resource/mesh_set.cpp @@ -1,4 +1,5 @@ #include "mesh_set.h" +#include #include #include #include @@ -131,6 +132,118 @@ std::vector> read_mesh_positions(std::span return result; } +namespace { +// The two layouts check_mesh_lod reads: the skinned one above, and the static one world meshes +// have (192, 176, 384, 0), whose levels of detail are laid out the same. A static mesh's sections +// are 384 or 416 bytes, and nothing found yet says which: check_mesh_lod takes the size under +// which every section reads as one. +struct Layout { + bool skinned{}; +}; +Layout layout_of(const Reader& r) { + const auto first = r.at(0), second = r.at(4), third = r.at(8); + if (first == 208 && second == 188 && third == 384) return {true}; + if (first == 192 && second == 176 && third == 384) return {false}; + require(false, "its layout is not one this reader knows"); + return {}; +} +// Whether every section, `stride` bytes apart, reads as one: a triangle list whose indices are in +// the index buffer and whose vertices start in the vertex buffer. +bool sections_read(const Reader& r, std::size_t table, std::uint32_t sections, std::size_t stride, std::size_t index_size, + std::size_t vertex_size, std::size_t index_total) { + for (std::uint32_t section = 0; section < sections; ++section) { + const auto s = table + section * stride; + if (s + stride > r.data.size()) return false; + const std::size_t triangles = r.at(s + triangle_count); + if (!triangles) continue; + if (r.at(s + primitive) != triangle_list || r.at(s + vertex_start) >= vertex_size || + std::size_t{r.at(s + first_index)} * index_size + triangles * 3 * index_size > index_total) + return false; + } + return true; +} +} + +std::vector check_mesh_lod(std::span resource, std::size_t lod, + std::span geometry, std::uint32_t* format_out) { + const Reader r{resource}; + const auto shape = layout_of(r); + require(lod < r.at(lod_count), "it has no such level of detail"); + const auto at = r.pointer(lod_table + lod * 8); + const std::size_t vertex_size = r.at(at + vertex_bytes), index_size_total = r.at(at + index_bytes); + const auto format = r.at(at + index_format); + if (format_out) *format_out = format; + const std::size_t index_size = format == sixteen_bit_indices ? 2 : 4; + std::vector problems; + const auto say = [&](std::string text) { if (problems.size() < 8) problems.push_back(std::move(text)); }; + if (geometry.size() < vertex_size + index_size_total) + say("the geometry chunk holds " + std::to_string(geometry.size()) + " bytes of the " + + std::to_string(vertex_size + index_size_total) + " its buffers need"); + const auto sections = r.at(at + section_count); + require(sections <= 4096, "it has too many sections"); + if (!sections) return problems; + const auto table = r.pointer(at + section_table); + std::size_t stride = section_size; + if (!shape.skinned) { + if (sections_read(r, table, sections, 384, index_size, vertex_size, index_size_total)) stride = 384; + else if (sections_read(r, table, sections, 416, index_size, vertex_size, index_size_total)) stride = 416; + else say("its sections read as neither 384 nor 416 bytes apart: one of them is out of its buffers"); + } + for (std::uint32_t section = 0; section < sections; ++section) { + const auto s = table + section * stride; + const auto name = "section " + std::to_string(section); + const auto type = r.at(s + primitive); + const std::size_t count = r.at(s + vertex_count), triangles = r.at(s + triangle_count); + if (!triangles) continue; + if (type != triangle_list) { say(name + " is primitive type " + std::to_string(type) + ", not a triangle list"); continue; } + if (!count) { say(name + " has " + std::to_string(triangles) + " triangles and no vertices"); continue; } + const std::size_t first = r.at(s + vertex_start); + if (first >= vertex_size) say(name + "'s vertices start at byte " + std::to_string(first) + " of a " + std::to_string(vertex_size) + "-byte vertex buffer"); + if (shape.skinned) { + // (Where its vertices end: each stream's follow the one before.) + const auto d = s + declaration; + const auto streams = r.at(d + stream_total); + std::size_t end = first; + for (std::size_t stream = 0; stream < streams && stream < 16; ++stream) + end += std::size_t{r.at(d + stream_table + stream * 2)} * count; + if (end > vertex_size) + say(name + "'s vertices end at byte " + std::to_string(end) + " of a " + std::to_string(vertex_size) + "-byte vertex buffer"); + } + const std::size_t start = std::size_t{r.at(s + first_index)} * index_size; + const std::size_t length = triangles * 3 * index_size; + if (start + length > index_size_total) { + say(name + "'s indices end at byte " + std::to_string(start + length) + " of a " + std::to_string(index_size_total) + "-byte index buffer"); + continue; + } + const auto base = vertex_size + start; + if (base + length > geometry.size()) continue; + std::size_t outside{}, largest{}; + for (std::size_t i = 0; i < triangles * 3; ++i) { + std::size_t index{}; + std::memcpy(&index, geometry.data() + base + i * index_size, index_size); + if (index >= count) ++outside, largest = std::max(largest, index); + } + if (outside) + say(name + ": " + std::to_string(outside) + " of " + std::to_string(triangles * 3) + " indices name a vertex outside its " + + std::to_string(count) + " (the largest " + std::to_string(largest) + ")"); + } + return problems; +} + +MeshLod any_mesh_lod_geometry(std::span resource, std::size_t lod, std::size_t* lods) { + const Reader r{resource}; + (void)layout_of(r); + const std::size_t total = r.at(lod_count); + if (lods) *lods = total; + require(lod < total, "it has no such level of detail"); + const auto at = r.pointer(lod_table + lod * 8); + MeshLod result; + result.chunk = r.at(at + geometry_chunk); + result.vertex_bytes = r.at(at + vertex_bytes); + result.index_bytes = r.at(at + index_bytes); + return result; +} + SkinnedMesh read_skinned_mesh(std::span resource, std::size_t lod, std::span geometry) { const auto layout = read_mesh_lod(resource, lod); const Reader r{resource}, g{geometry}; diff --git a/Engine/Resource/mesh_set.h b/Engine/Resource/mesh_set.h index 4cc2eac0..958e2ebb 100644 --- a/Engine/Resource/mesh_set.h +++ b/Engine/Resource/mesh_set.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace dingosdk::frostbite { @@ -28,6 +29,15 @@ struct MeshLod { [[nodiscard]] std::vector> read_mesh_positions(std::span resource, std::size_t lod, std::span geometry); +// A level of detail's geometry chunk and buffer sizes, of either layout check_mesh_lod reads (the +// skinned one, and world meshes' static one); `lods` gets how many levels there are. +[[nodiscard]] MeshLod any_mesh_lod_geometry(std::span resource, std::size_t lod, std::size_t* lods = nullptr); +// What is wrong with a level of detail's geometry as the GPU would read it, a skinned or a static +// MeshSet: a section whose vertices or indices run past their buffer, or a triangle that +// names a vertex outside its section. Empty when it is sound. `index_format` gets the level's +// index format (33: 16-bit; anything else is read as 32-bit). Throws for another layout. +[[nodiscard]] std::vector check_mesh_lod(std::span resource, std::size_t lod, + std::span geometry, std::uint32_t* index_format = nullptr); struct SkinnedVertex { std::array position{}; // model space std::array bones{}; // skeleton bones diff --git a/Engine/Resource/shader_lookup.cpp b/Engine/Resource/shader_lookup.cpp index c3f8fc6f..d720445d 100644 --- a/Engine/Resource/shader_lookup.cpp +++ b/Engine/Resource/shader_lookup.cpp @@ -1,9 +1,7 @@ #include "shader_lookup.h" #include -#include #include -#include #include #include #include @@ -80,176 +78,37 @@ TextureRows read_texture_rows(const Table& table) { } // namespace -namespace { -constexpr std::size_t programStride = 168; -struct ProgramBlob { - Table source; - std::span payload; - std::uint64_t records{}; - std::uint32_t count{}; - std::vector relocations; -}; -ProgramBlob program_blob(const Table& source) { - const auto end = lookup_offset(source); - const auto size = read(source.resourceMeta, 0); - if ((end - size) % 4) throw std::runtime_error("Program relocations are not aligned"); - ProgramBlob blob{source, source.resource.first(size)}; - blob.count = read(blob.payload, 0); - blob.records = read(blob.payload, 4); - if (blob.records > size || blob.count > (size - blob.records) / programStride) - throw std::runtime_error("Program records exceed their payload"); - for (auto at = std::size_t{size}; at < end; at += 4) { - const auto pointer = read(source.resource, at); - if (read(blob.payload, pointer) > size) - throw std::runtime_error("Program relocation points outside its payload"); - blob.relocations.push_back(pointer); - } - return blob; -} -struct ProgramArray { std::size_t pointer, stride; }; -constexpr std::array programArrays{ - ProgramArray{0x10, 16}, ProgramArray{0x1c, 8}, ProgramArray{0x28, 8}, ProgramArray{0x34, 16}, - ProgramArray{0x40, 6}, ProgramArray{0x4c, 8}, ProgramArray{0x58, 2}, - ProgramArray{0x64, 8}, ProgramArray{0x70, 8}, ProgramArray{0x80, 16}, ProgramArray{0x8c, 32} -}; -std::size_t record_at(const ProgramBlob& blob, std::uint32_t index) { - if (index >= blob.count) throw std::runtime_error("Material references an absent shader program"); - return static_cast(blob.records) + index * programStride; -} -std::span program_array(const ProgramBlob& blob, std::size_t record, ProgramArray array) { - const auto count = read(blob.payload, record + array.pointer - 4); - const auto pointer = read(blob.payload, record + array.pointer); - if (pointer > blob.payload.size() || count > (blob.payload.size() - pointer) / array.stride) - throw std::runtime_error("Shader program array exceeds its payload"); - return blob.payload.subspan(static_cast(pointer), count * array.stride); -} -bool same_program(const ProgramBlob& a, std::uint32_t ai, const ProgramBlob& b, std::uint32_t bi) { - const auto aa = record_at(a, ai), ba = record_at(b, bi); - for (std::size_t at = 0; at < programStride; ++at) { - if (std::ranges::any_of(programArrays, [&](auto array) { - return at >= array.pointer && at < array.pointer + 8; })) continue; - if (a.payload[aa + at] != b.payload[ba + at]) return false; - } - for (const auto array : programArrays) - if (!std::ranges::equal(program_array(a, aa, array), program_array(b, ba, array))) return false; - return true; -} -std::uint32_t native_offset(std::size_t offset) { - if (offset > std::numeric_limits::max()) - throw std::runtime_error("Merged program database exceeds the native address range"); - return static_cast(offset); -} -} // namespace - MergedTable merge_program_lookup(const Table& base, const std::span edits) { - std::vector blobs{program_blob(base)}; - for (const auto& edit : edits) blobs.push_back(program_blob(edit)); - struct Reference { std::size_t blob; std::uint32_t program; }; - std::map> references; + const auto start = lookup_offset(base); + const auto records = read(base.resource, 0); // the base blob's record count auto rows = read_program_rows(base); - for (const auto& [key, values] : rows) for (const auto index : values) { - record_at(blobs.front(), index); - references[key].push_back({0, index}); - } const auto baseRows = rows; + MergedTable merged; - for (std::size_t b = 1; b < blobs.size(); ++b) { - for (const auto& [key, values] : read_program_rows(blobs[b].source)) { - for (const auto index : values) record_at(blobs[b], index); + for (const auto& edit : edits) { + for (const auto& [key, values] : read_program_rows(edit)) { if (baseRows.contains(key)) continue; - if (const auto previous = references.find(key); previous != references.end()) { - bool same = previous->second.size() == values.size(); - for (std::size_t i = 0; same && i < values.size(); ++i) { - const auto ref = previous->second[i]; - same = same_program(blobs[ref.blob], ref.program, blobs[b], values[i]); - } - if (!same) ++merged.conflicts; + if (std::any_of(values.begin(), values.end(), [&](std::uint32_t index) { return index >= records; })) { + ++merged.outside; continue; } - for (const auto index : values) references[key].push_back({b, index}); - ++merged.added; + const auto [row, inserted] = rows.try_emplace(key, values); + if (inserted) ++merged.added; + else if (row->second != values) ++merged.conflicts; } } - // Preserve the exact native prefix when edits only add aliases to stock programs. - // Private programs need their own records and relocated backing data as well. - using Source = std::pair; - std::map remapped; - std::vector additions; - for (const auto& [key, refs] : references) { - auto& row = rows[key]; - row.clear(); - for (const auto ref : refs) { - auto index = ref.program; - if (ref.blob && (index >= blobs.front().count || - !same_program(blobs.front(), index, blobs[ref.blob], index))) { - const Source source{ref.blob, index}; - auto [entry, inserted] = remapped.try_emplace(source, - native_offset(std::size_t{blobs.front().count} + additions.size())); - if (inserted) additions.push_back(source); - index = entry->second; - } - row.push_back(index); - } - } - merged.resourceMeta.assign(base.resourceMeta.begin(), base.resourceMeta.end()); - if (additions.empty()) { - merged.resource.assign(base.resource.begin(), base.resource.begin() + lookup_offset(base)); - } else { - merged.resource.assign(blobs.front().payload.begin(), blobs.front().payload.end()); - auto relocations = blobs.front().relocations; - std::map offsets; - for (const auto& [b, index] : additions) { - if (offsets.contains(b)) continue; - merged.resource.resize((merged.resource.size() + 15) & ~std::size_t{15}); - const auto offset = merged.resource.size(); - offsets[b] = offset; - const auto& blob = blobs[b]; - // Copy the complete backing graph, including opaque resource bindings. - // This keeps runtime-mutated GUID arrays separate across mods. - merged.resource.insert(merged.resource.end(), blob.payload.begin(), blob.payload.end()); - for (const auto pointer : blob.relocations) { - const auto destination = native_offset(offset + pointer); - const auto target = read(blob.payload, pointer) + offset; - std::memcpy(merged.resource.data() + destination, &target, 8); - relocations.push_back(destination); - } - } - merged.resource.resize((merged.resource.size() + 7) & ~std::size_t{7}); - const auto table = static_cast(merged.resource.size()); - const auto count = native_offset(std::size_t{blobs.front().count} + additions.size()); - merged.resource.resize(native_offset(static_cast(table) + std::size_t{count} * programStride)); - const auto copy_record = [&](const ProgramBlob& blob, std::uint32_t index, - std::size_t offset, std::uint32_t destination) { - const auto source = record_at(blob, index); - const auto target = static_cast(table) + destination * programStride; - std::memcpy(merged.resource.data() + target, merged.resource.data() + offset + source, programStride); - for (const auto pointer : blob.relocations) - if (pointer >= source && pointer < source + programStride) - relocations.push_back(native_offset(target + pointer - source)); - }; - for (std::uint32_t i = 0; i < blobs.front().count; ++i) copy_record(blobs.front(), i, 0, i); - for (const auto& source : additions) - copy_record(blobs[source.first], source.second, offsets.at(source.first), remapped.at(source)); - std::memcpy(merged.resource.data(), &count, 4); - std::memcpy(merged.resource.data() + 4, &table, 8); - relocations.push_back(4); - std::ranges::sort(relocations); - relocations.erase(std::ranges::unique(relocations).begin(), relocations.end()); - const auto size = native_offset(merged.resource.size()); - for (const auto pointer : relocations) append(merged.resource, pointer); - const auto relocationSize = native_offset(relocations.size() * 4); - std::memcpy(merged.resourceMeta.data(), &size, 4); - std::memcpy(merged.resourceMeta.data() + 4, &relocationSize, 4); - } - const auto lookup = merged.resource.size(); - append(merged.resource, native_offset(rows.size())); + + merged.resource.assign(base.resource.begin(), + base.resource.begin() + static_cast(start)); + append(merged.resource, static_cast(rows.size())); for (const auto& [key, values] : rows) { append(merged.resource, key); - append(merged.resource, native_offset(values.size())); + append(merged.resource, static_cast(values.size())); for (const auto value : values) append(merged.resource, value); } - const auto size = native_offset(merged.resource.size() - lookup); - std::memcpy(merged.resourceMeta.data() + 8, &size, 4); + merged.resourceMeta.assign(base.resourceMeta.begin(), base.resourceMeta.end()); + const auto size = static_cast(merged.resource.size() - start); + std::memcpy(merged.resourceMeta.data() + 8, &size, sizeof(size)); return merged; } diff --git a/Engine/Resource/shader_lookup.h b/Engine/Resource/shader_lookup.h index fe140280..50457ae3 100644 --- a/Engine/Resource/shader_lookup.h +++ b/Engine/Resource/shader_lookup.h @@ -1,8 +1,9 @@ #pragma once -// Shader lookup tables carry material rows and compiled program records. -// Mods can alias stock programs or add private programs with their own -// permutations and backing arrays; both must survive a merge. +// The two shader lookup tables a level superbundle carries. A map mod adds a +// row per material it introduces, aliasing a shader program and texture set the +// base already ships, so two mods' tables differ from the base only by the rows +// each added and can be combined exactly. #include #include @@ -24,11 +25,18 @@ struct MergedTable { std::vector resourceMeta; std::size_t added{}; // rows taken from the edits std::size_t conflicts{}; // rows two edits disagreed on; the first won + std::size_t outside{}; // rows left out: they point past the base's programs }; -// `resourceMeta` is 16 bytes: program size, relocation size, lookup size. -// Private program records and their relocated backing data are copied from -// each edit, and material indices are remapped into the combined table. +// `resourceMeta` is 16 bytes: program size, relocation size, lookup size. The +// program blob and its relocations are identical across copies because a mod +// only ever points new keys at programs that are already there. The blob starts +// with its record count (u32), and every value of a lookup row is a record's +// index: the game takes it without a bounds check (an index past the end puts +// a pointer outside the database into its shader registry, Skate.exe 0x3f72280), +// so an added row with an index the base's blob does not have is left out +// (`outside`): a mod that brought programs of its own cannot have them merged +// onto the base's. [[nodiscard]] MergedTable merge_program_lookup(const Table& base, std::span edits); [[nodiscard]] MergedTable merge_texture_lookup(const Table& base, std::span edits); diff --git a/Engine/Vfs/Test/https_download_timeout_tests.cpp b/Engine/Vfs/Test/https_download_timeout_tests.cpp new file mode 100644 index 00000000..ac4963c9 --- /dev/null +++ b/Engine/Vfs/Test/https_download_timeout_tests.cpp @@ -0,0 +1,334 @@ +#include +#include +#include "../https_download.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fake_http { +enum class Stage { none, send, receive, read, eof, close }; +struct Handle { + DWORD_PTR context{}; + WINHTTP_STATUS_CALLBACK callback{}; + DWORD notifications{}; + bool closed{}; +}; +struct State { + dingosdk::https::Watch* watch{}; + Stage stop_at{Stage::none}, fail_at{Stage::none}; + bool timeout{}, callback_error{}, content_length{true}, downgrade{}; + unsigned opens{}, reads{}, closing_callbacks{}, pending_buffer_closes{}; + DWORD redirect{}, request_flags{}, session_flags{}; + void* pending_buffer{}; + Handle session, connection, request; + std::string body{"payload"}; +}; +State state; + +void stop() { + if (state.timeout) state.watch->deadline_us.store(1); + else state.watch->cancel.store(true); +} +void notify(DWORD status, void* info = nullptr, DWORD size = 0) { + state.request.callback(&state.request, state.request.context, status, info, size); +} +bool pending(Stage stage) { + if (state.stop_at != stage) return false; + stop(); + return true; +} +bool failure(Stage stage) { + if (state.fail_at != stage) return false; + if (state.callback_error) { + WINHTTP_ASYNC_RESULT error{}; + error.dwError = ERROR_WINHTTP_CONNECTION_ERROR; + notify(WINHTTP_CALLBACK_STATUS_REQUEST_ERROR, &error, sizeof(error)); + } else { + SetLastError(ERROR_WINHTTP_CONNECTION_ERROR); + } + return true; +} +HINTERNET WINAPI open(LPCWSTR, DWORD, LPCWSTR, LPCWSTR, DWORD flags) { + ++state.opens; + state.session_flags = flags; + return &state.session; +} +HINTERNET WINAPI connect(HINTERNET, LPCWSTR, INTERNET_PORT, DWORD) { + return &state.connection; +} +HINTERNET WINAPI open_request(HINTERNET, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR*, DWORD flags) { + state.request_flags = flags; + return &state.request; +} +BOOL WINAPI set_timeouts(HINTERNET, int, int, int, int) { return TRUE; } +BOOL WINAPI set_option(HINTERNET handle, DWORD option, LPVOID value, DWORD) { + if (option == WINHTTP_OPTION_CONTEXT_VALUE) + static_cast(handle)->context = *static_cast(value); + if (option == WINHTTP_OPTION_REDIRECT_POLICY) state.redirect = *static_cast(value); + return TRUE; +} +WINHTTP_STATUS_CALLBACK WINAPI set_callback(HINTERNET handle, WINHTTP_STATUS_CALLBACK callback, + DWORD notifications, DWORD_PTR) { + auto& request = *static_cast(handle); + const auto previous = request.callback; + request.callback = callback; + request.notifications = notifications; + return previous; +} +BOOL WINAPI send(HINTERNET, LPCWSTR, DWORD, LPVOID, DWORD, DWORD, DWORD_PTR context) { + state.request.context = context; + if (pending(Stage::send)) return TRUE; + if (failure(Stage::send)) return state.callback_error ? TRUE : FALSE; + notify(WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE); + return TRUE; +} +BOOL WINAPI receive(HINTERNET, LPVOID) { + if (pending(Stage::receive)) return TRUE; + if (failure(Stage::receive)) return state.callback_error ? TRUE : FALSE; + if (state.downgrade && state.redirect == WINHTTP_OPTION_REDIRECT_POLICY_DISALLOW_HTTPS_TO_HTTP) { + SetLastError(ERROR_WINHTTP_REDIRECT_FAILED); + return FALSE; + } + notify(WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE); + return TRUE; +} +BOOL WINAPI query(HINTERNET, DWORD query, LPCWSTR, LPVOID value, LPDWORD size, LPDWORD) { + if (query == (WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER)) { + *static_cast(value) = 200; + } else if (query == (WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER) && state.content_length) { + *static_cast(value) = static_cast(state.body.size()); + } else { + SetLastError(ERROR_WINHTTP_HEADER_NOT_FOUND); + return FALSE; + } + *size = sizeof(DWORD); + return TRUE; +} +BOOL WINAPI read(HINTERNET, LPVOID buffer, DWORD capacity, LPDWORD count) { + // The production async path must receive the count through its callback. + if (count || capacity < state.body.size()) { + SetLastError(ERROR_INVALID_PARAMETER); + return FALSE; + } + if (++state.reads == 1) { + std::memcpy(buffer, state.body.data(), state.body.size()); + notify(WINHTTP_CALLBACK_STATUS_READ_COMPLETE, buffer, static_cast(state.body.size())); + } else { + if (state.stop_at == Stage::read) { + state.pending_buffer = buffer; + stop(); + return TRUE; + } + if (failure(Stage::read)) return state.callback_error ? TRUE : FALSE; + if (state.stop_at == Stage::eof) stop(); + notify(WINHTTP_CALLBACK_STATUS_READ_COMPLETE, buffer, 0); + } + return TRUE; +} +BOOL WINAPI close(HINTERNET handle) { + auto& object = *static_cast(handle); + if (object.closed) { + SetLastError(ERROR_INVALID_HANDLE); + return FALSE; + } + object.closed = true; + if (handle == &state.request && object.callback) { + if (state.pending_buffer) { + // Pending I/O can still touch the caller's buffer during closure. + *static_cast(state.pending_buffer) = 'x'; + ++state.pending_buffer_closes; + state.pending_buffer = nullptr; + } + if (state.stop_at == Stage::close) stop(); + if (object.context && (object.notifications & WINHTTP_CALLBACK_FLAG_HANDLES)) { + ++state.closing_callbacks; + notify(WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING); + } + } + return TRUE; +} +} + +#define WinHttpOpen fake_http::open +#define WinHttpConnect fake_http::connect +#define WinHttpOpenRequest fake_http::open_request +#define WinHttpSetTimeouts fake_http::set_timeouts +#define WinHttpSetOption fake_http::set_option +#define WinHttpSetStatusCallback fake_http::set_callback +#define WinHttpSendRequest fake_http::send +#define WinHttpReceiveResponse fake_http::receive +#define WinHttpQueryHeaders fake_http::query +#define WinHttpReadData fake_http::read +#define WinHttpCloseHandle fake_http::close +#include "../https_download.cpp" +#undef WinHttpOpen +#undef WinHttpConnect +#undef WinHttpOpenRequest +#undef WinHttpSetTimeouts +#undef WinHttpSetOption +#undef WinHttpSetStatusCallback +#undef WinHttpSendRequest +#undef WinHttpReceiveResponse +#undef WinHttpQueryHeaders +#undef WinHttpReadData +#undef WinHttpCloseHandle + +namespace { +int failures{}; +void check(bool condition, const char* what) { + if (condition) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} +std::string contents(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; +} +struct Fixtures { + std::filesystem::path root; + bool created; + explicit Fixtures(std::filesystem::path path) : root(std::move(path)), + created(std::filesystem::create_directory(root)) {} + ~Fixtures() { + if (created) { + std::error_code error; + std::filesystem::remove_all(root, error); + } + } +}; +void reset(dingosdk::https::Watch& watch) { + fake_http::state = {}; + fake_http::state.watch = &watch; + watch.cancel.store(false); + watch.deadline_us.store(0); + watch.received.store(0); + watch.total.store(0); +} +} + +int main(int argc, char** argv) { + using namespace dingosdk::https; + using fake_http::Stage; + Fixtures fixtures(argc > 1 ? std::filesystem::path(argv[1]) : std::filesystem::current_path() / + ("https-download-timeout-fixtures-" + std::to_string(GetCurrentProcessId()))); + if (!fixtures.created) { + std::cerr << "Fixture directory already exists; refusing to overwrite it\n"; + return 1; + } + const auto path = fixtures.root / "download.bin"; + Watch watch; + const auto get = [&](std::uint64_t limit = 1024, std::uint32_t seconds = 5) { + return dingosdk::https::get(L"https://example.invalid/file", path, limit, seconds, L"test", &watch); + }; + const auto removed = [&] { return !std::filesystem::exists(path); }; + const auto closed = [&] { + return fake_http::state.closing_callbacks == 1 && fake_http::state.request.closed && + fake_http::state.connection.closed && fake_http::state.session.closed; + }; + + reset(watch); + auto result = get(); + check(result.ok && result.http_status == 200 && result.error == 0, "actual GET succeeds"); + check(contents(path) == "payload" && watch.received.load() == 7 && watch.total.load() == 7, + "successful bytes and progress are retained"); + check(closed(), "successful request receives final closing callback"); + check(fake_http::state.session_flags & WINHTTP_FLAG_ASYNC, "session uses asynchronous WinHTTP"); + check(fake_http::state.request_flags & WINHTTP_FLAG_SECURE, "request stays HTTPS"); + check(fake_http::state.redirect == WINHTTP_OPTION_REDIRECT_POLICY_DISALLOW_HTTPS_TO_HTTP, + "HTTPS downgrade redirect policy is configured"); + std::filesystem::remove(path); + + for (const auto stage : {Stage::send, Stage::receive, Stage::read, Stage::eof, Stage::close}) { + for (const bool timeout : {false, true}) { + reset(watch); + fake_http::state.stop_at = stage; + fake_http::state.timeout = timeout; + result = get(); + check(!result.ok && result.error == static_cast(timeout ? ERROR_TIMEOUT : ERROR_CANCELLED), + "pending operation or final completion reports the authoritative stop reason"); + check(removed(), "stopped GET removes only its created output, including partial bytes"); + check(closed(), "stopped request completes handle-closing lifetime handshake"); + if (stage == Stage::read) + check(watch.received.load() == 7 && fake_http::state.pending_buffer_closes == 1, + "partial output is cleaned while pending read buffer survives close"); + } + } + + for (const auto stage : {Stage::send, Stage::receive, Stage::read}) { + for (const bool callback_error : {false, true}) { + reset(watch); + fake_http::state.fail_at = stage; + fake_http::state.callback_error = callback_error; + result = get(); + check(!result.ok && result.error == ERROR_WINHTTP_CONNECTION_ERROR, + "immediate and callback API errors are reported"); + check(removed() && closed(), "API errors clean up output and close registered request"); + } + } + + reset(watch); + watch.deadline_us.store(1); + result = get(); + check(!result.ok && result.error == ERROR_TIMEOUT && fake_http::state.opens == 0 && removed(), + "already expired external deadline prevents network and output creation"); + reset(watch); + watch.deadline_us.store(static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()) + 60000000); + result = get(1024, 0); + check(!result.ok && result.error == ERROR_TIMEOUT && fake_http::state.opens == 0, + "internal timeout wins over a later external steady-clock deadline"); + reset(watch); + watch.cancel.store(true); + watch.deadline_us.store(1); + result = get(); + check(!result.ok && result.error == ERROR_CANCELLED && fake_http::state.opens == 0, + "explicit cancellation wins when both stop conditions hold"); + + reset(watch); + { + std::ofstream output(path, std::ios::binary); + output << "existing destination"; + } + result = get(); + check(!result.ok && contents(path) == "existing destination", + "CREATE_NEW failure preserves a pre-existing destination"); + check(closed(), "existing-destination failure still closes the request"); + std::filesystem::remove(path); + + for (const bool content_length : {false, true}) { + reset(watch); + fake_http::state.content_length = content_length; + result = get(3); + check(!result.ok && result.error == ERROR_FILE_TOO_LARGE && removed() && closed(), + "announced and streamed oversized bodies remain rejected and cleaned up"); + } + reset(watch); + fake_http::state.downgrade = true; + result = get(); + check(!result.ok && result.error == ERROR_WINHTTP_REDIRECT_FAILED && removed() && closed(), + "HTTPS-to-HTTP redirects remain refused"); + reset(watch); + fake_http::state.body.clear(); + result = get(); + check(!result.ok && result.error == ERROR_BAD_LENGTH && removed() && closed(), + "empty body remains rejected and its created output is removed"); + + reset(watch); + fake_http::state.stop_at = Stage::send; + const auto text = get_text(L"https://example.invalid/metadata", 1024, 5, L"test", &result, &watch); + check(!text && !result.ok && result.error == ERROR_CANCELLED && closed(), + "get_text forwards its Watch into the actual asynchronous GET"); + + if (failures) return 1; + std::cout << "HTTPS deadline/cancellation tests passed\n"; + return 0; +} diff --git a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp index 2a450d39..f52c1ec3 100644 --- a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp +++ b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp @@ -14,15 +14,20 @@ #include "Engine/Resource/cas_codec.h" #include "Engine/Resource/ebx_document.h" #include "Engine/Resource/toc.h" +#include "Engine/Vfs/game_archives.h" #include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/mod_merge_internal.h" #include "Engine/Vfs/native_db.h" #include +#include #include #include #include #include +#include +#include #include #include @@ -291,6 +296,8 @@ std::string describe(const mods::MergeReport& report) { constexpr char shared_toc[] = "test_shared.toc"; constexpr char map_toc[] = "levels/test/map.toc"; constexpr char map_superbundle[] = "Win32/levels/test/map"; +constexpr char second_map_toc[] = "levels/test/map2.toc"; +constexpr char second_map_superbundle[] = "Win32/levels/test/map2"; // The level mod alone: what its copy of the bundle holds with nothing carried into it. int map_only() { @@ -395,6 +402,27 @@ void two_music_mods_both_carried_into_maps_copy(int baseline) { std::to_string(files) + " vs " + std::to_string(baseline + 6)); } +// Two mods edit the playlist and two others carry the game's copy of the bundle untouched. Only +// the copies equal to the game's are left out for combining: every test/other, but neither edit +// of the playlist (the carriers' playlists take the edits, so they are not the game's either). +void copies_equal_to_the_games_are_not_edits() { + Fixture fixture("untouched-copies"); + const std::vector music1{{"test/playlist", 1, ebx_document(guid(1), {guid(10)})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song1", 0, ebx_document(guid(10), {})}}; + const std::vector music2{{"test/playlist", 2, ebx_document(guid(1), {guid(20)})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song2", 0, ebx_document(guid(20), {})}}; + fixture.add("carrier1", false, shared_toc, {}, game_copy()); + fixture.add("music1", false, shared_toc, {}, music1); + fixture.add("music2", false, shared_toc, {}, music2); + fixture.add("carrier2", false, shared_toc, {}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "untouched copies: the merge builds the patch\n" + describe(report)); + expect(report.unchangedCopies == 4, "untouched copies: " + std::to_string(report.unchangedCopies) + + " copies equal to the game's were left out, not 4"); +} + // The game keeps the playlist in a second bundle too, as it keeps the music playlist in eight. A // mod that ships its own copy of that one (it changed something else in it) takes the music mod's // playlist there. The song, its wave and the wave's resource were added beside the playlist in @@ -453,6 +481,177 @@ void replaced_while_the_game_runs() { report = mods::merge_mods(fixture.catalog, {}, {.live = true}); expect(report.built && report.issue.empty() && !noted("placed again"), "replaced: placed again without a change\n" + describe(report)); } +// The merged patch, read the way the merge reads it. +struct Patch { + explicit Patch(const Fixture& fixture) + : root(fixture.catalog.root / mods::generated_folder), base(fixture.catalog.data_root / "Data"), + game(fixture.catalog.data_root), layout(vfs::read_layout(root / "layout.toc")), store(base, root, layout.root) {} + // The bytes a placement names; empty, with a FAIL, when they are not on disk. + Bytes bytes(const fb::CasIdentifier& location, std::uint32_t offset, std::uint32_t size, const std::string& what) { + try { + const auto& from = location.patch ? root : base; + const auto length = store.archive_size(from, location); + expect(length && std::uint64_t{offset} + size <= *length, + what + " points at byte " + std::to_string(offset) + " of " + store.describe(location) + + (length ? ", past its end" : ", which is not there")); + if (!length || std::uint64_t{offset} + size > *length) return {}; + return store.read(from, location, offset, size); + } catch (const std::exception& error) { + expect(false, what + ": " + error.what()); + return {}; + } + } + // Where the merged copy of the bundle in `toc` keeps `asset`. + std::optional file(const std::string& toc, const std::string& asset) { + const auto path = root / "Win32" / fs::path(toc); + if (!fs::exists(path)) return std::nullopt; + for (const auto& bundle : fb::read_toc(read(path)).bundles) { + if (bundle.name != bundle_name) continue; + const auto listing = mods::detail::list_bundle(store, root, base, bundle, game); + if (!listing) return std::nullopt; + for (std::size_t index = 0; index < listing->manifest.ebx.size(); ++index) + if (listing->manifest.ebx[index].name == asset) return listing->files.at(listing->first + index); + } + return std::nullopt; + } + fs::path root, base, game; + vfs::Layout layout; + mods::detail::CasStore store; +}; +// How many times `part` is in `whole`. +std::size_t occurrences(const Bytes& whole, const Bytes& part) { + std::size_t count{}; + for (auto at = whole.begin(); (at = std::search(at, whole.end(), part.begin(), part.end())) != whole.end(); ++at) ++count; + return count; +} + +// Two maps each carry the music mod's song. Both copies read the one payload. +void carried_payload_written_once() { + Fixture fixture("carried-once"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "carried once: the merge builds the patch\n" + describe(report)); + if (!report.built) return; + Patch patch(fixture); + const auto first = patch.file(map_toc, "test/song"); + const auto second = patch.file(second_map_toc, "test/song"); + expect(first && second, "carried once: both maps' copies hold test/song"); + if (!first || !second) return; + expect(first->location == second->location && first->offset == second->offset, + "carried once: the two copies read different payloads (byte " + std::to_string(first->offset) + " and byte " + + std::to_string(second->offset) + ")"); + const auto song = encoded(ebx_document(guid(10), {guid(11)})); + expect(patch.bytes(first->location, first->offset, first->size, "carried once: test/song") == song, + "carried once: the placement does not hold the song's bytes"); + const auto size = patch.store.archive_size(patch.root, first->location); + const auto archive = size ? patch.bytes(first->location, 0, static_cast(*size), "carried once: its archive") : Bytes{}; + expect(occurrences(archive, song) == 1, + "carried once: the song is in its archive " + std::to_string(occurrences(archive, song)) + " times"); +} + +// Every placement in every merged TOC lies inside an archive that exists, and the merged layout +// declares only real archives. +void every_placement_lands_in_an_archive() { + Fixture fixture("placements"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + // The song's audio as a TOC chunk of the music mod's own, named by the song (its import): + // it follows the song into the maps' TOCs. + const Bytes wave{std::byte{7}, std::byte{8}, std::byte{9}, std::byte{10}}; + { + const auto toc = fixture.catalog.root / "music" / "Win32" / shared_toc; + const auto archive = fixture.catalog.root / "music" / "Win32" / "pkg" / "cas_01.cas"; + auto document = fb::read_toc(read(toc)); + auto bytes = read(archive); + const auto location = fb::read_bundle_region(document.bundles.front().region).files.front().location; + document.chunks.push_back({guid(11), location, static_cast(bytes.size()), static_cast(wave.size())}); + bytes.insert(bytes.end(), wave.begin(), wave.end()); + write(archive, bytes); + write(toc, fb::write_patch_toc(document.bundles, document.chunks, document.flags)); + } + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "placements: the merge builds the patch\n" + describe(report)); + if (!report.built) return; + Patch patch(fixture); + std::size_t checked{}; + std::set withAudio; + for (const auto& entry : fs::recursive_directory_iterator(patch.root / "Win32")) { + if (entry.path().extension() != ".toc") continue; + const auto document = fb::read_toc(read(entry.path())); + for (const auto& bundle : document.bundles) + for (const auto& file : fb::read_bundle_region(bundle.region).files) { + (void)patch.bytes(file.location, file.offset, file.size, "placements: " + bundle.name); + ++checked; + } + for (const auto& chunk : document.chunks) { + const auto held = patch.bytes(chunk.location, chunk.offset, chunk.size, "placements: chunk " + chunk.guid.string()); + if (chunk.guid != guid(11)) continue; + const auto name = fs::relative(entry.path(), patch.root / "Win32").generic_string(); + expect(held == wave, "placements: " + name + "'s chunk does not hold the song's audio"); + if (held == wave) withAudio.insert(name); + } + } + expect(checked > 0, "placements: the merged TOCs hold placements"); + expect(withAudio.contains(map_toc) && withAudio.contains(second_map_toc), "placements: both maps' TOCs carry the song's audio chunk"); + const auto* table = patch.layout.root.field("layeredInstallChunkFiles"); + expect(table != nullptr, "placements: the merged layout declares archives"); + if (table) + vfs::for_each_install_chunk_file(table->payload(), [&](std::uint32_t, std::uint16_t archive) { + expect(!mods::detail::CasStore::placeholder(archive), "placements: the layout declares placeholder archive " + std::to_string(archive)); + }); +} + +// A mod archive numbered like a placeholder takes a free number: its placements land on its bytes, +// and the layout never declares the placeholder. +void a_mod_archive_numbered_like_a_placeholder() { + Fixture fixture("placeholder-number"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + const auto mod = fixture.catalog.root / "map"; + const auto toc = mod / "Win32" / map_toc; + auto document = fb::read_toc(read(toc)); + for (auto& bundle : document.bundles) { + auto region = fb::read_bundle_region(bundle.region); + for (auto& file : region.files) file.location.archive = mods::detail::CasStore::carried_archive; + bundle.region = fb::write_bundle_region(region.files, region.inlineManifest); + } + write(toc, fb::write_patch_toc(document.bundles, document.chunks, document.flags)); + fs::rename(mod / "Win32" / "pkg" / "cas_01.cas", mod / "Win32" / "pkg" / vfs::archive_file(mods::detail::CasStore::carried_archive)); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "placeholder number: the merge builds the patch\n" + describe(report)); + if (!report.built) return; + Patch patch(fixture); + const auto other = patch.file(map_toc, "test/other"); + expect(other && !mods::detail::CasStore::placeholder(other->location) && + patch.bytes(other->location, other->offset, other->size, "placeholder number: test/other") == + encoded(ebx_document(guid(2), {})), + "placeholder number: the map's own asset does not read its bytes"); + if (const auto* table = patch.layout.root.field("layeredInstallChunkFiles")) + vfs::for_each_install_chunk_file(table->payload(), [&](std::uint32_t, std::uint16_t archive) { + expect(!mods::detail::CasStore::placeholder(archive), "placeholder number: the layout declares archive " + std::to_string(archive)); + }); +} + +// A merge built again from the same mods writes the same bytes. +void two_merges_write_the_same_bytes() { + Fixture fixture("same-bytes"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto patch = fixture.catalog.root / mods::generated_folder; + auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "same bytes: the first merge\n" + describe(report)); + const auto archive = read(patch / "Win32" / "pkg" / "cas_01.cas"); + const auto toc = read(patch / "Win32" / fs::path(second_map_toc)); + fs::remove(patch / mods::detail::stamp_file); + report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built && !report.reused, "same bytes: the second merge\n" + describe(report)); + expect(read(patch / "Win32" / "pkg" / "cas_01.cas") == archive, "same bytes: the patch's own archive differs"); + expect(read(patch / "Win32" / fs::path(second_map_toc)) == toc, "same bytes: the second map's TOC differs"); +} } // namespace int main() try { @@ -465,8 +664,13 @@ int main() try { same_name_from_two_mods(true); same_name_from_two_mods(false); two_music_mods_both_carried_into_maps_copy(baseline); + copies_equal_to_the_games_are_not_edits(); carried_into_another_bundle_with_the_list(true); carried_into_another_bundle_with_the_list(false); + carried_payload_written_once(); + every_placement_lands_in_an_archive(); + a_mod_archive_numbered_like_a_placeholder(); + two_merges_write_the_same_bytes(); if (failures) { std::cerr << failures << " failure(s)\n"; return 1; diff --git a/Engine/Vfs/Test/mod_mesh_check.cpp b/Engine/Vfs/Test/mod_mesh_check.cpp new file mode 100644 index 00000000..0e547e25 --- /dev/null +++ b/Engine/Vfs/Test/mod_mesh_check.cpp @@ -0,0 +1,191 @@ +// The meshes a mod ships, checked as the GPU would read them (frostbite::check_mesh_lod): a +// section whose vertices or indices run past their buffer, or a triangle naming a vertex outside +// its section, sends a draw, or a ray traced against the mesh, to memory that is not the mesh's. +// For finding what in a map hangs or faults the GPU when it loads. +// +// Arguments: []. Every MeshSet the mod's own files +// hold is read, each level of detail checked; what is wrong is listed by bundle and mesh, then +// how many meshes each bundle has and how many are wrong. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/mesh_set.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_merge_internal.h" +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +using namespace dingosdk::mods::detail; + +namespace { +constexpr std::uint32_t mesh_set_type = 0x49b156d4; + +struct Tally { + std::size_t meshes{}, wrong{}, unreadable{}; +}; +} + +int wmain(int argc, wchar_t** argv) { + if (argc < 3) { + std::cerr << "Usage: dingosdk_mod_mesh_check []\n"; + return 2; + } + const fs::path game_root = argv[1], mod = argv[2]; + std::string filter; + if (argc > 3) for (const wchar_t* c = argv[3]; *c; ++c) filter.push_back(static_cast(*c)); + const auto base_root = game_root / L"Data"; + try { + const auto layout = dingosdk::vfs::read_layout(base_root / L"layout.toc"); + // (Nothing is written: the store is only read through.) + const auto scratch = fs::temp_directory_path() / L"reskate-mesh-check"; + CasStore store(base_root, scratch, layout.root); + std::map tallies; + std::map formats; + for (const auto& relative : scan(mod).tocs) { + fb::TocDocument own; + try { + own = fb::read_toc(read_file(mod / fs::path(relative))); + } catch (const std::exception& error) { + std::cout << "unreadable toc " << fs::path(relative).string() << ": " << error.what() << "\n"; + continue; + } + for (const auto& bundle : own.bundles) { + if (!filter.empty() && bundle.name.find(filter) == std::string::npos) continue; + std::optional listing; + try { + listing = list_bundle(store, mod, base_root, bundle, game_root); + } catch (const std::exception& error) { + std::cout << "unreadable bundle " << bundle.name << ": " << error.what() << "\n"; + continue; + } + if (!listing) continue; + const auto& manifest = listing->manifest; + auto& tally = tallies[bundle.name]; + for (std::size_t index = 0; index < manifest.resources.size(); ++index) { + const auto file = manifest.ebx.size() + index; + const auto at = listing->first + file; + if (manifest.resources[index].resourceType != mesh_set_type || at >= listing->files.size() || + !listing->files[at].location.patch) + continue; // (the game's own meshes are not the mod's to check) + const auto& name = manifest.resources[index].name; + ++tally.meshes; + std::vector resource; + std::size_t lods{}; + try { + resource = read_asset(store, mod, base_root, *listing, file, game_root); + (void)fb::any_mesh_lod_geometry(resource, 0, &lods); + } catch (const std::exception& error) { + ++tally.unreadable; + // (Each layout this reader does not know, once: its first words and size.) + static std::map shown; + std::string head; + for (std::size_t word = 0; word < 16 && (word + 1) * 4 <= resource.size(); ++word) { + std::uint32_t value{}; + std::memcpy(&value, resource.data() + word * 4, 4); + head += std::to_string(value) + " "; + } + if (!shown[head.substr(0, 48)]) { + shown[head.substr(0, 48)] = true; + std::cout << "layout " << head << "(" << resource.size() << " bytes) e.g. " << name << ": " << error.what() << "\n"; + // (RESKATE_MESH_DUMP=1: the first one whole, eight bytes a line, for working out its layout.) + static bool dumped{}; + if (!dumped && GetEnvironmentVariableW(L"RESKATE_MESH_DUMP", nullptr, 0)) { + dumped = true; + for (std::size_t offset = 0; offset + 8 <= resource.size(); offset += 8) { + std::uint32_t low{}, high{}; + std::memcpy(&low, resource.data() + offset, 4); + std::memcpy(&high, resource.data() + offset + 4, 4); + std::cout << " +" << at << ": " << low << " " << high << "\n"; + } + } + } + continue; + } + // (RESKATE_MESH_DUMP_NAME=: that mesh's resource, offset and 32-bit word a line.) + if (wchar_t wanted_name[256]{}; GetEnvironmentVariableW(L"RESKATE_MESH_DUMP_NAME", wanted_name, 256)) { + std::string part; + for (const wchar_t* c = wanted_name; *c; ++c) part.push_back(static_cast(*c)); + if (name.find(part) != std::string::npos) + for (std::size_t offset = 0; offset + 4 <= resource.size(); offset += 4) { + std::uint32_t value{}; + std::memcpy(&value, resource.data() + offset, 4); + std::cout << "dump " << offset << " " << value << "\n"; + } + } + bool wrong{}; + for (std::size_t lod = 0; lod < lods; ++lod) { + try { + const auto wanted = fb::any_mesh_lod_geometry(resource, lod).chunk; + std::vector geometry; + for (std::size_t chunk = 0; chunk < manifest.chunks.size() && geometry.empty(); ++chunk) + if (manifest.chunks[chunk].guid == wanted) + geometry = read_asset(store, mod, base_root, *listing, + manifest.ebx.size() + manifest.resources.size() + chunk, game_root); + for (auto chunk = own.chunks.begin(); chunk != own.chunks.end() && geometry.empty(); ++chunk) + if (chunk->guid == wanted && !chunk->removed) + geometry = fb::decode_cas(store.read(chunk->location.patch ? mod : base_root, chunk->location, + chunk->offset, chunk->size), {game_root}); + if (geometry.empty()) { + std::cout << bundle.name << " | " << name << " | lod " << lod << " | its geometry chunk is in neither the bundle nor its superbundle\n"; + wrong = true; + continue; + } + std::uint32_t format{}; + const auto problems = fb::check_mesh_lod(resource, lod, geometry, &format); + // (RESKATE_MESH_STATS=1: each level's size, for comparing a map's meshes with another's: + // sections, buffer bytes, and the most vertices and triangles a section has.) + if (GetEnvironmentVariableW(L"RESKATE_MESH_STATS", nullptr, 0)) { + const auto word = [&](std::size_t offset) { + std::uint64_t value{}; + std::memcpy(&value, resource.data() + offset, 8); + return value; + }; + const auto lod_at = static_cast(word(48 + lod * 8)) + 16; + const auto sections = static_cast(word(lod_at + 8)); + const auto table = static_cast(word(lod_at + 12)) + 16; + const auto sizes = fb::any_mesh_lod_geometry(resource, lod); + // (Static meshes' sections are 384 or 416 bytes; the larger section counts read by either.) + std::uint64_t most_vertices{}, most_triangles{}; + for (std::uint32_t section = 0; section < sections; ++section) + for (const std::size_t stride : {384u, 416u}) { + const auto at_section = table + section * stride; + if (at_section + 48 > resource.size()) continue; + std::uint32_t triangles{}, vertices{}; + std::memcpy(&triangles, resource.data() + at_section + 32, 4); + std::memcpy(&vertices, resource.data() + at_section + 44, 4); + if (vertices <= sizes.vertex_bytes && triangles * 6ull <= sizes.index_bytes + 6ull) { + most_vertices = std::max(most_vertices, vertices); + most_triangles = std::max(most_triangles, triangles); + } + } + std::cout << "stats " << bundle.name << " " << name << " " << lod << " " << sections << " " << sizes.vertex_bytes + << " " << sizes.index_bytes << " " << most_vertices << " " << most_triangles << "\n"; + } + ++formats[format]; + for (const auto& problem : problems) std::cout << bundle.name << " | " << name << " | lod " << lod << " | " << problem << "\n"; + wrong = wrong || !problems.empty(); + } catch (const std::exception& error) { + std::cout << bundle.name << " | " << name << " | lod " << lod << " | unreadable: " << error.what() << "\n"; + } + } + tally.wrong += wrong; + } + } + } + std::cout << "\nBundle | meshes | wrong | unreadable\n"; + for (const auto& [name, tally] : tallies) + if (tally.meshes) std::cout << name << " | " << tally.meshes << " | " << tally.wrong << " | " << tally.unreadable << "\n"; + std::cout << "Index formats seen:"; + for (const auto& [format, count] : formats) std::cout << " " << format << " (" << count << " levels)"; + std::cout << "\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << "\n"; + return 1; + } + return 0; +} diff --git a/Engine/Vfs/builtin_pack_format.h b/Engine/Vfs/builtin_pack_format.h new file mode 100644 index 00000000..004544fd --- /dev/null +++ b/Engine/Vfs/builtin_pack_format.h @@ -0,0 +1,41 @@ +#pragma once +// The built-in pack format (Engine/Vfs/builtin_packs.h), as BuiltinPacker writes it at build time +// from a zip in assets/builtin and ReSkate unpacks it: one zstd frame (with its content size and a +// checksum) holding +// "RSKPACK1", u32 file count, +// for each file: u16 name length, the name (UTF-8, '/' between folders), u64 size, +// then every file's bytes in the same order. +// Numbers little-endian. +#include +#include +#include +#include + +namespace dingosdk::mods::builtin_pack { +inline constexpr std::array magic{'R', 'S', 'K', 'P', 'A', 'C', 'K', '1'}; +// The most a pack may unpack to, and the largest zstd window it may use (zstd's own default +// limit for decoding, 128 MiB). +inline constexpr std::uint64_t max_bytes = 512ull * 1024 * 1024; +inline constexpr int max_window_log = 27; + +// A path inside a pack: relative, forward slashes, no "..", plain characters (and spaces inside a +// name, as in a sound's "Body Impact 1.wav", but never at either end of one). +inline bool safe_name(std::string_view name) { + if (name.empty() || name.size() > 260 || name.front() == '/' || name.find(':') != std::string_view::npos || + name.find('\\') != std::string_view::npos) + return false; + std::size_t start = 0; + while (start <= name.size()) { + const auto end = std::min(name.find('/', start), name.size()); + const auto part = name.substr(start, end - start); + if (part.empty() || part == "." || part == ".." || part.front() == ' ' || part.back() == ' ') return false; + if (!std::all_of(part.begin(), part.end(), [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || + c == '-' || c == ' '; + })) + return false; + start = end + 1; + } + return true; +} +} diff --git a/Engine/Vfs/builtin_packs.cpp b/Engine/Vfs/builtin_packs.cpp new file mode 100644 index 00000000..52af0071 --- /dev/null +++ b/Engine/Vfs/builtin_packs.cpp @@ -0,0 +1,310 @@ +#include "builtin_packs.h" +#include "builtin_pack_format.h" +#include "Engine/Core/Platform/path_text.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::mods { +namespace { +namespace fs = std::filesystem; +// Each pack: its RCDATA resource in ReSkate.dll (cmake/Runtime.cmake) and its folder name. +struct Embedded { + const wchar_t* resource; + const char* name; +}; +constexpr Embedded embedded[]{{L"BUILTIN_PACK_FPV_DRONE", "fpv_drone"}}; +// Retired mods (builtin_packs.h), in small letters. +constexpr std::string_view retired[]{"fourth_studio-board_break"}; +std::string lower_text(std::string_view text) { + std::string out(text); + std::transform(out.begin(), out.end(), out.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return out; +} +constexpr std::uint64_t max_pack_bytes = builtin_pack::max_bytes; +constexpr std::string_view version_file = ".builtin-version"; + +struct Source { + HMODULE module{}; + bool owned{}; + Source() = default; + Source(const Source&) = delete; + Source& operator=(const Source&) = delete; + ~Source() { + if (owned && module) FreeLibrary(module); + } +}; +bool carries_packs(HMODULE module) { + return module && std::any_of(std::begin(embedded), std::end(embedded), [&](const Embedded& pack) { + return FindResourceW(module, pack.resource, MAKEINTRESOURCEW(10) /* RT_RCDATA */) != nullptr; + }); +} +// ReSkate.dll: this module when it is the DLL (the game), else ReSkate.dll beside the program +// (the launcher, which sits beside Skate.exe and the DLL), as data only. +void open_source(Source& source) { + HMODULE self{}; + if (GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&carries_packs), &self) && carries_packs(self)) { + source.module = self; + return; + } + if (const auto loaded = GetModuleHandleW(L"ReSkate.dll"); carries_packs(loaded)) { + source.module = loaded; + return; + } + std::wstring program(32768, L'\0'); + const auto length = GetModuleFileNameW(nullptr, program.data(), static_cast(program.size())); + if (!length || length >= program.size()) return; + program.resize(length); + const auto dll = fs::path(program).parent_path() / L"ReSkate.dll"; + source.module = LoadLibraryExW(dll.c_str(), nullptr, LOAD_LIBRARY_AS_DATAFILE | LOAD_LIBRARY_AS_IMAGE_RESOURCE); + source.owned = source.module != nullptr; +} +std::span resource_bytes(HMODULE module, const wchar_t* name) { + const auto resource = FindResourceW(module, name, MAKEINTRESOURCEW(10)); + if (!resource) return {}; + const auto loaded = LoadResource(module, resource); + const auto size = SizeofResource(module, resource); + const auto* data = loaded ? static_cast(LockResource(loaded)) : nullptr; + return data && size ? std::span(data, size) : std::span{}; +} +// Which build of a pack this is: enough to tell when the one embedded changed. +std::string version_of(std::span bytes) { + std::uint64_t hash = 0xcbf29ce484222325ULL; + for (const auto byte : bytes) hash = (hash ^ byte) * 0x100000001b3ULL; + return std::format("{:016x}-{}", hash, bytes.size()); +} +std::string read_text(const fs::path& path) { + std::ifstream input(path, std::ios::binary); + return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; +} +bool safe_entry(std::string_view name) { return builtin_pack::safe_name(name); } +// Writes `size` bytes to a new file at `path` (its folders made first). +bool write_file(const fs::path& path, const unsigned char* data, std::uint64_t size) { + std::error_code error; + fs::create_directories(path.parent_path(), error); + struct File { + HANDLE value; + ~File() { if (value != INVALID_HANDLE_VALUE) CloseHandle(value); } + } file{CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)}; + if (file.value == INVALID_HANDLE_VALUE) return false; + while (size) { + const auto chunk = static_cast(std::min(size, 64ull * 1024 * 1024)); + DWORD done{}; + if (!WriteFile(file.value, data, chunk, &done, nullptr) || done != chunk) return false; + data += chunk; + size -= chunk; + } + return true; +} +// A pack in ReSkate's own format (builtin_pack_format.h): one zstd frame, unpacked whole. +bool unpack_packed(std::span bytes, const fs::path& target, std::string& issue) { + const auto size = ZSTD_getFrameContentSize(bytes.data(), bytes.size()); + if (size == ZSTD_CONTENTSIZE_ERROR || size == ZSTD_CONTENTSIZE_UNKNOWN || size > max_pack_bytes + 16ull * 1024 * 1024) { + issue = "the embedded pack is not one ReSkate made"; + return false; + } + std::vector plain(static_cast(size)); + const auto unpacked = ZSTD_decompress(plain.data(), plain.size(), bytes.data(), bytes.size()); + if (ZSTD_isError(unpacked) || unpacked != plain.size()) { + issue = std::string("the embedded pack could not be decompressed: ") + (ZSTD_isError(unpacked) ? ZSTD_getErrorName(unpacked) : "short"); + return false; + } + std::size_t at = 0; + const auto take = [&](void* out, std::size_t count) { + if (count > plain.size() - at) return false; + std::memcpy(out, plain.data() + at, count); + at += count; + return true; + }; + std::array magic{}; + std::uint32_t count{}; + if (!take(magic.data(), magic.size()) || magic != builtin_pack::magic || !take(&count, sizeof(count)) || count > 100000) { + issue = "the embedded pack is not one ReSkate made"; + return false; + } + std::vector> files(count); + std::uint64_t total{}; + for (auto& [name, length] : files) { + std::uint16_t name_length{}; + if (!take(&name_length, sizeof(name_length))) { issue = "the embedded pack's list is cut short"; return false; } + name.resize(name_length); + if (!take(name.data(), name.size()) || !take(&length, sizeof(length))) { issue = "the embedded pack's list is cut short"; return false; } + total += length; + if (!safe_entry(name) || total > max_pack_bytes) { + issue = "the embedded pack has an entry it may not: " + name; + return false; + } + } + for (const auto& [name, length] : files) { + if (length > plain.size() - at) { issue = "the embedded pack is cut short at " + name; return false; } + if (!write_file(target / fs::path(name), plain.data() + at, length)) { issue = "could not unpack " + name; return false; } + at += static_cast(length); + } + if (at != plain.size()) { + issue = "the embedded pack has bytes past its last file"; + return false; + } + return true; +} +bool unpack(std::span bytes, const fs::path& target, std::string& issue) { + // ReSkate's own format (a zstd frame); else a plain zip, as a build without the packer embeds. + if (bytes.size() >= 4 && bytes[0] == 0x28 && bytes[1] == 0xb5 && bytes[2] == 0x2f && bytes[3] == 0xfd) + return unpack_packed(bytes, target, issue); + struct Zip { + mz_zip_archive value{}; + ~Zip() { mz_zip_reader_end(&value); } + } zip; + if (!mz_zip_reader_init_mem(&zip.value, bytes.data(), bytes.size(), 0)) { + issue = "the embedded pack is not a zip"; + return false; + } + const auto count = mz_zip_reader_get_num_files(&zip.value); + std::uint64_t total{}; + for (mz_uint index = 0; index < count; ++index) { + mz_zip_archive_file_stat info{}; + if (!mz_zip_reader_file_stat(&zip.value, index, &info) || info.m_is_encrypted) { + issue = "the embedded pack has an unreadable entry"; + return false; + } + if (info.m_is_directory) continue; + total += info.m_uncomp_size; + if (!safe_entry(info.m_filename) || total > max_pack_bytes) { + issue = std::string("the embedded pack has an entry it may not: ") + info.m_filename; + return false; + } + const auto path = target / fs::path(info.m_filename); + fs::create_directories(path.parent_path()); + struct File { + HANDLE value; + ~File() { if (value != INVALID_HANDLE_VALUE) CloseHandle(value); } + } file{CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)}; + struct Sink { + HANDLE file; + std::uint64_t written{}, limit; + } sink{file.value, 0, info.m_uncomp_size}; + const auto write = [](void* opaque, mz_uint64 offset, const void* buffer, size_t size) -> size_t { + auto& target = *static_cast(opaque); + if (offset != target.written || size > target.limit - target.written) return 0; + DWORD done{}; + if (!WriteFile(target.file, buffer, static_cast(size), &done, nullptr) || done != size) return 0; + target.written += size; + return size; + }; + if (file.value == INVALID_HANDLE_VALUE || !mz_zip_reader_extract_to_callback(&zip.value, index, write, &sink, 0) || + sink.written != info.m_uncomp_size) { + issue = std::string("could not unpack ") + info.m_filename; + return false; + } + } + return true; +} +} // namespace + +bool unpack_builtin_pack(std::span bytes, const fs::path& target, std::string& issue) noexcept { + try { + return unpack(bytes, target, issue); + } catch (const std::exception& failure) { + issue = failure.what(); + return false; + } +} + +std::vector official_packs(const fs::path& mods_root) noexcept { + std::vector packs; + try { + std::error_code error; + for (fs::directory_iterator at(mods_root / std::string(official_folder), error), end; !error && at != end; at.increment(error)) { + if (!at->is_directory(error) || !fs::is_regular_file(at->path() / std::string(official_stamp), error)) continue; + const auto wide = at->path().filename().wstring(); + if (wide.empty() || wide.front() == L'.' || wide.find(L".new") != std::wstring::npos) continue; + std::string name; + for (const auto c : wide) name.push_back(c < 0x80 ? static_cast(c) : '_'); + packs.push_back({std::move(name), at->path()}); + } + std::sort(packs.begin(), packs.end(), [](const BuiltinPack& a, const BuiltinPack& b) { return a.name < b.name; }); + } catch (...) {} + return packs; +} + +bool retired_mod(std::string_view folder) noexcept { + try { + return std::find(std::begin(retired), std::end(retired), lower_text(folder)) != std::end(retired); + } catch (...) { + return false; + } +} + +bool kept_by_reskate(const fs::path& mods_root, std::string_view folder) noexcept { + try { + if (folder.empty() || retired_mod(folder)) return !folder.empty(); + std::error_code error; + return fs::is_regular_file(mods_root / std::string(official_folder) / std::string(folder) / std::string(official_stamp), error); + } catch (...) { + return false; + } +} + +std::vector unpack_builtin_packs(const fs::path& mods_root, std::vector& notes) noexcept { + std::vector packs; + try { + Source source; + open_source(source); + for (const auto& pack : embedded) { + if (!source.module) break; + const auto bytes = resource_bytes(source.module, pack.resource); + if (bytes.empty()) continue; + const auto directory = mods_root / std::string(builtin_folder) / pack.name; + const auto version = version_of(bytes); + std::error_code error; + if (read_text(directory / version_file) != version) { + // Unpacked beside it, then swapped in: a pack is never half there. + const auto staging = fs::path(directory.wstring() + std::format(L".unpacking-{}", GetCurrentProcessId())); + fs::remove_all(staging, error); + std::string issue; + if (!unpack(bytes, staging, issue)) { + fs::remove_all(staging, error); + notes.push_back(std::format("Built-in content {} could not be unpacked: {}", pack.name, issue)); + continue; + } + std::ofstream(staging / version_file, std::ios::binary) << version; + fs::remove_all(directory, error); + fs::rename(staging, directory, error); + if (error) { + fs::remove_all(staging, error); + notes.push_back(std::format("Built-in content {} could not be put in place ({})", pack.name, error.message())); + continue; + } + notes.push_back(std::format("Built-in content {} unpacked ({} bytes)", pack.name, bytes.size())); + } + packs.push_back({std::string("reskate-builtin-") + pack.name, directory}); + } + // What an older ReSkate carried and this one does not (it became an official mod, or was + // dropped) goes: the Board Break deck list reads every folder in here. + std::error_code error; + const auto folder = mods_root / std::string(builtin_folder); + std::vector stale; + for (fs::directory_iterator at(folder, error), end; !error && at != end; at.increment(error)) + if (std::none_of(packs.begin(), packs.end(), [&](const BuiltinPack& pack) { return pack.directory == at->path(); })) + stale.push_back(at->path()); + for (const auto& path : stale) { + fs::remove_all(path, error); + if (!error) notes.push_back("Built-in content " + path_utf8(path.filename()) + " is no longer carried; removed"); + } + if (packs.empty()) fs::remove(folder, error); + } catch (const std::exception& failure) { + notes.push_back(std::string("Built-in content could not be prepared: ") + failure.what()); + } + return packs; +} +} // namespace dingosdk::mods diff --git a/Engine/Vfs/builtin_packs.h b/Engine/Vfs/builtin_packs.h new file mode 100644 index 00000000..17eb965c --- /dev/null +++ b/Engine/Vfs/builtin_packs.h @@ -0,0 +1,42 @@ +#pragma once +#include +#include +#include +#include + +// Game content ReSkate carries itself: packs of game assets built by ReSkate Studio, zipped into +// ReSkate.dll (the FPV drone's model, tools/models/build_drone_pack.py). Nobody installs or sees +// them. Official mods (below) are the other content ReSkate keeps for every player. Before a merge each is unpacked under Mods/.reskate-builtin/ (a dot folder, +// which no mod list shows) when the one embedded differs from what is there, and the catalogue +// merges it after every installed mod, always on (mod_catalog.cpp). +namespace dingosdk::mods { +inline constexpr std::string_view builtin_folder = ".reskate-builtin"; +struct BuiltinPack { + std::string name; // as the merge knows it: reskate-builtin- + std::filesystem::path directory; // Mods/.reskate-builtin/ +}; +// The packs ReSkate.dll carries, unpacked and current under `mods_root`. ReSkate.dll is the +// module running this when it is loaded (the game); otherwise the copy beside this program (the +// launcher) is opened as data. Never throws; `notes` says what was unpacked or went wrong. +std::vector unpack_builtin_packs(const std::filesystem::path& mods_root, + std::vector& notes) noexcept; +// One pack's bytes as ReSkate.dll carries them (builtin_pack_format.h, or a plain zip) unpacked +// into `target`; false with `issue` saying why. For tests and tools: unpack_builtin_packs uses it. +bool unpack_builtin_pack(std::span bytes, const std::filesystem::path& target, std::string& issue) noexcept; + +// Official mods (Launcher/official_mods.h): Thunderstore packages the ReSkate backend names, which +// the launcher keeps installed at their latest version for every player in +// Mods/.reskate-official/, each with a stamp file holding its version (written +// last, so a folder without one is not a whole mod). Always on: merged after every installed mod, +// like the packs above. Not in mods.json, and no mod list shows them. +inline constexpr std::string_view official_folder = ".reskate-official"; +inline constexpr std::string_view official_stamp = ".official-version"; +// The official mods in place under `mods_root`, by folder name. Never throws. +std::vector official_packs(const std::filesystem::path& mods_root) noexcept; +// Mods that came before an official one and are retired with it (Fourth Studio's deprecated first +// Board Break), by folder name; the launcher takes them away and nothing loads them. +bool retired_mod(std::string_view folder) noexcept; +// Whether the installed mod `folder` (Mods/) is left out because ReSkate keeps it itself: +// an official mod in place under `mods_root`, or a retired one. Never throws. +bool kept_by_reskate(const std::filesystem::path& mods_root, std::string_view folder) noexcept; +} // namespace dingosdk::mods diff --git a/Engine/Vfs/game_archives.cpp b/Engine/Vfs/game_archives.cpp index 5d5e6ac1..e6cdfd4e 100644 --- a/Engine/Vfs/game_archives.cpp +++ b/Engine/Vfs/game_archives.cpp @@ -74,12 +74,13 @@ GameArchives::GameArchives(fs::path root, const native_db::Node& layout) : root_ const auto win32 = root_ / "Win32"; for (fs::recursive_directory_iterator it(win32, error), end; it != end && !error; it.increment(error)) { if (!it->is_regular_file(error) || error) { error.clear(); continue; } - if (lower(it->path().extension().string()) != ".cas") continue; - const auto index = archive_index(it->path().stem().string()); + if (lower(path_utf8(it->path().extension())) != ".cas") continue; + const auto index = archive_index(path_utf8(it->path().stem())); if (!index) continue; // Relative to Win32, because read() adds that itself. - const auto directory = fs::relative(it->path().parent_path(), win32, error).generic_string(); + const auto relative = fs::relative(it->path().parent_path(), win32, error).generic_u8string(); if (error) { error.clear(); continue; } + const std::string directory(relative.begin(), relative.end()); directories_[directory].insert(*index); } for (const auto& [directory, archives] : directories_) byArchives[archives].push_back(directory); diff --git a/Engine/Vfs/https_download.cpp b/Engine/Vfs/https_download.cpp index 4ea1975a..1c2f4f88 100644 --- a/Engine/Vfs/https_download.cpp +++ b/Engine/Vfs/https_download.cpp @@ -3,26 +3,107 @@ #include #include #include +#include +#include #include #include +#include #include +#include namespace dingosdk::https { namespace { +std::uint64_t now_us() noexcept { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()).count()); +} + struct Handle { HANDLE value{INVALID_HANDLE_VALUE}; - ~Handle() { if (value != INVALID_HANDLE_VALUE) CloseHandle(value); } + const std::filesystem::path& path; + bool complete{}; + ~Handle() { + if (value != INVALID_HANDLE_VALUE) { + CloseHandle(value); + if (!complete) DeleteFileW(path.c_str()); + } + } }; struct Internet { HINTERNET value{}; ~Internet() { if (value) WinHttpCloseHandle(value); } }; +struct AsyncRequest { + HINTERNET value{}; + std::mutex mutex; + std::condition_variable changed; + DWORD expected{}, error{}, read{}; + bool completed{}, closed{}; + + ~AsyncRequest() { close(); } + + void close() { + if (!value) return; + // Async closure interrupts pending I/O. Keep the context and read buffer + // alive until WinHTTP's final callback, including on cancellation. + WinHttpCloseHandle(value); + std::unique_lock lock(mutex); + changed.wait(lock, [&] { return closed; }); + value = nullptr; + } + + static void CALLBACK callback(HINTERNET, DWORD_PTR context, DWORD status, void* info, DWORD size) { + if (!context) return; + auto& request = *reinterpret_cast(context); + std::lock_guard lock(request.mutex); + if (status == WINHTTP_CALLBACK_STATUS_HANDLE_CLOSING) { + request.closed = true; + } else if (status == WINHTTP_CALLBACK_STATUS_REQUEST_ERROR) { + request.error = static_cast(info)->dwError; + request.completed = true; + } else if (status == request.expected) { + if (status == WINHTTP_CALLBACK_STATUS_READ_COMPLETE) request.read = size; + request.completed = true; + } + request.changed.notify_all(); + } + + void prepare(DWORD status) { + std::lock_guard lock(mutex); + expected = status; + completed = false; + error = read = 0; + } + + template + DWORD wait(const Stop& stop) { + std::unique_lock lock(mutex); + for (;;) { + if (const auto stopped = stop()) return stopped; + if (completed) return error; + changed.wait_for(lock, std::chrono::milliseconds(20)); + } + } +}; } Download get(std::wstring_view url, const std::filesystem::path& destination, std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent, Watch* watch) { Download result; - const auto fail = [&](DWORD error) { result.error = error; return result; }; + const auto started = now_us(); + const auto timeout_deadline = started + std::uint64_t{timeout_seconds} * 1000000; + const auto stop = [&]() -> DWORD { + if (watch && watch->cancel.load()) return ERROR_CANCELLED; + const auto external = watch ? watch->deadline_us.load() : 0; + const auto deadline = external && external < timeout_deadline ? external : timeout_deadline; + return now_us() >= deadline ? ERROR_TIMEOUT : ERROR_SUCCESS; + }; + const auto fail = [&](DWORD error) { + const auto stopped = stop(); + result.error = stopped ? stopped : error; + return result; + }; + if (const auto error = stop()) return fail(error); if (url.empty() || url.size() >= 4096 || url.find_first_of(L"\r\n\t #") != std::wstring_view::npos) return fail(ERROR_INVALID_PARAMETER); const std::wstring address(url); @@ -35,7 +116,7 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, const std::wstring host(parts.lpszHostName, parts.dwHostNameLength); std::wstring resource = parts.dwUrlPathLength ? std::wstring(parts.lpszUrlPath, parts.dwUrlPathLength) : L"/"; if (parts.dwExtraInfoLength) resource.append(parts.lpszExtraInfo, parts.dwExtraInfoLength); - Internet session{WinHttpOpen(agent, WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY, nullptr, nullptr, 0)}; + Internet session{WinHttpOpen(agent, WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY, nullptr, nullptr, WINHTTP_FLAG_ASYNC)}; if (!session.value) return fail(GetLastError()); if (!WinHttpSetTimeouts(session.value, 5000, 5000, 10000, 10000)) return fail(GetLastError()); Internet connection{WinHttpConnect(session.value, host.c_str(), parts.nPort, 0)}; @@ -43,30 +124,56 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, Internet request{WinHttpOpenRequest(connection.value, L"GET", resource.c_str(), nullptr, nullptr, WINHTTP_DEFAULT_ACCEPT_TYPES, WINHTTP_FLAG_SECURE)}; if (!request.value) return fail(GetLastError()); + // Declared before AsyncRequest so pending reads cannot outlive their buffer. + std::array buffer{}; + AsyncRequest async; DWORD redirect = WINHTTP_OPTION_REDIRECT_POLICY_DISALLOW_HTTPS_TO_HTTP; DWORD logon = WINHTTP_AUTOLOGON_SECURITY_LEVEL_HIGH; if (!WinHttpSetOption(request.value, WINHTTP_OPTION_REDIRECT_POLICY, &redirect, sizeof(redirect)) || !WinHttpSetOption(request.value, WINHTTP_OPTION_AUTOLOGON_POLICY, &logon, sizeof(logon))) return fail(GetLastError()); - const auto started = GetTickCount64(); - if (!WinHttpSendRequest(request.value, nullptr, 0, nullptr, 0, 0, 0) || !WinHttpReceiveResponse(request.value, nullptr)) + DWORD_PTR context = reinterpret_cast(&async); + if (!WinHttpSetOption(request.value, WINHTTP_OPTION_CONTEXT_VALUE, &context, sizeof(context))) return fail(GetLastError()); + if (WinHttpSetStatusCallback(request.value, AsyncRequest::callback, + WINHTTP_CALLBACK_FLAG_SENDREQUEST_COMPLETE | WINHTTP_CALLBACK_FLAG_HEADERS_AVAILABLE | + WINHTTP_CALLBACK_FLAG_READ_COMPLETE | WINHTTP_CALLBACK_FLAG_REQUEST_ERROR | + WINHTTP_CALLBACK_FLAG_HANDLES, 0) == WINHTTP_INVALID_STATUS_CALLBACK) + return fail(GetLastError()); + async.value = std::exchange(request.value, nullptr); + const auto finish = [&](BOOL accepted) -> DWORD { + if (!accepted) { + const auto error = GetLastError(); + if (error != ERROR_IO_PENDING) return error; + } + return async.wait(stop); + }; + if (const auto error = stop()) return fail(error); + async.prepare(WINHTTP_CALLBACK_STATUS_SENDREQUEST_COMPLETE); + if (const auto error = finish(WinHttpSendRequest(async.value, nullptr, 0, nullptr, 0, 0, context))) + return fail(error); + async.prepare(WINHTTP_CALLBACK_STATUS_HEADERS_AVAILABLE); + if (const auto error = stop()) return fail(error); + if (const auto error = finish(WinHttpReceiveResponse(async.value, nullptr))) return fail(error); DWORD size = sizeof(result.http_status); - if (!WinHttpQueryHeaders(request.value, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, + if (!WinHttpQueryHeaders(async.value, WINHTTP_QUERY_STATUS_CODE | WINHTTP_QUERY_FLAG_NUMBER, nullptr, &result.http_status, &size, nullptr)) return fail(GetLastError()); if (result.http_status != 200) return fail(ERROR_BAD_NET_RESP); DWORD length{}; size = sizeof(length); - if (WinHttpQueryHeaders(request.value, WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER, + if (WinHttpQueryHeaders(async.value, WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER, nullptr, &length, &size, nullptr) && length > max_bytes) return fail(ERROR_FILE_TOO_LARGE); if (watch) watch->total.store(length), watch->received.store(0); - Handle output{CreateFileW(destination.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr)}; + if (const auto error = stop()) return fail(error); + Handle output{CreateFileW(destination.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr), + destination}; if (output.value == INVALID_HANDLE_VALUE) return fail(GetLastError()); - std::array buffer{}; std::uint64_t total{}; for (;;) { - if (GetTickCount64() - started >= std::uint64_t{timeout_seconds} * 1000) return fail(ERROR_TIMEOUT); - if (watch && watch->cancel.load()) return fail(ERROR_CANCELLED); - DWORD read{}, written{}; - if (!WinHttpReadData(request.value, buffer.data(), static_cast(buffer.size()), &read)) return fail(GetLastError()); + if (const auto error = stop()) return fail(error); + async.prepare(WINHTTP_CALLBACK_STATUS_READ_COMPLETE); + if (const auto error = finish(WinHttpReadData(async.value, buffer.data(), static_cast(buffer.size()), nullptr))) + return fail(error); + const auto read = async.read; + DWORD written{}; if (!read) break; if (total + read > max_bytes) return fail(ERROR_FILE_TOO_LARGE); if (!WriteFile(output.value, buffer.data(), read, &written, nullptr) || written != read) return fail(ERROR_WRITE_FAULT); @@ -75,27 +182,30 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, } if (!total) return fail(ERROR_BAD_LENGTH); if (!FlushFileBuffers(output.value)) return fail(GetLastError()); + async.close(); + if (const auto error = stop()) return fail(error); + output.complete = true; result.ok = true; return result; } std::optional get_text(std::wstring_view url, std::uint64_t max_bytes, - std::uint32_t timeout_seconds, const wchar_t* agent, Download* result) { + std::uint32_t timeout_seconds, const wchar_t* agent, Download* result, Watch* watch) { static std::atomic serial{}; wchar_t folder[MAX_PATH + 1]{}; const auto length = GetTempPathW(MAX_PATH, folder); const auto path = std::filesystem::path(length ? std::wstring(folder, length) : L".") / (L"reskate-get-" + std::to_wstring(GetCurrentProcessId()) + L"-" + std::to_wstring(++serial) + L".tmp"); std::error_code error; - std::filesystem::remove(path, error); // get needs a fresh destination - const auto download = get(url, path, max_bytes, timeout_seconds, agent); + std::filesystem::remove(path, error); // discard a stale file left by an earlier process + const auto download = get(url, path, max_bytes, timeout_seconds, agent, watch); if (result) *result = download; std::optional text; if (download.ok) { std::ifstream in(path, std::ios::binary); text.emplace(std::istreambuf_iterator(in), std::istreambuf_iterator()); } - std::filesystem::remove(path, error); + if (download.ok) std::filesystem::remove(path, error); return text; } } diff --git a/Engine/Vfs/https_download.h b/Engine/Vfs/https_download.h index 1b9073f2..aa71675a 100644 --- a/Engine/Vfs/https_download.h +++ b/Engine/Vfs/https_download.h @@ -16,6 +16,8 @@ struct Download { struct Watch { std::atomic received{}, total{}; std::atomic cancel{}; + // Absolute steady_clock epoch microseconds; zero leaves only the GET timeout. + std::atomic deadline_us{}; }; // Streams an HTTPS GET into `destination`, which must not exist yet. Follows // redirects that stay on HTTPS; refuses bodies over `max_bytes` or transfers @@ -25,5 +27,5 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, // The body of the same GET, for an answer small enough to hold: nothing when it // fails, and `result`, when given, says how. std::optional get_text(std::wstring_view url, std::uint64_t max_bytes, - std::uint32_t timeout_seconds, const wchar_t* agent, Download* result = nullptr); + std::uint32_t timeout_seconds, const wchar_t* agent, Download* result = nullptr, Watch* watch = nullptr); } diff --git a/Engine/Vfs/mod_catalog.cpp b/Engine/Vfs/mod_catalog.cpp index 39d0dfc3..313b107f 100644 --- a/Engine/Vfs/mod_catalog.cpp +++ b/Engine/Vfs/mod_catalog.cpp @@ -1,4 +1,6 @@ #include "mod_catalog.h" +#include "Engine/Core/Platform/path_text.h" +#include "builtin_packs.h" #include "mod_merge.h" #include "native_db.h" @@ -127,23 +129,36 @@ std::vector> decks_inside(const fs::path& mod) std::error_code error; for (fs::directory_iterator at(mod / L"decks", error), end; !error && at != end; at.increment(error)) { if (!at->is_directory(error) || !fs::is_regular_file(at->path() / L"layout.toc", error)) continue; - auto name = lower(at->path().filename().string()); + auto name = lower(path_utf8(at->path().filename())); if (deck_name(name)) found.emplace_back(std::move(name), at->path()); } std::sort(found.begin(), found.end(), [](const auto& a, const auto& b) { return deck_before(a.first, b.first); }); return found; } +// Every mod folder a deck can come from: those in the Mods folder (but one a built-in pack +// replaces), then the built-in packs (builtin_packs.h), which the merge also puts after them. +std::vector deck_mod_folders() { + std::vector found; + const auto mods = engine_data_root() / L"Mods"; + std::error_code error; + for (fs::directory_iterator at(mods, error), end; !error && at != end; at.increment(error)) + if (at->is_directory(error) && !kept_by_reskate(mods, path_utf8(at->path().filename()))) found.push_back(at->path()); + error.clear(); + for (fs::directory_iterator at(mods / std::string(builtin_folder), error), end; !error && at != end; at.increment(error)) + if (at->is_directory(error) && at->path().filename().wstring().find(L".unpacking-") == std::wstring::npos) + found.push_back(at->path()); + for (auto& official : official_packs(mods)) found.push_back(std::move(official.directory)); + return found; +} // Every deck in the Mods folder with the folder of the mod that brings it, in the menus' order, // whatever mods.json says of that mod: the decks inside a pack, or else the one a folder is named for. std::vector> installed_decks() { std::vector> decks; - std::error_code error; - for (fs::directory_iterator at(engine_data_root() / L"Mods", error), end; !error && at != end; at.increment(error)) { - if (!at->is_directory(error)) continue; - auto inside = decks_inside(at->path()); - for (auto& one : inside) decks.emplace_back(std::move(one.first), at->path()); + for (const auto& folder : deck_mod_folders()) { + auto inside = decks_inside(folder); + for (auto& one : inside) decks.emplace_back(std::move(one.first), folder); if (!inside.empty()) continue; - if (auto deck = board_break_deck_of(at->path().filename().string()); !deck.empty()) decks.emplace_back(std::move(deck), at->path()); + if (auto deck = board_break_deck_of(path_utf8(folder.filename())); !deck.empty()) decks.emplace_back(std::move(deck), folder); } std::stable_sort(decks.begin(), decks.end(), [](const auto& a, const auto& b) { return deck_before(a.first, b.first); }); return decks; @@ -220,6 +235,18 @@ std::vector board_break_folders() noexcept { } catch (...) {} return found; } +std::filesystem::path board_break_deck_folder(std::string_view deck) noexcept { + try { + const auto wanted = lower(deck); + for (const auto& folder : deck_mod_folders()) { + const auto inside = decks_inside(folder); + for (const auto& one : inside) + if (one.first == wanted) return one.second; + if (inside.empty() && board_break_deck_of(path_utf8(folder.filename())) == wanted) return folder; + } + } catch (...) {} + return {}; +} std::vector installed_board_break_decks() noexcept { std::vector found; try { @@ -291,7 +318,12 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver result.issue = "The engine data root could not be resolved"; return result; } + // Content ReSkate carries itself (builtin_packs.h), unpacked first: it makes the Mods folder + // when there is none, so the content loads without any mod installed. + std::vector builtin_notes; + const auto builtin = unpack_builtin_packs(data_root / mods_folder, builtin_notes); auto list = scan_mods(data_root); + for (auto& note : builtin_notes) list.notes.push_back(std::move(note)); result.root = std::move(list.root); result.present = list.present; if (!result.present) return result; @@ -305,6 +337,19 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver try { for (const auto& name : list.missing) result.notes.push_back("mods.json names a folder that is not present: " + name); + // A mod ReSkate keeps itself (an official mod, or one retired with it: builtin_packs.h) is + // left out, should the launcher not have taken it away yet: its content would load twice. + std::erase_if(list.entries, [&](const ModEntry& entry) { + if (!kept_by_reskate(data_root / mods_folder, entry.mod.name)) return false; + result.notes.push_back(entry.mod.name + " is an official mod ReSkate keeps itself; the copy in the Mods folder is left out"); + return true; + }); + // After every installed mod, so none of theirs is ever shadowed by it; always on. Official + // mods (builtin_packs.h) the same way. + for (const auto& pack : builtin) + list.entries.push_back({describe_mod(pack.directory, pack.name, result.notes), true}); + for (const auto& pack : official_packs(data_root / mods_folder)) + list.entries.push_back({describe_mod(pack.directory, pack.name, result.notes), true}); // Board Break: a deck pack holds several decks in one mod folder (mod_catalog.h). Each deck // inside becomes a mod of its own for the run, listed after the pack as #, so // that a pack's decks are chosen, set aside and switched like decks in folders of their own. diff --git a/Engine/Vfs/mod_catalog.h b/Engine/Vfs/mod_catalog.h index 66622347..9b6f8c69 100644 --- a/Engine/Vfs/mod_catalog.h +++ b/Engine/Vfs/mod_catalog.h @@ -63,8 +63,8 @@ struct Catalog { // A deck goes by its name in small letters. These are the decks the feature was made with, in the // order the menus list them, and the console also knows them by their place here (deck 1 is the // first). Any other deck comes after them, by its name: a new shape needs no change to ReSkate. -inline constexpr std::array board_break_decks{"standard", "wide", "concave", "powell", - "slim", "egg", "square", "shaped"}; +inline constexpr std::array board_break_decks{"standard", "wide", "concave", "powell", "slim", + "egg", "square", "shaped", "oldschool", "pig"}; // The name of the deck a mod holds going by the mod's name (a folder's, or a pack's deck's), or // nothing; and the folder name of the pack that one of a pack's decks is listed under, or nothing. std::string board_break_deck_of(std::string_view mod); @@ -78,6 +78,9 @@ bool save_board_break_deck(std::string_view deck) noexcept; // beside its decks. std::vector installed_board_break_decks() noexcept; std::vector board_break_folders() noexcept; +// The folder with a deck's own files (a pack's decks/, or the Board_Break_ folder): what +// it says of itself (deck.ini) is there. Empty when that deck is not installed. +std::filesystem::path board_break_deck_folder(std::string_view deck) noexcept; // What the mods in effect give the game: the launch's catalogue, or the one the last live apply // read (live_mods tells with board_break_applied). std::string board_break_in_play() noexcept; diff --git a/Engine/Vfs/mod_item_sources.cpp b/Engine/Vfs/mod_item_sources.cpp new file mode 100644 index 00000000..ea90b233 --- /dev/null +++ b/Engine/Vfs/mod_item_sources.cpp @@ -0,0 +1,75 @@ +#include "mod_item_sources.h" + +#include "Engine/Core/Json/json.h" +#include "Engine/Core/Platform/path_text.h" + +#include +#include +#include +#include + +namespace dingosdk::mods { +namespace fs = std::filesystem; + +std::vector read_item_sources(const fs::path& file) noexcept { + std::vector sources; + try { + std::ifstream input(file, std::ios::binary); + if (!input) return {}; + const std::string text{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + const auto root = Json::parse(text, {64u << 20, 32, 1u << 24}); + const auto& mods = root.at("mods"); + for (std::size_t i = 0; i < mods.size(); ++i) { + const auto& entry = mods.at(i); + ItemSource source{entry.at("mod").string(), entry.value("title", std::string()), entry.value("author", std::string()), + entry.value("version", std::string()), entry.value("folder", std::string()), {}}; + const auto& keys = entry.at("items"); + for (std::size_t k = 0; k < keys.size(); ++k) source.keys.push_back(keys.at(k).string()); + sources.push_back(std::move(source)); + } + } catch (...) { + return {}; // an unreadable file only means no card says where its item is from + } + return sources; +} + +void write_item_sources(const fs::path& file, const std::vector& sources) { + auto mods = Json::array(); + for (const auto& source : sources) { + auto entry = Json::object(); + entry["mod"] = source.mod; + entry["title"] = source.title; + entry["author"] = source.author; + entry["version"] = source.version; + entry["folder"] = source.folder; + entry["items"] = Json(source.keys); + mods.push_back(std::move(entry)); + } + auto root = Json::object(); + root["schema"] = 1u; + root["mods"] = std::move(mods); + const auto text = root.dump(1); + // Whole or not at all: the game may read it while a merge runs beside it. + auto temporary = file; + temporary += L".new"; + { + std::ofstream output(temporary, std::ios::binary | std::ios::trunc); + output.write(text.data(), static_cast(text.size())); + if (!output.flush()) throw std::runtime_error("Cannot write " + path_utf8(temporary)); + } + std::error_code error; + fs::rename(temporary, file, error); + if (error) { + fs::remove(temporary, error); + throw std::runtime_error("Cannot write " + path_utf8(file)); + } +} + +std::string item_source_name(const ItemSource& source) { + std::string name = source.title.empty() ? source.mod : source.title; + std::replace(name.begin(), name.end(), '_', ' '); + if (!source.author.empty()) name += " by " + source.author; + return name; +} + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_item_sources.h b/Engine/Vfs/mod_item_sources.h new file mode 100644 index 00000000..05cccf74 --- /dev/null +++ b/Engine/Vfs/mod_item_sources.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +// Which mod each cosmetic item came from, so its card in the customization menu can say so: +// Mods/.reskate/reskate-items.json. The merge writes it from the items the store-copy check reads +// in every mod (mod_store_copies.h); the game reads it when it builds its item catalogue. +namespace dingosdk::mods { + +inline constexpr char item_sources_file[] = "reskate-items.json"; + +struct ItemSource { + std::string mod; // its folder name + std::string title, author, version; // as its manifest gives them + std::string folder; // where it is, from Mods/ ("x" or ".reskate-official/x"), with its icon.png + std::vector keys; // the items it adds or changes, lower case +}; + +// Highest priority first, as the merge orders the mods: where two list one item, the game uses +// the first one's, so that is where it is from. Empty when there is no file or it cannot be read. +[[nodiscard]] std::vector read_item_sources(const std::filesystem::path& file) noexcept; +// Throws when the file cannot be written. +void write_item_sources(const std::filesystem::path& file, const std::vector& sources); + +// How a card names the mod: its title (Thunderstore names have underscores for spaces), and who +// made it when its manifest says: "ReSkate FPV Drone by zeex64". +[[nodiscard]] std::string item_source_name(const ItemSource& source); + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_list.cpp b/Engine/Vfs/mod_list.cpp index 04bc0f07..e2e3bd65 100644 --- a/Engine/Vfs/mod_list.cpp +++ b/Engine/Vfs/mod_list.cpp @@ -323,6 +323,28 @@ bool valid_mod_name(std::string_view name) { }); } +Mod describe_mod(const std::filesystem::path& directory, std::string name, std::vector& notes) { + std::error_code error; + Mod mod; + mod.name = std::move(name); + mod.title = mod.name; + mod.directory = directory; + mod.provides_layout = fs::is_regular_file(mod.directory / layout_name, error); + mod.provides_levels = fs::is_regular_file(mod.directory / levels_name, error); + for (fs::directory_iterator it(mod.directory / "parks", error), end; !error && it != end; it.increment(error)) { + auto file = ascii_path(it->path().filename()); + if (!file.ends_with(".park.json") || mod.park_maps.size() >= 16) continue; + file.resize(file.size() - 10); + mod.park_maps.push_back(std::move(file)); + } + std::sort(mod.park_maps.begin(), mod.park_maps.end()); + read_build_info(mod, notes); + read_mod_info(mod, notes); + read_levels(mod); + read_studio_marker(mod); + return mod; +} + ModList scan_mods(const std::filesystem::path& data_root) noexcept { ModList result; try { @@ -364,24 +386,7 @@ ModList scan_mods(const std::filesystem::path& data_root) noexcept { } ModEntry entry; entry.enabled = enabled; - auto& mod = entry.mod; - mod.name = found->second; - mod.title = mod.name; - mod.directory = result.root / mod.name; - mod.provides_layout = fs::is_regular_file(mod.directory / layout_name, error); - mod.provides_levels = fs::is_regular_file(mod.directory / levels_name, error); - for (fs::directory_iterator it(mod.directory / "parks", error), end; !error && it != end; it.increment(error)) { - auto file = ascii_path(it->path().filename()); - if (!file.ends_with(".park.json") || mod.park_maps.size() >= 16) continue; - file.resize(file.size() - 10); - mod.park_maps.push_back(std::move(file)); - } - error.clear(); - std::sort(mod.park_maps.begin(), mod.park_maps.end()); - read_build_info(mod, result.notes); - read_mod_info(mod, result.notes); - read_levels(mod); - read_studio_marker(mod); + entry.mod = describe_mod(result.root / found->second, found->second, result.notes); result.entries.push_back(std::move(entry)); } // Only mods named in the exclusions file are fingerprinted. diff --git a/Engine/Vfs/mod_list.h b/Engine/Vfs/mod_list.h index aff4fbd4..5237ca52 100644 --- a/Engine/Vfs/mod_list.h +++ b/Engine/Vfs/mod_list.h @@ -110,6 +110,9 @@ std::string ascii_path(const std::filesystem::path& path); // malformed, `issue` says why and `entries` still lists the folders (all // shown as enabled) so a manager can offer to rewrite it. ModList scan_mods(const std::filesystem::path& data_root) noexcept; +// One mod folder as scan_mods reads it: its layout, levels, parks, info files and build stamp. +// `name` is how it is known (for a folder under Mods/, the folder's name). Throws on I/O errors. +Mod describe_mod(const std::filesystem::path& directory, std::string name, std::vector& notes); // Writes mods.json for `entries` in order, replacing the file atomically. // Throws with a readable message on failure. diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index 219f5050..74d21571 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -2,9 +2,11 @@ #include "content_cache.h" #include "content_catalogs.h" +#include "mod_item_sources.h" #include "mod_merge_internal.h" #include "mod_store_copies.h" #include "native_db.h" +#include "Engine/Core/Platform/path_text.h" #include @@ -46,9 +48,9 @@ constexpr std::array launch_superbundles{"Win32/globals.toc // (mod_store_copies.h) gets a problem; true when any does. The problem says no // more than that the mod could not be merged: what was found is not for the // mod's author to read. The catalogue is only read once a mod turns out to add -// an item at all. +// an item at all. `added` gets the items each mod read adds, when the check ran. bool store_copy_problems(const Catalog& all, MergeReport& report, const std::vector& checked, - std::size_t threads, bool background) { + std::size_t threads, bool background, std::optional>& added) { // (Only when some are skipped is the catalogue copied, without them.) std::optional fewer; if (!checked.empty()) { @@ -63,6 +65,9 @@ bool store_copy_problems(const Catalog& all, MergeReport& report, const std::vec if (!store) store = content_cache::read_catalogs(content_cache::directory()); return store->reserved(key); }, &report.notes, threads, background); + // (No mod to read is a check that ran and found no items.) + if (found.read || std::none_of(catalog.mods.begin(), catalog.mods.end(), [](const Mod& mod) { return mod.provides_layout; })) + added = std::move(found.added); for (const auto& source : found.mods) report.problems[source.mod].emplace_back(store_copies_problem); return !found.mods.empty(); } @@ -100,7 +105,9 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con fingerprint += " inactive " + mod.name + " " + mod_fingerprint(mod.directory); if (options.live) { fs::remove(output / stamp_file, error); - } else if (auto previous = previous_merge(output, fingerprint)) { + } else if (auto previous = previous_merge(output, fingerprint); + previous && (!storeKnown || fs::exists(output / item_sources_file, error))) { + // (A patch from before items were traced to their mods is built again, once.) return std::move(*previous); } // A mod that adds copies of store items is not loaded at all. Found before @@ -111,8 +118,28 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // launch (the player is waiting on nothing else), half of them and fewer while the game runs. const auto cores = std::max(1U, std::thread::hardware_concurrency()); const std::size_t readers = std::min(options.live ? std::max(1U, cores / 2) : cores, options.live ? 6U : 8U); + std::optional> itemsRead; if (storeKnown && store_copy_problems(catalog, report, options.live ? options.checked : std::vector{}, - readers - 1, options.live)) return report; + readers - 1, options.live, itemsRead)) return report; + // Which mod each item is from (mod_item_sources.h): what was just read, and for a mod a + // live merge did not read again, what the launch's merge found. Read before the patch + // folder is cleared. + std::optional> itemSources; + if (itemsRead) { + const auto before = options.live ? read_item_sources(output / item_sources_file) : std::vector{}; + itemSources.emplace(); + for (const auto* mod : mods) { + std::error_code outside; + const auto folder = fs::relative(mod->directory, catalog.root, outside); + ItemSource source{mod->name, mod->title, mod->author, mod->version, + outside ? std::string() : path_utf8(folder.generic_wstring()), {}}; + const auto read = std::ranges::find(*itemsRead, mod->name, &StoreCopies::Added::mod); + const auto earlier = std::ranges::find(before, mod->name, &ItemSource::mod); + if (read != itemsRead->end()) source.keys = std::move(read->keys); + else if (earlier != before.end()) source.keys = earlier->keys; + if (!source.keys.empty()) itemSources->push_back(std::move(source)); + } + } if (!options.live) fs::remove_all(output, error); lap("checking the mods"); @@ -289,8 +316,8 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // Archive 1 stays the patch's own file, never a link to a mod's: // a live apply appends manifests and new mods to it while the // game has it open, and a linked file cannot be swapped for a - // private copy then (Windows keeps it in use). - claimed.insert(manifestArchive); + // private copy then (Windows keeps it in use). Placeholder indexes name no file. + claimed.insert({manifestArchive, CasStore::waiting_archive, CasStore::carried_archive}); if (number == manifestArchive) { if (const auto free = claim(directory); free) { spot.archive = *free; @@ -442,7 +469,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con for (std::size_t index = 0; index < job.merged.bundles.size(); ++index) { auto region = fb::read_bundle_region(job.merged.bundles[index].region); if (std::ranges::any_of(region.files, [](const fb::BundleFileInfo& file) { - return CasStore::waits(file.location); })) + return CasStore::placeholder(file.location); })) job.held.emplace_back(index, std::move(region)); } } catch (...) { @@ -509,9 +536,10 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con } for (auto& chunk : job.merged.chunks) job.store->settled(chunk.location, chunk.offset); for (const auto& [installChunk, archive] : job.used) - used.emplace(installChunk, archive == CasStore::waiting_archive ? manifestArchive : archive); + used.emplace(installChunk, CasStore::placeholder(archive) ? manifestArchive : archive); report.mergedBundles += job.report.mergedBundles; report.mergedAssets += job.report.mergedAssets; + report.unchangedCopies += job.report.unchangedCopies; report.notes.insert(report.notes.end(), std::make_move_iterator(job.report.notes.begin()), std::make_move_iterator(job.report.notes.end())); for (auto& [mod, problems] : job.report.problems) { @@ -690,6 +718,13 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con declare("layeredInstallChunkFiles"); declare("unlayeredInstallChunkFiles"); + if (itemSources) { + try { + write_item_sources(output / item_sources_file, *itemSources); + } catch (const std::exception& failure) { + report.notes.push_back(std::string("Item cards will not say which mod they are from: ") + failure.what()); + } + } if (!options.live) { const auto rebuilt = native_db::write(layout); std::vector file(native_db::envelope_size, 0); diff --git a/Engine/Vfs/mod_merge.h b/Engine/Vfs/mod_merge.h index 5d516a2b..ac9c7eb5 100644 --- a/Engine/Vfs/mod_merge.h +++ b/Engine/Vfs/mod_merge.h @@ -17,6 +17,7 @@ struct MergeReport { std::size_t superbundles{}; // superbundle TOCs written std::size_t mergedBundles{}; // bundles combined from two or more mods std::size_t mergedAssets{}; // EBX assets two or more mods had both edited + std::size_t unchangedCopies{}; // mods' asset copies equal to the game's, not kept for combining std::size_t archives{}; // cas files linked in std::string issue; // non-empty when nothing was produced std::vector notes; diff --git a/Engine/Vfs/mod_merge_cas_store.cpp b/Engine/Vfs/mod_merge_cas_store.cpp index c7e02288..bb10d7e1 100644 --- a/Engine/Vfs/mod_merge_cas_store.cpp +++ b/Engine/Vfs/mod_merge_cas_store.cpp @@ -61,7 +61,7 @@ fb::BundleFileInfo CasStore::write(std::uint32_t installChunk, std::uint16_t arc const auto path = archive_path(installChunk, archive); if (waiting_) { // A placement that waits names no archive, so there can be only the one to settle into. - if (archive == waiting_archive || (archive_ && *archive_ != archive)) + if (placeholder(archive) || (archive_ && *archive_ != archive)) throw std::logic_error("A waiting store writes to one archive index"); archive_ = archive; auto& held = held_[path.wstring()].bytes; @@ -83,19 +83,48 @@ CasStore CasStore::waiting() const { copy.archive_.reset(); copy.held_.clear(); copy.offsets_.clear(); + copy.carried_.clear(); + copy.carriedAt_.clear(); return copy; } +fb::BundleFileInfo CasStore::carry(std::uint32_t installChunk, std::uint16_t archive, std::span encoded) { + if (!waiting_) throw std::logic_error("Only a waiting store carries payloads"); + if (placeholder(archive) || (archive_ && *archive_ != archive)) + throw std::logic_error("A waiting store writes to one archive index"); + // An install chunk no package names fails here, for this payload alone. + (void)directory(installChunk); + archive_ = archive; + carried_.push_back({installChunk, sha1_of(encoded), encoded, {}}); + return {{true, installChunk, carried_archive}, static_cast(carried_.size() - 1), + static_cast(encoded.size())}; +} + void CasStore::settle(CasStore& waiting) { // In path order: which file comes first makes no difference to where anything lands. for (auto& [path, held] : waiting.held_) { held.at = append(fs::path(path), held.bytes); held.bytes = {}; } + // Then the carried assets, each written once per install chunk. + for (auto& carried : waiting.carried_) { + const std::pair key{carried.installChunk, carried.sha1}; + auto found = carriedAt_.find(key); + if (found == carriedAt_.end()) + found = carriedAt_.emplace(key, write(carried.installChunk, *waiting.archive_, carried.bytes)).first; + carried.at = found->second; + } } bool CasStore::settled(fb::CasIdentifier& location, std::uint32_t& offset) const { - if (!waits(location)) return false; + if (!placeholder(location)) return false; + if (location.archive == carried_archive) { + if (offset >= carried_.size()) throw std::logic_error("A carried placement names no asset its store holds"); + const auto& at = carried_[offset].at; + location = at.location; + offset = at.offset; + return true; + } const auto found = archive_ ? held_.find(archive_path(location.installChunk, *archive_).wstring()) : held_.end(); if (found == held_.end()) throw std::logic_error("A placement waits in a store that holds nothing for it"); const auto moved = found->second.at + offset; diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 6b95fe26..34a40323 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -1,6 +1,7 @@ #include "mod_merge_internal.h" #include "Engine/Resource/cas_codec.h" #include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_fixup_repair.h" #include "Engine/Resource/bundle_ref_table.h" #include "Engine/Resource/ebx_merge.h" #include "Engine/Resource/ebx_writer.h" @@ -46,7 +47,7 @@ struct Asset { // Where one contributor's copy of an asset lives, so conflicting copies can be // fetched and combined later. struct Contribution { - fs::path root; + const fs::path* root{}; // the source's folder, in `sources` or baseRoot: neither moves while combine runs fb::BundleFileInfo file; // still pointing at the archive it came from fb::Sha1 sha1; std::vector resourceMeta; @@ -112,7 +113,7 @@ struct BundleState { // are asset paths that share long beginnings, which an ordered map compares // byte by byte at every step, and nothing reads these in key order. std::unordered_map seen; - // Every copy of an asset more than one source supplies, base included. + // Each source's copy of an asset, base first; a mod's copy equal to the base's first is left out. std::unordered_map> history; // The base's own sha1 for each asset, so a mod that merely carries an // unchanged copy can be told apart from one that changed it. @@ -120,6 +121,12 @@ struct BundleState { // Assets a mod added that a higher-priority mod's different asset of the // same name then replaced: the key, and the sha1 of the copy that went. std::vector> displaced; + + // Whether `sha1` is the game's first copy of `key`: the copy edits are combined onto. + [[nodiscard]] bool matches_game_copy(const std::string& key, const fb::Sha1& sha1) const { + const auto copies = history.find(key); + return copies != history.end() && copies->second.front().base && copies->second.front().sha1 == sha1; + } }; // The guids of the EBX a merged bundle ends up holding, to tell whether what a @@ -370,8 +377,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, else if (addition.asset.kind == fb::AssetKind::resource) companions.emplace(std::tuple{bundleName, addition.mod, lower(addition.asset.name)}, &addition); } - // Where an addition's payload already went in this superbundle's patch - // archive, by install chunk: every copy that is given it reads the one. + // The carried place each addition got in this superbundle, by install chunk. std::map, fb::BundleFileInfo> carriedAt; // The names any mod changed, to rule the rest out without a copy of each (name_hash). std::unordered_set changedNames, scriptNames; @@ -439,7 +445,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // Studio writes the manifest as the region's first file, but a mod built // by another tool or an older build may keep it elsewhere, so every small // file is a candidate; a placement outside the patch lives in the base. - if (region.inlineManifest.empty() && (needed || propagate) && !region.files.empty()) { + if (region.inlineManifest.empty() && (needed || propagate || !isBase) && !region.files.empty()) { std::string firstFailure; constexpr std::uint32_t largestManifest = 32u << 20; const auto expected = region.files.size() - 1; @@ -568,6 +574,43 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(overridden.size()) + " asset(s) take another mod's change, e.g. " + example); } + // The mod's own EBX whose type-reference fixups sit on another fixup's slot, which crash + // the game as it loads (Engine/Resource/ebx_fixup_repair.h): the repaired copy goes into + // the merged patch. The one in the mod's folder, by where it lies there and its sha1, is + // still what a combine reads and compares. + std::map repaired; + std::map> repairedFrom; + if (!isBase && (casBacked || !region.inlineManifest.empty())) { + auto manifest = casBacked ? std::move(casManifest) : fb::read_binary_bundle(region.inlineManifest); + std::size_t removed{}; + std::string example; + for (std::size_t index = 0; index < manifest.ebx.size(); ++index) { + const auto at = (casBacked ? 1 : 0) + index; + if (at >= region.files.size() || overridden.contains(at)) continue; + const auto& file = region.files[at]; + if (!file.location.patch) continue; // the game's own copy + auto& asset = manifest.ebx[index]; + try { + auto encoded = store.read(root, file.location, file.offset, file.size); + const auto before = asset.sha1; + const auto count = repair_ebx_payload(encoded, asset, gameRoot); + if (!count) continue; + repaired[at] = store.write(file.location.installChunk, manifestArchive, encoded); + repairedFrom.emplace(at, std::pair{file, before}); + removed += count; + if (example.empty()) example = asset.name; + } catch (const std::exception& error) { + report.notes.push_back(modName + ": " + bundle.name + ": " + asset.name + + ": its type references could not be checked (" + error.what() + ")"); + } + } + if (casBacked) casManifest = std::move(manifest); + else if (!repaired.empty()) region.inlineManifest = fb::write_binary_bundle(manifest); + if (!repaired.empty()) + report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(repaired.size()) + + " asset(s) listed " + std::to_string(removed) + " type reference(s) on another fixup's slot, " + "which crash the game as it loads; taken out of the merged copies, e.g. " + example); + } // Assets another mod added to the game's copy of this bundle: this copy // loads instead of the game's on this mod's levels, so it gets them too. std::vector additions; @@ -615,7 +658,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, auto placed = carriedAt.find({installChunk, addition}); if (placed == carriedAt.end()) placed = carriedAt.emplace(std::pair{installChunk, addition}, - store.write(installChunk, manifestArchive, addition->encoded)).first; + store.carry(installChunk, manifestArchive, addition->encoded)).first; additions.push_back({addition->asset, placed->second}); addedFrom.insert(addition->mod); } catch (const std::exception& error) { @@ -629,7 +672,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, } } // Nothing moved in this bundle: it passes through whole, as before. - if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty() && additions.empty()) + if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty() && repaired.empty() && + additions.empty()) casBacked = false; for (auto& file : region.files) { @@ -641,7 +685,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // below moves every later file along, and the pass over the bundle's assets // further down asks by where a file is then, not where it was. std::vector inPatch(region.files.size()); - for (const auto* moved : {&renumbered, &overridden}) + for (const auto* moved : {&renumbered, &overridden, &repaired}) for (const auto& [index, file] : *moved) { region.files[index] = file; inPatch[index] = true; @@ -748,8 +792,15 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // (the item collections every cosmetic mod adds its items to) fell back // to one mod's copy. const bool rewritten = firstFile + position < inPatch.size() && inPatch[firstFile + position]; - if (!rewritten && (entry.asset.kind == fb::AssetKind::ebx || - entry.asset.kind == fb::AssetKind::resource)) { + // A mod's copy that is the game's adds nothing: the game's copy is recorded, first. + const bool combinable = !rewritten && (entry.asset.kind == fb::AssetKind::ebx || entry.asset.kind == fb::AssetKind::resource); + if (const auto original = repairedFrom.find(firstFile + position); original != repairedFrom.end()) { + // Repaired above: what this mod changed is its copy as it ships it. + state.history[id].push_back({&root, original->second.first, original->second.second, + entry.asset.resourceMeta, isBase}); + } else if (combinable && !isBase && state.matches_game_copy(id, entry.asset.sha1)) { + ++report.unchangedCopies; + } else if (combinable) { // The contributor's own copy still sits at its original offset in // its own folder, which is where a later merge reads it from. auto unshifted = entry.file; @@ -757,7 +808,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, if (const auto* directory = store.find_directory(unshifted.location.installChunk)) unshift(*placement, *directory, unshifted.location.archive, unshifted.offset); state.history[id].push_back( - {root, unshifted, entry.asset.sha1, entry.asset.resourceMeta, isBase}); + {&root, unshifted, entry.asset.sha1, entry.asset.resourceMeta, isBase}); } if (isBase) state.baseSha.insert_or_assign(id, entry.asset.sha1); const auto at = state.seen.find(id); @@ -788,10 +839,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // holds one asset per name, so the lower-priority mod's is gone, // and whatever of that mod's still names it by guid finds nothing // (an item list entry is dropped for that further down). - const bool shaderTable = entry.asset.kind == fb::AssetKind::resource && - (entry.asset.resourceType == fb::shader::programLookupType || - entry.asset.resourceType == fb::shader::textureLookupType); - if (entry.asset.kind != fb::AssetKind::chunk && !shaderTable && !rewritten && previous.mod && + if (entry.asset.kind != fb::AssetKind::chunk && !rewritten && previous.mod && previous.mod != owner && previous.asset.sha1 != entry.asset.sha1 && !state.baseSha.contains(at->first)) { auto& [count, example] = replaced[*previous.mod]; @@ -826,7 +874,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // base install, whichever mod's copy referenced it. const auto decode = [&](const Contribution& from) { return fb::decode_cas( - store.read(from.file.location.patch ? from.root : baseRoot, + store.read(from.file.location.patch ? *from.root : baseRoot, from.file.location, from.file.offset, from.file.size), {gameRoot}); }; @@ -849,11 +897,6 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, const Contribution* baseCopy{}; for (const auto& contribution : found->second) if (contribution.base) { baseCopy = &contribution; break; } - // Custom cosmetic bundles also carry the global shader tables, - // although the game's copy of that bundle does not. Use the - // highest-priority contributor as the starting table there. - const bool addedShaderTable = !baseCopy && shaderTable; - if (addedShaderTable) baseCopy = &found->second.back(); if (!baseCopy) continue; // Both mods rebuild the whole bundle, so only the assets whose // content actually moved away from the base are contested. @@ -864,8 +907,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // An item list is looked at even when one mod alone changed it: // what its entries name has to be in the bundle either way. const auto named = entry.asset.kind == fb::AssetKind::ebx && item_list_name(entry.asset.name); - if (edits.size() < 2 && !(named && edits.size() == 1) && - !(addedShaderTable && !edits.empty())) continue; + if (edits.size() < 2 && !(named && edits.size() == 1)) continue; try { const auto baseBytes = decode(*baseCopy); std::vector> editBytes; @@ -894,12 +936,13 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, if (!count++) example = clash.path + " and " + clash.holder; } for (const auto& [index, clash] : shadowed) - report.notes.push_back(path_utf8(edits[index]->root.filename()) + ": " + + report.notes.push_back(path_utf8(edits[index]->root->filename()) + ": " + std::to_string(clash.first) + " preset(s) share a short name with another mod's, e.g. " + clash.second + "; they are registered by their full path only"); } else if (shaderTable) { - // Keep each private program and its backing arrays, then - // remap material rows into the combined program table. + // Each mod aliases new material keys onto shader programs + // and texture sets the base already ships, so the union of + // the rows they added is exactly right. std::vector tables; for (std::size_t index = 0; index < editBytes.size(); ++index) tables.push_back({editBytes[index], edits[index]->resourceMeta}); @@ -907,6 +950,10 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, const auto table = entry.asset.resourceType == fb::shader::programLookupType ? fb::shader::merge_program_lookup(baseTable, tables) : fb::shader::merge_texture_lookup(baseTable, tables); + if (table.outside) + report.notes.push_back(entry.asset.name + ": " + std::to_string(table.outside) + + " material row(s) left out: they point past the shader programs the game ships, " + "which the game would read outside its shader database"); if (!table.added) continue; rebuilt = table.resource; entry.asset.resourceMeta = table.resourceMeta; @@ -954,7 +1001,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, fb::ebx::MergeSummary summary; auto combined = fb::ebx::merge_documents(baseDocument, pointers, &summary, itemList ? inBundle : notReplaced); - const auto mod_of = [&](std::size_t edit) { return path_utf8(edits[edit]->root.filename()); }; + const auto mod_of = [&](std::size_t edit) { return path_utf8(edits[edit]->root->filename()); }; if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries) { // One mod's copy with nothing to leave out stays as that mod wrote it. if (edits.size() < 2) continue; @@ -1013,6 +1060,12 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, ", which no enabled mod adds to the bundle")); } } + if (entry.asset.kind == fb::AssetKind::ebx) + if (auto repaired = fb::ebx::repair_type_fixups(rebuilt)) { + rebuilt = std::move(repaired->bytes); + report.notes.push_back(entry.asset.name + ": " + std::to_string(repaired->removed) + + " type reference(s) on another fixup's slot taken out of the merged copy"); + } const auto placement = store.write(state.manifestChunk, manifestArchive, fb::encode_cas(rebuilt, {gameRoot})); entry.file = placement; diff --git a/Engine/Vfs/mod_merge_files.cpp b/Engine/Vfs/mod_merge_files.cpp index 4a23ab62..80382d64 100644 --- a/Engine/Vfs/mod_merge_files.cpp +++ b/Engine/Vfs/mod_merge_files.cpp @@ -1,6 +1,8 @@ #include "mod_merge_internal.h" #include "Engine/Core/Json/json.h" #include "Engine/Core/Platform/path_text.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_fixup_repair.h" #include #include @@ -142,6 +144,20 @@ fb::Sha1 sha1_of(std::span bytes) { return digest; } +std::size_t repair_ebx_payload(std::vector& encoded, fb::BundleAsset& asset, const fs::path& gameRoot) { + std::optional repaired; + try { + repaired = fb::ebx::repair_type_fixups(fb::decode_cas(encoded, {gameRoot})); + } catch (const std::exception&) { + return 0; + } + if (!repaired) return 0; + encoded = fb::encode_cas(repaired->bytes, {gameRoot}); + asset.sha1 = sha1_of(repaired->bytes); + asset.originalSize = repaired->bytes.size(); + return repaired->removed; +} + PlacementRecord read_placements(const fs::path& file) { PlacementRecord record; std::ifstream input(file, std::ios::binary); diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index df9cbf52..0b1344b9 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -80,6 +80,12 @@ bool unshift(const ArchivePlacement& placement, std::string_view directory, std: fb::Sha1 sha1_of(std::span bytes); +// A mod's EBX as stored in cas, with the type-reference fixups that would crash the game's +// load taken out (Engine/Resource/ebx_fixup_repair.h): the payload is encoded again and the +// asset's sha1 and size follow it. How many came out; with none, or a payload that does not +// decode, both stay as they were. +std::size_t repair_ebx_payload(std::vector& encoded, fb::BundleAsset& asset, const fs::path& gameRoot); + // Where the launch's merge put every installed mod's archives, and which // superbundle TOCs the patch holds: Mods/.reskate/reskate-placements.json. inline constexpr wchar_t placements_file[] = L"reskate-placements.json"; @@ -132,11 +138,19 @@ public: // own has been settled, settle() appends what it holds to the real archives, and // settled() then moves a placement that points into it to where that went. static constexpr std::uint16_t waiting_archive = 0xFFFF; + // A carried payload, by its index in the waiting store's list: settle() writes it once per install chunk. + static constexpr std::uint16_t carried_archive = 0xFFFE; [[nodiscard]] CasStore waiting() const; - [[nodiscard]] bool holding() const noexcept { return !held_.empty(); } - [[nodiscard]] static bool waits(const fb::CasIdentifier& location) noexcept { - return location.patch && location.archive == waiting_archive; + [[nodiscard]] bool holding() const noexcept { return !held_.empty() || !carried_.empty(); } + [[nodiscard]] static bool placeholder(std::uint16_t archive) noexcept { + return archive == waiting_archive || archive == carried_archive; } + [[nodiscard]] static bool placeholder(const fb::CasIdentifier& location) noexcept { + return location.patch && placeholder(location.archive); + } + // A waiting store only. The bytes must stay valid until settle(). + [[nodiscard]] fb::BundleFileInfo carry(std::uint32_t installChunk, std::uint16_t archive, + std::span encoded); void settle(CasStore& waiting); // True when the placement was one of this waiting store's; it now names the archive. bool settled(fb::CasIdentifier& location, std::uint32_t& offset) const; @@ -160,6 +174,15 @@ private: bool waiting_{}; std::optional archive_; std::map held_; + // Carried payloads (waiting store) and where each landed (settling store). + struct Carried { + std::uint32_t installChunk{}; + fb::Sha1 sha1; + std::span bytes; + fb::BundleFileInfo at; + }; + std::vector carried_; + std::map, fb::BundleFileInfo> carriedAt_; }; using ArchiveUse = std::set>; diff --git a/Engine/Vfs/mod_merge_overrides.cpp b/Engine/Vfs/mod_merge_overrides.cpp index 295576ad..ec5412fd 100644 --- a/Engine/Vfs/mod_merge_overrides.cpp +++ b/Engine/Vfs/mod_merge_overrides.cpp @@ -74,7 +74,8 @@ struct Change { struct Script { std::string name; // lower case fb::Sha1 game; // the game's copy it replaces - AssetOverride value; + std::optional value; // none: it could not be read, and `note` says so + std::size_t note{}; // its place in the mod's notes }; // What one mod's files say, read without regard to any other mod: the part of the work that // is reading and decoding, done for several mods at once. What it means beside the mods @@ -146,12 +147,18 @@ ModRead read_mod(const Mod& mod, const RelativeFiles& files, const CasStore& sto const auto at = modListing->first + index; if (at >= modListing->files.size()) continue; const auto& file = modListing->files[at]; + // As the merge places it in the mod's own bundle: repaired when it would + // crash the game's load (repair_ebx_payload). + auto repairedAsset = asset; const auto payload = [&] { - return store.read(file.location.patch ? mod.directory : baseRoot, - file.location, file.offset, file.size); + auto encoded = store.read(file.location.patch ? mod.directory : baseRoot, + file.location, file.offset, file.size); + if (file.location.patch) repair_ebx_payload(encoded, repairedAsset, gameRoot); + return encoded; }; if (game_copy == original.end()) { - candidates.push_back({bundleName, {mod.name, asset, payload(), tocKey}}); + auto encodedPayload = payload(); + candidates.push_back({bundleName, {mod.name, repairedAsset, std::move(encodedPayload), tocKey}}); continue; } auto encodedPayload = payload(); @@ -163,8 +170,8 @@ ModRead read_mod(const Mod& mod, const RelativeFiles& files, const CasStore& sto if (const auto gameAt = gameListing->first + game_copy->second.second; gameAt < gameListing->files.size()) gameFile = gameListing->files[gameAt]; - changes.push_back({bundleName, name, asset.name, game_copy->second.first, asset.sha1, - asset.originalSize, std::move(encodedPayload), gameFile}); + changes.push_back({bundleName, name, asset.name, game_copy->second.first, repairedAsset.sha1, + repairedAsset.originalSize, std::move(encodedPayload), gameFile}); } for (std::size_t index = 0; index < modListing->manifest.resources.size(); ++index) { const auto& asset = modListing->manifest.resources[index]; @@ -176,14 +183,23 @@ ModRead read_mod(const Mod& mod, const RelativeFiles& files, const CasStore& sto // (A higher mod's version of the script, or this mod's own from another // bundle, wins when the mods are settled: read here either way.) if (std::ranges::any_of(scripts, [&](const Script& have) { - return have.name == name && have.game == originalScript->second->sha1; })) continue; + return have.name == name && have.game == originalScript->second->sha1 && have.value; })) continue; const auto at = modListing->first + modListing->manifest.ebx.size() + index; if (at >= modListing->files.size()) continue; const auto& file = modListing->files[at]; - scripts.push_back({name, originalScript->second->sha1, - AssetOverride{mod.name, asset.sha1, asset.originalSize, - store.read(file.location.patch ? mod.directory : baseRoot, - file.location, file.offset, file.size), asset}}); + try { + scripts.push_back({name, originalScript->second->sha1, + AssetOverride{mod.name, asset.sha1, asset.originalSize, + store.read(file.location.patch ? mod.directory : baseRoot, + file.location, file.offset, file.size), asset}}); + } catch (const std::exception& failed) { + // Read whether or not it will be wanted, so a failure is only this + // script's: where a higher mod's version stands, it is nobody's and + // the note goes (settled below). The rest of the TOC is read on. + notes.push_back(mod.name + ": " + relative + + ": its changes could not be read for other mods' copies (" + failed.what() + ")"); + scripts.push_back({name, originalScript->second->sha1, std::nullopt, notes.size() - 1}); + } } // Resources the mod adds, kept to go with an added EBX of the same name // (a wave's sound-bank resource registers the wave with the audio system). @@ -273,7 +289,6 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, if (read.failure) std::rethrow_exception(read.failure); auto& [candidates, companions, unreadable, addedIn, newChunks, imported, changes, scripts, notes, failure] = read; std::size_t changed{}; - report.notes.insert(report.notes.end(), std::make_move_iterator(notes.begin()), std::make_move_iterator(notes.end())); // A higher mod's version of an asset, which this mod's change is combined with // below, may name what this mod adds. for (const auto& change : changes) @@ -284,10 +299,16 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, } catch (const std::exception&) {} for (auto& script : scripts) { auto& versions = out.scripts[script.name]; - if (versions.contains(script.game)) continue; - versions.emplace(script.game, std::move(script.value)); + if (versions.contains(script.game)) { + if (!script.value) notes[script.note].clear(); // not wanted: as if never read + continue; + } + if (!script.value) continue; // wanted, and noted + versions.emplace(script.game, std::move(*script.value)); ++changed; } + for (auto& note : notes) + if (!note.empty()) report.notes.push_back(std::move(note)); // One addition per kind and name in a bundle, the highest-priority mod's. Two // mods adding the same asset is no clash; two different assets under one name // is, and only one of them can be what a copy gets: say whose, so a song or an diff --git a/Engine/Vfs/mod_store_copies.cpp b/Engine/Vfs/mod_store_copies.cpp index 7a436bc5..314e540a 100644 --- a/Engine/Vfs/mod_store_copies.cpp +++ b/Engine/Vfs/mod_store_copies.cpp @@ -671,10 +671,16 @@ StoreCopies check_store_copies(const Catalog& catalog, const StoreItem& sold, st work(); } for (std::size_t index = 0; index < asked.size(); ++index) { + StoreCopies::Added keys{asked[index]->name, {}}; + for (const auto& item : reads[index].items) keys.keys.push_back(lower(item.key)); + std::sort(keys.keys.begin(), keys.keys.end()); + keys.keys.erase(std::unique(keys.keys.begin(), keys.keys.end()), keys.keys.end()); + result.added.push_back(std::move(keys)); scan.say(said[index]); result.mod_meshes += reads[index].meshes.size(); if (!reads[index].items.empty() || !reads[index].meshes.empty()) added.push_back(std::move(reads[index])); } + result.read = true; } if (added.empty()) return result; // The costumes' meshes are only read when a mod ships a mesh at all. diff --git a/Engine/Vfs/mod_store_copies.h b/Engine/Vfs/mod_store_copies.h index 694356a7..179355ae 100644 --- a/Engine/Vfs/mod_store_copies.h +++ b/Engine/Vfs/mod_store_copies.h @@ -139,6 +139,15 @@ struct StoreCopies { StoreMeshes::Closest closest; }; std::vector meshes_seen; + // Every item each mod adds or changes (lower-case keys), copies or not, in the catalogue's + // order (highest priority first): which mod an item is from (mod_item_sources.h). Empty for + // a mod with none; `read` false when the check could not run. + struct Added { + std::string mod; + std::vector keys; + }; + std::vector added; + bool read{}; // How many of the game's own items the mods' were held against; none when no mod adds an item. std::size_t game_items{}; // And how many of the mods' meshes against how many of the game's costumes. diff --git a/Extension/Assets/Test/map_download_timeout_tests.cpp b/Extension/Assets/Test/map_download_timeout_tests.cpp new file mode 100644 index 00000000..fd8572f8 --- /dev/null +++ b/Extension/Assets/Test/map_download_timeout_tests.cpp @@ -0,0 +1,647 @@ +#include "Extension/Assets/map_download.h" +#include "Extension/Assets/live_mods.h" +#include "Extension/Multiplayer/Session/monotonic_clock.h" +#include "Extension/Multiplayer/Net/protocol.h" +#include "Engine/Core/Log/logging.h" +#include "Engine/Vfs/https_download.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Launcher/mod_manager.h" +#include "Extension/UI/Overlay/overlay.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fixture { +namespace fs = std::filesystem; +namespace policy = dingosdk::multiplayer::map_fetch_policy; +fs::path root; +std::atomic clock{1000000}; +constexpr auto use_download_deadline = static_cast(-1); +std::atomic active_deadline{use_download_deadline}; +std::atomic downloading{}, installing{}, release_download{}, release_install{}, merging{}; +std::atomic fetching{true}, held{}, observed_cancel{}, installed_cancel{}; +std::atomic downloads{}, installs{}, applies{}, stops{}, rejoins{}; +std::atomic queued_stop_deadline{}; +bool block_download{}, block_install{}, slow{}, transfer_error{}, fallback{}; +dingosdk::MultiplayerModel session; +std::mutex messages_mutex, gate_mutex; +std::condition_variable gate; +std::vector notices; +std::vector http_budgets; +fs::path archive; + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} +template +void wait(Predicate predicate, const char* message) { + const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!predicate()) { + if (std::chrono::steady_clock::now() >= end) throw std::runtime_error(message); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } +} +bool noticed(std::string_view text) { + std::lock_guard lock(messages_mutex); + for (const auto& notice : notices) + if (notice.find(text) != std::string::npos) return true; + return false; +} +} + +namespace dingosdk::multiplayer { +std::uint64_t fake_now_us() noexcept { return fixture::clock.load(); } +} + +// Compile the actual lifecycle, not a parallel model. Preincluded headers keep these +// substitutions local to this translation unit; archives also stay in the test fixture. +DWORD test_get_temp_path(DWORD capacity, LPWSTR output) { + const auto path = fixture::root.wstring() + L"\\"; + if (path.size() >= capacity) return 0; + std::copy(path.begin(), path.end(), output); + output[path.size()] = L'\0'; + return static_cast(path.size()); +} +#define now_us fake_now_us +#define GetTempPathW test_get_temp_path +#include "../map_download.cpp" +#undef GetTempPathW +#undef now_us + +namespace dingosdk::https { +std::optional get_text(std::wstring_view, std::uint64_t, std::uint32_t, + const wchar_t*, Download* result, Watch* watch) { + fixture::require(watch && watch->deadline_us.load() == map_download::deadline.load(), + "metadata shares the session deadline"); + if (result) *result = {true, 200, 0}; + return R"({"namespace":"Team","name":"Map","community_listings":[{"community":"reskate","categories":["Maps"]}], + "latest":{"version_number":"2.0.0","description":"Test map"}})"; +} +Download get(std::wstring_view, const std::filesystem::path& destination, + std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t*, Watch* watch) { + using namespace fixture; + require(max_bytes == (8ull << 30), "size restriction retained"); + require(timeout_seconds == 4 * 60 * 60, "HTTP keeps its own four hour limit, not the session's"); + archive = destination; + const auto request = ++downloads; + http_budgets.push_back(watch->deadline_us.load()); + std::ofstream(destination, std::ios::binary) << "partial archive"; + watch->received.store(10); + watch->total.store(100); + downloading.store(true); + if (block_download) { + std::unique_lock lock(gate_mutex); + gate.wait(lock, [] { return release_download.load(); }); + } + if (slow) { + // Real download progress still cannot extend either absolute deadline. + fixture::clock.fetch_add(10 * 60 * policy::second_us); + watch->received.store(90); + } + observed_cancel.store(watch->cancel.load()); + if (watch->cancel.load()) return {false, 0, ERROR_CANCELLED}; + if (fallback && request == 1) { + fixture::clock.fetch_add(60 * policy::second_us); + return {false, 404, 0}; + } + if (transfer_error) return {false, 0, ERROR_WINHTTP_TIMEOUT}; + watch->received.store(100); + return {true, 200, 0}; +} +} + +namespace dingosdk::mods { +std::filesystem::path engine_data_root() noexcept { return fixture::root; } +ModList scan_mods(const std::filesystem::path&) noexcept { return {}; } +void save_mod_order(const std::filesystem::path&, const std::vector&) {} +} +namespace dingosdk::launcher_mods { +std::string install(const std::filesystem::path& root, const std::filesystem::path& source, bool, + const Progress& progress, const std::atomic& cancel, const InstallOptions& options) { + using namespace fixture; + require(fs::exists(source), "installer receives completed archive"); + ++installs; + installing.store(true); + if (block_install) { + std::unique_lock lock(gate_mutex); + gate.wait(lock, [] { return release_install.load(); }); + } + progress(0.5f); + installed_cancel.store(cancel.load()); + if (cancel.load()) throw std::runtime_error("cancelled"); + fs::create_directories(root / options.folder); + std::ofstream(root / options.folder / "reskate-levels.json") << "{}"; + return options.folder; +} +void remove(const std::filesystem::path&, const std::string&) {} +} +namespace dingosdk::live_mods { +bool only_maps(const std::filesystem::path&) { return true; } +bool busy() noexcept { return fixture::merging.load(); } +std::string apply(bool reload) { + fixture::require(!reload, "map apply must not reload prematurely"); + ++fixture::applies; + fixture::merging.store(true); + return "applying"; +} +Progress progress() { return {1, 2, "map"}; } +std::vector applied_mods() { return {"Team-Map"}; } +std::string status() { return "applied"; } +} +namespace dingosdk::multiplayer { +MultiplayerModel model() { return fixture::session; } +bool fetching_map() noexcept { return fixture::fetching.load(); } +std::uint64_t map_fetch_deadline() noexcept { + const auto active = fixture::active_deadline.load(); + return active == fixture::use_download_deadline ? map_download::deadline.load() : active; +} +void hold_map_load(bool hold, std::string_view package) noexcept { + fixture::held.store(hold); + if (hold && package != "Team-Map-1.0.0") std::abort(); +} +bool queue_command(std::string_view action, std::string_view argument, std::string_view password) { + if (action == "stop") ++fixture::stops; + if (action == "stop-map-fetch") { + const auto expected = map_download::deadline.load(); + fixture::require(expected && argument == std::to_string(expected) && password.empty(), + "map stop carries the captured originating attempt deadline"); + fixture::queued_stop_deadline.store(expected); + ++fixture::stops; + } + if (action == "rejoin") ++fixture::rejoins; + return true; +} +std::string map_package_name(std::string_view folder, std::string_view version) { + return std::string(folder) + "-" + std::string(version); +} +} +namespace dingosdk::overlay { +void set_map_download_icon(const std::string&) noexcept {} +void notify(NoticeLevel, std::string title, std::string text) noexcept { + std::lock_guard lock(fixture::messages_mutex); + fixture::notices.push_back(title + ": " + text); +} +} +namespace dingosdk::logging { +bool enabled(Level) noexcept { return false; } +void write(Level, Context, Channel, std::string_view) noexcept {} +void write(Level, Channel, std::string_view) noexcept {} +} + +namespace fixture { +namespace download = dingosdk::map_download; +using Stage = download::Stage; + +void drain() { + release_download.store(true); + release_install.store(true); + gate.notify_all(); + wait([] { return download::workers.load() == 0; }, "workers did not drain"); +} +void reset() { + drain(); + require(download::view().stage == Stage::idle, "previous lifecycle did not finish"); + fs::remove_all(root / "Mods"); + clock.store(1000000); + active_deadline.store(use_download_deadline); + downloading.store(false); + installing.store(false); + release_download.store(false); + release_install.store(false); + merging.store(false); + fetching.store(true); + held.store(false); + observed_cancel.store(false); + installed_cancel.store(false); + downloads.store(0); + installs.store(0); + applies.store(0); + stops.store(0); + queued_stop_deadline.store(0); + rejoins.store(0); + block_download = block_install = slow = transfer_error = fallback = false; + session = {}; + session.active = true; + session.connected = false; + notices.clear(); + http_budgets.clear(); + archive.clear(); + download::last_failed.clear(); +} +void offer(std::uint64_t deadline = 0, bool update = false) { + dingosdk::multiplayer::MapNeed need{"Team-Map-1.0.0", "Test map", true, false, + update ? "0.9.0" : "", deadline}; + download::offer(need); + wait([] { return download::asking(); }, "metadata did not reach confirmation"); + wait([] { return download::workers.load() == 0; }, "metadata worker did not drain"); +} +void start_apply() { + wait([] { return download::view().stage == Stage::applying; }, "did not install and reach applying"); + drain(); + download::tick(); + require(applies.load() == 1 && held.load(), "apply starts once and holds session loading"); +} +void applying() { + download::answer(true); + start_apply(); +} +void joined() { + merging.store(false); + download::tick(); + require(download::view().stage == Stage::joining, "completed merge advances to joining"); + fetching.store(false); + session.connected = true; + clock.fetch_add(2500000); + download::tick(); + require(download::view().stage == Stage::idle, "joined session clears lifecycle"); + require(!held.load() && stops.load() == 0 && rejoins.load() == 0, "normal join does not leave or rejoin"); +} +void idle(const char* reason, unsigned expected_stops = 1) { + require(download::view().stage == Stage::idle, reason); + drain(); + require(stops.load() == expected_stops, "failure only stops its originating waiting session"); + if (expected_stops) + require(queued_stop_deadline.load() == download::deadline.load(), + "queued map stop identifies the originating attempt for dispatch-time ownership validation"); + download::tick(); + require(held.load() == (merging.load() && download::apply_started.load()), + "failure holds map loading only while an uncancellable live merge drains"); + require(archive.empty() || !fs::exists(archive), "partial archive removed"); +} + +void tests() { + reset(); + offer(); + applying(); + require(downloads.load() == 1 && installs.load() == 1, "success downloads and installs exactly once"); + joined(); + + reset(); + merging.store(true); + offer(); + download::answer(true); + wait([] { return download::view().stage == Stage::applying; }, "unrelated live merge blocked map offer"); + drain(); + download::tick(); + require(applies.load() == 0 && held.load(), "map apply waits for unrelated live merge without rejecting offer"); + merging.store(false); + download::tick(); + require(applies.load() == 1, "map apply starts after unrelated merge drains"); + joined(); + + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "download did not start for duplicate confirmation"); + clock.fetch_add(policy::second_us); + download::answer(true); + require(downloads.load() == 1 && download::workers.load() == 1, "only winning confirmation claims a worker"); + release_download.store(true); + gate.notify_all(); + start_apply(); + joined(); + + reset(); + block_download = true; + slow = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "slow download did not start"); + clock.fetch_add(10 * 60 * policy::second_us); + download::watch.received.store(50); + download::tick(); + require(download::view().stage == Stage::downloading && download::view().received == 50 && held.load(), + "progressing transfer remains active beyond old five minute deadline"); + release_download.store(true); + gate.notify_all(); + start_apply(); + require(clock.load() > 5 * 60 * policy::second_us, "slow progressing transfer exceeds former five minute limit"); + joined(); + + reset(); + transfer_error = true; + offer(); + download::answer(true); + wait([] { return download::view().stage == Stage::idle; }, "HTTP timeout did not fail"); + idle("transfer timeout"); + require(installs.load() == 0 && noticed("timed out"), "HTTP timeout cannot install"); + transfer_error = false; + stops.store(0); + offer(); + applying(); + require(downloads.load() == 2, "timeout allows same package retry"); + joined(); + + // The session stops waiting (its deadline passes and it ends itself) while the map downloads: + // the download is not thrown away, the map installs and is applied, and the player is told to + // join again (the session would not take them back past its deadline). + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "download did not start"); + clock.store(download::deadline.load()); + session.active = false; + fetching.store(false); + active_deadline.store(0); + download::tick(); + require(download::view().stage == Stage::downloading && !download::watch.cancel.load() && noticed("still downloading"), + "session expiry leaves an active download running and says so"); + release_download.store(true); + gate.notify_all(); + start_apply(); + require(!observed_cancel.load() && installs.load() == 1, "download that outlived its session installs"); + merging.store(false); + download::tick(); + require(download::view().stage == Stage::joining, "detached download is applied"); + clock.fetch_add(2500000); + download::tick(); + require(download::view().stage == Stage::idle && rejoins.load() == 0 && stops.load() == 0 && noticed("Map installed"), + "detached download ends installed, without rejoining an expired session"); + + reset(); + block_install = true; + offer(); + download::answer(true); + wait([] { return installing.load(); }, "install did not start"); + clock.store(download::deadline.load()); + download::tick(); + drain(); + require(!installed_cancel.load() && installs.load() == 1, "session expiry does not cancel an install under way"); + wait([] { return download::view().stage == Stage::applying; }, "expired install did not go on to applying"); + download::tick(); + require(applies.load() == 1, "expired install is still applied"); + merging.store(false); + download::tick(); + fetching.store(false); + session.active = false; + clock.fetch_add(2500000); + download::tick(); + require(download::view().stage == Stage::idle && rejoins.load() == 0 && noticed("Map installed"), + "map installed after the session's deadline is not rejoined"); + + // A live merge cannot be stopped: a fresh need waits for one left behind by a cancelled attempt. + reset(); + offer(); + applying(); + active_deadline.store(download::deadline.load() + 1); // another session's own attempt + download::tick(); + idle("apply cancelled by another session", 0); + require(merging.load() && rejoins.load() == 0, "cancelled atomic merge must not join"); + download::offer({"Team-Map-1.0.0", "Test map", true}); + require(download::view().stage == Stage::idle && noticed("still finishing"), "busy live merge blocks retry"); + merging.store(false); + download::tick(); + require(rejoins.load() == 0 && !held.load(), "finished cancelled apply cannot rejoin and releases load hold"); + + reset(); + offer(); + download::answer(false); + idle("decline"); + require(downloads.load() == 0 && noticed("cancelled"), "decline never downloads"); + + reset(); + offer(); + clock.store(download::ask_until.load() - 1); + download::tick(); + require(download::asking(), "confirmation remains open one microsecond before deadline"); + clock.fetch_add(1); + download::answer(true); + idle("confirmation exact boundary"); + require(downloads.load() == 0 && noticed("two minutes"), "late click cannot bypass confirmation timeout"); + + reset(); + offer(); + clock.store(download::ask_until.load()); + download::tick(); + idle("confirmation tick timeout"); + require(downloads.load() == 0 && noticed("two minutes"), "tick expires unanswered prompt"); + + reset(); + const auto short_session = clock.load() + policy::confirmation_us / 2; + offer(short_session); + require(download::ask_until.load() == short_session, "confirmation is capped by existing session deadline"); + clock.store(short_session); + download::answer(true); + idle("shorter session confirmation boundary"); + require(downloads.load() == 0 && noticed("timed out"), "confirmation never extends existing session budget"); + + reset(); + offer(clock.load() + 1); + download::answer(true); + wait([] { return download::view().stage == Stage::applying; }, "download confirmed in the session's last microsecond did not start"); + require(downloads.load() == 1 && http_budgets[0] == 0, "a download confirmed in time starts, with no session deadline on it"); + drain(); + download::tick(); + merging.store(false); + download::tick(); + fetching.store(false); + session.active = false; + clock.fetch_add(2500000); + download::tick(); + require(download::view().stage == Stage::idle && noticed("Map installed"), "late download still ends installed"); + + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "HTTP did not start for cancel race"); + download::answer(false); + require(download::watch.cancel.load(), "cancel flag is immediate"); + download::tick(); + require(held.load(), "cancelled HTTP keeps map loading held until worker drains"); + download::offer({"Team-Map-1.0.0", "Test map", true}); + require(downloads.load() == 1 && noticed("still finishing"), "retry cannot reuse a draining HTTP worker"); + drain(); + require(observed_cancel.load() && !fs::exists(archive) && installs.load() == 0, "cancel cleans archive without install"); + // The same package is retriable after a user cancellation. + block_download = false; + offer(); + applying(); + require(downloads.load() == 2, "retry fetches same package again"); + // The original user cancellation stopped its waiting attempt. + stops.store(0); + joined(); + + reset(); + fallback = true; + offer(); + applying(); + require(downloads.load() == 2 && http_budgets.size() == 2, "404 selects newest fallback"); + require(http_budgets[0] == 0 && http_budgets[1] == 0, "neither request is held to the session's deadline"); + joined(); + + reset(); + fallback = true; + offer(0, true); + download::answer(true); + wait([] { return download::view().stage == Stage::idle; }, "missing host version did not fail"); + idle("host version consistency"); + require(downloads.load() == 1 && installs.load() == 0, "version mismatch never falls back to another host version"); + + reset(); + offer(); + const auto original_deadline = download::deadline.load(); + download::offer({"Team-Other-1.0.0", "Other map", true}); + require(download::asking() && download::deadline.load() == original_deadline && stops.load() == 1, + "host changing maps does not replace in-flight request or extend its deadline"); + stops.store(0); + download::answer(false); + idle("host map change"); + + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "download did not start for session replacement"); + active_deadline.store(0); + download::tick(); + idle("session identity changed", 0); + require(observed_cancel.load() && installs.load() == 0, + "download cancels when the active session no longer owns its deadline"); + + reset(); + offer(); + active_deadline.store(0); + fetching.store(false); + session.connected = true; + download::tick(); + require(download::view().stage == Stage::idle && !held.load() && downloads.load() == 0 && + stops.load() == 0 && rejoins.load() == 0, + "abandoned confirmation cancels without holding or stopping a fresh unrelated ordinary join"); + download::answer(true); + require(downloads.load() == 0, "late confirmation cannot start download for abandoned session"); + + reset(); + offer(); + const auto abandoned_deadline = download::deadline.load(); + clock.fetch_add(policy::second_us); + const auto fresh_deadline = clock.load() + policy::session_us; + active_deadline.store(fresh_deadline); + download::offer({"Team-Map-1.0.0", "Test map", true, false, "", fresh_deadline}); + wait([] { return download::asking(); }, "fresh same-package need was not retaken"); + drain(); + require(download::deadline.load() == fresh_deadline && fresh_deadline != abandoned_deadline && stops.load() == 0, + "fresh same-package attempt replaces stale confirmation without stopping its new session"); + applying(); + joined(); + + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "old download did not start for queued replacement"); + clock.fetch_add(policy::second_us); + const auto queued_deadline = clock.load() + policy::session_us; + active_deadline.store(queued_deadline); + download::offer({"Team-Map-1.0.0", "Test map", true, false, "", queued_deadline}); + download::tick(); + require(download::view().stage == Stage::idle && download::watch.cancel.load() && + downloads.load() == 1 && stops.load() == 0 && download::pending && + download::pending->deadline_us == queued_deadline, + "fresh offer queues while cancelled worker drains without stopping replacement session"); + drain(); + require(observed_cancel.load() && installs.load() == 0 && !fs::exists(archive), + "old worker cleans partial archive before queued replacement can use its globals"); + block_download = false; + download::tick(); + wait([] { return download::asking(); }, "queued same-package need was not offered after drain"); + drain(); + require(download::deadline.load() == queued_deadline && !download::pending && stops.load() == 0, + "queued attempt keeps its fresh deadline"); + applying(); + require(downloads.load() == 2, "queued fresh attempt starts exactly one new transfer"); + joined(); + + reset(); + block_download = true; + offer(); + download::answer(true); + wait([] { return downloading.load(); }, "old download did not start for stale pending need"); + clock.fetch_add(policy::second_us); + const auto superseded_deadline = clock.load() + policy::session_us; + active_deadline.store(superseded_deadline); + download::offer({"Team-Map-1.0.0", "Test map", true, false, "", superseded_deadline}); + active_deadline.store(0); + drain(); + download::tick(); + require(download::view().stage == Stage::idle && !download::pending && downloads.load() == 1 && + stops.load() == 0 && !held.load(), + "queued need is discarded when its replacement session also ends before worker drain"); + + reset(); + offer(); + applying(); + merging.store(false); + download::tick(); + clock.store(download::join_at.load() + policy::installed_load_us - 1); + download::tick(); + require(download::view().stage == Stage::joining, "installed map load retains its last microsecond"); + clock.fetch_add(1); + download::tick(); + idle("installed map load timeout"); + require(noticed("still cannot load"), "installed map reports load failure at 30 seconds"); + + reset(); + offer(); + applying(); + merging.store(false); + download::tick(); + fetching.store(false); + session.active = false; + clock.fetch_add(2500000); + download::tick(); + require(download::view().stage == Stage::idle && rejoins.load() == 1 && stops.load() == 0, + "successful install rejoins when the original session ended independently"); + + reset(); + offer(); + applying(); + clock.store(download::deadline.load() - 2); + merging.store(false); + download::tick(); + require(download::view().stage == Stage::joining, "near-deadline apply reaches joining"); + clock.fetch_add(1); + session.connected = true; + fetching.store(false); + active_deadline.store(0); + clock.fetch_add(2); + download::tick(); + require(download::view().stage == Stage::idle && !held.load() && stops.load() == 0 && + rejoins.load() == 0 && !noticed("timed out"), + "join completed within session budget retires successfully even when next tick is after deadline"); + + reset(); + download::tick(); + require(!held.load() && downloads.load() == 0 && stops.load() == 0 && rejoins.load() == 0, + "ordinary join without download is untouched"); +} +} + +int main(int argc, char** argv) { + try { + fixture::require(argc == 2, "pass a fixture directory"); + fixture::root = std::filesystem::absolute(argv[1]); + std::filesystem::create_directories(fixture::root); + fixture::tests(); + fixture::drain(); + std::filesystem::remove_all(fixture::root); + std::cout << "Production map download timeout lifecycle tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + // A bounded wait failure can leave detached production workers alive. Do not + // destroy their fixture globals while they may still be using them. + std::_Exit(1); + } +} diff --git a/Extension/Assets/live_mods.cpp b/Extension/Assets/live_mods.cpp index 59d9f5c3..94dd5601 100644 --- a/Extension/Assets/live_mods.cpp +++ b/Extension/Assets/live_mods.cpp @@ -8,6 +8,7 @@ #include "Engine/Core/Platform/path_text.h" #include "Engine/Game/Build/20260929/live_mods.h" #include "Engine/Game/Build/addresses.h" +#include "Engine/Vfs/builtin_packs.h" #include "Engine/Vfs/mod_catalog.h" #include "Engine/Vfs/mod_scoring.h" #include "Extension/World/loading_screen.h" @@ -508,6 +509,8 @@ ApplyEffect preview(const mods::ModList &list) { // One of a Board Break deck pack's decks stands for the pack (mod_catalog.h). if (const auto pack = mods::board_break_pack_of(name); named(now, pack.empty() ? name : pack)) continue; const auto *entry = entry_of(name); + // Content ReSkate keeps itself (builtin_packs.h) is in no mod list, and stays. + if (!entry && (name.starts_with("reskate-builtin-") || mods::kept_by_reskate(list.root, name))) continue; // A folder deleted since: what it held is unknown, so assume the worst. if (!entry) { reload = true; diff --git a/Extension/Assets/map_download.cpp b/Extension/Assets/map_download.cpp index ade729e5..d6853183 100644 --- a/Extension/Assets/map_download.cpp +++ b/Extension/Assets/map_download.cpp @@ -6,15 +6,18 @@ #include "Engine/Vfs/mod_list.h" #include "Engine/Vfs/thunderstore_package.h" #include "Extension/Multiplayer/Net/protocol.h" +#include "Extension/Multiplayer/Session/monotonic_clock.h" #include "Extension/UI/Overlay/overlay.h" #include "Launcher/mod_manager.h" #include +#include #include #include #include #include #include #include +#include namespace dingosdk::map_download { namespace { @@ -24,13 +27,21 @@ namespace package = thunderstore_package; constexpr wchar_t agent[] = L"ReSkate"; constexpr char community[] = "reskate"; // A map is at most a few gigabytes (an archive of one holds four); the download may take as -// long as a slow line needs for that. +// long as a slow line needs for that, whether or not the session is still waiting on it. constexpr std::uint64_t largest_download = 8ull << 30; constexpr std::uint32_t download_seconds = 4 * 60 * 60; +namespace policy = multiplayer::map_fetch_policy; std::mutex mutex; std::atomic stage{Stage::idle}; +std::atomic workers{}; +std::atomic deadline{}, ask_until{}; +// The session stopped waiting while the map was coming (its deadline passed, the server gave up +// on the player, the connection went): the map is fetched and installed all the same, and the +// player is told to join again. +std::atomic detached{}; multiplayer::MapNeed need; // under mutex, while not idle +std::optional pending; // a fresh attempt waiting for cancelled work to drain package::Name wanted; // under mutex std::string version; // under mutex: the one to download std::string newest; // under mutex @@ -65,34 +76,107 @@ bool need_server() { } // The session that is waiting on the map is left: there is no map to wait for. -void leave_session() { - if (multiplayer::fetching_map()) multiplayer::queue_command("stop", "", ""); +void leave_session(std::uint64_t expected = 0) { + const auto owned = expected ? expected : multiplayer::map_fetch_deadline(); + if (multiplayer::fetching_map() && owned) + multiplayer::queue_command("stop-map-fetch", std::to_string(owned), ""); } // Back to idle with the reason said, as the plain "map not installed" notice is. -void fail(std::string reason) { +void fail(std::string reason, bool retry = false) { std::string text; bool update{}; + std::uint64_t failed_deadline{}; { std::lock_guard lock(mutex); + if (stage.exchange(Stage::idle) == Stage::idle) return; + failed_deadline = deadline.load(); + watch.cancel.store(true); + stop_install.store(true); + apply_wanted.store(false); update = !need.have.empty(); text = "The " + host_word(need.server) + (need.moved ? " changed to " : " is on ") + need.map + (update ? ", at another version (" + std::string(multiplayer::map_package_version(need.package)) + ") than the one installed on this PC (" + need.have + "). " : ", which is not installed on this PC. ") + reason; - last_failed = need.package; + if (!retry) last_failed = need.package; } - stage.store(Stage::idle); - leave_session(); + if (multiplayer::map_fetch_deadline() == failed_deadline) leave_session(failed_deadline); logging::log(logging::Level::warning, logging::Channel::assets, "Map download: {}", text); overlay::notify(overlay::NoticeLevel::warning, update ? "Map version differs" : "Map not installed", std::move(text)); } -// The packages of the maps installed here, by level (package_of): forgotten when one is installed. -struct Known { - std::string package; - std::uint64_t until{}; + +// Whether the download has begun (or is done): from then on it goes to the end on its own time. +bool under_way(Stage now) { + return now == Stage::downloading || now == Stage::installing || now == Stage::applying || now == Stage::joining; +} + +bool continuing() { + const auto now = stage.load(); + if (now == Stage::idle) return false; + // The session's deadline ends a check or a question it outlived; a download under way is + // not thrown away with it (a large map on a slow line would never come at all). + if (!under_way(now) && policy::expired(deadline.load(), multiplayer::now_us())) { + fail("Map download timed out. Please try joining the server again.", true); + return false; + } + return !watch.cancel.load(); +} + +bool advance(Stage from, Stage to) { + if (!continuing()) return false; + return stage.compare_exchange_strong(from, to); +} + +void ask() { + ask_until.store(std::min(deadline.load(), multiplayer::now_us() + policy::confirmation_us)); + advance(Stage::checking, Stage::asking); +} +// The packages of the maps installed here, by level (lower case), read from the Mods folder in +// one pass (package_of). Forgotten when a map is installed; read again in the background two +// minutes after it was read. It was read for each level asked, on the client thread, with every +// map offer a host sends (a guest joining, a map change): tens of milliseconds, a second and more +// with many mods (2026-10-15). +struct KnownPackages { + std::map> by_level; + std::uint64_t read_at{}; + bool valid{}, reading{}; }; std::mutex known_mutex; -std::map> known_packages; +KnownPackages known_packages; + +std::map> read_known_packages() { + std::map> table; + const auto list = mods::scan_mods(mods::engine_data_root()); + for (const auto& entry : list.entries) { + if (!entry.enabled || !entry.mod.provides_levels) continue; + const auto package = multiplayer::map_package_name(entry.mod.name, entry.mod.version); + for (auto level : entry.mod.levels) { + std::transform(level.begin(), level.end(), level.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + table.emplace(std::move(level), package); // (the first mod that has a level is the one) + } + } + return table; +} +// Reads the table on a thread of its own when there is none, or it is two minutes old. Under +// known_mutex. +void refresh_known_packages(std::uint64_t now) { + if (known_packages.reading || (known_packages.valid && now - known_packages.read_at < 120000)) return; + known_packages.reading = true; + try { + std::thread([] { + std::map> table; + bool ok{}; + try { table = read_known_packages(); ok = true; } catch (...) {} + std::lock_guard lock(known_mutex); + if (ok) { + known_packages.by_level = std::move(table); + known_packages.read_at = GetTickCount64(); + known_packages.valid = true; + } + known_packages.reading = false; + }).detach(); + } catch (...) { known_packages.reading = false; } +} // The mod's row in mods.json when its folder is here: whether it is switched on, and what it // says of itself. @@ -130,11 +214,12 @@ void check() { const auto bytes = read_bytes(mods_root() / name.folder() / L"icon.png", 4 * 1024 * 1024); overlay::set_map_download_icon(bytes); } catch (const std::exception&) {} - stage.store(Stage::asking); + ask(); return; } https::Download result; - const auto answer = https::get_text(package::details_url(name), 512 * 1024, 20, agent, &result); + const auto answer = https::get_text(package::details_url(name), 512 * 1024, 20, agent, &result, &watch); + if (!continuing()) return; if (!answer) return fail(result.http_status == 404 ? "Its mod is not on Thunderstore. Install its map mod yourself and join again." : "Thunderstore could not be reached to look for it."); @@ -151,9 +236,9 @@ void check() { } // Its icon for the card: small, and not worth holding the question up for long. if (!choice.icon.empty()) - if (const auto png = https::get_text(std::wstring(choice.icon.begin(), choice.icon.end()), 2 * 1024 * 1024, 8, agent)) + if (const auto png = https::get_text(std::wstring(choice.icon.begin(), choice.icon.end()), 2 * 1024 * 1024, 8, agent, nullptr, &watch)) overlay::set_map_download_icon(*png); - stage.store(Stage::asking); + ask(); } catch (const std::exception& failure) { fail(std::string("Looking for it failed: ") + failure.what()); } @@ -175,6 +260,7 @@ void fetch() { const auto root = mods_root(); const auto folder = name.folder(); try { + if (!continuing()) return; if (!off) { wchar_t temp[MAX_PATH + 1]{}; const auto length = GetTempPathW(MAX_PATH, temp); @@ -184,41 +270,54 @@ void fetch() { fs::remove(archive, error); watch.received.store(0); watch.total.store(0); - auto result = https::get(package::download_url(name, get), archive, largest_download, download_seconds, agent, &watch); + // (The session's deadline is not the download's: only its own time limit stops it.) + const auto download = [&](const std::string& selected) { + watch.deadline_us.store(0); + return https::get(package::download_url(name, selected), archive, largest_download, download_seconds, agent, &watch); + }; + auto result = download(get); // The host's version can be gone from Thunderstore: the newest is the same map. if (!update && !result.ok && result.http_status == 404 && get != fallback && !watch.cancel.load()) { fs::remove(archive, error); logging::log(logging::Level::info, logging::Channel::assets, "Map download: {} {} is not on Thunderstore any more; getting {}.", folder, get, fallback); - result = https::get(package::download_url(name, fallback), archive, largest_download, download_seconds, agent, &watch); + result = download(fallback); } if (!result.ok) { fs::remove(archive, error); - if (watch.cancel.load()) { stage.store(Stage::idle); leave_session(); return; } + if (!continuing()) return; + if (result.error == ERROR_TIMEOUT || result.error == ERROR_WINHTTP_TIMEOUT) + return fail("The download timed out. Please try joining the server again.", true); return fail(result.http_status && result.http_status != 200 ? "The download failed (Thunderstore answered " + std::to_string(result.http_status) + ")." : "The download failed (Windows error " + std::to_string(result.error) + ")."); } - stage.store(Stage::installing); + if (!advance(Stage::downloading, Stage::installing)) { + fs::remove(archive, error); + return; + } try { // (In the installed one's place when it is the same map at another version. The // game may hold the old one's files open when it was skated this run: then it // cannot be replaced until the game is closed.) - launcher_mods::install(root, archive, update, [](float) {}, stop_install, {folder, name.owner, true}); + launcher_mods::install(root, archive, update, [](float) { continuing(); }, stop_install, {folder, name.owner, true}); } catch (const std::exception& failure) { fs::remove(archive, error); + if (!continuing()) return; return fail(update ? std::string("It could not be replaced while the game runs (") + failure.what() + "). Close the game, update it from the launcher and join again." : std::string("It could not be installed: ") + failure.what()); } fs::remove(archive, error); + if (!continuing()) return; { std::lock_guard known(known_mutex); - known_packages.clear(); + known_packages.valid = false; } } else { - stage.store(Stage::installing); + if (!advance(Stage::downloading, Stage::installing)) return; } + if (!continuing()) return; // A map and nothing else: this way in is for maps. std::error_code error; if (!fs::exists(root / folder / L"reskate-levels.json", error) || !live_mods::only_maps(root / folder)) { @@ -228,16 +327,18 @@ void fetch() { return fail("Its package holds more than a map, so it is not installed from here. Install it from the launcher if you trust it."); } if (off || update) { // (a map at another version may have been switched off: it is wanted now) + if (!continuing()) return; auto list = mods::scan_mods(mods::engine_data_root()); if (!list.issue.empty()) return fail("mods.json could not be read: " + list.issue); for (auto& entry : list.entries) if (same(entry.mod.name, folder)) entry.enabled = true; mods::save_mod_order(list.root, list.entries); } + if (!continuing()) return; logging::log(logging::Level::info, logging::Channel::assets, "Map download: {} is in Mods; applying it.", folder); apply_started.store(false); apply_wanted.store(true); - stage.store(Stage::applying); + if (!advance(Stage::installing, Stage::applying)) apply_wanted.store(false); } catch (const std::exception& failure) { fail(std::string("It could not be installed: ") + failure.what()); } @@ -245,10 +346,37 @@ void fetch() { // One piece of work at a time, which the stage sees to: each is started by the move into its // stage. Detached, as the live merge's thread is: the game may close while a map downloads. -void start(void (*work)()) { std::thread(work).detach(); } +void start(void (*work)()) { + ++workers; + try { + std::thread([work] { + struct Finished { ~Finished() { --workers; } } finished; + work(); + }).detach(); + } catch (...) { + --workers; + throw; + } +} } // namespace void offer(const multiplayer::MapNeed& asked) { + std::unique_lock lock(mutex); + // (Another session's need cancels this attempt, unless its own session has stopped waiting + // on it already: then it finishes, and the new need is turned away below.) + if (stage.load() != Stage::idle && !detached.load() && asked.deadline_us && asked.deadline_us != deadline.load()) { + lock.unlock(); + fail("Map download cancelled.", true); + lock.lock(); + } + // A cancelled worker/atomic live merge must drain before its globals or files are reused. + if (stage.load() == Stage::idle && (workers.load() || (apply_started.load() && live_mods::busy()))) { + pending = asked; + overlay::notify(overlay::NoticeLevel::warning, "Map download", + "The previous map operation is still finishing. Waiting before starting this map."); + return; + } + pending.reset(); auto idle = Stage::idle; const auto name = package::parse_name(asked.package); // (Taken before anything is set up, so a second need while this one is in hand is dropped.) @@ -256,10 +384,7 @@ void offer(const multiplayer::MapNeed& asked) { // Another map is in hand (the session moved on while it came). One at a time: the one // being fetched is finished, and the session, which now waits on this other one, is left. bool other{}; - { - std::lock_guard lock(mutex); - other = asked.package != need.package; - } + other = asked.package != need.package; if (other) { overlay::notify(overlay::NoticeLevel::warning, "Map changed again", "The " + host_word(asked.server) + " moved on to " + asked.map + " while another map was downloading. Join again when it is done."); @@ -268,7 +393,6 @@ void offer(const multiplayer::MapNeed& asked) { return; } { - std::lock_guard lock(mutex); need = asked; if (!name || asked.package == last_failed) { // (Said once already, or nothing this can fetch: the plain notice.) @@ -287,23 +411,44 @@ void offer(const multiplayer::MapNeed& asked) { } overlay::set_map_download_icon({}); watch.cancel.store(false); + const auto now = multiplayer::now_us(); + deadline.store(asked.deadline_us ? asked.deadline_us : now + policy::session_us); + ask_until.store(0); + detached.store(false); + watch.deadline_us.store(deadline.load()); stop_install.store(false); + apply_wanted.store(false); + apply_started.store(false); pick_download.store(true); - start(check); + try { + start(check); + } catch (...) { + lock.unlock(); + fail("The map download could not be started. Please try joining again.", true); + } } void answer(bool download) noexcept { try { // (A click and a key can answer in the same moment: only the first does.) - auto asked = Stage::asking; - if (stage.compare_exchange_strong(asked, download ? Stage::downloading : Stage::idle)) { - if (download) start(fetch); - else leave_session(); - } else if (asked == Stage::downloading && !download) { - watch.cancel.store(true); + if (!continuing()) return; + const auto now = multiplayer::now_us(); + if (stage.load() == Stage::asking && policy::expired(ask_until.load(), now)) { + fail("The map download was not confirmed within two minutes. Please try joining again.", true); + return; + } + if (!download) { + if (stage.load() == Stage::asking || stage.load() == Stage::downloading) + fail("Map download cancelled.", true); + } else { + // Publish the stage and claim its worker together. Cancellation cannot allow + // a fresh offer to reuse the globals before this thread starts its worker. + std::lock_guard lock(mutex); + auto asked = Stage::asking; + if (!watch.cancel.load() && stage.compare_exchange_strong(asked, Stage::downloading)) start(fetch); } } catch (...) { - stage.store(Stage::idle); + fail("The map download could not be started. Please try joining again.", true); } } @@ -345,10 +490,47 @@ View view() { } void tick() { - const auto now = stage.load(); + warm_packages(); + std::optional next; + { + std::lock_guard lock(mutex); + if (pending && pending->deadline_us != multiplayer::map_fetch_deadline()) pending.reset(); + if (pending && stage.load() == Stage::idle && !workers.load() && + !(apply_started.load() && live_mods::busy())) + next = std::exchange(pending, std::nullopt); + } + if (next) offer(*next); + auto now = stage.load(); + if (now != Stage::idle) { + const auto session = multiplayer::model(); + // Admission (or completed travel) retires the session deadline. Do not turn an + // already successful join into a timeout while waiting out the UI's settling delay. + if (now == Stage::joining && session.active && session.connected && + !multiplayer::fetching_map() && !multiplayer::map_fetch_deadline()) { + std::lock_guard lock(mutex); + if (stage.compare_exchange_strong(now, Stage::idle)) last_failed.clear(); + now = stage.load(); + } + const bool gave_up = policy::expired(deadline.load(), multiplayer::now_us()) || + session.status.find("stopped waiting for the map download") != std::string::npos; + // The session stopped waiting while the download was under way: it finishes all the same. + if (!session.active && under_way(now) && !detached.exchange(true) && gave_up) + overlay::notify(overlay::NoticeLevel::warning, "Map still downloading", + "The " + [] { std::lock_guard lock(mutex); return host_word(need.server); }() + + " stopped waiting for the map. It goes on downloading and installs when it is done; join again then."); + if (session.active && !detached.load() && now != Stage::joining && multiplayer::map_fetch_deadline() != deadline.load()) + fail("Map download cancelled.", true); + if (!session.active && !under_way(now) && gave_up) + fail("The server stopped waiting for the map download.", true); + if (continuing() && now == Stage::asking && + policy::expired(ask_until.load(), multiplayer::now_us())) + fail("The map download was not confirmed within two minutes. Please try joining again.", true); + now = stage.load(); + } // The session leaves the map alone until it has been applied (an update's level is one the // game already has, and would load from files that are being replaced). - multiplayer::hold_map_load(now == Stage::downloading || now == Stage::installing || now == Stage::applying); + multiplayer::hold_map_load((now != Stage::idle && now != Stage::joining) || + workers.load() || (apply_started.load() && live_mods::busy()), need_package()); if (now == Stage::applying) { if (apply_wanted.load()) { // (A merge of the player's own may be running: wait for it and start this one.) @@ -371,50 +553,59 @@ void tick() { return fail("It is installed, but was left out when the mods were applied (" + live_mods::status() + ")."); // The level list takes the new map on the next ticks, and the session, which has been // trying the map all along, loads it then. - join_at.store(GetTickCount64()); - stage.store(Stage::joining); + join_at.store(multiplayer::now_us()); + advance(Stage::applying, Stage::joining); return; } if (now == Stage::joining) { - const auto waited = GetTickCount64() - join_at.load(); - if (waited < 2500) return; + const auto waited = multiplayer::now_us() - join_at.load(); + if (waited < 2500000) return; // Still waiting on it: the game has the mod and cannot find the map in it. - if (multiplayer::fetching_map()) { - if (waited > 30000) fail("It is installed, but the game still cannot load the map. Restart the game and join again."); + const auto session = multiplayer::model(); + if (multiplayer::fetching_map() || (session.active && !session.connected)) { + if (waited >= policy::installed_load_us) fail("It is installed, but the game still cannot load the map. Restart the game and join again."); return; } { // Should it not load after all, it is said and not offered round and round. std::lock_guard lock(mutex); - last_failed = need.package; + last_failed.clear(); } - stage.store(Stage::idle); - // The session ended while the map came (the host left, the connection dropped): back to it. - if (!multiplayer::model().active && !multiplayer::queue_command("rejoin", "", "")) - overlay::notify(overlay::NoticeLevel::warning, "Map installed", "Join the session again from the server browser."); + if (!continuing() || !stage.compare_exchange_strong(now, Stage::idle)) return; + // The session ended while the map came (the host left, the connection dropped): back to it, + // while it would still have the player. Past its deadline it would not. + if (!multiplayer::model().active && + (policy::expired(deadline.load(), multiplayer::now_us()) || !multiplayer::queue_command("rejoin", "", ""))) + overlay::notify(overlay::NoticeLevel::info, "Map installed", "Join the session again from the server browser."); } } +void warm_packages() noexcept { + try { + std::lock_guard lock(known_mutex); + if (!known_packages.valid) refresh_known_packages(GetTickCount64()); + } catch (...) {} +} + std::string package_of(std::string_view asset) { - auto& cache_mutex = known_mutex; - auto& cache = known_packages; if (asset.empty()) return {}; std::string key(asset); std::transform(key.begin(), key.end(), key.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); const auto now = GetTickCount64(); - std::lock_guard lock(cache_mutex); - if (const auto found = cache.find(key); found != cache.end() && now < found->second.until) return found->second.package; - // Asked with every map offer a host sends: the Mods folder is read at most every two - // minutes for a level. - std::string result; - const auto list = mods::scan_mods(mods::engine_data_root()); - for (const auto& entry : list.entries) { - if (!entry.enabled || !entry.mod.provides_levels) continue; - if (std::none_of(entry.mod.levels.begin(), entry.mod.levels.end(), [&](const std::string& level) { return same(level, key); })) continue; - result = multiplayer::map_package_name(entry.mod.name, entry.mod.version); - break; + std::unique_lock lock(known_mutex); + // (Only with no table at all, just after a map was installed, is the Mods folder read here.) + if (!known_packages.valid) { + lock.unlock(); + auto table = read_known_packages(); + lock.lock(); + if (!known_packages.valid) { + known_packages.by_level = std::move(table); + known_packages.read_at = now; + known_packages.valid = true; + } } - cache[key] = {result, now + 120000}; - return result; + refresh_known_packages(now); + const auto found = known_packages.by_level.find(key); + return found != known_packages.by_level.end() ? found->second : std::string{}; } } diff --git a/Extension/Assets/map_download.h b/Extension/Assets/map_download.h index 68f30ac6..13f4c1f0 100644 --- a/Extension/Assets/map_download.h +++ b/Extension/Assets/map_download.h @@ -9,6 +9,11 @@ // player is asked; on yes the package is downloaded, installed into Mods and applied as a live // mod (live_mods.h). The session stays connected meanwhile and loads the map once it is there; // on no, or when the map cannot be had, the session is left. +// The session waits 30 minutes on its local monotonic clock for the whole of it (checking, +// confirmation, download, installation, applying and joining); confirmation gets two of them. +// A download that has begun is not cut short when the session stops waiting: it finishes, +// installs and is applied, and the player is told to join again. A cancelled worker drains +// before another attempt can begin. // // Only maps come in this way. The package has to be listed as a map in ReSkate's Thunderstore // community, not deprecated and not rejected by its moderators; what is installed has to be a @@ -55,4 +60,7 @@ void tick(); // The Thunderstore package an enabled mod's level comes from (its folder and manifest), for a // host to tell its guests; empty when the level is the game's own or the mod is not a package. std::string package_of(std::string_view asset); +// Reads the table package_of answers from, on a thread of its own, if it has not been (so the +// first map offer does not wait on the Mods folder). Cheap to call each frame; tick() does. +void warm_packages() noexcept; } diff --git a/Extension/Assets/map_icons.cpp b/Extension/Assets/map_icons.cpp index 62f6aa82..2a168dfa 100644 --- a/Extension/Assets/map_icons.cpp +++ b/Extension/Assets/map_icons.cpp @@ -9,7 +9,9 @@ #include "Engine/Vfs/thunderstore_package.h" #include "Extension/Multiplayer/Session/session.h" #include "Extension/UI/Overlay/overlay.h" +#include #include +#include #include #include #include @@ -136,9 +138,56 @@ void fetch(unsigned serial, std::vector maps) { } fetching.store(false); } + +// The menu's map grid: each level's picture, a custom map's mod icon (its path found on the client +// thread) or an official map's loading screen. +std::atomic menu_fetching{}; +void fetch_menu(std::vector> maps) { + std::vector icons; + for (auto& [map, icon_path] : maps) { + overlay::MapVoteIcon icon; + if (!icon_path.empty()) icon.png = read_bytes(icon_path); + if (icon.png.empty()) { + icon.rgba = game_picture(map.name); + if (!icon.rgba.empty()) icon.side = picture_side; + } + if (icon.png.empty() && icon.rgba.empty()) continue; + icon.name = std::move(map.asset); + icons.push_back(std::move(icon)); + } + overlay::set_menu_map_pictures(icons); + try { + logging::log(logging::Level::info, logging::Channel::assets, "Menu map grid: {} of {} map(s) with a picture.", icons.size(), maps.size()); + } catch (...) {} + menu_fetching.store(false); +} +bool same_asset(std::string_view a, std::string_view b) { + return a.size() == b.size() && std::equal(a.begin(), a.end(), b.begin(), [](char x, char y) { + return std::tolower(static_cast(x)) == std::tolower(static_cast(y)); + }); +} +void menu_tick() { + if (menu_fetching.load()) return; + auto wanted = overlay::take_menu_map_request(); + if (wanted.empty()) return; + std::vector> maps; + const auto& catalog = mods::catalog(); + for (auto& map : wanted) { + fs::path icon; + for (const auto& mod : catalog.mods) + if (std::ranges::any_of(mod.levels, [&](const std::string& level) { return same_asset(level, map.asset); })) { + icon = mod.directory / L"icon.png"; + break; + } + maps.emplace_back(std::move(map), std::move(icon)); + } + menu_fetching.store(true); + std::thread(fetch_menu, std::move(maps)).detach(); +} } // namespace void tick() { + menu_tick(); const auto serial = multiplayer::next_maps_serial(); if (serial == wanted_serial.load() || fetching.load()) return; wanted_serial.store(serial); diff --git a/Extension/Assets/native_render_resource_pool.cpp b/Extension/Assets/native_render_resource_pool.cpp index b9abbc49..2e47243e 100644 --- a/Extension/Assets/native_render_resource_pool.cpp +++ b/Extension/Assets/native_render_resource_pool.cpp @@ -1,11 +1,14 @@ #include "native_render_resource_pool.h" #include "Engine/Core/Hooks/hooks.h" +#include "Engine/Core/Log/logging.h" +#include "Engine/Core/Platform/memory.h" #include "Engine/Game/Build/supported_build.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Game/Build/20260929/render_resource_pool.h" #include +#include #include #include #include @@ -13,6 +16,7 @@ #include #include #include +#include namespace dingosdk { namespace { @@ -61,6 +65,10 @@ static_assert(relation_mask_bytes == 0x00300000); using NativeConstructor = void* (__fastcall *)(void*, void*); using IndexTableConstructor = void* (__fastcall *)(void*, void*, void*); +// Records and metadata every renderer built on the constructor uses; the +// resource states and relation masks only the state owner has +// (contract::state_owner_call_rva). 115.5 MB of states went to the other +// renderer as well, which never touches them. struct Buffers { std::byte* records{}; std::uint32_t* metadata{}; @@ -68,9 +76,8 @@ struct Buffers { std::byte* resource_states{}; std::uint64_t* relation_masks{}; - explicit operator bool() const noexcept { - return records && metadata && resource_flags && resource_states && relation_masks; - } + explicit operator bool() const noexcept { return records && metadata && resource_flags; } + bool has_states() const noexcept { return resource_states && relation_masks; } }; struct IndexTableBuffers { @@ -89,6 +96,10 @@ std::array reserved_index_tables{}; std::size_t next_reserved_index_table{}; std::byte* resource_flag_stubs{}; std::uintptr_t installed_base{}; +// Where the state owner's constructor call returns to in the game. +std::atomic state_owner_return{}; +// The renderer that has resource states, once constructed. +std::atomic state_owner{}; bool read_memory(const void* source, void* destination, std::size_t size) noexcept { SIZE_T count{}; @@ -122,13 +133,22 @@ bool validate_contract(std::uintptr_t base) noexcept { contract::resource_flags_read_priority) || !matches(base + contract::resource_flags_visibility_rva, contract::resource_flags_read_visibility)) return false; + // The state owner's call to the constructor: an E8 call whose target is it. + std::array call{}; + std::int32_t displacement{}; + if (!read_memory(reinterpret_cast(base + contract::state_owner_call_rva), call.data(), call.size()) || + call[0] != 0xe8) return false; + std::memcpy(&displacement, call.data() + 1, sizeof(displacement)); + if (contract::state_owner_call_rva + call.size() + displacement != contract::constructor_rva) return false; for (const auto& patch : contract::patches) if (patch.expected.size() != patch.replacement.size() || !matches(base + patch.rva, patch.expected)) return false; return true; } -Buffers allocate_buffers() noexcept { +void free_buffers(Buffers& buffers) noexcept; + +Buffers allocate_buffers(bool states) noexcept { Buffers result{}; result.records = static_cast(VirtualAlloc( nullptr, record_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); @@ -140,19 +160,13 @@ Buffers allocate_buffers() noexcept { return {}; } result.resource_flags = reinterpret_cast(result.metadata) + metadata_bytes; + if (!states) return result; result.resource_states = static_cast(VirtualAlloc( nullptr, resource_state_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - if (!result.resource_states) { - VirtualFree(result.metadata, 0, MEM_RELEASE); - VirtualFree(result.records, 0, MEM_RELEASE); - return {}; - } - result.relation_masks = static_cast(VirtualAlloc( + result.relation_masks = !result.resource_states ? nullptr : static_cast(VirtualAlloc( nullptr, relation_mask_bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); - if (!result.relation_masks) { - VirtualFree(result.resource_states, 0, MEM_RELEASE); - VirtualFree(result.metadata, 0, MEM_RELEASE); - VirtualFree(result.records, 0, MEM_RELEASE); + if (!result.has_states()) { + free_buffers(result); return {}; } return result; @@ -195,16 +209,34 @@ void free_reserved_allocations() noexcept { next_reserved_index_table = 0; } -Buffers take_buffers() noexcept { +Buffers take_buffers(bool states) noexcept { std::lock_guard lock(allocation_mutex); + // Two renderers are built on the constructor: the first takes the reserved + // records and metadata, the state owner the reserved states, whichever + // comes first. Another instance costs an isolated allocation of its own + // and avoids aliasing records. + Buffers result{}; if (reserved_buffers) { - const auto result = reserved_buffers; - reserved_buffers = {}; + result.records = std::exchange(reserved_buffers.records, nullptr); + result.metadata = std::exchange(reserved_buffers.metadata, nullptr); + result.resource_flags = std::exchange(reserved_buffers.resource_flags, nullptr); + } else { + result = allocate_buffers(false); + if (!result) return {}; + } + if (!states) return result; + if (reserved_buffers.has_states()) { + result.resource_states = std::exchange(reserved_buffers.resource_states, nullptr); + result.relation_masks = std::exchange(reserved_buffers.relation_masks, nullptr); return result; } - // The renderer is normally a singleton. Supporting a subsequent native - // instance costs another isolated allocation and avoids aliasing records. - return allocate_buffers(); + const auto fresh = allocate_buffers(true); + if (!fresh) { free_buffers(result); return {}; } + result.resource_states = fresh.resource_states; + result.relation_masks = fresh.relation_masks; + VirtualFree(fresh.metadata, 0, MEM_RELEASE); + VirtualFree(fresh.records, 0, MEM_RELEASE); + return result; } IndexTableBuffers take_index_table_buffers() noexcept { @@ -259,9 +291,15 @@ void* __fastcall construct_index_table(void* self, void* first, void* second) { void* __fastcall construct_native_renderer(void* self, void* configuration) { const auto incoming_error = GetLastError(); - const auto buffers = take_buffers(); + // (The hook is jumped to, so this returns where the game's call does.) + const auto caller = reinterpret_cast(_ReturnAddress()); + const bool states = caller == state_owner_return.load(std::memory_order_acquire); + const auto buffers = take_buffers(states); const auto original = original_constructor.load(std::memory_order_acquire); - if (!self || !buffers || !original) allocation_failure(); + if (!self || !buffers || (states && !buffers.has_states()) || !original) allocation_failure(); + logging::log(logging::Level::info, logging::Channel::graphics, + "Render-resource pool: renderer built from Skate.exe+{:#x}, {}.", caller - installed_base, + states ? "with the enlarged resource states" : "without resource states (it has none)"); auto* const record_end = buffers.records + record_bytes; auto* const metadata_end = buffers.metadata + native_render_resource_capacity; @@ -270,10 +308,16 @@ void* __fastcall construct_native_renderer(void* self, void* configuration) { // The patched constructor loads its initial vector pointer from +0x90, // constructs the native default record there, and advances +0x98 itself. store_field(self, vector_begin_offset, buffers.records); + // Both renderers read the metadata through these pointers (render_resource_pool.h). The other + // renderer was allocated 0x2daa00 bytes, and they were written 18 MB past its end, into + // whatever lay there (a crash in store_field when nothing did): it is allocated the state + // owner's size now (the patch at 0x030c64cc), so they are its own. store_field(self, metadata_begin_offset, buffers.metadata); store_field(self, metadata_end_offset, metadata_end); - store_field(self, resource_state_vector_offset, buffers.resource_states); - store_field(self, relation_mask_vector_offset, relation_mask_begin); + if (states) { + store_field(self, resource_state_vector_offset, buffers.resource_states); + store_field(self, relation_mask_vector_offset, relation_mask_begin); + } SetLastError(incoming_error); auto* const result = original(self, configuration); @@ -281,6 +325,7 @@ void* __fastcall construct_native_renderer(void* self, void* configuration) { // stock build. Keep the external values authoritative across construction. store_field(self, metadata_begin_offset, buffers.metadata); store_field(self, metadata_end_offset, metadata_end); + if (states) state_owner.store(reinterpret_cast(self), std::memory_order_release); if (load_field(self, vector_begin_offset) != buffers.records || load_field(self, vector_capacity_offset) != record_end || load_field(self, vector_end_offset) < buffers.records || @@ -314,34 +359,6 @@ bool relative_jump(std::uintptr_t instruction, std::uintptr_t destination, return true; } -std::byte* allocate_near_game_image(std::uintptr_t base) noexcept { - SYSTEM_INFO information{}; - GetSystemInfo(&information); - const auto granularity = static_cast(information.dwAllocationGranularity); - const auto align_up = [granularity](std::uintptr_t value) { - return (value + granularity - 1) & ~(granularity - 1); - }; - auto cursor = align_up(base + supported_build::game_image_size); - const auto limit = base + 0x70000000ULL; - while (cursor < limit) { - MEMORY_BASIC_INFORMATION region{}; - if (!VirtualQuery(reinterpret_cast(cursor), ®ion, sizeof(region))) break; - const auto region_base = reinterpret_cast(region.BaseAddress); - const auto region_end = region_base + region.RegionSize; - if (region.State == MEM_FREE) { - const auto candidate = align_up(cursor > region_base ? cursor : region_base); - if (candidate <= region_end && region_end - candidate >= resource_flag_stub_page_bytes) { - if (auto* const result = static_cast(VirtualAlloc( - reinterpret_cast(candidate), resource_flag_stub_page_bytes, - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE))) return result; - } - } - if (region_end <= cursor) break; - cursor = align_up(region_end); - } - return nullptr; -} - template bool write_stub(std::byte* destination, std::span prefix, std::uintptr_t return_address) noexcept { @@ -357,7 +374,8 @@ bool write_stub(std::byte* destination, std::span prefix, } bool prepare_resource_flag_stubs(std::uintptr_t base) noexcept { - resource_flag_stubs = allocate_near_game_image(base); + resource_flag_stubs = static_cast( + memory::allocate_near_image(base, supported_build::game_image_size, resource_flag_stub_page_bytes)); if (!resource_flag_stubs) return false; std::memset(resource_flag_stubs, 0xcc, resource_flag_stub_page_bytes); if (!write_stub(resource_flag_stubs + priority_stub_offset, @@ -431,6 +449,33 @@ void restore_patches(std::uintptr_t base, std::size_t count) noexcept { } } // namespace +void report_native_render_resources() noexcept { + const auto owner = state_owner.load(std::memory_order_acquire); + std::uintptr_t begin{}, end{}; + if (!owner || !memory::peek(owner + vector_begin_offset, begin) || !memory::peek(owner + vector_end_offset, end) || + end < begin || end > begin + record_bytes) + return; + const auto used = (end - begin) / native_render_resource_record_size; + static std::size_t said{}; + static bool warned{}, full{}; + // (A new high, in steps of 1/32: how close a level came is worth having in a log.) + if (used < said + native_render_resource_capacity / 32 && !(used >= native_render_resource_capacity && !full)) return; + said = used; + const auto percent = used * 100 / native_render_resource_capacity; + if (used >= native_render_resource_capacity) { + full = true; + logging::log(logging::Level::error, logging::Channel::graphics, + "Render-resource pool: all {} records are taken; the level needs more than the game can address.", used); + } else if (percent >= 90 && !warned) { + warned = true; + logging::log(logging::Level::warning, logging::Channel::graphics, "Render-resource pool: {} of {} records taken ({}%).", used, + native_render_resource_capacity, percent); + } else { + logging::log(logging::Level::info, logging::Channel::graphics, "Render-resource pool: {} of {} records taken ({}%).", used, + native_render_resource_capacity, percent); + } +} + bool start_native_render_resource_pool(std::uintptr_t base, std::string& error) { std::lock_guard lock(install_mutex); error.clear(); @@ -446,7 +491,7 @@ bool start_native_render_resource_pool(std::uintptr_t base, std::string& error) { std::lock_guard allocation_lock(allocation_mutex); - reserved_buffers = allocate_buffers(); + reserved_buffers = allocate_buffers(true); for (auto& table : reserved_index_tables) table = allocate_index_table_buffers(); next_reserved_index_table = 0; if (!reserved_buffers || !reserved_index_tables[0] || !reserved_index_tables[1]) { @@ -520,6 +565,7 @@ bool start_native_render_resource_pool(std::uintptr_t base, std::string& error) return false; } + state_owner_return.store(base + contract::state_owner_call_rva + 5, std::memory_order_release); original_constructor.store(original, std::memory_order_release); original_index_table_constructor.store(original_index, std::memory_order_release); const auto queued_index = hook_queue_enable(index_target); diff --git a/Extension/Assets/native_render_resource_pool.h b/Extension/Assets/native_render_resource_pool.h index 7e266739..85df9974 100644 --- a/Extension/Assets/native_render_resource_pool.h +++ b/Extension/Assets/native_render_resource_pool.h @@ -14,4 +14,9 @@ inline constexpr std::size_t native_render_resource_record_size = 0xf0; // constructing its records normally, but its backing storage and all parallel // per-resource arrays are redirected to process-lifetime allocations. bool start_native_render_resource_pool(std::uintptr_t base, std::string& error); + +// How many of the pool's records the renderer that has resource states has taken: a level that +// needs more than native_render_resource_capacity cannot be drawn. Said in the log as it grows, +// checked every frame so the last count before a crash is there. Game thread. +void report_native_render_resources() noexcept; } // namespace dingosdk diff --git a/Extension/BoardBreak/Test/board_break_tests.cpp b/Extension/BoardBreak/Test/board_break_tests.cpp index 8dff1f87..d5653033 100644 --- a/Extension/BoardBreak/Test/board_break_tests.cpp +++ b/Extension/BoardBreak/Test/board_break_tests.cpp @@ -1,6 +1,7 @@ // Board Break's rules, apart from the game: what a break leaves of a board, how likely a slam // breaks it, and where the parts of a broken board lie. #include "Extension/BoardBreak/board_break_model.h" +#include "Extension/BoardBreak/board_break_feel.h" #include #include #include @@ -35,12 +36,12 @@ void a_break_leaves_two_pieces_or_one_that_hangs_together() { } void a_harder_slam_breaks_a_board_more_often() { - check(break_chance(10.0f, 10.0f, 0.25f) == 1.0f && break_chance(30.0f, 10.0f, 0.25f) == 1.0f, "A heavy slam always breaks it"); - check(break_chance(0.0f, 10.0f, 0.25f) == 0.25f && break_chance(6.0f, 10.0f, 0.25f) == 0.25f, "A light one now and then"); - check(near(break_chance(8.0f, 10.0f, 0.25f), 0.625f), "and in between it rises evenly"); + check(break_chance(10.0f, 10.0f) == 1.0f && break_chance(30.0f, 10.0f) == 1.0f, "A heavy slam always breaks it"); + check(break_chance(0.0f, 10.0f) == 0.0f && break_chance(6.0f, 10.0f) == 0.0f, "A light one never does"); + check(near(break_chance(8.0f, 10.0f), 0.5f), "and in between it rises smoothly"); float before{}; for (float force = 0; force <= 12.0f; force += 0.5f) { - const auto chance = break_chance(force, 10.0f, 0.25f); + const auto chance = break_chance(force, 10.0f); check(chance >= before && chance <= 1.0f, "It never falls with more force"); before = chance; } @@ -148,6 +149,66 @@ void a_baseplate_stays_on_the_deck_while_its_truck_steers() { } } } + +void the_settings_keep_to_their_ranges() { + Settings s; + check(set_setting_value(s, "sensitivity", 9999) && s.sensitivity == sensitivity_max, "A percent is kept within its range"); + check(set_setting_value(s, "always", 5) && s.always && setting_value(s, "always") == 1, "a switch is on or off"); + check(set_setting_value(s, "joined", -3) && s.joined == 0, "and nothing goes below its least"); + check(!set_setting_value(s, "nonsense", 1) && setting_value(s, "nonsense") == -1, "An unknown name is no setting"); + for (const auto& info : setting_info) check(setting_value(s, info.name) >= 0, "Every setting has a value"); +} + +void a_worn_or_touchy_board_breaks_sooner() { + check(slam_chance(heavy_slam, 100, 0.0f) == 1.0f, "A heavy slam always breaks a new board"); + check(slam_chance(6.0f, 100, 0.0f) < slam_chance(6.0f, 100, 1.0f), "A worn board breaks from less"); + check(slam_chance(heavy_slam * 0.5f, 200, 0.0f) == 1.0f, "At 200% half the force always breaks it"); + check(slam_chance(8.0f, 50, 0.0f) < slam_chance(8.0f, 100, 0.0f), "at 50% it takes more"); + check(slam_chance(2.0f, 100, 0.0f) == 0.0f && slam_chance(2.0f, 100, 0.6f) == 0.0f, "A stumble never breaks a board that is not done"); + check(slam_chance(0.0f, 400, 1.0f) == 0.0f, "nor does a bail whose force is not known, at any setting"); +} + +void every_hit_wears_a_board() { + check(landing_wear(3.0f) == 0.0f && landing_wear(5.0f) > 0.0f && landing_wear(9.0f) > landing_wear(5.0f), "Harder landings wear more, light ones none"); + check(rail_wear(3.0f, true) > rail_wear(3.0f, false), "A slide wears the deck more than a grind on the trucks"); + check(grind_wear_per_second(true) > grind_wear_per_second(false), "and sliding more than grinding"); + check(slam_wear(2.0f) == 0.0f && slam_wear(12.0f) > slam_wear(7.0f), "A stumble costs nothing, a hard slam a lot"); + check(add_wear(0.5f, 0.1f, 0) == 0.5f && near(add_wear(0.5f, 0.1f, 200), 0.7f) && add_wear(0.95f, 1.0f, 100) == 1.0f, + "Wear goes at the player's rate, and never past worn out"); +} + +void a_worn_board_handles_worse() { + check(handling(0.0f) == Handling{} && handling(0.2f) == Handling{}, "A board under a fifth worn handles as new"); + const auto half = handling(0.6f), done = handling(1.0f); + check(half.steering < 1.0f && done.steering < half.steering && done.spring < half.spring && done.twist > half.twist, + "The more worn, the lazier the turn and the softer the trucks"); + check(done.steering > 0.5f && done.spring > 0.2f, "but it can still be ridden"); + check(!gives_way_on_landing(20.0f, 400, done_wear - 0.01f), "A board that is not done never gives way on a landing"); + check(!gives_way_on_landing(6.0f, 100, done_wear) && gives_way_on_landing(6.5f, 100, done_wear), + "A board just done goes on a big drop"); + check(!gives_way_on_landing(3.0f, 100, 1.0f) && gives_way_on_landing(3.5f, 100, 1.0f), + "one worn right out on any real drop, but not on an ollie on flat ground"); + check(gives_way_on_landing(4.0f, 200, done_wear) && !gives_way_on_landing(4.0f, 100, done_wear), "Sensitivity makes it touchier"); +} + +void a_board_across_its_way_slides() { + const Vec level_up{0, 1, 0}, ahead{0, 0, 1}, across{1, 0, 0}, upside_down{0, -1, 0}, going{0, -1, 5}; + check(contact_of(ahead, level_up, going) == Contact::grind, "Along the rail it grinds on its trucks"); + check(contact_of(across, level_up, going) == Contact::slide, "Across it, it slides on the deck"); + check(contact_of(across, upside_down, going) == Contact::darkslide, "On its grip it is a darkslide"); + check(contact_of(across, level_up, {0, -2, 0.1f}) == Contact::grind, "A board that hardly moves along counts as grinding"); + check(rail_chance(9.0f, Contact::grind, 100, 0.0f) == 0.0f, "A grind on the trucks never snaps the board"); + check(rail_chance(2.0f, Contact::slide, 100, 0.0f) == 0.0f && rail_chance(6.0f, Contact::slide, 100, 0.0f) == 1.0f, + "A hop onto a rail is safe, a drop onto it snaps the board"); + check(rail_chance(3.5f, Contact::darkslide, 100, 0.0f) > rail_chance(3.5f, Contact::slide, 100, 0.0f), "a darkslide sooner"); +} + +void a_board_breaks_where_it_hit() { + check(place_of_break(0.8f, false, 0.1f) == Place::nose && place_of_break(-0.8f, false, 0.1f) == Place::tail, "Nose first at the nose, tail first at the tail"); + check(place_of_break(0.8f, true, 0.1f) == Place::mid, "Across a rail in the middle"); + check(place_of_break(0.0f, false, 0.2f) == Place::mid && place_of_break(0.0f, false, 0.9f) == Place::tail, "Flat mostly in the middle"); + check(body_loudness(2.0f) == 0.0f && body_loudness(body_full) == 1.0f && body_loudness(5.0f) < body_loudness(9.0f), "Harder hits sound louder"); +} } // namespace int main() { @@ -162,6 +223,12 @@ int main() { a_hanging_end_turns_about_the_break(); a_truck_goes_with_its_piece(); a_baseplate_stays_on_the_deck_while_its_truck_steers(); + the_settings_keep_to_their_ranges(); + a_worn_or_touchy_board_breaks_sooner(); + every_hit_wears_a_board(); + a_worn_board_handles_worse(); + a_board_across_its_way_slides(); + a_board_breaks_where_it_hit(); std::cout << "board break tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/Extension/BoardBreak/board_break.cpp b/Extension/BoardBreak/board_break.cpp index 9b9a1ee8..96823488 100644 --- a/Extension/BoardBreak/board_break.cpp +++ b/Extension/BoardBreak/board_break.cpp @@ -9,34 +9,51 @@ #include "Extension/Multiplayer/Remote/native_skater.h" #include "Extension/Profile/local_profile_runtime.h" #include "Extension/Skater/no_bail.h" +#include "board_break_audio.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/20260929/skater_state.h" +#include "Extension/HallOfMeat/hall_of_meat_model.h" +#include "Extension/HallOfMeat/hall_of_meat_skater.h" +#include "Extension/Multiplayer/Remote/native_vfx.h" +#include "Extension/Multiplayer/Session/session.h" +#include "Extension/Progression/script_natives.h" +#include "Extension/Skater/physics_tuning.h" +#include "Extension/UI/Overlay/overlay.h" #include -#include +#include #include +#include +#include #include #include #include +#include #include #include #include #include #include +#include #include +#include #include #include #include +#include #include #include -#pragma comment(lib, "winmm.lib") +#pragma comment(lib, "ole32.lib") +#pragma comment(lib, "shell32.lib") namespace dingosdk::board_break { namespace { constexpr const char* enabled_preference = "BoardBreak"; -// A slam of `heavy` force or more always breaks the board; a lighter one with `light`, rising to -// certain between 60% of `heavy` and `heavy` (break_chance). Half of all breaks stay joined by the -// grip tape. Now and then a board gives way under the rider: per second of riding, the nose goes. -constexpr float heavy = 10.0f, light = 0.25f, joined_share = 0.5f; -constexpr double gives_way = 0.0001; +// A slam of `heavy` force or more always breaks the board, one under 60% of it never, and the +// chance rises smoothly in between (break_chance). Half of all breaks stay joined by the grip tape. +// A board that is done gives way under the rider on a hard enough landing: the nose goes. +// (board_break_feel.h holds the forces, the chances and what wear does.) +namespace sstate = addr::skater_state; constexpr std::size_t cause_count = 35; // the collector's cause bytes (+0x28) and forces (+0x4c) constexpr std::uint64_t recent_ms = 1500; // a cause belongs to a wipeout this soon before it // The board is back in the skater's hand: close to the skater and not lying flat. @@ -50,6 +67,8 @@ constexpr int root_bone = 1, mount_bone = 2, deck_bone = 3, back_truck_bone = 4, constexpr int back_plate_bone = 5, front_plate_bone = 9; constexpr std::array carrier{12, 13, 15, 14}; constexpr std::uint32_t rig_bones = 16; +// The wheels, each on its truck's bone: back right and left, front right and left. +constexpr std::array wheel_bones{6, 7, 10, 11}; bool readable(std::uintptr_t p, void* out, std::size_t size) { return size ? memory::peek_bytes(p, out, size) : p >= 0x10000; } @@ -71,6 +90,7 @@ struct Broken { std::uint64_t since{}; bool gave_way{}; // it went under the rider: it stays broken while it is ridden bool asked{}; // it was broken from the console, to be looked at: it stays until it is mended there + std::string deck; // the deck it was: the one that gets a new board }; struct State { // Any thread. @@ -86,13 +106,32 @@ struct State { float angle{}; // how far a hanging end droops (radians), or how deep a fold sags (0 to 1) std::uint32_t flights{}; // counts the breaks, so each starts a flight of its own bool counted{}, hooked{}; - std::uint64_t wipeouts{}, back_since{}, last_error{}, rode_at{}, board_seen{}; + std::uint64_t wipeouts{}, back_since{}, last_error{}, board_seen{}; // The ground under the board: its own height while it rolls on it, and what of that was fresh // when the board broke (else not a number). float ride_height{}; std::uint64_t ride_when{}; float break_ground{std::numeric_limits::quiet_NaN()}; std::mt19937 random{std::random_device{}()}; + // The player's settings as the physics thread reads them (feel() holds the real ones). + std::atomic always{}, rails{true}, wear_on{true}, body_sounds{true}; + std::atomic sensitivity{100}; + // Wear: the board in the game's (for the physics thread), and what the physics thread has + // added since the tick took it. + std::atomic wear_now{}, wear_added{}; + // A board that came down too hard onto a rail, seen by the physics thread: the tick breaks it. + std::atomic rail_snap{}; + std::atomic landing_give{}; // the physics thread saw a done board land too hard + std::atomic landing_speed{}; + std::atomic rail_contact{}; + std::atomic rail_speed{}; + // The board's long axis (towards the nose) and its up axis in the world, from the tick. + std::mutex axes_mutex; + Vec along{}, up{}; + std::uint64_t axes_at{}; + // Client thread: the tuning set for wear, and when; when the settings were last saved. + std::uint64_t handled_at{}, saved_at{}, session_checked{}; + bool in_session{}, setup_read{}, effects_hooked{}; }; State& state() { static auto* value = new State; @@ -106,6 +145,8 @@ struct RenderEdit { std::atomic hinge{}; std::atomic angle{}; std::atomic park{}; // a breakable deck is in the game: whole boards hide their stretches + // The deck's own trucks (deck.ini: axle): how far apart the wheels sit against a standard board's. + std::atomic axle{1.0f}; }; RenderEdit& render_edit() { static RenderEdit value; return value; } @@ -170,6 +211,28 @@ void hold_baseplates(std::uintptr_t buffer) noexcept { hold(front_truck_bone, front_plate_bone, front_truck_rest); } +// A deck's own trucks (deck.ini: axle): the wheels further apart on a wider deck, closer together on +// a narrower one. The game puts each wheel on its truck's bone along the axle, about 0.0876 m out; +// that place is set here rather than scaled, so a pose that comes by twice is not moved twice. Only +// what is drawn changes: the board rides as it always does. +constexpr float wheel_offset = 0.0876f; +void spread_wheels(std::uintptr_t buffer) noexcept { + const float axle = render_edit().axle.load(std::memory_order_relaxed); + if (axle == 1.0f) return; + for (const int index : wheel_bones) { + PoseBone wheel{}; + if (!load_bone(bone_at(buffer, index), wheel)) continue; + int way = 0; // the axle, in the truck's axes: the largest part of where the wheel sits + for (int i = 1; i < 3; ++i) + if (std::abs(wheel.position[i]) > std::abs(wheel.position[way])) way = i; + const float out = std::abs(wheel.position[way]); + if (!(out > 0.05f && out < 0.16f)) continue; // not where the game puts a wheel + __try { + reinterpret_cast(bone_at(buffer, index))[8 + way] = std::copysign(wheel_offset * axle, wheel.position[way]); + } __except (EXCEPTION_EXECUTE_HANDLER) {} + } +} + // ---- the piece that comes off -------------------------------------------------------------------- // The client tick starts a flight (a new id) when the board breaks; the render pose hook moves the // piece through the world from there (a kick, gravity, the ground) and writes it relative to @@ -303,6 +366,7 @@ void draw_broken(std::uintptr_t base, std::uintptr_t holder, std::uint32_t kept, !load_bone(bone_at(buffer, deck_bone), deck)) return; hold_baseplates(buffer); + spread_wheels(buffer); // From where its animation put it this frame; a pose nobody has written since the last move stays as it is. if (const Vec at = place_of(deck); length(at) < 100.0f) (void)store_bone(bone_at(buffer, deck_bone), facing_of(deck), at + away); auto layout = lay_out(kept, hinge, angle, turn(facing_of(root), {0, 1, 0}).y, facing_of(mount)); @@ -323,71 +387,508 @@ void park_pieces(std::uintptr_t base, std::uintptr_t holder) { const auto buffer = board_pose(base, holder); if (!buffer) return; hold_baseplates(buffer); + spread_wheels(buffer); const auto bones = bones_of({}); for (std::size_t i = 0; i < bones.size(); ++i) (void)store_bone(bone_at(buffer, carrier[i]), bones[i].facing, bones[i].at); } -// ---- breaking and mending -------------------------------------------------------------------------- -// The crack of the deck, straight to the speakers (not through the game's mixer): a wave file out -// of the `sounds` folder of one of Board Break's deck mods. Its subfolders light, medium and heavy -// hold the sounds for a break of that size, as many as one likes under any names; a size with none -// of its own takes the nearest size that has some, and wave files in the folder itself serve when -// no subfolder does. -void crack_sound(State& s, Size size) { - static const std::array, 4> sounds = [] { - std::array, 4> found; - constexpr std::array names{L"light", L"medium", L"heavy", L""}; +// ---- the player's settings, the deck's setup, wear ------------------------------------------------- +// Kept in %LOCALAPPDATA%\ReSkate\BoardBreak.settings, a line each: the settings by name (as the +// console knows them), each deck's wear (wear.=0.37), the materials marked as metal. +struct Feel { + std::mutex mutex; + Settings settings; + std::map wear; // by deck + std::set metal; // marked by the player + std::set pack_metal; // a pack's list (sounds/metal/materials.txt) + bool loaded{}, dirty{}; +}; +Feel& feel() { + static auto* value = new Feel; + return *value; +} +std::filesystem::path settings_path() { + PWSTR raw{}; + std::filesystem::path result; + if (SUCCEEDED(SHGetKnownFolderPath(FOLDERID_LocalAppData, 0, nullptr, &raw)) && raw) + result = std::filesystem::path(raw) / L"ReSkate" / L"BoardBreak.settings"; + if (raw) CoTaskMemFree(raw); + return result; +} +std::string trimmed(std::string_view text) { + while (!text.empty() && std::isspace(static_cast(text.front()))) text.remove_prefix(1); + while (!text.empty() && std::isspace(static_cast(text.back()))) text.remove_suffix(1); + return std::string(text); +} +template bool number(std::string_view text, T& out) { + const auto parsed = std::from_chars(text.data(), text.data() + text.size(), out); + return parsed.ec == std::errc{} && parsed.ptr == text.data() + text.size(); +} +// Numbers out of a line of them ("12 40, 77"). +std::set numbers_in(std::string_view text) { + std::set out; + std::string one; + const auto take = [&] { + unsigned value{}; + if (!one.empty() && number(std::string_view(one), value) && value <= 0xffff) out.insert(static_cast(value)); + one.clear(); + }; + for (const char c : text) { + if (c >= '0' && c <= '9') one += c; + else take(); + } + take(); + return out; +} +// Under the lock. +void load_feel(Feel& f) { + if (f.loaded) return; + f.loaded = true; + try { + std::ifstream in(settings_path()); + std::string line; + while (std::getline(in, line)) { + const auto eq = line.find('='); + if (eq == std::string::npos) continue; + const auto key = trimmed(std::string_view(line).substr(0, eq)); + const auto value = trimmed(std::string_view(line).substr(eq + 1)); + if (key.starts_with("wear.")) { + float worn{}; + if (number(std::string_view(value), worn) && std::isfinite(worn)) f.wear[key.substr(5)] = std::clamp(worn, 0.0f, 1.0f); + } else if (key == "metal") f.metal = numbers_in(value); + else if (int setting{}; number(std::string_view(value), setting)) (void)set_setting_value(f.settings, key, setting); + } + } catch (...) {} +} +// Written on a thread of its own: this is called from the client tick every few seconds while +// wear changes, and a write (a folder lookup, a file truncated and written, whatever the virus +// scanner does on close) held the game's frame up (2026-10-15). The text is taken under the lock; +// the writes are kept in order by their own. +void save_feel(Feel& f) { + try { + std::ostringstream out; + for (const auto& info : setting_info) out << info.name << "=" << setting_value(f.settings, info.name) << "\n"; + for (const auto& [deck, worn] : f.wear) out << "wear." << deck << "=" << std::format("{:.4f}", worn) << "\n"; + if (!f.metal.empty()) { + out << "metal="; + for (const auto m : f.metal) out << m << " "; + out << "\n"; + } + f.dirty = false; + std::thread([text = out.str()] { + static std::mutex writing; + std::lock_guard lock(writing); + try { + const auto path = settings_path(); + if (path.empty()) return; + std::error_code error; + std::filesystem::create_directories(path.parent_path(), error); + std::ofstream file(path, std::ios::trunc); + file << text; + } catch (...) {} + }).detach(); + } catch (...) {} +} +// The settings into the atomics the physics thread reads. +void mirror(State& s, const Settings& st) { + s.always.store(st.always, std::memory_order_relaxed); + s.rails.store(st.rails, std::memory_order_relaxed); + s.wear_on.store(st.wear, std::memory_order_relaxed); + s.body_sounds.store(st.body_sounds, std::memory_order_relaxed); + s.sensitivity.store(st.sensitivity, std::memory_order_relaxed); +} +Settings settings_now() { + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + return f.settings; +} +std::string deck_now(State& s) { + std::lock_guard lock(s.names); + return s.deck; +} + +// What a deck says of itself, in its folder's deck.ini: `axle` for its trucks (1 = a standard board's; +// more on a wider deck) and `label`, its name in the menus. +struct DeckSetup { + float axle = 1.0f; + std::string label; +}; +DeckSetup read_deck_setup(std::string_view deck) { + DeckSetup out; + if (deck.empty()) return out; + try { + const auto folder = mods::board_break_deck_folder(deck); + if (folder.empty()) return out; + std::ifstream in(folder / L"deck.ini"); + std::string line; + while (std::getline(in, line)) { + if (const auto hash = line.find_first_of("#;"); hash != std::string::npos) line.erase(hash); + const auto eq = line.find('='); + if (eq == std::string::npos) continue; + auto key = trimmed(std::string_view(line).substr(0, eq)); + for (auto& c : key) c = static_cast(std::tolower(static_cast(c))); + const auto value = trimmed(std::string_view(line).substr(eq + 1)); + float amount{}; + if (key == "axle" && number(std::string_view(value), amount) && std::isfinite(amount)) out.axle = std::clamp(amount, 0.8f, 1.6f); + else if (key == "label") out.label = value; + } + } catch (...) {} + return out; +} + +// The same, kept a minute per deck: the menus ask every frame. +DeckSetup setup_of(std::string_view deck) { + struct Kept { + DeckSetup setup; + std::uint64_t at{}; + }; + static std::mutex mutex; + static std::map> kept; + const auto now = GetTickCount64(); + { + std::lock_guard lock(mutex); + // (A minute: each look lists every mod's folder, and the world model asked every few seconds + // on the client thread while a breakable deck was out.) + if (const auto at = kept.find(deck); at != kept.end() && now - at->second.at < 60000) return at->second.setup; + } + auto setup = read_deck_setup(deck); + std::lock_guard lock(mutex); + kept[std::string(deck)] = {setup, now}; + return setup; +} + +// ---- sounds --------------------------------------------------------------------------------------- +// Out of the `sounds` folder of one of Board Break's deck mods: light, medium and heavy for the +// crack of a break that size, metal for a board breaking on metal, body for the body's hits in a +// bail; as many wave files in each as one likes, under any names. A crack size with none of its own +// takes the nearest that has some, and wave files in the folder itself serve when no subfolder does. +// sounds/metal/materials.txt may list the level's materials that are metal. +void load_sounds() { + static std::once_flag once; + std::call_once(once, [] { + const auto wavs = [](const std::filesystem::path& folder) { + std::vector found; + std::error_code failed; + for (std::filesystem::directory_iterator at(folder, failed), end; !failed && at != end; at.increment(failed)) { + auto extension = at->path().extension().wstring(); + for (auto& c : extension) c = static_cast(towlower(c)); + if (at->is_regular_file(failed) && extension == L".wav") found.push_back(at->path()); + } + std::sort(found.begin(), found.end()); + return found; + }; for (const auto& mod : mods::board_break_folders()) { std::error_code failed; - if (!std::filesystem::is_directory(mod / L"sounds", failed)) continue; - for (std::size_t i = 0; i < names.size(); ++i) { - for (std::filesystem::directory_iterator at(mod / L"sounds" / names[i], failed), end; !failed && at != end; at.increment(failed)) { - auto extension = at->path().extension().wstring(); - for (auto& c : extension) c = static_cast(towlower(c)); - if (at->is_regular_file(failed) && extension == L".wav") found[i].push_back(at->path()); - } - std::sort(found[i].begin(), found[i].end()); + const auto folder = mod / L"sounds"; + if (!std::filesystem::is_directory(folder, failed)) continue; + const auto loose = wavs(folder); + constexpr std::array, 5> kinds{{{audio::Kind::light, L"light"}, {audio::Kind::medium, L"medium"}, + {audio::Kind::heavy, L"heavy"}, {audio::Kind::metal, L"metal"}, {audio::Kind::body, L"body"}}}; + for (const auto& [kind, name] : kinds) { + auto files = wavs(folder / name); + if (files.empty() && kind <= audio::Kind::heavy) files = loose; + audio::load(kind, files); } + try { + std::ifstream in(folder / L"metal" / L"materials.txt"); + std::string line, all; + while (std::getline(in, line)) + if (const auto hash = line.find('#'); hash != 0) all += " " + line.substr(0, hash); + auto& f = feel(); + std::lock_guard lock(f.mutex); + f.pack_metal = numbers_in(all); + } catch (...) {} break; } - return found; - }(); + }); +} +void crack_sound(Size size) { + if (!settings_now().crack_sounds) return; // Nearest first: a medium break without sounds of its own takes a heavy one before a light one. - static constexpr std::array, 3> order{{{0, 1, 2, 3}, {1, 2, 0, 3}, {2, 1, 0, 3}}}; - for (const auto from : order[static_cast(size)]) { - const auto& files = sounds[from]; - if (files.empty()) continue; - const auto& file = files[std::uniform_int_distribution(0, files.size() - 1)(s.random)]; - (void)PlaySoundW(file.c_str(), nullptr, SND_FILENAME | SND_ASYNC | SND_NODEFAULT); - return; + static constexpr std::array, 3> order{{{audio::Kind::light, audio::Kind::medium, audio::Kind::heavy}, + {audio::Kind::medium, audio::Kind::heavy, audio::Kind::light}, + {audio::Kind::heavy, audio::Kind::medium, audio::Kind::light}}}; + for (const auto kind : order[static_cast(size)]) + if (audio::has(kind)) return audio::play(kind, 1.0f); +} + +// ---- what a board broke on -------------------------------------------------------------------------- +// The material of the level the board touched most in the last half second, near `at`: the game's +// own contact effects say (Extension/Multiplayer/Remote/native_vfx.h). +std::optional material_near(Vec at, std::uint64_t within_ms, float reach) { + std::map count; + for (const auto& one : multiplayer::recent_local_impacts(within_ms)) { + const auto& p = one.impact.position; + if (std::hypot(p[0] - at.x, p[1] - at.y, p[2] - at.z) <= reach) ++count[one.impact.material]; + } + std::optional best; + int most{}; + for (const auto& [material, times] : count) + if (times > most) best = material, most = times; + return best; +} +bool is_metal(std::uint16_t material) { + auto& f = feel(); + std::lock_guard lock(f.mutex); + return f.metal.contains(material) || f.pack_metal.contains(material); +} + +// ---- wear ---------------------------------------------------------------------------------------- +// Client thread: what the physics thread added goes onto the deck in the game's board. +void take_wear(State& s, const std::string& deck, float extra = 0.0f) { + const float added = s.wear_added.exchange(0.0f, std::memory_order_acq_rel) + extra; + if (deck.empty() || added <= 0.0f) return; + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + auto& worn = f.wear[deck]; + const auto before = worn; + worn = add_wear(worn, added, f.settings.wear_rate); + s.wear_now.store(worn, std::memory_order_relaxed); + if (worn - before >= 0.01f) say(logging::Level::debug, "a hard hit: the board is {:.1f}% worn now (+{:.1f}).", worn * 100.0f, (worn - before) * 100.0f); + if (worn != before) f.dirty = true; + if (condition_name(before) != condition_name(worn)) { + say(logging::Level::info, "the {} board is {} now ({:.0f}% worn).", deck_label(deck), condition_name(worn), worn * 100.0f); + // Told as it gets close, so a break that follows is no surprise. + if (condition_name(worn) == "beat") + overlay::notify(overlay::NoticeLevel::info, "Your board is beat", + "It breaks from smaller slams now. New board: ReSkate menu (Insert) > Addons > Board Break."); + else if (condition_name(worn) == "done") + overlay::notify(overlay::NoticeLevel::warning, "Your board is done", + "The next big drop snaps it. New board: ReSkate menu (Insert) > Addons > Board Break."); } } +// A board that broke is a new one once it is whole again: no wear, no scratches. +void new_board(State& s, const std::string& deck) { + if (deck.empty()) return; + auto& f = feel(); + { + std::lock_guard lock(f.mutex); + load_feel(f); + f.wear[deck] = 0.0f; + f.dirty = true; + } + if (deck == deck_now(s)) { + s.wear_now.store(0.0f, std::memory_order_relaxed); + s.wear_added.store(0.0f, std::memory_order_relaxed); + profile_runtime::reset_board_wear(); + } +} +void add_physics_wear(State& s, float amount) { + if (!(amount > 0.0f)) return; + float current = s.wear_added.load(std::memory_order_relaxed); + while (!s.wear_added.compare_exchange_weak(current, current + amount, std::memory_order_acq_rel)) {} +} +// What wear does to the board's handling: four values of the game's physics tuning, the game's own +// value times handling()'s factor. A value the player changed (the trainer) is left alone; outside +// a multiplayer session only, which shares tuning between its players. +struct Tuned { + bool tried{}, known{}, wrote{}; + std::array offset{}; + std::array game{}, written{}; +}; +Tuned& tuned() { + static Tuned value; + return value; +} +void apply_handling(std::uintptr_t base, std::uintptr_t entity, const Handling& want) { + auto& t = tuned(); + if (!t.known) { + physics_tuning::prepare(); + const auto model = physics_tuning::game_tuning(); + if (!model) return; + constexpr std::array names{"PhysicsSteering.GeneralScalar", "PhysicsTrucks.StaticTruckSpringStrength", + "PhysicsTrucks.StaticTruckDamping", "PhysicsTrucks.TruckTwistLimit"}; + std::size_t found{}; + for (std::size_t i = 0; i < model->fields.size() && i < model->field_names.size(); ++i) + for (std::size_t k = 0; k < names.size(); ++k) + if (model->field_names[i] == names[k] && model->fields[i].real && model->fields[i].offset + 4u <= model->image.size()) { + t.offset[k] = model->fields[i].offset; + std::memcpy(&t.game[k], model->image.data() + t.offset[k], 4); + ++found; + } + if (!t.tried) { + t.tried = true; + if (found != names.size()) say(logging::Level::warning, "wear cannot change the handling: {} of the 4 tuning values found.", found); + } + if (found != names.size()) return; + t.known = true; + } + physics_tuning::Values live; + if (!physics_tuning::read_live(base, live)) return; + const std::array factor{want.steering, want.spring, want.damping, want.twist}; + bool change{}; + for (std::size_t k = 0; k < factor.size(); ++k) { + if (t.offset[k] + 4u > live.image.size()) return; + float now{}; + std::memcpy(&now, live.image.data() + t.offset[k], 4); + const auto same = [](float a, float b) { return std::abs(a - b) <= 1e-4f * (std::max)(1.0f, std::abs(b)); }; + if (!same(now, t.game[k]) && !(t.wrote && same(now, t.written[k]))) continue; + const float target = t.game[k] * factor[k]; + t.written[k] = target; + if (!same(now, target)) { + std::memcpy(live.image.data() + t.offset[k], &target, 4); + change = true; + } + } + t.wrote = true; + if (change) (void)physics_tuning::write_live(base, entity, live); +} +} // namespace + +std::optional handling_value(std::uint16_t offset) noexcept { + const auto& t = tuned(); + if (!t.known || !t.wrote) return {}; + for (std::size_t k = 0; k < t.offset.size(); ++k) + if (t.offset[k] == offset) return t.written[k]; + return {}; +} + +namespace { + +// ---- the board's axes, for the physics thread ------------------------------------------------------- +void note_axes(State& s, std::uintptr_t base, std::uintptr_t holder) { + const auto buffer = board_pose(base, holder); + PoseBone root{}, mount{}; + if (!buffer || !load_bone(bone_at(buffer, root_bone), root) || !load_bone(bone_at(buffer, mount_bone), mount)) return; + const auto facing = unit(facing_of(root) * facing_of(mount)); + std::lock_guard lock(s.axes_mutex); + s.along = turn(facing, {0, 0, 1}); + s.up = turn(facing, {0, 1, 0}); + s.axes_at = GetTickCount64(); +} +bool board_axes(State& s, Vec& along, Vec& up) { + std::lock_guard lock(s.axes_mutex); + if (!s.axes_at || GetTickCount64() - s.axes_at > 250) return false; + along = s.along; + up = s.up; + return true; +} + +// ---- every physics step of the local skater (No Bail's step listeners) ------------------------------ +// The board's root body velocity (as the grind exit control reads it). +bool board_velocity(const NoBailSkater& skater, std::array& velocity) noexcept { + std::uintptr_t holder{}, physics{}, type{}, bodies{}, owner{}; + std::uint32_t count{}; + if (!memory::peek(skater.core + sstate::board_holder_offset, holder) || + !memory::peek(holder + sstate::board_physics_offset, physics) || !memory::peek(physics, type) || + type != skater.base + sstate::board_physics_vtable || !memory::peek(physics + sstate::physics_bodies_offset, bodies) || + !memory::peek(bodies, count) || count != sstate::board_body_count) + return false; + const auto body = bodies + sstate::board_root_body * sstate::physics_body_size; + if (!memory::peek(body + sstate::body_owner_offset, owner) || owner != physics || !memory::peek(body + sstate::body_velocity_offset, velocity)) + return false; + return std::all_of(velocity.begin(), velocity.end(), [](float v) { return std::isfinite(v) && std::abs(v) < 1000; }); +} +struct StepWatch { + bool was_air{}; + float air_down{}; // how fast the board was coming down, in its last step in the air + std::array body_at{}; + std::uint64_t any_at{}; +}; +void on_step(const NoBailSkater& skater, float seconds, bool wipeout) noexcept { + auto& s = state(); + if (!s.ready.load(std::memory_order_acquire) || !s.on.load(std::memory_order_relaxed)) return; + static StepWatch w; // the physics thread's own + thread_local std::mt19937 roll{std::random_device{}()}; + const auto error = GetLastError(); + std::uint32_t physics{}; + if (memory::peek(skater.context + sstate::physics_state_offset, physics)) { + std::array v{}; + const bool known = board_velocity(skater, v); + const bool air = physics == 200 || physics == 201; + const bool grinding = physics >= 400 && physics < 500; + const bool rolling = physics >= 100 && physics < 200; + const bool wear_on = s.wear_on.load(std::memory_order_relaxed); + Vec along{}, up{}; + const auto contact = known && board_axes(s, along, up) ? contact_of(along, up, {v[0], v[1], v[2]}) : Contact::grind; + if (air && known) w.air_down = -v[1]; + if (w.was_air && (rolling || grinding)) { + const float down = w.air_down; + if (wear_on) add_physics_wear(s, rolling ? landing_wear(down) : rail_wear(down, contact != Contact::grind)); + const bool can_break = s.has_deck.load(std::memory_order_relaxed) && !s.rail_snap.load(std::memory_order_acquire); + // A done board gives way on a landing hard enough for how worn it is. + if (can_break && rolling && wear_on && !s.landing_give.load(std::memory_order_acquire) && + gives_way_on_landing(down, s.sensitivity.load(std::memory_order_relaxed), s.wear_now.load(std::memory_order_relaxed))) { + s.landing_speed.store(down, std::memory_order_relaxed); + s.landing_give.store(true, std::memory_order_release); + } + if (can_break && grinding && contact != Contact::grind && s.rails.load(std::memory_order_relaxed)) { + const float chance = s.always.load(std::memory_order_relaxed) ? (down > 1.5f ? 1.0f : 0.0f) + : rail_chance(down, contact, s.sensitivity.load(std::memory_order_relaxed), + s.wear_now.load(std::memory_order_relaxed)); + if (std::uniform_real_distribution(0.0f, 1.0f)(roll) < chance) { + s.rail_contact.store(static_cast(contact), std::memory_order_relaxed); + s.rail_speed.store(down, std::memory_order_relaxed); + s.rail_snap.store(true, std::memory_order_release); + // The board snaps under the rider: the rider goes down with it. Gently: the game reads the + // cause's force as how hard it was hit, and a slam's force (10 and more) with the board + // flat on the ground threw the rider high into the air. + no_bail_request_wipeout(2.5f); + } + } + } + if (grinding && wear_on) add_physics_wear(s, grind_wear_per_second(contact != Contact::grind) * seconds); + w.was_air = air; + } + // The body's hits in a bail, each where and as hard as it hit. + if (s.body_sounds.load(std::memory_order_relaxed) && hall_of_meat::skater_available()) { + const auto step = hall_of_meat::read_step(skater, seconds, wipeout); + if (step.ragdoll.value_or(false)) { + const auto now = GetTickCount64(); + for (std::size_t i = 1; i < skater_body::count; ++i) { + const auto& c = step.body.bodies[i]; + if (skater_body::foot(static_cast(i)) || !c.touching || c.impact < body_hit) continue; + if (now - w.body_at[i] < 220 || now - w.any_at < 70) continue; + w.body_at[i] = w.any_at = now; + audio::play(audio::Kind::body, body_loudness(c.impact)); + break; + } + } + } + SetLastError(error); +} + +// ---- breaking and mending -------------------------------------------------------------------------- // `carried`: the skater is on foot with the board. The grip tape always holds then: a piece that // came off would have to fall to the ground under the skater's feet, and where that is the board // in a hand does not tell (a loose piece takes the ground from where the board rolled or lies). bool snap(State& s, std::uintptr_t board, std::optional at, bool gave_way, bool carried, std::string_view why, bool asked = false) { if (!s.has_deck.load(std::memory_order_acquire) || s.broken || !board) return false; const auto place = at ? *at : static_cast(std::uniform_int_distribution(0, 2)(s.random)); - const bool joined = carried || std::uniform_real_distribution(0.0f, 1.0f)(s.random) < joined_share; + const bool joined = carried || std::uniform_real_distribution(0.0f, 1.0f)(s.random) < settings_now().joined / 100.0f; const auto now = GetTickCount64(); s.parts = broken_at(place, joined); s.angle = s.parts.hinge == Hinge::fold ? std::uniform_real_distribution(0.35f, 1.0f)(s.random) : std::uniform_real_distribution(0.30f, 0.65f)(s.random); s.break_ground = s.ride_when && now - s.ride_when < 400 ? s.ride_height : std::numeric_limits::quiet_NaN(); ++s.flights; - s.broken = Broken{board, now, gave_way, asked}; + s.broken = Broken{board, now, gave_way, asked, deck_now(s)}; s.back_since = 0; - crack_sound(s, break_size(place, joined, s.angle)); - say(logging::Level::debug, "the board broke at the {}{} ({}).", place_names[static_cast(place)], + crack_sound(break_size(place, joined, s.angle)); + say(logging::Level::info, "the board broke at the {}{} ({}).", place_names[static_cast(place)], joined ? ", held together by the grip tape" : "", why); return true; } void mend(State& s, std::string_view why) { if (!s.broken) return; say(logging::Level::debug, "the board is whole again after {:.1f} s ({}).", (GetTickCount64() - s.broken->since) / 1000.0, why); + // A board that really broke is a new one now; one broken from the console was only looked at. + if (!s.broken->asked) new_board(s, s.broken->deck); s.broken.reset(); s.parts = {}; } +// A board that broke on metal (a rail, a bench's edge, a car) sounds like it. +void metal_sound(std::uintptr_t base, std::uintptr_t holder) { + Vec at{}; + Quat facing{}; + if (!board_place(base, holder, at, facing)) return; + const auto material = material_near(at, 600, 3.0f); + const bool metal = material && is_metal(*material); + if (metal && settings_now().crack_sounds) audio::play(audio::Kind::metal, 1.0f); + say(logging::Level::debug, "it broke on material {}{}.", material ? std::to_string(*material) : std::string("unknown"), metal ? " (metal)" : ""); +} // Where the board is relative to the skater: how far, and how flat it lies (1 = flat on the ground // either way up, 0 = on its edge, as when it is carried). struct Hold { @@ -473,6 +974,16 @@ bool start(std::uintptr_t base) noexcept { } s.base.store(base, std::memory_order_release); s.on.store(profile_runtime::local_preference(enabled_preference).value_or(false), std::memory_order_release); + { + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + mirror(s, f.settings); + } + // Rail snaps, wear and the body's hits come from each physics step of the local skater. + if (!add_no_bail_step_listener(&on_step)) + logging::write(logging::Level::warning, logging::Channel::skater, + "Board Break: No Bail has no room for its physics step listener: no rail snaps, wear or body sounds."); const auto in_game = mods::board_break_in_play(); { std::lock_guard lock(s.names); @@ -545,11 +1056,32 @@ void on_client_tick(std::uintptr_t client) noexcept { wanted.swap(s.wanted_deck); } s.has_deck.store(in_game, std::memory_order_release); + if (changed || !s.setup_read) { + s.setup_read = true; + const auto setup = read_deck_setup(deck_in_game); + render_edit().axle.store(setup.axle, std::memory_order_release); + { + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + const auto worn = f.wear.find(deck_in_game); + s.wear_now.store(worn == f.wear.end() ? 0.0f : worn->second, std::memory_order_relaxed); + } + if (in_game) load_sounds(); + if (in_game && setup.axle != 1.0f) + say(logging::Level::info, "the {} deck's wheels sit {:.2f} as far apart as a standard board's.", deck_label(deck_in_game), setup.axle); + } if (changed) say(logging::Level::info, "the deck in the game is now: {}{}.", deck_label(deck_in_game), in_game ? "" : " (boards cannot break)"); if (!wanted.empty() && wanted != deck_in_game && mods::save_board_break_deck(wanted)) { mend(s, "another deck"); say(logging::Level::info, "deck {} chosen. {}", deck_label(wanted), live_mods::apply(true)); } + if (NoBailSkater probe; in_game && !s.effects_hooked && no_bail_skater(probe)) { + // What a board broke on comes from the game's contact effects (native_vfx.h). The hook is + // installed once, in a world (multiplayer would do it there too). + s.effects_hooked = true; + multiplayer::prepare_effects(base); + } if (in_game && !s.hooked) { // The boards' render pose hook is shared with multiplayer and first person, and only // installed when one of them asks. @@ -579,7 +1111,37 @@ void on_client_tick(std::uintptr_t client) noexcept { return; } s.board_seen = now; + note_axes(s, base, holder); if (s.broken && s.broken->board != board) mend(s, "the skater has another board"); + // What wear has done, onto the board; and what that does to its handling. + take_wear(s, deck_in_game); + if (now - s.handled_at > 500) { + s.handled_at = now; + if (now - s.session_checked > 2000) { + s.session_checked = now; + const auto session = multiplayer::model(); + s.in_session = session.active || session.connected; + } + const auto st = settings_now(); + const bool worn = s.on.load(std::memory_order_acquire) && st.wear && !s.in_session; + apply_handling(base, skater.entity, worn ? handling(s.wear_now.load(std::memory_order_relaxed)) : Handling{}); + } + if (now - s.saved_at > 5000) { + s.saved_at = now; + auto& f = feel(); + std::lock_guard lock(f.mutex); + if (f.dirty) save_feel(f); + } + // A board that came down too hard onto a rail in a slide (the physics thread saw it). + if (s.rail_snap.exchange(false, std::memory_order_acq_rel) && s.on.load(std::memory_order_acquire) && !s.broken) { + const auto contact = static_cast(s.rail_contact.load(std::memory_order_relaxed)); + const auto what = std::format("a {} onto a rail at {:.1f} m/s", contact_names[(std::min)(contact, contact_names.size() - 1)], s.rail_speed.load()); + if (snap(s, board, Place::mid, false, false, what)) metal_sound(base, holder); + } + // A done board that landed too hard (the physics thread saw it): the nose goes, and they ride on. + if (s.landing_give.exchange(false, std::memory_order_acq_rel) && s.on.load(std::memory_order_acquire) && !s.broken) + (void)snap(s, board, Place::nose, true, false, + std::format("a landing at {:.1f} m/s on a board that was done", s.landing_speed.load(std::memory_order_relaxed))); watch_physics_state(client, skater.entity); const auto watch = watched_physics_state(); if (s.wanted_mend.exchange(false, std::memory_order_acq_rel)) mend(s, "asked for"); @@ -608,16 +1170,20 @@ void on_client_tick(std::uintptr_t client) noexcept { if (hit->reason.load() < 0 || now - hit->at.load() > recent_ms) hit = &last_cause(); const bool known = now - hit->at.load() <= recent_ms && hit->reason.load() >= 0; const auto force = known ? hit->force.load() : 0.0f; - if (on && !s.broken && std::uniform_real_distribution(0.0f, 1.0f)(s.random) < break_chance(force, heavy, light)) - (void)snap(s, board, std::nullopt, false, watch.previous >= 500 && watch.previous < 600, std::format("a slam of force {:.1f}", force)); + const auto st = settings_now(); + if (on && st.wear) take_wear(s, deck_in_game, slam_wear(force)); + const float chance = st.always ? 1.0f : slam_chance(force, st.sensitivity, s.wear_now.load(std::memory_order_relaxed)); + if (on && !s.broken && std::uniform_real_distribution(0.0f, 1.0f)(s.random) < chance) { + // Nose first it goes at the nose, tail first at the tail, flat mostly in the middle. + Vec along{}, up{}; + const float slope = board_axes(s, along, up) ? -along.y : 0.0f; + const auto place = place_of_break(slope, false, std::uniform_real_distribution(0.0f, 1.0f)(s.random)); + if (snap(s, board, place, false, watch.previous >= 500 && watch.previous < 600, + std::format("a slam of force {:.1f}{}", force, st.always ? ", set to always break" : ""))) + metal_sound(base, holder); + } } - // Now and then a board simply gives way under the rider: the nose goes. - if (on && !s.broken && s.rode_at && riding(watch.state)) { - const auto seconds = (std::min)(now - s.rode_at, 250) / 1000.0; - if (std::uniform_real_distribution(0.0, 1.0)(s.random) < gives_way * seconds) - (void)snap(s, board, Place::nose, true, false, "it gave way while riding"); - } - s.rode_at = now; + if (s.broken) { // Whole again once the skater rides it, or has picked it up after the fall. A board that // gave way under the rider stays as it is while it is ridden; after a fall it is a @@ -660,6 +1226,7 @@ std::string deck() { } std::string deck_label(std::string_view deck) { if (deck.empty()) return "None"; + if (auto title = setup_of(deck).label; !title.empty()) return title; std::string name(deck); bool start = true; for (auto& c : name) { @@ -688,4 +1255,84 @@ std::string choose_deck(std::string wanted) { } void snap(int place) noexcept { state().wanted_snap.store(place >= 0 && place < 3 ? place : 3, std::memory_order_release); } void mend() noexcept { state().wanted_mend.store(true, std::memory_order_release); } + +Settings settings() noexcept { + try { + return settings_now(); + } catch (...) { return {}; } +} +bool set_setting(std::string_view name, int value) noexcept { + try { + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + if (!set_setting_value(f.settings, name, value)) return false; + mirror(state(), f.settings); + save_feel(f); + return true; + } catch (...) { return false; } +} +float wear() noexcept { return state().wear_now.load(std::memory_order_relaxed); } +void set_wear(float value) noexcept { + try { + auto& s = state(); + const auto deck = deck_now(s); + if (deck.empty()) return; + value = std::clamp(value, 0.0f, 1.0f); + { + auto& f = feel(); + std::lock_guard lock(f.mutex); + load_feel(f); + f.wear[deck] = value; + save_feel(f); + } + s.wear_now.store(value, std::memory_order_relaxed); + if (value == 0.0f) profile_runtime::reset_board_wear(); + } catch (...) {} +} +std::string metal_command(std::string_view action) { + auto& f = feel(); + if (action == "mark" || action == "unmark") { + const auto material = material_near({}, 1500, std::numeric_limits::infinity()); + if (!material) return "No contacts in the last second and a half: grind the rail first, then mark it."; + std::lock_guard lock(f.mutex); + load_feel(f); + if (action == "mark") f.metal.insert(*material); + else f.metal.erase(*material); + save_feel(f); + return std::format("Material {} is {} metal.", *material, action == "mark" ? "now" : "no longer"); + } + std::lock_guard lock(f.mutex); + load_feel(f); + if (action == "clear") { + f.metal.clear(); + save_feel(f); + return "No material is marked as metal now (a pack's own list still counts)."; + } + std::string list; + for (const auto m : f.metal) list += (list.empty() ? "" : ", ") + std::to_string(m); + std::string pack; + for (const auto m : f.pack_metal) pack += (pack.empty() ? "" : ", ") + std::to_string(m); + return "Metal: " + (list.empty() ? std::string("none marked") : list) + "; from the deck pack: " + (pack.empty() ? std::string("none") : pack) + "."; +} +std::string materials_report() { + std::map count; + for (const auto& one : multiplayer::recent_local_impacts(3000)) ++count[one.impact.material]; + if (count.empty()) return "No contacts in the last 3 seconds."; + std::string out = "Materials touched in the last 3 s:"; + for (const auto& [material, times] : count) out += std::format(" {} ({}x{})", material, times, is_metal(material) ? ", metal" : ""); + return out + "."; +} +std::string status() { + auto& s = state(); + if (!available()) return "Board Break is unavailable for this game build."; + const auto st = settings_now(); + const auto deck = deck_now(s); + const auto worn = s.wear_now.load(std::memory_order_relaxed); + return std::format("Board Break {}; deck {}; the board is {} ({:.0f}% worn); sensitivity {}%, always break {}, rail snaps {}, wear {} at {}%, " + "grip holds {}% of breaks, crack sounds {}, body sounds {}.", + s.on.load() ? "on" : "off", deck_label(deck), condition_name(worn), worn * 100.0f, + st.sensitivity, st.always ? "on" : "off", st.rails ? "on" : "off", st.wear ? "on" : "off", st.wear_rate, st.joined, + st.crack_sounds ? "on" : "off", st.body_sounds ? "on" : "off"); +} } // namespace dingosdk::board_break diff --git a/Extension/BoardBreak/board_break.h b/Extension/BoardBreak/board_break.h index 5771e9df..555c84e3 100644 --- a/Extension/BoardBreak/board_break.h +++ b/Extension/BoardBreak/board_break.h @@ -1,5 +1,7 @@ #pragma once +#include "board_break_feel.h" #include +#include #include #include @@ -60,4 +62,27 @@ std::string choose_deck(std::string wanted); // respawns. Or makes a broken board whole. void snap(int place) noexcept; void mend() noexcept; + +// ---- the feel of it ------------------------------------------------------------------------------- +// The player's settings (board_break_feel.h), kept in %LOCALAPPDATA%\ReSkate\BoardBreak.settings. +// Any thread; set_setting saves at once and is false for no such setting. +Settings settings() noexcept; +bool set_setting(std::string_view name, int value) noexcept; +// Wear and tear: how worn the board of the deck in the game is (0 new, 1 worn out), kept per deck +// with the settings. A board that broke is a new one once it is whole again. A worn board handles +// worse (softer trucks, a lazier turn: the game's own physics tuning, only outside multiplayer +// sessions, which share tuning), breaks more easily and, worn out, can give way under the rider. +// Its scratches are the game's own board wear (Mod Options > Board wear), reset with it. +float wear() noexcept; +void set_wear(float wear) noexcept; +// Client thread: what wear has set the game's physics tuning value at `offset` (in the tuning +// asset) to, when it is one of the four handling values wear changes and wear has written it; for +// the trainer, which hands such a value back to wear while the player has not changed it. +std::optional handling_value(std::uint16_t offset) noexcept; +// Metal: the level's materials a board breaking on them sounds like metal on. They are learned +// from the contacts while grinding (`boardbreak metal mark` marks what the board is on now as +// metal), and a pack can bring a list (sounds/metal/materials.txt). For the console. +std::string metal_command(std::string_view action); +std::string materials_report(); +std::string status(); } diff --git a/Extension/BoardBreak/board_break_audio.cpp b/Extension/BoardBreak/board_break_audio.cpp new file mode 100644 index 00000000..b7520960 --- /dev/null +++ b/Extension/BoardBreak/board_break_audio.cpp @@ -0,0 +1,333 @@ +#include "board_break_audio.h" +#include "Engine/Core/Platform/path_text.h" +#include "Engine/Core/Log/logging.h" +#include "Extension/Multiplayer/Remote/native_skater.h" +#include "Extension/Multiplayer/Voice/native_voice.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#pragma comment(lib, "xaudio2.lib") +#pragma comment(lib, "winmm.lib") +#pragma comment(lib, "ole32.lib") + +namespace dingosdk::board_break::audio { +namespace { +// One wave file, read whole: its format as the file gives it and its samples, and the same sound +// as the game's positioned sound takes it (16 kHz mono; empty when it could not be made). +struct Sound { + std::filesystem::path file; + std::vector format; // WAVEFORMATEX, or the larger WAVEFORMATEXTENSIBLE + std::vector samples; + std::vector voice; +}; +// The sound at the game sound's rate, mixed down to one channel: each output sample is the mean of +// the input samples it spans (a plain low-pass, so a crack's highs do not fold back as noise). +std::vector to_voice(const Sound& sound) { + WAVEFORMATEX format{}; + std::memcpy(&format, sound.format.data(), std::min(sound.format.size(), sizeof(format))); + auto tag = format.wFormatTag; + if (tag == WAVE_FORMAT_EXTENSIBLE && sound.format.size() >= 26) std::memcpy(&tag, sound.format.data() + 24, 2); // its SubFormat + const unsigned channels = format.nChannels, bits = format.wBitsPerSample, rate = format.nSamplesPerSec; + const bool floats = tag == WAVE_FORMAT_IEEE_FLOAT; + if (!channels || !rate || (tag != WAVE_FORMAT_PCM && !floats) || (floats && bits != 32) || + (!floats && bits != 8 && bits != 16 && bits != 24 && bits != 32)) + return {}; + const std::size_t stride = channels * (bits / 8), frames = sound.samples.size() / stride; + const auto at = [&](std::size_t frame) { + float sum = 0.0f; + for (unsigned c = 0; c < channels; ++c) { + const auto* p = sound.samples.data() + frame * stride + c * (bits / 8); + float v{}; + if (floats) std::memcpy(&v, p, 4); + else if (bits == 8) v = (static_cast(*p) - 128.0f) / 128.0f; + else if (bits == 16) { std::int16_t s{}; std::memcpy(&s, p, 2); v = s / 32768.0f; } + else if (bits == 24) v = static_cast(static_cast((p[0] << 8) | (p[1] << 16) | (p[2] << 24)) >> 8) / 8388608.0f; + else { std::int32_t s{}; std::memcpy(&s, p, 4); v = static_cast(s) / 2147483648.0f; } + sum += v; + } + return sum / static_cast(channels); + }; + const double step = static_cast(rate) / multiplayer::native_voice_rate; + const auto count = static_cast(static_cast(frames) / step); + std::vector out(count); + for (std::size_t i = 0; i < count; ++i) { + const auto from = static_cast(static_cast(i) * step); + const auto to = std::min(frames, std::max(from + 1, static_cast(static_cast(i + 1) * step))); + float sum = 0.0f; + for (auto f = from; f < to; ++f) sum += at(f); + const float v = std::isfinite(sum) ? sum / static_cast(to - from) : 0.0f; + out[i] = static_cast(std::clamp(v, -1.0f, 1.0f) * 32767.0f); + } + return out; +} +// A RIFF WAVE file's "fmt " and "data" chunks; false for anything else. +bool read_wave(const std::filesystem::path& file, Sound& out) { + std::ifstream in(file, std::ios::binary); + std::vector bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + if (bytes.size() < 12 || std::memcmp(bytes.data(), "RIFF", 4) || std::memcmp(bytes.data() + 8, "WAVE", 4)) return false; + std::size_t at = 12; + while (at + 8 <= bytes.size()) { + std::uint32_t size{}; + std::memcpy(&size, bytes.data() + at + 4, 4); + const auto body = at + 8; + if (body + size > bytes.size()) size = static_cast(bytes.size() - body); + if (!std::memcmp(bytes.data() + at, "fmt ", 4) && size >= 16) out.format.assign(bytes.begin() + body, bytes.begin() + body + size); + else if (!std::memcmp(bytes.data() + at, "data", 4)) out.samples.assign(bytes.begin() + body, bytes.begin() + body + size); + at = body + size + (size & 1); + } + if (out.format.size() < 16 || out.samples.empty()) return false; + std::uint16_t tag{}; + std::memcpy(&tag, out.format.data(), 2); + if (tag != WAVE_FORMAT_PCM && tag != WAVE_FORMAT_IEEE_FLOAT && tag != WAVE_FORMAT_EXTENSIBLE) return false; + // XAudio2 reads a whole WAVEFORMATEX (cbSize included). + if (out.format.size() < sizeof(WAVEFORMATEX)) out.format.resize(sizeof(WAVEFORMATEX), 0); + out.file = file; + return true; +} + +struct Request { + Kind kind{}; + float volume{}; +}; +// A sound the game's positioned sound is to play. +struct GameRequest { + const Sound* sound{}; + float volume{}; +}; +long long now_ms() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count(); +} +struct Engine { + std::mutex mutex; + std::condition_variable wake; + std::deque queue; + std::deque game_queue; + // When the game's sound last could play them (tick_game_sound): till then, and a moment after + // it stops (a level left), the speakers do. + std::atomic game_ms{}; + std::array, kinds> sounds; + std::array loaded{}; + std::thread worker; + std::atomic started{}, failed{}; + std::mt19937 random{std::random_device{}()}; +}; +Engine& engine() { + static auto* value = new Engine; + return *value; +} + +void run(Engine& e) { + const auto com = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + IXAudio2* xaudio{}; + IXAudio2MasteringVoice* master{}; + if (FAILED(XAudio2Create(&xaudio, 0, XAUDIO2_DEFAULT_PROCESSOR)) || FAILED(xaudio->CreateMasteringVoice(&master))) { + if (xaudio) xaudio->Release(); + xaudio = nullptr; + e.failed.store(true, std::memory_order_release); + logging::log(logging::Level::warning, logging::Channel::skater, + "Board Break: no sound device for its own sounds; the cracks play one at a time."); + } + std::vector voices; + for (;;) { + Request request; + const Sound* sound{}; + { + std::unique_lock lock(e.mutex); + e.wake.wait(lock, [&] { return !e.queue.empty(); }); + request = e.queue.front(); + e.queue.pop_front(); + auto& list = e.sounds[static_cast(request.kind)]; + if (list.empty()) continue; + sound = &list[std::uniform_int_distribution(0, list.size() - 1)(e.random)]; + } + if (!xaudio) { + // Without XAudio2: the cracks still sound, one at a time. + if (request.kind <= Kind::heavy) (void)PlaySoundW(sound->file.c_str(), nullptr, SND_FILENAME | SND_ASYNC | SND_NODEFAULT); + continue; + } + // Voices whose sound has played out go; at most a couple of dozen at once. + std::erase_if(voices, [](IXAudio2SourceVoice* voice) { + XAUDIO2_VOICE_STATE state{}; + voice->GetState(&state, XAUDIO2_VOICE_NOSAMPLESPLAYED); + if (state.BuffersQueued) return false; + voice->DestroyVoice(); + return true; + }); + if (voices.size() >= 24) { + voices.front()->DestroyVoice(); + voices.erase(voices.begin()); + } + IXAudio2SourceVoice* voice{}; + if (FAILED(xaudio->CreateSourceVoice(&voice, reinterpret_cast(sound->format.data())))) continue; + XAUDIO2_BUFFER buffer{}; + buffer.AudioBytes = static_cast(sound->samples.size()); + buffer.pAudioData = sound->samples.data(); + buffer.Flags = XAUDIO2_END_OF_STREAM; + voice->SetVolume(std::clamp(request.volume, 0.0f, 1.0f)); + if (FAILED(voice->SubmitSourceBuffer(&buffer)) || FAILED(voice->Start())) { + voice->DestroyVoice(); + continue; + } + voices.push_back(voice); + } + (void)com; +} +} + +void load(Kind kind, const std::vector& files) noexcept { + auto& e = engine(); + std::vector read; + for (const auto& file : files) { + try { + Sound sound; + if (read_wave(file, sound)) { + sound.voice = to_voice(sound); + read.push_back(std::move(sound)); + } + else logging::log(logging::Level::warning, logging::Channel::skater, "Board Break: {} is not a wave file it can play.", + path_utf8(file.filename())); + } catch (...) {} + } + std::lock_guard lock(e.mutex); + const auto at = static_cast(kind); + if (e.loaded[at]) return; + e.loaded[at] = true; + e.sounds[at] = std::move(read); +} +bool has(Kind kind) noexcept { + auto& e = engine(); + std::lock_guard lock(e.mutex); + return !e.sounds[static_cast(kind)].empty(); +} +void play(Kind kind, float volume) noexcept { + auto& e = engine(); + try { + // In a level: the game's own sound plays it, at the skater (tick_game_sound). + if (now_ms() - e.game_ms.load(std::memory_order_relaxed) < 400) { + std::lock_guard lock(e.mutex); + auto& list = e.sounds[static_cast(kind)]; + if (list.empty()) return; + const auto& sound = list[std::uniform_int_distribution(0, list.size() - 1)(e.random)]; + if (!sound.voice.empty()) { + if (e.game_queue.size() < 32) e.game_queue.push_back({&sound, volume}); + return; + } + } + if (!e.started.exchange(true)) { + e.worker = std::thread([&e] { run(e); }); + e.worker.detach(); + } + std::lock_guard lock(e.mutex); + if (e.queue.size() < 32) e.queue.push_back({kind, volume}); + } catch (...) { return; } + e.wake.notify_one(); +} +namespace { +using Clock = std::chrono::steady_clock; +// Board Break's sounds in the game's sample provider ("RSKB"), apart from voices and drones. +constexpr std::uint64_t source_tag = 0x52534b4200000000ULL; +// Their loudness through the game's sound, against the speakers' (1: the same samples). +constexpr float game_level = 1.0f; +// Sound made ahead for a sound just started, so its first blocks do not run dry; at most this +// much in one frame; and how long a finished sound keeps its place for its last blocks to play. +constexpr double preroll_seconds = 0.06; +constexpr float longest_frame = 0.1f; +constexpr float tail_seconds = 0.4f; +struct Stream { + const Sound* sound{}; + float gain{}; + std::size_t cursor{}; + double owed{}; + float tail{}; +}; +struct Player { + multiplayer::NativeVoicePlayback playback{source_tag}; + std::map streams; + std::uint64_t next_id = 1; + bool started{}; + Clock::time_point last{}; +}; +// Never destroyed: its sounds belong to the game, which is gone by then. +Player& player() { + static auto* value = new Player; + return *value; +} +} + +void tick_game_sound(std::uintptr_t base, std::uintptr_t client, bool ready) noexcept { + auto& e = engine(); + auto& p = player(); + try { + const auto now = Clock::now(); + if (!p.started) { + p.playback.reset(1); + p.started = true; + p.last = now; + } + const float elapsed = std::clamp(std::chrono::duration(now - p.last).count(), 0.0f, longest_frame); + p.last = now; + { + std::lock_guard lock(e.mutex); + for (const auto& request : e.game_queue) + if (ready && p.streams.size() < 24) { + Stream stream; + stream.sound = request.sound; + stream.gain = std::clamp(request.volume, 0.0f, 1.0f) * game_level; + stream.owed = preroll_seconds * multiplayer::native_voice_rate; + p.streams.emplace(p.next_id++, stream); + } + e.game_queue.clear(); + } + std::array at{}; + bool placed{}; + if (ready && !p.streams.empty()) { + const auto frame = multiplayer::capture_local(base, client, false); + if (frame.ready) { + at = {frame.pose.root.position[0], frame.pose.root.position[1] + 0.3f, frame.pose.root.position[2]}; + placed = true; + } + } + if (!placed) p.streams.clear(); // (their sounds are let go once they have no place for a moment) + std::vector block; + for (auto it = p.streams.begin(); it != p.streams.end();) { + auto& stream = it->second; + const auto& samples = stream.sound->voice; + p.playback.position(it->first, 1, at, stream.gain); + if (stream.cursor >= samples.size()) { + stream.tail += elapsed; + if (stream.tail >= tail_seconds) { it = p.streams.erase(it); continue; } + ++it; + continue; + } + stream.owed += static_cast(elapsed) * multiplayer::native_voice_rate; + const auto count = std::min({static_cast(stream.owed), samples.size() - stream.cursor, + static_cast(multiplayer::native_voice_rate * longest_frame)}); + stream.owed -= static_cast(count); + if (count) { + const auto* data = reinterpret_cast(samples.data() + stream.cursor); + if (p.playback.submit(it->first, 1, {data, count * sizeof(std::int16_t)})) stream.cursor += count; + } + ++it; + } + p.playback.process(base); + if (ready && p.playback.available()) e.game_ms.store(now_ms(), std::memory_order_relaxed); + } catch (...) { + p.streams.clear(); + } +} +} diff --git a/Extension/BoardBreak/board_break_audio.h b/Extension/BoardBreak/board_break_audio.h new file mode 100644 index 00000000..f9eff1b8 --- /dev/null +++ b/Extension/BoardBreak/board_break_audio.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include + +// Board Break's sounds: wave files out of a deck pack's `sounds` folder, over each other and at the +// loudness asked for, so a crack, the board's metal and the body's hits of one bail can all be heard. +// In a level they play through the game's own positioned sound, where the skater is, so the game's +// volume, mix and echo apply (16 kHz mono, as that sound takes). Elsewhere, or until the game's +// sound is ready, straight to the speakers: XAudio2 on a thread of its own, and without it (no +// sound device) the cracks one at a time with PlaySound. +namespace dingosdk::board_break::audio { +enum class Kind { light, medium, heavy, metal, body }; +inline constexpr std::size_t kinds = 5; +// The wave files of a kind (16-bit or float PCM, any rate and channels). Any thread; a kind is +// given its files once, later calls for it change nothing. +void load(Kind kind, const std::vector& files) noexcept; +bool has(Kind kind) noexcept; +// One of them, at random, at `volume` (0 to 1). Any thread; returns at once. +void play(Kind kind, float volume) noexcept; +// Client thread, every frame (Runtime/client_tick.cpp): plays what play() gave the game's sound, +// at the local skater. `ready`: a level is loaded and the skater can be read. +void tick_game_sound(std::uintptr_t base, std::uintptr_t client, bool ready) noexcept; +} diff --git a/Extension/BoardBreak/board_break_commands.cpp b/Extension/BoardBreak/board_break_commands.cpp index 76b26943..6ae13c05 100644 --- a/Extension/BoardBreak/board_break_commands.cpp +++ b/Extension/BoardBreak/board_break_commands.cpp @@ -71,5 +71,78 @@ void register_board_break_commands(Commands ®istry) { out("Mended."); }; registry.add(std::move(mend)); + // The player's settings (board_break_feel.h), kept: with no value each says what it is. + struct Setting { + const char *name, *description; + }; + const Setting settings[]{ + {"always", "Board Break: every slam breaks the board, and every slide that comes down onto a rail (for replay clips)"}, + {"sensitivity", "Board Break: how easily a board breaks, percent (100; 200 takes half the force, 50 twice as much)"}, + {"joined", "Board Break: percent of breaks the grip tape holds together (50)"}, + {"rails", "Board Break: a boardslide or darkslide that comes down too hard onto a rail can snap the board"}, + {"wear", "Board Break: boards wear out with every hit, handle worse and break more easily as they do"}, + {"wearrate", "Board Break: how fast boards wear, percent (100)"}, + {"cracks", "Board Break: the crack (and metal) sound of a break"}, + {"bodysounds", "Board Break: the body's hits in a bail, where and as hard as it hits"}, + }; + for (const auto &setting : settings) { + auto entry = action(std::string("boardbreak ") + setting.name, setting.description, Group::gameplay, + {argument("value", Type::text, true)}); + entry.run = [name = std::string(setting.name)](const Model &, const Values &values, const Output &out) { + const auto *text = values.empty() ? nullptr : std::get_if(&values[0]); + if (!text || text->empty()) { + out(name + " is " + std::to_string(board_break::setting_value(board_break::settings(), name)) + "."); + return; + } + int value{}; + if (*text == "on" || *text == "yes") value = 1; + else if (*text == "off" || *text == "no") value = 0; + else { + try { + value = std::stoi(*text); + } catch (...) { + out("Give a number (or on/off)."); + return; + } + } + out(board_break::set_setting(name, value) ? name + " is now " + std::to_string(board_break::setting_value(board_break::settings(), name)) + "." + : "No such setting."); + }; + registry.add(std::move(entry)); + } + + auto worn = action("boardbreak worn", "How worn the board of the deck in the game is, percent; 0 is a new board", Group::gameplay, + {argument("percent", Type::text, true)}); + worn.run = [](const Model &, const Values &values, const Output &out) { + const auto *text = values.empty() ? nullptr : std::get_if(&values[0]); + if (text && !text->empty()) { + try { + board_break::set_wear(std::stof(*text) / 100.0f); + } catch (...) { + out("Give a percent, 0 to 100."); + return; + } + } + const auto wear = board_break::wear(); + out("The board is " + std::string(board_break::condition_name(wear)) + ", " + std::to_string(static_cast(wear * 100.0f + 0.5f)) + "% worn."); + }; + registry.add(std::move(worn)); + + auto status = action("boardbreak status", "Everything Board Break is set to, and how worn the board is", Group::gameplay); + status.run = [](const Model &, const Values &, const Output &out) { out(board_break::status()); }; + registry.add(std::move(status)); + + auto metal = action("boardbreak metal", "Which materials a board breaking on them sounds like metal on: mark (what you grind now), unmark, clear, list", + Group::gameplay, {argument("mark|unmark|clear|list", Type::text, true)}); + metal.run = [](const Model &, const Values &values, const Output &out) { + const auto *text = values.empty() ? nullptr : std::get_if(&values[0]); + out(board_break::metal_command(text ? *text : std::string("list"))); + }; + registry.add(std::move(metal)); + + auto materials = action("boardbreak materials", "The level's materials your skater touched in the last 3 seconds, by number (to find a rail's)", + Group::gameplay); + materials.run = [](const Model &, const Values &, const Output &out) { out(board_break::materials_report()); }; + registry.add(std::move(materials)); } } // namespace dingosdk::console diff --git a/Extension/BoardBreak/board_break_feel.h b/Extension/BoardBreak/board_break_feel.h new file mode 100644 index 00000000..57828b45 --- /dev/null +++ b/Extension/BoardBreak/board_break_feel.h @@ -0,0 +1,178 @@ +#pragma once +#include "board_break_model.h" +#include +#include +#include +#include + +// How a board wears and how hard it has to be hit to break, apart from the game: the player's +// settings, the damage each kind of hit does, what a worn board does to its handling, and what kind +// of contact a board makes with a rail. Plain numbers, so the tests can hold them. +namespace dingosdk::board_break { +// ---- the player's settings ---------------------------------------------------------------------- +// Each is kept in %LOCALAPPDATA%\ReSkate\BoardBreak.settings and set from the console as +// `boardbreak ` (the menus send the same). +struct Settings { + bool always = false; // every slam breaks the board, for replay clips + int sensitivity = 100; // percent: at 200 half the force breaks a board, at 50 twice as much + int joined = 50; // percent of breaks the grip tape holds together + bool rails = true; // a hard boardslide or darkslide onto a rail or ledge can break it + bool wear = true; // boards wear out, handle worse and break more easily as they do + int wear_rate = 100; // percent: how fast they wear + bool crack_sounds = true; + bool body_sounds = true; // the body's hits during a bail +}; +inline constexpr int sensitivity_min = 10, sensitivity_max = 400, rate_min = 0, rate_max = 500; +struct SettingInfo { + std::string_view name; + bool on_off; // else a percent + int low, high; +}; +inline constexpr std::array setting_info{{ + {"always", true, 0, 1}, {"sensitivity", false, sensitivity_min, sensitivity_max}, {"joined", false, 0, 100}, + {"rails", true, 0, 1}, {"wear", true, 0, 1}, {"wearrate", false, rate_min, rate_max}, + {"cracks", true, 0, 1}, {"bodysounds", true, 0, 1}, +}}; +// The setting `name` as a number (on/off as 1/0), or -1 for no such setting. +inline int setting_value(const Settings& s, std::string_view name) noexcept { + if (name == "always") return s.always; + if (name == "sensitivity") return s.sensitivity; + if (name == "joined") return s.joined; + if (name == "rails") return s.rails; + if (name == "wear") return s.wear; + if (name == "wearrate") return s.wear_rate; + if (name == "cracks") return s.crack_sounds; + if (name == "bodysounds") return s.body_sounds; + return -1; +} +// Sets it, kept within its range; false for no such setting. +inline bool set_setting_value(Settings& s, std::string_view name, int value) noexcept { + for (const auto& info : setting_info) { + if (info.name != name) continue; + value = std::clamp(value, info.low, info.high); + if (name == "always") s.always = value != 0; + else if (name == "sensitivity") s.sensitivity = value; + else if (name == "joined") s.joined = value; + else if (name == "rails") s.rails = value != 0; + else if (name == "wear") s.wear = value != 0; + else if (name == "wearrate") s.wear_rate = value; + else if (name == "cracks") s.crack_sounds = value != 0; + else s.body_sounds = value != 0; + return true; + } + return false; +} + +// ---- how hard a slam has to be ------------------------------------------------------------------ +// The slam forces measured in play run from about 2 (a stumble) to 10 and more (a hard slam). +inline constexpr float heavy_slam = 10.0f; +// A worn board takes less: at the end of its life about half the force it took new. +inline constexpr float worn_strength = 0.5f; +// How likely a slam of `force` breaks the board, with the player's sensitivity (percent) and the +// board's wear (0 new, 1 worn out). A new board at 100%: never under 6, half the time at 8, always +// from 10. A bail whose force is not known (0) never breaks it. +inline float slam_chance(float force, int sensitivity, float wear) noexcept { + if (!(force > 0.0f)) return 0.0f; + const float s = std::clamp(sensitivity, sensitivity_min, sensitivity_max) / 100.0f; + const float strength = 1.0f - (1.0f - worn_strength) * std::clamp(wear, 0.0f, 1.0f); + return break_chance(force, heavy_slam * strength / s); +} + +// ---- wear and tear ------------------------------------------------------------------------------ +// A board's wear runs from 0 (new) to 1 (worn out). Every hit adds some, at the player's rate +// (percent of these). A fresh board lasts a good while: a few hundred hard landings, or some +// dozens of big drops, or half an hour of boardslides. +// A landing, by how fast the board was coming down as it touched (m/s): nothing up to an ollie off +// a curb's speed, then more and more. +inline constexpr float landing_free = 3.5f; +inline float landing_wear(float down_speed) noexcept { + const float over = (std::max)(0.0f, down_speed - landing_free); + return (std::min)(0.0008f * over * over, 0.04f); +} +// Onto a rail or ledge: the board's own hit, harder on a slide (the deck takes it) than on the trucks. +inline float rail_wear(float down_speed, bool slide) noexcept { + const float hit = landing_wear(down_speed + (slide ? 1.0f : 0.0f)); + return hit + (slide ? 0.001f : 0.0003f); +} +// Every second of grinding or sliding. +inline float grind_wear_per_second(bool slide) noexcept { return slide ? 0.0005f : 0.00015f; } +// A slam, by its force. +inline float slam_wear(float force) noexcept { return std::clamp((force - 4.0f) / 6.0f, 0.0f, 1.0f) * 0.025f; } +inline float add_wear(float wear, float amount, int rate) noexcept { + return std::clamp(wear + amount * std::clamp(rate, rate_min, rate_max) / 100.0f, 0.0f, 1.0f); +} +// What wear does to the board's handling, as factors on the game's own tuning: the steering answers +// less, the trucks' springs and their damping go soft (the board wobbles and leans further), and +// they twist further. Nothing shows until a board is a fifth worn. +struct Handling { + float steering = 1, spring = 1, damping = 1, twist = 1; + bool operator==(const Handling&) const = default; +}; +inline Handling handling(float wear) noexcept { + const float w = std::clamp((wear - 0.2f) / 0.8f, 0.0f, 1.0f); + const float eased = w * w * (3.0f - 2.0f * w); + return {1.0f - 0.25f * eased, 1.0f - 0.6f * eased, 1.0f - 0.5f * eased, 1.0f + 0.4f * eased}; +} +// What a board's state is called on the menus. A board is "done" from `done_wear` on. +inline constexpr float done_wear = 0.85f; +inline std::string_view condition_name(float wear) noexcept { + return wear < 0.15f ? "new" : wear < 0.4f ? "used" : wear < 0.65f ? "worn" : wear < done_wear ? "beat" : "done"; +} +// A done board gives way under the rider on a landing (the nose goes, and they ride on): from a +// big drop (6.5 m/s coming down) as it becomes done, down to any real drop (3.5 m/s, a curb's) +// when it is worn right out. Never a board less worn, and never on flat ground riding: a board +// breaks from something the rider did. Sensitivity as for a slam. +inline bool gives_way_on_landing(float down_speed, int sensitivity, float wear) noexcept { + if (wear < done_wear) return false; + const float s = std::clamp(sensitivity, sensitivity_min, sensitivity_max) / 100.0f; + const float t = std::clamp((wear - done_wear) / (1.0f - done_wear), 0.0f, 1.0f); + return down_speed >= (6.5f - 3.0f * t) / s; +} + +// ---- the board on a rail ------------------------------------------------------------------------ +enum class Contact { grind, slide, darkslide }; +constexpr std::array contact_names{"grind", "slide", "darkslide"}; +// From the board's long axis and its up axis in the world, and how it moves: a board lying across +// its way (a boardslide, a lipslide) slides, one on its grip is in a darkslide, the rest grind on +// their trucks. A board that hardly moves grinds. +inline Contact contact_of(Vec along, Vec up, Vec velocity) noexcept { + if (up.y < -0.35f) return Contact::darkslide; + const float a = std::hypot(along.x, along.z), v = std::hypot(velocity.x, velocity.z); + if (a < 0.2f || v < 0.5f) return Contact::grind; + const float cosine = std::abs((along.x * velocity.x + along.z * velocity.z) / (a * v)); + return cosine < 0.5f ? Contact::slide : Contact::grind; +} +// How likely a board snaps as it comes down onto a rail in a slide, by how fast it was coming down +// (m/s): from 3 m/s up (a hop onto a rail is about 2), certain 3 m/s above that; a darkslide on the +// grip goes sooner. Grinds on the trucks do not snap a board. Sensitivity and wear as for a slam. +inline float rail_chance(float down_speed, Contact contact, int sensitivity, float wear) noexcept { + if (contact == Contact::grind) return 0.0f; + const float s = std::clamp(sensitivity, sensitivity_min, sensitivity_max) / 100.0f; + const float strength = 1.0f - (1.0f - worn_strength) * std::clamp(wear, 0.0f, 1.0f); + const float start = (contact == Contact::darkslide ? 2.2f : 3.0f) * strength / s; + const float span = 3.0f * strength / s; + return std::clamp((down_speed - start) / span, 0.0f, 1.0f); +} + +// ---- where a board breaks ----------------------------------------------------------------------- +// From how the board came down: nose first it breaks at the nose, tail first at the tail, flat (or +// across a rail) in the middle. `slope` is how far the nose points down (the board's long axis' +// downward part, -1 to 1); `roll` is 0 to 1, chance. Flat landings still break at an end now and +// then, as real boards do. +inline Place place_of_break(float slope, bool across_rail, float roll) noexcept { + if (across_rail) return Place::mid; + if (slope > 0.35f) return roll < 0.75f ? Place::nose : Place::mid; + if (slope < -0.35f) return roll < 0.75f ? Place::tail : Place::mid; + return roll < 0.5f ? Place::mid : roll < 0.75f ? Place::nose : Place::tail; +} + +// ---- a body's hit, for its sound ----------------------------------------------------------------- +// The ragdoll's hits along their normal (Extension/HallOfMeat/hall_of_meat_model.h): a graze is 2 to +// 3 m/s, a fall on flat ground 3 to 9, a drop from height 13 and more. Below `body_hit` nothing is +// heard; the loudness rises to full at `body_full`. +inline constexpr float body_hit = 3.0f, body_full = 11.0f; +inline float body_loudness(float speed) noexcept { + if (speed < body_hit) return 0.0f; + return std::clamp(0.3f + 0.7f * (speed - body_hit) / (body_full - body_hit), 0.3f, 1.0f); +} +} diff --git a/Extension/BoardBreak/board_break_model.h b/Extension/BoardBreak/board_break_model.h index d78b3f0a..1f17ac62 100644 --- a/Extension/BoardBreak/board_break_model.h +++ b/Extension/BoardBreak/board_break_model.h @@ -82,12 +82,14 @@ constexpr Size break_size(Place place, bool joined, float fold_depth) noexcept { if (place == Place::mid) return !joined || fold_depth >= 0.7f ? Size::heavy : Size::medium; return joined ? Size::light : Size::medium; } -// How likely a slam of `force` breaks the board: always from `heavy` up, with `light` up to 60% of -// it, rising evenly in between. -constexpr float break_chance(float force, float heavy, float light) noexcept { +// How likely a slam of `force` breaks the board: never up to 60% of `heavy`, always from `heavy` up, +// rising smoothly in between. A board only breaks from a hit the player can feel was a big one. +constexpr float break_chance(float force, float heavy) noexcept { if (heavy <= 0 || force >= heavy) return 1.0f; const auto low = 0.6f * heavy; - return force <= low ? light : light + (1.0f - light) * (force - low) / (heavy - low); + if (force <= low) return 0.0f; + const auto t = (force - low) / (heavy - low); + return t * t * (3.0f - 2.0f * t); } // ---- where the parts of a broken board lie ------------------------------------------------------ diff --git a/Extension/Boot/descriptor_heap_size.cpp b/Extension/Boot/descriptor_heap_size.cpp new file mode 100644 index 00000000..aadd2623 --- /dev/null +++ b/Extension/Boot/descriptor_heap_size.cpp @@ -0,0 +1,61 @@ +#include "descriptor_heap_size.h" + +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/supported_build.h" +#include "Engine/Game/Build/20260929/descriptor_heap_size.h" + +#include +#include +#include + +namespace dingosdk { +namespace { +namespace contract = addr::descriptor_heap_size; + +bool supported_image(std::uintptr_t base) noexcept { + IMAGE_DOS_HEADER dos{}; + IMAGE_NT_HEADERS64 nt{}; + return base && memory::read(base, dos) && dos.e_magic == IMAGE_DOS_SIGNATURE && dos.e_lfanew > 0 && + dos.e_lfanew <= 0x1000 && memory::read(base + dos.e_lfanew, nt) && nt.Signature == IMAGE_NT_SIGNATURE && + nt.OptionalHeader.SizeOfImage == supported_build::game_image_size; +} + +bool write_code(std::uintptr_t address, const void* bytes, std::size_t size) noexcept { + auto* const destination = reinterpret_cast(address); + DWORD previous{}; + if (!VirtualProtect(destination, size, PAGE_EXECUTE_READWRITE, &previous)) return false; + std::memcpy(destination, bytes, size); + const bool flushed = FlushInstructionCache(GetCurrentProcess(), destination, size) != FALSE; + DWORD discarded{}; + return VirtualProtect(destination, size, previous, &discarded) != FALSE && flushed; +} +} // namespace + +bool start_descriptor_heap_size(std::uintptr_t base, std::string& error) { + std::array actual{}; + std::array total{}; + if (!supported_image(base) || !memory::read_bytes(base + contract::cap_load_rva, actual.data(), actual.size()) || + actual != contract::cap_load || !memory::read_bytes(base + contract::total_cap_rva, total.data(), total.size()) || + total != contract::total_cap) { + error = "the renderer's descriptor heap setup does not match the supported game"; + return false; + } + // The same divide, with the cap a constant instead of the setting (which is never set): + // div r9d / mov edi, eax / mov ecx, cap / or eax, -1 / nop + std::array code{ + 0x41, 0xf7, 0xf1, 0x8b, 0xf8, 0xb9, 0, 0, 0, 0, 0x83, 0xc8, 0xff, 0x90}; + std::memcpy(code.data() + 6, &frame_descriptor_cap, 4); + // The total, a constant, then capped at 1,000,000 as before: mov ecx, total / nop (4 bytes). + std::array load{0xb9, 0, 0, 0, 0, 0x0f, 0x1f, 0x40, 0x00}; + std::memcpy(load.data() + 1, &heap_descriptor_total, 4); + if (!write_code(base + contract::cap_load_rva, code.data(), code.size()) || + !write_code(base + contract::total_cap_rva, load.data(), load.size())) { + (void)write_code(base + contract::cap_load_rva, contract::cap_load.data(), contract::cap_load.size()); + (void)write_code(base + contract::total_cap_rva, contract::total_cap.data(), contract::total_cap.size()); + error = "the heap sizes could not be patched"; + return false; + } + return true; +} +} // namespace dingosdk diff --git a/Extension/Boot/descriptor_heap_size.h b/Extension/Boot/descriptor_heap_size.h new file mode 100644 index 00000000..40b74b3e --- /dev/null +++ b/Extension/Boot/descriptor_heap_size.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace dingosdk { +// Each frame's share of the renderer's shader-visible descriptor heap is capped at 131,072 +// descriptors, in a heap of 333,333. A dense level draws more than that in a frame (the Los Santos +// City sublevels with more than about 65,000 mesh sections), and the renderer then writes past +// the end of the heap: shaders read descriptors that are not there and fault or hang the GPU. +// Each heap is allowed D3D12's 1,000,000 instead, and a frame's share is capped at this, which +// makes each frame's heap 644,288 descriptors. +inline constexpr std::uint32_t heap_descriptor_total = 1'000'000; +inline constexpr std::uint32_t frame_descriptor_cap = 524'288; + +// Installs before executable entry, while the game is suspended. +bool start_descriptor_heap_size(std::uintptr_t base, std::string& error); +} // namespace dingosdk diff --git a/Extension/Boot/eadp_errors_arena.cpp b/Extension/Boot/eadp_errors_arena.cpp new file mode 100644 index 00000000..0992a8af --- /dev/null +++ b/Extension/Boot/eadp_errors_arena.cpp @@ -0,0 +1,93 @@ +#include "eadp_errors_arena.h" + +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/supported_build.h" +#include "Engine/Game/Build/20260929/eadp_errors_arena.h" + +#include +#include +#include +#include + +namespace dingosdk { +namespace { +namespace contract = addr::eadp_errors_arena; + +constexpr std::size_t stub_page_bytes = 0x1000; + +bool supported_image(std::uintptr_t base) noexcept { + IMAGE_DOS_HEADER dos{}; + IMAGE_NT_HEADERS64 nt{}; + return base && memory::read(base, dos) && dos.e_magic == IMAGE_DOS_SIGNATURE && dos.e_lfanew > 0 && + dos.e_lfanew <= 0x1000 && memory::read(base + dos.e_lfanew, nt) && nt.Signature == IMAGE_NT_SIGNATURE && + nt.OptionalHeader.SizeOfImage == supported_build::game_image_size; +} + +bool rel32(std::uintptr_t next_instruction, std::uintptr_t target, std::int32_t& displacement) noexcept { + const auto delta = static_cast(target) - static_cast(next_instruction); + if (delta < std::numeric_limits::min() || delta > std::numeric_limits::max()) return false; + displacement = static_cast(delta); + return true; +} + +bool write_code(std::uintptr_t address, const void* bytes, std::size_t size) noexcept { + auto* const destination = reinterpret_cast(address); + DWORD previous{}; + if (!VirtualProtect(destination, size, PAGE_EXECUTE_READWRITE, &previous)) return false; + std::memcpy(destination, bytes, size); + const bool flushed = FlushInstructionCache(GetCurrentProcess(), destination, size) != FALSE; + DWORD discarded{}; + return VirtualProtect(destination, size, previous, &discarded) != FALSE && flushed; +} +} // namespace + +bool start_eadp_errors_arena(std::uintptr_t base, std::string& error) { + std::array actual{}; + if (!supported_image(base) || !memory::read_bytes(base + contract::size_load_rva, actual.data(), actual.size()) || + actual != contract::size_load) { + error = "the arena's factory does not match the supported game"; + return false; + } + auto* const stub = static_cast(memory::allocate_near_image(base, supported_build::game_image_size, stub_page_bytes)); + if (!stub) { + error = "no room for its stub near the game"; + return false; + } + const auto at = reinterpret_cast(stub); + // The two moved instructions, with the size capped between them: + // mov eax, [rax+0x7c] / cmp eax, cap / jbe +5 / mov eax, cap / lea rcx, [vtable] / jmp resume + std::array code{0x8b, 0x40, 0x7c, 0x3d, 0, 0, 0, 0, 0x76, 0x05, 0xb8, 0, 0, 0, 0, + 0x48, 0x8d, 0x0d, 0, 0, 0, 0, 0xe9, 0, 0, 0, 0}; + std::memcpy(code.data() + 4, &eadp_errors_arena_bytes, 4); + std::memcpy(code.data() + 11, &eadp_errors_arena_bytes, 4); + std::int32_t vtable{}, resume{}, into{}; + if (!rel32(at + 22, base + contract::vtable_rva, vtable) || !rel32(at + 27, base + contract::resume_rva, resume) || + !rel32(base + contract::size_load_rva + 5, at, into)) { + VirtualFree(stub, 0, MEM_RELEASE); + error = "its stub is out of reach of the game"; + return false; + } + std::memcpy(code.data() + 18, &vtable, 4); + std::memcpy(code.data() + 23, &resume, 4); + std::memset(stub, 0xcc, stub_page_bytes); + std::memcpy(stub, code.data(), code.size()); + DWORD previous{}; + if (!VirtualProtect(stub, stub_page_bytes, PAGE_EXECUTE_READ, &previous) || + !FlushInstructionCache(GetCurrentProcess(), stub, stub_page_bytes)) { + VirtualFree(stub, 0, MEM_RELEASE); + error = "its stub could not be made executable"; + return false; + } + std::array jump{0xe9}; + std::memcpy(jump.data() + 1, &into, 4); + std::memset(jump.data() + 5, 0x90, jump.size() - 5); + if (!write_code(base + contract::size_load_rva, jump.data(), jump.size())) { + (void)write_code(base + contract::size_load_rva, contract::size_load.data(), contract::size_load.size()); + VirtualFree(stub, 0, MEM_RELEASE); + error = "the size load could not be redirected"; + return false; + } + return true; +} +} // namespace dingosdk diff --git a/Extension/Boot/eadp_errors_arena.h b/Extension/Boot/eadp_errors_arena.h new file mode 100644 index 00000000..ddfd6e59 --- /dev/null +++ b/Extension/Boot/eadp_errors_arena.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include +#include + +namespace dingosdk { +// The arena EA's error reporting allocates from is 80 MiB, made at startup and kept in RAM, of +// which about 10 MB was in use: reporting is off (ea_service_block, and the EadpErrorsData +// settings in client_tick.cpp), so its reports are never built. It is made before those settings +// can be changed, so its size is capped where the game reads it instead. A smaller configured +// size is left as it is. +inline constexpr std::uint32_t eadp_errors_arena_bytes = 32u << 20; + +// Installs before executable entry, while the game is suspended. +bool start_eadp_errors_arena(std::uintptr_t base, std::string& error); +} // namespace dingosdk diff --git a/Extension/Boot/exit_watch.cpp b/Extension/Boot/exit_watch.cpp index b329111e..6986665a 100644 --- a/Extension/Boot/exit_watch.cpp +++ b/Extension/Boot/exit_watch.cpp @@ -10,9 +10,11 @@ namespace { constexpr ULONGLONG grace_ms = 8000, watch_ms = 120000; std::atomic watching{}; std::atomic destroyed_at{}; +std::atomic close_notice{}; void watch(HWND window) noexcept { const auto asked = GetTickCount64(); + if (const auto notice = close_notice.load()) notice(); ULONGLONG stuck_since{}; for (;;) { Sleep(500); @@ -37,6 +39,7 @@ void watch(HWND window) noexcept { } } // namespace +void when_closing(CloseNotice notice) noexcept { close_notice = notice; } void note_window_message(HWND window, UINT message, WPARAM wp) noexcept { const bool closing = message == WM_CLOSE || message == WM_DESTROY || (message == WM_SYSCOMMAND && (wp & 0xfff0) == SC_CLOSE) || (message == WM_ENDSESSION && wp); diff --git a/Extension/Boot/exit_watch.h b/Extension/Boot/exit_watch.h index 94d7c433..4f5c9e62 100644 --- a/Extension/Boot/exit_watch.h +++ b/Extension/Boot/exit_watch.h @@ -7,4 +7,8 @@ namespace dingosdk::exit_watch { // asked the game to close, this gives it a few seconds and then ends the process. // Called from the game window's procedure with each message of that window. void note_window_message(HWND window, UINT message, WPARAM wp) noexcept; +// Called, on the watch's own thread, each time the player asks the game to close, before the +// close is watched: for what has to be sent before the process goes. It may take a few seconds. +using CloseNotice = void (*)() noexcept; +void when_closing(CloseNotice notice) noexcept; } diff --git a/Extension/Boot/game_fatal_dump.cpp b/Extension/Boot/game_fatal_dump.cpp new file mode 100644 index 00000000..bcf487fb --- /dev/null +++ b/Extension/Boot/game_fatal_dump.cpp @@ -0,0 +1,71 @@ +#include "game_fatal_dump.h" + +#include "Engine/Core/Debug/crash_report.h" +#include "Engine/Core/Platform/memory.h" +#include "Engine/Game/Build/addresses.h" +#include "Engine/Game/Build/20260929/game_fatal_dump.h" + +#include +#include +#include +#include +#include + +namespace dingosdk { +namespace { +namespace contract = addr::game_fatal_dump; +using WriteDump = BOOL(WINAPI*)(HANDLE, DWORD, HANDLE, MINIDUMP_TYPE, PMINIDUMP_EXCEPTION_INFORMATION, + PMINIDUMP_USER_STREAM_INFORMATION, PMINIDUMP_CALLBACK_INFORMATION); +std::atomic original_write{}; +std::uintptr_t installed_base{}; +std::mutex install_mutex; + +// Only a dump of this process with an exception to show is a crash; the game's dump follows +// whether or not the reporter took one. (Its pointers are this process's either way.) +BOOL WINAPI write_dump(HANDLE process, DWORD process_id, HANDLE file, MINIDUMP_TYPE type, + PMINIDUMP_EXCEPTION_INFORMATION exception, PMINIDUMP_USER_STREAM_INFORMATION streams, + PMINIDUMP_CALLBACK_INFORMATION callback) { + if (exception && exception->ExceptionPointers && process_id == GetCurrentProcessId()) + crash_report::capture_game_fatal(exception->ThreadId, exception->ExceptionPointers); + return original_write.load(std::memory_order_acquire)(process, process_id, file, type, exception, streams, callback); +} + +// The slot holds the export it was bound to: MiniDumpWriteDump of the module that address is in. +bool bound_to_minidump(std::uintptr_t value) noexcept { + HMODULE module{}; + if (!value || !GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(value), &module)) return false; + return reinterpret_cast(GetProcAddress(module, "MiniDumpWriteDump")) == value; +} +} // namespace + +bool start_game_fatal_dump(std::uintptr_t base, std::string& error) { + std::lock_guard lock(install_mutex); + error.clear(); + if (installed_base) { + if (installed_base == base) return true; + error = "already installed on another image"; + return false; + } + std::array thunk{}; + std::uintptr_t bound{}; + if (!memory::read_bytes(base + contract::thunk, thunk.data(), thunk.size()) || + !std::equal(thunk.begin(), thunk.end(), contract::thunk_bytes.begin()) || + !memory::read(base + contract::import_slot, bound) || !bound_to_minidump(bound)) { + error = "the game's MiniDumpWriteDump import does not match the supported game"; + return false; + } + original_write.store(reinterpret_cast(bound), std::memory_order_release); + auto* const slot = reinterpret_cast(base + contract::import_slot); + DWORD previous{}; + if (!VirtualProtect(slot, sizeof(void*), PAGE_READWRITE, &previous)) { + error = "its import slot could not be made writable"; + return false; + } + InterlockedExchangePointer(slot, reinterpret_cast(&write_dump)); + DWORD ignored{}; + VirtualProtect(slot, sizeof(void*), previous, &ignored); + installed_base = base; + return true; +} +} // namespace dingosdk diff --git a/Extension/Boot/game_fatal_dump.h b/Extension/Boot/game_fatal_dump.h new file mode 100644 index 00000000..e359f076 --- /dev/null +++ b/Extension/Boot/game_fatal_dump.h @@ -0,0 +1,12 @@ +#pragma once + +#include +#include + +namespace dingosdk { +// When one of the game's fatal handlers is about to write its minidump, the crash reporter first +// takes its own of the still-running game (crash_report::capture_game_fatal), with the memory the +// stacks point at. Points the game's MiniDumpWriteDump import at ReSkate; the game's dump is +// still written after. Install after crash reporting has started, before the game runs. +bool start_game_fatal_dump(std::uintptr_t base, std::string& error); +} // namespace dingosdk diff --git a/Extension/Console/commands.cpp b/Extension/Console/commands.cpp index 63f68c12..b449231c 100644 --- a/Extension/Console/commands.cpp +++ b/Extension/Console/commands.cpp @@ -44,24 +44,23 @@ void register_console_commands(Commands ®istry) { if (args.empty()) { out("Enter a variable by itself to read it, or follow it with a value: " "noclip 1, nobail 0, flyspeed 60."); - out("Use commands [prefix] to list entries, help for details, " - "and Tab to complete names and " - "arguments."); + out("Use commands [prefix] to list entries, help for details and the commands under it " + "(help perf, help mp, help trainer), and Tab to complete names and arguments."); out("Groups: Movement, Gameplay, World, Graphics, Progression, Objects, " "Engine, Console."); return; } const auto &name = std::get(args[0]); const auto *entry = registry.find(name); + const auto below = registry.under(entry ? entry->name : name); + const auto list_below = [&below, &out] { + for (const auto *item : below) + out(Commands::usage(*item) + (item->description.empty() ? "" : " - " + item->description)); + }; if (!entry) { - bool found = false; - for (const auto &item : registry.entries()) - if (ascii_case_insensitive_starts_with(item.name, name + " ")) { - out(Commands::usage(item) + (item.description.empty() ? "" : " - " + item.description)); - found = true; - } - if (!found) + if (below.empty()) out("error: Unknown help topic."); + list_below(); return; } out(Commands::usage(*entry)); @@ -83,6 +82,10 @@ void register_console_commands(Commands ®istry) { } if (entry->reset) out("Restore: reset " + entry->name); + // `perf` is a command of its own and the first word of `perf hud`, `perf status` and the rest. + if (!below.empty()) + out(entry->name + " commands:"); + list_below(); }; registry.add(std::move(help)); auto list = action("commands", "List commands and variables, optionally filtered by prefix", Group::console, @@ -229,6 +232,7 @@ const Commands &game_commands() { register_road_rash_commands(*result); register_board_break_commands(*result); register_grind_exit_commands(*result); + register_replay_keyframes_commands(*result); return result; }(); return *registry; diff --git a/Extension/Console/commands.h b/Extension/Console/commands.h index 0caee4c7..828ecd15 100644 --- a/Extension/Console/commands.h +++ b/Extension/Console/commands.h @@ -30,6 +30,7 @@ void register_hall_of_meat_commands(Commands &); void register_road_rash_commands(Commands &); void register_board_break_commands(Commands &); void register_grind_exit_commands(Commands &); +void register_replay_keyframes_commands(Commands &); // Runtime adapters. Invoked only by the verified game-thread dispatcher. void request_debug(overlay::DebugAction, bool enabled = false, float value = 0); void request_feature(overlay::OfflineFeatureGroup, bool enabled); diff --git a/Extension/Console/memory_ledger.cpp b/Extension/Console/memory_ledger.cpp new file mode 100644 index 00000000..dfe8fa63 --- /dev/null +++ b/Extension/Console/memory_ledger.cpp @@ -0,0 +1,812 @@ +#include "Extension/Console/memory_ledger.h" +#include "Engine/Core/Hooks/hooks.h" +#include "Engine/Core/Log/logging.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::memory_ledger { +namespace { +constexpr std::size_t depth = 14; +constexpr std::size_t caller_slots = 8192; +constexpr unsigned block_bits = 18; +constexpr std::size_t block_slots = std::size_t{1} << block_bits; +constexpr std::uint32_t gone = 0xffffffff; + +struct Caller { + std::uint64_t hash{}; + std::array frames{}; + unsigned frame_count{}; + // What was asked for while `perf memory trace` was on: reservations and commits of the + // traced sizes. smallest/largest: the sizes of those. + std::uint64_t asked_count{}, asked_bytes{}, smallest{}, largest{}; +}; +// A reservation the process still holds and who made it. caller: an index into callers plus +// one; 0 is an empty slot, `gone` one whose reservation was given back (kept so a search +// for one further along still finds it). +struct Block { + std::uintptr_t base{}; + std::uint32_t caller{}; + std::uint32_t made{}; // seconds after the ledger started +}; + +// Never held while allocating: the heap may grow and come back through the hook on the same +// thread. What is copied out under it goes into storage allocated before. +// (None of this is ever destroyed: the hooks stay on while the process exits, and what other +// modules give back then still passes them.) +std::mutex& mutex = *new std::mutex; +std::vector& callers = *new std::vector(caller_slots); +std::vector& blocks = *new std::vector(block_slots); +std::size_t blocks_live{}, blocks_gone{}; +std::uint64_t untracked{}; // reservations the tables had no room for +std::atomic running{}, counting{}; +std::atomic least{1}, most{UINT64_MAX}; +std::atomic began{}; +std::mutex& start_mutex = *new std::mutex; +bool prepared{}; + +using Allocate = LONG (NTAPI*)(HANDLE, PVOID*, ULONG_PTR, PSIZE_T, ULONG, ULONG); +using AllocateEx = LONG (NTAPI*)(HANDLE, PVOID*, PSIZE_T, ULONG, ULONG, void*, ULONG); +using Free = LONG (NTAPI*)(HANDLE, PVOID*, PSIZE_T, ULONG); +Allocate original{}; +AllocateEx original_ex{}; +Free original_free{}; + +std::size_t block_slot(std::uintptr_t base) noexcept { + return static_cast(((base >> 16) * 0x9e3779b97f4a7c15ull) >> (64 - block_bits)); +} +// Under the lock. The search is bounded: a hook cannot take long. +void remember(std::uintptr_t base, std::uint32_t caller, std::uint32_t made) noexcept { + std::size_t free_at = block_slots; + for (std::size_t probe = 0; probe < 2048; ++probe) { + const auto at = (block_slot(base) + probe) & (block_slots - 1); + auto& block = blocks[at]; + if (!block.caller) { if (free_at == block_slots) free_at = at; break; } + if (block.caller == gone) { if (free_at == block_slots) free_at = at; continue; } + // (A release that was not seen: the same base is made again.) + if (block.base == base) { block.caller = caller; block.made = made; return; } + } + if (free_at == block_slots || (!blocks[free_at].caller && (blocks_live + blocks_gone) * 8 >= block_slots * 7)) { + ++untracked; + return; + } + if (blocks[free_at].caller == gone) --blocks_gone; + blocks[free_at] = {base, caller, made}; + ++blocks_live; +} +void forget(std::uintptr_t base) noexcept { + for (std::size_t probe = 0; probe < 2048; ++probe) { + auto& block = blocks[(block_slot(base) + probe) & (block_slots - 1)]; + if (!block.caller) return; + if (block.caller != gone && block.base == base) { + block.caller = gone; + --blocks_live; + ++blocks_gone; + return; + } + } +} +const Block* find(const std::vector& table, std::uintptr_t base) noexcept { + for (std::size_t probe = 0; probe < 2048; ++probe) { + const auto& block = table[(block_slot(base) + probe) & (block_slots - 1)]; + if (!block.caller) return nullptr; + if (block.caller != gone && block.base == base) return █ + } + return nullptr; +} +// Under the lock: the caller's index plus one, made if new; 0 when the table is full. +std::uint32_t caller_of(std::uint64_t hash, const std::array& frames, unsigned count) noexcept { + for (std::size_t probe = 0; probe < 128; ++probe) { + const auto at = (hash + probe) % caller_slots; + auto& caller = callers[at]; + if (caller.hash && caller.hash != hash) continue; + if (!caller.hash) { + caller = {}; + caller.hash = hash; + caller.frames = frames; + caller.frame_count = count; + } + return static_cast(at + 1); + } + return 0; +} + +// This process, by its pseudo-handle or a real handle to it (which some code opens for itself). +bool own(HANDLE process) noexcept { + return process == GetCurrentProcess() || (process && GetProcessId(process) == GetCurrentProcessId()); +} + +// (Not inlined: the stack is captured from a known depth, two frames under the caller of +// NtAllocateVirtualMemory.) +__declspec(noinline) void note(HANDLE process, void* base, SIZE_T asked, ULONG type) noexcept { + if (!running.load(std::memory_order_relaxed) || !(type & (MEM_COMMIT | MEM_RESERVE)) || !own(process)) + return; + const bool reserve = (type & MEM_RESERVE) != 0; + const bool counted = counting.load(std::memory_order_relaxed) && asked >= least.load(std::memory_order_relaxed) && + asked <= most.load(std::memory_order_relaxed); + if (!reserve && !counted) return; // (a commit inside a reservation that is already noted) + std::array frames{}; + const unsigned count = CaptureStackBackTrace(2, static_cast(depth), frames.data(), nullptr); + std::uint64_t hash = 1469598103934665603ull; + for (unsigned i = 0; i < count; ++i) hash = hash * 1099511628211ull ^ reinterpret_cast(frames[i]); + if (!hash) hash = 1; + const auto made = static_cast((GetTickCount64() - began.load(std::memory_order_relaxed)) / 1000); + std::lock_guard lock(mutex); + const auto index = caller_of(hash, frames, count); + if (!index) { if (reserve) ++untracked; return; } + if (counted) { + auto& caller = callers[index - 1]; + caller.smallest = caller.asked_count ? std::min(caller.smallest, asked) : asked; + caller.largest = std::max(caller.largest, asked); + ++caller.asked_count; + caller.asked_bytes += asked; + } + if (reserve && base) remember(reinterpret_cast(base), index, made); +} +LONG NTAPI hooked(HANDLE process, PVOID* base, ULONG_PTR zero_bits, PSIZE_T size, ULONG type, ULONG protect) { + const auto asked = size ? *size : 0; + const auto status = original(process, base, zero_bits, size, type, protect); + if (status >= 0) note(process, base ? *base : nullptr, asked, type); + return status; +} +// The newer call for the same (VirtualAlloc2, which graphics drivers and the D3D12 runtime use). +LONG NTAPI hooked_ex(HANDLE process, PVOID* base, PSIZE_T size, ULONG type, ULONG protect, void* parameters, ULONG parameter_count) { + const auto asked = size ? *size : 0; + const auto status = original_ex(process, base, size, type, protect, parameters, parameter_count); + if (status >= 0) note(process, base ? *base : nullptr, asked, type); + return status; +} +LONG NTAPI hooked_free(HANDLE process, PVOID* base, PSIZE_T size, ULONG type) { + const auto at = base ? reinterpret_cast(*base) : 0; + const auto status = original_free(process, base, size, type); + // (A placeholder split or joined is still reserved: MEM_COALESCE/PRESERVE_PLACEHOLDER.) + if (status >= 0 && at && (type & MEM_RELEASE) && !(type & 0x3) && running.load(std::memory_order_relaxed) && + own(process)) { + std::lock_guard lock(mutex); + forget(at); + } + return status; +} + +void* ntdll_function(const char* name) { + return reinterpret_cast(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), name)); +} + +std::string megabytes(std::uint64_t bytes) { + return bytes >= (1ull << 30) ? std::format("{:.2f} GB", static_cast(bytes) / (1ull << 30)) + : std::format("{:.1f} MB", static_cast(bytes) / (1ull << 20)); +} +std::string age(std::uint32_t seconds) { + return seconds < 120 ? std::format("{} s", seconds) : seconds < 7200 ? std::format("{} min", seconds / 60) + : std::format("{:.1f} h", seconds / 3600.0); +} + +// ---- reading what the process holds ---- +struct Range { + std::uintptr_t base{}; + std::uint64_t bytes{}; + DWORD protect{}; +}; +struct Allocation { + std::uint64_t committed{}, reserved{}; // reserved: the whole reservation, committed or not + DWORD reserved_as{}; // the protection it was reserved with + DWORD committed_as{}; // every protection its committed parts have, together + std::vector ranges; // what of it is committed (kept only when asked for) +}; +struct Held { + std::map private_memory; + std::uint64_t committed{}, mapped{}, image{}; +}; +Held read_held(bool keep_ranges) { + Held held; + SYSTEM_INFO system{}; + GetSystemInfo(&system); + auto at = reinterpret_cast(system.lpMinimumApplicationAddress); + const auto end = reinterpret_cast(system.lpMaximumApplicationAddress); + MEMORY_BASIC_INFORMATION info{}; + while (at < end && VirtualQuery(reinterpret_cast(at), &info, sizeof(info)) == sizeof(info)) { + if (info.State != MEM_FREE) { + if (info.Type == MEM_PRIVATE) { + auto& allocation = held.private_memory[reinterpret_cast(info.AllocationBase)]; + allocation.reserved += info.RegionSize; + allocation.reserved_as = info.AllocationProtect; + if (info.State == MEM_COMMIT) { + allocation.committed += info.RegionSize; + allocation.committed_as |= info.Protect; + held.committed += info.RegionSize; + if (keep_ranges) + allocation.ranges.push_back({reinterpret_cast(info.BaseAddress), info.RegionSize, info.Protect}); + } + } else if (info.State == MEM_COMMIT) { + (info.Type == MEM_IMAGE ? held.image : held.mapped) += info.RegionSize; + } + } + at = reinterpret_cast(info.BaseAddress) + info.RegionSize; + } + return held; +} + +struct Copy { + std::vector callers; + std::vector blocks; + std::uint64_t untracked{}; + std::size_t live{}; +}; +Copy copy_tables() { + Copy copy; + copy.callers.resize(caller_slots); + copy.blocks.resize(block_slots); + std::lock_guard lock(mutex); + std::copy(callers.begin(), callers.end(), copy.callers.begin()); + std::copy(blocks.begin(), blocks.end(), copy.blocks.begin()); + copy.untracked = untracked; + copy.live = blocks_live; + return copy; +} + +// The module an address is in, by name; empty when it is in none. +struct Module { + std::string name; + std::uintptr_t base{}; +}; +Module module_at(void* address) { + HMODULE module{}; + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(address), &module) || !module) + return {}; + wchar_t path[MAX_PATH]{}; + GetModuleFileNameW(module, path, MAX_PATH); + const wchar_t* name = path; + for (const wchar_t* c = path; *c; ++c) if (*c == L'\\') name = c + 1; + Module result{{}, reinterpret_cast(module)}; + for (; *name; ++name) result.name += static_cast(*name < 128 ? *name : '?'); + return result; +} +std::string place(void* address) { + const auto module = module_at(address); + if (module.name.empty()) return std::format("{:#x}", reinterpret_cast(address)); + return std::format("{}+{:#x}", module.name, reinterpret_cast(address) - module.base); +} +std::string lower(std::string text) { + for (auto& c : text) c = static_cast(c >= 'A' && c <= 'Z' ? c + 32 : c); + return text; +} +// Who a caller's memory belongs to: the first module on its stack that is not Windows' or the +// C runtime's way to memory. Through a heap when Windows' heap asked (the heap grew for +// whoever was allocating when it ran out). +std::string owner_of(const Caller& caller) { + static const std::array passing{"ntdll.dll", "kernelbase.dll", "kernel32.dll", "ucrtbase.dll", + "vcruntime140.dll", "vcruntime140_1.dll", "msvcp140.dll"}; + bool heap{}; + for (unsigned i = 0; i < caller.frame_count; ++i) { + const auto module = module_at(caller.frames[i]); + const auto name = lower(module.name); + if (i == 0 && name == "ntdll.dll") heap = true; + if (name.empty() || std::ranges::find(passing, name) != passing.end()) continue; + return heap ? module.name + " (through a heap)" : module.name; + } + return heap ? "Windows (a heap)" : "Windows"; +} +// "read/write, write-combined": what a page protection allows, in words. +std::string protection(DWORD protect) { + std::string text; + const auto add = [&](std::string_view word) { text += (text.empty() ? "" : "+") + std::string(word); }; + constexpr std::pair kinds[] = {{PAGE_NOACCESS, "no access"}, {PAGE_READONLY, "read"}, + {PAGE_READWRITE, "read/write"}, {PAGE_WRITECOPY, "write-copy"}, {PAGE_EXECUTE, "execute"}, + {PAGE_EXECUTE_READ, "execute/read"}, {PAGE_EXECUTE_READWRITE, "execute/read/write"}, + {PAGE_EXECUTE_WRITECOPY, "execute/write-copy"}}; + for (const auto& [bit, word] : kinds) if (protect & bit) add(word); + if (protect & PAGE_GUARD) text += ", guard"; + if (protect & PAGE_NOCACHE) text += ", uncached"; + if (protect & PAGE_WRITECOMBINE) text += ", write-combined"; + return text.empty() ? std::format("{:#x}", protect) : text; +} + +std::string stack_of(const Caller& caller) { + std::string line; + for (unsigned i = 0; i < caller.frame_count; ++i) line += " < " + place(caller.frames[i]); + return line; +} + +constexpr std::string_view unseen = "not seen being made (before the ledger started, or by Windows itself: thread stacks)"; + +// How much of an allocation is in RAM, and of a few of those pages, how many hold only zeros. +struct Use { + std::uint64_t resident{}; + unsigned sampled{}, zero{}; +}; +Use use_of(const Allocation& allocation, std::vector& pages) { + Use use; + const auto total_pages = allocation.committed / 4096; + const std::uint64_t stride = std::max(1, total_pages / 8); + std::uint64_t page_number{}; + std::array bytes{}; + for (const auto& range : allocation.ranges) { + // (Pages that would be spoiled or slow to read are not read: a guard page would lose + // its guard, and write-combined or uncached memory is the GPU's.) + const bool readable = !(range.protect & (PAGE_GUARD | PAGE_NOACCESS | PAGE_WRITECOMBINE | PAGE_NOCACHE)); + for (std::uint64_t done = 0; done < range.bytes;) { + const auto count = static_cast(std::min(pages.size(), (range.bytes - done) / 4096)); + if (!count) break; + for (std::size_t i = 0; i < count; ++i) pages[i].VirtualAddress = reinterpret_cast(range.base + done + i * 4096); + if (!QueryWorkingSetEx(GetCurrentProcess(), pages.data(), static_cast(count * sizeof(pages[0])))) return use; + for (std::size_t i = 0; i < count; ++i, ++page_number) { + if (!pages[i].VirtualAttributes.Valid) continue; + use.resident += 4096; + if (!readable || page_number % stride || use.sampled >= 8) continue; + SIZE_T got{}; + if (!ReadProcessMemory(GetCurrentProcess(), pages[i].VirtualAddress, bytes.data(), bytes.size(), &got) || + got != bytes.size()) + continue; + ++use.sampled; + use.zero += std::all_of(bytes.begin(), bytes.end(), [](unsigned char byte) { return byte == 0; }); + } + done += count * 4096ull; + } + } + return use; +} + +struct Totals { + std::uint64_t committed{}, reserved{}, resident{}, zero{}, blocks{}; + std::uint32_t first_made{UINT32_MAX}, last_made{}; // seconds after the ledger started + std::uint64_t smallest{UINT64_MAX}, largest{}; + void add(const Allocation& allocation, const Use& use, std::uint32_t made, bool known) { + committed += allocation.committed; + reserved += allocation.reserved; + resident += use.resident; + // (An estimate: the share of the pages looked at that were all zero.) + if (use.sampled) zero += use.resident * use.zero / use.sampled; + ++blocks; + smallest = std::min(smallest, allocation.committed); + largest = std::max(largest, allocation.committed); + if (known) first_made = std::min(first_made, made), last_made = std::max(last_made, made); + } +}; +std::string totals_text(const Totals& t) { + return std::format("{} committed, {} in RAM, {} never used or moved out, ~{} in RAM but all zero; {} reserved in {} block(s)", + megabytes(t.committed), megabytes(t.resident), megabytes(t.committed - std::min(t.committed, t.resident)), megabytes(t.zero), + megabytes(t.reserved), t.blocks); +} + +std::uint32_t seconds_running() noexcept { + return static_cast((GetTickCount64() - began.load()) / 1000); +} + +// ---- the leak watcher ---- +std::mutex& watch_mutex = *new std::mutex; +std::string& watched_owners = *new std::string; +struct History { + std::array committed{}; + unsigned count{}; + std::uint64_t said_at{}; +}; +std::unordered_map& histories = *new std::unordered_map; // by caller hash; the watcher's own +std::unordered_map& owner_names = *new std::unordered_map; + +void rebuild_if_worn() { + { + std::lock_guard lock(mutex); + if (blocks_gone * 8 < block_slots) return; + } + std::vector fresh(block_slots); + { + std::lock_guard lock(mutex); + fresh.swap(blocks); + blocks_live = blocks_gone = 0; + for (const auto& block : fresh) + if (block.caller && block.caller != gone) remember(block.base, block.caller, block.made); + } +} + +void watch_once() { + rebuild_if_worn(); + const auto held = read_held(false); + const auto copy = copy_tables(); + std::unordered_map by_caller; + std::map by_owner; + std::uint64_t unseen_bytes{}; + for (const auto& [base, allocation] : held.private_memory) { + if (!allocation.committed) continue; + if (const auto* block = find(copy.blocks, base)) by_caller[block->caller] += allocation.committed; + else unseen_bytes += allocation.committed; + } + const auto now = GetTickCount64(); + for (const auto& [index, committed] : by_caller) { + const auto& caller = copy.callers[index - 1]; + auto& name = owner_names[caller.hash]; + if (name.empty()) name = owner_of(caller); + by_owner[name] += committed; + auto& history = histories[caller.hash]; + std::rotate(history.committed.begin(), history.committed.begin() + 1, history.committed.end()); + history.committed.back() = committed; + history.count = std::min(history.count + 1, static_cast(history.committed.size())); + // Growing: over the last ten minutes, by 64 MB or more, and larger at nearly every look. + constexpr unsigned span = 10; + if (history.count <= span || now - history.said_at < 600000) continue; + const auto* last = history.committed.data() + history.committed.size() - span - 1; + unsigned rises{}, falls{}; + for (unsigned i = 1; i <= span; ++i) rises += last[i] > last[i - 1], falls += last[i] < last[i - 1]; + if (last[span] < last[0] + (64ull << 20) || rises < span * 7 / 10 || falls > 1) continue; + history.said_at = now; + logging::log(logging::Level::warning, logging::Channel::runtime, + "Memory keeps growing: {} has {} committed, {} more than {} min ago; made by{}", name, megabytes(last[span]), + megabytes(last[span] - last[0]), span, stack_of(caller)); + } + // (The callers no longer holding anything are dropped from the history.) + std::unordered_set holding; + for (const auto& [index, committed] : by_caller) holding.insert(copy.callers[index - 1].hash); + std::erase_if(histories, [&](const auto& entry) { return !holding.contains(entry.first); }); + std::vector> order; + for (const auto& [name, committed] : by_owner) order.emplace_back(committed, name); + order.emplace_back(unseen_bytes, "not seen made"); + std::ranges::sort(order, std::greater{}); + std::string line = "Committed by owner:"; + for (std::size_t i = 0; i < order.size() && i < 5; ++i) + line += std::format("{} {} {}", i ? "," : "", order[i].second, megabytes(order[i].first)); + std::lock_guard lock(watch_mutex); + watched_owners = std::move(line); +} + +void watch() { + SetThreadDescription(GetCurrentThread(), L"ReSkate memory ledger"); + for (;;) { + Sleep(60000); + try { watch_once(); } catch (...) {} + } +} + +// ---- heaps ---- +struct SizeCount { + std::uint64_t size{}, count{}; +}; +// Block sizes go into storage made before a heap is locked: allocating from a heap while +// walking it would change what is walked. +struct SizeTable { + static constexpr std::size_t slots = std::size_t{1} << 16; + std::vector counts = std::vector(slots); + std::uint64_t other_count{}, other_bytes{}; + void add(std::uint64_t size) noexcept { + for (std::size_t probe = 0; probe < 64; ++probe) { + auto& slot = counts[((size * 0x9e3779b97f4a7c15ull >> 48) + probe) & (slots - 1)]; + if (slot.count && slot.size != size) continue; + slot.size = size; + ++slot.count; + return; + } + ++other_count; + other_bytes += size; + } +}; +struct HeapSeen { + HANDLE heap{}; + std::uint64_t committed{}, busy{}, free{}, largest_free{}, busy_blocks{}; + bool whole{}; +}; +// (A fault in the walk is caught: a heap made without its own lock may change under it.) +void walk_heap(HeapSeen& heap, SizeTable& sizes) noexcept { + if (!HeapLock(heap.heap)) return; + heap.whole = true; + __try { + PROCESS_HEAP_ENTRY entry{}; + while (HeapWalk(heap.heap, &entry)) { + if (entry.wFlags & PROCESS_HEAP_REGION) { + heap.committed += entry.Region.dwCommittedSize; + } else if (entry.wFlags & PROCESS_HEAP_ENTRY_BUSY) { + heap.busy += entry.cbData; + ++heap.busy_blocks; + sizes.add(entry.cbData); + } else if (!(entry.wFlags & PROCESS_HEAP_UNCOMMITTED_RANGE)) { + heap.free += entry.cbData; + heap.largest_free = std::max(heap.largest_free, entry.cbData); + } + } + } __except (EXCEPTION_EXECUTE_HANDLER) { + heap.whole = false; + } + HeapUnlock(heap.heap); +} +std::mutex heap_mutex; +std::map heap_mark; // block size -> count +bool heap_marked{}; +} // namespace + +bool start(std::string& error) { + std::lock_guard start_lock(start_mutex); + if (running.load()) return true; + if (!prepared) { + const auto allocate = ntdll_function("NtAllocateVirtualMemory"); + const auto release = ntdll_function("NtFreeVirtualMemory"); + if (!allocate || !release || hook_prepare(allocate, reinterpret_cast(&hooked), reinterpret_cast(&original)) != HookOk) { + error = "the allocation hook could not be prepared"; + return false; + } + if (hook_prepare(release, reinterpret_cast(&hooked_free), reinterpret_cast(&original_free)) != HookOk) { + hook_remove(allocate); + error = "the release hook could not be prepared"; + return false; + } + // (Missing on an older Windows: then only the first is followed.) + if (const auto ex = ntdll_function("NtAllocateVirtualMemoryEx"); + ex && hook_prepare(ex, reinterpret_cast(&hooked_ex), reinterpret_cast(&original_ex)) != HookOk) + original_ex = nullptr; + prepared = true; + } + began.store(GetTickCount64()); + running.store(true); + // (The release hook first: a block noted and given back before it is on would stay noted.) + if (hook_enable(ntdll_function("NtFreeVirtualMemory")) != HookOk || hook_enable(ntdll_function("NtAllocateVirtualMemory")) != HookOk) { + running.store(false); + hook_disable(ntdll_function("NtFreeVirtualMemory")); + error = "the allocation hooks could not be attached"; + return false; + } + if (original_ex) hook_enable(ntdll_function("NtAllocateVirtualMemoryEx")); + std::thread(watch).detach(); + logging::log(logging::Level::info, logging::Channel::runtime, + "Memory ledger on: who makes each reservation of memory is noted (`perf memory owners`), and an owner whose memory keeps growing is said here."); + return true; +} +bool on() noexcept { return running.load(); } + +bool start_asking(std::uint64_t smallest, std::uint64_t largest, std::string& error) { + if (!start(error)) return false; + { + std::lock_guard lock(mutex); + for (auto& caller : callers) caller.asked_count = caller.asked_bytes = caller.smallest = caller.largest = 0; + } + least.store(smallest); + most.store(largest); + counting.store(true); + return true; +} +void stop_asking() noexcept { counting.store(false); } +bool asking() noexcept { return counting.load(); } + +void asked_report(const Line& out) { + const auto copy = copy_tables(); + std::vector seen; + for (const auto& caller : copy.callers) if (caller.asked_count) seen.push_back(&caller); + std::ranges::sort(seen, [](const Caller* a, const Caller* b) { return a->asked_bytes > b->asked_bytes; }); + out(std::format("Memory asked for while tracing (blocks of {} and up), by who asked ({} callers), the most first:", + megabytes(least.load()), seen.size())); + for (std::size_t i = 0; i < seen.size() && i < 16; ++i) { + out(seen[i]->smallest == seen[i]->largest + ? std::format(" {} x {} bytes = {}", seen[i]->asked_count, seen[i]->smallest, megabytes(seen[i]->asked_bytes)) + : std::format(" {} blocks of {} to {} bytes = {}", seen[i]->asked_count, seen[i]->smallest, seen[i]->largest, + megabytes(seen[i]->asked_bytes))); + out(" " + stack_of(*seen[i])); + } +} + +void owners_report(const Line& out, std::size_t most_callers) { + if (std::string error; !start(error)) { out("error: " + error); return; } + const auto copy = copy_tables(); + const auto held = read_held(true); + std::vector pages(16384); + std::unordered_map by_caller; + std::map by_owner; + Totals all, unseen_totals; + struct Unseen { std::uintptr_t base; const Allocation* allocation; Use use; }; + std::vector unseen_blocks; + for (const auto& [base, allocation] : held.private_memory) { + if (!allocation.committed) continue; + const auto use = use_of(allocation, pages); + const auto* block = find(copy.blocks, base); + all.add(allocation, use, 0, false); + if (!block) { + unseen_totals.add(allocation, use, 0, false); + unseen_blocks.push_back({base, &allocation, use}); + continue; + } + by_caller[block->caller].add(allocation, use, block->made, true); + } + std::unordered_map names; + for (const auto& [index, totals] : by_caller) { + auto& name = names[index] = owner_of(copy.callers[index - 1]); + auto& owner = by_owner[name]; + owner.committed += totals.committed; + owner.reserved += totals.reserved; + owner.resident += totals.resident; + owner.zero += totals.zero; + owner.blocks += totals.blocks; + } + by_owner[std::string(unseen)] = unseen_totals; + + out(std::format("Memory: {} committed, {} of it in RAM; {} of files mapped in, {} of programs.", megabytes(held.committed), + megabytes(all.resident), megabytes(held.mapped), megabytes(held.image))); + out(std::format("Of what is committed: {} never used or moved out of RAM by Windows, and about {} in RAM but all zero.", + megabytes(all.committed - std::min(all.committed, all.resident)), megabytes(all.zero))); + gpu_report(out); + out(std::format("Ledger: on for {}, {} reservations held by {} callers{}.", age(seconds_running()), copy.live, + std::ranges::count_if(copy.callers, [](const Caller& c) { return c.hash != 0; }), + copy.untracked ? std::format(", {} not noted (its tables were full)", copy.untracked) : std::string())); + out("By owner, the most committed first:"); + std::vector> owners; + for (const auto& [name, totals] : by_owner) owners.emplace_back(name, &totals); + std::ranges::sort(owners, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; }); + for (const auto& [name, totals] : owners) + if (totals->committed) out(std::format(" {}: {}", name, totals_text(*totals))); + + // What the memory no hook saw being made is: memory Windows or a driver maps in from the + // kernel (a GPU's memory is write-combined or uncached) never passes NtAllocateVirtualMemory. + std::map unseen_by_kind; + for (const auto& block : unseen_blocks) + unseen_by_kind[std::format("committed {}; reserved {}", protection(block.allocation->committed_as), + protection(block.allocation->reserved_as))].add(*block.allocation, block.use, 0, false); + out("Not seen being made, by protection:"); + std::vector> kinds; + for (const auto& [kind, totals] : unseen_by_kind) kinds.emplace_back(kind, &totals); + std::ranges::sort(kinds, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; }); + for (const auto& [kind, totals] : kinds) out(std::format(" {}: {}", kind, totals_text(*totals))); + std::ranges::sort(unseen_blocks, [](const Unseen& a, const Unseen& b) { return a.allocation->committed > b.allocation->committed; }); + out("The largest of them:"); + for (std::size_t i = 0; i < unseen_blocks.size() && i < 16; ++i) { + const auto& block = unseen_blocks[i]; + out(std::format(" {:#x}: {} committed of {} reserved, {} in RAM{}; committed {}, reserved {}, in {} part(s)", block.base, + megabytes(block.allocation->committed), megabytes(block.allocation->reserved), megabytes(block.use.resident), + block.use.sampled ? std::format(" ({} of {} pages looked at all zero)", block.use.zero, block.use.sampled) : std::string(), + protection(block.allocation->committed_as), protection(block.allocation->reserved_as), block.allocation->ranges.size())); + } + + std::vector> order; + for (const auto& [index, totals] : by_caller) order.emplace_back(index, &totals); + std::ranges::sort(order, [](const auto& a, const auto& b) { return a.second->committed > b.second->committed; }); + out(std::format("By caller ({}), the most committed first:", order.size())); + for (std::size_t i = 0; i < order.size() && i < most_callers; ++i) { + const auto& totals = *order[i].second; + out(std::format(" {}: {}", names[order[i].first], totals_text(totals))); + out(std::format(" blocks of {} to {}; the newest made {} ago, the oldest {} ago", megabytes(totals.smallest), + megabytes(totals.largest), age(seconds_running() - std::min(seconds_running(), totals.last_made)), + age(seconds_running() - std::min(seconds_running(), totals.first_made)))); + out(" " + stack_of(copy.callers[order[i].first - 1])); + } +} + +void heap_report(const Line& out, bool mark, std::size_t most_sizes) { + std::lock_guard report_lock(heap_mutex); + struct Summary { DWORD size; SIZE_T allocated, committed, reserved, most; }; + static const auto heap_summary = reinterpret_cast( + GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "HeapSummary")); + std::array heaps{}; + const auto heap_count = std::min(GetProcessHeaps(static_cast(heaps.size()), heaps.data()), static_cast(heaps.size())); + SizeTable sizes; + std::vector seen(heap_count); + std::uint64_t total_committed{}, total_busy{}, total_free{}; + std::size_t partial{}; + for (DWORD i = 0; i < heap_count; ++i) { + auto& heap = seen[i]; + heap.heap = heaps[i]; + walk_heap(heap, sizes); + partial += !heap.whole; + // (A heap whose segments the walk does not show, such as one of the segment heap's: its + // summary says what it holds.) + if (Summary summary{sizeof(Summary)}; heap_summary && heap_summary(heaps[i], 0, &summary) && summary.committed > heap.committed) + heap.committed = summary.committed; + total_committed += heap.committed; + total_busy += heap.busy; + total_free += heap.free; + } + out(std::format("Heaps: {} committed in {} heap(s), {} of it in use, {} free inside them{}.", megabytes(total_committed), + heap_count, megabytes(total_busy), megabytes(total_free), + partial ? std::format(" ({} could not be walked whole)", partial) : std::string())); + std::ranges::sort(seen, [](const HeapSeen& a, const HeapSeen& b) { return a.committed > b.committed; }); + for (std::size_t i = 0; i < seen.size() && i < 8; ++i) { + if (!seen[i].committed) continue; + out(std::format(" heap {:#x}{}: {} committed, {} in use in {} blocks, {} free (the largest free piece {})", + reinterpret_cast(seen[i].heap), seen[i].heap == GetProcessHeap() ? " (the process heap)" : "", + megabytes(seen[i].committed), megabytes(seen[i].busy), seen[i].busy_blocks, megabytes(seen[i].free), + megabytes(seen[i].largest_free))); + } + std::map now; + for (const auto& slot : sizes.counts) if (slot.count) now[slot.size] = slot.count; + std::vector> order(now.begin(), now.end()); // size, count + std::ranges::sort(order, [](const auto& a, const auto& b) { return a.first * a.second > b.first * b.second; }); + out("Blocks in use by size, the sizes that add up to most first:"); + for (std::size_t i = 0; i < order.size() && i < most_sizes; ++i) + out(std::format(" {} x {} bytes = {}", order[i].second, order[i].first, megabytes(order[i].first * order[i].second))); + if (sizes.other_count) out(std::format(" (and {} blocks of other sizes, {})", sizes.other_count, megabytes(sizes.other_bytes))); + if (mark) { + heap_mark = std::move(now); + heap_marked = true; + out("Marked. Do what you want to measure, then run `perf memory heap` again."); + return; + } + if (!heap_marked) return; + // What was added since the mark, by size: a leak shows as a size whose count only climbs. + std::vector> added; // size, count added + for (const auto& [size, count] : now) { + const auto before = heap_mark.find(size); + const auto was = before == heap_mark.end() ? 0 : before->second; + if (count > was) added.emplace_back(size, static_cast(count - was)); + } + std::ranges::sort(added, [](const auto& a, const auto& b) { + return std::get<0>(a) * std::get<1>(a) > std::get<0>(b) * std::get<1>(b); + }); + out("Added since the mark, by size:"); + for (std::size_t i = 0; i < added.size() && i < most_sizes; ++i) + out(std::format(" +{} x {} bytes = {}", std::get<1>(added[i]), std::get<0>(added[i]), + megabytes(std::get<0>(added[i]) * static_cast(std::get<1>(added[i]))))); +} + +namespace { +struct Gpu { + std::string name; + std::uint64_t local{}, local_budget{}, shared{}, shared_budget{}; +}; +std::vector read_gpus() { + std::vector gpus; + IDXGIFactory4* factory{}; + if (FAILED(CreateDXGIFactory1(__uuidof(IDXGIFactory4), reinterpret_cast(&factory))) || !factory) return gpus; + IDXGIAdapter1* adapter{}; + for (UINT i = 0; factory->EnumAdapters1(i, &adapter) != DXGI_ERROR_NOT_FOUND; ++i) { + DXGI_ADAPTER_DESC1 description{}; + IDXGIAdapter3* adapter3{}; + if (SUCCEEDED(adapter->GetDesc1(&description)) && !(description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) && + SUCCEEDED(adapter->QueryInterface(__uuidof(IDXGIAdapter3), reinterpret_cast(&adapter3)))) { + DXGI_QUERY_VIDEO_MEMORY_INFO local{}, shared{}; + adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_LOCAL, &local); + adapter3->QueryVideoMemoryInfo(0, DXGI_MEMORY_SEGMENT_GROUP_NON_LOCAL, &shared); + // (Only an adapter this process uses.) + if (local.CurrentUsage || shared.CurrentUsage) { + Gpu gpu{{}, local.CurrentUsage, local.Budget, shared.CurrentUsage, shared.Budget}; + for (const wchar_t* c = description.Description; *c; ++c) gpu.name += static_cast(*c < 128 ? *c : '?'); + gpus.push_back(std::move(gpu)); + } + adapter3->Release(); + } + adapter->Release(); + } + factory->Release(); + return gpus; +} +} // namespace + +void gpu_report(const Line& out) { + for (const auto& gpu : read_gpus()) + out(std::format("GPU {}: {} of video memory (budget {}), {} of system memory (budget {}).", gpu.name, + megabytes(gpu.local), megabytes(gpu.local_budget), megabytes(gpu.shared), megabytes(gpu.shared_budget))); +} + +std::string owners_line() { + if (!running.load()) return {}; + std::lock_guard lock(watch_mutex); + return watched_owners; +} +std::string gpu_line() { + std::string line; + for (const auto& gpu : read_gpus()) + line += std::format("{}GPU: {} video memory, {} system memory", line.empty() ? "" : "; ", megabytes(gpu.local), megabytes(gpu.shared)); + return line; +} + +std::filesystem::path write_report() { + SYSTEMTIME now{}; + GetLocalTime(&now); + const auto path = logging::status().directory / std::format("memory-{:04}{:02}{:02}-{:02}{:02}{:02}.txt", now.wYear, now.wMonth, + now.wDay, now.wHour, now.wMinute, now.wSecond); + std::string text; + const Line add = [&](const std::string& line) { text += line + "\n"; }; + owners_report(add, 60); + add(""); + heap_report(add, false, 60); + if (asking()) { + add(""); + asked_report(add); + } + std::ofstream file(path, std::ios::binary); + file << text; + return file ? path : std::filesystem::path{}; +} +} diff --git a/Extension/Console/memory_ledger.h b/Extension/Console/memory_ledger.h new file mode 100644 index 00000000..174f4078 --- /dev/null +++ b/Extension/Console/memory_ledger.h @@ -0,0 +1,41 @@ +#pragma once +#include +#include +#include +#include + +// Who owns the memory the process holds, and how much of it is used (`perf memory owners`). +// While it is on, every reservation of memory from Windows is noted with the code that asked +// for it, and forgotten when it is given back; a report then joins what the process holds now +// with who made it: what is committed, what of that is in RAM, and what is in RAM but all zero. +// A minute at a time it also looks for an owner whose memory only grows, and says so in the log. +// It is a research aid and off until asked for: every allocation in the process passes it. +namespace dingosdk::memory_ledger { +using Line = std::function; + +// Starts noting reservations. What was made before is reported as not seen being made. +bool start(std::string& error); +bool on() noexcept; +// `perf memory trace`: also counts what is asked for (commits too) of sizes in [least, most] +// by who asked, from now on. Starts the ledger if it is off. +bool start_asking(std::uint64_t least, std::uint64_t most, std::string& error); +void stop_asking() noexcept; +bool asking() noexcept; +void asked_report(const Line& out); + +// What the process holds by owner and by caller (the ledger is started if it is off). +// `most` callers are listed, each with its stack. +void owners_report(const Line& out, std::size_t most = 12); +// The process's heaps: what each holds and has free inside it, and its blocks by size; with +// `mark`, the sizes are remembered and the next report says what was added since. +void heap_report(const Line& out, bool mark, std::size_t most = 20); +// Video memory and the system memory the GPU uses for this process, one line per adapter. +void gpu_report(const Line& out); +// Everything above at length, into memory-