diff --git a/CMakeLists.txt b/CMakeLists.txt index 85c792a..50afea5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ endif() option(DINGOSDK_BUILD_TESTS "Build logging regression tests" OFF) option(DINGOSDK_BUILD_PARK_EDITOR_TESTS "Build park editor regression tests" OFF) +option(DINGOSDK_BUILD_TRAINER_TESTS "Build the trainer's tools and regression tests" OFF) if(DINGOSDK_BUILD_PARK_EDITOR_TESTS AND WIN32) enable_testing() add_executable(dingosdk_park_editor_tests Extension/Objects/ParkEditor/Test/park_editor_tests.cpp) @@ -66,6 +67,9 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) Extension/Throwdowns/throwdown_wire.cpp) target_include_directories(dingosdk_server_activity_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME server_activity COMMAND dingosdk_server_activity_tests) + add_executable(dingosdk_server_text_tests Server/Test/server_text_tests.cpp) + target_include_directories(dingosdk_server_text_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME server_text COMMAND dingosdk_server_text_tests) add_executable(dingosdk_speed_check_tests Server/Test/speed_check_tests.cpp) target_include_directories(dingosdk_speed_check_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME server_speed_check COMMAND dingosdk_speed_check_tests) @@ -80,6 +84,15 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) dingosdk_lz4 dingosdk_zstd) endif() add_test(NAME server_config COMMAND dingosdk_server_config_tests) + add_executable(dingosdk_server_release_tests Server/Test/server_release_tests.cpp Server/server_release.cpp) + target_include_directories(dingosdk_server_release_tests PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_server_release_tests PRIVATE dingosdk_json) + add_test(NAME server_release COMMAND dingosdk_server_release_tests) + add_executable(dingosdk_global_bans_tests Server/Test/global_bans_tests.cpp Server/global_bans.cpp + Extension/Multiplayer/developer_identity.cpp) + target_include_directories(dingosdk_global_bans_tests PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_global_bans_tests PRIVATE dingosdk_json $<$:dingosdk_https>) + add_test(NAME server_global_bans COMMAND dingosdk_global_bans_tests) add_executable(dingosdk_word_filter_tests Engine/Core/Text/Test/word_filter_tests.cpp) target_link_libraries(dingosdk_word_filter_tests PRIVATE dingosdk_word_filter) add_test(NAME word_filter COMMAND dingosdk_word_filter_tests) @@ -88,6 +101,12 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) Extension/Multiplayer/Net/wire_codec.cpp Extension/Multiplayer/Net/delta_codec.cpp) target_link_libraries(dingosdk_multiplayer_bandwidth_tests PRIVATE dingosdk_lz4 dingosdk_zstd) add_test(NAME multiplayer_bandwidth COMMAND dingosdk_multiplayer_bandwidth_tests) + # With a pose-dump recording as its argument it prints what each encoding costs on real play. + add_executable(dingosdk_pose_codec_tests + Extension/Multiplayer/Test/pose_codec_tests.cpp Extension/Multiplayer/Net/protocol.cpp Extension/Multiplayer/Remote/playback_buffers.cpp + Extension/Multiplayer/Net/wire_codec.cpp Extension/Multiplayer/Net/delta_codec.cpp) + target_link_libraries(dingosdk_pose_codec_tests PRIVATE dingosdk_lz4 dingosdk_zstd) + add_test(NAME multiplayer_pose_codec COMMAND dingosdk_pose_codec_tests) add_executable(dingosdk_multiplayer_lane_tests Extension/Multiplayer/Test/steam_lanes_tests.cpp) target_include_directories(dingosdk_multiplayer_lane_tests SYSTEM PRIVATE "${PROJECT_SOURCE_DIR}/External/steam_networking") add_test(NAME multiplayer_lanes COMMAND dingosdk_multiplayer_lane_tests) @@ -96,6 +115,8 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS) add_test(NAME multiplayer_prediction COMMAND dingosdk_multiplayer_prediction_tests) endif() if(DINGOSDK_BUILD_MULTIPLAYER_TESTS AND WIN32) + add_executable(dingosdk_velocity_boost_tests Engine/Game/Skater/Test/velocity_boost_tests.cpp) + add_test(NAME velocity_boost COMMAND dingosdk_velocity_boost_tests) if(EXISTS "${PROJECT_SOURCE_DIR}/tests/first_person_spring_tests.cpp") add_executable(dingosdk_first_person_spring_tests tests/first_person_spring_tests.cpp) target_include_directories(dingosdk_first_person_spring_tests PRIVATE "${PROJECT_SOURCE_DIR}") @@ -132,8 +153,8 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS AND WIN32) add_executable(dingosdk_multiplayer_tests Extension/Multiplayer/Test/protocol_tests.cpp Extension/Multiplayer/Net/protocol.cpp Extension/Multiplayer/Remote/playback_buffers.cpp Extension/Multiplayer/Net/wire_codec.cpp Extension/Multiplayer/Steam/steam_server_browser.cpp Server/steam_server.cpp - Extension/Throwdowns/throwdown_wire.cpp) - target_link_libraries(dingosdk_multiplayer_tests PRIVATE dingosdk_lz4 dingosdk_zstd dingosdk_logging dingosdk_word_filter) + Extension/Throwdowns/throwdown_wire.cpp Extension/Multiplayer/developer_identity.cpp) + target_link_libraries(dingosdk_multiplayer_tests PRIVATE dingosdk_lz4 dingosdk_zstd dingosdk_logging dingosdk_word_filter dingosdk_json) add_test(NAME multiplayer_protocol COMMAND dingosdk_multiplayer_tests) add_executable(dingosdk_throwdown_relay_tests Extension/Throwdowns/Test/throwdown_relay_tests.cpp Extension/Throwdowns/throwdown_relay.cpp Extension/Throwdowns/throwdown_wire.cpp) @@ -151,11 +172,16 @@ if(DINGOSDK_BUILD_MULTIPLAYER_TESTS AND WIN32) add_executable(dingosdk_multiplayer_mesh_tests Extension/Multiplayer/Test/mesh_tests.cpp Extension/Multiplayer/Net/protocol.cpp Extension/Multiplayer/Remote/playback_buffers.cpp Extension/Multiplayer/Net/wire_codec.cpp Extension/Multiplayer/Net/delta_codec.cpp - Extension/Multiplayer/Session/password.cpp Extension/Multiplayer/Steam/steam_lobbies.cpp - Extension/Multiplayer/Steam/steam_server_browser.cpp) + Extension/Multiplayer/Session/password.cpp Extension/Multiplayer/Session/party_book.cpp + Extension/Multiplayer/Steam/steam_lobbies.cpp Extension/Multiplayer/Steam/steam_server_browser.cpp) target_link_libraries(dingosdk_multiplayer_mesh_tests PRIVATE dingosdk_logging dingosdk_launcher_support dingosdk_lz4 dingosdk_zstd bcrypt dingosdk_word_filter) add_test(NAME multiplayer_mesh COMMAND dingosdk_multiplayer_mesh_tests) + add_executable(dingosdk_developer_identity_tests + Extension/Multiplayer/Test/developer_identity_tests.cpp Extension/Multiplayer/developer_identity.cpp + Extension/Multiplayer/developer_identity_fetch.cpp) + target_link_libraries(dingosdk_developer_identity_tests PRIVATE dingosdk_json dingosdk_https dingosdk_logging winhttp) + add_test(NAME developer_identity COMMAND dingosdk_developer_identity_tests) add_executable(dingosdk_settings_lock_tests Engine/Game/Settings/Test/multiplayer_settings_lock_tests.cpp) target_include_directories(dingosdk_settings_lock_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME multiplayer_settings_lock COMMAND dingosdk_settings_lock_tests) @@ -185,10 +211,28 @@ if(WIN32) file(GLOB_RECURSE game_headers CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/Engine/Game/*.h") target_sources(dingosdk_runtime PRIVATE ${game_headers}) endif() +if(DINGOSDK_BUILD_TRAINER_TESTS) + enable_testing() + add_executable(dingosdk_trainer_tuning_dump Extension/Trainer/Test/tuning_dump.cpp Extension/Skater/physics_tuning_model.cpp) + target_include_directories(dingosdk_trainer_tuning_dump PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_trainer_tuning_dump PRIVATE dingosdk_game_archives) + # Fails when the game's data stops naming a tuning value (the trainer's table comes from those names). + add_test(NAME trainer_tuning_names COMMAND dingosdk_trainer_tuning_dump "${DINGOSDK_TEST_GAME_ROOT}") + # What a session's host shares beyond its tuning: the encoding, and what a guest refuses. + add_executable(dingosdk_trainer_session_tests Extension/Trainer/Test/session_extras_tests.cpp Extension/Trainer/trainer_session.cpp) + target_include_directories(dingosdk_trainer_session_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME trainer_session_extras COMMAND dingosdk_trainer_session_tests) + # Ground-safe arrivals: the downward ray, the landing watch, and the remembered map waypoint. + add_executable(dingosdk_trainer_landing_tests Extension/Trainer/Test/landing_tests.cpp) + target_include_directories(dingosdk_trainer_landing_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME trainer_landing COMMAND dingosdk_trainer_landing_tests) + add_executable(dingosdk_trainer_asset_list Extension/Trainer/Test/asset_list.cpp) + target_include_directories(dingosdk_trainer_asset_list PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_trainer_asset_list PRIVATE dingosdk_game_archives) +endif() get_property(project_targets DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) foreach(target IN LISTS project_targets) if(NOT target MATCHES "^(dingosdk_imgui|dingosdk_detours|dingosdk_sqlite|dingosdk_lz4|dingosdk_zstd|dingosdk_miniz)$") dingosdk_configure_target(${target}) endif() endforeach() - diff --git a/EmotePacker/main.cpp b/EmotePacker/main.cpp index 7b18bc5..54a59d9 100644 --- a/EmotePacker/main.cpp +++ b/EmotePacker/main.cpp @@ -8,6 +8,7 @@ // // Dropping a folder on the exe writes the pack next to it, in "_pack". #include "Engine/Core/Json/json.h" +#include "Engine/Core/Platform/path_text.h" #include #include #include @@ -26,6 +27,7 @@ using Microsoft::WRL::ComPtr; namespace fs = std::filesystem; using dingosdk::Json; +using dingosdk::path_utf8; namespace { // ReSkate's loader limits (chat_emotes.cpp): frames at most 256 x 256, names of letters, @@ -198,10 +200,12 @@ std::vector tidy(std::vector frames, std::size_t limit) { return thinned; } +// Built from the wide name: path::string() throws on a character outside the ANSI code page, +// and one such file would stop the whole pack. std::string emote_name(const fs::path &file) { std::string name; - for (const char c : file.stem().string()) - name += std::isalnum(static_cast(c)) ? c : '_'; + for (const wchar_t c : file.stem().wstring()) + name += c < 128 && std::isalnum(static_cast(c)) ? static_cast(c) : '_'; if (name.size() > max_name) name.resize(max_name); return name.empty() ? std::string("emote") : name; } @@ -266,7 +270,7 @@ Options parse(int argc, wchar_t **argv) { if (arg == L"--size") o.size = std::stoi(value()); else if (arg == L"--max-frames") o.max_frames = static_cast(std::stoul(value())); else if (arg == L"--max-width") o.max_width = std::stoi(value()); - else if (arg == L"--name") o.name = fs::path(value()).string(); + else if (arg == L"--name") o.name = path_utf8(value()); else if (arg == L"--help" || arg == L"-h" || arg == L"/?") throw std::invalid_argument(""); else positional.emplace_back(arg); } @@ -281,16 +285,16 @@ Options parse(int argc, wchar_t **argv) { } int run(const Options &o) { - if (!fs::is_directory(o.input)) throw std::runtime_error("not a folder: " + o.input.string()); + if (!fs::is_directory(o.input)) throw std::runtime_error("not a folder: " + path_utf8(o.input)); std::vector files; for (const auto &entry : fs::directory_iterator(o.input)) { - const auto extension = lowered(entry.path().extension().string()); + const auto extension = lowered(path_utf8(entry.path().extension())); if (entry.is_regular_file() && (extension == ".png" || extension == ".gif" || extension == ".jpg" || extension == ".jpeg" || extension == ".bmp" || extension == ".webp")) files.push_back(entry.path()); } std::sort(files.begin(), files.end()); - if (files.empty()) throw std::runtime_error("no .png, .gif, .jpg, .bmp or .webp files in " + o.input.string()); + if (files.empty()) throw std::runtime_error("no .png, .gif, .jpg, .bmp or .webp files in " + path_utf8(o.input)); std::vector emotes; std::set names, folded; @@ -298,6 +302,8 @@ int run(const Options &o) { for (const auto &file : files) { try { auto name = emote_name(file); + if (std::none_of(name.begin(), name.end(), [](unsigned char c) { return std::isalnum(c); })) + throw std::runtime_error("no letters or digits to name it by; rename the file"); for (int n = 2; names.contains(name); ++n) { const auto suffix = "_" + std::to_string(n); name = emote_name(file).substr(0, max_name - suffix.size()) + suffix; @@ -315,10 +321,10 @@ int run(const Options &o) { names.insert(name); std::printf(" :%s: %dx%d%s\n", name.c_str(), frames.front().image.width, frames.front().image.height, frames.size() > 1 ? (", " + std::to_string(frames.size()) + " frames").c_str() : ""); - emotes.push_back({name, file.filename().string(), std::move(frames)}); + emotes.push_back({name, path_utf8(file.filename()), std::move(frames)}); } catch (const std::exception &e) { ++failed; - std::printf(" skipped %s: %s\n", file.filename().string().c_str(), e.what()); + std::printf(" skipped %s: %s\n", path_utf8(file.filename()).c_str(), e.what()); } } if (emotes.empty()) throw std::runtime_error("no pictures could be read"); @@ -367,13 +373,14 @@ int run(const Options &o) { std::printf("\n%zu emotes (%zu frames) in a %dx%d atlas%s.\nWrote %s\\emotes.json and emotes.png.\n" "Copy both into the repository's assets\\emotes and rebuild ReSkate.dll.\n", emotes.size(), total, o.max_width, height, failed ? (", " + std::to_string(failed) + " skipped").c_str() : "", - o.output.string().c_str()); + path_utf8(o.output).c_str()); return failed ? 2 : 0; } } // namespace int wmain(int argc, wchar_t **argv) { std::setvbuf(stdout, nullptr, _IONBF, 0); // progress shows as it happens, even through a pipe + SetConsoleOutputCP(CP_UTF8); // file names are printed as UTF-8 const bool own_console = [] { DWORD processes[2]; return GetConsoleProcessList(processes, 2) == 1; // started by double-click or drag-and-drop diff --git a/Engine/Core/Platform/launcher_pe.cpp b/Engine/Core/Platform/launcher_pe.cpp index affed24..64a72a4 100644 --- a/Engine/Core/Platform/launcher_pe.cpp +++ b/Engine/Core/Platform/launcher_pe.cpp @@ -1,10 +1,12 @@ #include "launcher_support.h" #include "launcher_support_internal.h" +#include "path_text.h" #ifdef _WIN32 #include #include #else +#include #include #include #endif @@ -137,15 +139,56 @@ std::string rva_string(const MappedFile& file, const ParsedPe& pe, std::uint32_t fail("Unterminated PE export name"); } +// Opens a file to read all of it. A file that was written a moment ago (a download) is often +// held by the antivirus scanning it, so a sharing violation is waited out for a few seconds. +// A file that still cannot be opened is named along with the reason: with ReSkate.dll that is +// nearly always an antivirus blocking it, and "cannot open file" told the player nothing. +Handle open_to_read(const fs::path& path) { + DWORD error{}; + for (int attempt = 0; attempt < 50; ++attempt) { + Handle file(CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr)); + if (file.get() != INVALID_HANDLE_VALUE) return file; + error = GetLastError(); + if (error != ERROR_SHARING_VIOLATION && error != ERROR_LOCK_VIOLATION) break; + Sleep(100); + } + const auto name = path_utf8(path.filename()); + const auto folder = path_utf8(path.parent_path()); + std::string message; + switch (error) { + case ERROR_VIRUS_INFECTED: + case ERROR_VIRUS_DELETED: + message = "Your antivirus blocked " + name + ". Add the folder " + folder + + " to its exclusions (in Windows Security: Virus & threat protection > Exclusions), then try again."; + break; + case ERROR_ACCESS_DENIED: + message = name + " cannot be read: access was denied. An antivirus is probably blocking it: add the folder " + folder + + " to its exclusions, then try again."; + break; + case ERROR_SHARING_VIOLATION: + case ERROR_LOCK_VIOLATION: + message = name + " is in use by another program, usually an antivirus scan. Close Skate if it is running, wait a moment and try again; " + "if it keeps happening, add the folder " + folder + " to your antivirus exclusions."; + break; + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + message = name + " is missing from " + folder + ". If it was just downloaded, an antivirus removed it: add the folder to its " + "exclusions, then try again."; + break; + default: + message = "Cannot read " + name + " in " + folder; + break; + } + throw std::runtime_error(message + " (Windows error " + std::to_string(error) + ")"); +} } // namespace #endif // _WIN32 std::string sha256_file(const fs::path& path) { #ifdef _WIN32 - Handle file(CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, - nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN, nullptr)); - if (file.get() == INVALID_HANDLE_VALUE) detail::fail("Cannot open file for SHA-256"); + const auto file = open_to_read(path); Algorithm algorithm; if (BCryptOpenAlgorithmProvider(&algorithm.value, BCRYPT_SHA256_ALGORITHM, nullptr, 0) < 0) @@ -179,8 +222,15 @@ std::string sha256_file(const fs::path& path) { } return output; #else + // Name the file and the reason, as open_to_read does on Windows: a server admin reading + // "cannot open file" in the log has no idea which file, or what to fix. std::ifstream input(path, std::ios::binary); - if (!input) detail::fail("Cannot open file for SHA-256"); + if (!input) { + const int error = errno; + throw std::runtime_error("Cannot open " + path_utf8(path.filename()) + " in " + + path_utf8(path.parent_path()) + " for SHA-256: " + + (error ? std::strerror(error) : "unknown error")); + } SHA256_CTX context; if (!SHA256_Init(&context)) detail::fail("Cannot initialize SHA-256"); std::vector bytes(1024 * 1024); diff --git a/Engine/Core/Platform/memory.h b/Engine/Core/Platform/memory.h index edc53d7..589416b 100644 --- a/Engine/Core/Platform/memory.h +++ b/Engine/Core/Platform/memory.h @@ -40,8 +40,10 @@ inline std::ptrdiff_t peek_cstring(std::uintptr_t address, char* destination, st if (!destination || !capacity || address < 0x10000 || address > highest_user_address - capacity) return -1; __try { const auto* source = reinterpret_cast(address); - for (std::size_t i = 0; i < capacity; ++i) - if (!(destination[i] = source[i])) return static_cast(i); + for (std::size_t i = 0; i < capacity; ++i) { + destination[i] = source[i]; + if (destination[i] == '\0') return static_cast(i); + } return -1; } __except (EXCEPTION_EXECUTE_HANDLER) { return -1; diff --git a/Engine/Core/Profiling/sampler.cpp b/Engine/Core/Profiling/sampler.cpp index 3d93b7d..fc62615 100644 --- a/Engine/Core/Profiling/sampler.cpp +++ b/Engine/Core/Profiling/sampler.cpp @@ -186,6 +186,7 @@ const HANDLE symbol_session = reinterpret_cast(static_cast symbols_loaded; +const char waiting_marker{}; // an address inside this module, to tell it from the others std::string symbol_name(const Module& module, std::uintptr_t address) { std::lock_guard lock(symbols_mutex); if (!symbols_ready) { @@ -200,8 +201,16 @@ std::string symbol_name(const Module& module, std::uintptr_t address) { info->SizeOfStruct = sizeof(SYMBOL_INFO); info->MaxNameLen = 511; DWORD64 displacement{}; - if (symbols_ready && SymFromAddr(symbol_session, address, &displacement, info) && info->Name[0]) - return module.name + "!" + info->Name; + if (symbols_ready && SymFromAddr(symbol_session, address, &displacement, info) && info->Name[0]) { + // A release ReSkate.dll ships without its symbols, so all this finds in it is the nearest + // export, megabytes away: every function of ours then reads as one of three exports and + // a report cannot say which was slow. Unless the address is inside a symbol with a size + // (the real ones have one), give our own module's address instead: the release's symbols + // (build/release/ReSkate--symbols) name it afterwards. + const auto here = reinterpret_cast(&waiting_marker); + const bool own = module.base <= here && here < module.end; + if (!own || (info->Size && displacement < info->Size)) return module.name + "!" + info->Name; + } return std::format("{}+0x{:x}", module.name, address - module.base); } diff --git a/Engine/Game/Build/20260929/local_music.h b/Engine/Game/Build/20260929/local_music.h index 99b1b57..b472fe4 100644 --- a/Engine/Game/Build/20260929/local_music.h +++ b/Engine/Game/Build/20260929/local_music.h @@ -2,6 +2,7 @@ #include "Engine/Game/Build/fingerprint.h" #include "engine.h" #include +#include #include namespace dingosdk::game::build::v20260929::local_music { @@ -40,6 +41,11 @@ inline constexpr std::uintptr_t context_insert = 0x6c6120; inline constexpr std::uintptr_t complete_delegate = 0x58b1f00; // Music UI initializer (hook target). inline constexpr Fingerprint music_ui_initialize_contract{0x868760, {0x48,0x89,0x6c,0x24,0x18,0x48,0x89,0x74,0x24,0x20,0x41,0x54,0x41,0x56,0x41,0x57,0x48,0x81,0xec,0x80,0x00,0x00,0x00,0x49,0x8b,0xe9,0x4d,0x8b,0xf0,0x4c,0x8b,0xfa}}; +// Queue selector shared by UI Next and natural end-of-track selection. The queue +// supplies its own channel and registered playlist asset; do not hard-code IDs. +inline constexpr Fingerprint playback_select_next_contract{0x14f8ed0, {0x48,0x89,0x54,0x24,0x10,0x55,0x53,0x56,0x57,0x41,0x54,0x41,0x55,0x41,0x56,0x41,0x57,0x48,0x8d,0x6c,0x24,0xc8,0x48,0x81,0xec,0x38,0x01,0x00,0x00,0x4c,0x8b,0xe1}}; +// Native playlist playback-mode setter (PlaylistPlaybackMode, not MusicPlaylistType). +inline constexpr Fingerprint playback_mode_setter_contract{0x443aa0, {0x44,0x8b,0x91,0xa8,0x00,0x00,0x00,0x4c,0x8b,0x99,0xa0,0x00,0x00,0x00,0x44,0x8b,0xca,0x8b,0xc2,0x33,0xd2,0x49,0xf7,0xf2,0x8b,0xc2,0x49,0x8b,0x04,0xc3,0x48,0x85}}; inline constexpr std::array music_ui_contracts{{ music_ui_initialize_contract, {construct_playlist, {0x48,0x89,0x5c,0x24,0x08,0x57,0x48,0x83,0xec,0x20,0x48,0x8b,0xda,0x48,0x8b,0xf9,0x48,0x8d,0x15,0x21,0x95,0x32,0x04,0xe8,0x94,0x30,0x5e,0x00,0x33,0xc9,0x48,0x8d}}, @@ -49,4 +55,20 @@ inline constexpr std::array music_ui_contracts{{ {context_insert, {0x40,0x53,0x55,0x56,0x57,0x41,0x54,0x41,0x55,0x41,0x56,0x41,0x57,0x48,0x83,0xec,0x38,0x49,0x8b,0xf9,0x49,0x8b,0xd8,0x48,0x8b,0xf2,0x4c,0x8b,0xf9,0x33,0xed,0x48}}, {complete_delegate, {0x48,0x83,0xec,0x28,0x48,0x8b,0x11,0x48,0x8b,0x0a,0x48,0x8b,0x41,0x40,0x48,0x85,0xc0,0x74,0x0d,0x80,0x79,0x68,0x00,0x75,0x07,0xff,0xd0,0x48,0x83,0xc4,0x28,0xc3}} }}; +// Playlist lookup crash guard. The shipped loop loads a song entry (rcx), masks it with +// `and rcx, -5` and tests a flag at [rcx+0x14]; an unresolved or NULL entry makes that +// dereference fault and crashes the game. The 13-byte patch replaces the leading 5-byte +// `mov eax, 0` with a 2-byte `xor eax, eax`, a 2-byte `jrcxz` that jumps past the whole +// dereference when rcx is zero, and a padding `nop`, so the untouched tail stays aligned. +inline constexpr std::uintptr_t playlist_lookup_safety_rva = 0x14f80c7; +inline constexpr std::array playlist_lookup_safety_contract{ + 0xb8, 0x00, 0x00, 0x00, 0x00, 0x48, 0x83, 0xe1, 0xfb, 0x66, 0x85, 0x71, 0x14}; +inline constexpr std::array playlist_lookup_patch_bytes{ + 0x31, 0xc0, 0x48, 0x83, 0xe1, 0xfb, 0xe3, 0x29, 0x66, 0x85, 0x71, 0x14, 0x90}; +// The jrcxz sits at +6 (after the 2-byte xor and 4-byte and) and ends at +8; its +0x29 +// displacement lands on the loop increment (`inc r9d; add r8, 4`) instead of the fault. +inline constexpr std::size_t playlist_lookup_jump_next = 8; +inline constexpr std::size_t playlist_lookup_jump_displacement = 0x29; +inline constexpr std::uintptr_t playlist_lookup_jump_target = + playlist_lookup_safety_rva + playlist_lookup_jump_next + playlist_lookup_jump_displacement; } diff --git a/Engine/Game/Build/20260929/offboard_flight.h b/Engine/Game/Build/20260929/offboard_flight.h index c0d6540..5d2c731 100644 --- a/Engine/Game/Build/20260929/offboard_flight.h +++ b/Engine/Game/Build/20260929/offboard_flight.h @@ -35,4 +35,16 @@ inline constexpr std::array offboard_flight_states{{ 0xe3,0x71,0x21,0x4f,0x18,0x20,0xc5,0xe8,0x58,0xc1,0xc5,0xf8,0x11,0x06,0x4c,0x8b}}}, }}; inline constexpr std::uintptr_t offboard_flight_vtable = 0x65e8f40; +// FollowRagdoll / FollowAnimatedRagdoll / FollowSimulatedRagdoll also keep +// world velocity at +10. Their shared update derives it from displacement +// (RVA 48380DE), including Y at +14. Boosts must update this cached velocity +// as well as the skeleton bodies when diving or gliding. +inline constexpr std::array, 3> offboard_ragdoll_states{{ + {0x720, 0x65e8df8, 0x4832c80}, + {0x8e0, 0x65e8e60, 0x4830a70}, + {0xaa0, 0x65e8ed0, 0x48332f0}, +}}; +inline constexpr Fingerprint ragdoll_velocity_write{0x48380de, { + 0xc4,0xe2,0x79,0x18,0x80,0xec,0x17,0x00,0x00,0xc5,0xe8,0x5e,0xc8,0xc5,0xf8,0x11, + 0x4f,0x10,0xc5,0xf8,0x10,0x43,0x30,0xc5,0xf8,0x11,0x87,0x60,0x01,0x00,0x00,0x4c}}; } diff --git a/Engine/Game/Build/20260929/physics_world_size.h b/Engine/Game/Build/20260929/physics_world_size.h new file mode 100644 index 0000000..90ef97a --- /dev/null +++ b/Engine/Game/Build/20260929/physics_world_size.h @@ -0,0 +1,72 @@ +// Reviewed native contract for supported_build::game_sha256 (September 29 2026). +// fb::FBPhysicsWorld creation copies the 0xa8-byte FBPhysicsWorldSize at the +// front of its descriptor into the job arguments (+0x2428) before any pool, +// free list or per-body array is sized from it. Both native callers build that +// descriptor on their own stack immediately before the call. +#pragma once +#include +#include +#include +#include + +namespace dingosdk::game::build::v20260929::physics_world_size { +template consteval auto hex_bytes(const char (&hex)[N]) { + static_assert(N % 2 == 1); + std::array result{}; + const auto nibble = [](char c) -> unsigned { + if (c >= '0' && c <= '9') return static_cast(c - '0'); + if (c >= 'a' && c <= 'f') return static_cast(c - 'a' + 10); + throw "Invalid physics world size contract hex"; + }; + for (std::size_t i = 0; i < result.size(); ++i) + result[i] = static_cast((nibble(hex[i * 2]) << 4) | nibble(hex[i * 2 + 1])); + return result; +} + +// World creation entry, through the stack cookie. +inline constexpr auto create_entry = hex_bytes( + "48895c241848897424205741544155415641574881eca0030000c5f829b42490030000488b05a6d3bb044833c4" + "4889842480030000"); +// rbx = this->m_jobArguments; copy descriptor[0x00..0xa8) to rbx+0x2428. +inline constexpr auto size_copy = hex_bytes( + "488b5e28c4c17c1006c5fc118328240000c4c17c104e20c5fc118b48240000c4c17c104640c5fc118368240000" + "c4c17c104e60c5fc118b88240000c4c17c108680000000c5fc1183a8240000c4c17b108ea0000000c5fb118bc8" + "240000"); +// The world reserves one internal body on top of BodyCount (data 8200 -> 8201 slots). +inline constexpr auto internal_body = hex_bytes("ff833024000083832824000003"); +// Both callers: lea rdx,[stack descriptor]; mov rcx,rax; call create. +inline constexpr auto caller_a_call = hex_bytes("e851195200"); +inline constexpr auto caller_b_call = hex_bytes("e8306f8dfe"); + +struct Contract { + std::uintptr_t rva; + std::span bytes; +}; +inline constexpr Contract contracts[]{ + {0x02606ff0, create_entry}, + {0x02607fec, size_copy}, + {0x026080b0, internal_body}, + {0x020e569a, caller_a_call}, + {0x03d300bb, caller_b_call}, +}; +inline constexpr std::size_t max_contract_size = [] { + std::size_t size = 0; + for (const auto& contract : contracts) size = contract.bytes.size() > size ? contract.bytes.size() : size; + return size; +}(); + +// unsigned-int fields of FBPhysicsWorldSize in reflected field order (index * 4). +inline constexpr std::size_t size_bytes = 0xa8; +inline constexpr std::size_t geometry_count = 0x00; +inline constexpr std::size_t owner_count = 0x04; +inline constexpr std::size_t body_count = 0x08; +inline constexpr std::size_t hull_and_mesh_count = 0x14; +inline constexpr std::size_t shape_count = 0x18; +inline constexpr std::size_t aggregate_shape_count = 0x1c; +inline constexpr std::size_t fixed_shape_count = 0x20; +inline constexpr std::size_t fixed_partition_count = 0x3c; +inline constexpr std::size_t persistent_spatial_index_count = 0xa0; + +// fb::FBPhysicsWorld creation: (void* world, Descriptor* descriptor). +inline constexpr std::uintptr_t create_rva = 0x02606ff0; +} // namespace dingosdk::game::build::v20260929::physics_world_size diff --git a/Engine/Game/Input/controller_bindings.h b/Engine/Game/Input/controller_bindings.h index f833dac..cffcf00 100644 --- a/Engine/Game/Input/controller_bindings.h +++ b/Engine/Game/Input/controller_bindings.h @@ -1,5 +1,7 @@ #pragma once +#include #include +#include #include #include #include @@ -7,10 +9,69 @@ namespace dingosdk { // XInput button bits, plus two synthetic trigger bits (above the dead zone). inline constexpr std::uint32_t controller_button_mask = 0x3f3ff; +// Single keys retain their saved format. Chords pack up to five sorted key IDs +// into the remaining 30 bits of a distinct tag in the same profile field. +inline constexpr std::uint32_t keyboard_binding_tag = 0x80000000u; +inline constexpr std::uint32_t keyboard_chord_tag = 0x40000000u; +inline constexpr std::uint8_t keyboard_key_id(std::uint32_t key) noexcept { + if (key >= '0' && key <= '9') return static_cast(key - '0' + 1); + if (key >= 'A' && key <= 'Z') return static_cast(key - 'A' + 11); + if (key == 0x20) return 37; // Space + if (key >= 0x70 && key <= 0x7b) return static_cast(key - 0x70 + 38); // F1-F12 + if (key == 0x11) return 50; // Ctrl + if (key == 0x10) return 51; // Shift + if (key == 0x12) return 52; // Alt + return 0; +} +inline constexpr std::uint8_t keyboard_key_from_id(std::uint32_t id) noexcept { + if (id >= 1 && id <= 10) return static_cast('0' + id - 1); + if (id >= 11 && id <= 36) return static_cast('A' + id - 11); + if (id == 37) return 0x20; + if (id >= 38 && id <= 49) return static_cast(0x70 + id - 38); + if (id == 50) return 0x11; + if (id == 51) return 0x10; + if (id == 52) return 0x12; + return 0; +} +inline constexpr bool bindable_keyboard_key(std::uint32_t key) noexcept { + return keyboard_key_id(key) != 0; +} +inline constexpr bool keyboard_binding(std::uint32_t value) noexcept { + if ((value & ~0xffu) == keyboard_binding_tag) return bindable_keyboard_key(value & 0xffu); + if ((value & 0xc0000000u) != keyboard_chord_tag) return false; + unsigned count = 0, previous = 0; + bool ended = false; + for (unsigned slot = 0; slot < 5; ++slot) { + const unsigned id = (value >> (slot * 6)) & 0x3fu; + if (!id) { ended = true; continue; } + if (ended || !keyboard_key_from_id(id) || id <= previous) return false; + previous = id; + ++count; + } + return count >= 2; +} +inline std::uint32_t keyboard_binding_from_keys(const std::array& keys) noexcept { + std::uint32_t value = keyboard_chord_tag; + unsigned count = 0; + for (unsigned id = 1; id <= 52; ++id) { + const auto key = keyboard_key_from_id(id); + if (!(keys[key / 64] & (std::uint64_t{1} << (key % 64)))) continue; + if (count == 5) return 0; + value |= id << (count++ * 6); + } + if (count == 1) { + const auto id = value & 0x3fu; + return keyboard_binding_tag | keyboard_key_from_id(id); + } + return count >= 2 ? value : 0; +} inline constexpr bool valid_controller_combo(std::uint32_t value) noexcept { return (value & ~controller_button_mask) == 0; } -// Button names shown to the player. Bindings are always stored as XInput bits. +inline constexpr bool valid_action_binding(std::uint32_t value) noexcept { + return valid_controller_combo(value) || keyboard_binding(value); +} +// Button names shown to the player. Controller chords keep their XInput bits. enum class ControllerStyle : std::uint8_t { xbox, dualshock4, dualsense }; struct ControllerInput { bool available{}; @@ -19,16 +80,46 @@ struct ControllerInput { // (a pad connecting, disconnecting or switching source) resets latches. unsigned device{}; ControllerStyle style{}; + std::array keys{}; }; struct ControllerBindingsModel { bool available{}; + bool freecam_controller{}; + std::uint32_t freecam_controller_combo{}; + std::uint32_t freecam_combo{}; + std::uint32_t tp_to_freecam_combo{}; std::uint32_t noclip_combo{}; std::uint32_t forward_velocity_combo{}; std::uint32_t up_velocity_combo{}; + std::uint32_t offboard_up_velocity_combo{}; std::string status; }; inline std::string controller_combo_label(std::uint32_t value, ControllerStyle style = ControllerStyle::xbox) { if (!value) return "Not bound"; + if (keyboard_binding(value)) { + const auto key_name = [](std::uint32_t key) { + if (key == 0x20) return std::string("Space"); + if (key == 0x11) return std::string("Ctrl"); + if (key == 0x10) return std::string("Shift"); + if (key == 0x12) return std::string("Alt"); + if (key >= 0x70) return "F" + std::to_string(key - 0x6fu); + return std::string(1, static_cast(key)); + }; + if ((value & 0xc0000000u) == keyboard_binding_tag) return key_name(value & 0xffu); + std::string label; + const auto append = [&](unsigned wanted) { + for (unsigned slot = 0; slot < 5; ++slot) { + const auto id = (value >> (slot * 6)) & 0x3fu; + if (!id) break; + if (id != wanted) continue; + if (!label.empty()) label += " + "; + label += key_name(keyboard_key_from_id(id)); + } + }; + for (unsigned id = 50; id <= 52; ++id) append(id); + for (unsigned id = 1; id < 50; ++id) append(id); + return label; + } const bool playstation = style != ControllerStyle::xbox; struct Button { std::uint32_t bit; const char* xbox; const char* playstation; }; constexpr Button buttons[]{ @@ -46,18 +137,34 @@ inline std::string controller_combo_label(std::uint32_t value, ControllerStyle s return label; } // A held combo never repeats. Returning from menus, reconnecting, changing a -// binding, and changing level all require releasing every bound button first. +// binding, and changing level all require releasing the bound inputs first. struct ControllerComboLatch { std::uint32_t combo{}; unsigned device{}; bool armed{}; bool update(std::uint32_t binding, ControllerInput input, bool inhibited) noexcept { - if (binding != combo || input.device != device) { - combo = binding; device = input.device; armed = false; + const bool keyboard = keyboard_binding(binding); + const unsigned source = keyboard ? 0 : input.device; + if (binding != combo || source != device) { + combo = binding; device = source; armed = false; } - if (inhibited || !input.available || !combo || !valid_controller_combo(combo)) { + if (inhibited || (!keyboard && !input.available) || !combo || !valid_action_binding(combo)) { armed = false; return false; } + if (keyboard) { + bool any = false, all = true; + for (unsigned slot = 0; slot < 5; ++slot) { + const auto key = (combo & 0xc0000000u) == keyboard_binding_tag ? + (slot ? 0u : combo & 0xffu) : keyboard_key_from_id((combo >> (slot * 6)) & 0x3fu); + if (!key) break; + const bool held = (input.keys[key / 64] & (std::uint64_t{1} << (key % 64))) != 0; + any |= held; all &= held; + } + if (!any) armed = true; + if (!armed || !all) return false; + armed = false; + return true; + } const auto held = input.buttons & combo; if (!held) armed = true; if (!armed || held != combo) return false; @@ -71,13 +178,30 @@ struct ControllerComboCapture { std::uint32_t chord{}; unsigned device{}; bool ready{}; - std::optional update(ControllerInput input) noexcept { - if (!input.available || input.device != device) { - chord = 0; ready = false; device = input.device; + std::array keyboard_chord{}; + std::optional update(ControllerInput input, bool allow_keyboard = false) noexcept { + if (input.device != device || (!input.available && !allow_keyboard)) { + chord = 0; ready = false; keyboard_chord = {}; device = input.device; + } + if (!input.available && !allow_keyboard) return {}; + const auto buttons = input.buttons & controller_button_mask; + const bool keys_held = std::any_of(input.keys.begin(), input.keys.end(), [](auto word) { return word != 0; }); + if (!ready) { ready = buttons == 0 && (!allow_keyboard || !keys_held); return {}; } + if (allow_keyboard && keys_held && !chord) { + if (std::popcount(input.keys[0]) + std::popcount(input.keys[1]) + + std::popcount(input.keys[2]) + std::popcount(input.keys[3]) > + std::popcount(keyboard_chord[0]) + std::popcount(keyboard_chord[1]) + + std::popcount(keyboard_chord[2]) + std::popcount(keyboard_chord[3])) keyboard_chord = input.keys; + } + if (std::any_of(keyboard_chord.begin(), keyboard_chord.end(), [](auto word) { return word != 0; })) { + if (!keys_held) { + const auto result = keyboard_binding_from_keys(keyboard_chord); + keyboard_chord = {}; ready = false; + if (result) return result; + } + return {}; } if (!input.available) return {}; - const auto buttons = input.buttons & controller_button_mask; - if (!ready) { ready = buttons == 0; return {}; } if (std::popcount(buttons) > std::popcount(chord)) chord = buttons; if (!buttons && chord) { const auto result = chord; chord = 0; ready = false; diff --git a/Engine/Game/Multiplayer/object_placement.h b/Engine/Game/Multiplayer/object_placement.h index e53010a..443b9b1 100644 --- a/Engine/Game/Multiplayer/object_placement.h +++ b/Engine/Game/Multiplayer/object_placement.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -27,4 +28,21 @@ inline std::optional parse_object_placement(std::string_view ar : current == ObjectPlacement::host_only ? ObjectPlacement::nobody : ObjectPlacement::everyone; return {}; } +// How many objects each player may have placed in a session, set by the host or the server: +// 0 is no limit beyond the protocol's own (max_object_limit). The host and a server's admins +// are not held to it. A player over it keeps what everyone already sees of theirs; only the +// objects past the limit are left out. +inline constexpr unsigned max_object_limit = 1024; +// What a server or a host starts with, until they choose otherwise. +inline constexpr unsigned default_object_limit = 100; +inline constexpr bool valid_object_limit(std::uint64_t value) noexcept { return value <= max_object_limit; } +// Accepts a number from 1 to max_object_limit, and off|none|0 for no limit. +inline std::optional parse_object_limit(std::string_view argument) noexcept { + if (argument == "off" || argument == "none") return 0U; + unsigned value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (argument.empty() || parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || !valid_object_limit(value)) + return {}; + return value; +} } // namespace dingosdk diff --git a/Engine/Game/Multiplayer/session_limits.h b/Engine/Game/Multiplayer/session_limits.h index 22acef3..dd8c407 100644 --- a/Engine/Game/Multiplayer/session_limits.h +++ b/Engine/Game/Multiplayer/session_limits.h @@ -2,7 +2,7 @@ namespace dingosdk { // Everyone a session can hold: the wire protocol's limit, reached by dedicated servers. inline constexpr int multiplayer_player_limit = 250; -// Players in a lobby a player hosts (their own game relays everyone, and the lobby is one party). +// Players in a lobby a player hosts (their own game relays everyone). inline constexpr int multiplayer_lobby_player_limit = 32; static_assert(multiplayer_lobby_player_limit <= multiplayer_player_limit); } diff --git a/Engine/Game/Multiplayer/session_model.h b/Engine/Game/Multiplayer/session_model.h index 6dd3472..04ed311 100644 --- a/Engine/Game/Multiplayer/session_model.h +++ b/Engine/Game/Multiplayer/session_model.h @@ -8,9 +8,25 @@ #include #include #include +#include #include namespace dingosdk { +constexpr unsigned max_map_rotation = 1440; // minutes a server's rotation keeps one map, at most +// A dedicated server's name: 1 to 64 letters, digits, spaces and - _ / [ ] ( ), with a +// letter or digit among them and no space at either end. A server refuses any other +// name, and the server browser does not show one. +inline constexpr char server_name_rule[] = "1 to 64 letters, numbers, spaces and - _ / [ ] ( )"; +[[nodiscard]] constexpr bool valid_server_name(std::string_view name) noexcept { + if (name.empty() || name.size() > 64 || name.front() == ' ' || name.back() == ' ') return false; + bool named{}; + for (const auto c : name) { + const bool word = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); + if (!word && std::string_view(" -_/[]()").find(c) == std::string_view::npos) return false; + named |= word; + } + return named; +} struct MultiplayerLobby { std::uint64_t id{}, owner{}; std::string name, map, code; @@ -18,8 +34,23 @@ struct MultiplayerLobby { int players = 1, capacity = multiplayer_player_limit; // A dedicated server rather than a player's lobby: joined by its code. bool dedicated{}; + // A dedicated server the ReSkate team runs (developer_identity.h): first in the browser. + bool official{}; int ping = -1; // ms, when the server answered directly + // A dedicated server that takes direct connections: its address (host byte order) and UDP + // port, 0 for none. + std::uint32_t direct_ip{}; + int direct_port{}; + // Steam friends skating there now, by name (steam_social.h). + std::vector friends; }; +// "Ana, Ben and 2 more": the friends in a server, as a browser row has room for. +inline std::string lobby_friends_text(const MultiplayerLobby &lobby, std::size_t named = 2) { + std::string text; + for (std::size_t i = 0; i < lobby.friends.size() && i < named; ++i) text += (i ? ", " : "") + lobby.friends[i]; + if (lobby.friends.size() > named) text += " and " + std::to_string(lobby.friends.size() - named) + " more"; + return text; +} struct MultiplayerPlayer { std::uint64_t id{}; std::string name; @@ -66,12 +97,20 @@ inline constexpr std::size_t multiplayer_chat_history = 50; struct MultiplayerChatLine { std::uint64_t sequence{}; // local arrival order, increasing within the process std::uint64_t sender{}; // Steam id; 0 for a notice from ReSkate itself + std::uint64_t received{}; // local monotonic time (microseconds) the line arrived std::string name, text; bool local{}; // sent by this player // The sender's role, as their nametag shows it: its colour (IM_COL32 layout, 0 = none) - // and a tag shown in a box before the name ("Dev", "Admin", "Host", "Friend" or empty). + // and a tag shown in a box before the name ("Dev", "Staff", "Creator", "Centrix", "Homie", "Admin", "Host", "Friend", "Server" or empty). std::uint32_t color{}; std::string tag; + // With the chat filter on, `text` is masked and this is the line as sent (same length), so + // the overlay can keep emote names the filter caught; empty when nothing was masked. + std::string unmasked; + bool server{}; // said by the dedicated server itself: its console, welcome or answers + // The colour of the line's own text when it is not the usual one (IM_COL32 layout, 0 = usual): + // a dedicated server's lines, in the colour its owner chose. + std::uint32_t text_color{}; }; // A command typed into chat with a leading "/" (shown as the player types "/"). struct MultiplayerChatCommand { @@ -94,26 +133,69 @@ struct MultiplayerBan { std::string name; // as they were known when banned; may be empty std::int64_t added{}; // Unix time }; +inline constexpr float player_distance_least = 50.f, player_distance_unlimited = 1000.f; struct MultiplayerModel { VoiceModel voice; std::vector bans; unsigned tps = multiplayer_default_tps; ObjectPlacement object_placement = ObjectPlacement::everyone; + // Objects each player may have placed in this session (0: no limit), the limit this + // player is held to (0 for the host and a server's admins), and how many they have placed. + unsigned object_limit{}, object_limit_own{}, objects_placed{}; // Host setting: whether guests may use noclip / No Bail (the host and server admins always may). bool guest_noclip{true}, guest_no_bail{true}, guest_boosts{true}; // Host setting: guests skate with the host's physics tuning (on a dedicated server: the // game's own). Guest: what that does here, while enforced. bool enforce_tuning{true}; std::string tuning_status; - // Local display preferences: a lobby shown as the game's own party (Lobby party), - // and the floating name label above each peer. - bool party_overlay{true}, nametags{true}; - // Nametag style: ReSkate's own (name, distance, role colours, dots) or the game's. - bool custom_nametags{true}; + // Local display preference: the floating name label above each peer. + bool nametags{true}; // Local: whether session text chat shows at all (and T opens it). bool chat_visible{true}; // Local: bad words in chat names and messages show as **** (on by default). bool chat_filter{true}; + // Local, for ReSkate's nametags: `nametag_distance` is how far away a player's name still + // shows (past it they are a dot), `nametag_dots` whether those dots and the ones at the + // screen's edge for off-screen players show at all, `nametags_friends` names only the + // player's Steam friends. + // Local: how far away another player still gets a skater built for them, in metres. Past + // it they keep their nametag or dot. Each skater costs memory and frame time, in view or + // not. `player_distance_unlimited` and above: every player, however far. + float player_distance{120.f}; + // Local: connect straight to dedicated servers that offer it (their "connection": "direct") + // instead of through Steam's relays. Off: always the relays, and no server sees this PC's address. + bool prefer_direct{true}; + float nametag_distance{120.f}; + bool nametag_dots{true}; + bool nametags_friends{}; + // Local: chat bubbles above each skater's head. `chat_bubbles_distance` is how far away + // a player may be and still show one, `chat_bubbles_duration` how many seconds a line + // stays before it fades, `chat_bubbles_history` how many recent lines stack up. + // `chat_bubbles_own` also shows this player's own lines. + bool chat_bubbles{true}; + bool chat_bubbles_own{}; + float chat_bubbles_distance{40.f}; + float chat_bubbles_duration{5.f}; + int chat_bubbles_history{3}; + // Local: the tag the ReSkate backend gives this player ("Dev", "Staff", "Creator", "Centrix" or "Homie"; empty + // for most players) and its role colour, and whether they show it, and the animated items + // that come with it, to everyone. + std::string identity_tag; + std::uint32_t identity_tag_colour{}; + bool identity_tag_shown{true}, identity_items_shown{true}; + // Local: how each of this player's marked cosmetics is coloured, for the Special page: what + // they wear in each slot ("Top", "Shoes", ...), then the parts of their board. mode: 0 what + // their list gives (`identity_animation`: "RAINBOW", "GREEN", "RED", "BLUE" or "GOLD"), 1 off, 2 a gradient + // between the two colours they picked, 3 the first of them alone, 4 the rainbow (offered + // when `identity_rainbow`: to the staff). speed: 0 normal, 1 slow, 2 fast. + struct IdentityStyle { + std::string name; + int mode{}, speed{}; + std::array from{}, to{}; + }; + std::vector identity_styles; + bool identity_rainbow{}; + std::string identity_animation; float voice_range = default_voice_range; // how far the host (or server) forwards proximity voice // In a dedicated server's session: the server is the host but not a player. // Admins it lists may change its settings, and its map through Levels. @@ -123,6 +205,9 @@ struct MultiplayerModel { unsigned server_ban_total{}; // For its admins: the levels (assets) the dedicated server can switch to. std::vector server_maps; + std::vector server_map_pool; // in rotation order; empty: every map + unsigned server_map_rotation{}; // minutes per map (0: off) + bool server_map_votes{}; // players may vote for a map MultiplayerHostPreferences saved_host; bool force_world_layers{}; std::uint64_t host_id{}; @@ -134,8 +219,8 @@ struct MultiplayerModel { bool password_required{}; int players = 1, capacity = multiplayer_player_limit; std::vector roster; - // The local player's party: a lobby is always one party led by the host; on a dedicated - // server players form their own (`parties`: invite, leave, kick... are available). + // The local player's party. Players form their own, in a lobby as on a dedicated server + // (`parties`: invite, leave, kick... are available in any session). std::uint32_t party{}; bool party_leader{}, party_open{}, parties{}; std::vector party_invites; // newest last diff --git a/Engine/Game/Multiplayer/session_physics.h b/Engine/Game/Multiplayer/session_physics.h new file mode 100644 index 0000000..18eb4cd --- /dev/null +++ b/Engine/Game/Multiplayer/session_physics.h @@ -0,0 +1,71 @@ +#pragma once +// What a session does to the local player's physics, between the session (which knows the +// host and its rules) and the code that changes physics (the trainer). The tuning asset itself +// goes through Extension/Skater/physics_tuning.h; this is the rest. +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk { +// Physics a player changes outside Gameplay/SkatePhysicsTuning: the trainer's tuning-class +// values and trick multipliers (Extension/Trainer/trainer_session.h reads and writes the +// bytes; they are opaque here and on the wire). Empty: everything the game's own. +inline constexpr std::size_t max_physics_extras = 4096; + +namespace session_physics_detail { +inline std::atomic enforced{}; +struct Box { + std::mutex mutex; + std::vector bytes; + std::uint64_t revision = 1; // moves whenever `bytes` does; nobody's `seen` starts at it +}; +inline Box &local_box() { static Box value; return value; } +inline Box &host_box() { static Box value; return value; } +inline void put(Box &box, std::span bytes) { + std::lock_guard lock(box.mutex); + if (std::ranges::equal(box.bytes, bytes)) return; + box.bytes.assign(bytes.begin(), bytes.end()); + ++box.revision; +} +inline bool take(Box &box, std::uint64_t &seen, std::vector &bytes) { + std::lock_guard lock(box.mutex); + if (seen == box.revision) return false; + seen = box.revision; + bytes = box.bytes; + return true; +} +} // namespace session_physics_detail + +// The local player is a guest of a session whose host (or dedicated server) sets everyone's +// physics (the roster's enforce_tuning): their own physics edits stand down, and what the host +// shares is theirs. False outside a session and for a host. Set by the session every tick. +inline bool session_tuning_enforced() noexcept { + return session_physics_detail::enforced.load(std::memory_order_acquire); +} +inline void set_session_tuning_enforced(bool enforced) noexcept { + session_physics_detail::enforced.store(enforced, std::memory_order_release); +} + +// The local player's own extras, published by whoever edits them; a host's session sends them +// to its guests. The readers copy `bytes` out only when they are not the ones `seen` names +// (start `seen` at 0), and say whether they did. +inline void set_local_physics_extras(std::span bytes) { + session_physics_detail::put(session_physics_detail::local_box(), bytes); +} +inline bool local_physics_extras(std::uint64_t &seen, std::vector &bytes) { + return session_physics_detail::take(session_physics_detail::local_box(), seen, bytes); +} +// A guest's host's extras, set by the session: empty until the host sends some, and always for +// a dedicated server, whose physics are the game's own. Only meaningful while +// session_tuning_enforced(). +inline void set_host_physics_extras(std::span bytes) { + session_physics_detail::put(session_physics_detail::host_box(), bytes); +} +inline bool host_physics_extras(std::uint64_t &seen, std::vector &bytes) { + return session_physics_detail::take(session_physics_detail::host_box(), seen, bytes); +} +} // namespace dingosdk diff --git a/Engine/Game/Multiplayer/tick_settings.h b/Engine/Game/Multiplayer/tick_settings.h index 3f53440..43fa9b3 100644 --- a/Engine/Game/Multiplayer/tick_settings.h +++ b/Engine/Game/Multiplayer/tick_settings.h @@ -18,4 +18,10 @@ constexpr bool valid_pose_interval(std::uint32_t interval) noexcept { if (interval == multiplayer_pose_interval(rate)) return true; return false; } +// The most poses a second one player is sent on a dedicated server, whatever the crowd +// around them (Extension/Multiplayer/Session/room.h, crowd_limits); 0: no limit. +inline constexpr unsigned crowd_pose_budget = 600, min_crowd_budget = 300, max_crowd_budget = 20000; +constexpr bool valid_crowd_budget(unsigned budget) noexcept { + return !budget || (budget >= min_crowd_budget && budget <= max_crowd_budget); +} } // namespace dingosdk diff --git a/Engine/Game/Skater/Test/velocity_boost_tests.cpp b/Engine/Game/Skater/Test/velocity_boost_tests.cpp new file mode 100644 index 0000000..f9aa04f --- /dev/null +++ b/Engine/Game/Skater/Test/velocity_boost_tests.cpp @@ -0,0 +1,57 @@ +#include "Engine/Game/Skater/velocity_boost.h" +#include +#include +#include + +namespace { +void check(bool condition, const char* message) { + if (!condition) { std::cerr << message << '\n'; std::exit(1); } +} +bool near(float a, float b) { return std::abs(a - b) < .001f; } +} + +int main() { + using dingosdk::VelocityBoostDirection; + using dingosdk::velocity_boost_delta; + constexpr auto forward = VelocityBoostDirection::forward; + constexpr auto up = VelocityBoostDirection::up; + std::array transform{1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1}; + // Rotate through world headings; changing preferred stance does not change + // this contract: facing-aligned travel boosts forward, fakie travel reverses it. + for (int heading = 0; heading < 360; heading += 15) { + const float angle = static_cast(heading) * 3.14159265f / 180.f; + transform[8] = std::sin(angle); + transform[10] = std::cos(angle); + for (const float speed : {1.f, 75.f, 300.f}) { + for (const float travel : {12.f, -12.f}) { + const std::array motion{travel * transform[8], 0, travel * transform[10]}; + const auto delta = velocity_boost_delta(transform, speed, forward, motion); + check(delta.has_value(), "Valid riding boost rejected"); + const float sign = travel < 0 ? -1.f : 1.f; + check(near((*delta)[0], sign * speed * transform[8]) && near((*delta)[2], sign * speed * transform[10]), + "Boost opposes riding direction"); + check(near(std::hypot((*delta)[0], (*delta)[2]), speed), "Configured boost magnitude changed"); + } + } + } + transform[8] = 0; transform[10] = 1; + for (const auto motion : {std::array{0, 0, 0}, {0, 0, -.05f}, {12, 0, 0}, {0, -50, 0}, {0, 50, 0}}) { + const auto delta = velocity_boost_delta(transform, 75, forward, motion); + check(delta && near((*delta)[2], 75), "Stationary, sideways or vertical motion reversed boost"); + } + // Preserve the original three-dimensional facing axis and magnitude on ramps. + transform[9] = .6f; transform[10] = .8f; + const auto ramp = velocity_boost_delta(transform, 75, forward, {0, -100, -12}); + check(ramp && near((*ramp)[1], -45) && near((*ramp)[2], -60), "Fakie ramp direction or magnitude changed"); + const auto rising = velocity_boost_delta(transform, 75, forward, {0, -100, 12}); + check(rising && near((*rising)[1], 45) && near((*rising)[2], 60), "Vertical motion overrode forward travel"); + const auto upward = velocity_boost_delta(transform, 25, up, {0, -100, -12}); + check(upward && *upward == std::array{0, 25, 0}, "Up Boost changed in fakie"); + for (const float speed : {0.f, 301.f, std::numeric_limits::quiet_NaN()}) + check(!velocity_boost_delta(transform, speed, forward), "Invalid forward speed accepted"); + check(!velocity_boost_delta(transform, 26, up), "Invalid up speed accepted"); + check(!velocity_boost_delta({}, 75, forward), "Degenerate facing accepted"); + check(!velocity_boost_delta(transform, 75, forward, {0, 0, std::numeric_limits::infinity()}), + "Invalid travel velocity accepted"); + std::cout << "Velocity boost regressions passed.\n"; +} diff --git a/Engine/Game/Skater/velocity_boost.h b/Engine/Game/Skater/velocity_boost.h index 7bb2b9b..00edd6d 100644 --- a/Engine/Game/Skater/velocity_boost.h +++ b/Engine/Game/Skater/velocity_boost.h @@ -7,7 +7,8 @@ namespace dingosdk { enum class VelocityBoostDirection { forward, up }; inline std::optional> velocity_boost_delta( - const std::array& transform, float speed, VelocityBoostDirection direction) noexcept { + const std::array& transform, float speed, VelocityBoostDirection direction, + const std::array& board_velocity = {}) noexcept { const float maximum = direction == VelocityBoostDirection::up ? 25.0f : 300.0f; if (!std::isfinite(speed) || speed < 1.0f || speed > maximum) return {}; if (direction == VelocityBoostDirection::up) return std::array{0, speed, 0}; @@ -17,6 +18,16 @@ inline std::optional> velocity_boost_delta( if (!std::isfinite(length) || length <= .01f) return {}; const auto scale = speed / std::sqrt(length); for (auto& component : delta) component *= scale; + // Fakie travel opposes the skater's facing axis. Use horizontal motion so + // falling/rising cannot reverse a boost, and retain facing at rest or sideways. + const float horizontal_length = std::hypot(delta[0], delta[2]); + if (horizontal_length > .01f) { + const float travel = (delta[0] * board_velocity[0] + delta[2] * board_velocity[2]) / horizontal_length; + if (!std::isfinite(travel)) return {}; + // Ignore tiny backwards drift around standstill (0.1 world units/s). + if (travel < -.1f) + for (auto& component : delta) component = -component; + } return delta; } } diff --git a/Engine/Resource/cas_codec.cpp b/Engine/Resource/cas_codec.cpp index 8249df8..9149e1f 100644 --- a/Engine/Resource/cas_codec.cpp +++ b/Engine/Resource/cas_codec.cpp @@ -1,4 +1,5 @@ #include "cas_codec.h" +#include "Engine/Core/Platform/path_text.h" #ifdef _WIN32 #include @@ -68,7 +69,7 @@ HMODULE oodle(const std::filesystem::path& gameRoot) { std::ranges::sort(matches); handle = LoadLibraryExW(matches.back().c_str(), nullptr, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_SYSTEM32); - if (!handle) throw std::runtime_error("Cannot load " + matches.back().string() + + if (!handle) throw std::runtime_error("Cannot load " + path_utf8(matches.back()) + " (Windows error " + std::to_string(GetLastError()) + ")"); return handle; } diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index a7eee7c..3f43973 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -631,4 +631,11 @@ Document read_document(const std::span bytes) { return result; } +Guid read_file_guid(const std::span bytes) { + BinaryReader reader(bytes); + const auto chunks = read_chunks(reader); + reader.seek(unique_chunk(chunks, efix, "EFIX").begin); + return reader.guid(); +} + } diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index 8d50e39..4ae9809 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -194,5 +194,8 @@ struct Document { }; [[nodiscard]] Document read_document(std::span bytes); +// The document's own guid, which is what another document's import names it by, +// without reading the rest. +[[nodiscard]] Guid read_file_guid(std::span bytes); } diff --git a/Engine/Resource/ebx_merge.cpp b/Engine/Resource/ebx_merge.cpp index b4d3d85..224698a 100644 --- a/Engine/Resource/ebx_merge.cpp +++ b/Engine/Resource/ebx_merge.cpp @@ -5,12 +5,16 @@ #include #include +#include #include #include #include +#include #include #include +#include #include +#include namespace dingosdk::frostbite::ebx { namespace { @@ -81,6 +85,170 @@ const Value::Array* root_array(const Document& document, std::string_view name) return nullptr; } +// A type's name, which is what two documents that number their types +// independently have in common. Empty for a descriptor that names none. +std::string type_or_none(const Document& document, std::int32_t descriptor) { + return descriptor >= 0 && static_cast(descriptor) < document.types.size() + ? document.types[static_cast(descriptor)].name : std::string{}; +} + +constexpr auto unmatched = static_cast(-1); + +// Which of an edit's instances each of the base's is: the root is the root, an +// exported one is found by its guid, and an internal one, which has nothing +// else to go by, by its place among the internal ones. An edit that added or +// dropped an internal instance has moved the rest along, so then none of them +// is matched. Two that are not of one type are not a match either. +std::vector match_instances(const Document& base, const Document& edit) { + std::vector match(base.instances.size(), unmatched); + if (base.instances.empty() || edit.instances.empty()) return match; + match[0] = 0; + std::map exported; + std::vector internal, baseInternal; + for (std::size_t index = 1; index < edit.instances.size(); ++index) { + if (edit.instances[index].exported) exported.emplace(edit.instances[index].instanceGuid, index); + else internal.push_back(index); + } + for (std::size_t index = 1; index < base.instances.size(); ++index) { + if (!base.instances[index].exported) { baseInternal.push_back(index); continue; } + if (const auto found = exported.find(base.instances[index].instanceGuid); found != exported.end()) + match[index] = found->second; + } + if (internal.size() == baseInternal.size()) + for (std::size_t at = 0; at < internal.size(); ++at) match[baseInternal[at]] = internal[at]; + for (std::size_t index = 0; index < match.size(); ++index) + if (match[index] != unmatched && type_or_none(base, base.instances[index].descriptor) != + type_or_none(edit, edit.instances[match[index]].descriptor)) + match[index] = unmatched; + return match; +} + +bool same(double left, double right) { return std::bit_cast(left) == std::bit_cast(right); } +bool same(const ResourceReference& left, const ResourceReference& right) { return left.id == right.id; } +template bool same(const Type& left, const Type& right) { return left == right; } + +// Finds what one edit changed in place. The base and the edit are walked side +// by side, along with the result: it began as a copy of the base, so it has the +// same shape, and may already hold what an earlier edit changed. Nothing is +// written until the whole edit has been seen, so an edit either has all its +// values taken or none of them. +class InPlace final { +public: + InPlace(const Document& base, const Document& edit, const std::vector& match) + : base_(base), edit_(edit), match_(match) {} + + void fields(const Object& base, const Object& edit, Object& result, bool root) { + if (base.fields.size() != edit.fields.size() || base.fields.size() != result.fields.size()) { uncarried = true; return; } + for (std::size_t index = 0; index < base.fields.size(); ++index) { + if (base.fields[index].name != edit.fields[index].name) { uncarried = true; return; } + value(base.fields[index].value, edit.fields[index].value, result.fields[index].value, root); + } + } + + void apply() { + for (const auto& [to, from] : changes_) to->data = from->data; + } + + std::size_t values{}; // values the edit changed from the base + std::size_t contested{}; // of those, ones the result already had changed to something else + bool uncarried{}; // a change was seen that cannot be taken + bool unread{}; // somewhere a change could not be looked for + +private: + template bool plain(const Value& base, const Value& edit, Value& result) { + const auto* before = std::get_if(&base.data); + if (!before) return false; + const auto& after = std::get(edit.data); + if (same(*before, after)) return true; + ++values; + const auto& current = std::get(result.data); + if (same(current, after)) return true; + if (!same(current, *before)) ++contested; + changes_.emplace_back(&result, &edit); + return true; + } + + void value(const Value& base, const Value& edit, Value& result, bool rootList) { + if (base.data.index() != edit.data.index() || base.data.index() != result.data.index()) { uncarried = true; return; } + if (std::holds_alternative(base.data)) return; + if (plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result)) return; + if (const auto* before = std::get_if(&base.data)) { + reference(*before, std::get(edit.data)); + return; + } + if (const auto* before = std::get_if>(&base.data)) { + const auto& after = std::get>(edit.data); + const auto& mine = std::get>(result.data); + if (!*before && !after) return; + if (!*before || !after || !mine) { uncarried = true; return; } + fields(**before, *after, *mine, false); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + auto& mine = std::get(result.data); + // Only a list on the root grows at its end, and what was appended + // comes across on its own. A list that shrank, or one deeper in + // that changed length, has entries no longer told apart by place. + if (rootList ? after.size() < before->size() : after.size() != before->size()) { uncarried = true; return; } + if (mine.size() < before->size()) { uncarried = true; return; } + for (std::size_t index = 0; index < before->size(); ++index) + value((*before)[index], after[index], mine[index], false); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + if (before->primitive != after.primitive) uncarried = true; + else if (before->primitive) uncarried |= before->primitiveType != after.primitiveType; + else uncarried |= type_or_none(base_, before->descriptor) != type_or_none(edit_, after.descriptor); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + // What a boxed value holds is not read, so a change inside one is not seen. + if ((before->encodedType == 0) != (after.encodedType == 0)) uncarried = true; + else if (before->encodedType != 0) unread = true; + } + } + + // A reference still has to lead to the same place; one pointed elsewhere + // cannot be taken, because what it points at may not be in the result. + void reference(const PointerReference& before, const PointerReference& after) { + if (before.kind != after.kind) { uncarried = true; return; } + if (before.kind == PointerKind::null) return; + if (before.index < 0 || after.index < 0) { uncarried |= before.index != after.index; return; } + const auto from = static_cast(before.index), to = static_cast(after.index); + if (before.kind == PointerKind::external) { + uncarried |= from >= base_.imports.size() || to >= edit_.imports.size() || + base_.imports[from].fileGuid != edit_.imports[to].fileGuid || + base_.imports[from].classGuid != edit_.imports[to].classGuid; + return; + } + if (from >= match_.size()) { uncarried = true; return; } + // An internal instance that could not be matched is not known to have moved. + if (match_[from] == unmatched) (base_.instances[from].exported ? uncarried : unread) = true; + else uncarried |= match_[from] != to; + } + + const Document& base_; + const Document& edit_; + const std::vector& match_; + std::vector> changes_; +}; + +template void each_pointer(Value& value, const Visit& visit) { + if (auto* pointer = std::get_if(&value.data)) { visit(*pointer); return; } + if (auto* nested = std::get_if>(&value.data)) { + if (*nested) for (auto& field : (*nested)->fields) each_pointer(field.value, visit); + return; + } + if (auto* list = std::get_if(&value.data)) + for (auto& element : *list) each_pointer(element, visit); +} + } // namespace namespace detail { @@ -289,7 +457,7 @@ void sort_instances(Document& document) { } // namespace detail Document merge_documents(const Document& base, const std::span edits, - MergeSummary* summary) { + MergeSummary* summary, const CarryImport& carry) { auto result = detail::clone_document(base); const auto* baseRoot = base.root(); if (!baseRoot || !baseRoot->object) throw std::runtime_error("EBX base has no root object"); @@ -302,11 +470,44 @@ Document merge_documents(const Document& base, const std::spansize()); MergeSummary totals; - for (const auto* edit : edits) { + // The references the edits so far appended to each root array. + std::map>, std::less<>> appended; + for (std::size_t index = 0; index < edits.size(); ++index) { + const auto* edit = edits[index]; if (!edit) continue; const auto* editRoot = edit->root(); if (!editRoot || !editRoot->object) throw std::runtime_error("EBX edit has no root object"); + + // What the edit changed in place. The result's first instances are still + // the base's, in the base's order: carried ones are added behind them, + // and nothing is sorted until every edit is in. + const auto match = match_instances(base, *edit); + InPlace changes(base, *edit, match); + for (std::size_t at = 0; at < base.instances.size(); ++at) { + const auto& before = base.instances[at]; + if (match[at] == unmatched) { + // An exported instance is found whenever the edit still has it. + (before.exported ? changes.uncarried : changes.unread) = true; + continue; + } + const auto& after = edit->instances[match[at]]; + if (!before.object || !after.object || !result.instances[at].object) { + changes.unread |= before.object || after.object; + continue; + } + changes.fields(*before.object, *after.object, *result.instances[at].object, at == 0); + } + if (changes.uncarried) { + totals.uncarried.push_back(index); + } else { + changes.apply(); + totals.values += changes.values; + totals.contested += changes.contested; + } + if (changes.unread) totals.unread.push_back(index); + detail::Carrier mapper(*edit, result); + std::map>, std::less<>> mine; // Whatever an edit appended to one of the root's arrays comes across, // pulling in the instances those entries point at. @@ -320,10 +521,34 @@ Document merge_documents(const Document& base, const std::spansize(); ++at) { - target->push_back(mapper.value((*editArray)[at])); + const auto& entry = (*editArray)[at]; + const auto* pointer = carry ? std::get_if(&entry.data) : nullptr; + if (pointer && pointer->kind == PointerKind::external && pointer->index >= 0 && + static_cast(pointer->index) < edit->imports.size()) { + const auto& reference = edit->imports[static_cast(pointer->index)]; + if (!carry(reference)) { + totals.dropped.push_back({index, reference}); + continue; + } + } + auto carried = mapper.value(entry); + // Two edits that append the same thing mean it once. What one edit + // lists twice on its own is left as its author wrote it. + if (const auto* reference = std::get_if(&carried.data); + reference && reference->kind != PointerKind::null) { + const std::pair key{reference->kind, reference->index}; + if (const auto earlier = appended.find(field.name); + earlier != appended.end() && earlier->second.contains(key)) { + ++totals.repeated; + continue; + } + mine[field.name].insert(key); + } + target->push_back(std::move(carried)); ++totals.arrayEntries; } } + for (auto& [name, references] : mine) appended[name].merge(references); // An instance an edit added but never linked from the root still has to // exist, or the asset it belongs to goes missing. @@ -346,4 +571,51 @@ Document merge_documents(const Document& base, const std::span(document.root()); + if (!keep || !root || !root->object) return 0; + std::size_t dropped{}; + for (auto& field : root->object->fields) { + auto* array = std::get_if(&field.value.data); + if (!array) continue; + dropped += std::erase_if(*array, [&](const Value& entry) { + const auto* pointer = std::get_if(&entry.data); + return pointer && pointer->kind == PointerKind::external && pointer->index >= 0 && + static_cast(pointer->index) < document.imports.size() && + !keep(document.imports[static_cast(pointer->index)]); + }); + } + // A boxed value is kept as the bytes it was read as, import numbers and + // all, so a document with one keeps its imports where they are. + if (!dropped || !document.boxedValues.empty()) return dropped; + + std::vector used(document.imports.size()); + for (auto& instance : document.instances) + if (instance.object) + for (auto& field : instance.object->fields) + each_pointer(field.value, [&](const PointerReference& pointer) { + if (pointer.kind == PointerKind::external && pointer.index >= 0 && + static_cast(pointer.index) < used.size()) + used[static_cast(pointer.index)] = true; + }); + std::vector moveTo(used.size(), -1); + std::vector kept; + for (std::size_t index = 0; index < used.size(); ++index) { + if (!used[index]) continue; + moveTo[index] = static_cast(kept.size()); + kept.push_back(document.imports[index]); + } + if (kept.size() == document.imports.size()) return dropped; + for (auto& instance : document.instances) + if (instance.object) + for (auto& field : instance.object->fields) + each_pointer(field.value, [&](PointerReference& pointer) { + if (pointer.kind == PointerKind::external && pointer.index >= 0 && + static_cast(pointer.index) < moveTo.size()) + pointer.index = moveTo[static_cast(pointer.index)]; + }); + document.imports = std::move(kept); + return dropped; +} + } // namespace dingosdk::frostbite::ebx diff --git a/Engine/Resource/ebx_merge.h b/Engine/Resource/ebx_merge.h index fe3d055..294ed25 100644 --- a/Engine/Resource/ebx_merge.h +++ b/Engine/Resource/ebx_merge.h @@ -3,6 +3,7 @@ #include "ebx_document.h" #include +#include #include #include @@ -12,15 +13,46 @@ struct MergeSummary { std::size_t instances{}; // instances taken from the edits std::size_t arrayEntries{};// root array entries taken from the edits std::size_t renumbered{}; // carried instances whose object id was re-derived + // Root array entries left behind because `carry` turned their target down. + struct Dropped { + std::size_t edit{}; // index into `edits` + ImportReference target; + }; + std::vector dropped; + std::size_t repeated{}; // appended references an earlier entry had already added, left out + std::size_t values{}; // plain values an edit changed in place, taken from it + std::size_t contested{}; // of those, values an earlier edit had changed to something else + // Edits (indices into `edits`) with a change that did not come across: a reference + // pointed somewhere else, a list resized anywhere but at the root's end, an instance gone. + std::vector uncarried; + // Edits whose in-place changes could not all be looked for, because their internal + // instances do not line up with the base's or the document holds boxed values. + std::vector unread; + + // Every edit's in-place changes are in the result, as far as anyone can tell. + [[nodiscard]] bool exact() const noexcept { return uncarried.empty() && unread.empty(); } }; +// Whether a root array entry an edit appended comes across, asked for an entry +// that is a reference into another document. Such a reference only resolves +// while the document it names is loaded; one that is not loads as a null object. +using CarryImport = std::function; + // Combines several independently edited copies of one EBX asset with the base -// they were all derived from. Each edit contributes the instances it added and -// the entries it appended to the root's arrays; anything an edit merely changed -// in place stays as the base had it, because two edits to one value cannot both -// be honoured. Edits are applied in the order given, so the last one wins a tie. +// they were all derived from. Each edit contributes the instances it added, the +// entries it appended to the root's arrays (a reference two edits both append is +// listed once), and the plain values it changed in place: a number, a string, a +// guid. Two edits to one value cannot both be honoured, so edits are applied in +// the order given and the last one wins. A change of any other shape (a +// reference pointed elsewhere, a list resized in the middle of the document) +// stays as the base had it, and the summary names the edit it was in. [[nodiscard]] Document merge_documents(const Document& base, std::span edits, - MergeSummary* summary = nullptr); + MergeSummary* summary = nullptr, + const CarryImport& carry = {}); + +// Takes out the root array entries that refer to a document `keep` turns down, +// and the imports nothing refers to afterwards. Returns how many entries went. +std::size_t drop_root_references(Document& document, const CarryImport& keep); } // namespace dingosdk::frostbite::ebx diff --git a/Engine/Vfs/Test/chunk_metadata_merge_tests.cpp b/Engine/Vfs/Test/chunk_metadata_merge_tests.cpp new file mode 100644 index 0000000..67aab2d --- /dev/null +++ b/Engine/Vfs/Test/chunk_metadata_merge_tests.cpp @@ -0,0 +1,203 @@ +// A bundle's chunk metadata has one record for each of its chunks: whose the chunk is (the hash of +// the resource's name) and, for a texture, the first mip the bundle holds. In the game's bundles +// the records are in the order of the chunks' guids, which is not the order the chunks are listed +// in. A mod's tool starts from the game's list and writes a record where each chunk it adds or +// replaces is LISTED: right for a chunk it adds at the end, but a replaced chunk's record lands +// on another chunk's, which is then gone from the mod's list. +// +// When two mods each ship a copy of one game bundle with chunks of their own (two costumes built +// on the same game costume), the merged bundle needs a list that describes every chunk and is in +// the game's form. With the game's list alone, or with records taken from where the mods' lists +// have them, the costumes' meshes and textures never streamed in. +#include "Engine/Vfs/mod_merge_internal.h" + +#include +#include +#include +#include + +namespace db = dingosdk::native_db; +namespace fb = dingosdk::frostbite; +using namespace dingosdk::mods::detail; + +namespace { +using Bytes = std::vector; + +int failures = 0; +void check(bool condition, const char* what) { + if (!condition) { std::cerr << "FAILED: " << what << "\n"; ++failures; } +} + +// A chunk's guid: the first byte decides where it sorts. +fb::Guid guid(unsigned char first) { + fb::Guid value; + value.bytes.fill(std::byte{0x11}); + value.bytes[0] = static_cast(first); + return value; +} +// One chunk's record: the hash of what it belongs to, and for a texture its first mip. +struct Record { + std::uint64_t owner{}; + std::optional firstMip; + bool operator==(const Record&) const = default; +}; +db::Node number(unsigned type, const char* name, std::uint64_t value, std::size_t size) { + db::Node node; + node.type = type; + node.named = true; + node.name = name; + for (std::size_t byte = 0; byte < size; ++byte) node.owned.push_back(static_cast(value >> (8 * byte))); + return node; +} +Bytes list_of(const std::vector& records) { + db::Node list; + list.type = 1; + list.named = true; + list.name = "chunkMeta"; + list.terminated = true; + for (const auto& record : records) { + db::Node meta; + meta.type = 2; + meta.named = true; + meta.name = "meta"; + meta.terminated = true; + if (record.firstMip) meta.children.push_back(number(8, "firstMip", *record.firstMip, 4)); + db::Node row; + row.type = 2; + row.terminated = true; + row.children.push_back(number(9, "h64", record.owner, 8)); + row.children.push_back(std::move(meta)); + list.children.push_back(std::move(row)); + } + const auto written = db::write(list); + const auto* bytes = reinterpret_cast(written.data()); + return {bytes, bytes + written.size()}; +} +// What a list says, record by record; a record that says nothing reads as owner 0. +std::vector read_list(const Bytes& list) { + const std::span bytes(reinterpret_cast(list.data()), list.size()); + std::size_t used{}; + const auto root = db::read(bytes, "chunk metadata", &used); + check(used == bytes.size() && root.type == 1 && root.name == "chunkMeta", "the merged list is one list, named as the game names it"); + std::vector records; + for (const auto& row : root.children) { + Record record; + if (const auto* owner = row.field("h64"); owner && owner->payload().size() == 8) + std::memcpy(&record.owner, owner->payload().data(), 8); + if (const auto* meta = row.field("meta")) + if (const auto* mip = meta->field("firstMip"); mip && mip->payload().size() == 4) { + std::uint32_t value{}; + std::memcpy(&value, mip->payload().data(), 4); + record.firstMip = value; + } + records.push_back(record); + } + return records; +} +ChunkRecord chunk(unsigned char first, std::size_t copy, std::size_t index, std::size_t shipped = ChunkRecord::none) { + return {guid(first), copy, index, shipped}; +} +} // namespace + +int main() { + // The game's bundle lists five chunks: something of its own, a texture, a mesh in two chunks + // around another texture. Its list has their records by guid: 10, 30, 50, 60, 70. + const Record mesh{0x1111}, textureA{0xAAAA, 2}, textureB{0xBBBB, 3}, other{0x9999}; + const std::vector shipped{guid(0x50), guid(0x30), guid(0x70), guid(0x10), guid(0x60)}; + const std::vector game{mesh, textureA, other, textureB, mesh}; + constexpr std::size_t gameList = 0; + + // One costume adds a texture (guid 20) and a mesh chunk (guid 80), listed after the game's five, + // with their records at those two places. + const Record firstTexture{0xF001, 0}, firstMesh{0xF002}; + auto first = game; + first.insert(first.end(), {firstTexture, firstMesh}); + // Another, made with an older tool, replaces the game's second texture (guid 60, listed fifth) + // with one whose first mip is 1: its record went over the fifth record, the mesh's, with no hash. + // It also adds a texture (guid 40). + const Record secondTexture{0x5001, 0}; + auto second = game; + second[4] = Record{0, 1}; + second.push_back(secondTexture); + const std::vector lists{list_of(game), list_of(first), list_of(second)}; + + // The merged bundle: the game's five as the higher-priority mod has them, then each mod's own. + const std::vector merged{chunk(0x50, 2, 0, 0), chunk(0x30, 2, 1, 1), chunk(0x70, 2, 2, 2), chunk(0x10, 2, 3, 3), + chunk(0x60, 2, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6), chunk(0x40, 2, 5)}; + const auto replaced = Record{textureB.owner, 1}; + const std::vector expected{mesh, firstTexture, textureA, secondTexture, other, replaced, mesh, firstMesh}; + const auto list = read_list(merge_chunk_metadata(lists, merged, gameList, shipped)); + check(list == expected, "a record for every chunk, in the order of the chunks' guids"); + check(list.size() == merged.size(), "as many records as chunks"); + check(std::ranges::count(list, mesh) == 2, "the record the older tool wrote over is back: the mesh has both of its"); + check(std::ranges::find(list, replaced) != list.end() && std::ranges::find(list, textureB) == list.end(), + "the replaced texture keeps its own name and takes the first mip the mod gave it"); + check(std::ranges::none_of(list, [](const Record& record) { return record.owner == 0; }), "no record is left without a hash"); + // The fault this is for: the game's own list is three records short, and the mods' lists have + // their records where the chunks are listed, which is not where the game looks. + check(read_list(lists[gameList]).size() + 3 == merged.size(), "the game's own list is three records short of the merged bundle"); + + // No mod's chunk in it: the game's list is not written again. + const std::vector untouched{chunk(0x50, 0, 0, 0), chunk(0x30, 0, 1, 1), chunk(0x70, 0, 2, 2), chunk(0x10, 0, 3, 3), chunk(0x60, 0, 4, 4)}; + check(merge_chunk_metadata(lists, untouched, gameList, shipped) == lists[gameList], "a bundle that is all the game's keeps the game's list as it is"); + // One mod's copy alone is still written in the game's form: its additions go where their guids put them. + const std::vector oneMod{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3), + chunk(0x60, 1, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6)}; + check(read_list(merge_chunk_metadata(lists, oneMod, gameList, shipped)) == + std::vector{mesh, firstTexture, textureA, other, textureB, mesh, firstMesh}, + "one mod's additions are put in guid order with the game's"); + // A newer tool writes the replaced texture's own hash with its first mip; the first mip is what counts. + auto newer = game; + newer[4] = Record{textureB.owner, 0}; + const std::vector withNewer{list_of(game), list_of(newer)}; + const std::vector keepsNewer{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3), chunk(0x60, 1, 4, 4)}; + check(read_list(merge_chunk_metadata(withNewer, keepsNewer, gameList, shipped)) == + std::vector{mesh, textureA, other, Record{textureB.owner, 0}, mesh}, + "a replaced texture's new first mip goes into its own record, and the record written over is back"); + // A mod that only carries the game's version of a chunk another mod replaced changes nothing of it. + const std::vector carried{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3), + chunk(0x60, 1, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6)}; + check(read_list(merge_chunk_metadata(lists, carried, gameList, shipped))[4] == textureB, "a chunk a mod only carries keeps the game's record"); + + // A chunk whose copy has no list, or a shorter one, still takes up a place. + const std::vector unknown{chunk(0x50, 0, 0, 0), chunk(0x30, 0, 1, 1), chunk(0x70, 0, 2, 2), chunk(0x10, 0, 3, 3), + chunk(0x60, 0, 4, 4), chunk(0x20, ChunkRecord::none, 5), chunk(0x80, 1, 9)}; + check(read_list(merge_chunk_metadata(lists, unknown, gameList, shipped)) == + std::vector{mesh, Record{}, textureA, other, textureB, mesh, Record{}}, + "a chunk no list describes gets a record that says nothing, and the others stay in step"); + + // A bundle the game has no list for (it has no chunks there, or does not ship the bundle): every + // chunk is a mod's, with the record of the copy it came in. Maps each add their loading screen + // to one such bundle. + const std::vector modsOnly{list_of({firstTexture, firstMesh}), list_of({firstTexture, firstMesh, secondTexture})}; + const std::vector theirs{chunk(0x90, 1, 0), chunk(0x20, 1, 1), chunk(0x40, 1, 2)}; + check(merge_chunk_metadata(modsOnly, theirs) == modsOnly[1], "a bundle that is one mod's copy alone keeps that copy's list"); + const std::vector both{chunk(0x90, 0, 0), chunk(0x20, 0, 1), chunk(0x40, 1, 2)}; + check(read_list(merge_chunk_metadata(modsOnly, both)) == std::vector{firstMesh, secondTexture, firstTexture}, + "and otherwise each chunk's record from the copy it came in, by guid like any other"); + // A game list that is not one record a chunk is of some other kind: nothing is put in order by it. + const std::vector shortGame{list_of({mesh, textureA}), list_of(first)}; + check(read_list(merge_chunk_metadata(shortGame, oneMod, gameList, shipped)) == first, + "a game list that is not a record a chunk leaves the records where the copies have them"); + + // No chunks, no list to write: what the first copy has stays. + check(merge_chunk_metadata({}, {}).empty(), "no list from no lists"); + check(merge_chunk_metadata(lists, {}, gameList, shipped) == lists[gameList], "a bundle without chunks keeps the first copy's list"); + const std::vector none{Bytes{}, Bytes{}}; + const std::vector bare{chunk(0x10, ChunkRecord::none, 0), chunk(0x20, ChunkRecord::none, 1)}; + check(merge_chunk_metadata(none, bare).empty(), "nor from copies that have none"); + + // A list that is needed and cannot be read is the caller's to deal with. + const std::vector broken{lists[0], Bytes(7, std::byte{0x41})}; + bool threw{}; + try { (void)merge_chunk_metadata(broken, oneMod, gameList, shipped); } + catch (const std::exception&) { threw = true; } + check(threw, "an unreadable list throws"); + + if (failures) { + std::cerr << "chunk metadata merge: " << failures << " check(s) failed\n"; + return 1; + } + std::cout << "chunk metadata merge: ok\n"; + return 0; +} diff --git a/Engine/Vfs/Test/content_catalogs_tests.cpp b/Engine/Vfs/Test/content_catalogs_tests.cpp index 2b1f62f..798c066 100644 --- a/Engine/Vfs/Test/content_catalogs_tests.cpp +++ b/Engine/Vfs/Test/content_catalogs_tests.cpp @@ -62,6 +62,20 @@ int main(int argc, char** argv) { check(stadium != travel.access_points.end() && stadium->second == std::vector{"location_stadium"}, "stadium entrance offers only the first stadium"); check(travel.access_points.size() == 5, "five access points"); + check(catalogs.music_playlists.size() == 54, "54 music playlists"); + if (const auto station = catalogs.music_playlists.find("0_Licensed_All"); station != catalogs.music_playlists.end()) + check(station->second.name == "San Van Soundtrack" && station->second.tracks.size() == 107 && + station->second.tracks.front() == "2hollis - flash" && + station->second.artwork == "cdn:/0e63e2111f9088448ae261ab0f930a4a", "San Van Soundtrack station"); + else check(false, "San Van Soundtrack station"); + if (const auto ballroom = catalogs.music_playlists.find("1920sBallroom"); ballroom != catalogs.music_playlists.end()) + check(ballroom->second.name == "Ballroom 237" && ballroom->second.tracks.size() == 3, "Ballroom station"); + else check(false, "Ballroom station"); + if (const auto art = catalogs.music_song_artwork.find("Freddie Gibbs, The Alchemist, Anderson .Paak - Ensalada"); + art != catalogs.music_song_artwork.end()) + check(art->second == "cdn:/ad89945108fd5d7d6fc94ccb33c0f500", "a song's cover art"); + else check(false, "a song's cover art"); + check(catalogs.music_song_artwork.size() >= 100, "the licensed songs have cover art"); if (argc > 2) { Json dump{{"items", catalogs.items}, {"challenges", catalogs.challenges}, {"entitlements", Json::array()}}; for (const auto& id : catalogs.entitlements) dump["entitlements"].push_back(id); diff --git a/Engine/Vfs/Test/ebx_merge_tests.cpp b/Engine/Vfs/Test/ebx_merge_tests.cpp new file mode 100644 index 0000000..a821183 --- /dev/null +++ b/Engine/Vfs/Test/ebx_merge_tests.cpp @@ -0,0 +1,280 @@ +// The document merge, on documents made in memory, so it needs no game files. +// +// Several mods each ship their own copy of one of the game's assets. merge_documents puts the +// copies together over the game's: what each added, and what each changed in place. The cases: +// * values two edits changed in place both arrive, and one both changed is the later edit's; +// * an edit that also changed something that cannot be taken (a reference pointed elsewhere, a +// list resized) has none of its values taken, and the summary names it; +// * an exported instance is found by its guid wherever the edit keeps it, and internal +// instances that do not line up are reported as not looked at; +// * a reference two edits both append is listed once, and one edit's own repeat is left alone; +// * drop_root_references takes entries out of the root's lists with the imports they used. +#include "Engine/Resource/ebx_carry.h" +#include "Engine/Resource/ebx_merge.h" + +#include +#include +#include +#include +#include + +namespace ebx = dingosdk::frostbite::ebx; +using dingosdk::frostbite::Guid; + +namespace { +int failures = 0; +void expect(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +Guid guid(std::uint8_t seed) { + Guid value; + value.bytes.fill(static_cast(seed)); + return value; +} + +template ebx::Value value_of(Type data) { + ebx::Value value; + value.data = std::move(data); + return value; +} +ebx::Value number(std::int64_t value) { return value_of(value); } +ebx::Value real(double value) { return value_of(value); } +ebx::Value text(std::string value) { return value_of(std::move(value)); } +// A reference to another document, by the place of its import, and one to an instance of this one. +ebx::Value outside(std::int32_t import) { return value_of(ebx::PointerReference{ebx::PointerKind::external, import}); } +ebx::Value inside(std::int32_t instance) { return value_of(ebx::PointerReference{ebx::PointerKind::internal, instance}); } +ebx::Value list(ebx::Value::Array entries) { return value_of(std::move(entries)); } + +std::shared_ptr object(std::int32_t type, std::vector> fields) { + auto result = std::make_shared(); + result->descriptor = type; + for (auto& [name, value] : fields) result->fields.push_back({result->fields.size(), std::move(name), std::move(value)}); + return result; +} +ebx::InstanceRecord instance(std::int32_t type, std::shared_ptr fields, const Guid* exported = nullptr) { + ebx::InstanceRecord record; + record.descriptor = type; + record.exported = exported != nullptr; + if (exported) record.instanceGuid = *exported; + record.object = std::move(fields); + return record; +} + +constexpr std::int32_t list_type = 0, entry_type = 1, detail_type = 2; +const Guid entry_guid = guid(0xA1); + +// The game's asset: a root with plain values, a list of two other documents and a nested +// structure; an exported instance the root points at; and an internal one that one points at. +ebx::Document game() { + ebx::Document document; + document.fileGuid = guid(1); + document.rootType = "ListAsset"; + document.types.resize(3); + document.types[list_type].name = "ListAsset"; + document.types[entry_type].name = "Entry"; + document.types[detail_type].name = "Detail"; + document.imports = {{guid(0x10), guid(0xC0)}, {guid(0x11), guid(0xC0)}}; + document.instances.push_back(instance(list_type, object(list_type, { + {"Volume", number(5)}, {"Gain", real(1.0)}, {"Title", text("game")}, + {"Items", list({outside(0), outside(1)})}, {"First", inside(1)}, + {"Tuning", value_of(object(detail_type, {{"Weights", list({number(1), number(2)})}, {"Scale", real(2.0)}}))}}))); + document.instances.push_back(instance(entry_type, object(entry_type, {{"Cost", number(10)}, {"Detail", inside(2)}}), &entry_guid)); + document.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(3)}}))); + return document; +} +ebx::Document copy(const ebx::Document& document) { return ebx::detail::clone_document(document); } + +ebx::Value& field(ebx::Object& from, std::string_view name) { + for (auto& entry : from.fields) + if (entry.name == name) return entry.value; + throw std::runtime_error("no field " + std::string(name)); +} +ebx::Value& field(ebx::Document& document, std::size_t at, std::string_view name) { return field(*document.instances.at(at).object, name); } +template const Type& read(const ebx::Document& document, std::size_t at, std::string_view name) { + return std::get(field(*document.instances.at(at).object, name).data); +} +ebx::Object& tuning(ebx::Document& document) { return *std::get>(field(document, 0, "Tuning").data); } +ebx::Value::Array& items(ebx::Document& document) { return std::get(field(document, 0, "Items").data); } +// The documents a list names, in order. +std::vector named(const ebx::Document& document) { + std::vector result; + for (const auto& entry : std::get(field(*document.instances.front().object, "Items").data)) + result.push_back(document.imports.at(static_cast(std::get(entry.data).index)).fileGuid); + return result; +} +void append(ebx::Document& document, const Guid& target) { + document.imports.push_back({target, guid(0xC0)}); + items(document).push_back(outside(static_cast(document.imports.size() - 1))); +} +// Where the exported instance with this guid is. +std::size_t find(const ebx::Document& document, const Guid& wanted) { + for (std::size_t index = 0; index < document.instances.size(); ++index) + if (document.instances[index].exported && document.instances[index].instanceGuid == wanted) return index; + throw std::runtime_error("no such exported instance"); +} + +ebx::Document merge(const ebx::Document& base, std::initializer_list edits, ebx::MergeSummary& summary) { + const std::vector documents(edits); + return ebx::merge_documents(base, documents, &summary); +} + +void values_from_every_edit() { + const auto base = game(); + auto first = copy(base), second = copy(base); + field(first, 0, "Volume") = number(9); + field(first, 0, "Title") = text("first"); + field(second, 0, "Gain") = real(0.5); + field(tuning(second), "Scale") = real(4.0); + std::get(field(tuning(second), "Weights").data)[1] = number(7); + field(second, 1, "Cost") = number(20); + field(second, 2, "Depth") = number(8); + ebx::MergeSummary summary; + auto merged = merge(base, {&first, &second}, summary); + expect(read(merged, 0, "Volume") == 9 && read(merged, 0, "Title") == "first", + "values: the first edit's values are in the result"); + expect(read(merged, 0, "Gain") == 0.5 && std::get(field(tuning(merged), "Scale").data) == 4.0 && + std::get(std::get(field(tuning(merged), "Weights").data)[1].data) == 7, + "values: the second edit's values are in the result, the nested ones too"); + expect(read(merged, 1, "Cost") == 20 && read(merged, 2, "Depth") == 8, + "values: values in an exported and in an internal instance are in the result"); + expect(summary.values == 7 && summary.contested == 0 && summary.exact() && !summary.instances && !summary.arrayEntries, + "values: seven values taken, none disagreed, nothing added (" + std::to_string(summary.values) + " taken)"); + expect(named(merged) == named(base) && merged.instances.size() == base.instances.size(), "values: nothing else moved"); +} + +void one_value_from_two_edits(bool same) { + const std::string label = same ? "same value twice" : "contested value"; + const auto base = game(); + auto lower = copy(base), higher = copy(base); + field(lower, 0, "Volume") = number(9); + field(higher, 0, "Volume") = number(same ? 9 : 11); + ebx::MergeSummary summary; + const auto merged = merge(base, {&lower, &higher}, summary); + expect(read(merged, 0, "Volume") == (same ? 9 : 11), label + ": the later edit's value is the one kept"); + expect(summary.values == 2 && summary.contested == (same ? 0u : 1u) && summary.exact(), + label + ": " + (same ? "no" : "one") + " disagreement is counted"); +} + +// What cannot be taken: the edit that made such a change keeps all its values out, so the result +// never has half of what a mod did to the game's own content. +void a_change_that_cannot_be_taken(int kind) { + static constexpr std::array labels{"retargeted reference", "shorter root list", "resized nested list", "instance gone"}; + const std::string label = labels[static_cast(kind)]; + const auto base = game(); + auto reshaped = copy(base), plain = copy(base); + field(reshaped, 0, "Volume") = number(9); + if (kind == 0) items(reshaped)[0] = outside(1); + else if (kind == 1) items(reshaped).pop_back(); + else if (kind == 2) std::get(field(tuning(reshaped), "Weights").data).push_back(number(3)); + else { + // The exported instance is not in the edit: the root points at the internal one instead. + reshaped.instances.erase(reshaped.instances.begin() + 1); + field(reshaped, 0, "First") = inside(1); + } + field(plain, 0, "Title") = text("plain"); + ebx::MergeSummary summary; + const auto merged = merge(base, {&reshaped, &plain}, summary); + expect(read(merged, 0, "Volume") == 5, label + ": none of that edit's values are taken"); + expect(read(merged, 0, "Title") == "plain", label + ": the other edit's value is"); + expect(summary.uncarried == std::vector{0} && !summary.exact() && summary.values == 1, + label + ": the summary names the edit"); + expect(named(merged) == named(base), label + ": the game's list is as it was"); +} + +// An edit that added an exported instance has moved the game's along; they are still found. +void instances_found_by_guid() { + const auto base = game(); + auto edit = copy(base); + const auto added = guid(0x05); // sorts ahead of the game's + edit.instances.insert(edit.instances.begin() + 1, + instance(entry_type, object(entry_type, {{"Cost", number(1)}, {"Detail", value_of(ebx::PointerReference{})}}), &added)); + field(edit, 0, "First") = inside(2); + field(edit, 2, "Detail") = inside(3); + field(edit, 2, "Cost") = number(30); + ebx::MergeSummary summary; + const auto merged = merge(base, {&edit}, summary); + expect(summary.uncarried.empty() && summary.unread.empty() && summary.values == 1 && summary.instances == 1, + "by guid: one value taken and one instance added, nothing left behind"); + expect(read(merged, find(merged, entry_guid), "Cost") == 30, "by guid: the game's instance has the edit's value"); + expect(read(merged, find(merged, added), "Cost") == 1, "by guid: the added instance came across"); + expect(find(merged, added) < find(merged, entry_guid), "by guid: exported instances are in guid order"); + expect(static_cast(read(merged, 0, "First").index) == find(merged, entry_guid), + "by guid: the root still points at the game's instance"); +} + +// An internal instance has no guid. With one more of them in the edit the places say nothing, +// so they are not compared, and the summary says the edit was not looked at in full. +void internal_instances_that_do_not_line_up() { + const auto base = game(); + auto edit = copy(base); + edit.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(4)}}))); + field(edit, 0, "Volume") = number(9); + ebx::MergeSummary summary; + const auto merged = merge(base, {&edit}, summary); + expect(read(merged, 0, "Volume") == 9, "internal: the value on the root is taken"); + expect(summary.uncarried.empty() && summary.unread == std::vector{0} && !summary.exact(), + "internal: the edit is reported as not looked at in full"); +} + +void appended_references() { + const auto base = game(); + auto first = copy(base), second = copy(base), third = copy(base); + append(first, guid(0x20)); + append(first, guid(0x21)); + append(second, guid(0x21)); // the first edit's as well + append(second, guid(0x22)); + append(third, guid(0x23)); // twice in one edit: its author's doing + append(third, guid(0x23)); + ebx::MergeSummary summary; + const auto merged = merge(base, {&first, &second, &third}, summary); + const std::vector expected{guid(0x10), guid(0x11), guid(0x20), guid(0x21), guid(0x22), guid(0x23), guid(0x23)}; + expect(named(merged) == expected, "appended: each reference once, in the order of the edits, one edit's own repeat kept (" + + std::to_string(named(merged).size()) + " entries)"); + expect(summary.arrayEntries == 5 && summary.repeated == 1 && summary.exact(), "appended: five entries taken, one repeat left out"); +} + +void dropped_references() { + auto document = game(); + append(document, guid(0x20)); + append(document, guid(0x21)); + const auto gone = ebx::drop_root_references(document, [](const ebx::ImportReference& reference) { + return reference.fileGuid != guid(0x11) && reference.fileGuid != guid(0x20); }); + expect(gone == 2, "dropped: two entries went (" + std::to_string(gone) + ")"); + expect((named(document) == std::vector{guid(0x10), guid(0x21)}), "dropped: the others are left, in order"); + expect(document.imports.size() == 2, "dropped: the imports nothing uses went with them (" + + std::to_string(document.imports.size()) + " left)"); + + // A boxed value is kept as bytes with its import numbers in them, so the imports stay put. + auto boxed = game(); + boxed.boxedValues.emplace_back(); + expect(ebx::drop_root_references(boxed, [](const ebx::ImportReference& reference) { return reference.fileGuid != guid(0x10); }) == 1 && + boxed.imports.size() == 2 && named(boxed) == std::vector{guid(0x11)}, + "dropped: a document with a boxed value keeps its imports where they are"); + auto untouched = game(); + expect(ebx::drop_root_references(untouched, [](const ebx::ImportReference&) { return true; }) == 0 && + untouched.imports.size() == 2, "dropped: nothing goes when every entry is kept"); +} +} // namespace + +int main() try { + values_from_every_edit(); + one_value_from_two_edits(false); + one_value_from_two_edits(true); + for (int kind = 0; kind < 4; ++kind) a_change_that_cannot_be_taken(kind); + instances_found_by_guid(); + internal_instances_that_do_not_line_up(); + appended_references(); + dropped_references(); + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "ebx merge: ok\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/item_list_merge_tests.cpp b/Engine/Vfs/Test/item_list_merge_tests.cpp new file mode 100644 index 0000000..41d6209 --- /dev/null +++ b/Engine/Vfs/Test/item_list_merge_tests.cpp @@ -0,0 +1,577 @@ +// Cosmetic mods whose item lists name an item the merged bundle does not hold. +// +// Every cosmetic mod ships its own copy of the game's shared customization bundle: the game's +// assets, one of the game's item lists with the mod's item appended, and the item. The merge +// combines the lists, and a bundle keeps one asset per name. The game reads each entry of an +// item list at startup without checking it, so an entry whose item is not in the bundle is a +// crash on every launch (Skate.exe+0x891e07, 2026-10-04). Two ways to get one: +// * two mods each add a different item under the same asset name, and only one can stay; +// * a mod's list names an item no installed mod ships (it was built on top of another mod). +// Either way the entry has to go, and everything else of every mod has to stay. (A bundle only +// one mod ships is not merged at all, it passes through as that mod built it, so every case +// here has two mods shipping the shared bundle.) +// +// The same mods also show what else the merge does with two copies of one of the game's assets: +// * an item two mods both list is in the merged list once; +// * values two mods changed in place in one asset are both in the merged asset, and where that +// cannot be done the merge keeps one mod's copy whole and says whose changes are missing; +// * the list other mods' copies of the bundle are given leaves out an item that is not in them. +// +// The mods are made here from the installed game's own list and one of its items. +// Arguments: []; skipped when the game is not there. With a second +// argument the two mods of the first case are also left in that folder, stamped for this game, +// to try a build against in the game itself. +#include "Engine/Game/Build/supported_build.h" +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_carry.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_writer.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/mod_merge_internal.h" + +#include + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +using namespace dingosdk; +using namespace dingosdk::mods::detail; + +namespace { +using Bytes = std::vector; +constexpr char shared_bundle[] = "win32/characters/customization/configs/cas_main_sharedbundle"; +constexpr char list_type[] = "delmaritemcollectionasset"; + +int failures = 0; +void expect(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +fb::Guid guid(std::uint8_t seed) { + fb::Guid value; + for (std::size_t index = 0; index < value.bytes.size(); ++index) + value.bytes[index] = static_cast(static_cast(seed + index)); + return value; +} + +// The shared bundle as one data layer has it: the game's, a mod's or the merged patch's. +struct Shared { + fb::TocBundle bundle; + Listing listing; +}; +Shared shared_in(const CasStore& store, const fs::path& root, const fs::path& baseRoot, const fs::path& gameRoot) { + for (const auto& bundle : fb::read_toc(read_file(root / L"Win32" / L"items.toc")).bundles) { + if (lower(bundle.name) != shared_bundle) continue; + if (auto listing = list_bundle(store, root, baseRoot, bundle, gameRoot)) return {bundle, std::move(*listing)}; + break; + } + throw std::runtime_error(std::string("no readable ") + shared_bundle + " in " + root.string()); +} + +bool list_name(std::string_view name) { + const auto text = lower(name); + return text.starts_with("items/") && text.find("collection") != std::string::npos; +} + +// The items a list names, by the guid of each one's own document. +std::vector listed(const fb::ebx::Document& list) { + const auto* root = list.root(); + const auto* items = root && root->object ? root->object->find("Items") : nullptr; + const auto* entries = items ? std::get_if(&items->value.data) : nullptr; + if (!entries) throw std::runtime_error("an item list has no Items"); + std::vector result; + for (const auto& entry : *entries) { + const auto* pointer = std::get_if(&entry.data); + if (!pointer || pointer->kind != fb::ebx::PointerKind::external) + throw std::runtime_error("an item list entry is not a reference to another document"); + result.push_back(list.imports.at(static_cast(pointer->index)).fileGuid); + } + return result; +} + +// The game's list with more items appended, the way a cosmetic mod ships it. +Bytes list_with(const fb::ebx::Document& game, std::initializer_list items) { + auto list = fb::ebx::detail::clone_document(game); + for (auto& field : list.instances.front().object->fields) { + if (field.name != "Items") continue; + auto& entries = std::get(field.value.data); + for (const auto& item : items) { + fb::ebx::Value entry; + entry.data = fb::ebx::PointerReference{fb::ebx::PointerKind::external, + static_cast(list.imports.size())}; + list.imports.push_back(item); + entries.push_back(std::move(entry)); + } + } + return fb::ebx::write_document(list); +} + +// One of the game's items under a name and identity of its own: what a cosmetic mod adds. +struct Item { + std::string name; + fb::ebx::ImportReference identity; + Bytes payload; +}; +Item item_from(const fb::ebx::Document& donor, const std::string& name, std::uint8_t seed) { + auto item = fb::ebx::detail::clone_document(donor); + auto& root = item.instances.front(); + item.fileGuid = guid(seed); + root.instanceGuid = guid(static_cast(seed + 0x40)); + const auto key = name.substr(name.rfind('/') + 1); + std::uint32_t hash = 5381; // the game's own hash of an item's key + for (const unsigned char character : key) hash = hash * 33U ^ character; + for (auto& field : root.object->fields) { + if (field.name == "Name") field.value.data = name; + else if (field.name == "Key") field.value.data = key; + else if (field.name == "HashedAssetKey") { + if (std::holds_alternative(field.value.data)) field.value.data = static_cast(hash); + else field.value.data = static_cast(hash); + } + } + return {name, {item.fileGuid, root.instanceGuid}, fb::ebx::write_document(item)}; +} + +// What the mods are made from: the game's shared bundle, its shortest item list and an item. +struct Game { + fs::path root, base; + Shared shared; + std::size_t listIndex{}; + std::string listName; + fb::ebx::Document list; + std::size_t listed{}; + std::size_t itemIndex{}; + std::string itemName; + fb::ebx::Document item; +}; +Game read_game(const fs::path& root, const CasStore& store) { + Game game{root, root / L"Data"}; + game.shared = shared_in(store, game.base, game.base, root); + const auto& assets = game.shared.listing.manifest.ebx; + bool list{}, item{}; + for (std::size_t index = 0; index < assets.size(); ++index) { + const auto name = lower(assets[index].name); + if (!name.starts_with("items/") || (item && !list_name(name))) continue; + auto document = fb::ebx::read_document(read_asset(store, game.base, game.base, game.shared.listing, index, root)); + const auto type = lower(document.rootType); + if (type == list_type) { + const auto count = listed(document).size(); + if (list && count >= game.listed) continue; + list = true; + game.listIndex = index; + game.listName = assets[index].name; + game.listed = count; + game.list = std::move(document); + } else if (!item && type.find("itemasset") != std::string::npos) { + item = true; + game.itemIndex = index; + game.itemName = assets[index].name; + game.item = std::move(document); + } + } + if (!list || !item) throw std::runtime_error("the game's shared bundle has no item list or no item"); + return game; +} + +// A cosmetic mod's folder: its copy of the shared bundle in items.toc (the game's assets, the +// item list replaced and its items added) and one archive holding the manifest and those payloads. +// With no list the mod leaves the game's as it is, as a mod does whose items go in another one. +// `changed` replaces assets of the game's, by their place in the bundle. +void write_mod(const fs::path& directory, const CasStore& store, const Game& game, const Bytes* list, + std::initializer_list items, const std::map& changed = {}) { + auto manifest = game.shared.listing.manifest; + const auto& shipped = game.shared.listing.files; + std::vector files(shipped.begin() + static_cast(game.shared.listing.first), shipped.end()); + const auto installChunk = shipped.front().location.installChunk; + Bytes archive; + const auto stored = [&](std::span bytes) { + const fb::BundleFileInfo file{{true, installChunk, 1}, static_cast(archive.size()), + static_cast(bytes.size())}; + archive.insert(archive.end(), bytes.begin(), bytes.end()); + return file; + }; + const auto encoded = [](const Bytes& payload) { return fb::encode_cas(payload, {.compression = fb::CasCompression::raw}); }; + if (list) { + manifest.ebx[game.listIndex].sha1 = sha1_of(*list); + manifest.ebx[game.listIndex].originalSize = list->size(); + files[game.listIndex] = stored(encoded(*list)); + } + for (const auto& [index, payload] : changed) { + manifest.ebx[index].sha1 = sha1_of(payload); + manifest.ebx[index].originalSize = payload.size(); + files[index] = stored(encoded(payload)); + } + for (const auto& item : items) { + fb::BundleAsset asset; + asset.kind = fb::AssetKind::ebx; + asset.name = item.name; + asset.sha1 = sha1_of(item.payload); + asset.originalSize = item.payload.size(); + files.insert(files.begin() + static_cast(manifest.ebx.size()), stored(encoded(item.payload))); + manifest.ebx.push_back(std::move(asset)); + } + files.insert(files.begin(), stored(fb::write_binary_bundle(manifest))); + const std::vector bundles{{game.shared.bundle.name, fb::write_bundle_region(files), 1}}; + fs::create_directories(directory / L"Win32" / fs::path(store.directory(installChunk))); + write_file(directory / L"Win32" / L"items.toc", fb::write_patch_toc(bundles)); + write_file(directory / L"Win32" / fs::path(store.directory(installChunk)) / L"cas_01.cas", archive); + fs::copy_file(game.base / L"layout.toc", directory / L"layout.toc", fs::copy_options::overwrite_existing); +} + +// A Mods folder of its own beside nothing, merged against the installed game. +struct Merge { + fs::path root; + mods::Catalog catalog; + + Merge(const Game& game, const std::string& name) { + wchar_t temp[MAX_PATH]{}; + GetTempPathW(MAX_PATH, temp); + root = fs::path(temp) / ("reskate-item-lists-" + std::to_string(GetCurrentProcessId()) + "-" + name); + fs::remove_all(root); + catalog.data_root = game.root; + catalog.root = root / "Mods"; + catalog.present = true; + } + ~Merge() { + std::error_code ignored; + fs::remove_all(root, ignored); + } + // Mods are added highest priority first. + fs::path add(const std::string& name) { + mods::Mod mod; + mod.name = name; + mod.directory = catalog.root / name; + mod.provides_layout = true; + catalog.mods.push_back(mod); + return mod.directory; + } +}; + +// The merged shared bundle: the guid of every document in it, and what each item list names. +struct Merged { + std::set held; + std::map> lists; +}; +Merged read_merged(const Merge& merge, const Game& game) { + const auto output = merge.catalog.root / mods::generated_folder; + const auto layout = vfs::read_layout(output / L"layout.toc"); + const CasStore store(game.base, merge.root / L"unused", layout.root); + const auto shared = shared_in(store, output, game.base, game.root); + Merged merged; + for (std::size_t index = 0; index < shared.listing.manifest.ebx.size(); ++index) { + const auto bytes = read_asset(store, output, game.base, shared.listing, index, game.root); + merged.held.insert(fb::ebx::read_file_guid(bytes)); + const auto& name = shared.listing.manifest.ebx[index].name; + if (!list_name(name)) continue; + const auto document = fb::ebx::read_document(bytes); + if (lower(document.rootType) == list_type) merged.lists.emplace(name, listed(document)); + } + return merged; +} + +bool noted(const mods::MergeReport& report, const std::string& text) { + return std::ranges::find(report.notes, text) != report.notes.end(); +} +bool noted_part(const mods::MergeReport& report, std::string_view part) { + return std::ranges::any_of(report.notes, [&](const std::string& note) { return note.find(part) != std::string::npos; }); +} +std::string describe(const mods::MergeReport& report) { + std::string text = "issue: '" + report.issue + "'"; + for (const auto& note : report.notes) text += "\n note: " + note; + for (const auto& [mod, problems] : report.problems) + for (const auto& problem : problems) text += "\n " + mod + ": " + problem; + return text; +} + +// What the game needs of any merge: every item every list names is in the bundle. Then how many +// entries the mods' list ended with, and that no mod was turned away over it. +std::vector merged_list(const Merge& merge, const Game& game, const mods::MergeReport& report, + const std::string& label, std::size_t added) { + expect(report.issue.empty() && report.built && report.problems.empty(), + label + ": the merge builds the patch and keeps every mod\n" + describe(report)); + if (!report.built) return {}; + const auto merged = read_merged(merge, game); + for (const auto& [name, items] : merged.lists) + for (const auto& item : items) + expect(merged.held.contains(item), label + ": " + name + " names " + item.string() + + ", which the merged bundle does not hold"); + const auto found = merged.lists.find(game.listName); + expect(found != merged.lists.end(), label + ": the merged bundle still has " + game.listName); + if (found == merged.lists.end()) return {}; + expect(found->second.size() == game.listed + added, label + ": the list has the game's " + std::to_string(game.listed) + + " item(s) and " + std::to_string(added) + " more (" + std::to_string(found->second.size()) + ")\n" + describe(report)); + return found->second; +} +bool names(const std::vector& list, const Item& item) { + return std::ranges::find(list, item.identity.fileGuid) != list.end(); +} + +constexpr char same_name[] = "items/reskate_test/own_same_name"; + +// Two mods add different items under one name. The higher-priority mod's is the one the bundle +// keeps, so the other's entry goes, and the merge says which mod lost an item to which. +void same_name_from_two_mods(const Game& game, const CasStore& store, const fs::path* keep) { + Merge merge(game, "same-name"); + const auto upper = item_from(game.item, same_name, 0x10); + const auto under = item_from(game.item, same_name, 0x20); + const auto upperList = list_with(game.list, {upper.identity}), underList = list_with(game.list, {under.identity}); + const auto build = [&](const fs::path& top, const fs::path& bottom) { + write_mod(top, store, game, &upperList, {upper}); + write_mod(bottom, store, game, &underList, {under}); + }; + const auto top = merge.add("upper"), bottom = merge.add("under"); + build(top, bottom); + const auto report = mods::merge_mods(merge.catalog); + const auto list = merged_list(merge, game, report, "same name", 1); + expect(names(list, upper) && !names(list, under), "same name: the list keeps the higher-priority mod's item only"); + expect(noted(report, "under: " + game.shared.bundle.name + ": 1 added asset(s) share a name with ones upper adds, e.g. " + + same_name + "; the merged bundle keeps upper's"), + "same name: the merge says whose asset the name went to\n" + describe(report)); + expect(noted(report, "under: " + game.listName + ": 1 item(s) left out of the list because the merged bundle does not " + "hold them, e.g. " + same_name + ", replaced by upper's asset of the same name"), + "same name: the merge says which item was left out and why\n" + describe(report)); + if (!keep) return; + // The same two for the game's own Mods folder, where a mod has to say what it was built for. + const std::string stamp = "ReSkate Studio native Patch v1\nskate_sha256=" + std::string(supported_build::game_sha256) + "\n"; + for (const auto* name : {L"ReSkateTest-SameNameUpper", L"ReSkateTest-SameNameUnder"}) fs::create_directories(*keep / name); + build(*keep / L"ReSkateTest-SameNameUpper", *keep / L"ReSkateTest-SameNameUnder"); + for (const auto* name : {L"ReSkateTest-SameNameUpper", L"ReSkateTest-SameNameUnder"}) + write_file(*keep / name / mods::studio_marker_file, std::as_bytes(std::span(stamp))); + std::cout << "The two same-name mods are in " << keep->string() << ".\n"; +} + +// Two mods add items under names of their own: both stay, and nothing is said about leaving any out. +void different_names_from_two_mods(const Game& game, const CasStore& store) { + Merge merge(game, "different-names"); + const auto first = item_from(game.item, "items/reskate_test/own_first", 0x10); + const auto second = item_from(game.item, "items/reskate_test/own_second", 0x20); + const auto firstList = list_with(game.list, {first.identity}), secondList = list_with(game.list, {second.identity}); + write_mod(merge.add("first"), store, game, &firstList, {first}); + write_mod(merge.add("second"), store, game, &secondList, {second}); + const auto report = mods::merge_mods(merge.catalog); + const auto list = merged_list(merge, game, report, "different names", 2); + expect(names(list, first) && names(list, second), "different names: the list has both mods' items"); + expect(!noted_part(report, "left out of the list") && !noted_part(report, "share a name"), + "different names: nothing is reported\n" + describe(report)); +} + +// One mod alone changes the list, beside a mod that ships the bundle with the game's. Its list +// as it wrote it is what the bundle gets, unless that list names an item nothing ships: then +// that entry goes and its own item stays. +void one_mod_changes_the_list(const Game& game, const CasStore& store, bool missing) { + const std::string label = missing ? "an item nothing ships" : "one mod's list"; + Merge merge(game, missing ? "missing-item" : "one-list"); + const auto own = item_from(game.item, "items/reskate_test/own_alone", 0x10); + const fb::ebx::ImportReference absent{guid(0x70), guid(0x90)}; + const auto shipped = missing ? list_with(game.list, {own.identity, absent}) : list_with(game.list, {own.identity}); + write_mod(merge.add("alone"), store, game, &shipped, {own}); + write_mod(merge.add("bystander"), store, game, nullptr, {}); + const auto report = mods::merge_mods(merge.catalog); + const auto list = merged_list(merge, game, report, label, 1); + expect(names(list, own), label + ": the list has the mod's own item"); + const auto said = noted(report, "alone: " + game.listName + ": 1 item(s) left out of the list because the merged bundle " + "does not hold them, e.g. the asset with guid " + absent.fileGuid.string() + + ", which no enabled mod adds to the bundle"); + expect(said == missing, label + (missing ? ": the merge says which item was left out\n" : ": nothing is reported\n") + + describe(report)); +} + +// Two packs that both carry one item, each with an item of its own besides. Both list the shared +// one, and it is one item: the merged list has it once. +void an_item_two_mods_both_list(const Game& game, const CasStore& store) { + Merge merge(game, "listed-twice"); + const auto both = item_from(game.item, "items/reskate_test/own_in_both", 0x10); + const auto first = item_from(game.item, "items/reskate_test/own_first", 0x20); + const auto second = item_from(game.item, "items/reskate_test/own_second", 0x30); + const auto firstList = list_with(game.list, {both.identity, first.identity}); + const auto secondList = list_with(game.list, {both.identity, second.identity}); + write_mod(merge.add("first"), store, game, &firstList, {both, first}); + write_mod(merge.add("second"), store, game, &secondList, {both, second}); + const auto report = mods::merge_mods(merge.catalog); + const auto list = merged_list(merge, game, report, "listed by both", 3); + expect(names(list, both) && names(list, first) && names(list, second), "listed by both: the list has all three items"); + expect(std::ranges::count(list, both.identity.fileGuid) == 1, "listed by both: the item both mods list is in it once"); + expect(noted(report, game.listName + ": combined 2 edits (0 instance(s), 3 list entries, 1 repeated entry(ies) listed once)"), + "listed by both: the merge says one entry was a repeat\n" + describe(report)); +} + +// The game's item as a mod that retunes it ships it: the same document, changed in place. +Bytes item_with(const fb::ebx::Document& game, const std::function& change) { + auto item = fb::ebx::detail::clone_document(game); + change(*item.instances.front().object); + return fb::ebx::write_document(item); +} +fb::ebx::Value& field(fb::ebx::Object& object, std::string_view name) { + for (auto& entry : object.fields) + if (entry.name == name) return entry.value; + throw std::runtime_error("the game's item has no " + std::string(name)); +} +void set_hash(fb::ebx::Object& object, std::uint32_t hash) { + auto& value = field(object, "HashedAssetKey"); + if (std::holds_alternative(value.data)) value.data = static_cast(hash); + else value.data = static_cast(hash); +} +std::uint32_t hash_of(const fb::ebx::Object& object) { + const auto& value = field(const_cast(object), "HashedAssetKey"); + const auto* number = std::get_if(&value.data); + return number ? static_cast(*number) : static_cast(std::get(value.data)); +} +// A change the merge cannot take value by value: the first list of the item made one shorter, +// or failing that its first reference cleared. False when the item has neither. +bool reshape(fb::ebx::Object& object) { + for (auto& entry : object.fields) + if (auto* list = std::get_if(&entry.value.data); list && !list->empty()) { + list->pop_back(); + return true; + } + for (auto& entry : object.fields) + if (auto* pointer = std::get_if(&entry.value.data); + pointer && pointer->kind != fb::ebx::PointerKind::null) { + *pointer = {}; + return true; + } + return false; +} + +// The merged shared bundle's copy of the game's item, and the sha1 the bundle lists it under. +std::pair merged_item(const Merge& merge, const Game& game) { + const auto output = merge.catalog.root / mods::generated_folder; + const auto layout = vfs::read_layout(output / L"layout.toc"); + const CasStore store(game.base, merge.root / L"unused", layout.root); + const auto shared = shared_in(store, output, game.base, game.root); + const auto& assets = shared.listing.manifest.ebx; + for (std::size_t index = 0; index < assets.size(); ++index) + if (assets[index].name == game.itemName) + return {fb::ebx::read_document(read_asset(store, output, game.base, shared.listing, index, game.root)), assets[index].sha1}; + throw std::runtime_error("the merged bundle has no " + game.itemName); +} + +// Two mods each ship the game's item with values of their own changed in place. Nothing was added +// to it, so the merge has no list to put together: it has the values. Each mod's are in the +// merged item, and where both changed one the higher mod's is. +void values_two_mods_changed(const Game& game, const CasStore& store, bool contested) { + const std::string label = contested ? "one value from two mods" : "values from two mods"; + Merge merge(game, contested ? "one-value" : "two-values"); + const auto upper = item_with(game.item, [&](fb::ebx::Object& root) { + if (contested) set_hash(root, 0x1111); + else field(root, "Key").data = std::string("reskate_upper"); + }); + const auto under = item_with(game.item, [&](fb::ebx::Object& root) { + set_hash(root, 0x2222); + if (contested) field(root, "Key").data = std::string("reskate_under"); + }); + write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}}); + write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}}); + const auto report = mods::merge_mods(merge.catalog); + expect(report.issue.empty() && report.built && report.problems.empty(), + label + ": the merge builds the patch and keeps every mod\n" + describe(report)); + if (!report.built) return; + const auto item = merged_item(merge, game).first; + auto& root = *item.instances.front().object; + expect(std::get(field(root, "Key").data) == (contested ? "reskate_under" : "reskate_upper"), + label + ": the merged item has the key one mod changed"); + expect(hash_of(root) == (contested ? 0x1111u : 0x2222u), + label + (contested ? ": the value both changed is the higher mod's" : ": and the value the other mod changed")); + expect(noted(report, game.itemName + ": combined 2 edits (0 instance(s), 0 list entries, " + + (contested ? "3 changed value(s), 1 disagreed)" : "2 changed value(s))")), + label + ": the merge says what it combined\n" + describe(report)); +} + +// One of the two also changed the item in a way that is not a value (a list made shorter). That +// cannot be put together with the other mod's values, so the higher mod's copy stays whole, as +// it always did, and the merge now says whose changes are not in the game. +void a_change_that_does_not_combine(const Game& game, const CasStore& store) { + Merge merge(game, "not-combined"); + bool reshaped{}; + const auto upper = item_with(game.item, [&](fb::ebx::Object& root) { + field(root, "Key").data = std::string("reskate_upper"); + reshaped = reshape(root); + }); + if (!reshaped) { + std::cout << "The game's item has no list or reference to change; that case is skipped.\n"; + return; + } + const auto under = item_with(game.item, [&](fb::ebx::Object& root) { set_hash(root, 0x2222); }); + write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}}); + write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}}); + const auto report = mods::merge_mods(merge.catalog); + expect(report.issue.empty() && report.built && report.problems.empty(), + "not combined: the merge builds the patch and keeps every mod\n" + describe(report)); + if (!report.built) return; + expect(merged_item(merge, game).second == sha1_of(upper), "not combined: the bundle has the higher mod's copy as that mod built it"); + expect(noted(report, game.itemName + ": kept upper's copy whole; the changes under made to it are of a kind that cannot " + "be combined with it and are not in the game"), + "not combined: the merge says whose changes are missing\n" + describe(report)); +} + +// What other mods' copies of the bundle are given (a map carries one for its levels): one list +// for all of them, put together from every mod's. Under a name two mods share, such a copy gets +// the higher mod's asset, so the lower mod's entry for its own has to be out of that list too. +void the_list_other_copies_are_given(const Game& game, const CasStore& store) { + Merge merge(game, "carried-list"); + const auto upper = item_from(game.item, same_name, 0x10); + const auto under = item_from(game.item, same_name, 0x20); + const auto upperList = list_with(game.list, {upper.identity}), underList = list_with(game.list, {under.identity}); + write_mod(merge.add("upper"), store, game, &upperList, {upper}); + write_mod(merge.add("under"), store, game, &underList, {under}); + std::vector order; + std::map files; + for (const auto& mod : merge.catalog.mods) { + order.push_back(&mod); + files.emplace(&mod, scan(mod.directory)); + } + mods::MergeReport report; + const auto overrides = collect_asset_overrides(order, files, store, game.base, game.root, report); + const auto versions = overrides.changed.find(lower(game.listName)); + expect(versions != overrides.changed.end() && versions->second.size() == 1, + "carried list: there is one changed list for other mods' copies\n" + describe(report)); + if (versions == overrides.changed.end() || versions->second.empty()) return; + const auto& change = versions->second.begin()->second; + const auto list = listed(fb::ebx::read_document(fb::decode_cas(change.encoded, {game.root}))); + expect(list.size() == game.listed + 1 && names(list, upper) && !names(list, under), + "carried list: it has the game's items and the higher mod's, not the one no copy is given (" + + std::to_string(list.size()) + " entries)"); + expect(change.names == std::set{upper.identity.fileGuid}, "carried list: the item that goes with it is the higher mod's"); + expect(noted(report, "under: " + game.listName + ": 1 entry(ies) left out of the copies other mods carry, where another " + "mod's asset of the same name stands in for what they name"), + "carried list: the merge says which mod's entry was left out\n" + describe(report)); +} +} // namespace + +int main(int argc, char** argv) try { + if (argc < 2 || !fs::exists(fs::path(argv[1]) / L"Data" / L"Win32" / L"items.toc")) { + std::cout << "No game given; skipped.\n"; + return 0; + } + const fs::path root = argv[1]; + const auto layout = vfs::read_layout(root / L"Data" / L"layout.toc"); + const CasStore store(root / L"Data", fs::temp_directory_path() / L"reskate-item-lists-unused", layout.root); + const auto game = read_game(root, store); + const fs::path keep = argc > 2 ? fs::path(argv[2]) : fs::path{}; + same_name_from_two_mods(game, store, argc > 2 ? &keep : nullptr); + different_names_from_two_mods(game, store); + one_mod_changes_the_list(game, store, false); + one_mod_changes_the_list(game, store, true); + an_item_two_mods_both_list(game, store); + values_two_mods_changed(game, store, false); + values_two_mods_changed(game, store, true); + a_change_that_does_not_combine(game, store); + the_list_other_copies_are_given(game, store); + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "item list merge: ok (" << game.listName << ", " << game.listed << " item(s) in the game)\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp new file mode 100644 index 0000000..2923651 --- /dev/null +++ b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp @@ -0,0 +1,448 @@ +// Assets a mod ADDS to a bundle (new songs, say) while another mod ships its own copy of that +// bundle (a map does, for its levels). The merged bundle is built from one mod's copy, so +// without the carrying step the added assets are missing from it whenever the map's copy wins. +// +// The fixture is a small made-up game and two mods in a temp folder, written with the repo's +// own writers, so it needs no game files: +// game bundle "win32/test/shared": test/playlist, test/other +// bundle "win32/test/second": test/playlist again, test/extra +// map a level mod: its own level superbundle (a new TOC) with a copy of that bundle, as a +// custom map carries the core assets for its level; it edits nothing +// music an asset mod that changes test/playlist to name a new song, and adds the song, the +// wave the song names, the wave's sound-bank resource, and an asset nothing names. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/native_db.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +namespace db = dingosdk::native_db; +using namespace dingosdk; + +namespace { +using Bytes = std::vector; +constexpr std::uint32_t package = 0x1234; +constexpr char bundle_name[] = "win32/test/shared"; +constexpr char second_bundle[] = "win32/test/second"; + +int failures = 0; +void expect(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +// ---- Bytes --------------------------------------------------------------------------------------- +void put32(Bytes& out, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) out.push_back(static_cast(value >> shift)); +} +void put_bytes(Bytes& out, std::string_view text) { + for (const char c : text) out.push_back(static_cast(c)); +} +void put_guid(Bytes& out, const fb::Guid& guid) { out.insert(out.end(), guid.bytes.begin(), guid.bytes.end()); } +fb::Guid guid(std::uint8_t seed) { + fb::Guid value; + value.bytes.fill(static_cast(seed)); + return value; +} +void write(const fs::path& path, std::span bytes) { + fs::create_directories(path.parent_path()); + std::ofstream(path, std::ios::binary).write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} +Bytes read(const fs::path& path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + Bytes bytes(static_cast(in.tellg())); + in.seekg(0); + in.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +// ---- An EBX document with no instances: a file GUID and the files it imports. ----------------- +// That is all the merge reads from an asset to follow what it refers to. +void chunk(Bytes& out, const char (&id)[5], const Bytes& body) { + put_bytes(out, std::string_view(id, 4)); + put32(out, static_cast(body.size())); + out.insert(out.end(), body.begin(), body.end()); + if (body.size() & 1) out.push_back(std::byte{0}); +} +Bytes ebx_document(const fb::Guid& file, const std::vector& imports) { + Bytes fixup; + put_guid(fixup, file); + for (int i = 0; i < 6; ++i) put32(fixup, 0); // types, signatures, exported, instances, pointers, resource refs + put32(fixup, static_cast(imports.size())); + for (const auto& import : imports) { + put_guid(fixup, import); + put_guid(fixup, guid(0xEE)); // the imported class + } + for (int i = 0; i < 2; ++i) put32(fixup, 0); // import relocations, type-info references + for (int i = 0; i < 3; ++i) put32(fixup, 0); // array, boxed-value and string sections + const Bytes data(16); + Bytes extra; + put32(extra, 0); // arrays + put32(extra, 0); // boxed values + Bytes reflection; + for (int i = 0; i < 5; ++i) put32(reflection, 0); // signatures, types, fields, groups, mappings + Bytes body; + put_bytes(body, std::string_view("EBX\0", 4)); + chunk(body, "EFIX", fixup); + chunk(body, "EBXD", data); + chunk(body, "EBXX", extra); + chunk(body, "REFL", reflection); + Bytes riff; + put_bytes(riff, "RIFF"); + put32(riff, static_cast(body.size())); + riff.insert(riff.end(), body.begin(), body.end()); + return riff; +} + +// ---- The fixture's files ----------------------------------------------------------------------- +// layout.toc with one install chunk, `package`, whose archives live in Win32/pkg. +Bytes layout_toc(const std::vector& extra = {}) { + const auto leaf = [](unsigned type, const char* name, std::vector payload) { + db::Node node; + node.type = type; + node.named = true; + node.name = name; + node.owned = std::move(payload); + return node; + }; + const auto container = [](unsigned type, const char* name, std::vector children) { + db::Node node; + node.type = type; + node.named = *name != '\0'; + node.name = name; + node.terminated = true; + node.children = std::move(children); + return node; + }; + const auto records = [](const std::vector& names) { + std::vector rows; + for (const auto& name : names) rows.push_back(db::make_named_record("name", name)); + return rows; + }; + const auto low = static_cast(package & 0xFF), high = static_cast(package >> 8); + const auto chunk_entry = container(2, "", {db::make_named_record("name", "pkg").children.front(), + leaf(8, "persistentIndex", {low, high, 0, 0}), + container(1, "superbundles", records(extra))}); + const auto root = container(2, "", { + container(1, "superBundles", [&] { auto rows = records({"Win32/globals"}); for (auto& row : records(extra)) rows.push_back(std::move(row)); return rows; }()), + container(2, "installManifest", {container(1, "installChunks", {chunk_entry})}), + leaf(19, "layeredInstallChunkFiles", {1, 0, low, high, 0, 0, 0, 0})}); + const auto tree = db::write(root); + Bytes file(db::envelope_size + tree.size()); + std::memcpy(file.data(), db::magic, sizeof(db::magic)); + std::memcpy(file.data() + db::envelope_size, tree.data(), tree.size()); + return file; +} + +Bytes encoded(const Bytes& payload) { return fb::encode_cas(payload, {.compression = fb::CasCompression::raw}); } + +fb::Sha1 sha(const std::string& name, unsigned version) { + fb::Sha1 value; + for (std::size_t i = 0; i < name.size(); ++i) value.bytes[i % 19] ^= static_cast(name[i]); + value.bytes[19] = static_cast(version); + return value; +} + +struct Ebx { std::string name; unsigned version; Bytes payload; }; +struct Resource { std::string name; Bytes payload; }; + +fb::BinaryBundle manifest_of(const std::vector& assets, const std::vector& resources) { + fb::BinaryBundle manifest; + for (const auto& asset : assets) { + fb::BundleAsset entry; + entry.kind = fb::AssetKind::ebx; + entry.name = asset.name; + entry.sha1 = sha(asset.name, asset.version); + entry.originalSize = asset.payload.size(); + manifest.ebx.push_back(std::move(entry)); + } + for (const auto& resource : resources) { + fb::BundleAsset entry; + entry.kind = fb::AssetKind::resource; + entry.name = resource.name; + entry.sha1 = sha(resource.name, 0); + entry.resourceId = 7; + entry.resourceType = 0xb2c465f6; + entry.resourceMeta.assign(16, std::byte{0}); + entry.originalSize = resource.payload.size(); + manifest.resources.push_back(std::move(entry)); + } + return manifest; +} + +struct Fixture { + fs::path root; + mods::Catalog catalog; + + explicit Fixture(const std::string& name) { + wchar_t temp[MAX_PATH]{}; + GetTempPathW(MAX_PATH, temp); + root = fs::path(temp) / ("reskate-merge-added-" + std::to_string(GetCurrentProcessId()) + "-" + name); + fs::remove_all(root); + catalog.data_root = root / "game"; + catalog.root = catalog.data_root / "Mods"; + catalog.present = true; + // The game: two assets in one bundle, and a second bundle that has the playlist as well + // (the game keeps a list in several bundles), stored in the game's own archive. + const std::vector assets{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}}; + const std::vector second{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 0, ebx_document(guid(3), {})}}; + Bytes archive; + const auto region = [&](const std::vector& list) { + // The asset list is the region's first file, as it is in every bundle of the game: + // a bundle one mod alone ships then passes through as that mod built it. + const auto listing = fb::write_binary_bundle(manifest_of(list, {})); + std::vector files{{{false, package, 1}, static_cast(archive.size()), + static_cast(listing.size())}}; + archive.insert(archive.end(), listing.begin(), listing.end()); + for (const auto& asset : list) { + const auto payload = encoded(asset.payload); + files.push_back({{false, package, 1}, static_cast(archive.size()), static_cast(payload.size())}); + archive.insert(archive.end(), payload.begin(), payload.end()); + } + return fb::write_bundle_region(files); + }; + const std::vector bundles{{bundle_name, region(assets), 1}, {second_bundle, region(second), 1}}; + write(catalog.data_root / "Data" / "layout.toc", layout_toc()); + write(catalog.data_root / "Data" / "Win32" / "test_shared.toc", fb::write_patch_toc(bundles)); + write(catalog.data_root / "Data" / "Win32" / "pkg" / "cas_01.cas", archive); + } + ~Fixture() { + std::error_code ignored; + fs::remove_all(root, ignored); + } + + // A mod with its own copy of a bundle: the manifest first (raw), then every payload. + void add(const std::string& name, bool levels, const std::string& toc, const std::vector& superbundles, + const std::vector& assets, const std::vector& resources = {}, const char* bundle = bundle_name) { + const auto listing = fb::write_binary_bundle(manifest_of(assets, resources)); + Bytes archive(listing); + std::vector files{{{true, package, 1}, 0, static_cast(listing.size())}}; + const auto store = [&](const Bytes& payload) { + const auto bytes = encoded(payload); + files.push_back({{true, package, 1}, static_cast(archive.size()), static_cast(bytes.size())}); + archive.insert(archive.end(), bytes.begin(), bytes.end()); + }; + for (const auto& asset : assets) store(asset.payload); + for (const auto& resource : resources) store(resource.payload); + const std::vector bundles{{bundle, fb::write_bundle_region(files), 1}}; + const auto directory = catalog.root / name; + write(directory / "layout.toc", layout_toc(superbundles)); + write(directory / "Win32" / fs::path(toc), fb::write_patch_toc(bundles)); + write(directory / "Win32" / "pkg" / "cas_01.cas", archive); + mods::Mod mod; + mod.name = name; + mod.directory = directory; + mod.provides_layout = true; + mod.provides_levels = levels; + catalog.mods.push_back(std::move(mod)); + } + + // How many files the merged copy of a bundle in `toc` lists, or -1 when the merge left none. + int merged_files(const std::string& relative, const std::string& name = bundle_name) const { + const auto toc = catalog.root / mods::generated_folder / "Win32" / fs::path(relative); + if (!fs::exists(toc)) return -1; + for (const auto& bundle : fb::read_toc(read(toc)).bundles) + if (bundle.name == name) return static_cast(fb::read_bundle_region(bundle.region).files.size()); + return -1; + } +}; + +// The game's copy of the bundle, as a level mod carries it. +std::vector game_copy() { + return {{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}}; +} +// The music mod's bundle: the playlist now names the song (a change), the song names its wave, +// and one more asset nothing names. +std::vector music_assets() { + return {{"test/playlist", 1, ebx_document(guid(1), {guid(10)})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song", 0, ebx_document(guid(10), {guid(11)})}, + {"test/wave", 0, ebx_document(guid(11), {})}, + {"test/unrelated", 0, ebx_document(guid(12), {})}}; +} +std::vector music_resources() { return {{"test/wave", {std::byte{1}, std::byte{2}, std::byte{3}}}}; } + +bool noted(const mods::MergeReport& report, const std::string& text) { + for (const auto& note : report.notes) + if (note == text) return true; + return false; +} +std::string describe(const mods::MergeReport& report) { + std::string text = "issue: '" + report.issue + "'"; + for (const auto& note : report.notes) text += "\n note: " + note; + for (const auto& [mod, problems] : report.problems) + for (const auto& problem : problems) text += "\n " + mod + ": " + problem; + return text; +} + +constexpr char shared_toc[] = "test_shared.toc"; +constexpr char map_toc[] = "levels/test/map.toc"; +constexpr char map_superbundle[] = "Win32/levels/test/map"; + +// The level mod alone: what its copy of the bundle holds with nothing carried into it. +int map_only() { + Fixture fixture("map-only"); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "map only: the merge builds the patch\n" + describe(report)); + return fixture.merged_files(map_toc); +} + +// The music mod edits the game's TOC; the map's copy of the bundle is in its own TOC, so nothing +// of the music mod reaches it unless the merge carries it. The playlist change always did (it is a +// change to an asset the map's copy has); the song, its wave and the wave's resource are what +// the changed playlist leads to. The asset nothing names must stay out. Either mod may come first. +void carried_into_the_maps_copy(bool map_first, int baseline) { + const std::string order = map_first ? "map first" : "music first"; + Fixture fixture(map_first ? "map-first" : "music-first"); + const auto add_map = [&] { fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); }; + const auto add_music = [&] { fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); }; + if (map_first) { add_map(); add_music(); } else { add_music(); add_map(); } + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, order + ": the merge builds the patch\n" + describe(report)); + expect(noted(report, std::string("map: ") + bundle_name + ": 3 asset(s) added by other mods, e.g. test/song"), + order + ": the map's copy receives them\n" + describe(report)); + const auto files = fixture.merged_files(map_toc); + expect(files == baseline + 3, order + ": the map's copy gained those three files and not the unnamed asset (" + + std::to_string(files) + " files, " + std::to_string(baseline) + " without the music mod)"); +} + +// Another asset mod ships the game's copy of the bundle in the TOC the music mod ships, the way +// cosmetic mods share one. That copy merges with the music mod's bundle, which already has the +// song, so nothing is carried into it (no "added by other mods" note), and the merged bundle is +// exactly the union: the manifest, the music mod's five EBX and its resource. +void copy_in_the_adders_toc_is_left_alone() { + Fixture fixture("same-toc"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("other", false, shared_toc, {}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "same TOC: the merge builds the patch\n" + describe(report)); + for (const auto& note : report.notes) + expect(note.find("added by other mods") == std::string::npos, + "same TOC: nothing is carried into a copy in the adder's own TOC: " + note); + const auto expected = static_cast(1 + music_assets().size() + music_resources().size()); + const auto files = fixture.merged_files(shared_toc); + expect(files == expected, "same TOC: the merged bundle is the union, " + std::to_string(expected) + " files (" + + std::to_string(files) + " files)"); +} +// Two asset mods each add test/song, each named by a change of its own (the rival's is to the +// other asset). Only one can be carried into the map's copy: the higher-priority mod's. When the +// two are different assets the merge says whose the name went to; the same asset added by both +// is no clash and is not reported. The bundle the two mods share keeps one asset per name as +// well, and says so the same way. +void same_name_from_two_mods(bool different) { + const std::string label = different ? "name clash" : "same asset twice"; + Fixture fixture(different ? "name-clash" : "same-twice"); + const auto song = guid(different ? 20 : 10); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("rival", false, shared_toc, {}, + {{"test/playlist", 0, ebx_document(guid(1), {})}, + {"test/other", 1, ebx_document(guid(2), {song})}, + {"test/song", different ? 1u : 0u, ebx_document(song, {})}}); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, label + ": the merge builds the patch\n" + describe(report)); + const bool said = noted(report, "rival: 1 added asset(s) share a name with ones music adds, e.g. test/song; " + "other mods' copies of the bundle get music's"); + expect(said == different, label + (different ? ": the merge says whose asset the name went to\n" + : ": nothing is reported\n") + describe(report)); + const bool kept = noted(report, std::string("rival: ") + bundle_name + ": 1 added asset(s) share a name with ones " + "music adds, e.g. test/song; the merged bundle keeps music's"); + expect(kept == different, label + (different ? ": the merge says whose asset the shared bundle keeps\n" + : ": nothing is reported about the shared bundle\n") + describe(report)); + expect(noted(report, std::string("map: ") + bundle_name + ": 3 asset(s) added by other mods, e.g. test/song"), + label + ": the map's copy still receives the first mod's three\n" + describe(report)); +} + +// Two music mods both change test/playlist to add their own songs. Both mods' songs, +// waves and resources must be carried into the map's copy of the bundle, not just the first mod's. +void two_music_mods_both_carried_into_maps_copy(int baseline) { + Fixture fixture("two-music-mods"); + const auto song1 = guid(10), wave1 = guid(11); + const auto song2 = guid(20), wave2 = guid(21); + fixture.add("music1", false, shared_toc, {}, + {{"test/playlist", 1, ebx_document(guid(1), {song1})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song1", 0, ebx_document(song1, {wave1})}, + {"test/wave1", 0, ebx_document(wave1, {})}}, + {{"test/wave1", {std::byte{1}}}}); + fixture.add("music2", false, shared_toc, {}, + {{"test/playlist", 2, ebx_document(guid(1), {song2})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song2", 0, ebx_document(song2, {wave2})}, + {"test/wave2", 0, ebx_document(wave2, {})}}, + {{"test/wave2", {std::byte{2}}}}); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "two music mods: the merge builds the patch\n" + describe(report)); + expect(noted(report, std::string("map: ") + bundle_name + ": 6 asset(s) added by other mods, e.g. test/song1"), + "two music mods: the map's copy receives all six assets from both mods\n" + describe(report)); + const auto files = fixture.merged_files(map_toc); + expect(files == baseline + 6, "two music mods: the map's copy gained 6 files (3 per music mod): " + + std::to_string(files) + " vs " + std::to_string(baseline + 6)); +} + +// The game keeps the playlist in a second bundle too, as it keeps the music playlist in eight. A +// mod that ships its own copy of that one (it changed something else in it) takes the music mod's +// playlist there. The song, its wave and the wave's resource were added beside the playlist in +// the first bundle; they have to be beside it in this one as well, or the list names a song its +// bundle does not hold. The asset nothing names stays out, and the music mod's own bundle is as +// that mod built it. Either mod may come first. +void carried_into_another_bundle_with_the_list(bool music_first) { + const std::string order = music_first ? "second bundle, music first" : "second bundle, music last"; + Fixture fixture(music_first ? "second-bundle-music-first" : "second-bundle-music-last"); + const auto add_music = [&] { fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); }; + const auto add_tweak = [&] { + fixture.add("tweak", false, shared_toc, {}, + {{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 1, ebx_document(guid(3), {})}}, {}, second_bundle); + }; + if (music_first) { add_music(); add_tweak(); } else { add_tweak(); add_music(); } + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, order + ": the merge builds the patch\n" + describe(report)); + expect(noted(report, std::string("tweak: ") + second_bundle + ": 1 asset(s) take another mod's change, e.g. test/playlist from music"), + order + ": the copy of the second bundle takes the changed playlist\n" + describe(report)); + expect(noted(report, std::string("tweak: ") + second_bundle + ": 3 asset(s) added by other mods, e.g. test/song"), + order + ": and with it the song, its wave and the wave's resource\n" + describe(report)); + const auto files = fixture.merged_files(shared_toc, second_bundle); + expect(files == 1 + 2 + 3, order + ": that copy lists its manifest, its two assets and those three (" + std::to_string(files) + " files)"); + const auto own = fixture.merged_files(shared_toc); + const auto built = static_cast(1 + music_assets().size() + music_resources().size()); + expect(own == built, order + ": the music mod's own bundle is as it built it (" + std::to_string(own) + " files, " + + std::to_string(built) + " built)"); +} +} // namespace + +int main() try { + const auto baseline = map_only(); + expect(baseline > 0, "map only: the map's copy exists (" + std::to_string(baseline) + ")"); + carried_into_the_maps_copy(true, baseline); + carried_into_the_maps_copy(false, baseline); + copy_in_the_adders_toc_is_left_alone(); + same_name_from_two_mods(true); + same_name_from_two_mods(false); + two_music_mods_both_carried_into_maps_copy(baseline); + carried_into_another_bundle_with_the_list(true); + carried_into_another_bundle_with_the_list(false); + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "mod merge added assets: ok\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/mod_merge_audit_tests.cpp b/Engine/Vfs/Test/mod_merge_audit_tests.cpp new file mode 100644 index 0000000..321ba3e --- /dev/null +++ b/Engine/Vfs/Test/mod_merge_audit_tests.cpp @@ -0,0 +1,1066 @@ +// Runs the whole mod merge on a data root and audits the patch it wrote, against the game and +// against every mod, so a Mods folder of real mods can be checked without starting the game. +// +// What has to hold of any merge: +// * every superbundle TOC reads back, and every placement lies inside an archive that is there +// and that the merged layout declares; +// * every payload the patch stores decodes to the size its asset list gives, and every EBX in +// it parses; +// * every bundle, asset and TOC chunk a mod ships is in the patch; +// * a document two mods' assets fought over by name is named by nothing that is left; +// * a document the merge wrote keeps its instances in the order the engine searches them, has +// every reference a mod appended to one of its lists, and has every value a mod changed in +// place changed: no value in it is the merge's own invention; +// * a list whose entries the game keeps beside it (an item list, the music playlist) names +// nothing its own bundle does not hold, in every copy of that bundle the patch carries; +// * a bundle's chunk metadata has one record for each of its chunks, in the order of the +// chunks' guids as the game writes it: every texture's own record, with the first mip its +// header gives, is at its chunk's place. +// What is only reported: assets two mods add under one name, and assets of the game two mods +// change where the patch has one mod's copy whole. +// +// Arguments: [], the root being a folder holding the game's Data/ and +// a Mods/ folder; skipped when none is given. The merge runs there as the game would run it and +// writes Mods/.reskate. With a second argument, every asset the patch itself stores is written +// to that file, one line each, to compare two merges of the same mods. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/mod_merge_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +using namespace dingosdk; +using namespace dingosdk::mods::detail; + +namespace { +using Bytes = std::vector; + +std::map failures; +// The first of each kind in full, the rest counted: one broken mod can fail ten thousand times. +void fail(const std::string& kind, const std::string& text) { + const auto count = ++failures[kind]; + if (count <= 25) std::cout << "FAIL [" << kind << "] " << text << '\n'; + else if (count == 26) std::cout << "FAIL [" << kind << "] ... more of these, counted below\n"; +} +void section(const std::string& title) { std::cout << "\n=== " << title << '\n'; } + +std::string hex(const fb::Sha1& sha) { + static constexpr char digits[] = "0123456789abcdef"; + std::string text; + for (const auto value : sha.bytes) { + text += digits[static_cast(value) >> 4]; + text += digits[static_cast(value) & 15]; + } + return text; +} +std::string key_of(const fb::BundleAsset& asset) { + return std::to_string(static_cast(asset.kind)) + ':' + + (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : lower(asset.name)); +} +const char* kind_name(fb::AssetKind kind) { + return kind == fb::AssetKind::ebx ? "ebx" : kind == fb::AssetKind::resource ? "res" : "chunk"; +} +std::string name_of(const fb::BundleAsset& asset) { + return std::string(kind_name(asset.kind)) + " " + (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : asset.name); +} + +struct Entry { + const fb::BundleAsset* asset{}; + fb::BundleFileInfo file; +}; +// One copy of a bundle: its asset list, and the assets in region order (ebx, resources, chunks). +struct Bundle { + std::string name; + std::optional listing; + std::vector assets; + std::map at; // asset key -> index into assets +}; +struct Toc { + std::string relative; // Win32/..., generic separators + fb::TocDocument document; + std::vector bundles; + std::map bundleAt; // lower-case name + std::set chunks; +}; + +// Where a payload really lives: 0 the game, 1 the merged patch, 2 and up a mod. +struct Place { + int layer{}; + std::uint32_t chunk{}; + std::uint16_t archive{}; + std::uint32_t offset{}, size{}; + auto operator<=>(const Place&) const = default; +}; + +struct EbxFacts { + bool parsed{}; + fb::Guid file; + std::string rootType; + std::vector imports; +}; + +// One mod's own copy of an asset that is not the game's copy of it. +struct Copy { + std::size_t mod{}; + fb::Sha1 sha1; + fb::BundleFileInfo file; +}; +// An asset at least one mod ships a copy of its own of, and what the patch has for it. +struct Contested { + std::string bundle, name; + fb::AssetKind kind{}; + bool shipped{}; // the game has this asset in this bundle + fb::Sha1 game, merged; + Place gamePlace, mergedPlace; + std::vector copies; +}; + +void walk(const fb::ebx::Value& value, const std::function& visit) { + if (const auto* pointer = std::get_if(&value.data)) { visit(*pointer); return; } + if (const auto* object = std::get_if>(&value.data)) { + if (*object) for (const auto& field : (*object)->fields) walk(field.value, visit); + return; + } + if (const auto* array = std::get_if(&value.data)) + for (const auto& element : *array) walk(element, visit); +} + +// What a list entry refers to, so the same entry can be found in another copy of the document; +// empty for an entry that is not a reference to something with a guid. +std::string identity(const fb::ebx::Document& document, const fb::ebx::Value& value) { + const auto* pointer = std::get_if(&value.data); + if (!pointer || pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) return {}; + const auto index = static_cast(pointer->index); + if (pointer->kind == fb::ebx::PointerKind::external) + return index < document.imports.size() + ? "x:" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string{}; + return index < document.instances.size() && document.instances[index].exported + ? "i:" + document.instances[index].instanceGuid.string() : std::string{}; +} +// The document a list entry names, when it names one in another file. +const fb::Guid* named(const fb::ebx::Document& document, const fb::ebx::Value& value) { + const auto* pointer = std::get_if(&value.data); + if (!pointer || pointer->kind != fb::ebx::PointerKind::external || pointer->index < 0 || + static_cast(pointer->index) >= document.imports.size()) return nullptr; + return &document.imports[static_cast(pointer->index)].fileGuid; +} + +// Every value of a document by where it is: its instance (the root, an exported one's guid, an +// internal one's place among the internal ones), then the fields and list positions down to it. +// A plain value is spelled with a lower-case letter in front; a reference (what it leads to) and +// a list's length are spelled with an upper-case one, and are the document's shape. +struct Flat { + std::map values; + std::size_t internal{}; // internal instances: a place is all one is known by +}; +bool plain(const std::string& spelled) { return !spelled.empty() && spelled.front() >= 'a' && spelled.front() <= 'z'; } +void flatten(const fb::ebx::Document& document, const std::vector& instances, + const fb::ebx::Value& value, const std::string& path, Flat& out) { + const auto& data = value.data; + if (const auto* flag = std::get_if(&data)) out.values[path] = *flag ? "b1" : "b0"; + else if (const auto* number = std::get_if(&data)) out.values[path] = "i" + std::to_string(*number); + else if (const auto* count = std::get_if(&data)) out.values[path] = "u" + std::to_string(*count); + else if (const auto* real = std::get_if(&data)) out.values[path] = "d" + std::to_string(std::bit_cast(*real)); + else if (const auto* text = std::get_if(&data)) out.values[path] = "s" + *text; + else if (const auto* guid = std::get_if(&data)) out.values[path] = "g" + guid->string(); + else if (const auto* sha = std::get_if(&data)) out.values[path] = "h" + hex(*sha); + else if (const auto* resource = std::get_if(&data)) out.values[path] = "r" + std::to_string(resource->id); + else if (const auto* pointer = std::get_if(&data)) { + const auto index = pointer->index < 0 ? std::size_t{} : static_cast(pointer->index); + if (pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) out.values[path] = "P"; + else if (pointer->kind == fb::ebx::PointerKind::external) + out.values[path] = index < document.imports.size() + ? "P" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string("P?"); + else out.values[path] = index < instances.size() ? "P" + instances[index] : std::string("P?"); + } else if (const auto* object = std::get_if>(&data)) { + if (*object) for (const auto& field : (*object)->fields) flatten(document, instances, field.value, path + "/" + field.name, out); + } else if (const auto* array = std::get_if(&data)) { + out.values[path + "/#"] = "N" + std::to_string(array->size()); + for (std::size_t index = 0; index < array->size(); ++index) + flatten(document, instances, (*array)[index], path + "/" + std::to_string(index), out); + } +} +Flat flatten(const fb::ebx::Document& document) { + Flat out; + std::vector instances; + for (std::size_t index = 0; index < document.instances.size(); ++index) { + const auto& instance = document.instances[index]; + instances.push_back(index == 0 ? std::string("root") + : instance.exported ? "x:" + instance.instanceGuid.string() : "n:" + std::to_string(out.internal++)); + } + for (std::size_t index = 0; index < document.instances.size(); ++index) + if (const auto& object = document.instances[index].object) + for (const auto& field : object->fields) flatten(document, instances, field.value, instances[index] + "/" + field.name, out); + return out; +} + +struct Audit { + fs::path root, base, out; + mods::Catalog catalog; + vfs::Layout baseLayout, mergedLayout; + std::unique_ptr store; + // A deque: bundles hold pointers into their own asset lists, which must not move. + std::deque tocs; + std::map tocAt; // lower-case relative path + // The game's bundles by name, wherever they are, to tell a mod's own content from the game's + // copy of something it carries. + std::map gameBundles; + std::map> gameListings; + std::map facts; + std::map guids; + std::vector contested; + // The patch's bundles (by TOC and bundle) that a mod ships a readable copy of: the ones whose + // chunk metadata the merge may have written. + std::set> chunkBundles; + // Those of them the patch has exactly as one mod ships them, chunks and list: the merge wrote + // nothing there. + std::set> chunkBundlesAsShipped; + std::chrono::steady_clock::time_point started = std::chrono::steady_clock::now(); + + void progress(const std::string& what) const { + const auto seconds = std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count(); + std::cerr << "[" << seconds << "s] " << what << std::endl; + } + const fs::path& layer(int index) const { + return index == 0 ? base : index == 1 ? out : catalog.mods.at(static_cast(index - 2)).directory; + } + // A placement that is not a patch one is in the game, whichever layer's bundle names it. + Place place(int from, const fb::BundleFileInfo& file) const { + return {file.location.patch ? from : 0, file.location.installChunk, file.location.archive, file.offset, file.size}; + } + Bytes payload(const Place& where) const { + fb::CasDecodeOptions options{root}; + options.maximumOutputSize = 2048ULL * 1024ULL * 1024ULL; + return fb::decode_cas(store->read(layer(where.layer), {where.layer != 0, where.chunk, where.archive}, where.offset, where.size), options); + } + static EbxFacts facts_of(const Bytes& bytes) { + EbxFacts result; + try { + const auto document = fb::ebx::read_document(bytes); + result.parsed = true; + result.file = document.fileGuid; + result.rootType = document.rootType; + result.imports = document.imports; + } catch (const std::exception&) {} + return result; + } + const EbxFacts& ebx(const Place& where) { + if (const auto found = facts.find(where); found != facts.end()) return found->second; + EbxFacts result; + try { result = facts_of(payload(where)); } catch (const std::exception&) {} + return facts.emplace(where, std::move(result)).first->second; + } + const fb::Guid* guid(const Place& where) { + if (const auto found = guids.find(where); found != guids.end()) return &found->second; + if (const auto known = facts.find(where); known != facts.end() && known->second.parsed) + return &guids.emplace(where, known->second.file).first->second; + try { return &guids.emplace(where, fb::ebx::read_file_guid(payload(where))).first->second; } + catch (const std::exception&) { return nullptr; } + } + + Bundle read_bundle(int from, const fb::TocBundle& bundle) const { + Bundle result; + result.name = bundle.name; + try { + result.listing = list_bundle(*store, layer(from), base, bundle, root); + if (!result.listing) return result; + std::size_t at = result.listing->first; + for (const auto* list : {&result.listing->manifest.ebx, &result.listing->manifest.resources, &result.listing->manifest.chunks}) + for (const auto& asset : *list) { + if (at >= result.listing->files.size()) throw std::runtime_error("more assets than files"); + result.at.emplace(key_of(asset), result.assets.size()); + result.assets.push_back({&asset, result.listing->files[at++]}); + } + } catch (const std::exception&) { + result.listing.reset(); + result.assets.clear(); + result.at.clear(); + } + return result; + } + const Bundle* game_bundle(const std::string& name) { + const auto key = lower(name); + if (const auto found = gameListings.find(key); found != gameListings.end()) return found->second ? &*found->second : nullptr; + auto& slot = gameListings[key]; + if (const auto shipped = gameBundles.find(key); shipped != gameBundles.end()) + if (auto bundle = read_bundle(0, shipped->second); bundle.listing) slot = std::move(bundle); + return slot ? &*slot : nullptr; + } + // The guid of every document a copy of a bundle holds. + std::set held(int from, const Bundle& bundle, bool& unsure) { + std::set result; + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + if (const auto* id = guid(place(from, entry.file))) result.insert(*id); else unsure = true; + } + return result; + } + // One record of a bundle's chunk metadata: whose the chunk is, and a texture's first mip. + struct ChunkMeta { + std::optional owner; + std::optional firstMip; + }; + // A copy's chunk metadata, record by record; nothing when the list cannot be read. + static std::optional> records_of(const Bundle& bundle) { + std::vector records; + const auto& list = bundle.listing->manifest.chunkMetadata; + if (list.empty()) return records; + try { + const std::span bytes(reinterpret_cast(list.data()), list.size()); + const auto root = native_db::read(bytes, "chunk metadata", nullptr, {.unique_fields = false, .max_entries = bytes.size()}); + for (const auto& row : root.children) { + ChunkMeta record; + if (const auto* hash = row.field("h64"); hash && hash->payload().size() == 8) + std::memcpy(&record.owner.emplace(), hash->payload().data(), 8); + if (const auto* meta = row.field("meta")) + if (const auto* mip = meta->field("firstMip"); mip && mip->payload().size() == 4) + std::memcpy(&record.firstMip.emplace(), mip->payload().data(), 4); + records.push_back(record); + } + } catch (const std::exception&) { + return std::nullopt; + } + return records; + } + // The hash a record names a resource by: of its lower-case name. + static std::uint64_t owner_hash(std::string_view name) { + std::uint64_t value = 5381; + for (const auto c : name) value = value * 33U ^ static_cast(std::tolower(static_cast(c))); + return value; + } + // The textures of a copy of a bundle whose pixels are one of the bundle's chunks, each against + // the record at its chunk's place when the chunks are in the order of their guids: how many + // there are, how many have another resource's record (or none) there, how many their own with + // another first mip than the texture's header gives, and one of those that are wrong. + struct TextureRecords { + std::size_t textures{}, misplaced{}, mips{}; + std::string example; + }; + TextureRecords texture_records(int from, const Bundle& bundle, const std::vector& records) const { + constexpr std::uint32_t textureType = 0x6BDE20BA; + const auto& manifest = bundle.listing->manifest; + std::vector order(manifest.chunks.size()); + for (std::size_t index = 0; index < order.size(); ++index) order[index] = index; + std::ranges::sort(order, [&](std::size_t left, std::size_t right) { return manifest.chunks[left].guid < manifest.chunks[right].guid; }); + std::map guidPlace; + for (std::size_t at = 0; at < order.size(); ++at) guidPlace.emplace(manifest.chunks[order[at]].guid, at); + TextureRecords result; + for (std::size_t index = 0; index < manifest.resources.size(); ++index) { + const auto& resource = manifest.resources[index]; + if (resource.resourceType != textureType || resource.resourceMeta.size() < 4) continue; + Bytes header; + try { header = payload(place(from, bundle.assets[manifest.ebx.size() + index].file)); } catch (const std::exception&) { continue; } + if (header.size() != 144 && header.size() != 180) continue; + // The header: type and format, a version's extra field, then flags and sizes, the mip + // count and first mip, padding, and the chunk the pixels are in. + std::uint32_t version{}; + std::memcpy(&version, resource.resourceMeta.data(), 4); + const std::size_t mips = 8 + 4 + 4 + (version >= 12 ? 4 : 0) + 10; + const auto firstMip = static_cast(header[mips + 1]); + fb::Guid chunk; + std::memcpy(chunk.bytes.data(), header.data() + mips + 2 + (header.size() == 180 ? 8 : 4), 16); + const auto at = guidPlace.find(chunk); + if (at == guidPlace.end()) continue; // its pixels are not in this bundle + ++result.textures; + const auto* record = at->second < records.size() ? &records[at->second] : nullptr; + if (!record || record->owner != owner_hash(resource.name)) { + if (!result.misplaced++ && result.example.empty()) result.example = resource.name + " has " + (record ? "another resource's record" : "no record") + " at its chunk's place"; + } else if (record->firstMip != firstMip) { + if (!result.mips++ && result.example.empty()) + result.example = resource.name + " is told first mip " + (record->firstMip ? std::to_string(*record->firstMip) : std::string("none")) + + ", its header says " + std::to_string(firstMip); + } + } + return result; + } + std::string mod_names(const std::vector& list) const { + std::set unique; + for (const auto index : list) unique.insert(catalog.mods[index].name); + std::string text; + for (const auto& name : unique) text += (text.empty() ? "" : ", ") + name; + return text; + } + + // ---------------------------------------------------------------- the merge, and reading what it wrote + bool merge() { + section("Merge"); + catalog = mods::load_catalog(root); + std::cout << catalog.mods.size() << " mod(s) merged, " << catalog.excluded.size() << " left out, " + << catalog.disabled.size() << " disabled; " << catalog.notes.size() << " note(s)\n"; + if (!catalog.issue.empty()) fail("merge", catalog.issue); + for (const auto& warning : catalog.warnings) fail("left out", warning); + if (!catalog.merged) fail("merge", "no merged patch was produced"); + return catalog.merged; + } + + void load() { + base = root / L"Data"; + out = root / L"Mods" / mods::generated_folder; + baseLayout = vfs::read_layout(base / L"layout.toc"); + mergedLayout = vfs::read_layout(out / L"layout.toc"); + store = std::make_unique(base, fs::temp_directory_path() / L"reskate-merge-audit-unused", mergedLayout.root); + + section("Superbundles"); + const auto toc_files = [](const fs::path& under) { + std::vector files; + std::error_code error; + for (fs::recursive_directory_iterator it(under / L"Win32", error), end; it != end && !error; it.increment(error)) + if (it->is_regular_file(error) && lower(it->path().extension().string()) == ".toc") files.push_back(it->path()); + std::ranges::sort(files); + return files; + }; + std::error_code error; + std::size_t bundles{}, chunks{}, unreadable{}; + for (const auto& file : toc_files(out)) { + Toc toc; + toc.relative = fs::relative(file, out, error).generic_string(); + try { + toc.document = fb::read_toc(read_file(file)); + fb::verify_toc(toc.document); + } catch (const std::exception& failure) { + fail("toc", toc.relative + ": " + failure.what()); + continue; + } + toc.bundles.reserve(toc.document.bundles.size()); + for (const auto& bundle : toc.document.bundles) { + if (!toc.bundleAt.emplace(lower(bundle.name), toc.bundles.size()).second) + fail("toc", toc.relative + " lists " + bundle.name + " twice"); + toc.bundles.push_back(read_bundle(1, bundle)); + unreadable += !toc.bundles.back().listing; + ++bundles; + } + for (const auto& chunk : toc.document.chunks) + if (!chunk.removed) toc.chunks.insert(chunk.guid); + chunks += toc.document.chunks.size(); + tocAt.emplace(lower(toc.relative), tocs.size()); + tocs.push_back(std::move(toc)); + } + std::cout << "the patch: " << tocs.size() << " superbundle TOC(s), " << bundles << " bundle(s) (" << unreadable + << " without a readable asset list), " << chunks << " TOC chunk(s)\n"; + std::size_t shipped{}; + for (const auto& file : toc_files(base)) { + ++shipped; + try { + for (auto& bundle : fb::read_toc(read_file(file)).bundles) gameBundles.try_emplace(lower(bundle.name), std::move(bundle)); + } catch (const std::exception&) {} + } + std::cout << "the game: " << shipped << " superbundle TOC(s), " << gameBundles.size() << " bundle name(s)\n"; + } + + // ---------------------------------------------------------------- placements and the layout + void placements() { + section("Placements and layout"); + using Archive = std::pair; + const auto declared_in = [](const native_db::Node& layout) { + std::set result; + for (const auto* field : {"layeredInstallChunkFiles", "unlayeredInstallChunkFiles"}) + if (const auto* node = layout.field(field); node && node->type == 19) + vfs::for_each_install_chunk_file(node->payload(), [&](std::uint32_t id, std::uint16_t archive) { result.emplace(id, archive); }); + return result; + }; + const auto declared = declared_in(mergedLayout.root), shipped = declared_in(baseLayout.root); + std::set used; + std::size_t checked{}; + const auto check = [&](const std::string& what, const fb::CasIdentifier& location, std::uint32_t offset, std::uint32_t size) { + ++checked; + const auto on_disk = store->archive_size(location.patch ? out : base, location); + if (!on_disk) { + fail("placement", what + " is in " + store->describe(location) + (location.patch ? " of the patch" : " of the game") + ", which is not there"); + return; + } + if (static_cast(offset) + size > *on_disk) + fail("placement", what + ": " + std::to_string(size) + " bytes at " + std::to_string(offset) + " run past the end of " + + store->describe(location) + " (" + std::to_string(*on_disk) + " bytes)"); + if (location.patch) used.emplace(location.installChunk, location.archive); + }; + for (const auto& toc : tocs) { + for (const auto& bundle : toc.document.bundles) { + const auto region = fb::read_bundle_region(bundle.region); + for (std::size_t index = 0; index < region.files.size(); ++index) + check(toc.relative + ": " + bundle.name + " file " + std::to_string(index), region.files[index].location, + region.files[index].offset, region.files[index].size); + } + for (const auto& chunk : toc.document.chunks) + if (!chunk.removed) check(toc.relative + ": chunk " + chunk.guid.string(), chunk.location, chunk.offset, chunk.size); + } + // An archive the layout does not declare is never opened; one it declares and the patch + // lacks sends the engine looking for a file that is absent. + for (const auto& archive : used) + if (!declared.contains(archive)) + fail("layout", "the patch reads archive " + std::to_string(archive.second) + " of install chunk " + + std::to_string(archive.first) + ", which the merged layout does not declare"); + std::size_t added{}; + for (const auto& archive : declared) { + if (shipped.contains(archive)) continue; + ++added; + if (!store->archive_size(out, {true, archive.first, archive.second})) + fail("layout", "the merged layout declares archive " + std::to_string(archive.second) + " of install chunk " + + std::to_string(archive.first) + ", and the patch has no such file"); + } + std::cout << checked << " placement(s) checked; " << used.size() << " patch archive(s) in use, " << added << " declared beyond the game's\n"; + + // The superbundle list is searched by name, so it stays in order; a mod's own superbundles + // have to be in it, in an install chunk, and have a TOC. + const auto names = [](const native_db::Node& layout) { + std::vector result; + if (const auto* list = layout.field("superBundles")) + for (const auto& row : list->children) + if (const auto* name = row.field("name"); name && name->type == 7) result.push_back(lower(name->text)); + return result; + }; + const auto merged = names(mergedLayout.root), game = names(baseLayout.root); + const std::set known(merged.begin(), merged.end()), stock(game.begin(), game.end()); + if (!std::ranges::is_sorted(merged)) fail("layout", "the merged superbundle list is not in name order"); + if (known.size() != merged.size()) fail("layout", "the merged superbundle list names one twice"); + std::set placed; + if (const auto* manifest = mergedLayout.root.field("installManifest")) + if (const auto* chunks = manifest->field("installChunks")) + for (const auto& chunk : chunks->children) + if (const auto* list = chunk.field("superbundles")) + for (const auto& entry : list->children) + if (entry.type == 7) placed.insert(lower(entry.text)); + std::size_t own{}; + for (const auto& mod : catalog.mods) { + if (!mod.provides_layout) continue; + try { + const auto layout = vfs::read_layout(mod.directory / L"layout.toc"); + for (const auto& name : names(layout.root)) { + if (stock.contains(name)) continue; + ++own; + if (!known.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is not in the merged layout"); + if (!placed.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is in no install chunk of the merged layout"); + if (!tocAt.contains(name + ".toc")) fail("layout", mod.name + ": its superbundle " + name + " has no TOC in the patch"); + } + } catch (const std::exception& failure) { + fail("layout", mod.name + ": layout.toc could not be read (" + failure.what() + ")"); + } + } + std::cout << merged.size() << " superbundle(s) in the merged layout, " << own << " listed by mods beyond the game's\n"; + } + + // ---------------------------------------------------------------- asset lists and payloads + void payloads(const fs::path& listing) { + section("Asset lists and payloads"); + // What a listed size means is learnt from the game's own assets first: only a kind whose + // every sample agrees is held to it. + std::array sampled{}, agreed{}; + std::set seen; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) + for (const auto& entry : bundle.assets) { + const auto kind = static_cast(entry.asset->kind); + const auto where = place(1, entry.file); + if (where.layer != 0 || sampled[kind] >= 150 || entry.file.size > (8u << 20) || !seen.insert(where).second) continue; + ++sampled[kind]; + try { agreed[kind] += payload(where).size() == entry.asset->originalSize; } catch (const std::exception&) {} + } + std::array sized{}; + for (std::size_t kind = 0; kind < 3; ++kind) sized[kind] = sampled[kind] >= 20 && agreed[kind] == sampled[kind]; + + std::map> stored; + std::size_t listed{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) { + if (bundle.at.size() != bundle.assets.size()) { + std::set names; + std::string example; + for (const auto& entry : bundle.assets) + if (!names.insert(key_of(*entry.asset)).second && example.empty()) example = name_of(*entry.asset); + fail("asset list", toc.relative + ": " + bundle.name + " lists an asset twice, e.g. " + example); + } + for (const auto& entry : bundle.assets) { + ++listed; + if (const auto where = place(1, entry.file); where.layer == 1) stored.try_emplace(where, &entry, toc.relative + ": " + bundle.name); + } + } + std::cout << listed << " asset(s) listed across the patch; " << stored.size() << " payload(s) are stored in the patch's own archives\n"; + progress("decoding " + std::to_string(stored.size()) + " payloads"); + std::ofstream written; + if (!listing.empty()) written.open(listing, std::ios::binary); + std::array decoded{}; + std::size_t bytesIn{}, bytesOut{}, unparsed{}; + for (const auto& [where, what] : stored) { + const auto& [entry, from] = what; + const auto kind = static_cast(entry->asset->kind); + try { + const auto bytes = payload(where); + ++decoded[kind]; + bytesIn += where.size; + bytesOut += bytes.size(); + if (sized[kind] && bytes.size() != entry->asset->originalSize) + fail("payload", from + ": " + name_of(*entry->asset) + " decodes to " + std::to_string(bytes.size()) + + " bytes, its asset list says " + std::to_string(entry->asset->originalSize)); + if (entry->asset->kind == fb::AssetKind::ebx) { + auto result = facts_of(bytes); + if (!result.parsed) { ++unparsed; fail("payload", from + ": " + name_of(*entry->asset) + " is not a document the merge can read"); } + facts.emplace(where, std::move(result)); + } + if (written.is_open()) written << from << '\t' << name_of(*entry->asset) << '\t' << bytes.size() << '\t' << hex(sha1_of(bytes)) << '\n'; + } catch (const std::exception& failure) { + fail("payload", from + ": " + name_of(*entry->asset) + " at " + store->describe({true, where.chunk, where.archive}) + "+" + + std::to_string(where.offset) + " cannot be decoded (" + failure.what() + ")"); + } + } + std::cout << decoded[0] << " ebx, " << decoded[1] << " res and " << decoded[2] << " chunk payload(s) decoded: " << bytesIn / (1024 * 1024) + << " MB stored, " << bytesOut / (1024 * 1024) << " MB decoded; " << unparsed << " EBX do not parse\n"; + } + + // ---------------------------------------------------------------- every mod against the patch + void mods_against_patch() { + section("Every mod against the patch"); + std::map found; // by bundle name, asset and what the patch has + std::size_t bundles{}, assets{}, chunks{}, byName{}; + for (std::size_t index = 0; index < catalog.mods.size(); ++index) { + const auto& mod = catalog.mods[index]; + if (!mod.provides_layout) continue; + if (index % 20 == 0) progress("mod " + std::to_string(index + 1) + " of " + std::to_string(catalog.mods.size())); + for (const auto& relative : scan(mod.directory).tocs) { + const auto merged = tocAt.find(lower(relative)); + if (merged == tocAt.end()) { fail("mod content", mod.name + ": the patch has no " + relative); continue; } + const auto& toc = tocs[merged->second]; + fb::TocDocument own; + try { own = fb::read_toc(read_file(mod.directory / fs::path(relative))); } + catch (const std::exception& failure) { fail("mod content", mod.name + ": " + relative + " cannot be read (" + failure.what() + ")"); continue; } + for (const auto& chunk : own.chunks) { + if (chunk.removed) continue; + ++chunks; + if (!toc.chunks.contains(chunk.guid)) + fail("mod content", mod.name + ": " + relative + ": its chunk " + chunk.guid.string() + " is not in the patch's TOC"); + } + for (const auto& bundle : own.bundles) { + ++bundles; + const auto at = toc.bundleAt.find(lower(bundle.name)); + if (at == toc.bundleAt.end()) { fail("mod content", mod.name + ": " + relative + ": its bundle " + bundle.name + " is not in the patch"); continue; } + const auto& patched = toc.bundles[at->second]; + const auto mine = read_bundle(static_cast(index + 2), bundle); + if (!mine.listing || !patched.listing) { ++byName; continue; } + const auto* game = game_bundle(bundle.name); + chunkBundles.emplace(merged->second, at->second); + if (mine.listing->manifest.chunkMetadata == patched.listing->manifest.chunkMetadata && + std::ranges::equal(mine.listing->manifest.chunks, patched.listing->manifest.chunks, {}, + &fb::BundleAsset::guid, &fb::BundleAsset::guid)) + chunkBundlesAsShipped.emplace(merged->second, at->second); + for (const auto& entry : mine.assets) { + ++assets; + const auto key = key_of(*entry.asset); + const auto there = patched.at.find(key); + if (there == patched.at.end()) { + fail("mod content", mod.name + ": " + relative + ": " + bundle.name + ": its " + name_of(*entry.asset) + " is not in the patch's copy"); + continue; + } + const auto& kept = patched.assets[there->second]; + const Entry* shipped{}; + if (game) if (const auto it = game->at.find(key); it != game->at.end()) shipped = &game->assets[it->second]; + // The game's copy carried along is not this mod's content. + if (shipped && shipped->asset->sha1 == entry.asset->sha1) continue; + auto& record = found[lower(bundle.name) + '|' + key + '|' + hex(kept.asset->sha1)]; + if (record.copies.empty()) { + record.bundle = bundle.name; + record.name = entry.asset->name; + record.kind = entry.asset->kind; + record.shipped = shipped != nullptr; + if (shipped) { record.game = shipped->asset->sha1; record.gamePlace = place(0, shipped->file); } + record.merged = kept.asset->sha1; + record.mergedPlace = place(1, kept.file); + } + if (std::ranges::none_of(record.copies, [&](const Copy& copy) { return copy.mod == index && copy.sha1 == entry.asset->sha1; })) + record.copies.push_back({index, entry.asset->sha1, entry.file}); + } + } + } + } + std::cout << bundles << " mod bundle(s), " << assets << " listed asset(s) and " << chunks << " TOC chunk(s) looked up in the patch; " + << byName << " bundle(s) could only be checked by name\n"; + for (auto& [key, record] : found) contested.push_back(std::move(record)); + } + + // ---------------------------------------------------------------- what became of each mod's own content + void outcomes() { + section("What became of each mod's own content"); + struct Clash { const Contested* asset; std::vector kept, lost; }; + std::vector clashes, overridden; + std::vector written; + std::size_t asShipped{}, rewritten{}; + for (const auto& record : contested) { + std::set versions; + for (const auto& copy : record.copies) versions.insert(copy.sha1); + const bool one = versions.contains(record.merged); + if (one && versions.size() == 1) { ++asShipped; continue; } + if (record.shipped && record.merged == record.game) { + fail("lost edit", record.bundle + ": " + record.name + ": " + std::to_string(record.copies.size()) + " mod(s) change it (e.g. " + + catalog.mods[record.copies.front().mod].name + ") and the patch has the game's own copy"); + continue; + } + if (!one) { + // No one mod's copy: the merge wrote it (edits combined, a change carried in). + if (record.kind == fb::AssetKind::ebx) written.push_back(&record); else ++rewritten; + continue; + } + Clash clash{&record, {}, {}}; + for (const auto& copy : record.copies) (copy.sha1 == record.merged ? clash.kept : clash.lost).push_back(copy.mod); + (record.shipped ? overridden : clashes).push_back(std::move(clash)); + } + std::cout << asShipped << " asset(s) are in the patch as their one mod ships them; the merge wrote " << written.size() + << " EBX document(s) and " << rewritten << " other asset(s) itself\n"; + + std::cout << "\nassets two mods add under one name, with different content: " << clashes.size() << '\n'; + std::map> pairs; + for (const auto& clash : clashes) pairs[mod_names(clash.lost) + " lost them to " + mod_names(clash.kept)].push_back(&clash); + for (const auto& [pair, list] : pairs) + std::cout << " " << pair << ": " << list.size() << ", e.g. " << list.front()->asset->name << " in " << list.front()->asset->bundle << '\n'; + + std::cout << "assets of the game two mods change, where the patch has one mod's copy whole: " << overridden.size() << '\n'; + std::size_t printed{}; + for (const auto& clash : overridden) { + if (++printed > 40) { std::cout << " ... and " << overridden.size() - 40 << " more\n"; break; } + std::cout << " " << kind_name(clash.asset->kind) << ' ' << clash.asset->name << " (" << clash.asset->bundle << "): kept " + << mod_names(clash.kept) << "; not in it: " << mod_names(clash.lost) << '\n'; + } + + // A reference is by the guid of the document it names. Where two mods' assets shared a + // name and one went, whatever still names the one that went finds nothing. + section("References to documents that lost their name to another mod"); + struct Gone { std::string mod, name, keptBy; }; + std::map gone; + for (const auto& clash : clashes) { + if (clash.asset->kind != fb::AssetKind::ebx) continue; + const auto& kept = ebx(clash.asset->mergedPlace); + for (const auto& copy : clash.asset->copies) { + if (copy.sha1 == clash.asset->merged) continue; + const auto& lost = ebx(place(static_cast(copy.mod + 2), copy.file)); + // The same guid under both copies still resolves, to the copy that stayed. + if (lost.parsed && kept.parsed && lost.file != kept.file) + gone.emplace(lost.file, Gone{catalog.mods[copy.mod].name, clash.asset->name, mod_names(clash.kept)}); + } + } + std::map> holders; + std::size_t references{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + const auto where = place(1, entry.file); + if (where.layer != 1) continue; + for (const auto& import : ebx(where).imports) { + const auto lost = gone.find(import.fileGuid); + if (lost == gone.end()) continue; + ++references; + auto& [count, example] = holders[bundle.name + ": " + entry.asset->name + " names " + lost->second.mod + "'s " + + lost->second.name + ", which " + lost->second.keptBy + "'s replaced"]; + if (!count++) example = toc.relative; + } + } + for (const auto& [text, where] : holders) + fail("dangling reference", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + std::cout << gone.size() << " document(s) went with a guid nothing in the patch has; " << references << " reference(s) in the patch still name one\n"; + + section("Documents the merge wrote"); + std::size_t checked{}, lists{}, kept{}, inPlace{}, valueEdits{}, reshaped{}; + for (const auto* record : written) { + fb::ebx::Document merged; + try { merged = fb::ebx::read_document(payload(record->mergedPlace)); } + catch (const std::exception& failure) { fail("merged document", record->bundle + ": " + record->name + " cannot be read back (" + failure.what() + ")"); continue; } + ++checked; + const auto label = record->bundle + ": " + record->name + " (" + merged.rootType + ")"; + // The engine binary-searches the exported instances after the root by guid, and + // internal ones have to follow them. + bool internal{}; + const fb::Guid* previous{}; + for (std::size_t index = 1; index < merged.instances.size(); ++index) { + const auto& instance = merged.instances[index]; + if (!instance.exported) { internal = true; continue; } + if (internal) { fail("merged document", label + ": an exported instance follows an internal one"); break; } + if (previous && !(*previous < instance.instanceGuid)) { fail("merged document", label + ": its exported instances are not in guid order"); break; } + previous = &instance.instanceGuid; + } + std::size_t stray{}; + for (const auto& instance : merged.instances) + if (instance.object) + for (const auto& field : instance.object->fields) + walk(field.value, [&](const fb::ebx::PointerReference& pointer) { + const auto limit = pointer.kind == fb::ebx::PointerKind::internal ? merged.instances.size() + : pointer.kind == fb::ebx::PointerKind::external ? merged.imports.size() : std::size_t{}; + stray += pointer.kind != fb::ebx::PointerKind::null && (pointer.index < 0 || static_cast(pointer.index) >= limit); + }); + if (stray) fail("merged document", label + ": " + std::to_string(stray) + " reference(s) index past the instance or import table"); + if (!record->shipped) continue; + fb::ebx::Document game; + try { game = fb::ebx::read_document(payload(record->gamePlace)); } catch (const std::exception&) { continue; } + if (!merged.root() || !merged.root()->object || !game.root() || !game.root()->object) continue; + // Every reference a mod appended to one of the root's lists is in the merged list, + // unless what it names lost its name to another mod's asset. + std::set done; + for (const auto& copy : record->copies) { + if (!done.insert(copy.sha1).second) continue; + fb::ebx::Document edit; + try { edit = fb::ebx::read_document(payload(place(static_cast(copy.mod + 2), copy.file))); } catch (const std::exception&) { continue; } + if (!edit.root() || !edit.root()->object) continue; + for (const auto& field : edit.root()->object->fields) { + const auto* added = std::get_if(&field.value.data); + const auto* before = game.root()->object->find(field.name); + const auto* after = merged.root()->object->find(field.name); + const auto* beforeList = before ? std::get_if(&before->value.data) : nullptr; + const auto* afterList = after ? std::get_if(&after->value.data) : nullptr; + if (!added || !beforeList || !afterList || added->size() <= beforeList->size()) continue; + ++lists; + std::set present; + for (const auto& value : *afterList) present.insert(identity(merged, value)); + for (auto at = beforeList->size(); at < added->size(); ++at) { + const auto id = identity(edit, (*added)[at]); + if (id.empty()) continue; + if (present.contains(id)) { ++kept; continue; } + if (const auto* target = named(edit, (*added)[at]); target && gone.contains(*target)) continue; + fail("merged list", label + ": " + field.name + " lacks an entry " + catalog.mods[copy.mod].name + " added (" + id + ")"); + } + } + } + + // Values changed in place. What the merged document has that the game's does not is + // some mod's value, and a mod that changed values and nothing else has every one of + // them changed in the merged document: to its own, or to another mod's that changed + // the same one. + const auto before = flatten(game), after = flatten(merged); + // An internal instance is known by its place alone, which says nothing once one was + // added or taken away. + const auto told = [&](const Flat& other, const std::string& key) { + return !key.starts_with("n:") || other.internal == before.internal; + }; + std::vector> edits; + done.clear(); + for (const auto& copy : record->copies) { + if (!done.insert(copy.sha1).second) continue; + try { edits.emplace_back(copy.mod, flatten(fb::ebx::read_document(payload(place(static_cast(copy.mod + 2), copy.file))))); } + catch (const std::exception&) {} + } + std::size_t invented{}; + std::string example; + for (const auto& [key, value] : before.values) { + if (!plain(value) || !told(after, key)) continue; + const auto now = after.values.find(key); + if (now == after.values.end() || now->second == value) continue; + ++inPlace; + if (std::ranges::none_of(edits, [&](const std::pair& edit) { + const auto theirs = edit.second.values.find(key); + return told(edit.second, key) && theirs != edit.second.values.end() && theirs->second == now->second; })) + if (!invented++) example = key; + } + if (invented) + fail("merged value", label + ": " + std::to_string(invented) + " value(s) are neither the game's nor any mod's, e.g. " + example); + for (const auto& [mod, edit] : edits) { + bool shape{}; // a reference moved, a list resized, something of the game's gone + std::vector changed; + for (const auto& [key, value] : before.values) { + if (!told(edit, key)) continue; + const auto theirs = edit.values.find(key); + if (theirs == edit.values.end()) { shape = true; continue; } + if (theirs->second == value) continue; + if (plain(value) && plain(theirs->second)) { changed.push_back(&key); continue; } + // A list on the root that grew at its end is the one change of shape that + // is combined, and is checked above. + const bool rootList = key.starts_with("root/") && key.ends_with("/#") && std::ranges::count(key, '/') == 2; + if (!(rootList && std::stoull(theirs->second.substr(1)) > std::stoull(value.substr(1)))) shape = true; + } + if (changed.empty()) continue; + if (shape) { ++reshaped; continue; } + std::size_t lost{}; + for (const auto* key : changed) { + const auto now = after.values.find(*key); + if (told(after, *key) && (now == after.values.end() || now->second == before.values.at(*key)) && !lost++) example = *key; + } + if (lost) + fail("lost value", label + ": " + std::to_string(lost) + " of the " + std::to_string(changed.size()) + " value(s) " + + catalog.mods[mod].name + " changed are the game's again, e.g. " + example); + ++valueEdits; + } + } + std::cout << checked << " document(s) read back; " << lists << " list(s) a mod appended to, " << kept << " appended reference(s) found in them\n"; + std::cout << inPlace << " value(s) in them differ from the game's, each one some mod's; " << valueEdits + << " mod copy(ies) that changed values alone have them all in; " << reshaped + << " that also moved a reference or resized a list are not combined that way\n"; + } + + // ---------------------------------------------------------------- lists and what they name + // The game keeps what some lists name in the list's own bundle (an item list and its items, + // the music playlist and its songs), and reads such a list without checking an entry. So in + // every copy of that bundle the patch carries, the entries mods added have to be there too. + void lists() { + section("Lists whose entries the game keeps beside them"); + struct Shipped { fb::ebx::Document document; std::set held; bool unsure{}; }; + std::map> shipped; // by bundle and asset key + std::map> missing, twice; + std::size_t copies{}, entries{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) { + const auto* game = game_bundle(bundle.name); + if (!game) continue; + std::optional> here; + bool unsure{}; + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + const auto where = place(1, entry.file); + const auto key = key_of(*entry.asset); + const auto original = game->at.find(key); + if (where.layer != 1 || original == game->at.end()) continue; + auto& known = shipped[lower(bundle.name) + '|' + key]; + if (!known) { + Shipped value; + try { value.document = fb::ebx::read_document(payload(place(0, game->assets[original->second].file))); } catch (const std::exception&) {} + known = std::move(value); + } + const auto* gameRoot = known->document.root(); + if (!gameRoot || !gameRoot->object) continue; + // Only a document with a list of references to other documents is read at all. + const auto lists_references = std::ranges::any_of(gameRoot->object->fields, [&](const fb::ebx::FieldValue& field) { + const auto* list = std::get_if(&field.value.data); + return list && !list->empty() && named(known->document, list->front()); + }); + if (!lists_references) continue; + fb::ebx::Document merged; + try { merged = fb::ebx::read_document(payload(where)); } catch (const std::exception&) { continue; } + if (!merged.root() || !merged.root()->object) continue; + for (const auto& field : merged.root()->object->fields) { + const auto* list = std::get_if(&field.value.data); + const auto* before = gameRoot->object->find(field.name); + const auto* beforeList = before ? std::get_if(&before->value.data) : nullptr; + if (!list || !beforeList || beforeList->empty() || list->size() <= beforeList->size()) continue; + // Does the game keep every one of its own entries in this bundle? + if (known->held.empty() && !known->unsure) known->held = held(0, *game, known->unsure); + if (known->unsure || !std::ranges::all_of(*beforeList, [&](const fb::ebx::Value& value) { + const auto* target = named(known->document, value); + return target && known->held.contains(*target); })) continue; + if (!here) here = held(1, bundle, unsure); + if (unsure) continue; + ++copies; + std::set seen; + std::size_t absent{}, repeated{}; + for (std::size_t index = 0; index < list->size(); ++index) { + const auto* target = named(merged, (*list)[index]); + if (!target) { if (index >= beforeList->size()) ++absent; continue; } + if (index < beforeList->size()) continue; + ++entries; + absent += !here->contains(*target); + repeated += !seen.insert(*target).second; + } + const auto what = bundle.name + ": " + field.name + " of " + entry.asset->name; + if (absent) { auto& [count, example] = missing[what + ": " + std::to_string(absent) + " of " + std::to_string(list->size() - beforeList->size()) + + " added entr" + (absent == 1 ? "y names" : "ies name") + " a document that bundle does not hold"]; + if (!count++) example = toc.relative; } + if (repeated) { auto& [count, example] = twice[what + ": " + std::to_string(repeated) + " added entr" + (repeated == 1 ? "y repeats" : "ies repeat") + " one a mod already added"]; + if (!count++) example = toc.relative; } + } + } + } + for (const auto& [text, where] : missing) + fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + for (const auto& [text, where] : twice) + fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + std::cout << copies << " such list(s) with entries mods added, across every copy in the patch; " << entries << " added entries checked\n"; + } + + // ---------------------------------------------------------------- chunk metadata + // The engine reads a chunk's record before it streams the chunk. The game writes a bundle's + // list with a record for each chunk, in the order of the chunks' guids, so a texture's record + // (the hash of its name, and its first mip) is at its chunk's place in that order. A bundle + // the patch has from more than the game's copy (two mods' costumes built on one game costume, + // each with chunks of its own) has to be in that form as well: with fewer records than + // chunks, or with records where the mods' own lists have them, textures and meshes get + // another's record or none and never appear. Checked wherever the game's own copy of the + // bundle is in that form, which is everywhere it has been looked at. + void chunk_metadata() { + section("Chunk metadata"); + std::size_t bundles{}, untouched{}, asShipped{}, checked{}, textures{}, unknownForm{}; + for (const auto& where : chunkBundles) { + const auto& toc = tocs[where.first]; + const auto& patched = toc.bundles[where.second]; + const auto& kept = patched.listing->manifest.chunks; + if (kept.empty()) continue; + ++bundles; + const auto what = toc.relative + ": " + patched.name; + const auto* game = game_bundle(patched.name); + // The game's own list on the game's own chunks is what the game shipped. + if (game && game->listing->manifest.chunkMetadata == patched.listing->manifest.chunkMetadata && + game->listing->manifest.chunks.size() == kept.size() && + std::ranges::equal(game->listing->manifest.chunks, kept, {}, &fb::BundleAsset::guid, &fb::BundleAsset::guid)) { + ++untouched; + continue; + } + // One mod's copy passed through whole is that mod's to have right: its tool's list is + // not in the game's form, and the merge does not rewrite what only one mod ships. + if (chunkBundlesAsShipped.contains(where)) { ++asShipped; continue; } + const auto records = records_of(patched); + if (!records) { fail("chunk metadata", what + ": the patch's list cannot be read"); continue; } + if (records->size() != kept.size() && !(records->empty() && (!game || game->listing->manifest.chunkMetadata.empty()))) { + fail("chunk metadata", what + ": " + std::to_string(records->size()) + " record(s) for " + + std::to_string(kept.size()) + " chunk(s)"); + continue; + } + // Only a bundle whose own copy in the game is in guid order says what the patch's has to be. + if (!game) { ++unknownForm; continue; } + const auto shipped = records_of(*game); + if (!shipped) { ++unknownForm; continue; } + const auto original = texture_records(0, *game, *shipped); + if (original.misplaced || original.mips) { ++unknownForm; continue; } + ++checked; + const auto merged = texture_records(1, patched, *records); + textures += merged.textures; + if (merged.misplaced || merged.mips) + fail("chunk metadata", what + ": of " + std::to_string(merged.textures) + " texture(s) with a chunk in the bundle, " + + std::to_string(merged.misplaced) + " do not have their own record at their chunk's place and " + + std::to_string(merged.mips) + " have another first mip than their header, e.g. " + merged.example); + } + std::cout << bundles << " bundle(s) mods ship that hold chunks: " << untouched << " with the game's own list on the game's own chunks, " + << asShipped << " exactly as one mod ships them, " << checked << " others checked, " << textures + << " texture record(s) in them; " << unknownForm << " whose own copy in the game gives nothing to check against\n"; + } + + int summary() const { + section("Summary"); + std::size_t total{}; + for (const auto& [kind, count] : failures) { std::cout << count << " x " << kind << '\n'; total += count; } + std::cout << (total ? "mod merge audit: " + std::to_string(total) + " problem(s)" : std::string("mod merge audit: ok")) << '\n'; + return total ? 1 : 0; + } +}; +} // namespace + +int main(int argc, char** argv) try { + if (argc < 2 || !fs::exists(fs::path(argv[1]) / "Data") || !fs::exists(fs::path(argv[1]) / "Mods")) { + std::cout << "No data root with Data and Mods given; skipped.\n"; + return 0; + } + Audit audit; + audit.root = argv[1]; + if (!audit.merge()) return audit.summary(); + audit.load(); + audit.placements(); + audit.payloads(argc > 2 ? fs::path(argv[2]) : fs::path{}); + audit.mods_against_patch(); + audit.outcomes(); + audit.lists(); + audit.chunk_metadata(); + return audit.summary(); +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/mod_merge_unshift_tests.cpp b/Engine/Vfs/Test/mod_merge_unshift_tests.cpp index f92717c..e5f6513 100644 --- a/Engine/Vfs/Test/mod_merge_unshift_tests.cpp +++ b/Engine/Vfs/Test/mod_merge_unshift_tests.cpp @@ -12,6 +12,10 @@ int failures = 0; void check(bool condition, const char* what) { if (!condition) { std::cerr << "FAILED: " << what << "\n"; ++failures; } } +// A mod's archive: the package folder it is in and its number there. +std::pair of(const std::string& directory, std::uint16_t archive) { + return {directory, archive}; +} bool lands(const ArchivePlacement& placement, std::string_view directory, std::uint16_t archive, std::uint32_t offset, std::uint16_t expected_archive, std::uint32_t expected_offset) { return unshift(placement, directory, archive, offset) && archive == expected_archive && offset == expected_offset; @@ -24,8 +28,8 @@ int main() { // The usual launch placement: each archive at the start of an index of its own. ArchivePlacement own; - own.at[{initial, 1}] = {4, 0}; - own.at[{standard, 1}] = {9, 0}; + own.at[of(initial, 1)] = {4, 0}; + own.at[of(standard, 1)] = {9, 0}; check(lands(own, initial, 4, 1234, 1, 1234), "an archive with an index of its own keeps its offsets"); check(lands(own, standard, 9, 0, 1, 0), "the other package folder has its own blocks"); std::uint16_t archive = 9; @@ -35,9 +39,9 @@ int main() { // A mod added while the game runs: archives 1, 2 and 3 appended to the patch's archive 1. ArchivePlacement appended; - appended.at[{initial, 1}] = {1, 5000}; - appended.at[{initial, 2}] = {1, 9096}; - appended.at[{initial, 3}] = {1, 20000}; + appended.at[of(initial, 1)] = {1, 5000}; + appended.at[of(initial, 2)] = {1, 9096}; + appended.at[of(initial, 3)] = {1, 20000}; check(lands(appended, initial, 1, 5000, 1, 0), "the first block starts at its own byte 0"); check(lands(appended, initial, 1, 9095, 1, 4095), "the last byte of the first block is still the first archive"); check(lands(appended, initial, 1, 9096, 2, 0), "an offset in the second block is the second archive"); @@ -49,8 +53,8 @@ int main() { // An empty archive's block starts where the next one does; the data is in the next. ArchivePlacement empty_first; - empty_first.at[{initial, 1}] = {1, 700}; - empty_first.at[{initial, 2}] = {1, 700}; + empty_first.at[of(initial, 1)] = {1, 700}; + empty_first.at[of(initial, 2)] = {1, 700}; check(lands(empty_first, initial, 1, 700, 2, 0) && lands(empty_first, initial, 1, 900, 2, 200), "an empty archive's block yields to the one that follows it"); diff --git a/Engine/Vfs/Test/mod_music_tests.cpp b/Engine/Vfs/Test/mod_music_tests.cpp new file mode 100644 index 0000000..79495b4 --- /dev/null +++ b/Engine/Vfs/Test/mod_music_tests.cpp @@ -0,0 +1,87 @@ +// Checks how a mod's reskate-music.json is read: the playlist ids and names, which song entries are +// kept, the limits, and that a document of the wrong shape is refused rather than half-read. +#include "Engine/Vfs/mod_music.h" +#include +#include +#include + +namespace { +int failures = 0; +void check(bool condition, const std::string& message) { + if (!condition) { std::cerr << "FAIL: " << message << '\n'; ++failures; } +} + +using dingosdk::mods::parse_music_playlists; + +bool refused(const std::string& json) { + try { parse_music_playlists("mod", json); } catch (const std::exception&) { return true; } + return false; +} + +std::string playlist_of(const std::string& songs) { + return R"({"schema":1,"playlists":[{"name":"Mix","songs":)" + songs + "}]}"; +} +} + +int main() try { + { + const auto lists = parse_music_playlists("KevinMacLeod", R"({ + "schema": 1, + "playlists": [ + {"name": "Kevin MacLeod", "songs": ["Kevin MacLeod - Cipher", "Kevin MacLeod - Sneaky Snitch"]}, + {"name": "Second", "songs": []} + ]})"); + check(lists.size() == 2, "both playlists are read"); + check(lists.at(0).id == "mod:KevinMacLeod:Kevin MacLeod", "id is mod::"); + check(lists.at(0).name == "Kevin MacLeod", "name is the shown name"); + check(lists.at(0).songs == std::vector{"Kevin MacLeod - Cipher", "Kevin MacLeod - Sneaky Snitch"}, + "songs keep their order and spelling"); + check(lists.at(1).songs.empty(), "an empty playlist is kept (the catalog drops it when nothing matches)"); + } + { + const auto lists = parse_music_playlists("mod", playlist_of( + R"(["A - One", "", 7, null, "B - Tw\to", ")" + std::string(256, 'x') + R"(", "C - Three"])")); + check(lists.at(0).songs == std::vector{"A - One", "C - Three"}, + "empty, non-string, control-character and over-long entries are dropped, the rest kept"); + } + { + std::string songs = "["; + for (int i = 0; i < 1100; ++i) songs += (i ? "," : "") + std::string("\"A - ") + std::to_string(i) + "\""; + const auto lists = parse_music_playlists("mod", playlist_of(songs + "]")); + check(lists.at(0).songs.size() == 1024, "at most 1024 songs per playlist"); + + std::string many = R"({"schema":1,"playlists":[)"; + for (int i = 0; i < 70; ++i) many += (i ? "," : "") + std::string(R"({"name":"P)") + std::to_string(i) + R"(","songs":[]})"; + check(parse_music_playlists("mod", many + "]}").size() == 64, "at most 64 playlists"); + } + check(parse_music_playlists("mod", R"({"schema":1,"playlists":[]})").empty(), "no playlists is fine"); + { + const auto lists = parse_music_playlists("mod", R"({"schema":1,"playlists":[ + {"name":"Mix","artwork":"artwork/playlist.png","songs":["A - One","B - Two"]}], + "song_artwork":{"A - One":"artwork/track.png","B - Two":"../outside.png","Unlisted":"artwork/other.png"}})"); + check(lists[0].artwork == "artwork/playlist.png", "playlist cover is read"); + check(lists[0].song_artwork.size() == 1 && lists[0].song_artwork.at("A - One") == "artwork/track.png", + "song covers are limited to members and safe paths"); + for (const auto* path : {"../cover.png", "/cover.png", "C:/cover.png", "a\\cover.png", "a/../cover.png", + "a//cover.png", "a/./cover.png", "cover.png:stream", "a%2fb.png", "cover.jpg"}) + check(!dingosdk::mods::music_artwork_path(path), std::string("unsafe artwork path: ") + path); + const auto bad = parse_music_playlists("mod", R"({"schema":1,"playlists":[ + {"name":"Mix","artwork":42,"songs":["A - One"]}],"song_artwork":[]})"); + check(bad[0].artwork.empty() && bad[0].songs.size() == 1, "bad optional artwork does not discard music"); + } + check(refused(R"({"schema":2,"playlists":[]})"), "another schema version is refused"); + check(refused(R"({"playlists":[]})"), "a missing schema is refused"); + check(refused(R"({"schema":1})"), "missing playlists is refused"); + check(refused(R"({"schema":1,"playlists":{}})"), "playlists that is not an array is refused"); + check(refused(R"({"schema":1,"playlists":[{"songs":[]}]})"), "a playlist without a name is refused"); + check(refused(R"({"schema":1,"playlists":[{"name":"","songs":[]}]})"), "an empty name is refused"); + check(refused(R"({"schema":1,"playlists":[{"name":"X"}]})"), "a playlist without songs is refused"); + check(refused(R"({"schema":1,"playlists":[{"name":"X","songs":"A - B"}]})"), "songs that is not an array is refused"); + check(refused("not json"), "text that is not JSON is refused"); + + if (failures == 0) std::cout << "mod music tests passed\n"; + return failures == 0 ? 0 : 1; +} catch (const std::exception& error) { + std::cerr << "FAIL: uncaught: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/store_copies_tests.cpp b/Engine/Vfs/Test/store_copies_tests.cpp new file mode 100644 index 0000000..47b544d --- /dev/null +++ b/Engine/Vfs/Test/store_copies_tests.cpp @@ -0,0 +1,372 @@ +// Mods that add copies of items the game's store sells (Engine/Vfs/mod_store_copies.h). +// +// ReSkate never hands out a store item, and tells one by its key. The mods that got round that +// (2026-10-05) all did it the same way: each store item again under a key of its own, which is +// in no catalogue and so was handed out like any mod's item. The check tells a copy by what it +// is made of, and the merge leaves a mod that adds one out whole, saying only that it could +// not be merged. What it must not do is take a mod's own item for one: mods fit textures of +// their own on the game's meshes, a store item's included, and that is theirs. +// +// Arguments: [ [...]]. With the game and its content cache (which +// says what the store sells) the check is run on mods made here from the game's own items; the +// mods in each further folder are then checked and listed, to try it on real ones. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_carry.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_writer.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/content_cache.h" +#include "Engine/Vfs/content_catalogs.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/mod_merge.h" +#include "Engine/Vfs/mod_merge_internal.h" +#include "Engine/Vfs/mod_store_copies.h" + +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +using namespace dingosdk; +using namespace dingosdk::mods; +using namespace dingosdk::mods::detail; + +namespace { +using Bytes = std::vector; +constexpr char shared_bundle[] = "win32/characters/customization/configs/cas_main_sharedbundle"; + +int failures = 0; +void check(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +ItemContent made_of(std::string key, std::vector looks, std::vector shapes, std::string data) { + return {std::move(key), std::move(looks), std::move(shapes), std::move(data)}; +} + +// What counts as a copy, on items written out here. +void rules() { + StoreItems game; + game.add(made_of("Own_Store_Hoodie", {"asset 3 store_hoodie_ap"}, {"asset 1 hoodie_dmpreset"}, "a"), true); + game.add(made_of("Own_Free_Hoodie", {"asset 3 free_hoodie_ap"}, {"asset 1 hoodie_dmpreset"}, "b"), false); + game.add(made_of("Own_Store_Coat", {"asset 3 store_coat_ap"}, {"asset 1 coat_dmpreset"}, "c"), true); + game.add(made_of("Own_Store_Costume", {}, {"asset 1 costume_dmpreset"}, "d"), true); + game.add(made_of("Own_Store_Title", {}, {}, ""), true); + game.add(made_of("Own_Free_Title", {}, {}, ""), false); + game.add(made_of("Own_Store_Voice", {}, {}, "e"), true); + game.add(made_of("Own_Store_Frame", {}, {}, "f"), true); + game.add(made_of("Own_Free_Frame", {}, {}, "f"), false); + + const auto copies = [&](std::vector looks, std::vector shapes, std::string data = "mod") { + return game.original(made_of("Own_Mod_Item", std::move(looks), std::move(shapes), std::move(data))); + }; + check(copies({"asset 3 store_hoodie_ap"}, {"asset 1 hoodie_dmpreset"}) == "Own_Store_Hoodie", "a store item's look is that store item"); + check(copies({"asset 3 junk_ap", "asset 3 store_hoodie_ap"}, {}) == "Own_Store_Hoodie", "and stays it with more beside it"); + check(copies({"asset 3 free_hoodie_ap"}, {"asset 1 hoodie_dmpreset"}).empty(), "a free item's look is free"); + check(copies({"asset 3 mod_ap"}, {"asset 1 coat_dmpreset"}).empty(), "a mod's own look on a store item's shape is the mod's"); + check(copies({}, {"asset 1 coat_dmpreset"}).empty(), "the shape of a store item that has a look is not that item"); + check(copies({}, {"asset 1 costume_dmpreset"}) == "Own_Store_Costume", "a store costume that is only a shape is told by it"); + check(copies({"asset 3 mod_ap"}, {"asset 1 costume_dmpreset"}).empty(), "unless the mod gives it a look of its own"); + check(copies({}, {"asset 1 hoodie_dmpreset"}).empty(), "a shape free items use is free"); + check(copies({}, {}, "e") == "Own_Store_Voice", "a store item's data, field for field, is that store item"); + check(copies({}, {}, "f").empty(), "data a free item has as well is free"); + check(copies({}, {}, "").empty(), "data with nothing in it says nothing"); + check(copies({}, {}).empty(), "an item of the mod's own is no copy"); +} + +fb::Guid guid(std::uint8_t seed) { + fb::Guid value; + for (std::size_t index = 0; index < value.bytes.size(); ++index) + value.bytes[index] = static_cast(static_cast(seed + index)); + return value; +} + +// The game's shared customization bundle, which holds every item, and two of them to make mods +// from: one the store sells and one it does not, both clothing (a look fitted on a shape). +struct Game { + fs::path root, base; + fb::TocBundle bundle; + Listing listing; + fb::ebx::Document sold, free; + std::string soldKey, freeKey; + // A player-card background the store sells: its picture is another asset, and it is kept + // outside items/. + fb::ebx::Document card; + std::string cardKey; +}; +Game read_game(const fs::path& root, const CasStore& store, const StoreItem& sold) { + Game game{root, root / L"Data"}; + bool found{}; + for (const auto& bundle : fb::read_toc(read_file(game.base / L"Win32" / L"items.toc")).bundles) { + if (lower(bundle.name) != shared_bundle) continue; + if (auto listing = list_bundle(store, game.base, game.base, bundle, root)) { + game.bundle = bundle; + game.listing = std::move(*listing); + found = true; + } + break; + } + if (!found) throw std::runtime_error(std::string("no readable ") + shared_bundle); + const auto& assets = game.listing.manifest.ebx; + for (std::size_t index = 0; index < assets.size() && (game.soldKey.empty() || game.freeKey.empty() || game.cardKey.empty()); ++index) { + const auto name = lower(assets[index].name); + if (!name.starts_with("items/cust_") && !name.starts_with("thumbnail/pcard/bg/ownable/")) continue; + auto document = fb::ebx::read_document(read_asset(store, game.base, game.base, game.listing, index, root)); + const auto item = item_content(document); + if (!item || item->looks.size() != 1) continue; + const auto store_item = sold(lower(item->key)); + if (store_item && game.cardKey.empty() && item->shapes.empty() && item->looks.front().starts_with("picture ")) { + game.cardKey = item->key; + game.card = std::move(document); + continue; + } + if (item->shapes.empty() || !item->looks.front().starts_with("asset 3 ")) continue; + auto& key = store_item ? game.soldKey : game.freeKey; + if (!key.empty()) continue; + key = item->key; + (store_item ? game.sold : game.free) = std::move(document); + } + if (game.soldKey.empty() || game.freeKey.empty() || game.cardKey.empty()) + throw std::runtime_error("the game has no store item, free item or store card background to copy"); + return game; +} + +// The item's data, to change in a copy of it. +fb::ebx::Object& item_data(fb::ebx::Document& item) { + for (auto& field : item.instances.front().object->fields) { + if (field.name != "ItemData") continue; + const auto& pointer = std::get(field.value.data); + return *item.instances.at(static_cast(pointer.index)).object; + } + throw std::runtime_error("an item has no ItemData"); +} +fb::ebx::Value& field_of(fb::ebx::Object& object, std::string_view name) { + for (auto& field : object.fields) + if (field.name == name) return field.value; + throw std::runtime_error("an item has no " + std::string(name)); +} + +// One of the game's items under a name and identity of its own: what a cosmetic mod adds. +struct Item { + std::string name; + Bytes payload; +}; +Item item_from(const fb::ebx::Document& donor, const std::string& name, std::uint8_t seed, + const std::function& change = {}) { + auto item = fb::ebx::detail::clone_document(donor); + auto& root = item.instances.front(); + item.fileGuid = guid(seed); + root.instanceGuid = guid(static_cast(seed + 0x40)); + const auto key = name.substr(name.rfind('/') + 1); + std::uint32_t hash = 5381; // the game's own hash of an item's key + for (const unsigned char character : key) hash = hash * 33U ^ character; + for (auto& field : root.object->fields) { + if (field.name == "Name") field.value.data = name; + else if (field.name == "Key") field.value.data = key; + else if (field.name == "HashedAssetKey") { + if (std::holds_alternative(field.value.data)) field.value.data = static_cast(hash); + else field.value.data = static_cast(hash); + } + } + if (change) change(item); + return {name, fb::ebx::write_document(item)}; +} + +// A cosmetic mod's folder: its copy of the shared bundle in items.toc (the game's assets and its +// own items) and one archive holding the manifest and those items. +Mod write_mod(const fs::path& mods, const std::string& name, const CasStore& store, const Game& game, + std::initializer_list items) { + const auto directory = mods / name; + auto manifest = game.listing.manifest; + const auto& shipped = game.listing.files; + std::vector files(shipped.begin() + static_cast(game.listing.first), shipped.end()); + const auto installChunk = shipped.front().location.installChunk; + Bytes archive; + const auto stored = [&](std::span bytes) { + const fb::BundleFileInfo file{{true, installChunk, 1}, static_cast(archive.size()), + static_cast(bytes.size())}; + archive.insert(archive.end(), bytes.begin(), bytes.end()); + return file; + }; + for (const auto& item : items) { + fb::BundleAsset asset; + asset.kind = fb::AssetKind::ebx; + asset.name = item.name; + asset.sha1 = sha1_of(item.payload); + asset.originalSize = item.payload.size(); + files.insert(files.begin() + static_cast(manifest.ebx.size()), + stored(fb::encode_cas(item.payload, {.compression = fb::CasCompression::raw}))); + manifest.ebx.push_back(std::move(asset)); + } + files.insert(files.begin(), stored(fb::write_binary_bundle(manifest))); + const std::vector bundles{{game.bundle.name, fb::write_bundle_region(files), 1}}; + fs::create_directories(directory / L"Win32" / fs::path(store.directory(installChunk))); + write_file(directory / L"Win32" / L"items.toc", fb::write_patch_toc(bundles)); + write_file(directory / L"Win32" / fs::path(store.directory(installChunk)) / L"cas_01.cas", archive); + fs::copy_file(game.base / L"layout.toc", directory / L"layout.toc", fs::copy_options::overwrite_existing); + Mod mod; + mod.name = name; + mod.directory = directory; + mod.provides_layout = true; + return mod; +} + +std::string describe(const StoreCopies& copies, const std::vector& notes) { + std::string text; + for (const auto& [copy, original] : copies.items) text += "\n " + copy + " copies " + original; + for (const auto& note : notes) text += "\n note: " + note; + return text; +} + +// Mods made from the game's own items, checked against the game. +void made_mods(const Game& game, const CasStore& store, const StoreItem& sold) { + const auto root = fs::temp_directory_path() / ("reskate-store-copies-" + std::to_string(GetCurrentProcessId())); + std::error_code error; + fs::remove_all(root, error); + Catalog catalog; + catalog.data_root = game.root; + catalog.root = root / "Mods"; + catalog.present = true; + + // The store item under another key, as the mods did it; and again with its colour changed, so + // it is no longer the same data but still names the store item's appearance. + const auto plain = item_from(game.sold, "items/reskate_test/own_test_copy", 0x10); + const auto tinted = item_from(game.sold, "items/reskate_test/own_test_tinted", 0x20, [](fb::ebx::Document& item) { + auto& colour = std::get>(field_of(item_data(item), "BaseColor").data); + field_of(*colour, "x").data = 0.75; + }); + // A mod's own item on the store item's shape: every appearance it names is the mod's. + const auto own = item_from(game.sold, "items/reskate_test/own_test_own_look", 0x30, [](fb::ebx::Document& item) { + for (auto& entry : std::get(field_of(item_data(item), "AssetPaths").data)) { + auto& path = *std::get>(entry.data); + const auto& kind = field_of(path, "AssetTypeId").data; + const auto shape = std::holds_alternative(kind) ? std::get(kind) < 3 + : std::get(kind) < 3; + if (!shape) field_of(path, "AssetName").data = std::string("ReSkate_Test_Own_Look_AP"); + } + }); + // An item the store does not sell, under another key. + const auto freebie = item_from(game.free, "items/reskate_test/own_test_free_copy", 0x50); + // A store card background under another key: no preset or texture name, only its picture. + const auto card = item_from(game.card, "items/reskate_test/own_test_card", 0x60); + + catalog.mods.push_back(write_mod(catalog.root, "honest", store, game, {own, freebie})); + catalog.mods.push_back(write_mod(catalog.root, "copies", store, game, {plain, tinted, card})); + catalog.mods.push_back(write_mod(catalog.root, "nothing", store, game, {})); + std::vector notes; + const auto found = check_store_copies(catalog, sold, ¬es); + const auto said = describe(found, notes); + const auto store_key = lower(game.soldKey); + check(found.game_items > 3000, "the game's items were read (" + std::to_string(found.game_items) + ")"); + check(found.items.size() == 3, "three of the five items are copies of a store item" + said); + check(found.items.contains("own_test_copy") && found.items.at("own_test_copy") == store_key, + "the store item under another key is a copy of it" + said); + check(found.items.contains("own_test_tinted") && found.items.at("own_test_tinted") == store_key, + "and still is with its colour changed" + said); + check(found.items.contains("own_test_card") && found.items.at("own_test_card") == lower(game.cardKey), + "a store card background under another key is a copy of it" + said); + check(!found.items.contains("own_test_own_look"), "a look of the mod's own on the store item's shape is not" + said); + check(!found.items.contains("own_test_free_copy"), "a copy of an item the store does not sell is not" + said); + check(found.mods.size() == 1 && found.mods.front().mod == "copies" && found.mods.front().count == 3, + "only the mod that adds them is named" + said); + check(notes.empty(), "nothing went unread" + said); + // With no store at all (no content cache) nothing is anyone's copy. + check(check_store_copies(catalog, [](const std::string&) { return false; }).items.empty(), "nothing is a copy when the store sells nothing"); + + // The merge builds nothing while such a mod is among them, and says whose problem it is, so + // the caller leaves that mod out; without it the others merge as they always did. The + // problem is one line that tells the mod's author nothing: no item, no count, no reason. + const auto told = [](const MergeReport& report) { + std::string text = "\n issue: '" + report.issue + "'"; + for (const auto& [mod, list] : report.problems) + for (const auto& problem : list) text += "\n " + mod + ": " + problem; + return text; + }; + const auto refused = merge_mods(catalog); + check(refused.issue.empty() && !refused.built, "nothing is built with a mod that copies store items among them" + told(refused)); + check(refused.problems.size() == 1 && refused.problems.contains("copies") && + refused.problems.at("copies") == std::vector{std::string(store_copies_problem)}, + "and the problem is that mod's alone, and only the one line" + told(refused)); + const auto lower_case = [](std::string text) { return lower(text); }; + for (const auto* word : {"store", "copy", "copies", "item", "own_"}) + check(lower_case(std::string(store_copies_problem)).find(word) == std::string::npos && + lower_case(std::string(store_copies_check)).find(word) == std::string::npos, + std::string("what the mod is told does not say '") + word + "'"); + check(!fs::exists(catalog.root / generated_folder / L"layout.toc", error), "no patch was written for them"); + std::erase_if(catalog.mods, [](const Mod& mod) { return mod.name == "copies"; }); + const auto merged = merge_mods(catalog); + check(merged.issue.empty() && merged.built && merged.problems.empty(), "the other mods merge without it" + told(merged)); + fs::remove_all(root, error); +} + +// Real mods, listed rather than judged. +void installed_mods(const fs::path& game, const fs::path& folder, const StoreItem& sold) { + Catalog catalog; + catalog.data_root = game; + catalog.root = fs::temp_directory_path() / L"reskate-store-copies-unused"; + catalog.present = true; + std::error_code error; + for (fs::directory_iterator it(folder, error), end; it != end && !error; it.increment(error)) { + if (!it->is_directory(error) || it->path().filename().wstring().starts_with(L".") || + !fs::exists(it->path() / L"layout.toc", error)) continue; + Mod mod; + mod.name = it->path().filename().string(); + mod.directory = it->path(); + mod.provides_layout = true; + catalog.mods.push_back(std::move(mod)); + } + std::vector notes; + const auto start = std::chrono::steady_clock::now(); + const auto found = check_store_copies(catalog, sold, ¬es); + const auto ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + std::cout << folder.string() << ": " << catalog.mods.size() << " mod(s) checked in " << ms << " ms against " + << found.game_items << " of the game's items, " << found.items.size() << " copy(ies) of store items in " + << found.mods.size() << " of them\n"; + for (const auto& source : found.mods) + std::cout << " " << source.mod << ": " << source.count << ", e.g. " << source.example << " is " << source.original << '\n'; + for (const auto& note : notes) std::cout << " note: " << note << '\n'; +} +} // namespace + +int main(int argc, char** argv) try { + rules(); + { + // No mods: nothing to read, the game's own files included. + std::vector notes; + check(check_store_copies(Catalog{}, [](const std::string&) { return true; }, ¬es).items.empty() && notes.empty(), + "no mods, no copies and nothing to say"); + } + std::error_code error; + if (argc < 2 || !fs::exists(fs::path(argv[1]) / L"Data" / L"Win32" / L"items.toc", error)) { + std::cout << "No game given; the check was not run on mods.\n"; + } else if (!content_cache::installed()) { + std::cout << "No content cache, so nothing says what the store sells; the check was not run on mods.\n"; + } else { + const auto catalogs = content_cache::read_catalogs(content_cache::directory()); + const StoreItem sold = [&](const std::string& key) { return catalogs.reserved(key); }; + const fs::path root = argv[1]; + const auto layout = vfs::read_layout(root / L"Data" / L"layout.toc"); + const CasStore store(root / L"Data", fs::temp_directory_path() / L"reskate-store-copies-unused", layout.root); + const auto game = read_game(root, store, sold); + made_mods(game, store, sold); + std::cout << "store copies: made from " << game.soldKey << " and " << game.cardKey << " (sold) and " + << game.freeKey << " (free)\n"; + for (int index = 2; index < argc; ++index) installed_mods(root, argv[index], sold); + } + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "store copies: ok\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/content_catalogs.cpp b/Engine/Vfs/content_catalogs.cpp index d6bc99e..0000e9b 100644 --- a/Engine/Vfs/content_catalogs.cpp +++ b/Engine/Vfs/content_catalogs.cpp @@ -54,6 +54,40 @@ void add_item(Catalogs& out, const Fields& record) { out.items[key] = std::move(entry); } +// Music playlists: field 1 names the playlist, repeated field 2 lists tracks +// ("Artist - Title"), and field 10 is a message carrying the display name and +// artwork. Owned-item records put a varint in field 10, so requiring the byte +// wire separates the two. First record for an id wins. +void add_music_playlist(Catalogs& out, const std::string& id, const Fields& record) { + const auto content = message(record, 10); + if (!content) return; + const auto name = text(*content, 10); + if (!name) return; + std::vector tracks; + for (const auto& field : record) { + if (field.number != 2 || field.wire != Wire::bytes) continue; + const auto value = text(&field); + if (!value || value->find(" - ") == std::string::npos) continue; + tracks.push_back(*value); + } + if (tracks.size() < 2) return; + Catalogs::MusicPlaylistEntry entry; + entry.name = *name; + entry.artwork = text(*content, 11).value_or(""); + entry.tracks = std::move(tracks); + out.music_playlists.try_emplace(id, std::move(entry)); +} + +// Songs: field 1 is "Artist - Title" and field 10 a message with the artist (10), title (11) and +// cdn:/ cover art (12). First record for an id wins. +void add_music_song(Catalogs& out, const std::string& id, const Fields& record) { + if (id.find(" - ") == std::string::npos) return; + const auto content = message(record, 10); + if (!content || !text(*content, 10) || !text(*content, 11)) return; + if (const auto artwork = text(*content, 12); artwork && artwork->starts_with("cdn:/")) + out.music_song_artwork.try_emplace(id, *artwork); +} + // Grant lists: repeated `number` entries, each {1: kind, 2: id}. struct Grant { std::string kind, id; }; std::vector grants(const Fields& fields, std::uint32_t number) { @@ -221,6 +255,8 @@ Catalogs read_catalogs(const std::filesystem::path& folder) { if (!id) continue; if (text(*fields, 3) == "entitlement") entitlements.insert(*id); add_item(out, *fields); + add_music_playlist(out, *id, *fields); + add_music_song(out, *id, *fields); add_challenge(out, *id, *fields); add_object_category(out, *id, *fields); add_travel(out, *id, *fields); diff --git a/Engine/Vfs/content_catalogs.h b/Engine/Vfs/content_catalogs.h index d0bcef7..71f8b85 100644 --- a/Engine/Vfs/content_catalogs.h +++ b/Engine/Vfs/content_catalogs.h @@ -2,6 +2,7 @@ #include "Engine/Core/Json/json.h" #include #include +#include #include #include #include @@ -40,6 +41,17 @@ struct Catalogs { std::vector travel_locations; // Access point id (accesspoint_*) -> the location ids it offers. std::vector>> travel_access_points; + // Music playlist id -> {display name, artwork id, track ids} from the music + // chunk. A music record carries a field-10 message (10.10 name, 10.11 + // artwork) and repeated field-2 tracks; owned items use a varint field 10. + struct MusicPlaylistEntry { + std::string name, artwork; + std::vector tracks; + }; + std::map> music_playlists; + // Song ("Artist - Title") -> its cdn:/ cover art: a song record carries a field-10 message + // (10.10 artist, 10.11 title, 10.12 artwork). + std::map> music_song_artwork; bool reserved(const std::string& key) const; }; // Read once, on first use. Empty (available == false) when no pack is installed. diff --git a/Engine/Vfs/https_download.cpp b/Engine/Vfs/https_download.cpp index 9d654b5..d655c72 100644 --- a/Engine/Vfs/https_download.cpp +++ b/Engine/Vfs/https_download.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include namespace dingosdk::https { @@ -72,4 +75,24 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, result.ok = true; return result; } + +std::optional get_text(std::wstring_view url, std::uint64_t max_bytes, + std::uint32_t timeout_seconds, const wchar_t* agent, Download* result) { + static std::atomic serial{}; + wchar_t folder[MAX_PATH + 1]{}; + const auto length = GetTempPathW(MAX_PATH, folder); + const auto path = std::filesystem::path(length ? std::wstring(folder, length) : L".") / + (L"reskate-get-" + std::to_wstring(GetCurrentProcessId()) + L"-" + std::to_wstring(++serial) + L".tmp"); + std::error_code error; + std::filesystem::remove(path, error); // get needs a fresh destination + const auto download = get(url, path, max_bytes, timeout_seconds, agent); + if (result) *result = download; + std::optional text; + if (download.ok) { + std::ifstream in(path, std::ios::binary); + text.emplace(std::istreambuf_iterator(in), std::istreambuf_iterator()); + } + std::filesystem::remove(path, error); + return text; +} } diff --git a/Engine/Vfs/https_download.h b/Engine/Vfs/https_download.h index e0cfe49..91b777a 100644 --- a/Engine/Vfs/https_download.h +++ b/Engine/Vfs/https_download.h @@ -1,6 +1,8 @@ #pragma once #include #include +#include +#include #include namespace dingosdk::https { @@ -13,4 +15,8 @@ struct Download { // that take longer than `timeout_seconds` in total. Download get(std::wstring_view url, const std::filesystem::path& destination, std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent); +// The body of the same GET, for an answer small enough to hold: nothing when it +// fails, and `result`, when given, says how. +std::optional get_text(std::wstring_view url, std::uint64_t max_bytes, + std::uint32_t timeout_seconds, const wchar_t* agent, Download* result = nullptr); } diff --git a/Engine/Vfs/mod_catalog.cpp b/Engine/Vfs/mod_catalog.cpp index 3e964b3..ff670f7 100644 --- a/Engine/Vfs/mod_catalog.cpp +++ b/Engine/Vfs/mod_catalog.cpp @@ -139,15 +139,9 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver if (!entry.enabled) { result.notes.push_back("Mod disabled in mods.json: " + entry.mod.name); result.disabled.push_back(entry.mod.name); - if (entry.mod.provides_layout && result.inactive.size() < maximum_mods) - result.inactive.push_back(std::move(entry.mod)); + if (entry.mod.provides_layout) result.inactive.push_back(std::move(entry.mod)); continue; } - if (result.mods.size() >= maximum_mods) { - result.notes.push_back("More than " + std::to_string(maximum_mods) + - " mods are present; " + entry.mod.name + " and any later mod were skipped"); - break; - } if (!entry.mod.provides_layout && !entry.mod.provides_levels && entry.mod.park_maps.empty()) result.notes.push_back("Mod " + entry.mod.name + " has no layout.toc or reskate-levels.json"); result.mods.push_back(std::move(entry.mod)); diff --git a/Engine/Vfs/mod_list.cpp b/Engine/Vfs/mod_list.cpp index 3d68090..04bc0f0 100644 --- a/Engine/Vfs/mod_list.cpp +++ b/Engine/Vfs/mod_list.cpp @@ -23,7 +23,9 @@ constexpr char levels_name[] = "reskate-levels.json"; // Written by the tool that built the mod (Studio, since 2026-09-24): who made // it and when. Purely informational, so it is read loosely and never rejected. constexpr char build_name[] = "reskate-build.json"; -constexpr std::size_t maximum_order_bytes = 256 * 1024; +// mods.json and the list of mods left out: as much as the JSON reader itself takes, so the +// number of mods is not held down by the size of the file that lists them. +constexpr std::size_t maximum_order_bytes = 4 * 1024 * 1024; constexpr std::size_t maximum_build_bytes = 16 * 1024; constexpr std::size_t maximum_info_bytes = 32 * 1024; constexpr std::size_t maximum_levels_bytes = 64 * 1024; @@ -199,8 +201,6 @@ std::vector> read_order(const fs::path& path) { throw std::runtime_error("mods.json schema must be integer 1"); const auto& rows = root.at("mods"); if (!rows.is_array()) throw std::runtime_error("mods.json mods must be an array"); - if (rows.size() > maximum_mods) - throw std::runtime_error("mods.json lists more than " + std::to_string(maximum_mods) + " mods"); std::vector> order; std::set> seen; @@ -398,8 +398,6 @@ ModList scan_mods(const std::filesystem::path& data_root) noexcept { } void save_mod_order(const std::filesystem::path& mods_root, const std::vector& entries) { - if (entries.size() > maximum_mods) - throw std::runtime_error("At most " + std::to_string(maximum_mods) + " mods can be listed"); auto rows = Json::array(); for (const auto& entry : entries) { if (!valid_mod_name(entry.mod.name)) diff --git a/Engine/Vfs/mod_list.h b/Engine/Vfs/mod_list.h index 0f3a8ec..aff4fbd 100644 --- a/Engine/Vfs/mod_list.h +++ b/Engine/Vfs/mod_list.h @@ -12,7 +12,9 @@ // happens here, so the launcher's mod manager and the runtime share it. namespace dingosdk::mods { -inline constexpr std::size_t maximum_mods = 64; +// There is no limit on how many mods are listed or loaded: nothing in the merge needs one +// (archives are numbered with 16 bits). There was one of 64 once, which players reached: past +// it the launcher could not save the list at all, and the game skipped the later mods. inline constexpr std::size_t maximum_mod_name = 64; // Generated state the SDK owns. Folders beginning with a dot are never mods. inline constexpr char generated_folder[] = ".reskate"; diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index 4180511..605fc93 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -1,6 +1,9 @@ #include "mod_merge.h" +#include "content_cache.h" +#include "content_catalogs.h" #include "mod_merge_internal.h" +#include "mod_store_copies.h" #include "native_db.h" #include @@ -29,6 +32,21 @@ constexpr std::string_view root_level = "win32/levels/game/dingolevel_root/dingo // installed while the game runs has a mounted copy to replace in memory // (Extension/Assets/live_mods.cpp) and one disabled can be swapped back out. constexpr std::array launch_superbundles{"Win32/globals.toc", "Win32/items.toc"}; + +// Each enabled mod that adds copies of items the game's store sells +// (mod_store_copies.h) gets a problem; true when any does. The problem says no +// more than that the mod could not be merged: what was found is not for the +// mod's author to read. The catalogue is only read once a mod turns out to add +// an item at all. +bool store_copy_problems(const Catalog& catalog, MergeReport& report) { + std::optional store; + const auto found = check_store_copies(catalog, [&store](const std::string& key) { + if (!store) store = content_cache::read_catalogs(content_cache::directory()); + return store->reserved(key); + }, &report.notes); + for (const auto& source : found.mods) report.problems[source.mod].emplace_back(store_copies_problem); + return !found.mods.empty(); +} } // namespace MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, const MergeOptions& options) noexcept { @@ -45,16 +63,23 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con std::map modFiles; for (const auto* mod : mods) modFiles[mod] = scan(mod->directory); + // What the store sells comes from the content cache, which the launcher installs. + const bool storeKnown = content_cache::installed(); // Disabled mods count too: their archives and map registration are placed at launch. - auto fingerprint = merge_fingerprint(catalog, mods, modFiles); + auto fingerprint = merge_fingerprint(catalog, mods, modFiles, storeKnown); for (const auto& mod : catalog.inactive) fingerprint += "\ninactive " + mod.name + " " + mod_fingerprint(mod.directory); if (options.live) { fs::remove(output / stamp_file, error); - } else { - if (auto previous = previous_merge(output, fingerprint)) return std::move(*previous); - fs::remove_all(output, error); + } else if (auto previous = previous_merge(output, fingerprint)) { + return std::move(*previous); } + // A mod that adds copies of store items is not loaded at all. Found before + // anything is built: the caller merges again without it, as it does for a + // mod that cannot be merged, and the patch on disk stays for that merge to + // reuse or replace. + if (storeKnown && store_copy_problems(catalog, report)) return report; + if (!options.live) fs::remove_all(output, error); // Progress: each mod's archives, each superbundle, then the layout. std::set distinctTocs; @@ -291,7 +316,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con std::erase_if(superbundles, [](const std::string& relative) { return lower(relative) == root_level; }); } if (options.live) report.load_screens = read_load_screens(mods, modFiles, store, baseRoot, gameRoot, report); - const auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report); + auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report); + // Carried chunks point into their mod's archives; move them to where those landed. + for (auto& [name, chunks] : overrides.chunks) + for (const auto* mod : mods) + if (mod->name == name) + for (auto& chunk : chunks) store.shift(chunk.location, chunk.offset, &placements[mod]); for (const auto& relative : superbundles) { if (keepRoot && lower(relative) == root_level) { diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 5de90c4..2824a87 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -8,8 +8,11 @@ #include "Engine/Core/Platform/path_text.h" #include +#include #include +#include #include +#include namespace dingosdk::mods::detail { namespace { @@ -32,6 +35,10 @@ struct Asset { fb::BundleAsset asset; fb::BundleFileInfo file; bool base{}; + const std::string* mod{}; // whose copy of the bundle it came in; null for the game's + // For a chunk: where its record is in the chunk metadata of the copy this version + // of it came in, and where the game's copy lists the chunk (merge_chunk_metadata). + ChunkRecord record; }; // Where one contributor's copy of an asset lives, so conflicting copies can be @@ -68,6 +75,16 @@ std::string asset_key(const fb::BundleAsset& asset) { (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : lower(asset.name)); } +// The game's item lists (items/itemmastercollection, a season's +// 0.30.0_itemcollection), which every cosmetic mod adds its items to. The game +// reads each entry of one at startup without checking it, so an entry naming an +// item that is not loaded is a crash on every launch, not a missing item. +bool item_list_name(std::string_view name) { + const auto text = lower(name); + return text.starts_with("items/") && text.find("collection") != std::string::npos; +} +constexpr std::string_view item_list_type = "delmaritemcollectionasset"; + // Some bundles carry no inline manifest: their region only lists placements, // so their assets cannot be named and the bundle can only be passed through // whole, with the highest-priority copy winning. @@ -81,7 +98,13 @@ struct BundleState { std::uint32_t manifestChunk{}; std::vector assets; std::vector files; - std::vector metadata; + // Each copy's chunk metadata; copies that carry the same list share an entry. + // Every chunk remembers its record in one of these (Asset::record), and the + // list is written again when the manifest is rebuilt (merge_chunk_metadata). + std::vector> metadata; + // Which of those is the game's own, and the chunks the game's copy lists. + std::size_t gameMetadata{ChunkRecord::none}; + std::vector shippedChunks; // Keyed position of each asset, so replacing one stays constant time: these // bundles hold tens of thousands of entries. std::map> seen; @@ -90,6 +113,78 @@ struct BundleState { // The base's own sha1 for each asset, so a mod that merely carries an // unchanged copy can be told apart from one that changed it. std::map> baseSha; + // Assets a mod added that a higher-priority mod's different asset of the + // same name then replaced: the key, and the sha1 of the copy that went. + std::vector> displaced; +}; + +// The guids of the EBX a merged bundle ends up holding, to tell whether what a +// list entry names is in it. Read on first use: the assets mods added, which is +// where a mod's entry nearly always points, and the game's own thousands only +// when one of those is not the answer. +class HeldAssets final { +public: + using Decode = std::function(const Contribution&)>; + HeldAssets(const BundleState& state, Decode decode) : state_(state), decode_(std::move(decode)) {} + + [[nodiscard]] bool holds(const fb::Guid& guid) { + if (!added_) { read(false); added_ = true; } + if (guids_.contains(guid)) return true; + if (!shipped_ && !unsure_) { read(true); shipped_ = true; } + // A copy that could not be read may be the very one asked about. + return unsure_ || guids_.contains(guid); + } + + // The asset that took the place of the one with this guid, when it was a + // mod's addition another mod's of the same name replaced; null otherwise. + [[nodiscard]] const Asset* replacement(const fb::Guid& guid) { + if (!displacedRead_) { + displacedRead_ = true; + for (const auto& [key, sha1] : state_.displaced) { + const auto copies = state_.history.find(key); + const auto at = state_.seen.find(key); + if (copies == state_.history.end() || at == state_.seen.end()) continue; + for (const auto& copy : copies->second) { + if (copy.sha1 != sha1) continue; + try { + displaced_.emplace(fb::ebx::read_file_guid(decode_(copy)), &state_.assets[at->second]); + } catch (const std::exception&) {} + break; + } + } + } + const auto found = displaced_.find(guid); + return found == displaced_.end() ? nullptr : found->second; + } + +private: + void read(bool shipped) { + for (const auto& asset : state_.assets) { + if (asset.asset.kind != fb::AssetKind::ebx) continue; + const auto key = asset_key(asset.asset); + if (state_.baseSha.contains(key) != shipped) continue; + // The copy the bundle keeps. One combined from several mods' is none + // of theirs, and has the guid of the game's it was combined onto. + const Contribution* kept{}; + if (const auto copies = state_.history.find(key); copies != state_.history.end()) + for (const auto& copy : copies->second) { + if (copy.sha1 == asset.asset.sha1) { kept = © break; } + if (copy.base) kept = © + } + try { + if (!kept) throw std::runtime_error("no copy to read"); + guids_.insert(fb::ebx::read_file_guid(decode_(*kept))); + } catch (const std::exception&) { + unsure_ = true; + } + } + } + + const BundleState& state_; + Decode decode_; + std::set guids_; + std::map displaced_; + bool added_{}, shipped_{}, unsure_{}, displacedRead_{}; }; // Rebuilds the manifest and the matching placement list in the order the @@ -110,8 +205,115 @@ void build_manifest(const std::vector& assets, std::span emit(fb::AssetKind::chunk, manifest.chunks); } +std::span unsigned_bytes(std::span data) { + return {reinterpret_cast(data.data()), data.size()}; +} + } // namespace +std::vector merge_chunk_metadata(std::span> lists, + std::span chunks, std::size_t game, + std::span shipped) { + // A bundle with no chunks has nothing to describe; what the first copy has stays. + if (chunks.empty()) return lists.empty() ? std::vector{} : lists.front(); + const bool shippedList = game < lists.size() && !lists[game].empty(); + + // Nothing moved: nearly every bundle is this, and its list is not read at all. + // With the game's list, every chunk is the game's own where the game lists it; + // without one, every chunk keeps one copy's record at its own place. + const auto first = shippedList ? game : chunks.front().copy; + bool whole = first < lists.size() && (!shippedList || chunks.size() == shipped.size()); + for (std::size_t index = 0; whole && index < chunks.size(); ++index) + whole = chunks[index].copy == first && chunks[index].index == index && + (!shippedList || chunks[index].shipped == index); + if (whole) return lists[first]; + + std::vector> read(lists.size()); + const auto list = [&](std::size_t copy) -> const native_db::Node* { + if (copy >= lists.size() || lists[copy].empty()) return nullptr; + // A level's root bundle has thousands of chunks; every value takes a byte at least. + if (!read[copy]) + read[copy] = native_db::read(unsigned_bytes(lists[copy]), "chunk metadata", nullptr, + {.unique_fields = false, .max_entries = lists[copy].size()}); + return &*read[copy]; + }; + const auto row = [&](std::size_t copy, std::size_t index) -> const native_db::Node* { + const auto* from = list(copy); + return from && index < from->children.size() ? &from->children[index] : nullptr; + }; + + // The game's list, when it is the kind this is written as: a record for each of + // the game's chunks, in the order of their guids. + const auto* original = shippedList ? list(game) : nullptr; + if (original && original->children.size() != shipped.size()) original = nullptr; + std::vector place(shipped.size()); // where each of the game's chunks has its record + if (original) { + std::vector order(shipped.size()); + for (std::size_t index = 0; index < order.size(); ++index) order[index] = index; + std::ranges::sort(order, [&](std::size_t left, std::size_t right) { return shipped[left] < shipped[right]; }); + for (std::size_t at = 0; at < order.size(); ++at) place[order[at]] = at; + } + + std::vector records; + std::vector described(chunks.size()); + records.reserve(chunks.size()); + for (std::size_t index = 0; index < chunks.size(); ++index) { + const auto& chunk = chunks[index]; + const auto* written = row(chunk.copy, chunk.index); + const native_db::Node* kept = written; + std::optional changed; + if (original && chunk.shipped < shipped.size()) { + kept = &original->children[place[chunk.shipped]]; + // A mod's version of the chunk: its tool started from the game's list and wrote a + // record where the chunk is listed, in place of whatever the game has there. The + // first mip it gave is the one the mod's texture has; the rest of it is not the + // chunk's (the hash is another chunk's, or empty). + const auto* before = chunk.index < original->children.size() ? &original->children[chunk.index] : nullptr; + if (chunk.copy != game && written && (!before || native_db::write(*written) != native_db::write(*before))) + if (const auto* meta = written->field("meta")) + if (const auto* mip = meta->field("firstMip")) { + changed = *kept; + if (auto* own = changed->field("meta")) { + if (auto* current = own->field("firstMip")) *current = *mip; + else own->children.push_back(*mip); + kept = &*changed; + } + } + } + described[index] = kept != nullptr; + records.push_back(kept ? *kept : native_db::Node{}); + } + const native_db::Node* shape = original; + for (std::size_t copy = 0; !shape && copy < read.size(); ++copy) + if (read[copy]) shape = &*read[copy]; + if (!shape) return {}; // no copy has a list at all + // A chunk no list describes still takes up its place, with a record that says nothing. + native_db::Node blank; + blank.type = 2; + for (std::size_t index = 0; index < records.size(); ++index) + if (described[index]) { + blank = records[index]; + blank.children.clear(); + break; + } + for (std::size_t index = 0; index < records.size(); ++index) + if (!described[index]) records[index] = blank; + + // As the game writes it: by the chunks' guids. Only a bundle whose own list in the + // game is of another kind keeps the order its chunks are listed in. + std::vector order(chunks.size()); + for (std::size_t index = 0; index < order.size(); ++index) order[index] = index; + if (original || !shippedList) + std::ranges::stable_sort(order, [&](std::size_t left, std::size_t right) { return chunks[left].guid < chunks[right].guid; }); + auto merged = *shape; + merged.children.clear(); + merged.children.reserve(records.size()); + for (const auto index : order) merged.children.push_back(std::move(records[index])); + const auto bytes = native_db::write(merged); + const auto* data = reinterpret_cast(bytes.data()); + return {data, data + bytes.size()}; +} + // One superbundle, combined across every mod that ships it. `used` collects the // patch archives the result actually references, keyed by install chunk, so the // layout declares those and only those. @@ -147,6 +349,26 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, std::string example; }; std::map archiveFaults; + // Mods whose added assets went into a copy here: their added chunks come too. + std::set addedFrom; + // The mods absorbed so far, so each asset can say whose copy it is. + std::set owners; + // Every EBX a mod added to one of the game's bundles, by its document's guid + // and with the bundle it was added to: what a changed list names is looked + // up here, whichever bundle the mod put it in. + std::multimap> addedByGuid; + // The resource a mod added to a bundle under a name, by bundle, mod and name. + std::map, const AssetAddition*> companions; + for (const auto& [bundleName, list] : overrides.added) + for (const auto& addition : list) { + if (addition.asset.kind == fb::AssetKind::ebx) + addedByGuid.emplace(addition.file, std::pair{&bundleName, &addition}); + else if (addition.asset.kind == fb::AssetKind::resource) + companions.emplace(std::tuple{bundleName, addition.mod, lower(addition.asset.name)}, &addition); + } + // Where an addition's payload already went in this superbundle's patch + // archive, by install chunk: every copy that is given it reads the one. + std::map, fb::BundleFileInfo> carriedAt; const auto absorb = [&](const fb::TocBundle& bundle, const ArchivePlacement* placement, bool isBase, const fs::path& root) { @@ -296,43 +518,136 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // The mod's unchanged copies of assets another mod changed: the change // goes into the merged patch, next to this bundle's other files. std::map overridden; - if (propagate && casBacked) { + // The documents those changes refer to: the ones a mod added come along below. + std::set wanted; + if (propagate && (casBacked || !region.inlineManifest.empty())) { std::string example; - for (std::size_t index = 0; index < casManifest.ebx.size(); ++index) { - auto& asset = casManifest.ebx[index]; - const auto named = overrides.find(lower(asset.name)); - if (named == overrides.end()) continue; - const auto change = named->second.find(asset.sha1); - const auto at = 1 + index; - if (change == named->second.end() || change->second.mod == modName || at >= region.files.size()) - continue; - try { - overridden[at] = store.write(region.files[at].location.installChunk, manifestArchive, - change->second.encoded); - asset.sha1 = change->second.sha1; - asset.originalSize = change->second.originalSize; - if (example.empty()) example = asset.name + " from " + change->second.mod; - } catch (const std::exception& error) { - report.notes.push_back(modName + ": " + bundle.name + ": " + asset.name + - " kept the game's copy, the change could not be copied (" + error.what() + ")"); + auto manifest = casBacked ? std::move(casManifest) : fb::read_binary_bundle(region.inlineManifest); + const auto replace = [&](std::vector& assets, const auto& changes, std::size_t first) { + for (std::size_t index = 0; index < assets.size(); ++index) { + auto& asset = assets[index]; + const auto named = changes.find(lower(asset.name)); + if (named == changes.end()) continue; + const auto change = named->second.find(asset.sha1); + const auto at = (casBacked ? 1 : 0) + first + index; + if (change == named->second.end() || change->second.mod == modName || at >= region.files.size()) + continue; + const auto& resource = change->second.resource; + if (asset.kind == fb::AssetKind::resource && + (!resource || resource->resourceType != asset.resourceType || resource->resourceId != asset.resourceId)) + continue; + try { + overridden[at] = store.write(region.files[at].location.installChunk, manifestArchive, + change->second.encoded); + asset.sha1 = change->second.sha1; + asset.originalSize = change->second.originalSize; + if (resource) asset.resourceMeta = resource->resourceMeta; + wanted.insert(change->second.names.begin(), change->second.names.end()); + if (example.empty()) example = asset.name + " from " + change->second.mod; + } catch (const std::exception& error) { + report.notes.push_back(modName + ": " + bundle.name + ": " + asset.name + + " kept the game's copy, the change could not be copied (" + error.what() + ")"); + } } - } + }; + replace(manifest.ebx, overrides.changed, 0); + replace(manifest.resources, overrides.scripts, manifest.ebx.size()); + if (casBacked) casManifest = std::move(manifest); + else if (!overridden.empty()) region.inlineManifest = fb::write_binary_bundle(manifest); if (!overridden.empty()) report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(overridden.size()) + " asset(s) take another mod's change, e.g. " + example); } + // Assets another mod added to the game's copy of this bundle: this copy + // loads instead of the game's on this mod's levels, so it gets them too. + std::vector additions; + if (propagate && casBacked) { + std::vector take; + std::set taking; + // A copy in the TOC the adder ships itself merges with the adder's bundle and has + // its assets already; only copies in other superbundles (maps) need them carried. + const auto here = lower(relative); + if (const auto found = overrides.added.find(key); found != overrides.added.end()) + for (const auto& addition : found->second) { + if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk || + addition.toc == here) continue; + take.push_back(&addition); + taking.insert(&addition); + } + // What the changes this copy took name. The game keeps a list in more + // bundles than the one a mod added to (the music playlist is in eight), + // and a copy of any of them takes the changed list: it needs what the + // mod added next to that list, and what those refer to in turn. + for (std::vector open(wanted.begin(), wanted.end()); !open.empty();) { + const auto guid = open.back(); + open.pop_back(); + const auto [first, last] = addedByGuid.equal_range(guid); + for (auto at = first; at != last; ++at) { + const auto& [from, addition] = at->second; + if ((*from == key && addition->toc == here) || !taking.insert(addition).second) continue; + take.push_back(addition); + for (const auto& name : addition->names) + if (wanted.insert(name).second) open.push_back(name); + // The resource added under the same name goes where the asset goes. + if (const auto companion = companions.find({*from, addition->mod, lower(addition->asset.name)}); + companion != companions.end() && taking.insert(companion->second).second) + take.push_back(companion->second); + } + } + if (!take.empty()) { + std::set> present; + for (const auto& asset : casManifest.ebx) present.insert(asset_key(asset)); + for (const auto& asset : casManifest.resources) present.insert(asset_key(asset)); + const auto installChunk = region.files.front().location.installChunk; + for (const auto* addition : take) { + if (!present.insert(asset_key(addition->asset)).second) continue; + try { + auto placed = carriedAt.find({installChunk, addition}); + if (placed == carriedAt.end()) + placed = carriedAt.emplace(std::pair{installChunk, addition}, + store.write(installChunk, manifestArchive, addition->encoded)).first; + additions.push_back({addition->asset, placed->second}); + addedFrom.insert(addition->mod); + } catch (const std::exception& error) { + report.notes.push_back(modName + ": " + bundle.name + ": " + addition->asset.name + + " from " + addition->mod + " could not be added (" + error.what() + ")"); + } + } + if (!additions.empty()) + report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(additions.size()) + + " asset(s) added by other mods, e.g. " + additions.front().asset.name); + } + } // Nothing moved in this bundle: it passes through whole, as before. - if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty()) casBacked = false; + if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty() && additions.empty()) + casBacked = false; for (auto& file : region.files) { store.shift(file.location, file.offset, placement); if (file.location.patch) used.emplace(file.location.installChunk, file.location.archive); } + // Which of the region's files now lie in the merged patch rather than in this + // mod's own archives, by position. Kept in step with region.files: an addition + // below moves every later file along, and the pass over the bundle's assets + // further down asks by where a file is then, not where it was. + std::vector inPatch(region.files.size()); for (const auto* moved : {&renumbered, &overridden}) for (const auto& [index, file] : *moved) { region.files[index] = file; + inPatch[index] = true; used.emplace(file.location.installChunk, file.location.archive); } + // Added EBX go after this copy's own EBX and added resources after its own + // resources, ahead of its chunks; after the indexed moves above so their + // positions still hold. + for (auto& addition : additions) { + const auto ebx = addition.asset.kind == fb::AssetKind::ebx; + const auto at = static_cast(1 + casManifest.ebx.size() + (ebx ? 0 : casManifest.resources.size())); + (ebx ? casManifest.ebx : casManifest.resources).push_back(std::move(addition.asset)); + region.files.insert(region.files.begin() + at, addition.file); + inPatch.insert(inPatch.begin() + at, true); + used.emplace(addition.file.location.installChunk, addition.file.location.archive); + } const auto fresh = !states.contains(key); if (fresh) { order.push_back(bundle.name); states.emplace(key, BundleState{}); } auto& state = states.at(key); @@ -362,6 +677,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, state.opaqueWhy = manifestFailure; state.files = std::move(region.files); state.assets.clear(); state.seen.clear(); + state.metadata.clear(); state.shippedChunks.clear(); + state.gameMetadata = ChunkRecord::none; } return; } @@ -381,20 +698,46 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, } if (casBacked) state.casBacked = true; if (!region.files.empty()) state.manifestChunk = region.files.front().location.installChunk; - if (fresh || state.metadata.empty()) state.metadata = manifest.chunkMetadata; + // Each chunk remembers where this copy lists it, and with that its record + // in this copy's chunk metadata, whichever version of the chunk is kept. + { + auto copy = ChunkRecord::none; + if (!manifest.chunkMetadata.empty()) { + auto list = std::find(state.metadata.begin(), state.metadata.end(), manifest.chunkMetadata); + if (list == state.metadata.end()) list = state.metadata.insert(state.metadata.end(), manifest.chunkMetadata); + copy = static_cast(list - state.metadata.begin()); + } + if (isBase) { + state.gameMetadata = copy; + state.shippedChunks.clear(); + for (const auto& chunk : manifest.chunks) state.shippedChunks.push_back(chunk.guid); + } + const auto firstChunk = manifest.ebx.size() + manifest.resources.size(); + for (std::size_t index = 0; index < manifest.chunks.size() && firstChunk + index < flat.size(); ++index) { + auto& record = flat[firstChunk + index].record; + record.guid = manifest.chunks[index].guid; + record.copy = copy; + record.index = index; + if (isBase) record.shipped = index; + } + } const std::size_t firstFile = casBacked ? 1 : 0; + const auto* owner = isBase ? nullptr : &*owners.insert(modName).first; + // By the mod whose added assets this copy's replace: how many, and one of them. + std::map> replaced; for (std::size_t position = 0; position < flat.size(); ++position) { auto& entry = flat[position]; entry.base = isBase; + entry.mod = owner; auto id = asset_key(entry.asset); // A copy rewritten above (another mod's change, renumbered collision) - // is not this mod's edit and no longer lies in this mod's archives: it - // is in the merged patch. Recorded as a contribution it was read back - // from the mod's folder at the patch's offset, the read failed, and - // the assets that really had two edits (the item collections every - // cosmetic mod adds its items to) fell back to one mod's copy. - const bool rewritten = renumbered.contains(firstFile + position) || - overridden.contains(firstFile + position); + // or added from another mod is not this mod's edit and does not lie in + // this mod's archives: it is in the merged patch. Recorded as a + // contribution it was read back from the mod's folder at the patch's + // offset, the read failed, and the assets that really had two edits + // (the item collections every cosmetic mod adds its items to) fell back + // to one mod's copy. + const bool rewritten = firstFile + position < inPatch.size() && inPatch[firstFile + position]; if (!rewritten && (entry.asset.kind == fb::AssetKind::ebx || entry.asset.kind == fb::AssetKind::resource)) { // The contributor's own copy still sits at its original offset in @@ -431,9 +774,26 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, previous.asset.sha1 != entry.asset.sha1) report.notes.push_back("contested chunk " + entry.asset.name + ": kept the highest-priority copy"); + // Two mods each adding a different asset under one name: a bundle + // holds one asset per name, so the lower-priority mod's is gone, + // and whatever of that mod's still names it by guid finds nothing + // (an item list entry is dropped for that further down). + if (entry.asset.kind != fb::AssetKind::chunk && !rewritten && previous.mod && + previous.mod != owner && previous.asset.sha1 != entry.asset.sha1 && + !state.baseSha.contains(at->first)) { + auto& [count, example] = replaced[*previous.mod]; + if (!count++) example = entry.asset.name; + state.displaced.emplace_back(at->first, previous.asset.sha1); + } + // Whichever version is kept, it is still the chunk the game lists there. + entry.record.shipped = previous.record.shipped; state.assets[at->second] = std::move(entry); } } + for (const auto& [other, clash] : replaced) + report.notes.push_back(other + ": " + bundle.name + ": " + std::to_string(clash.first) + + " added asset(s) share a name with ones " + modName + " adds, e.g. " + clash.second + + "; the merged bundle keeps " + modName + "'s"); }; if (base) for (const auto& bundle : base->bundles) absorb(bundle, nullptr, true, baseRoot); @@ -449,6 +809,15 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, if (state.opaque) { region = fb::write_bundle_region(state.files, {}); } else { + // A placement that is not a patch one still lives in the + // base install, whichever mod's copy referenced it. + const auto decode = [&](const Contribution& from) { + return fb::decode_cas( + store.read(from.file.location.patch ? from.root : baseRoot, + from.file.location, from.file.offset, from.file.size), + {gameRoot}); + }; + HeldAssets held(state, decode); // Two mods rewriting one EBX asset is the case layering can never // answer: whichever copy wins, the other mod's additions are gone. // Combining the documents is the only shape that keeps both. @@ -474,16 +843,11 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, for (const auto& contribution : found->second) if (!contribution.base && contribution.sha1 != baseCopy->sha1) edits.push_back(&contribution); - if (edits.size() < 2) continue; + // An item list is looked at even when one mod alone changed it: + // what its entries name has to be in the bundle either way. + const auto named = entry.asset.kind == fb::AssetKind::ebx && item_list_name(entry.asset.name); + if (edits.size() < 2 && !(named && edits.size() == 1)) continue; try { - // A placement that is not a patch one still lives in the - // base install, whichever mod's copy referenced it. - const auto decode = [&](const Contribution& from) { - return fb::decode_cas( - store.read(from.file.location.patch ? from.root : baseRoot, - from.file.location, from.file.offset, from.file.size), - {gameRoot}); - }; const auto baseBytes = decode(*baseCopy); std::vector> editBytes; for (const auto* edit : edits) editBytes.push_back(decode(*edit)); @@ -533,6 +897,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, what += ", " + std::to_string(table.conflicts) + " disagreed"; } else { const auto baseDocument = fb::ebx::read_document(baseBytes); + const auto itemList = lower(baseDocument.rootType) == item_list_type; + if (edits.size() < 2 && !itemList) continue; // A material grid is a square interaction matrix whose rows are // addressed by slot numbers baked into each map's collision. // Two mods that each appended surfaces both claim the same @@ -550,12 +916,84 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, editDocuments.push_back(fb::ebx::read_document(bytes)); std::vector pointers; for (const auto& document : editDocuments) pointers.push_back(&document); + // An item a mod lists is an asset that mod adds to this bundle. + // When it is not here (another mod added a different asset + // under its name, or the mod was built on top of a mod that + // is not installed) the entry goes: the mod loses that item, + // which it had already lost, and the game still starts. + const fb::ebx::CarryImport inBundle = [&](const fb::ebx::ImportReference& item) { + // What the game's own list names is the game's to answer for. + return std::ranges::any_of(baseDocument.imports, [&](const fb::ebx::ImportReference& known) { + return known.fileGuid == item.fileGuid; }) || + held.holds(item.fileGuid); + }; + // Any other list loses an entry only when what it names was in + // this bundle and another mod's asset of the same name took its + // place: nothing else is known about where its entries live. + const fb::ebx::CarryImport notReplaced = [&](const fb::ebx::ImportReference& named) { + return !held.replacement(named.fileGuid) || held.holds(named.fileGuid); + }; fb::ebx::MergeSummary summary; - auto combined = fb::ebx::merge_documents(baseDocument, pointers, &summary); - if (!summary.instances && !summary.arrayEntries) continue; + auto combined = fb::ebx::merge_documents(baseDocument, pointers, &summary, + itemList ? inBundle : notReplaced); + const auto mod_of = [&](std::size_t edit) { return path_utf8(edits[edit]->root.filename()); }; + if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries) { + // One mod's copy with nothing to leave out stays as that mod wrote it. + if (edits.size() < 2) continue; + // The same change from each of them is one change. + if (std::ranges::all_of(edits, [&](const Contribution* edit) { + return edit->sha1 == edits.front()->sha1; })) continue; + // Nothing was added to it, so each mod changed what was there. + // Values changed in place combine; when that is not all of + // some mod's change, the copy already kept stays whole, and + // the others' changes to it are not in the game. + if (!summary.values || !summary.exact()) { + std::set others; + for (std::size_t index = 0; index < edits.size(); ++index) + if (edits[index]->sha1 != entry.asset.sha1) others.insert(mod_of(index)); + if (entry.mod) others.erase(*entry.mod); + std::string names; + for (const auto& other : others) names += (names.empty() ? "" : ", ") + other; + if (!others.empty()) + report.notes.push_back(entry.asset.name + ": kept " + + (entry.mod ? *entry.mod + "'s copy" : std::string("the highest-priority copy")) + + " whole; the changes " + names + " made to it are of a kind that cannot be " + "combined with it and are not in the game"); + continue; + } + } rebuilt = fb::ebx::write_document(combined); what = std::to_string(summary.instances) + " instance(s), " + std::to_string(summary.arrayEntries) + " list entries"; + if (summary.values) what += ", " + std::to_string(summary.values) + " changed value(s)"; + if (summary.contested) what += ", " + std::to_string(summary.contested) + " disagreed"; + if (summary.repeated) + what += ", " + std::to_string(summary.repeated) + " repeated entry(ies) listed once"; + // The merged copy is the game's with what the mods added: a mod's + // change to what was already there is in it only when it is one + // that could be taken. + for (const auto index : summary.uncarried) + report.notes.push_back(mod_of(index) + ": " + entry.asset.name + + ": the merged copy has what this mod added to it, but not its changes to what " + "the game's copy already held"); + std::map> left; // by edit + for (const auto& gone : summary.dropped) left[gone.edit].push_back(gone.target.fileGuid); + for (const auto& [index, guids] : left) { + // Name the mod whose asset took the item's name, when that is why. + const Asset* taken{}; + for (const auto& guid : guids) { + taken = held.replacement(guid); + if (taken) break; + } + report.notes.push_back(mod_of(index) + ": " + entry.asset.name + + ": " + std::to_string(guids.size()) + (itemList ? " item(s)" : " entry(ies)") + + " left out of the list because the merged " + "bundle does not hold them, e.g. " + + (taken ? taken->asset.name + ", replaced by " + (taken->mod ? *taken->mod : "another mod") + + "'s asset of the same name" + : "the asset with guid " + guids.front().string() + + ", which no enabled mod adds to the bundle")); + } } const auto placement = store.write(state.manifestChunk, manifestArchive, fb::encode_cas(rebuilt, {gameRoot})); @@ -563,6 +1001,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, entry.asset.sha1 = sha1_of(rebuilt); entry.asset.originalSize = rebuilt.size(); used.emplace(placement.location.installChunk, placement.location.archive); + if (edits.size() < 2) continue; report.notes.push_back(entry.asset.name + ": combined " + std::to_string(edits.size()) + " edits (" + what + ")"); ++report.mergedAssets; @@ -589,9 +1028,36 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, } } + // The merged bundle holds every mod's chunks, so its chunk metadata has + // to describe them all, and in the game's own form: a record for each + // chunk, in the order of the chunks' guids. With the game's list alone, + // or with the lists the mods ship (which are not in that order, and + // have lost the records their tools wrote over), the meshes and textures + // of costumes built on one game costume never stream in. + std::vector metadata; + { + std::vector records; + std::size_t fromMods{}; + for (const auto& entry : state.assets) { + if (entry.asset.kind != fb::AssetKind::chunk) continue; + records.push_back(entry.record); + fromMods += entry.record.shipped == ChunkRecord::none || entry.record.copy != state.gameMetadata; + } + try { + metadata = merge_chunk_metadata(state.metadata, records, state.gameMetadata, state.shippedChunks); + if (fromMods && state.gameMetadata != ChunkRecord::none && + metadata != state.metadata[state.gameMetadata]) + report.notes.push_back(name + ": chunk metadata written again for " + std::to_string(records.size()) + + " chunks, " + std::to_string(fromMods) + " of them added, changed or carried by mods"); + } catch (const std::exception& failure) { + if (!state.metadata.empty()) metadata = state.metadata.front(); + report.notes.push_back(name + ": the copies' chunk metadata could not be put together (" + failure.what() + + "); kept the first copy's, so chunks mods add to this bundle may not load"); + } + } fb::BinaryBundle manifest; std::vector files; - build_manifest(state.assets, state.metadata, manifest, files); + build_manifest(state.assets, metadata, manifest, files); if (!state.casBacked) { region = fb::write_bundle_region(files, fb::write_binary_bundle(manifest)); } else { @@ -637,6 +1103,20 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, merged.chunks.push_back(chunk); } } + // This superbundle resolves chunks from its own TOC: the audio of a song another + // mod added to a bundle copied here has to be listed here as well. + for (const auto& name : addedFrom) { + const auto found = overrides.chunks.find(name); + if (found == overrides.chunks.end()) continue; + std::size_t listed{}; + for (const auto& chunk : found->second) { + if (!chunkAt.emplace(chunk.guid, merged.chunks.size()).second) continue; + merged.chunks.push_back(chunk); + used.emplace(chunk.location.installChunk, chunk.location.archive); + ++listed; + } + if (listed) report.notes.push_back(relative + ": " + std::to_string(listed) + " chunk(s) from " + name); + } if (sources.size() > 1) { report.notes.push_back(relative + ": combined " + std::to_string(sources.size()) + diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 8f04fb1..722ae95 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -80,8 +80,11 @@ struct PlacementRecord { PlacementRecord read_placements(const fs::path& file); void write_placements(const fs::path& file, const PlacementRecord& record); +// `storeKnown`: the content cache that says what the game's store sells is +// installed, so the mods were checked for copies of store items; a patch built +// without it is built again once it is there. std::string merge_fingerprint(const Catalog& catalog, const std::vector& mods, - const std::map& modFiles); + const std::map& modFiles, bool storeKnown); inline constexpr wchar_t stamp_file[] = L"reskate-merge.stamp"; std::optional previous_merge(const fs::path& output, const std::string& fingerprint); void write_stamp(const fs::path& output, const std::string& fingerprint, const MergeReport& report); @@ -174,18 +177,93 @@ struct AssetOverride { fb::Sha1 sha1; // the mod's version std::uint64_t originalSize{}; std::vector encoded; // its payload, as stored in cas + // Present for a Lua resource replacement: its identity and metadata must + // travel with the script bytes, including source and bytecode sizes. + std::optional resource; + // The documents the changed EBX refers to that its mod added to the bundle it + // made the change in, by their guids. They go wherever the change goes: the + // game keeps a list in more bundles than one, each with what the list names. + std::set names; }; -// By lower-case asset name, then the game's sha1 the change replaces. -using AssetOverrides = std::map, std::less<>>; +inline constexpr std::uint32_t luaScriptResourceType = 0xEC383B87; +// An EBX asset a mod added to one of the game's bundles, or a resource it added +// under the same name as such an asset (a wave's sound-bank). Maps carry their own +// copy of a bundle like bam_coregameassets, and that copy loads instead of the +// game's on their levels; an edit to a list there (the master music playlist) +// propagates as a change, so the asset it now names has to come along too. The +// game also keeps such a list in more bundles than the one the mod put its +// assets in, and a copy of any of those takes the change the same way. +struct AssetAddition { + std::string mod; + fb::BundleAsset asset; + std::vector encoded; // its payload, as stored in cas + // The lower-case relative path of the TOC the mod ships this bundle in. A copy of the + // bundle in that same TOC merges with the mod's own bundle and already has the asset; + // only copies in other superbundles (a map's level TOC) need it carried. + std::string toc; + // For an EBX: its document's guid, which is what a list names it by, and the + // guids of the documents it refers to in turn (a song names its wave). + fb::Guid file; + std::vector names; +}; +struct AssetOverrides { + // By lower-case asset name, then the game's sha1 the change replaces. + std::map, std::less<>> changed; + // Separate from EBX: a script's EBX wrapper can have the same asset name. + std::map, std::less<>> scripts; + // By lower-case bundle name, in priority order. + std::map, std::less<>> added; + // By mod folder name: TOC chunks the mod adds that its carried additions name + // (a new wave's audio). A map's superbundle resolves chunks from its own TOC, + // so wherever the additions go these entries go too. Collected pointing into + // the mod's own archives; the merge shifts them to where those archives landed. + std::map, std::less<>> chunks; -// The changes asset mods (mods that add no levels) make to the game's own EBX, -// the highest-priority mod's change winning. Never throws: an unreadable mod -// is noted and simply changes nothing elsewhere. + [[nodiscard]] bool empty() const noexcept { return changed.empty() && scripts.empty() && added.empty(); } +}; + +// The changes and additions asset mods (mods that add no levels) make to the +// game's own EBX and Lua scripts, the highest-priority mod's version winning. Never throws: an +// unreadable mod is noted and simply changes nothing elsewhere. AssetOverrides collect_asset_overrides(const std::vector& mods, const std::map& modFiles, const CasStore& store, const fs::path& baseRoot, const fs::path& gameRoot, MergeReport& report); +// A bundle's chunk metadata is a list with one record for each of its chunks: the +// hash of the name of the resource the chunk belongs to ("h64") and what the +// engine needs before it streams the chunk (a texture's first mip). In the game's +// own bundles the records are in the order of the chunks' guids, not the order +// the chunks are listed in. A mod's copy starts from the game's list as it is and +// has, at the place each chunk it adds or replaces is listed at, a record its +// tool wrote: for an added chunk that is the chunk's own record, but for a +// replaced one it has taken the place of another chunk's (and older tools left +// its hash empty), so the game's record for that other chunk is gone from the +// mod's list. +struct ChunkRecord { + static constexpr std::size_t none = static_cast(-1); + fb::Guid guid; + std::size_t copy{none}; // the list of the copy this version of the chunk came in; none when that copy has no list + std::size_t index{}; // where that copy lists the chunk + std::size_t shipped{none}; // where the game's copy lists it; none for a chunk mods add +}; +// The list for a merged bundle, written as the game writes one: a record for each +// of `chunks` (the bundle's chunks, as listed), in the order of their guids. A +// chunk of the game's keeps the game's record for it, whichever copy carries it +// (`game` is the game's list among `lists`, `shipped` the chunks the game's copy +// lists); when the kept version is one a mod replaced it with, the first mip that +// mod's record gives goes into it. A chunk a mod adds has that mod's record, and +// one nothing describes gets a record that says nothing. In a bundle the game +// does not ship, or ships without a list, every chunk is one a mod adds. When +// every chunk is the game's own where the game lists it, the game's list comes +// back as it is; so does one mod's, for a bundle without a list in the game that +// is that mod's copy alone. +// Throws when a list that is needed cannot be read. +[[nodiscard]] std::vector merge_chunk_metadata(std::span> lists, + std::span chunks, + std::size_t game = ChunkRecord::none, + std::span shipped = {}); + fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, std::vector& sources, MergeReport& report, const std::string& relative, ArchiveUse& used, CasStore& store, diff --git a/Engine/Vfs/mod_merge_overrides.cpp b/Engine/Vfs/mod_merge_overrides.cpp index 0bb612d..049e7c7 100644 --- a/Engine/Vfs/mod_merge_overrides.cpp +++ b/Engine/Vfs/mod_merge_overrides.cpp @@ -1,5 +1,12 @@ #include "mod_merge_internal.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_merge.h" +#include "Engine/Resource/ebx_writer.h" +#include +#include +#include #include namespace dingosdk::mods::detail { @@ -17,6 +24,30 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto files = modFiles.find(mod); if (files == modFiles.end()) continue; std::size_t changed{}; + // This mod's new assets, and the partitions its recorded changes import. + // An added asset only follows a change that refers to it: copied on its + // own, it could arrive in another mod's bundle without the resources + // and chunks it was added with. + std::vector> candidates, companions; + std::vector newChunks; // TOC chunks the game's TOCs do not have + std::set imported; + // The game's assets this mod changed. They wait until its additions are + // settled: one that another mod's asset of the same name stands in for + // is not where a copy can find it, and a change must not name it there. + struct Change { + std::string bundle; // lower case: where the mod changed it + std::string name; // lower case + std::string asset; // as the bundle spells it + fb::Sha1 game; // the game's copy it replaces + fb::Sha1 sha1; + std::uint64_t originalSize{}; + std::vector encoded; + std::optional gameFile; // where the game's copy is + }; + std::vector changes; + const auto decoded = [&](std::span encoded) { + return fb::ebx::read_document(fb::decode_cas(encoded, {gameRoot})); + }; for (const auto& relative : files->second.tocs) { std::error_code error; const auto baseToc = baseRoot / fs::path(relative); @@ -26,36 +57,244 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto game = fb::read_toc(read_file(baseToc)); std::map> gameBundles; for (const auto& bundle : game.bundles) gameBundles.emplace(lower(bundle.name), &bundle); + std::set gameChunks; + for (const auto& chunk : game.chunks) gameChunks.insert(chunk.guid); + for (const auto& chunk : own.chunks) + if (!chunk.removed && chunk.location.patch && !gameChunks.contains(chunk.guid)) newChunks.push_back(chunk); for (const auto& bundle : own.bundles) { const auto shipped = gameBundles.find(lower(bundle.name)); if (shipped == gameBundles.end()) continue; const auto modListing = list_bundle(store, mod->directory, baseRoot, bundle, gameRoot); const auto gameListing = list_bundle(store, baseRoot, baseRoot, *shipped->second, gameRoot); if (!modListing || !gameListing) continue; - std::map> original; - for (const auto& asset : gameListing->manifest.ebx) original.emplace(lower(asset.name), asset.sha1); + // The game's copy of each asset: its sha1, and where in the listing it is. + std::map, std::less<>> original; + for (std::size_t index = 0; index < gameListing->manifest.ebx.size(); ++index) { + const auto& asset = gameListing->manifest.ebx[index]; + original.emplace(lower(asset.name), std::pair{asset.sha1, index}); + } for (std::size_t index = 0; index < modListing->manifest.ebx.size(); ++index) { const auto& asset = modListing->manifest.ebx[index]; const auto name = lower(asset.name); const auto game_copy = original.find(name); - if (game_copy == original.end() || game_copy->second == asset.sha1) continue; - auto& versions = out[name]; - if (versions.contains(game_copy->second)) continue; + if (game_copy != original.end() && game_copy->second.first == asset.sha1) continue; const auto at = modListing->first + index; if (at >= modListing->files.size()) continue; const auto& file = modListing->files[at]; - versions.emplace(game_copy->second, + const auto payload = [&] { + return store.read(file.location.patch ? mod->directory : baseRoot, + file.location, file.offset, file.size); + }; + if (game_copy == original.end()) { + candidates.push_back({lower(bundle.name), {mod->name, asset, payload(), lower(relative)}}); + continue; + } + auto encodedPayload = payload(); + try { + for (const auto& reference : decoded(encodedPayload).imports) + imported.insert(reference.fileGuid); + } catch (const std::exception&) {} + // A higher mod's version of the asset, which this change is + // combined with below, may name what this mod adds. + if (const auto versions = out.changed.find(name); versions != out.changed.end()) + if (const auto existing = versions->second.find(game_copy->second.first); + existing != versions->second.end()) try { + for (const auto& reference : decoded(existing->second.encoded).imports) + imported.insert(reference.fileGuid); + } catch (const std::exception&) {} + std::optional gameFile; + if (const auto gameAt = gameListing->first + game_copy->second.second; + gameAt < gameListing->files.size()) + gameFile = gameListing->files[gameAt]; + changes.push_back({lower(bundle.name), name, asset.name, game_copy->second.first, asset.sha1, + asset.originalSize, std::move(encodedPayload), gameFile}); + } + // Maps carry stock SkaterLoader too. Replacing it only in the game's + // bundles loses custom cosmetic material targets on a map transition. + // Other resource types retain their own table/dependency merge rules. + std::map> gameScripts; + for (const auto& asset : gameListing->manifest.resources) + if (asset.resourceType == luaScriptResourceType) gameScripts.emplace(lower(asset.name), &asset); + for (std::size_t index = 0; index < modListing->manifest.resources.size(); ++index) { + const auto& asset = modListing->manifest.resources[index]; + if (asset.resourceType != luaScriptResourceType) continue; + const auto name = lower(asset.name); + const auto originalScript = gameScripts.find(name); + if (originalScript == gameScripts.end() || originalScript->second->sha1 == asset.sha1 || + originalScript->second->resourceId != asset.resourceId) continue; + auto& versions = out.scripts[name]; + if (versions.contains(originalScript->second->sha1)) continue; + const auto at = modListing->first + modListing->manifest.ebx.size() + index; + if (at >= modListing->files.size()) continue; + const auto& file = modListing->files[at]; + versions.emplace(originalScript->second->sha1, AssetOverride{mod->name, asset.sha1, asset.originalSize, - store.read(file.location.patch ? mod->directory : baseRoot, - file.location, file.offset, file.size)}); + store.read(file.location.patch ? mod->directory : baseRoot, + file.location, file.offset, file.size), asset}); ++changed; } + // Resources the mod adds, kept to go with an added EBX of the same name + // (a wave's sound-bank resource registers the wave with the audio system). + std::set> shippedResources; + for (const auto& asset : gameListing->manifest.resources) shippedResources.insert(lower(asset.name)); + for (std::size_t index = 0; index < modListing->manifest.resources.size(); ++index) { + const auto& asset = modListing->manifest.resources[index]; + const auto at = modListing->first + modListing->manifest.ebx.size() + index; + if (shippedResources.contains(lower(asset.name)) || at >= modListing->files.size()) continue; + const auto& file = modListing->files[at]; + companions.push_back({lower(bundle.name), {mod->name, asset, + store.read(file.location.patch ? mod->directory : baseRoot, file.location, file.offset, file.size), lower(relative)}}); + } } } catch (const std::exception& failure) { report.notes.push_back(mod->name + ": " + relative + ": its changes could not be read for other mods' copies (" + failure.what() + ")"); } } + // One addition per kind and name in a bundle, the highest-priority mod's. Two + // mods adding the same asset is no clash; two different assets under one name + // is, and only one of them can be what a copy gets: say whose, so a song or an + // item that goes missing on a map can be traced to the mod that took its name. + std::map, std::less<>> shadowed; // by the mod kept + // This mod's added EBX that another mod's different document stands in for: what + // names one by its guid finds nothing in a copy that was given the other's. + std::set lost; + const auto keep = [&](const std::string& bundle, AssetAddition addition) { + auto& list = out.added[bundle]; + const auto name = lower(addition.asset.name); + const auto holder = std::ranges::find_if(list, [&](const AssetAddition& other) { + return other.asset.kind == addition.asset.kind && lower(other.asset.name) == name; }); + if (holder != list.end()) { + if (holder->mod != addition.mod && holder->asset.sha1 != addition.asset.sha1) { + auto& [count, example] = shadowed[holder->mod]; + if (!count++) example = addition.asset.name; + if (addition.asset.kind == fb::AssetKind::ebx && holder->file != addition.file) + lost.insert(addition.file); + } + return false; + } + list.push_back(std::move(addition)); + return true; + }; + // Transitively: an addition a following addition imports follows too (a new song + // imports its new wave, and only the song is named by the changed playlist). + std::vector followed(candidates.size()); + // The documents this mod added to each bundle, by guid. + std::map, std::less<>> addedIn; + for (std::size_t index = 0; index < candidates.size(); ++index) { + auto& addition = candidates[index].second; + try { + const auto document = decoded(addition.encoded); + addition.file = document.fileGuid; + for (const auto& reference : document.imports) addition.names.push_back(reference.fileGuid); + addedIn[candidates[index].first].insert(addition.file); + } catch (const std::exception&) { followed[index] = true; } // unreadable: never follows + } + for (bool grew = true; grew;) { + grew = false; + for (std::size_t index = 0; index < candidates.size(); ++index) { + auto& [bundle, addition] = candidates[index]; + if (followed[index] || !imported.contains(addition.file)) continue; + followed[index] = grew = true; + imported.insert(addition.names.begin(), addition.names.end()); + keep(bundle, std::move(addition)); + } + } + for (auto& [bundle, companion] : companions) { + // Looked up, not made: a bundle with nothing carried must stay out of `added`. + const auto found = out.added.find(bundle); + if (found == out.added.end()) continue; + const auto name = lower(companion.asset.name); + if (std::ranges::any_of(found->second, [&](const AssetAddition& addition) { + return addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx && + lower(addition.asset.name) == name; })) + keep(bundle, std::move(companion)); + } + for (auto& change : changes) { + // What the change names of the assets the mod added beside it. The mod + // put them in one bundle with what names them, so they stay together. + std::set names; + const auto beside = addedIn.find(change.bundle); + try { + auto document = decoded(change.encoded); + const auto stays = [&](const fb::ebx::ImportReference& reference) { + return !lost.contains(reference.fileGuid); }; + if (!std::ranges::all_of(document.imports, stays)) + if (const auto gone = fb::ebx::drop_root_references(document, stays)) { + const auto rebuilt = fb::ebx::write_document(document); + change.sha1 = sha1_of(rebuilt); + change.originalSize = rebuilt.size(); + change.encoded = fb::encode_cas(rebuilt, {gameRoot}); + report.notes.push_back(mod->name + ": " + change.asset + ": " + std::to_string(gone) + + " entry(ies) left out of the copies other mods carry, where another mod's asset of " + "the same name stands in for what they name"); + } + if (beside != addedIn.end()) + for (const auto& reference : document.imports) + if (beside->second.contains(reference.fileGuid) && stays(reference)) + names.insert(reference.fileGuid); + } catch (const std::exception&) {} + auto& versions = out.changed[change.name]; + const auto existing = versions.find(change.game); + if (existing == versions.end()) { + versions.emplace(change.game, AssetOverride{mod->name, change.sha1, change.originalSize, + std::move(change.encoded), std::nullopt, std::move(names)}); + ++changed; + continue; + } + // The same change again, from another of the mod's bundles or another mod. + if (existing->second.sha1 == change.sha1) { + existing->second.names.insert(names.begin(), names.end()); + continue; + } + if (!change.gameFile) continue; + try { + const auto baseDoc = decoded(store.read(baseRoot, change.gameFile->location, + change.gameFile->offset, change.gameFile->size)); + const auto existingDoc = decoded(existing->second.encoded); + const auto thisDoc = decoded(change.encoded); + // Lowest priority first, the order a bundle's own copies are combined + // in: where two mods changed one value, the higher mod's is kept. + const std::array editDocs{&thisDoc, &existingDoc}; + fb::ebx::MergeSummary summary; + auto combined = fb::ebx::merge_documents(baseDoc, editDocs, &summary); + // Changed values alone are only worth a copy that has all of them. + if (!summary.instances && !summary.arrayEntries && !(summary.values && summary.exact())) continue; + const auto rebuilt = fb::ebx::write_document(combined); + existing->second.sha1 = sha1_of(rebuilt); + existing->second.originalSize = rebuilt.size(); + existing->second.encoded = fb::encode_cas(rebuilt, {gameRoot}); + if (existing->second.mod != mod->name && !existing->second.mod.ends_with(" + " + mod->name)) + existing->second.mod += " + " + mod->name; + existing->second.names.insert(names.begin(), names.end()); + } catch (const std::exception&) {} + } + for (const auto& [other, clash] : shadowed) + report.notes.push_back(mod->name + ": " + std::to_string(clash.first) + + " added asset(s) share a name with ones " + other + " adds, e.g. " + clash.second + + "; other mods' copies of the bundle get " + other + "'s"); + // The new TOC chunks this mod's carried EBX name (as raw GUID bytes, the way an + // EBX stores a ChunkId), so they can follow those assets into other superbundles. + if (!newChunks.empty()) { + std::vector> carried; + for (const auto& [bundle, list] : out.added) + for (const auto& addition : list) + if (addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx) try { + carried.push_back(fb::decode_cas(addition.encoded, {gameRoot})); + } catch (const std::exception&) {} + std::set taken; + for (const auto& chunk : newChunks) { + const auto& id = chunk.guid.bytes; + if (!taken.contains(chunk.guid) && std::ranges::any_of(carried, [&](const std::vector& bytes) { + return std::search(bytes.begin(), bytes.end(), id.begin(), id.end()) != bytes.end(); })) { + out.chunks[mod->name].push_back(chunk); + taken.insert(chunk.guid); + } + } + if (!taken.empty()) + report.notes.push_back(mod->name + ": " + std::to_string(taken.size()) + + " added chunk(s) follow its added assets into other mods' superbundles"); + } if (changed) report.notes.push_back(mod->name + ": " + std::to_string(changed) + " changed asset(s) also apply to the copies other mods carry"); diff --git a/Engine/Vfs/mod_merge_stamp.cpp b/Engine/Vfs/mod_merge_stamp.cpp index 3161150..92c80ef 100644 --- a/Engine/Vfs/mod_merge_stamp.cpp +++ b/Engine/Vfs/mod_merge_stamp.cpp @@ -45,9 +45,10 @@ fs::path sdk_module() { } // namespace std::string merge_fingerprint(const Catalog& catalog, const std::vector& mods, - const std::map& modFiles) { + const std::map& modFiles, bool storeKnown) { std::string inputs = stamp_header; inputs += '\n'; + inputs += storeKnown ? "store known\n" : "store unknown\n"; const auto describe = [&](std::string_view label, const fs::path& path) { inputs += label; std::error_code error; diff --git a/Engine/Vfs/mod_music.cpp b/Engine/Vfs/mod_music.cpp new file mode 100644 index 0000000..5fa56da --- /dev/null +++ b/Engine/Vfs/mod_music.cpp @@ -0,0 +1,60 @@ +#include "Engine/Vfs/mod_music.h" +#include "Engine/Core/Json/json.h" +#include +#include + +namespace dingosdk::mods { +namespace { +bool text_ok(const std::string& value) { + return !value.empty() && value.size() <= 255 && + std::none_of(value.begin(), value.end(), [](unsigned char c) { return c < 32 || c == 127; }); +} +} + +bool music_artwork_path(const std::string& path) { + if (!text_ok(path) || path.front() == '/' || path.find_first_of("\\:%?#") != std::string::npos || + !path.ends_with(".png")) return false; + std::size_t start = 0; + while (start < path.size()) { + const auto end = path.find('/', start); + const auto part = path.substr(start, end == std::string::npos ? end : end - start); + if (part.empty() || part == "." || part == ".." || part.back() == ' ' || part.back() == '.') return false; + if (end == std::string::npos) break; + start = end + 1; + } + return true; +} + +std::vector parse_music_playlists(const std::string& mod, const std::string& json) { + std::vector result; + const auto root = Json::parse(json); + if (!root.contains("schema") || root.at("schema").get() != 1 || !root.contains("playlists") || + !root.at("playlists").is_array()) + throw std::runtime_error("expected schema 1 with a playlists array"); + const auto& list = root.at("playlists"); + std::map covers; + if (root.contains("song_artwork") && root.at("song_artwork").is_object()) { + for (const auto& [id, value] : root.at("song_artwork").items()) { + if (covers.size() >= 4096) break; + if (text_ok(id) && value.is_string() && music_artwork_path(value.string())) covers.emplace(id, value.string()); + } + } + for (std::size_t i = 0; i < list.size() && i < 64; ++i) { + const auto& entry = list.at(i); + if (!entry.contains("name") || !entry.at("name").is_string() || !text_ok(entry.at("name").string()) || + !entry.contains("songs") || !entry.at("songs").is_array()) + throw std::runtime_error("a playlist needs a name and a songs array"); + MusicPlaylist playlist{"mod:" + mod + ":" + entry.at("name").string(), entry.at("name").string(), {}}; + if (entry.contains("artwork") && entry.at("artwork").is_string() && music_artwork_path(entry.at("artwork").string())) + playlist.artwork = entry.at("artwork").string(); + const auto& songs = entry.at("songs"); + for (std::size_t s = 0; s < songs.size() && s < 1024; ++s) + if (songs.at(s).is_string() && text_ok(songs.at(s).string())) playlist.songs.push_back(songs.at(s).string()); + for (const auto& id : playlist.songs) + if (const auto found = covers.find(id); found != covers.end()) playlist.song_artwork.emplace(*found); + if (text_ok(playlist.id)) result.push_back(std::move(playlist)); + } + return result; +} + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_music.h b/Engine/Vfs/mod_music.h new file mode 100644 index 0000000..ea30e0f --- /dev/null +++ b/Engine/Vfs/mod_music.h @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include + +namespace dingosdk::mods { + +// A playlist a mod declares for the game's music screen. `songs` are song ids, the +// "artist - title" the game registers a song under. +struct MusicPlaylist { + std::string id; // "mod::" + std::string name; // shown in the music screen + std::vector songs; + std::string artwork; // relative PNG path inside the mod; empty means no cover + std::map song_artwork; +}; + +// Reads a mod's reskate-music.json: +// {"schema":1,"playlists":[{"name":"...","songs":["artist - title", ...]}]} +// Optional playlist "artwork" and root "song_artwork" (song-id -> PNG path) use mod-relative paths. +// Invalid optional artwork is ignored, without dropping valid music membership. +// Text must be non-empty, at most 255 bytes and free of control characters; a song entry that is +// not such text is dropped, as are playlists beyond 64 and songs beyond 1024 per playlist. +// Throws std::exception on a document that is not this shape, so the caller can skip the file. +std::vector parse_music_playlists(const std::string& mod, const std::string& json); + +// A portable mod-relative PNG path: no traversal, drive names or alternate streams. +bool music_artwork_path(const std::string& path); + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_store_copies.cpp b/Engine/Vfs/mod_store_copies.cpp new file mode 100644 index 0000000..5c566a8 --- /dev/null +++ b/Engine/Vfs/mod_store_copies.cpp @@ -0,0 +1,401 @@ +#include "mod_store_copies.h" + +#include "mod_catalog.h" +#include "mod_merge_internal.h" +#include "Engine/Resource/ebx_document.h" + +#include +#include + +namespace dingosdk::mods { +using namespace detail; +namespace { +// Where the game keeps its items, the store's among them, and how it names them: +// most under items/, the rest (player-card art, skin tones) after their key, +// and every key the store sells starts with own_. A cheap way to leave the +// textures and presets that fill the rest of the bundle unread. +constexpr std::string_view items_toc = "Win32/items.toc"; +bool item_name(std::string_view name) { + const auto slash = name.rfind('/'); + return name.starts_with("items/") || + name.substr(slash == std::string_view::npos ? 0 : slash + 1).starts_with("own_"); +} + +// An item's AssetPaths say what kind of asset each one is: 1 a morph preset and +// 2 a base mesh, which are what the item is fitted on; every other kind (3, an +// appearance preset, for nearly all of them) is what it looks like. Of the +// game's own items, store and free ones share shapes and never a look. +constexpr bool shape_kind(std::int64_t kind) noexcept { return kind == 1 || kind == 2; } + +std::string hex(const fb::Sha1& sha1) { + static constexpr char digits[] = "0123456789abcdef"; + std::string text; + for (const auto value : sha1.bytes) { + const auto byte = static_cast(value); + text += digits[byte >> 4]; + text += digits[byte & 15]; + } + return text; +} + +const std::string* text_of(const fb::ebx::Object& object, std::string_view name) { + const auto* field = object.find(name); + return field ? std::get_if(&field->value.data) : nullptr; +} +std::optional integer_of(const fb::ebx::Object& object, std::string_view name) { + const auto* field = object.find(name); + if (!field) return std::nullopt; + if (const auto* value = std::get_if(&field->value.data)) return *value; + if (const auto* value = std::get_if(&field->value.data)) return static_cast(*value); + return std::nullopt; +} + +// Walks an item's data: every field goes into `text`, in order and with what +// its pointers lead to, and the fields that say what the item is go into its +// looks and shapes. +class ItemReader final { +public: + ItemReader(const fb::ebx::Document& document, ItemContent& item, std::string& text) + : document_(document), item_(item), text_(text) {} + + void value(const fb::ebx::Value& value, int depth = 0) { + // Deeper than any item's data goes; the walk has to end on any document. + if (depth > 32) { text_ += '?'; return; } + struct Visit { + ItemReader& reader; + int depth; + void operator()(std::monostate) const { reader.text_ += '~'; } + void operator()(bool held) const { reader.text_ += held ? "true" : "false"; } + void operator()(std::int64_t held) const { reader.text_ += std::to_string(held); } + void operator()(std::uint64_t held) const { reader.text_ += std::to_string(held); } + void operator()(double held) const { reader.text_ += std::to_string(held); } + void operator()(const std::string& held) const { reader.text_ += '"' + lower(held) + '"'; } + void operator()(const fb::Guid& held) const { reader.text_ += held.string(); } + void operator()(const fb::Sha1& held) const { reader.text_ += hex(held); } + void operator()(const fb::ebx::ResourceReference& held) const { reader.text_ += "resource " + std::to_string(held.id); } + void operator()(const fb::ebx::PointerReference& held) const { reader.pointer(held, depth); } + void operator()(const fb::ebx::TypeReference& held) const { reader.text_ += "type " + std::to_string(held.encoded); } + void operator()(const fb::ebx::BoxedReference& held) const { reader.text_ += "boxed " + std::to_string(held.encodedType); } + void operator()(const std::shared_ptr& held) const { + if (held) reader.object(*held, depth); + else reader.text_ += "null"; + } + void operator()(const fb::ebx::Value::Array& held) const { + reader.text_ += '['; + for (const auto& entry : held) { reader.value(entry, depth + 1); reader.text_ += ','; } + reader.text_ += ']'; + } + }; + std::visit(Visit{*this, depth}, value.data); + } + // Whether the data has a field at all; the player-card items have none. + [[nodiscard]] bool any() const noexcept { return fields_ != 0; } + +private: + [[nodiscard]] const fb::ebx::ImportReference* import(const fb::ebx::PointerReference& reference) const { + if (reference.kind != fb::ebx::PointerKind::external || reference.index < 0 || + static_cast(reference.index) >= document_.imports.size()) + return nullptr; + return &document_.imports[static_cast(reference.index)]; + } + static std::string named(const fb::ebx::ImportReference& reference) { + return reference.fileGuid.string() + '/' + reference.classGuid.string(); + } + + void pointer(const fb::ebx::PointerReference& reference, int depth) { + if (reference.kind == fb::ebx::PointerKind::null) { text_ += "null"; return; } + if (const auto* other = import(reference)) { text_ += "import " + named(*other); return; } + const auto index = static_cast(reference.index); + if (reference.kind != fb::ebx::PointerKind::internal || reference.index < 0 || + index >= document_.instances.size() || !document_.instances[index].object) { + text_ += "nothing"; + return; + } + // An instance that leads back to itself is written once. + if (std::find(open_.begin(), open_.end(), index) != open_.end()) { text_ += "again"; return; } + open_.push_back(index); + object(*document_.instances[index].object, depth); + open_.pop_back(); + } + + void object(const fb::ebx::Object& object, int depth) { + parts(object); + if (object.descriptor >= 0 && static_cast(object.descriptor) < document_.types.size()) + text_ += document_.types[static_cast(object.descriptor)].name; + text_ += '{'; + for (const auto& field : object.fields) { + ++fields_; + text_ += field.name; + text_ += '='; + value(field.value, depth + 1); + text_ += ';'; + } + text_ += '}'; + } + + void parts(const fb::ebx::Object& object) { + // Clothing, decks, wheels, trucks: the assets the item is made of, by name. + if (const auto* name = text_of(object, "AssetName"); name && !name->empty()) + if (const auto kind = integer_of(object, "AssetTypeId")) + (shape_kind(*kind) ? item_.shapes : item_.looks) + .push_back("asset " + std::to_string(*kind) + ' ' + lower(*name)); + // Stickers and tattoos: the textures of the decal. + if (const auto* name = text_of(object, "TextureName"); name && !name->empty()) + item_.looks.push_back("texture " + lower(*name)); + // Quick Drop objects: what is spawned, and the bundle it is in. + if (const auto* name = text_of(object, "BuildKitPrefabName"); name && !name->empty()) + item_.looks.push_back("prefab " + lower(*name)); + if (const auto* name = text_of(object, "BuildKitBundleName"); name && !name->empty()) + item_.looks.push_back("bundle " + lower(*name)); + // Player-card backgrounds and icons: the picture, which is another asset. + if (const auto* texture = object.find("ItemTexture")) + if (const auto* reference = std::get_if(&texture->value.data)) + if (const auto* other = import(*reference)) item_.looks.push_back("picture " + named(*other)); + // Emotes: a value of the game's gesture state, which is the animation played. + if (const auto* state = object.find("GameState")) + if (const auto gesture = integer_of(object, "Value")) + if (const auto* held = std::get_if>(&state->value.data); held && *held) + if (const auto* asset = (*held)->find("AssetRef")) + if (const auto* reference = std::get_if(&asset->value.data)) + if (const auto* other = import(*reference)) + item_.looks.push_back("gesture " + named(*other) + ' ' + std::to_string(*gesture)); + } + + const fb::ebx::Document& document_; + ItemContent& item_; + std::string& text_; + std::vector open_; + std::size_t fields_{}; +}; + +class Scan final { +public: + Scan(const Catalog& catalog, std::vector* notes) + : baseRoot_(catalog.data_root / L"Data"), gameRoot_(catalog.data_root), + layout_(vfs::read_layout(baseRoot_ / L"layout.toc")), + store_(baseRoot_, catalog.root / generated_folder, layout_.root), notes_(notes) {} + + // The items a mod adds to the game's bundles, and the game's own it changes. + // An item only loads from a bundle the game loads, and the lists the game + // reads its items from are in one of those, so a mod's own bundles (a map's + // levels) are not looked at. + [[nodiscard]] std::vector read_mod(const Mod& mod) { + std::vector items; + for (const auto& relative : scan(mod.directory).tocs) { + const auto* game = game_toc(relative); + if (!game) continue; + fb::TocDocument own; + try { + own = fb::read_toc(read_file(mod.directory / fs::path(relative))); + } catch (const std::exception& failure) { + note(mod.name + ": " + relative + " could not be read for " + std::string(store_copies_check) + " (" + failure.what() + ")"); + continue; + } + for (const auto& bundle : own.bundles) { + const auto shipped = game->find(lower(bundle.name)); + if (shipped == game->end()) continue; + try { + // A bundle the mod only passes through reads every file from the game. + const auto region = fb::read_bundle_region(bundle.region); + if (std::none_of(region.files.begin(), region.files.end(), + [](const fb::BundleFileInfo& file) { return file.location.patch; })) + continue; + const auto listing = list_bundle(store_, mod.directory, baseRoot_, bundle, gameRoot_); + const auto* known = listing ? game_assets(relative, *shipped->second) : nullptr; + if (!known) continue; + for (std::size_t index = 0; index < listing->manifest.ebx.size(); ++index) { + const auto& asset = listing->manifest.ebx[index]; + const auto at = listing->first + index; + // The mod's own payloads are in its own archives. + if (at >= listing->files.size() || !listing->files[at].location.patch) continue; + const auto original = known->find(lower(asset.name)); + if (original != known->end() && original->second == asset.sha1) continue; + try { + const auto document = fb::ebx::read_document( + read_asset(store_, mod.directory, baseRoot_, *listing, index, gameRoot_)); + if (auto item = item_content(document)) items.push_back(std::move(*item)); + } catch (const std::exception&) { + // Not an asset this reads; the game may not read it either. + } + } + } catch (const std::exception& failure) { + note(mod.name + ": " + bundle.name + " could not be read for " + std::string(store_copies_check) + " (" + failure.what() + ")"); + } + } + } + return items; + } + + // Every item the game ships; how many were read. + [[nodiscard]] std::size_t read_game(StoreItems& items, const StoreItem& sold) { + const auto* toc = game_toc(items_toc); + if (!toc) return 0; + std::size_t read{}; + for (const auto& [name, bundle] : *toc) { + try { + const auto listing = list_bundle(store_, baseRoot_, baseRoot_, *bundle, gameRoot_); + if (!listing) continue; + for (std::size_t index = 0; index < listing->manifest.ebx.size(); ++index) { + if (!item_name(lower(listing->manifest.ebx[index].name))) continue; + try { + const auto document = fb::ebx::read_document( + read_asset(store_, baseRoot_, baseRoot_, *listing, index, gameRoot_)); + const auto item = item_content(document); + if (!item) continue; + items.add(*item, sold(lower(item->key))); + ++read; + } catch (const std::exception&) {} + } + } catch (const std::exception& failure) { + note("The game's " + bundle->name + " could not be read for " + std::string(store_copies_check) + " (" + failure.what() + ")"); + } + } + return read; + } + +private: + using Bundles = std::map>; + using Assets = std::map>; + + const Bundles* game_toc(std::string_view relative) { + const auto key = lower(relative); + if (const auto found = tocs_.find(key); found != tocs_.end()) + return found->second.index.empty() ? nullptr : &found->second.index; + auto& entry = tocs_[key]; + std::error_code error; + const auto path = baseRoot_ / fs::path(std::string(relative)); + if (!fs::is_regular_file(path, error)) return nullptr; + try { + entry.document = fb::read_toc(read_file(path)); + } catch (const std::exception& failure) { + note("The game's " + std::string(relative) + " could not be read for " + std::string(store_copies_check) + " (" + failure.what() + ")"); + return nullptr; + } + for (const auto& bundle : entry.document.bundles) entry.index.emplace(lower(bundle.name), &bundle); + return entry.index.empty() ? nullptr : &entry.index; + } + + // The game's copy of a bundle: each EBX it holds, to tell a mod's additions + // and changes from what it carries untouched. Null when it cannot be read. + const Assets* game_assets(std::string_view relative, const fb::TocBundle& bundle) { + const auto key = lower(relative) + '|' + lower(bundle.name); + if (const auto found = assets_.find(key); found != assets_.end()) + return found->second ? &*found->second : nullptr; + auto& entry = assets_[key]; + const auto listing = list_bundle(store_, baseRoot_, baseRoot_, bundle, gameRoot_); + if (!listing) return nullptr; + entry.emplace(); + for (const auto& asset : listing->manifest.ebx) entry->emplace(lower(asset.name), asset.sha1); + return &*entry; + } + + void note(std::string text) { + if (notes_) notes_->push_back(std::move(text)); + } + + struct GameToc { + fb::TocDocument document; + Bundles index; + }; + fs::path baseRoot_, gameRoot_; + vfs::Layout layout_; + CasStore store_; + std::vector* notes_; + std::map> tocs_; + std::map, std::less<>> assets_; +}; +} // namespace + +std::optional item_content(const fb::ebx::Document& document) { + const auto* root = document.root(); + if (!root || !root->object) return std::nullopt; + const auto* key = text_of(*root->object, "Key"); + const auto* data = root->object->find("ItemData"); + if (!key || key->empty() || !data) return std::nullopt; + ItemContent item; + item.key = *key; + std::string text; + ItemReader reader(document, item, text); + reader.value(data->value); + for (auto* parts : {&item.looks, &item.shapes}) { + std::sort(parts->begin(), parts->end()); + parts->erase(std::unique(parts->begin(), parts->end()), parts->end()); + } + // Data with no field in it is every such item's alike, and says nothing. + if (reader.any()) item.data = hex(sha1_of(std::as_bytes(std::span(text)))); + return item; +} + +const std::string* StoreItems::Parts::owner(std::string_view part) const { + if (free.contains(part)) return nullptr; + const auto found = sold.find(part); + return found == sold.end() ? nullptr : &found->second; +} + +void StoreItems::add(const ItemContent& item, bool sold) { + const auto file = [&](Parts& parts, const std::string& part) { + if (sold) parts.sold.try_emplace(part, item.key); + else parts.free.insert(part); + }; + for (const auto& look : item.looks) file(looks_, look); + // A store item with a look is told by that; its shape is anyone's to use. + if (!sold || item.looks.empty()) + for (const auto& shape : item.shapes) file(shapes_, shape); + if (!item.data.empty()) file(data_, item.data); +} + +std::string StoreItems::original(const ItemContent& item) const { + for (const auto& look : item.looks) + if (const auto* key = looks_.owner(look)) return *key; + if (item.looks.empty()) + for (const auto& shape : item.shapes) + if (const auto* key = shapes_.owner(shape)) return *key; + if (!item.data.empty()) + if (const auto* key = data_.owner(item.data)) return *key; + return {}; +} + +StoreCopies check_store_copies(const Catalog& catalog, const StoreItem& sold, std::vector* notes) noexcept { + StoreCopies result; + try { + if (!sold || std::none_of(catalog.mods.begin(), catalog.mods.end(), [](const Mod& mod) { return mod.provides_layout; })) + return result; + Scan scan(catalog, notes); + std::vector>> added; + for (const auto& mod : catalog.mods) { + if (!mod.provides_layout) continue; + if (auto items = scan.read_mod(mod); !items.empty()) added.emplace_back(&mod, std::move(items)); + } + if (added.empty()) return result; + StoreItems game; + result.game_items = scan.read_game(game, sold); + if (!result.game_items) { + if (notes) notes->push_back(std::string(store_copies_check) + " could not read the game's own files and did not run"); + return result; + } + for (const auto& [mod, items] : added) { + StoreCopies::Source source{mod->name}; + for (const auto& item : items) { + const auto key = lower(item.key); + // A store item under its own key is held by that key already. + if (sold(key)) continue; + const auto original = game.original(item); + if (original.empty()) continue; + if (!source.count++) { + source.example = item.key; + source.original = original; + } + result.items.insert_or_assign(key, lower(original)); + } + if (source.count) result.mods.push_back(std::move(source)); + } + } catch (const std::exception& failure) { + if (notes) notes->push_back(std::string(store_copies_check) + " could not run: " + failure.what()); + result = {}; + } catch (...) { + result = {}; + } + return result; +} + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_store_copies.h b/Engine/Vfs/mod_store_copies.h new file mode 100644 index 0000000..c1952ee --- /dev/null +++ b/Engine/Vfs/mod_store_copies.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::frostbite::ebx { +struct Document; +} + +namespace dingosdk::mods { + +struct Catalog; + +// ReSkate hands out every cosmetic except the ones the game's store sells, and +// tells those apart by key (content_cache::Catalogs::reserved). A mod that adds +// a store item again under a key of its own gets round that: the copy is in no +// catalogue, so it is handed out like any other mod's item. What an item is +// does not change with its key, though. It still names the store item's +// appearance preset, decal textures, object or gesture, and those are what this +// check holds a mod's items against. A mod that adds such a copy is not loaded +// at all: the merge leaves it out whole (mod_merge.cpp), as it does a mod that +// cannot be merged. + +// All such a mod is told, in the launcher, the game and the log (which the +// mod's author reads too): that it could not be merged, and this. Nothing of +// what was looked for or found. The number is how ReSkate's own people tell it +// from a mod that really is damaged; the log names the check by it as well, +// when something could not be read for it. +inline constexpr std::string_view store_copies_problem = "merge error 0x5343"; +inline constexpr std::string_view store_copies_check = "merge check 0x5343"; + +// What an item is made of, whatever it is called. Lower-case; the lists sorted. +struct ItemContent { + std::string key; // the item's Key, as written + // What it looks like or does: appearance presets, decal textures, a + // player card's picture, an object's prefab and bundle, a gesture. + std::vector looks; + // The meshes and morph presets it is fitted on. Mods fit looks of their own + // on the game's shapes, a store item's included, so a shape alone says + // nothing. + std::vector shapes; + std::string data; // a digest of every field of its item data, to compare whole +}; +// Empty when the document is not an item (an asset with a Key and ItemData). +[[nodiscard]] std::optional item_content(const frostbite::ebx::Document& document); + +// The game's own items, which a mod's are held against. +class StoreItems final { +public: + // `sold`: the store sells it, so ReSkate never hands it out. + void add(const ItemContent& item, bool sold); + // The key of the store item `item` is a copy of, or empty. It is one when + // it has a look only a store item has, when it has no look of its own and + // is fitted on a shape only a store item with no look has (a whole + // costume), or when its item data is a store item's field for field. + // Anything an item the store does not sell has as well is nobody's. + [[nodiscard]] std::string original(const ItemContent& item) const; + +private: + struct Parts { + std::map> sold; // part -> the store item's key + std::set> free; // parts of the other items + [[nodiscard]] const std::string* owner(std::string_view part) const; + }; + Parts looks_, shapes_, data_; +}; + +struct StoreCopies { + // Lower-case key of each copy -> the lower-case key of the store item it copies. + std::map> items; + // The mods that add them, highest priority first. + struct Source { + std::string mod; + std::size_t count{}; + std::string example, original; // one of its copies and what that copies, as written + }; + std::vector mods; + // How many of the game's own items the mods' were held against; none when no mod adds an item. + std::size_t game_items{}; +}; + +// Whether the store sells the item with this lower-case key. +using StoreItem = std::function; + +// Reads the items every enabled mod adds to (or changes in) the game's bundles +// and holds them against the game's own. The game's are only read when a mod +// has an item at all. Never throws: a bundle that cannot be read is left out +// (the merge leaves its mod out of the game too) and noted in `notes`. +[[nodiscard]] StoreCopies check_store_copies(const Catalog& catalog, const StoreItem& sold, + std::vector* notes = nullptr) noexcept; + +} // namespace dingosdk::mods diff --git a/Engine/Vfs/native_db.cpp b/Engine/Vfs/native_db.cpp index 0e46518..8a428d9 100644 --- a/Engine/Vfs/native_db.cpp +++ b/Engine/Vfs/native_db.cpp @@ -12,7 +12,7 @@ public: : input_(input), context_(context), options_(options) {} Node read(unsigned depth = 0) { - if (depth > 32 || ++nodes_ > 32768) fail("DB nesting/entry limit exceeded"); + if (depth > 32 || ++nodes_ > options_.max_entries) fail("DB nesting/entry limit exceeded"); const auto tag = take(1)[0]; Node node; node.type = tag & 31; diff --git a/Engine/Vfs/native_db.h b/Engine/Vfs/native_db.h index 5d0c0d1..e6380ac 100644 --- a/Engine/Vfs/native_db.h +++ b/Engine/Vfs/native_db.h @@ -41,6 +41,10 @@ struct Options { // an entry. layout.toc's meta section repeats keys by design, so readers of // that file turn the rule off. bool unique_fields = true; + // How many values a file may hold: more than any the game reads whole at + // startup. A reader of something longer by nature (the chunk list of a + // level's root bundle) raises it to what its input can hold. + std::size_t max_entries = 32768; }; // Throws std::runtime_error prefixed with `context`, for example "InitFS". diff --git a/Extension/Assets/live_mods.cpp b/Extension/Assets/live_mods.cpp index 33accd7..f58dfcd 100644 --- a/Extension/Assets/live_mods.cpp +++ b/Extension/Assets/live_mods.cpp @@ -5,6 +5,7 @@ #include "Engine/Game/Build/20260929/engine.h" #include "Engine/Game/World/custom_level_manifest.h" #include "Engine/Core/Platform/memory.h" +#include "Engine/Core/Platform/path_text.h" #include "Engine/Game/Build/20260929/live_mods.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Vfs/mod_catalog.h" @@ -211,7 +212,7 @@ bool refresh_toc(const std::string &superbundle, const std::filesystem::path &fi std::ifstream in(file, std::ios::binary); std::vector bytes((std::istreambuf_iterator(in)), std::istreambuf_iterator()); if (bytes.size() <= native::toc_header + 4) { - problem = file.filename().string() + " could not be read"; + problem = path_utf8(file.filename()) + " could not be read"; return false; } const auto base = reinterpret_cast(GetModuleHandleW(nullptr)); diff --git a/Extension/Assets/mod_layers.cpp b/Extension/Assets/mod_layers.cpp index 840d7be..fa33671 100644 --- a/Extension/Assets/mod_layers.cpp +++ b/Extension/Assets/mod_layers.cpp @@ -6,6 +6,7 @@ #include "Engine/Vfs/mod_scoring.h" #include "Engine/Core/Hooks/hooks.h" #include "Engine/Core/Log/logging.h" +#include "Engine/Core/Platform/path_text.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Game/Build/20260929/mod_layers.h" #include "Extension/UI/Startup/startup_window.h" @@ -268,7 +269,7 @@ bool start_mod_layers(std::uintptr_t base, std::string& error) { } if (!catalog.present) { logging::log(logging::Level::info, logging::Channel::assets, - "No Mods folder at {}; only Patch and Data are layered.", catalog.root.string()); + "No Mods folder at {}; only Patch and Data are layered.", path_utf8(catalog.root)); return true; } if (!catalog.merged) { @@ -322,7 +323,7 @@ bool start_mod_layers(std::uintptr_t base, std::string& error) { } installed_base = base; logging::log(logging::Level::info, logging::Channel::assets, - "Merged mod patch ready at {}", state().merged_root.string()); + "Merged mod patch ready at {}", path_utf8(state().merged_root)); return true; } LayoutObjects layout_objects() noexcept { return {seen_manager.load(), seen_layers.load()}; } diff --git a/Extension/Console/commands.cpp b/Extension/Console/commands.cpp index abbc270..3a599b6 100644 --- a/Extension/Console/commands.cpp +++ b/Extension/Console/commands.cpp @@ -1,5 +1,6 @@ #include "commands.h" #include "Engine/Core/Log/logging.h" +#include "Engine/Core/Platform/path_text.h" #include namespace dingosdk::console { @@ -47,7 +48,7 @@ void register_console_commands(Commands ®istry) { "and Tab to complete names and " "arguments."); out("Groups: Movement, Gameplay, World, Graphics, Progression, Objects, " - "Engine, Developer, Console."); + "Engine, Console."); return; } const auto &name = std::get(args[0]); @@ -179,7 +180,7 @@ void register_console_commands(Commands ®istry) { log.run = [](const Model &, const Values &, const Output &out) { const auto s = logging::status(); out(std::string("Logging level: ") + std::string(logging::name(s.level))); - out((s.directory / L"ReSkate.log").string()); + out(path_utf8(s.directory / L"ReSkate.log")); }; registry.add(std::move(log)); auto level = argument("level"); @@ -222,6 +223,7 @@ const Commands &game_commands() { register_park_editor_commands(*result); register_multiplayer_commands(*result); register_perf_commands(*result); + register_trainer_commands(*result); return result; }(); return *registry; diff --git a/Extension/Console/commands.h b/Extension/Console/commands.h index 153b495..dde960b 100644 --- a/Extension/Console/commands.h +++ b/Extension/Console/commands.h @@ -22,6 +22,7 @@ void register_object_commands(Commands &); void register_park_editor_commands(Commands &); void register_multiplayer_commands(Commands &); void register_perf_commands(Commands &); +void register_trainer_commands(Commands &); // Runtime adapters. Invoked only by the verified game-thread dispatcher. void request_debug(overlay::DebugAction, bool enabled = false, float value = 0); void request_feature(overlay::OfflineFeatureGroup, bool enabled); diff --git a/Extension/Console/perf_commands.cpp b/Extension/Console/perf_commands.cpp index 2af5ebf..644fe3f 100644 --- a/Extension/Console/perf_commands.cpp +++ b/Extension/Console/perf_commands.cpp @@ -101,7 +101,7 @@ void register_perf_commands(Commands& registry) { seconds.minimum = 0.5; seconds.maximum = 60; auto target = argument("client|present|all|", Type::text, true); - target.choices = {"client", "present", "all"}; + target.complete = [](const Model&, auto) { return std::vector{"client", "present", "all"}; }; auto rate = argument("rate", Type::unsigned_integer, true); rate.minimum = 50; rate.maximum = 4000; diff --git a/Extension/Customization/developer_board.cpp b/Extension/Customization/developer_board.cpp index 0df1cbf..5b5442f 100644 --- a/Extension/Customization/developer_board.cpp +++ b/Extension/Customization/developer_board.cpp @@ -20,12 +20,12 @@ void report_board(std::uintptr_t entity, std::uint64_t steam_id, std::string_vie } if (entry->entity == entity && (entry->message == hash || now - entry->at < 5000)) return; *entry = {entity, now, hash}; - logging::log(level, logging::Channel::customization, "Developer board (Steam {}, actor {:#x}): {}", steam_id, entity, message); + logging::log(level, logging::Channel::customization, "Board items (Steam {}, actor {:#x}): {}", steam_id, entity, message); } bool write_color(std::uintptr_t base, const developer_board_detail::Binding &binding, const developer_board_detail::Color &color) noexcept { __try { if (!binding.node) { - const developer_board_detail::NativeColor value{color, 0, binding.type}; + const developer_board_detail::NativeColor value{color, 0, base + addr::native_cosmetics::color_rgb_type}; // Let the engine allocate/copy/hash a missing typed parameter. Priority // zero allows later native appearance presets to replace this override. return reinterpret_cast; +Animations board_animations(std::uint64_t steam_id, const multiplayer::MarkStyles &styles) noexcept { + const auto mark = multiplayer::identity_mark(steam_id); + Animations out; + for (std::size_t i = 0; i < out.size(); ++i) + out[i] = developer_board_detail::item_animation(mark, styles[multiplayer::skater_mark_items + i]); + return out; +} +bool any_on(const Animations &animations) noexcept { + return std::any_of(animations.begin(), animations.end(), [](const auto &animation) { return animation.on; }); +} } // namespace void update_developer_board(std::uintptr_t base, std::uintptr_t board, std::uint64_t steam_id, - std::uint64_t generation, DeveloperBoardState &state) noexcept { - const bool developer = multiplayer::reskate_developer(steam_id); + std::uint64_t generation, DeveloperBoardState &state, + const multiplayer::MarkStyles &styles) noexcept { + const auto animations = board_animations(steam_id, styles); + const bool listed = any_on(animations); if (!board) { state = {}; return; } - if (!developer && !state.count) return; + if (!listed && state.colors.empty()) return; try { auto read = [](std::uintptr_t address, void *out, std::size_t size) { return memory::peek_bytes(address, out, size); }; std::array publisher{}; @@ -64,14 +78,16 @@ void update_developer_board(std::uintptr_t base, std::uintptr_t board, std::uint bool published = true; auto write = [base](const auto &binding, const auto &color) { return write_color(base, binding, color); }; auto publish = [base, &published](std::uintptr_t item) { if (!publish_materials(base, item)) published = false; }; - developer_board_detail::animate(state, live, board, generation, developer, GetTickCount64(), write, publish); - const auto wanted = static_cast(std::count_if(live.bindings.begin(), live.bindings.begin() + live.count, - [](const auto &binding) { return binding.eligible; })); + developer_board_detail::animate(state, live, board, generation, animations, GetTickCount64(), write, publish); + // The colours of the parts that are on: found, and taken. + const auto wanted = std::count_if(live.bindings.begin(), live.bindings.end(), + [&](const auto &binding) { return animations[binding.part].on; }); + const auto taken = std::count_if(state.colors.begin(), state.colors.end(), + [&](const auto &saved) { return animations[saved.binding.part].on; }); if (!published) report_board(board, steam_id, "native material publication failed", logging::Level::warning); - else if (developer && !live.pending && state.count < wanted) + else if (listed && !live.pending && taken < wanted) report_board(board, steam_id, "waiting for the remaining board color overrides", logging::Level::warning); - else if (developer && state.count) - report_board(board, steam_id, "RGB active on the owned pink board cosmetics", logging::Level::info); + else if (listed && taken) report_board(board, steam_id, "animation active on the board", logging::Level::info); } catch (const std::exception &error) { report_board(board, steam_id, error.what(), logging::Level::warning); } catch (...) {} @@ -81,15 +97,16 @@ void tick_local_developer_board(std::uintptr_t base, std::uintptr_t client, bool if (!ready) return; try { const auto social = multiplayer::steam_social_snapshot(); - const auto id = social ? social->local.id : 0; - if (!multiplayer::reskate_developer(id) && !state.count) return; + const auto id = social && multiplayer::own_items_shown() ? social->local.id : 0; + const auto styles = developer_hoodie_detail::own_styles.load(); + if (!any_on(board_animations(id, styles)) && state.colors.empty()) return; const auto local = multiplayer::capture_local(base, client, false); if (!local.ready) return; auto read = [](std::uintptr_t address, void *out, std::size_t size) { return memory::peek_bytes(address, out, size); }; // Cosmetic changes can replace the board while retaining the skater. // Re-resolve the actual local association, rather than capture's cached board. const auto board = multiplayer::read_native_board(read, base, local.entity); - update_developer_board(base, board.entity, id, local.context, state); + update_developer_board(base, board.entity, id, local.context, state, styles); } catch (...) {} } } // namespace dingosdk diff --git a/Extension/Customization/developer_board.h b/Extension/Customization/developer_board.h index 678eb74..fbb203e 100644 --- a/Extension/Customization/developer_board.h +++ b/Extension/Customization/developer_board.h @@ -2,7 +2,10 @@ #include "developer_board_material.h" namespace dingosdk { +// Colours the board a listed player rides: `styles` is how they have each marked cosmetic, the +// board's deck, grip tape, trucks and wheels after the skater's. void update_developer_board(std::uintptr_t base, std::uintptr_t board, std::uint64_t steam_id, - std::uint64_t generation, DeveloperBoardState &state) noexcept; + std::uint64_t generation, DeveloperBoardState &state, + const multiplayer::MarkStyles &styles) noexcept; void tick_local_developer_board(std::uintptr_t base, std::uintptr_t client, bool ready) noexcept; } // namespace dingosdk diff --git a/Extension/Customization/developer_board_material.h b/Extension/Customization/developer_board_material.h index 1b432b9..fb0c95b 100644 --- a/Extension/Customization/developer_board_material.h +++ b/Extension/Customization/developer_board_material.h @@ -4,9 +4,15 @@ namespace dingosdk { namespace developer_board_detail { +using developer_hoodie_detail::Binding; using developer_hoodie_detail::Color; +using developer_hoodie_detail::ItemAnimation; +using developer_hoodie_detail::ParameterKey; +using developer_hoodie_detail::animate; +using developer_hoodie_detail::item_animation; +using developer_hoodie_detail::item_color; using developer_hoodie_detail::material_hash; -using developer_hoodie_detail::rainbow; +using developer_hoodie_detail::settle_publishes; inline constexpr std::uint64_t graphic_color = 0x873f48af36e50d59ULL; inline constexpr std::uint64_t base_color = 0x8c46cd354b18a9e2ULL; inline constexpr Color shader_default{.5f, .5f, .5f}; @@ -14,58 +20,40 @@ inline constexpr std::array slots{ "Characters/Customization/Board/Slots/Skateboard_Deck_Slot", "Characters/Customization/Board/Slots/Skateboard_Trucks_Slot", "Characters/Customization/Board/Slots/Skateboard_Wheels_Slot"}; +// One colour of the board: a named material of the controller assigned one of `slots`, the +// shader key, and which of the board's marked cosmetics it colours (deck, grip tape, trucks, +// wheels: the order of their styles after the skater's, multiplayer::mark_item_names). struct Target { - std::string_view asset, material; - std::uint32_t recipe_slot; + std::string_view material; std::size_t controller; std::uint64_t key; + std::uint8_t part; }; +inline constexpr std::size_t part_count = multiplayer::mark_items - multiplayer::skater_mark_items; // The board loader registers Deck_Mat / DeckTop_Mat for the compiled sections -// Skateboard_base_mat / mat_skateboard_top. Use the live named map identities. -// Recipe identities, assigned board controllers and serialized shader keys from -// the four pink cosmetics. The deck's wood follows its Solid Pink graphic. +// Skateboard_base_mat / mat_skateboard_top, whatever deck is on it: use the live named map +// identities. The deck's wood follows its graphic. inline constexpr std::array targets{{ - {"Own_DeckGraphic_Gen_Popsicle_00021", "Deck_Mat", 0x66071628, 0, graphic_color}, - {"Own_DeckGraphic_Gen_Popsicle_00021", "Deck_Mat", 0x66071628, 0, base_color}, - {"Own_DeckGripColor_Gen_Popsicle_00006", "DeckTop_Mat", 0x755d5ee0, 0, graphic_color}, - {"Own_Truck_Gen_Default_00002", "Truck_Mat", 0x0add98be, 1, base_color}, - {"Own_WheelColor_Gen_Classic_00005", "Wheel_mat", 0xc83bb94e, 2, base_color}, + {"Deck_Mat", 0, graphic_color, 0}, + {"Deck_Mat", 0, base_color, 0}, + {"DeckTop_Mat", 0, graphic_color, 1}, + {"Truck_Mat", 1, base_color, 2}, + {"Wheel_mat", 2, base_color, 3}, }}; -// Native ParamDbKey and inline NativeValue layout used by addEsVector's setter. -// The key's first word is the final two serialized GUID bytes. Its type index -// comes from the live ColorRgb descriptor, not a serialized Vec3 or boxed value. -struct ParameterKey { - std::uint16_t identity{}, size{}, type{}, count{}; - std::uint64_t hash{}; -}; +// The inline NativeValue layout used by addEsVector's setter. Its type is the live ColorRgb +// descriptor, not a serialized Vec3 or boxed value. struct NativeColor { Color color{}; std::uint32_t padding{}; std::uintptr_t type{}; std::uint32_t flags{3}, reserved{}; }; -static_assert(sizeof(ParameterKey) == 16 && offsetof(ParameterKey, hash) == 8); static_assert(sizeof(NativeColor) == 32 && offsetof(NativeColor, type) == 16 && offsetof(NativeColor, flags) == 24); inline ParameterKey parameter_key(std::uint64_t hash, std::uint16_t type) noexcept { return {static_cast(hash == graphic_color ? 0xd442 : 0x8c54), sizeof(Color), type, 1, hash}; } -struct Binding { - std::uintptr_t item{}, material{}, node{}, type{}; - ParameterKey key{}; - Color color{}; - bool eligible{}; -}; -struct SavedColor { - Binding binding; - Color original{}, last{}; -}; inline constexpr std::size_t max_bindings = targets.size() * 4; -struct Materials { - std::uintptr_t component{}, appearance{}; - std::array bindings{}; - std::size_t count{}; - bool pending{}; -}; +using developer_hoodie_detail::Materials; template std::uintptr_t parameter(Memory &m, std::uintptr_t material, std::uint64_t key) { const auto buckets = m.ptr(material, 0x268); @@ -85,21 +73,9 @@ template Materials materials(Read &read, std::uintptr_t base, std::u using namespace multiplayer; CosmeticMemory m{read, base}; Materials out; - m.check(m.ptr(entity) == base + addr::engine::board_entity_vtable, "RGB board actor type differs."); + m.check(m.ptr(entity) == base + addr::engine::board_entity_vtable, "Board actor type differs."); out.component = m.component(entity); out.pending = m.template get(out.component, 0x10a) || m.template get(out.component, 0x10d); - const auto recipe = m.ptr(out.component, 0x158); - const auto count = m.count(recipe, sizeof(NativeCosmeticItem), max_cosmetic_slots); - std::array eligible{}; - for (std::size_t i = 0; i < count; ++i) { - const auto item = m.template get(recipe, i * sizeof(NativeCosmeticItem)); - for (std::size_t t = 0; t < targets.size(); ++t) { - const auto &target = targets[t]; - if (item.slot != target.recipe_slot || item.hash != cosmetic_asset_hash(target.asset)) continue; - const auto name = item.asset ? m.text(reinterpret_cast(item.asset)) : std::string{}; - eligible[t] = name.empty() || name == target.asset; - } - } const auto type = base + addr::native_cosmetics::color_rgb_type; const auto info = m.ptr(type); m.check(info >= base && info < base + 0x09144000 && m.template get(info) == 0x885eff59 && @@ -126,8 +102,7 @@ template Materials materials(Read &read, std::uintptr_t base, std::u if (!total) { out.pending = true; continue; } m.check(buckets && size && size <= 256 && total <= 128, "Board material map exceeds bounds."); const auto sentinel = m.ptr(buckets, size * 8ULL); - for (std::size_t t = 0; t < targets.size(); ++t) { - const auto &target = targets[t]; + for (const auto &target : targets) { if (target.controller != slot) continue; const auto hash = material_hash(target.material); auto node = m.ptr(buckets, (hash % size) * 8ULL); @@ -152,62 +127,17 @@ template Materials materials(Read &read, std::uintptr_t base, std::u color = m.template get(p, 0x10); m.check(std::all_of(color.begin(), color.end(), [](float c) { return std::isfinite(c); }), "Board color is invalid."); } - m.check(out.count < out.bindings.size(), "Board colors exceed bounds."); - out.bindings[out.count++] = {item, material, p, type, key, color, eligible[t]}; + m.check(out.bindings.size() < max_bindings, "Board colors exceed bounds."); + out.bindings.push_back({item, material, p, key, color, target.part}); } } } return out; } -inline bool same_binding(const Binding &live, const Binding &saved) noexcept { - // A successful native insertion acquires its node on the next fresh walk. - return live.item == saved.item && live.material == saved.material && live.type == saved.type && live.key.hash == saved.key.hash && - (saved.node ? live.node == saved.node : live.node != 0); -} } // namespace developer_board_detail -struct DeveloperBoardState { - std::uintptr_t entity{}, component{}, appearance{}; - std::uint64_t generation{}; - std::array colors{}; - std::size_t count{}; -}; -namespace developer_board_detail { -// Re-discover ownership before every write. Changing the recipe restores only -// colors that still equal our last write; native preset updates take precedence. -// Missing keys inherit .5 from these compiled board shaders. Keep inserted -// overrides at native preset priority and restore that default on deactivation. +// What was saved of a board's parts while they are coloured. A key a part's material does not +// have inherits .5 from the compiled board shaders: one is added at native preset priority, so +// a later native preset replaces it, and goes back to that default when the colouring stops. // Never clear a NativeValue: the renderer still copies the key's full payload. -template -void animate(DeveloperBoardState &state, const Materials &live, std::uintptr_t entity, std::uint64_t generation, - bool developer, std::uint64_t milliseconds, Write &write, Publish &publish) { - if (state.entity != entity || state.generation != generation || state.component != live.component || state.appearance != live.appearance) - state = {}; - if (live.pending) return; - DeveloperBoardState next; - next.entity = entity; next.generation = generation; next.component = live.component; next.appearance = live.appearance; - const auto color = rainbow(milliseconds); - std::array changed_items{}; - std::size_t changed{}; - for (std::size_t i = 0; i < live.count; ++i) { - const auto &binding = live.bindings[i]; - const SavedColor *saved{}; - for (std::size_t j = 0; j < state.count; ++j) - if (same_binding(binding, state.colors[j].binding)) { saved = &state.colors[j]; break; } - bool wrote{}; - if (developer && binding.eligible) { - const auto original = saved && binding.color == saved->last ? saved->original : binding.color; - if ((binding.node && binding.color == color) || (wrote = write(binding, color))) - next.colors[next.count++] = {binding, original, color}; - else if (saved) next.colors[next.count++] = *saved; - } else if (saved && binding.color == saved->last && binding.color != saved->original) { - wrote = write(binding, saved->original); - if (!wrote) next.colors[next.count++] = *saved; - } - if (wrote && std::find(changed_items.begin(), changed_items.begin() + changed, binding.item) == changed_items.begin() + changed) - changed_items[changed++] = binding.item; - } - state = next; - for (std::size_t i = 0; i < changed; ++i) publish(changed_items[i]); -} -} // namespace developer_board_detail +struct DeveloperBoardState : developer_hoodie_detail::ItemState {}; } // namespace dingosdk diff --git a/Extension/Customization/developer_hoodie.cpp b/Extension/Customization/developer_hoodie.cpp index 0cb4dc8..a5541ac 100644 --- a/Extension/Customization/developer_hoodie.cpp +++ b/Extension/Customization/developer_hoodie.cpp @@ -5,6 +5,7 @@ #include "Extension/Multiplayer/Remote/native_skater.h" #include #include +#include namespace dingosdk { namespace { @@ -23,7 +24,7 @@ void report_hoodie(std::uintptr_t entity, std::uint64_t steam_id, std::string_vi } if (entry->entity == entity && (entry->message == hash || now - entry->at < 5000)) return; *entry = {entity, now, hash}; - logging::log(level, logging::Channel::customization, "Developer hoodie (Steam {}, actor {:#x}): {}", + logging::log(level, logging::Channel::customization, "Skater items (Steam {}, actor {:#x}): {}", steam_id, entity, message); } // NativeValue is owned by this material. Preserve its padding, type, priority, @@ -41,12 +42,34 @@ bool publish_materials(std::uintptr_t base, std::uintptr_t item) noexcept { return true; } __except (EXCEPTION_EXECUTE_HANDLER) { return false; } } +// What each of the skater's marked cosmetics does for this player: top, bottoms, shoes, ... +using Animations = std::array; +Animations skater_animations(std::uint64_t steam_id, const multiplayer::MarkStyles &styles) noexcept { + const auto mark = multiplayer::identity_mark(steam_id); + Animations out; + for (std::size_t i = 0; i < out.size(); ++i) out[i] = developer_hoodie_detail::item_animation(mark, styles[i]); + return out; +} +bool any_on(const Animations &animations) noexcept { + return std::any_of(animations.begin(), animations.end(), [](const auto &animation) { return animation.on; }); +} +// Which cosmetics are coloured, and how many colours of each: what is equipped decides. +std::string coloured(const DeveloperHoodieState &state) { + std::array colors{}; + for (const auto &saved : state.colors) ++colors[saved.binding.part]; + std::string out; + for (std::size_t i = 0; i < colors.size(); ++i) + if (colors[i]) out += std::format("{}{} ({})", out.empty() ? "" : ", ", multiplayer::mark_item_names[i], colors[i]); + return out.empty() ? "nothing equipped has a color to change" : "animation active on " + out; +} } // namespace void update_developer_hoodie(std::uintptr_t base, std::uintptr_t entity, std::uint64_t steam_id, - std::uint64_t generation, DeveloperHoodieState &state) noexcept { - const bool developer = multiplayer::reskate_developer(steam_id); + std::uint64_t generation, DeveloperHoodieState &state, + const multiplayer::MarkStyles &styles) noexcept { + const auto animations = skater_animations(steam_id, styles); + const bool listed = any_on(animations); if (!entity) { state = {}; return; } - if (!developer && !state.count) return; + if (!listed && state.colors.empty()) return; try { auto read = [](std::uintptr_t address, void *out, std::size_t size) { return memory::peek_bytes(address, out, size); }; std::array prefix{}; @@ -57,12 +80,10 @@ void update_developer_hoodie(std::uintptr_t base, std::uintptr_t entity, std::ui } const auto live = developer_hoodie_detail::materials(read, base, entity); bool published = true; - auto publish = [base, &published](std::uintptr_t item) { published = publish_materials(base, item); }; - developer_hoodie_detail::animate(state, live, entity, generation, developer, GetTickCount64(), write_color, publish); + auto publish = [base, &published](std::uintptr_t item) { if (!publish_materials(base, item)) published = false; }; + developer_hoodie_detail::animate(state, live, entity, generation, animations, GetTickCount64(), write_color, publish); if (!published) report_hoodie(entity, steam_id, "native material publication failed", logging::Level::warning); - else if (state.count) report_hoodie(entity, steam_id, "RGB active on the owned hoodie material", logging::Level::info); - else if (developer && live.eligible) - report_hoodie(entity, steam_id, "waiting for the assigned top slot's color parameters", logging::Level::warning); + else if (listed && !live.pending) report_hoodie(entity, steam_id, coloured(state), logging::Level::info); } catch (const std::exception &error) { report_hoodie(entity, steam_id, error.what(), logging::Level::warning); } catch (...) { @@ -77,10 +98,11 @@ void tick_local_developer_hoodie(std::uintptr_t base, std::uintptr_t client, boo if (!ready) return; try { const auto social = multiplayer::steam_social_snapshot(); - const auto id = social ? social->local.id : 0; - if (!multiplayer::reskate_developer(id) && !state.count) return; + const auto id = social && multiplayer::own_items_shown() ? social->local.id : 0; + const auto styles = developer_hoodie_detail::own_styles.load(); + if (!any_on(skater_animations(id, styles)) && state.colors.empty()) return; const auto local = multiplayer::capture_local(base, client, false); - if (local.ready) update_developer_hoodie(base, local.entity, id, local.context, state); + if (local.ready) update_developer_hoodie(base, local.entity, id, local.context, state, styles); else report_hoodie(client, id, local.detail, logging::Level::warning); } catch (...) {} } diff --git a/Extension/Customization/developer_hoodie.h b/Extension/Customization/developer_hoodie.h index 3842763..97235e9 100644 --- a/Extension/Customization/developer_hoodie.h +++ b/Extension/Customization/developer_hoodie.h @@ -4,7 +4,10 @@ namespace dingosdk { // Verified client update thread only. Remote identity comes from the Steam roster, // generation from the current native actor; neither is a client cosmetic parameter. +// Colours what a listed player wears (developer_hoodie_material.h): `styles` is how they have +// each marked cosmetic, the skater's first. void update_developer_hoodie(std::uintptr_t base, std::uintptr_t entity, std::uint64_t steam_id, - std::uint64_t generation, DeveloperHoodieState &state) noexcept; + std::uint64_t generation, DeveloperHoodieState &state, + const multiplayer::MarkStyles &styles) noexcept; void tick_local_developer_hoodie(std::uintptr_t base, std::uintptr_t client, bool ready) noexcept; } // namespace dingosdk diff --git a/Extension/Customization/developer_hoodie_material.h b/Extension/Customization/developer_hoodie_material.h index 1c3d2b2..d4a2bbf 100644 --- a/Extension/Customization/developer_hoodie_material.h +++ b/Extension/Customization/developer_hoodie_material.h @@ -1,48 +1,93 @@ #pragma once #include "Extension/Multiplayer/developer_identity.h" +#include "Extension/Multiplayer/Remote/cosmetics.h" #include "Extension/Multiplayer/Remote/native_cosmetics_layout.h" #include #include +#include #include +#include #include +#include +#include +#include +#include +// The cosmetics of a player on one of the backend's lists (developer_identity.h) are coloured +// for them: whatever they wear and ride, on their own material instances, as they choose on the +// menu's Special page. It began as the developers' rainbow hoodie and pink board, which the +// names here still say. namespace dingosdk { namespace developer_hoodie_detail { -inline constexpr std::string_view asset = "Own_TopShirt_Gen_HoodieRelaxed_00001"; -// The compiled appearance preset names controllers by their assigned CAS slot. -// "TopItem" is a Lua construction name, not the live controller's name. -inline constexpr std::string_view top_slot = "Characters/MainCharacters/Generic/CAS/Common/Slots/CAS_Top_Slot"; -// Serialized ParamDbKey identities from this hoodie's AppearanceShaderExpressionPreset. -// Only the six fabric color regions: textures, stitches and other clothing stay intact. -inline constexpr std::array color_keys{ - 0x8c46cd354b18a9e2ULL, 0x1567b5742f4ec0b2ULL, 0x1564a942cb33c53bULL, - 0xf526305270d94cb3ULL, 0x99879bf202d1960aULL, 0x998450425dbd9044ULL}; +// The compiled appearance preset names a skater's item controllers by their assigned CAS slot +// ("TopItem" and the like are Lua construction names, not the live controllers' names). +inline constexpr std::string_view slot_folder = "Characters/MainCharacters/Generic/CAS/Common/Slots/"; +// The skater's marked cosmetics, in the order of their styles (multiplayer::mark_item_names): +// top, bottoms, shoes, socks, hat, glasses, outfit, costume. +inline constexpr std::array slots{ + "CAS_Top_Slot", "CAS_Bottom_Slot", "CAS_Footwear_Slot", "CAS_Sock_Slot", + "CAS_Headgear_Slot", "CAS_Eyewear_Slot", "CAS_Outfit_Slot", "CAS_CostumeFullBody_Slot"}; +// Serialized ParamDbKey identities of the ColorRgb parameters the game's clothing presets set +// (AppearanceShaderExpressionPreset.ParamInfo; read from 114 presets of every kind of item, +// 2026-10-04): the colours of an item's regions. Every kind of clothing draws on the same +// ones, the first thirteen by far the most, and each item colours some and leaves the rest at +// the shaders' neutral grey (`neutral`), where its textures show as they are. +// On the relaxed hoodie, as seen in game: the first is the whole body, sleeves and hood; the +// fourth the cuffs and hem; the fifth the drawstrings and hood lining. +inline constexpr std::array color_keys{ + 0x8c46cd354b18a9e2ULL, 0x1567b5742f4ec0b2ULL, 0x1564a942cb33c53bULL, 0xf526305270d94cb3ULL, + 0x99879bf202d1960aULL, 0x998450425dbd9044ULL, 0x18984e85c679da8dULL, 0x2bf96fc9d8da57bbULL, + 0x13cfb51bd8474129ULL, 0x7c06880748ded394ULL, 0xfca496001ea60718ULL, 0xfca79ddc325249ceULL, + 0x5e1943a7c20ab77cULL, 0x0adafe3e13df4057ULL, 0xe3ab5f6a33dd1684ULL, 0xc31ad6ca613dcf43ULL, + 0xc1c70aa67f8bde7cULL, 0xc1c4554f6b58eee8ULL, 0x24e6723cc9f17eccULL, 0xd98b435969485311ULL, + 0x27e785e804edea5cULL, 0x27e45bf8d67d959fULL, 0xae823ac5acb2abbaULL, 0x89cbc6b088ae9b1bULL, + 0x593aacb9e12dcd6bULL, 0x4ec6aeb742a2c383ULL, 0xa19416e977675f88ULL, 0x0a94b0e28d102968ULL}; using Color = std::array; +inline constexpr Color neutral{.5f, .5f, .5f}; +// Native ParamDbKey, as a material's parameter node holds it. The first word is the final two +// serialized GUID bytes; the type index is the live ColorRgb descriptor's. +struct ParameterKey { + std::uint16_t identity{}, size{}, type{}, count{}; + std::uint64_t hash{}; +}; +static_assert(sizeof(ParameterKey) == 16 && offsetof(ParameterKey, hash) == 8); +// One colour of one material instance. `part` is which marked cosmetic it belongs to. struct Binding { - std::uintptr_t material{}, node{}, type{}; - std::uint64_t key{}; + std::uintptr_t item{}, material{}, node{}; // node 0: no such parameter yet (the board adds one) + ParameterKey key{}; Color color{}; - bool operator==(const Binding &) const = default; + std::uint8_t part{}; }; struct SavedColor { Binding binding; Color original{}, last{}; + std::uint8_t settling{}; // publishes still owed for `last` (settle_publishes) }; -inline constexpr std::size_t max_bindings = color_keys.size() * 4; +// The colours found on an actor in one walk. struct Materials { - std::uintptr_t component{}, appearance{}, item{}; - std::array bindings{}; - std::size_t count{}; - bool eligible{}; + std::uintptr_t component{}, appearance{}; + std::vector bindings; + bool pending{}; + // Clothing: a colour at `neutral` is a region the item leaves to its textures, and is left + // alone, unless the item colours none of its regions: then all of them are coloured. + bool regions_only{}; }; +// What was saved of an actor's colours while they are coloured here. +struct ItemState { + std::uintptr_t entity{}, component{}, appearance{}; + std::uint64_t generation{}; + std::vector colors; +}; +// What is equipped can be anything: this many colours of a skater, and no more. +inline constexpr std::size_t max_bindings = 512; inline std::uint32_t material_hash(std::string_view name) noexcept { std::uint32_t hash = 5381; for (auto c : name) hash = hash * 33 ^ static_cast(c >= 'A' && c <= 'Z' ? c + 32 : c); return hash; } -inline Color rainbow(std::uint64_t milliseconds) noexcept { - // One continuous hue cycle every eight seconds, in the material's existing gamut. - const auto hue = static_cast(milliseconds % 8000) * (6.f / 8000.f); +inline Color rainbow(std::uint64_t milliseconds, std::uint64_t period = 8000) noexcept { + // One continuous hue cycle every eight seconds (or `period`), in the material's existing gamut. + const auto hue = static_cast(milliseconds % period) * (6.f / static_cast(period)); const auto sector = static_cast(hue); constexpr float high = .8f, low = .025f; const auto rising = low + (high - low) * (hue - static_cast(sector)); @@ -56,126 +101,276 @@ inline Color rainbow(std::uint64_t milliseconds) noexcept { default: return {high, low, falling}; } } +// What one of a listed player's marked cosmetics does right now: nothing, the rainbow, or a +// walk through three colours and back (one colour three times stands still). +struct ItemAnimation { + bool on{}, rainbow{}; + std::array stops{}; + std::uint64_t period = 3000; // milliseconds for a round +}; +// A colour as picked on screen, as the material takes it: linear, and inside the gamut the +// rainbow keeps to. +inline Color material_color(const std::array &picked) noexcept { + Color out{}; + for (std::size_t i = 0; i < out.size(); ++i) + out[i] = std::max(.004f, std::pow(static_cast(picked[i]) / 255.f, 2.2f) * .8f); + return out; +} +// The colours the pickers start from: about where each list's own animation begins and ends. +inline std::pair, std::array> standard_picks(multiplayer::IdentityList mark) noexcept { + using multiplayer::IdentityList; + if (mark == IdentityList::content_creator) return {{0x88, 0x23, 0x23}, {0xff, 0x48, 0x70}}; + if (mark == IdentityList::homie) return {{0xad, 0x6c, 0x1c}, {0xff, 0xe6, 0x67}}; + if (mark == IdentityList::centrix) return {{0x1f, 0x7b, 0xff}, {0xff, 0xff, 0xff}}; + if (mark == IdentityList::staff) return {{0x10, 0x80, 0x38}, {0x6a, 0xf0, 0x8c}}; + return {{0x6e, 0x20, 0xff}, {0xb4, 0x78, 0xff}}; +} +// What a cosmetic does for a player on `mark`'s list (nobody else's does anything) with the +// style they chose on the Special page. Left standard, a developer's cycles the rainbow, a +// content creator's goes from deep red through bright red to a reddish pink and back, and a +// homie's from deep gold through bright gold to yellow and back, Centrix's from blue +// through light blue to white and back, and the staff's from deep green through bright green +// to a light one and back. The rainbow is a developer's standard, and a style a developer or +// a member of staff can pick (rainbow_style); nobody else's item takes it. +inline ItemAnimation item_animation(std::optional mark, const multiplayer::MarkStyle &style) noexcept { + using multiplayer::IdentityList; + using multiplayer::MarkMode; + ItemAnimation out; + if (!mark || style.mode == MarkMode::off) return out; + out.on = true; + out.rainbow = (style.mode == MarkMode::standard && *mark == IdentityList::developer) || + (multiplayer::rainbow_style(style) && (*mark == IdentityList::developer || *mark == IdentityList::staff)); + const std::uint64_t normal = out.rainbow ? 8000 : 3000; + out.period = style.speed == 1 ? normal * 2 : style.speed == 2 ? normal / 2 : normal; + if (out.rainbow) { + } else if (style.mode == MarkMode::solid) { + out.stops.fill(material_color(style.from)); + } else if (style.mode == MarkMode::gradient) { + const auto from = material_color(style.from), to = material_color(style.to); + out.stops = {from, Color{(from[0] + to[0]) * .5f, (from[1] + to[1]) * .5f, (from[2] + to[2]) * .5f}, to}; + } else if (!out.rainbow) { + // One colour getting lighter and darker is hard to see, so each goes on into the colour + // next to it: a deep shade, a bright one, then a reddish pink or a yellow. + constexpr std::array red{{{.2f, .004f, .01f}, {.8f, .03f, .03f}, {.8f, .05f, .13f}}}; + constexpr std::array gold{{{.34f, .12f, .006f}, {.8f, .5f, .06f}, {.8f, .64f, .11f}}}; + constexpr std::array blue{{{.01f, .16f, .8f}, {.3f, .5f, .8f}, {.8f, .8f, .8f}}}; + constexpr std::array green{{{.006f, .2f, .04f}, {.05f, .8f, .2f}, {.35f, .8f, .4f}}}; + out.stops = *mark == IdentityList::content_creator ? red : *mark == IdentityList::centrix ? blue : + *mark == IdentityList::staff ? green : gold; + } + return out; +} +inline Color item_color(const ItemAnimation &animation, std::uint64_t milliseconds) noexcept { + if (animation.rainbow) return rainbow(milliseconds, animation.period); + // 0 to 2 and back, once a round. + const auto along = 1.f - std::cos(static_cast(milliseconds % animation.period) * + (6.2831853f / static_cast(animation.period))); + const auto &from = animation.stops[along < 1.f ? 0 : 1], &to = animation.stops[along < 1.f ? 1 : 2]; + const auto amount = along < 1.f ? along : along - 1.f; + return {from[0] + (to[0] - from[0]) * amount, from[1] + (to[1] - from[1]) * amount, + from[2] + (to[2] - from[2]) * amount}; +} +// The local player's own styles (the Special page): sent with their appearance, and kept in +// their profile as text, two hex digits a byte. +inline std::atomic own_styles{}; +inline std::string mark_styles_text(const multiplayer::MarkStyles &styles) { + constexpr char digits[] = "0123456789abcdef"; + std::string text; + const auto add = [&](std::uint8_t byte) { text += digits[byte >> 4], text += digits[byte & 15]; }; + for (const auto &style : styles) { + add(static_cast(style.mode)); + for (const auto part : style.from) add(part); + for (const auto part : style.to) add(part); + add(style.speed); + } + return text; +} +inline std::optional parse_mark_styles(std::string_view text) noexcept { + multiplayer::MarkStyles styles{}; + if (text.size() != styles.size() * 16) return std::nullopt; + std::size_t at = 0; + bool valid = true; + const auto next = [&]() -> std::uint8_t { + unsigned byte = 0; + for (int digit = 0; digit < 2; ++digit) { + const char c = text[at++]; + if (c >= '0' && c <= '9') byte = byte * 16 + static_cast(c - '0'); + else if (c >= 'a' && c <= 'f') byte = byte * 16 + static_cast(c - 'a' + 10); + else valid = false; + } + return static_cast(byte); + }; + for (auto &style : styles) { + style.mode = static_cast(next()); + for (auto &part : style.from) part = next(); + for (auto &part : style.to) part = next(); + style.speed = next(); + valid = valid && multiplayer::valid_mark_style(style); + } + return valid ? std::optional(styles) : std::nullopt; +} +// A material instance's own parameters: a hash map of typed nodes. +struct ParameterMap { + std::uintptr_t buckets{}, stop{}; + std::uint32_t size{}, total{}; +}; +template ParameterMap parameter_map(Memory &m, std::uintptr_t material) { + ParameterMap out{m.ptr(material, 0x268), 0, m.template get(material, 0x270), + m.template get(material, 0x274)}; + if (!out.buckets || !out.size || out.size > 1024 || !out.total || out.total > 512) return {}; + out.stop = m.ptr(out.buckets, out.size * 8ULL); + return out; +} +// The node of a material's ColorRgb parameter with this key, or 0. The EBX boxed ColorRgb has a +// serialized gamut word; the loaded native ColorRgb is three floats (12 bytes): this goes by +// the actual type descriptor, not the serialized asset size. A parameter of another shape is +// passed over, not reported: what is equipped can be anything. +template +std::uintptr_t color_parameter(Memory &m, std::uintptr_t base, const ParameterMap &map, std::uint64_t key, ParameterKey &native, + Color &color) { + if (!map.size) return 0; + auto node = m.ptr(map.buckets, (key % map.size) * 8ULL); + for (std::uint32_t visited = 0; node && node != map.stop && visited < map.total; ++visited, node = m.ptr(node, 0x40)) { + if (m.template get(node, 8) != key) continue; + native = m.template get(node); + const auto type = m.ptr(node, 0x20); + if (type < base || type >= base + 0x09144000 || native.size != sizeof(Color) || native.count != 1 || + !(m.template get(node, 0x28) & 1) || m.template get(type, 8) != native.type) + return 0; + const auto info = m.ptr(type); + if (m.template get(info) != 0x885eff59U || m.template get(info, 6) != sizeof(Color)) return 0; + color = m.template get(node, 0x10); + return std::all_of(color.begin(), color.end(), [](float c) { return std::isfinite(c); }) ? node : 0; + } + return 0; +} // All discovery is bounded and follows the actor's own component/controller/material // backlinks. Never edit the shared AppearanceShaderExpressionPreset or recipe. template Materials materials(Read &read, std::uintptr_t base, std::uintptr_t entity) { using namespace multiplayer; CosmeticMemory m{read, base}; Materials out; - m.check(m.ptr(entity) == base + addr::engine::skater_entity_vtable, "Hoodie actor type differs."); + out.regions_only = true; + m.check(m.ptr(entity) == base + addr::engine::skater_entity_vtable, "Skater actor type differs."); out.component = m.component(entity); - const bool pending = m.template get(out.component, 0x10a) || m.template get(out.component, 0x10d); - const auto recipe = m.ptr(out.component, 0x158); - const auto items = m.count(recipe, sizeof(NativeCosmeticItem), max_cosmetic_slots); - const auto wanted = cosmetic_asset_hash(asset); - for (std::size_t i = 0; i < items; ++i) { - const auto item = m.template get(recipe, i * sizeof(NativeCosmeticItem)); - if (item.hash == wanted) { - // Native recipes may store only the hash (an empty CString). - const auto name = m.text(reinterpret_cast(item.asset)); - out.eligible = !pending && (name.empty() || name == asset); - } - } + out.pending = m.template get(out.component, 0x10a) || m.template get(out.component, 0x10d); out.appearance = read_native_component(read, entity, base + addr::engine::skater_appearance_vtable); const auto begin = m.ptr(out.appearance, 0x90), end = m.ptr(out.appearance, 0x98), capacity = m.ptr(out.appearance, 0xa0); m.check(begin <= end && end <= capacity && (end - begin) % 8 == 0 && (capacity - begin) % 8 == 0 && - (capacity - begin) / 8 <= 64 && (begin || !capacity), "Hoodie controller array exceeds bounds."); + (capacity - begin) / 8 <= 64 && (begin || !capacity), "Skater controller array exceeds bounds."); for (auto entry = begin; entry < end; entry += 8) { const auto item = m.ptr(entry); - if (!item || m.ptr(item, 0x48) != out.appearance + 0x40 || m.text(m.ptr(item, 0x30)) != top_slot) continue; - m.check(!out.item, "Hoodie has multiple top controllers."); - out.item = item; - if (!(m.template get(item, 0xb0) & 2)) { out.eligible = false; continue; } + if (!item || m.ptr(item, 0x48) != out.appearance + 0x40) continue; + const auto name = m.text(m.ptr(item, 0x30)); + if (!name.starts_with(slot_folder)) continue; + const auto slot = std::find(slots.begin(), slots.end(), std::string_view(name).substr(slot_folder.size())); + if (slot == slots.end()) continue; + const auto part = static_cast(slot - slots.begin()); + // A slot with nothing in it, or with an item still on its way, has no materials. + if (!(m.template get(item, 0xb0) & 2)) continue; const auto buckets = m.ptr(item, 0x1f0); const auto size = m.template get(item, 0x1f8), total = m.template get(item, 0x1fc); if (!total) continue; - m.check(size && size <= 256 && total <= 128, "Hoodie material map exceeds bounds."); - const auto sentinel = m.ptr(buckets, size * 8ULL); - const auto hash = material_hash("Top_mat"); - auto node = m.ptr(buckets, (hash % size) * 8ULL); - for (std::uint32_t visited = 0; node && node != sentinel && visited < total; ++visited, node = m.ptr(node, 0x28)) { - if (m.template get(node) != hash) continue; - std::array seen{}; - for (std::size_t variant = 0; variant < seen.size(); ++variant) { - const auto material = m.ptr(node, 8 + variant * 8); - if (!material || std::find(seen.begin(), seen.begin() + variant, material) != seen.begin() + variant) continue; - seen[variant] = material; - const auto params = m.ptr(material, 0x268); - const auto n = m.template get(material, 0x270), count = m.template get(material, 0x274); - if (!count) continue; - m.check(n && n <= 1024 && count <= 512, "Hoodie parameter map exceeds bounds."); - const auto stop = m.ptr(params, n * 8ULL); - for (const auto key : color_keys) { - auto parameter = m.ptr(params, (key % n) * 8ULL); - for (std::uint32_t v = 0; parameter && parameter != stop && v < count; ++v, parameter = m.ptr(parameter, 0x40)) { - if (m.template get(parameter, 8) != key) continue; - // The EBX boxed ColorRgb has a serialized gamut word; the - // loaded native ColorRgb is three floats (12 bytes). Validate - // the actual type descriptor, not the serialized asset size. - const auto type = m.ptr(parameter, 0x20); - m.check(type >= base && type < base + 0x09144000 && - m.template get(parameter, 2) == sizeof(Color) && - m.template get(m.ptr(type)) == 0x885eff59U && - m.template get(m.ptr(type), 6) == sizeof(Color) && m.template get(parameter, 6) == 1 && - (m.template get(parameter, 0x28) & 1) && - m.template get(type, 8) == m.template get(parameter, 4), - "Hoodie color parameter type differs."); - const auto color = m.template get(parameter, 0x10); - m.check(std::all_of(color.begin(), color.end(), [](float c) { return std::isfinite(c); }) && - out.count < out.bindings.size(), "Hoodie color exceeds bounds."); - out.bindings[out.count++] = {material, parameter, type, key, color}; - break; + m.check(buckets && size && size <= 256 && total <= 128, "Skater material map exceeds bounds."); + // Every named material of the item, whatever its mesh calls them ("Top_mat", "Bottom_mat", ...). + std::array heads{}; + m.check(read(buckets, heads.data(), (size + 1ULL) * 8), "Skater material map cannot be read."); + std::uint32_t visited{}; + for (std::uint32_t bucket = 0; bucket < size; ++bucket) { + for (auto node = heads[bucket]; node && node != heads[size]; node = m.ptr(node, 0x28)) { + if (m.template get(node) % size != bucket) break; // the next bucket's + m.check(visited++ < total, "Skater material chain exceeds bounds."); + std::array seen{}; + for (std::size_t variant = 0; variant < seen.size(); ++variant) { + const auto material = m.ptr(node, 8 + variant * 8); + if (!material || std::find(seen.begin(), seen.begin() + variant, material) != seen.begin() + variant) continue; + seen[variant] = material; + const auto map = parameter_map(m, material); + for (const auto key : color_keys) { + if (out.bindings.size() == max_bindings) break; + ParameterKey native; + Color color{}; + if (const auto parameter = color_parameter(m, base, map, key, native, color)) + out.bindings.push_back({item, material, parameter, native, color, static_cast(part)}); } } } - break; } } return out; } -inline bool same_binding(const Binding &a, const Binding &b) noexcept { - return a.material == b.material && a.node == b.node && a.type == b.type && a.key == b.key; +inline bool same_binding(const Binding &live, const Binding &saved) noexcept { + // A parameter the board added has its node from the next fresh walk on. + return live.item == saved.item && live.material == saved.material && live.key.hash == saved.key.hash && + (saved.node ? live.node == saved.node : live.node != 0); } -} // namespace developer_hoodie_detail -struct DeveloperHoodieState { - std::uintptr_t entity{}, component{}, appearance{}, item{}; - std::uint64_t generation{}; - std::array colors{}; - std::size_t count{}; -}; -namespace developer_hoodie_detail { -// Writes are only to freshly rediscovered bindings. Restoration additionally -// requires the current color to still be ours; a new native preset takes precedence. -template -void animate(DeveloperHoodieState &state, const Materials &live, std::uintptr_t entity, std::uint64_t generation, - bool developer, std::uint64_t milliseconds, Write &write, Publish &publish) { - if (state.entity != entity || state.generation != generation || state.component != live.component || - state.appearance != live.appearance || state.item != live.item) state = {}; - const bool enabled = developer && live.eligible; - const auto color = rainbow(milliseconds); - DeveloperHoodieState next; - if (enabled) { - next.entity = entity; next.generation = generation; next.component = live.component; - next.appearance = live.appearance; next.item = live.item; - } - bool changed{}; - for (std::size_t i = 0; i < live.count; ++i) { +// The renderer shows a color one publish late. While a color keeps changing the next tick's +// publish brings it; one that stands still (a solid color, or the item's own once the +// animation is off) needs publishing again after the write that set it, or the item keeps the +// color before it on screen (seen in game, 2026-10-04). +inline constexpr std::uint8_t settle_publishes = 3; +// Ownership is discovered afresh before every write, and writes go only to what that walk +// found. A color is restored only while it is still the one written here: a new native +// preset takes precedence. `animations`: what each part does, or nothing. +template +void animate(ItemState &state, const Materials &live, std::uintptr_t entity, std::uint64_t generation, + const std::array &animations, std::uint64_t milliseconds, Write &write, Publish &publish) { + if (state.entity != entity || state.generation != generation || state.component != live.component || state.appearance != live.appearance) + state = {}; + // Pausing, loading and a recipe on its way keep what was saved until a walk succeeds. + if (live.pending) return; + // What was saved of each color found: a walk finds them in the same order as the last. + std::vector saved(live.bindings.size()); + std::array regions{}; + for (std::size_t i = 0, from = 0; i < live.bindings.size(); ++i) { const auto &binding = live.bindings[i]; - const SavedColor *saved{}; - for (std::size_t j = 0; j < state.count; ++j) - if (same_binding(binding, state.colors[j].binding)) { saved = &state.colors[j]; break; } - if (enabled) { - const auto original = saved && binding.color == saved->last ? saved->original : binding.color; - if (binding.color == color || write(binding, color)) { - next.colors[next.count++] = {binding, original, color}; - changed = changed || binding.color != color; - } - } else if (saved && binding.color == saved->last && binding.color != saved->original) { - changed = write(binding, saved->original) || changed; + for (std::size_t n = 0; n < state.colors.size(); ++n) { + const auto at = (from + n) % state.colors.size(); + if (same_binding(binding, state.colors[at].binding)) { saved[i] = &state.colors[at], from = at + 1; break; } } + const auto &own = saved[i] && binding.color == saved[i]->last ? saved[i]->original : binding.color; + regions[binding.part] = regions[binding.part] || own != neutral; } - state = next; - if (changed) publish(live.item); + ItemState next; + next.entity = entity; next.generation = generation; next.component = live.component; next.appearance = live.appearance; + next.colors.reserve(live.bindings.size()); + std::array changed_items{}; + std::size_t changed{}; + for (std::size_t i = 0; i < live.bindings.size(); ++i) { + const auto &binding = live.bindings[i]; + const auto *was = saved[i]; + const auto &animation = animations[binding.part]; + const bool ours = was && binding.color == was->last; + const auto original = ours ? was->original : binding.color; + bool show{}; + if (animation.on && !(live.regions_only && regions[binding.part] && original == neutral)) { + const auto color = item_color(animation, milliseconds); + if (binding.node && binding.color == color) { + // Already this color: nothing to write, but it may still be owed to the renderer. + const std::uint8_t left = ours ? was->settling : 0; + next.colors.push_back({binding, original, color, static_cast(left ? left - 1 : 0)}); + show = left != 0; + } else if (write(binding, color)) { + next.colors.push_back({binding, original, color, settle_publishes}); + show = true; + } else if (was) next.colors.push_back(*was); + } else if (was && (binding.color == was->last || binding.color == was->original)) { + // Back to the item's own color, and published until it shows. + const bool first = was->last != was->original; + show = binding.color == was->original || write(binding, was->original); + const std::uint8_t left = first ? settle_publishes : was->settling ? was->settling - 1 : 0; + if (!show) next.colors.push_back(*was); + else if (left) next.colors.push_back({binding, was->original, was->original, left}); + } + if (show && changed < changed_items.size() && + std::find(changed_items.begin(), changed_items.begin() + changed, binding.item) == changed_items.begin() + changed) + changed_items[changed++] = binding.item; + } + state.entity = next.entity, state.generation = next.generation, state.component = next.component, state.appearance = next.appearance; + state.colors = std::move(next.colors); + for (std::size_t i = 0; i < changed; ++i) publish(changed_items[i]); } } // namespace developer_hoodie_detail +// What was saved of a skater's items while they are coloured. +struct DeveloperHoodieState : developer_hoodie_detail::ItemState {}; } // namespace dingosdk diff --git a/Extension/Customization/local_customization_runtime.cpp b/Extension/Customization/local_customization_runtime.cpp index f5cb037..3ff43b4 100644 --- a/Extension/Customization/local_customization_runtime.cpp +++ b/Extension/Customization/local_customization_runtime.cpp @@ -20,8 +20,7 @@ bool cosmetic_diagnostic(const char* operation, const char* reason, std::string_ // ..._rejected are), so a player's ordinary log says why; the rest are routine. const std::string_view why(reason); const bool routine = why == "callback_entered" || why == "unchanged" || why == "no_saved_loadout" || - why == "card_only_preset_recovered" || why == "card_is_not_skater_preset" || - why == "retried_after_catalog_ready"; + why == "card_only_preset_recovered" || why == "card_is_not_skater_preset"; const dingosdk::Json event{{"event", routine ? "local_cosmetic_loadout_diagnostic" : "local_cosmetic_loadout_rejected"}, {"operation", operation}, {"reason", reason}, {"preset", id}}; dingosdk::logging::event(dingosdk::logging::Channel::customization, event.dump().c_str()); @@ -298,20 +297,14 @@ bool recover_card_only_preset(std::string_view id, const profile::CosmeticLoadou return true; } -// The loadout manager's record array, or false when it cannot be read. The -// same shape the save hook demands of a record it is handed. -bool loadout_array(std::uintptr_t& manager, std::uintptr_t& begin, std::uintptr_t& end) { - auto& s = local_runtime(); - return read(s.base + addr::engine::loadout_manager, manager) && manager && - read(manager + 0x18, begin) && read(manager + 0x20, end) && begin <= end && - (end - begin) % 0x48 == 0 && (end - begin) / 0x48 <= 10; -} - -// A load, taking the preset id instead of the game's wrapper so that -// retry_pending_cosmetic_loads can run it again. The caller holds native_mutex. -bool apply_saved_loadout(const std::string& id, void* destination) { +bool load_cosmetic_hook(const void* wrapper, void* destination) { auto& s = local_runtime(); + if (!s.active.load(std::memory_order_acquire)) return s.load_cosmetic(wrapper, destination); + PreserveError preserve; + std::lock_guard lock(s.native_mutex); + std::string id; try { + if (!identifier(wrapper, id)) return cosmetic_diagnostic("load", "invalid_preset_id"); const auto saved = s.store->cosmetic_loadout(id); if (!saved) { profile::CosmeticLoadout starter; std::vector native; @@ -331,15 +324,14 @@ bool apply_saved_loadout(const std::string& id, void* destination) { profile::CosmeticLoadout defaults; std::vector native; if (!read_cosmetic_recipes(destination, defaults, native)) return cosmetic_diagnostic("load", "invalid_native_recipe", id); if (recover_card_only_preset(id, *saved, defaults)) return false; - if (!refresh_cosmetic_catalog()) { - // The cosmetics manager is not ready this early in a level load. - // The preset stays blocked, so nothing is written over the outfit - // on disk, and the pump applies this record once it is ready -- - // but only while this is still the array the record came from. - CosmeticRuntime::PendingLoad pending{destination}; - if (loadout_array(pending.manager, pending.begin, pending.end)) c.pending_loads[id] = pending; - return cosmetic_diagnostic("load", "catalog_not_ready", id); - } + // The cosmetics manager cannot be read for the first seconds of a level + // load, which is when the game asks for every slot up to the selected + // one. The catalog read before the load says the same thing (what is + // installed does not change with the level), so the outfit is checked + // against that one. Only with no catalog read yet this session is the + // load refused; the selection that asks is held back until one is + // (local_customization_outfits_loadable). + if (!refresh_cosmetic_catalog() && c.items.empty()) return cosmetic_diagnostic("load", "catalog_not_ready", id); if (saved->recipes.size() != defaults.recipes.size()) return cosmetic_diagnostic("load", "recipe_count_mismatch", id); const auto snapshot_shared = s.store->shared_snapshot(); const auto& snapshot = *snapshot_shared; @@ -350,7 +342,7 @@ bool apply_saved_loadout(const std::string& id, void* destination) { for (std::size_t i = 0; i < saved->recipes.size(); ++i) { const auto& recipe = saved->recipes[i]; const auto& expected = defaults.recipes[i]; if (recipe.template_key != expected.template_key || recipe.template_version != expected.template_version || - recipe.scalar_bits.size() != expected.scalar_bits.size() || recipe.items.size() != expected.items.size()) + recipe.scalar_bits.size() != expected.scalar_bits.size()) return cosmetic_diagnostic("load", "recipe_template_mismatch", id); std::map slot_categories; if (!cosmetic_slot_categories(native[i].resource, slot_categories)) @@ -358,11 +350,23 @@ bool apply_saved_loadout(const std::string& id, void* destination) { words.emplace_back(recipe.scalar_bits.size()); std::copy(recipe.scalar_bits.begin(), recipe.scalar_bits.end(), words.back().data()); native[i].scalars = words.back().data(); - item_arrays.emplace_back(recipe.items.size()); - for (std::size_t j = 0; j < recipe.items.size(); ++j) { - const auto* slot = &recipe.items[j]; - if (slot->slot != expected.items[j].slot) return cosmetic_diagnostic("load", "slot_mismatch", id); - if (!slot->asset.empty()) { + // The template says which slots there are and in what order; the saved + // outfit is matched to it slot by slot. A mod that adds slots to a + // template (a wheel slot per wheel on the board) changes that list + // under every outfit saved before it, and again when it is switched + // off: a slot the outfit has no item for starts on its default, and a + // saved slot the template no longer has is held, so the next save + // keeps it for when the slot is back. + std::map saved_slots; + for (const auto& item : recipe.items) saved_slots.emplace(item.slot, &item); + std::size_t added{}; + item_arrays.emplace_back(expected.items.size()); + for (std::size_t j = 0; j < expected.items.size(); ++j) { + const auto match = saved_slots.find(expected.items[j].slot); + const auto* slot = match == saved_slots.end() ? &expected.items[j] : match->second; + if (match == saved_slots.end()) ++added; + else saved_slots.erase(match); + if (slot != &expected.items[j] && !slot->asset.empty()) { // An item that is no longer installed (a removed mod, a catalog // change) or no longer fits its slot falls back to the slot's // default. Rejecting the whole outfit would also block every @@ -387,13 +391,19 @@ bool apply_saved_loadout(const std::string& id, void* destination) { } words.emplace_back(slot->parameter_bits.size()); std::copy(slot->parameter_bits.begin(), slot->parameter_bits.end(), words.back().data()); - item_arrays.back().data()[j] = {slot->asset.c_str(), words.back().data(), cosmetic_hash(slot->asset), slot->slot}; + item_arrays.back().data()[j] = {slot->asset.c_str(), words.back().data(), cosmetic_hash(slot->asset), + expected.items[j].slot}; } + for (const auto& [hash, item] : saved_slots) held.push_back({i, *item, {}}); + if (added || !saved_slots.empty()) + dingosdk::logging::log(dingosdk::logging::Level::info, dingosdk::logging::Channel::customization, + "Saved outfit \"{}\": its slots differ from the game's (a mod that adds or removes slots): {} " + "new slot(s) start on their default, {} saved slot(s) the game does not have now are kept.", + id, added, saved_slots.size()); native[i].items = item_arrays.back().data(); } s.copy_cosmetic(destination, native.data(), native.data() + native.size()); c.blocked_loadouts.erase(id); - c.pending_loads.erase(id); if (held.empty()) c.held_slots.erase(id); else c.held_slots[id] = std::move(held); dingosdk::logging::event(dingosdk::logging::Channel::customization, "{\"event\":\"local_cosmetic_loadout_loaded\"}"); @@ -405,44 +415,6 @@ bool apply_saved_loadout(const std::string& id, void* destination) { } } -bool load_cosmetic_hook(const void* wrapper, void* destination) { - auto& s = local_runtime(); - if (!s.active.load(std::memory_order_acquire)) return s.load_cosmetic(wrapper, destination); - PreserveError preserve; - std::lock_guard lock(s.native_mutex); - std::string id; - try { - if (!identifier(wrapper, id)) return cosmetic_diagnostic("load", "invalid_preset_id"); - } catch (...) { - return cosmetic_diagnostic("load", "invalid_preset_id"); - } - return apply_saved_loadout(id, destination); -} - -void retry_pending_cosmetic_loads() { - auto& c = cosmetic_runtime(); - if (c.pending_loads.empty()) return; - std::uintptr_t manager{}, begin{}, end{}; - // Unreadable now: keep the records and come back, rather than losing them. - if (!loadout_array(manager, begin, end)) return; - auto pending = std::move(c.pending_loads); - c.pending_loads.clear(); - for (const auto& [id, load] : pending) { - const auto p = reinterpret_cast(load.destination); - // The same array, the same bounds, and the record still inside it on - // its stride. Anything else and this address may now hold another - // preset: give up on the outfit rather than write it over a different - // saved character. - if (!load.destination || manager != load.manager || begin != load.begin || end != load.end || - p < begin || p >= end || (p - begin) % 0x48) { - cosmetic_diagnostic("load", "retry_record_moved", id); - continue; - } - if (apply_saved_loadout(id, load.destination)) - cosmetic_diagnostic("load", "retried_after_catalog_ready", id); - } -} - void save_cosmetic_hook(std::uintptr_t manager, void* record, const char* raw_id, const char* reason, bool changed_only) { auto& s = local_runtime(); if (!s.active.load(std::memory_order_acquire)) { s.save_cosmetic(manager, record, raw_id, reason, changed_only); return; } @@ -470,9 +442,10 @@ void save_cosmetic_hook(std::uintptr_t manager, void* record, const char* raw_id if (value.recipes.size() == 1 && value.recipes.front().template_key == 2169419386U) { cosmetic_diagnostic("save", "card_is_not_skater_preset", id); return; } - // Keep items from costume mods that are not installed right now: a slot - // still showing the stand-in default saves the original item, while a - // slot the player changed saves the new choice and releases the hold. + // Keep items from costume mods that are not installed right now, and + // whole slots a mod added that is switched off: a slot still showing the + // stand-in default saves the original item, while a slot the player + // changed saves the new choice and releases the hold. if (const auto held = cosmetic_runtime().held_slots.find(id); held != cosmetic_runtime().held_slots.end()) { std::erase_if(held->second, [&](const CosmeticRuntime::HeldSlot& h) { if (h.recipe >= value.recipes.size()) return true; @@ -482,7 +455,10 @@ void save_cosmetic_hook(std::uintptr_t manager, void* record, const char* raw_id item = h.saved; return false; } - return true; + // The game has no such slot now (the mod that added it is off): + // the saved item rides along until the slot is back. + value.recipes[h.recipe].items.push_back(h.saved); + return false; }); if (held->second.empty()) cosmetic_runtime().held_slots.erase(held); } diff --git a/Extension/Customization/local_customization_runtime.h b/Extension/Customization/local_customization_runtime.h index d6744f0..5e4c144 100644 --- a/Extension/Customization/local_customization_runtime.h +++ b/Extension/Customization/local_customization_runtime.h @@ -15,27 +15,15 @@ struct CosmeticRuntime { std::uintptr_t item_manager{}; std::uint32_t item_count{}; std::map> items; + // Presets whose last load was refused. A save to one is refused too, so the + // standard skater the game shows in its place is never written over the + // outfit on disk. std::set blocked_loadouts; - // Presets whose load was refused only because the cosmetics catalog was not - // ready yet, with the record the game asked us to fill. The game asks once - // per level load and does not ask again, so the pump retries these once the - // catalog is ready. Until one succeeds the preset stays in - // `blocked_loadouts`, so a save cannot write over the outfit on disk. - // - // The loadout manager's array is recorded with it. A record is only written - // to when that array is still the same one, bounds and all: inside the - // array and on its stride says a record is there, not that it is still - // this preset's, and a level load can rebuild the array under us. Writing - // one preset's outfit into another's slot would lose a saved character, - // which is worse than not applying it at all. - struct PendingLoad { - void* destination{}; - std::uintptr_t manager{}, begin{}, end{}; - }; - std::map> pending_loads; // Saved outfit items that were not installed when their preset loaded (a // disabled or removed costume mod). The slot shows its default this session; - // saves keep the original item unless the player changes that slot. + // saves keep the original item unless the player changes that slot. Also + // saved slots the template did not have then (a mod that adds slots, + // switched off): those have no slot to show in, and saves carry them along. struct HeldSlot { std::size_t recipe{}; profile::CosmeticSlot saved; std::string fallback; }; std::map, std::less<>> held_slots; std::set diagnostics; @@ -122,11 +110,11 @@ void initialize_starter_from_expression(std::uintptr_t vm, std::uint32_t pc) noe bool recover_card_only_preset(std::string_view id, const profile::CosmeticLoadout& saved, const profile::CosmeticLoadout& defaults); +// The game calls this with a temporary of its own (a slot's default recipes) and +// builds the slot from what is left in it: the saved outfit when this returns +// true, the standard skater when it returns false. It asks once, when the slot +// is first needed after a level load, and there is no record to fill in later. bool load_cosmetic_hook(const void* wrapper, void* destination); -// Applies the outfits whose load was refused for a catalog that was not ready. -// Call from the customization pump, holding native_mutex, once -// refresh_cosmetic_catalog() is true. -void retry_pending_cosmetic_loads(); void save_cosmetic_hook(std::uintptr_t manager, void* record, const char* raw_id, const char* reason, bool changed_only); } diff --git a/Extension/Multiplayer/Hud/custom_nametags.cpp b/Extension/Multiplayer/Hud/custom_nametags.cpp index 922af76..8c4dadd 100644 --- a/Extension/Multiplayer/Hud/custom_nametags.cpp +++ b/Extension/Multiplayer/Hud/custom_nametags.cpp @@ -26,6 +26,8 @@ struct State { std::vector tags; Clock::time_point published; bool hidden{}; // the game hides its own nametags (menus, hidden UI; game_ui_state.h) + bool show_names{true}, show_bubbles{}, dots{true}; + float bubble_distance{40.f}, name_distance{120.f}; }; State &state() { static auto *value = new State; @@ -34,7 +36,8 @@ State &state() { } // namespace void publish_custom_nametags(std::uintptr_t base, std::vector players, - std::optional> local) noexcept { + std::optional> local, bool show_names, + bool show_bubbles, float bubble_distance, float name_distance, bool dots) noexcept { auto &s = state(); if (!s.enabled.load(std::memory_order_acquire)) return; s.base.store(base, std::memory_order_release); @@ -54,6 +57,11 @@ void publish_custom_nametags(std::uintptr_t base, std::vector pla tag.color = player.color; tag.tag = std::move(player.tag); tag.talking = player.talking; + tag.bubbles.reserve(player.bubbles.size()); + for (auto &line : player.bubbles) + tag.bubbles.push_back({std::move(line.text), std::move(line.raw), line.appear, line.fade}); + tag.self = player.self; + tag.nameless = player.nameless; if (local) { const float dx = player.head[0] - (*local)[0], dy = player.head[1] - (*local)[1], dz = player.head[2] - (*local)[2]; tag.distance = std::sqrt(dx * dx + dy * dy + dz * dz); @@ -62,6 +70,11 @@ void publish_custom_nametags(std::uintptr_t base, std::vector pla } std::lock_guard lock(s.mutex); s.hidden = hidden; + s.show_names = show_names; + s.show_bubbles = show_bubbles; + s.bubble_distance = bubble_distance; + s.name_distance = name_distance; + s.dots = dots; if (!view) { s.tags.clear(); return; @@ -113,6 +126,11 @@ overlay::Nametags custom_nametags() { std::lock_guard lock(s.mutex); if (s.hidden || s.tags.empty() || Clock::now() - s.published > std::chrono::milliseconds(250)) return {}; result.tags = s.tags; + result.show_names = s.show_names; + result.show_bubbles = s.show_bubbles; + result.bubble_distance = s.bubble_distance; + result.name_distance = s.name_distance; + result.dots = s.dots; view = s.view; } read_live_view(s.base.load(std::memory_order_acquire), view); diff --git a/Extension/Multiplayer/Hud/custom_nametags.h b/Extension/Multiplayer/Hud/custom_nametags.h index 16bc9b8..baa24da 100644 --- a/Extension/Multiplayer/Hud/custom_nametags.h +++ b/Extension/Multiplayer/Hud/custom_nametags.h @@ -10,24 +10,55 @@ // ReSkate's own nametags (drawn by the overlay, Extension/UI/Overlay/nametag_overlay.cpp) // in place of the game's nametag and compass arrows; the pause map's player icons stay. namespace dingosdk::multiplayer { +// One line in a player's bubble stack: the text (masked when the chat filter is on), the line +// as sent when the filter changed it (so emote names survive), how far its pop-in has come +// (0 just arrived, 1 settled) and how opaque it still is (1, down to 0 as it goes). +struct NametagBubble { + std::string text, raw; + float appear{1}, fade{1}; +}; struct NametagPlayer { std::array head{}; // world position above the skater std::string name; std::uint32_t color{0xffffffffU}; // IM_COL32 layout std::string tag; // role badge before the name, as in chat ("" for none) bool talking{}; + // The player's recent chat lines to show as bubbles above the head, oldest first. Empty + // draws no bubbles. + std::vector bubbles; + // The local player's own tag: only ever draws bubbles, never a name or dot. + bool self{}; + // Another player whose name is not shown (friends-only nametags): bubbles only, like `self`. + bool nameless{}; }; // Role colours; friends are the viewer's own Steam friends. inline constexpr std::uint32_t nametag_white = 0xffffffffU, nametag_developer = 0xffff78b4U, // purple - nametag_admin = 0xff3dc8ffU, // gold + nametag_creator = 0xff6464ffU, // red + nametag_homie = 0xff7ae2ffU, // gold + nametag_centrix = 0xffff7b1fU, // blue (#1F7BFF) + nametag_staff = 0xff71cc2eU, // green (#2ECC71) + nametag_admin = 0xffc674ffU, // pink nametag_host = 0xffffc85eU, // blue - nametag_friend = 0xff8ae07bU; // green -inline constexpr std::uint32_t nametag_developer_start = 0xffff206eU; // #6E20FF, IM_COL32 layout + nametag_friend = 0xff8ae07bU, // green + nametag_server = 0xffff5c8eU, // violet (#8E5CFF) + nametag_server_text = 0xffffc8d9U; // lavender (#D9C8FF), its lines' text +// Developers, staff, content creators, homies and Centrix shimmer from these to their role +// colour (Centrix between white and its blue). +inline constexpr std::uint32_t nametag_developer_start = 0xffff206eU, // #6E20FF, IM_COL32 layout + nametag_creator_start = 0xff2e10c8U, // #C8102E + nametag_homie_start = 0xff008ae0U, // #E08A00 + nametag_centrix_start = 0xffffffffU, // white + nametag_staff_start = 0xff3a6b0bU; // #0B6B3A // Client thread, once per rendered frame: everyone to label and the local skater's position // (for distances). Also reads whether the game is hiding its own nametags right now. +// `show_names` draws the name, distance and role badge; `show_bubbles` draws chat bubbles +// within `bubble_distance` metres. Names show within `name_distance` metres; past it, and for +// players off screen, a dot when `dots` is on. void publish_custom_nametags(std::uintptr_t base, std::vector players, - std::optional> local) noexcept; + std::optional> local, bool show_names, + bool show_bubbles, float bubble_distance, float name_distance = 120.f, + bool dots = true) noexcept; void set_custom_nametags_enabled(bool enabled) noexcept; bool custom_nametags_enabled() noexcept; // The overlay's feed (any thread): nothing while off, stale, or while the game hides nametags. diff --git a/Extension/Multiplayer/Hud/native_party.cpp b/Extension/Multiplayer/Hud/native_party.cpp index c694f21..930e4fe 100644 --- a/Extension/Multiplayer/Hud/native_party.cpp +++ b/Extension/Multiplayer/Hud/native_party.cpp @@ -388,7 +388,7 @@ void update_native_party(std::uintptr_t base, const PartyRoster &roster, unsigne if (!roster[i].id) continue; auto &record = next->records[i]; record.player = roster[i]; record.player_id = 0x7e000000U + static_cast(i); - // The game's colour list is sized for its eight-member parties: a bigger lobby party repeats them. + // The game's colour list is sized for its eight-member parties: a bigger one repeats them. record.colour = roster[i].member ? colours++ % static_cast(authored_party_slots) : -1; record.friend_of = std::any_of(steam->friends.begin(), steam->friends.end(), [&](const auto &friend_info) { return friend_info.id == roster[i].id; diff --git a/Extension/Multiplayer/Hud/native_party.h b/Extension/Multiplayer/Hud/native_party.h index 026b02b..856e908 100644 --- a/Extension/Multiplayer/Hud/native_party.h +++ b/Extension/Multiplayer/Hud/native_party.h @@ -37,8 +37,8 @@ void spectate_party_member(std::uint64_t id) noexcept; // Returns only a record owned by this adapter, validated against this manager. std::uint64_t native_party_player_info(std::uintptr_t manager, std::size_t slot) noexcept; std::string native_party_status(); -// Whether players form their own parties (a dedicated server); a lobby is one fixed party, so -// the game's party buttons are refused there. +// Whether players can form parties: in a session, a lobby or a dedicated server. Outside one +// the game's party buttons are refused. void set_native_party_changes(bool allowed) noexcept; // A party invite from `from` (a session player) as the game's own invite toast, whose Accept and // Decline come back through the request hooks. False when it could not be shown. Game thread. diff --git a/Extension/Multiplayer/Hud/native_party_hooks.cpp b/Extension/Multiplayer/Hud/native_party_hooks.cpp index eba5afb..f74c419 100644 --- a/Extension/Multiplayer/Hud/native_party_hooks.cpp +++ b/Extension/Multiplayer/Hud/native_party_hooks.cpp @@ -303,7 +303,7 @@ void update_map_markers() { auto &marker = s.markers[slot]; const auto record = published->slot_record(slot); const auto value = record ? target(record->player.id) : std::optional{}; - // Party markers are for the local player's party (a lobby is one party); everyone + // Party markers are for the local player's party; everyone // else shows as the game's plain player dot (native_player_ui.cpp). if (!value || !available(*value) || !record->player.member) { remove(marker); continue; } if (marker.id != record->player.id || marker.epoch != record->player.epoch || diff --git a/Extension/Multiplayer/Hud/native_party_requests.cpp b/Extension/Multiplayer/Hud/native_party_requests.cpp index d5564df..a8e538a 100644 --- a/Extension/Multiplayer/Hud/native_party_requests.cpp +++ b/Extension/Multiplayer/Hud/native_party_requests.cpp @@ -12,7 +12,7 @@ // The game's own party buttons (Social menu, player cards, invites) end in a handful of request // natives that need a native Blaze game group, which ReSkate never has. While the SDK owns the // party, these hooks turn each request into a ReSkate party request (the session's "party" -// command, which a dedicated server answers) and complete the button's callback themselves. +// command, which whoever hosts answers) and complete the button's callback themselves. // Party queries are answered from the published roster (analysis/party-re/native-core.md). namespace dingosdk::multiplayer { using namespace native_party_detail; @@ -40,7 +40,7 @@ struct Requests { }; Requests &requests() { static auto *value = new Requests; return *value; } std::atomic changes_allowed{}; -constexpr const char *lobby_party = "Everyone in a lobby is in one party with the host."; +constexpr const char *lobby_party = "Parties need a multiplayer session."; std::uint64_t now_ms() { return static_cast(std::chrono::duration_cast( std::chrono::steady_clock::now().time_since_epoch()).count()); @@ -333,7 +333,7 @@ bool post_event(std::uintptr_t type, const void *payload) { } void post_party_changes(const Published *previous, const Published &next) { if (!requests().installed.load(std::memory_order_acquire)) return; - // The local party as the game shows it (a hidden lobby party is none). + // The local party as the game shows it. const auto members = [](const Published *p) { std::map result; // id -> leads if (!p) return result; diff --git a/Extension/Multiplayer/Net/delta_codec.cpp b/Extension/Multiplayer/Net/delta_codec.cpp index 2ad5c1d..1d71a4c 100644 --- a/Extension/Multiplayer/Net/delta_codec.cpp +++ b/Extension/Multiplayer/Net/delta_codec.cpp @@ -43,7 +43,8 @@ WireUpdate DeltaSender::build(const Packet &p, std::span raw } const auto found = bases_.find({p.source, p.kind}); if (found == bases_.end() || found->second.epoch != p.epoch || found->second.world != p.world || p.time_us < found->second.time || - (p.kind != PacketKind::cosmetics && p.time_us - found->second.time >= 2000000)) { + (p.kind != PacketKind::cosmetics && + (aligned_ ? p.time_us / refresh_us_ != found->second.time / refresh_us_ : p.time_us - found->second.time >= refresh_us_))) { if (wire.size() + 4 > max_packet) return out; out.bytes.assign(wire.begin(), wire.end()); diff --git a/Extension/Multiplayer/Net/delta_codec.h b/Extension/Multiplayer/Net/delta_codec.h index 7ca5fbf..ba43e47 100644 --- a/Extension/Multiplayer/Net/delta_codec.h +++ b/Extension/Multiplayer/Net/delta_codec.h @@ -32,6 +32,8 @@ class DeltaSender { }; std::map bases_; std::uint64_t clock_{}; + std::uint64_t refresh_us_ = 2000000; + bool aligned_{}; WireUpdate build(const Packet &, std::span raw, std::span wire, DeltaCache *cache) const; @@ -44,6 +46,18 @@ class DeltaSender { WireUpdate prepare(const Packet &, std::span raw, std::span wire, DeltaCache &cache) const; void sent(const Packet &, WireUpdate &&); + // Drops a stream's reference: the next packet of it goes as a whole state. + void forget(std::uint64_t source, PacketKind kind) { bases_.erase(StreamKey{source, kind}); } + // How old a pose or sound reference may get before a whole state replaces it. Each one + // is sent reliably, so Steam resends it until it arrives: a dedicated server, with a + // stream for every pair of players, keeps them rarer than the two seconds a game does. + // `aligned`: on the source's own clock, at each multiple of the time, instead of that + // long after this recipient's last one. Every recipient of a source then takes the same + // packet as its reference, and one patch serves them all (DeltaCache). + void set_refresh(std::uint64_t microseconds, bool aligned = false) { + refresh_us_ = microseconds; + aligned_ = aligned; + } }; class DeltaReceiver { struct Base { diff --git a/Extension/Multiplayer/Net/pose_batch.h b/Extension/Multiplayer/Net/pose_batch.h new file mode 100644 index 0000000..53e45e5 --- /dev/null +++ b/Extension/Multiplayer/Net/pose_batch.h @@ -0,0 +1,458 @@ +#pragma once +#include "pose_codec.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// Poses from a dedicated server to a player: several players' poses in one message, each as its +// differences (pose_codec.h) from a pose of that player the receiver has said it holds. +// +// A message is a header and entries. A whole pose opens a stream: it names the player and gives +// the stream a number, and every later entry is that number, which earlier pose it builds on, and +// the differences. The receiver answers with which messages it could read in full (an ack: the +// newest and a bit for each of the 64 before it), and the server builds only on poses in messages +// that were acked, or on the whole pose it sent reliably. Messages are kept to one packet, so a +// lost packet loses a handful of poses, each of which the next one makes good. +namespace dingosdk::multiplayer::pose_batch { +inline constexpr std::array batch_magic{'R', 'M', 'Q', '1'}, ack_magic{'R', 'M', 'A', '1'}; +inline constexpr std::size_t header_size = 16, ack_size = 16; +// Entries are added to a message while it stays under this, which with Steam's own framing fits a packet. +inline constexpr std::size_t message_room = 1100; +// How many poses a receiver keeps of each stream to build on, and how many a server may send +// past the newest one acked before it sends the stream whole again. +inline constexpr std::size_t receiver_keeps = 32, sender_runs = 24; + +inline bool is_batch(std::span bytes) noexcept { + return bytes.size() >= header_size && std::equal(batch_magic.begin(), batch_magic.end(), bytes.begin()); +} +inline bool is_ack(std::span bytes) noexcept { + return bytes.size() == ack_size && std::equal(ack_magic.begin(), ack_magic.end(), bytes.begin()); +} + +namespace detail { +inline void put(std::vector &out, std::uint64_t value, unsigned bytes) { + for (unsigned i = 0; i < bytes; ++i) out.push_back(static_cast(value >> (8 * i))); +} +inline void varint(std::vector &out, std::uint64_t value) { + while (value >= 0x80) { + out.push_back(static_cast(value | 0x80)); + value >>= 7; + } + out.push_back(static_cast(value)); +} +struct Reader { + std::span bytes; + std::size_t at{}; + std::uint64_t get(unsigned count) { + if (at > bytes.size() || count > bytes.size() - at) throw std::runtime_error("Truncated pose batch"); + std::uint64_t value{}; + for (unsigned i = 0; i < count; ++i) value |= std::uint64_t{bytes[at++]} << (8 * i); + return value; + } + std::uint64_t varint() { + std::uint64_t value{}; + for (unsigned shift = 0; shift < 64; shift += 7) { + const auto byte = get(1); + value |= (byte & 0x7f) << shift; + if (!(byte & 0x80)) return value; + } + throw std::runtime_error("Invalid pose batch number"); + } + std::span take(std::size_t count) { + if (at > bytes.size() || count > bytes.size() - at) throw std::runtime_error("Truncated pose batch"); + const auto part = bytes.subspan(at, count); + at += count; + return part; + } +}; +constexpr std::uint64_t zigzag(std::int64_t value) noexcept { + return (static_cast(value) << 1) ^ static_cast(value >> 63); +} +constexpr std::int64_t unzigzag(std::uint64_t value) noexcept { return static_cast(value >> 1) ^ -static_cast(value & 1); } +} // namespace detail + +// How often the pose's player is being sent, which the receiver plays it back by: at the +// session's own rate, or every 100 or 200 ms. +enum class Rate : std::uint8_t { full, half, low }; + +struct Entry { + std::uint16_t stream{}; + bool whole{}, collision{}; + Rate rate{Rate::full}; + std::uint64_t source{}, epoch{}; // whole only: who the stream is + std::uint32_t sequence{}; // whole only: the pose's own number + std::uint16_t reference{}; // difference only: the low bits of the number of the pose it builds on + std::uint32_t ahead{}; // difference only: how far past that pose's number its own is (1 or more) + std::uint64_t time_us{}; // whole: the pose's time. Difference: left 0, see time_step + std::int64_t time_step{}; // difference only: its time less its reference's + std::span pose; // pose_codec bytes +}; + +// The first 16 bytes: which message this is, and the world and map it belongs to (their low bits: +// enough to tell a message from before a map change). +inline std::vector begin(std::uint32_t number, std::uint64_t world, std::uint64_t map) { + std::vector out(batch_magic.begin(), batch_magic.end()); + out.reserve(message_room + 64); + detail::put(out, number, 4); + detail::put(out, world & 0xffffffff, 4); + detail::put(out, map & 0xffffffff, 4); + return out; +} +inline void add(std::vector &out, const Entry &entry) { + detail::varint(out, entry.stream); + out.push_back(static_cast((entry.whole ? 1U : 0U) | (entry.collision ? 2U : 0U) | (static_cast(entry.rate) << 2))); + if (entry.whole) { + detail::put(out, entry.source, 8); + detail::put(out, entry.epoch, 8); + detail::put(out, entry.sequence, 4); + detail::put(out, entry.time_us, 8); + } else { + detail::put(out, entry.reference, 2); + detail::varint(out, entry.ahead); + detail::varint(out, detail::zigzag(entry.time_step)); + } + detail::varint(out, entry.pose.size()); + out.insert(out.end(), entry.pose.begin(), entry.pose.end()); +} +// The room an entry takes beyond its pose bytes, at most. +inline constexpr std::size_t entry_room = 3 + 1 + 28 + 3; + +struct Batch { + std::uint32_t number{}, world{}, map{}; + std::vector entries; // their pose bytes point into the message read +}; +// A message's header and entries, or nothing when it is not one or is cut short. +inline std::optional read(std::span bytes) noexcept { + try { + if (!is_batch(bytes)) return {}; + detail::Reader r{bytes, 4}; + Batch batch; + batch.number = static_cast(r.get(4)); + batch.world = static_cast(r.get(4)); + batch.map = static_cast(r.get(4)); + while (r.at < bytes.size()) { + Entry entry; + const auto stream = r.varint(); + if (stream > 0xffff) return {}; + entry.stream = static_cast(stream); + const auto flags = r.get(1); + if (flags & ~0xfULL || ((flags >> 2) & 3) > 2) return {}; + entry.whole = flags & 1; + entry.collision = flags & 2; + entry.rate = static_cast((flags >> 2) & 3); + if (entry.whole) { + entry.source = r.get(8); + entry.epoch = r.get(8); + entry.sequence = static_cast(r.get(4)); + entry.time_us = r.get(8); + } else { + entry.reference = static_cast(r.get(2)); + const auto ahead = r.varint(); + if (!ahead || ahead > 0xffffff) return {}; + entry.ahead = static_cast(ahead); + entry.time_step = detail::unzigzag(r.varint()); + } + const auto length = r.varint(); + if (length > bytes.size()) return {}; + entry.pose = r.take(static_cast(length)); + batch.entries.push_back(entry); + if (batch.entries.size() > 512) return {}; + } + return batch; + } catch (...) { + return {}; + } +} + +// Which messages a receiver read in full: the newest, and a bit for each of the 64 before it +// (bit 0 the one just before). +struct Ack { + std::uint32_t newest{}; + std::uint64_t earlier{}; + bool any{}; + void note(std::uint32_t number) noexcept { + if (!any) { + any = true; + newest = number; + earlier = 0; + } else if (number > newest) { + const auto step = number - newest; + earlier = step >= 64 ? 0 : earlier << step; + if (step <= 64) earlier |= std::uint64_t{1} << (step - 1); // the one that was newest + newest = number; + } else if (number < newest && newest - number <= 64) { + earlier |= std::uint64_t{1} << (newest - number - 1); + } + } + bool holds(std::uint32_t number) const noexcept { + return any && (number == newest || (number < newest && newest - number <= 64 && (earlier >> (newest - number - 1)) & 1)); + } + std::array bytes() const noexcept { + std::array out{}; + std::copy(ack_magic.begin(), ack_magic.end(), out.begin()); + for (unsigned i = 0; i < 4; ++i) out[4 + i] = static_cast(newest >> (8 * i)); + for (unsigned i = 0; i < 8; ++i) out[8 + i] = static_cast(earlier >> (8 * i)); + return out; + } + static std::optional read(std::span bytes) noexcept { + if (!is_ack(bytes)) return {}; + Ack ack; + ack.any = true; + for (unsigned i = 0; i < 4; ++i) ack.newest |= std::uint32_t{bytes[4 + i]} << (8 * i); + for (unsigned i = 0; i < 8; ++i) ack.earlier |= std::uint64_t{bytes[8 + i]} << (8 * i); + return ack; + } +}; + +// A receiver's side of one stream: who it is, and the last poses of theirs it rebuilt. +struct Stream { + std::uint64_t source{}, epoch{}; + struct Held { + std::uint32_t sequence{}; + std::uint64_t time_us{}; + pose_codec::QuantPose pose; + }; + std::deque held; + const Held *find_low(std::uint16_t low) const noexcept { + for (auto it = held.rbegin(); it != held.rend(); ++it) + if (static_cast(it->sequence) == low) return &*it; + return nullptr; + } + bool holds(std::uint32_t sequence) const noexcept { + for (const auto &pose : held) + if (pose.sequence == sequence) return true; + return false; + } + void keep(Held pose) { + held.push_back(std::move(pose)); + while (held.size() > receiver_keeps) held.pop_front(); + } +}; +// What one entry rebuilds to on the receiver, or nothing when it cannot be (a stream it was never +// told of, a reference it does not hold, bytes that are not a pose). `streams` is updated: a whole +// pose opens or reopens its stream, and every rebuilt pose is kept to build on. +struct Rebuilt { + std::uint64_t source{}, epoch{}, time_us{}; + std::uint32_t sequence{}; + bool collision{}; + Rate rate{Rate::full}; + pose_codec::QuantPose pose; + bool repeat{}; // this stream's pose of that number was already held: nothing new +}; +template std::optional rebuild(Streams &streams, const Entry &entry) { + Rebuilt out; + out.collision = entry.collision; + out.rate = entry.rate; + if (entry.whole) { + if (!pose_codec::whole(entry.pose)) return {}; + auto pose = pose_codec::decode(entry.pose, nullptr); + if (!pose) return {}; + auto &stream = streams[entry.stream]; + if (stream.source != entry.source || stream.epoch != entry.epoch) stream = {entry.source, entry.epoch, {}}; + out.source = entry.source; + out.epoch = entry.epoch; + out.sequence = entry.sequence; + out.time_us = entry.time_us; + out.repeat = stream.holds(entry.sequence); + out.pose = std::move(*pose); + if (!out.repeat) stream.keep({out.sequence, out.time_us, out.pose}); + return out; + } + const auto found = streams.find(entry.stream); + if (found == streams.end() || pose_codec::whole(entry.pose)) return {}; + auto &stream = found->second; + const auto *reference = stream.find_low(entry.reference); + if (!reference) return {}; + auto pose = pose_codec::decode(entry.pose, &reference->pose); + if (!pose) return {}; + out.source = stream.source; + out.epoch = stream.epoch; + out.sequence = reference->sequence + entry.ahead; + out.time_us = static_cast(static_cast(reference->time_us) + entry.time_step); + out.repeat = stream.holds(out.sequence); + out.pose = std::move(*pose); + if (!out.repeat) stream.keep({out.sequence, out.time_us, out.pose}); + return out; +} + +// A server's side of what one player is sent: for each other player a stream, the newest pose of +// it the receiver acked, and the messages sent and not yet acked. Poses are handed to add() as +// they fall due; it packs them into messages and gives each to `emit(bytes, reliable)`, which +// answers whether it went out. `find(sequence)` gives a kept pose of the player being added, or +// nothing when it is no longer kept. +struct KeptView { + std::uint32_t sequence{}; + std::uint64_t time_us{}; + const pose_codec::QuantPose *pose{}; +}; +class Sender { + public: + // What add() did with a pose, for counting. + enum class Did : std::uint8_t { nothing, difference, whole }; + struct Added { + Did did{Did::nothing}; + std::size_t bytes{}; + }; + void begin(std::uint64_t world, std::uint64_t map) { + world_ = world; + map_ = map; + message_.clear(); + batch_ = {}; + } + template + Added add(std::uint64_t source, std::uint64_t epoch, const KeptView &pose, Find &&find, bool hold_fingers, bool collision, Rate rate, + std::uint64_t now, Emit &&emit) { + using pose_codec::Fingers; + auto &out = streams_[source]; + if (!out.stream || out.epoch != epoch) { + out = {}; + out.epoch = epoch; + out.stream = next_stream_++; + if (!next_stream_) next_stream_ = 1; + } + // The pose to build on: the newest they acked, else the whole one sent reliably (it will + // arrive; a difference that overtakes it is only dropped). Neither when too many have + // gone since for the receiver to still hold it, or it is no longer kept here. + std::optional reference; + if (out.acked && out.sends - out.acked_send <= sender_runs) reference = find(out.acked_sequence); + if (!reference && out.whole_sent && out.sends - out.whole_send <= sender_runs) reference = find(out.whole_sequence); + if (reference && pose.sequence <= reference->sequence) return {}; // nothing newer than they hold + Entry entry; + entry.stream = out.stream; + entry.collision = collision; + entry.rate = rate; + std::vector bytes; + bool fingers{}; + if (reference) { + // Fingers: not sent to a player too far to see them; and after that, sent whole until + // an ack shows the receiver has them again, since its copy of the reference has old + // ones in it. + const auto mode = hold_fingers ? Fingers::held : out.fingers_stale ? Fingers::whole : Fingers::sent; + bytes = pose_codec::encode_delta(*pose.pose, *reference->pose, mode); + fingers = mode == Fingers::whole; + if (bytes.empty()) reference.reset(); // another skeleton (a changed rig): whole + } + if (!reference) { + // The whole pose, in a message of its own, reliably. Not again while the last is still + // on its way: at most one a second for a stream. + if (out.whole_sent && now - out.whole_at < 1000000) return {}; + flush(emit); + open(); + bytes = pose_codec::encode_whole(*pose.pose); + entry.whole = true; + entry.source = source; + entry.epoch = epoch; + entry.sequence = pose.sequence; + entry.time_us = pose.time_us; + entry.pose = bytes; + pose_batch::add(message_, entry); + out.acked = false; + out.whole_sent = true; + out.whole_sequence = pose.sequence; + out.whole_send = ++out.sends; + out.whole_at = now; + out.fingers_stale = false; + batch_.poses.push_back({source, epoch, pose.sequence, out.sends, true}); + send(emit, true); + return {Did::whole, bytes.size()}; + } + entry.reference = static_cast(reference->sequence); + entry.ahead = pose.sequence - reference->sequence; + entry.time_step = static_cast(pose.time_us) - static_cast(reference->time_us); + entry.pose = bytes; + if (message_.empty()) open(); + if (!batch_.poses.empty() && message_.size() + bytes.size() + entry_room > message_room) { + send(emit, false); + open(); + } + pose_batch::add(message_, entry); + if (hold_fingers) { + out.fingers_stale = true; + out.last_held_batch = batch_.number; + } + batch_.poses.push_back({source, epoch, pose.sequence, ++out.sends, fingers}); + return {Did::difference, bytes.size()}; + } + // Sends what add() has packed so far. + template void flush(Emit &&emit) { send(emit, false); } + // The receiver said which messages it read in full: the poses in those are ones it holds. + void ack(const Ack &ack) { + for (auto it = sent_.begin(); it != sent_.end();) { + if (!ack.holds(it->number)) { + ++it; + continue; + } + for (const auto &pose : it->poses) { + const auto found = streams_.find(pose.source); + if (found == streams_.end() || found->second.epoch != pose.epoch) continue; + auto &out = found->second; + if (!out.acked || pose.sequence > out.acked_sequence) { + out.acked = true; + out.acked_sequence = pose.sequence; + out.acked_send = pose.send; + } + // (Not when a pose without fingers went out after this one.) + if (pose.fingers && it->number > out.last_held_batch) out.fingers_stale = false; + } + it = sent_.erase(it); + } + while (!sent_.empty() && sent_.front().number + 128 < ack.newest) sent_.pop_front(); + } + // A new world: every stream starts over. The messages' numbers go on, so the receiver's + // acks need no starting over with them. + void restart() { + streams_.clear(); + sent_.clear(); + message_.clear(); + batch_ = {}; + } + // The player left: their stream is nobody's. + void forget(std::uint64_t source) { streams_.erase(source); } + std::size_t unacked() const noexcept { return sent_.size(); } + + private: + struct Out { + std::uint64_t epoch{}; + std::uint16_t stream{}; + bool acked{}, whole_sent{}, fingers_stale{}; + std::uint32_t acked_sequence{}, whole_sequence{}, sends{}, acked_send{}, whole_send{}, last_held_batch{}; + std::uint64_t whole_at{}; + }; + struct SentPose { + std::uint64_t source{}, epoch{}; + std::uint32_t sequence{}, send{}; + bool fingers{}; // it carried the fingers whole + }; + struct SentBatch { + std::uint32_t number{}; + std::vector poses; + }; + void open() { + batch_ = {next_batch_++, {}}; + message_ = pose_batch::begin(batch_.number, world_, map_); + } + // Unreliable messages are let go when the connection is behind (the next poses replace + // them); nothing is then recorded as sent, so nothing is built on them either. + template void send(Emit &&emit, bool reliable) { + if (batch_.poses.empty()) return; + if (emit(std::span(message_), reliable)) { + sent_.push_back(std::move(batch_)); + while (sent_.size() > 256) sent_.pop_front(); + } + batch_ = {}; + message_.clear(); + } + std::map streams_; + std::deque sent_; + std::vector message_; + SentBatch batch_; + std::uint64_t world_{}, map_{}; + std::uint16_t next_stream_ = 1; + std::uint32_t next_batch_ = 1; +}; +} // namespace dingosdk::multiplayer::pose_batch diff --git a/Extension/Multiplayer/Net/pose_codec.h b/Extension/Multiplayer/Net/pose_codec.h new file mode 100644 index 0000000..c9a9385 --- /dev/null +++ b/Extension/Multiplayer/Net/pose_codec.h @@ -0,0 +1,361 @@ +#pragma once +#include "protocol.h" +#include +#include +#include +#include +#include +#include +#include +#include + +// A pose as the differences from an earlier one, each in as few bits as it needs. +// +// Measured on a minute of recorded play (tools/research, pose-dump): of a skater's 413 bones 87 +// ever turn and about 70 turn in any one pose, by a degree or so; 40 of the 87 are fingers. The +// older encoding sends each changed field whole (6 bytes a rotation) and leaves the rest to a +// general compressor: 661 bytes a pose. Here a pose is first rounded to whole steps (QuantPose), +// and what is sent is, for each bone that changed, the small integer it changed by: about 310 +// bytes against a pose 67 ms older, 215 without the fingers. +// +// Both ends work on the rounded integers, never on the floats, so a chain of differences cannot +// drift: applying a difference to the reference gives exactly the sender's rounded pose. +namespace dingosdk::multiplayer::pose_codec { +// Rotations are rounded to 2^rotation_shift of the 16-bit form a compact pose packs them in: +// 0.079 degrees. Positions to a millimetre. +inline constexpr unsigned rotation_shift = 5; +inline constexpr int rotation_bits = 16 - rotation_shift + 1; // a whole component, signed +inline constexpr float rotation_scale = 46339.5358f; // as Writer::compact_transform +inline constexpr std::int32_t position_limit = 1000000000; // mm: valid_transform's million metres + +struct QuantBone { + std::int16_t rotation[3]{}; // the three smaller components, in steps + std::uint8_t largest{3}; // which component is left out (and is positive) + bool scaled{}; + std::int32_t position[3]{}; // mm + float scale[3]{1, 1, 1}; // sent whole, and only when it is not 1: mods aside it always is + bool operator==(const QuantBone &) const = default; +}; +// The root, the skater's bones, then the board's. +struct QuantPose { + std::uint16_t skater{}, board{}; + std::vector bones; + bool operator==(const QuantPose &) const = default; +}; +// The skater's finger bones, as indices into QuantPose::bones (the root is 0): two runs of +// twenty. Only in the skeleton this was measured on; any other bone count has none. +inline constexpr std::size_t finger_skeleton = 395; +inline constexpr bool finger_bone(const QuantPose &pose, std::size_t bone) noexcept { + return pose.skater == finger_skeleton && ((bone >= 52 && bone < 72) || (bone >= 281 && bone < 301)); +} + +inline QuantBone quantize(const Transform &t) noexcept { + QuantBone bone; + unsigned largest{}; + float norm{}; + for (unsigned i = 0; i < 4; ++i) { + norm += t.rotation[i] * t.rotation[i]; + if (std::abs(t.rotation[i]) > std::abs(t.rotation[largest])) largest = i; + } + const float factor = norm > 0.f && std::isfinite(norm) ? (t.rotation[largest] < 0 ? -1.f : 1.f) * rotation_scale / std::sqrt(norm) : 0.f; + const float step = static_cast(1U << rotation_shift); + const auto most = static_cast(32767 >> rotation_shift); + bone.largest = static_cast(largest); + unsigned out{}; + for (unsigned i = 0; i < 4; ++i) + if (i != largest) bone.rotation[out++] = static_cast(std::clamp(std::lround(t.rotation[i] * factor / step), -most, most)); + for (unsigned i = 0; i < 3; ++i) { + const auto mm = std::isfinite(t.position[i]) ? std::llround(static_cast(t.position[i]) * 1000.0) : 0LL; + bone.position[i] = static_cast(std::clamp(mm, -position_limit, position_limit)); + } + bone.scaled = t.scale != std::array{1, 1, 1}; + if (bone.scaled) std::copy(t.scale.begin(), t.scale.end(), bone.scale); + return bone; +} +inline Transform restore(const QuantBone &bone) noexcept { + Transform t; + const float step = static_cast(1U << rotation_shift) / rotation_scale; + float sum{}; + unsigned in{}; + for (unsigned i = 0; i < 4; ++i) { + if (i == bone.largest) continue; + t.rotation[i] = static_cast(bone.rotation[in++]) * step; + sum += t.rotation[i] * t.rotation[i]; + } + t.rotation[bone.largest & 3] = std::sqrt(std::max(0.f, 1.f - sum)); + for (unsigned i = 0; i < 3; ++i) t.position[i] = static_cast(static_cast(bone.position[i]) / 1000.0); + if (bone.scaled) std::copy(bone.scale, bone.scale + 3, t.scale.begin()); + return t; +} +inline QuantPose quantize(const Pose &pose) { + QuantPose out; + out.skater = static_cast(pose.skater.size()); + out.board = static_cast(pose.board.size()); + out.bones.reserve(1 + pose.skater.size() + pose.board.size()); + out.bones.push_back(quantize(pose.root)); + for (const auto &t : pose.skater) out.bones.push_back(quantize(t)); + for (const auto &t : pose.board) out.bones.push_back(quantize(t)); + return out; +} +inline Pose restore(const QuantPose &pose) { + Pose out; + if (pose.bones.size() != std::size_t{1} + pose.skater + pose.board) return out; + out.root = restore(pose.bones[0]); + out.skater.reserve(pose.skater); + out.board.reserve(pose.board); + for (std::size_t i = 0; i < pose.skater; ++i) out.skater.push_back(restore(pose.bones[1 + i])); + for (std::size_t i = 0; i < pose.board; ++i) out.board.push_back(restore(pose.bones[1 + pose.skater + i])); + return out; +} + +namespace detail { +struct BitWriter { + std::vector bytes; + std::uint64_t hold{}; + unsigned held{}; + void put(std::uint64_t value, unsigned bits) { // the low `bits` of value, at most 32 + hold |= (value & ((std::uint64_t{1} << bits) - 1)) << held; + held += bits; + while (held >= 8) { + bytes.push_back(static_cast(hold)); + hold >>= 8; + held -= 8; + } + } + // A count of 1 or more, shorter the smaller it is (Elias gamma): bones that changed come in + // runs, and the step from one to the next is nearly always 1, which is a single bit. + void gap(std::uint32_t value) { + const auto width = static_cast(std::bit_width(value)); + put(0, width - 1); + put(1, 1); + put(value, width - 1); + } + std::vector finish() { + if (held) bytes.push_back(static_cast(hold)); + hold = held = 0; + return std::move(bytes); + } +}; +struct BitReader { + std::span bytes; + std::size_t at{}; + std::uint64_t hold{}; + unsigned held{}; + std::uint64_t get(unsigned bits) { + while (held < bits) { + if (at >= bytes.size()) throw std::runtime_error("Truncated pose"); + hold |= std::uint64_t{bytes[at++]} << held; + held += 8; + } + const auto value = hold & ((std::uint64_t{1} << bits) - 1); + hold >>= bits; + held -= bits; + return value; + } + std::uint32_t gap() { + unsigned width{1}; + while (!get(1)) + if (++width > 32) throw std::runtime_error("Invalid pose gap"); + return static_cast((std::uint64_t{1} << (width - 1)) | get(width - 1)); + } +}; +constexpr std::uint32_t zigzag(std::int64_t value) noexcept { + return static_cast(value >= 0 ? static_cast(value) * 2 : static_cast(-value) * 2 - 1); +} +constexpr std::int64_t unzigzag(std::uint64_t value) noexcept { + return (value & 1) ? -static_cast((value + 1) / 2) : static_cast(value / 2); +} +// A signed number as its length (a fixed few bits) and then that many bits. +inline void small(BitWriter &w, std::int64_t value, unsigned length_bits) { + const auto coded = zigzag(value); + const auto width = static_cast(std::bit_width(coded)); + w.put(width, length_bits); + if (width > 32 - 1) { + w.put(coded & 0xffff, 16); + w.put(coded >> 16, width - 16); + } else { + w.put(coded, width); + } +} +inline std::int64_t small(BitReader &r, unsigned length_bits) { + const auto width = static_cast(r.get(length_bits)); + if (width > 32) throw std::runtime_error("Invalid pose number"); + if (width > 31) { + const auto low = r.get(16); + return unzigzag(low | (r.get(width - 16) << 16)); + } + return unzigzag(r.get(width)); +} +inline void whole_rotation(BitWriter &w, const QuantBone &bone) { + w.put(bone.largest, 2); + for (const auto component : bone.rotation) w.put(zigzag(component), rotation_bits); +} +inline void whole_rotation(BitReader &r, QuantBone &bone) { + bone.largest = static_cast(r.get(2)); + for (auto &component : bone.rotation) component = static_cast(unzigzag(r.get(rotation_bits))); +} +inline void scale(BitWriter &w, const QuantBone &bone) { + w.put(bone.scaled, 1); + if (bone.scaled) + for (const float axis : bone.scale) { + const auto bits = std::bit_cast(axis); + w.put(bits & 0xffff, 16); + w.put(bits >> 16, 16); + } +} +inline void scale(BitReader &r, QuantBone &bone) { + bone.scaled = r.get(1) != 0; + for (float &axis : bone.scale) { + if (!bone.scaled) { + axis = 1; + continue; + } + const auto low = static_cast(r.get(16)); + axis = std::bit_cast(low | (static_cast(r.get(16)) << 16)); + if (!std::isfinite(axis) || std::abs(axis) > 1000.f) throw std::runtime_error("Invalid pose scale"); + } +} +inline bool same_rotation(const QuantBone &a, const QuantBone &b) noexcept { + return a.largest == b.largest && std::equal(a.rotation, a.rotation + 3, b.rotation); +} +inline bool same_position(const QuantBone &a, const QuantBone &b) noexcept { return std::equal(a.position, a.position + 3, b.position); } +inline bool same_scale(const QuantBone &a, const QuantBone &b) noexcept { + return a.scaled == b.scaled && (!a.scaled || std::equal(a.scale, a.scale + 3, b.scale)); +} +} // namespace detail + +// How a pose's fingers are sent (finger_bone). `held`: not at all, the receiver keeps the ones it +// has (a player too far away to make out a hand). `whole`: every finger bone in full, whatever +// the reference holds: the first poses after `held`, until the receiver is known to have them, +// since its copy of the reference has old fingers in it. +enum class Fingers : std::uint8_t { sent, held, whole }; + +// The whole pose, needing no reference: the first of a stream, and whenever the receiver holds +// nothing recent enough to build on. +inline std::vector encode_whole(const QuantPose &pose) { + detail::BitWriter w; + w.put(1, 1); + w.put(rotation_shift, 3); + w.put(pose.skater, 10); + w.put(pose.board, 7); + for (const auto &bone : pose.bones) { + detail::whole_rotation(w, bone); + for (const auto mm : bone.position) detail::small(w, mm, 6); + detail::scale(w, bone); + } + return w.finish(); +} +// The pose as its differences from `reference`, which the receiver must hold exactly. Empty when +// the two are not the same skeleton: send it whole. +inline std::vector encode_delta(const QuantPose &pose, const QuantPose &reference, Fingers fingers = Fingers::sent) { + if (pose.skater != reference.skater || pose.board != reference.board || pose.bones.size() != reference.bones.size()) return {}; + detail::BitWriter w; + w.put(0, 1); + w.put(rotation_shift, 3); + w.put(static_cast(fingers), 2); + const auto count = pose.bones.size(); + const auto listed = [&](std::size_t bone) { return fingers == Fingers::sent || !finger_bone(pose, bone); }; + // Three lists of the bones that changed, each as how many and then the steps between them. + const auto list = [&](auto &&differs, auto &&write) { + std::uint32_t changed{}; + for (std::size_t bone = 0; bone < count; ++bone) changed += listed(bone) && differs(pose.bones[bone], reference.bones[bone]); + w.put(changed, 10); + std::size_t last{}; + for (std::size_t bone = 0; bone < count; ++bone) { + if (!listed(bone) || !differs(pose.bones[bone], reference.bones[bone])) continue; + w.gap(static_cast(bone + 1 - last)); + last = bone + 1; + write(pose.bones[bone], reference.bones[bone]); + } + }; + list([](const QuantBone &a, const QuantBone &b) { return !detail::same_rotation(a, b); }, [&](const QuantBone &now, const QuantBone &was) { + // A different component left out: the three sent are other ones, so they go whole. + const bool flipped = now.largest != was.largest; + w.put(flipped, 1); + if (flipped) return detail::whole_rotation(w, now); + for (unsigned i = 0; i < 3; ++i) detail::small(w, std::int64_t{now.rotation[i]} - was.rotation[i], 4); + }); + list([](const QuantBone &a, const QuantBone &b) { return !detail::same_position(a, b); }, [&](const QuantBone &now, const QuantBone &was) { + for (unsigned i = 0; i < 3; ++i) detail::small(w, std::int64_t{now.position[i]} - was.position[i], 6); + }); + list([](const QuantBone &a, const QuantBone &b) { return !detail::same_scale(a, b); }, + [&](const QuantBone &now, const QuantBone &) { detail::scale(w, now); }); + if (fingers == Fingers::whole) + for (std::size_t bone = 0; bone < count; ++bone) { + if (!finger_bone(pose, bone)) continue; + detail::whole_rotation(w, pose.bones[bone]); + for (const auto mm : pose.bones[bone].position) detail::small(w, mm, 6); + detail::scale(w, pose.bones[bone]); + } + return w.finish(); +} +inline bool whole(std::span bytes) noexcept { return !bytes.empty() && (bytes[0] & 1); } +// The pose `bytes` holds. `reference`: the pose its differences are from, as this end holds it +// (ignored by a whole pose). Nothing when the bytes are not a pose, or need a reference of +// another skeleton. +inline std::optional decode(std::span bytes, const QuantPose *reference) noexcept { + try { + detail::BitReader r{bytes}; + const bool is_whole = r.get(1) != 0; + if (r.get(3) != rotation_shift) return {}; + const auto valid = [](QuantBone &bone) { + if (bone.largest > 3) throw std::runtime_error("Invalid pose rotation"); + const long most = 32767 >> rotation_shift; + for (const auto component : bone.rotation) + if (component > most || component < -most) throw std::runtime_error("Invalid pose rotation"); + for (const auto mm : bone.position) + if (mm > position_limit || mm < -position_limit) throw std::runtime_error("Invalid pose position"); + }; + QuantPose pose; + if (is_whole) { + pose.skater = static_cast(r.get(10)); + pose.board = static_cast(r.get(7)); + if (pose.skater > max_skater_bones || pose.board > max_board_bones) return {}; + pose.bones.resize(std::size_t{1} + pose.skater + pose.board); + for (auto &bone : pose.bones) { + detail::whole_rotation(r, bone); + for (auto &mm : bone.position) mm = static_cast(std::clamp(detail::small(r, 6), INT32_MIN, INT32_MAX)); + detail::scale(r, bone); + valid(bone); + } + return pose; + } + if (!reference || reference->bones.size() != std::size_t{1} + reference->skater + reference->board) return {}; + pose = *reference; + const auto fingers = static_cast(r.get(2)); + if (fingers > Fingers::whole) return {}; + const auto count = pose.bones.size(); + const auto list = [&](auto &&read) { + const auto changed = r.get(10); + std::size_t last{}; + for (std::uint64_t i = 0; i < changed; ++i) { + const auto bone = last + r.gap() - 1; + if (bone >= count) throw std::runtime_error("Invalid pose bone"); + last = bone + 1; + read(pose.bones[bone]); + valid(pose.bones[bone]); + } + }; + list([&](QuantBone &bone) { + if (r.get(1)) return detail::whole_rotation(r, bone); + for (auto &component : bone.rotation) + component = static_cast(std::clamp(component + detail::small(r, 4), INT16_MIN, INT16_MAX)); + }); + list([&](QuantBone &bone) { + for (auto &mm : bone.position) mm = static_cast(std::clamp(mm + detail::small(r, 6), INT32_MIN, INT32_MAX)); + }); + list([&](QuantBone &bone) { detail::scale(r, bone); }); + if (fingers == Fingers::whole) + for (std::size_t bone = 0; bone < count; ++bone) { + if (!finger_bone(pose, bone)) continue; + detail::whole_rotation(r, pose.bones[bone]); + for (auto &mm : pose.bones[bone].position) mm = static_cast(std::clamp(detail::small(r, 6), INT32_MIN, INT32_MAX)); + detail::scale(r, pose.bones[bone]); + valid(pose.bones[bone]); + } + return pose; + } catch (...) { + return {}; + } +} +} // namespace dingosdk::multiplayer::pose_codec diff --git a/Extension/Multiplayer/Net/protocol.cpp b/Extension/Multiplayer/Net/protocol.cpp index fc8eada..f897297 100644 --- a/Extension/Multiplayer/Net/protocol.cpp +++ b/Extension/Multiplayer/Net/protocol.cpp @@ -9,6 +9,7 @@ namespace dingosdk::multiplayer { namespace { constexpr std::uint32_t magic = 0x31504d52; // RMP1, little endian constexpr std::size_t header_size = packet_header_size; +std::size_t utf8_length(std::string_view text, std::size_t at) noexcept; // defined with the text checks below std::uint8_t encode_park(unsigned lot, std::string_view choice) { if (choice.empty() || choice == "empty") return 0; for (unsigned family = 0; family < park_families.size(); ++family) @@ -180,9 +181,14 @@ bool valid_roster(std::span members, unsigned capacity) noexcept { // A server is in no party; only a party's leader leads or opens it. if ((server && m.party) || (!m.party && (m.party_leader || m.party_open)) || (m.party_open && !m.party_leader)) return false; - for (unsigned char c : m.name) - if (c < 32 || c == 127) + // Names reach the game's own UI: whole UTF-8 characters only, and no controls. + for (std::size_t at = 0; at < m.name.size();) { + const auto length = utf8_length(m.name, at); + const auto c = static_cast(m.name[at]); + if (!length || c < 32 || c == 127) return false; + at += length; + } unsigned leaders = m.party_leader ? 1U : 0U, size = 1; for (std::size_t j = 0; j < members.size(); ++j) { if (j < i && members[j].id == m.id) @@ -231,7 +237,7 @@ bool valid_appearance(const Appearance &a) noexcept { if (a.skater.key != skater_recipe_key || a.skater.version != 2 || a.board.key != board_recipe_key || a.board.version != 1) return false; - std::size_t size = header_size + 12; + std::size_t size = header_size + 13 + mark_items * 8; for (const auto *r : {&a.skater, &a.board}) { if (r->scalars.size() > max_cosmetic_scalars || r->items.empty() || r->items.size() > max_cosmetic_slots) @@ -251,7 +257,7 @@ bool valid_appearance(const Appearance &a) noexcept { size += 8 + item.asset.size() + item.parameters.size() * 4; } } - return size <= max_packet; + return size <= max_appearance_bytes; } bool valid_transform(const Transform &t) noexcept { for (float v : t.position) @@ -309,6 +315,46 @@ std::string_view greeting_error(const Packet &p, std::uint64_t session, std::uin return "Peer changed level or session. Join again."; return {}; } +namespace { +void coarsen(Transform &t, unsigned bits) noexcept { + if (!bits) return; + unsigned largest{}; + float norm{}; + for (unsigned i = 0; i < 4; ++i) { + norm += t.rotation[i] * t.rotation[i]; + if (std::abs(t.rotation[i]) > std::abs(t.rotation[largest])) largest = i; + } + if (!(norm > 0.f) || !std::isfinite(norm)) return; + // As Writer::compact_transform packs it: the three smaller components, the largest positive. + constexpr float scale = 46339.5358f; + const float factor = (t.rotation[largest] < 0 ? -1.f : 1.f) * scale / std::sqrt(norm); + const float step = static_cast(1U << bits); + const float most = std::floor(32767.f / step) * step; // a rounded value must still fit 16 bits + float sum{}; + for (unsigned i = 0; i < 4; ++i) { + if (i == largest) continue; + const float packed = std::clamp(std::round(t.rotation[i] * factor / step) * step, -most, most); + t.rotation[i] = packed / scale; + sum += t.rotation[i] * t.rotation[i]; + } + t.rotation[largest] = std::sqrt(std::max(0.f, 1.f - sum)); +} +} // namespace +void limit_bone_scale(Pose &pose, float limit) noexcept { + if (!(limit >= 1.f)) return; + const auto hold = [&](Transform &t) { + for (float &axis : t.scale) axis = std::isfinite(axis) ? std::clamp(axis, 1.f / limit, limit) : 1.f; + }; + hold(pose.root); + for (auto &t : pose.skater) hold(t); + for (auto &t : pose.board) hold(t); +} +void coarsen_rotations(Pose &pose, unsigned bits) noexcept { + bits = std::min(bits, 12U); + coarsen(pose.root, std::min(bits, 6U)); // which way they face matters from further off than a finger does + for (auto &t : pose.skater) coarsen(t, bits); + for (auto &t : pose.board) coarsen(t, bits); +} std::vector encode(const Packet &p, bool compact_pose) { return encode(p, compact_pose, p.pose_interval_us); } @@ -334,6 +380,8 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 throw std::invalid_argument("Invalid throwdown message"); if (p.kind == PacketKind::physics_tuning && (!p.source || p.tuning.size() > max_physics_tuning)) throw std::invalid_argument("Invalid physics tuning"); + if (p.kind == PacketKind::physics_extras && (!p.source || p.extras.size() > max_physics_extras)) + throw std::invalid_argument("Invalid physics extras"); if (p.kind == PacketKind::party && (!p.source || !valid_party_request(p.party_action, p.party_player))) throw std::invalid_argument("Invalid party message"); if (p.kind == PacketKind::scoring && (!p.source || (!p.text.empty() && !valid_admin_text(p.text)))) @@ -348,7 +396,8 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 }))) throw std::invalid_argument("Invalid ban list"); if (p.kind == PacketKind::maps && - (!p.source || p.maps.size() > max_server_maps || + (!p.source || p.maps.size() > max_server_maps || !valid_map_pool(p.map_pool, p.maps.size()) || + p.map_rotation > max_map_rotation || std::any_of(p.maps.begin(), p.maps.end(), [](const auto &asset) { return !valid_map_asset(asset); }))) throw std::invalid_argument("Invalid server map list"); if (p.kind == PacketKind::hello && !p.text.empty() && !valid_member_name(p.text)) @@ -358,6 +407,8 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 if (p.kind == PacketKind::map_offer && (!valid_map_destination(p.destination) || p.map != map_hash(p.destination))) throw std::invalid_argument("Invalid host map destination"); + if ((p.kind == PacketKind::map_offer || p.kind == PacketKind::world_state) && !valid_map_label(p.map_label)) + throw std::invalid_argument("Invalid map name"); if (p.kind == PacketKind::world_state && (p.destination.empty() ? (p.map != 0 || p.world_ready) : (!valid_map_destination(p.destination) || p.map != map_hash(p.destination)))) @@ -373,7 +424,7 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 p.kind != PacketKind::objects && p.kind != PacketKind::voice && p.kind != PacketKind::chat && p.kind != PacketKind::admin && p.kind != PacketKind::bans && p.kind != PacketKind::maps && p.kind != PacketKind::throwdown && p.kind != PacketKind::teleport && p.kind != PacketKind::physics_tuning && - p.kind != PacketKind::party && p.kind != PacketKind::scoring) + p.kind != PacketKind::party && p.kind != PacketKind::scoring && p.kind != PacketKind::physics_extras) throw std::invalid_argument("Unknown packet kind"); const auto payload = greeting ? 72 : p.kind == PacketKind::away @@ -400,6 +451,8 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(p.map_authorized ? 1 : 0, 1); w.integer(p.destination.size(), 2); w.bytes.insert(w.bytes.end(), p.destination.begin(), p.destination.end()); + w.integer(p.map_label.size(), 1); + w.bytes.insert(w.bytes.end(), p.map_label.begin(), p.map_label.end()); } if (p.kind == PacketKind::hello) { w.integer(p.text.size(), 1); @@ -410,6 +463,8 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(p.world_ready ? 1 : 0, 1); w.integer(p.destination.size(), 2); w.bytes.insert(w.bytes.end(), p.destination.begin(), p.destination.end()); + w.integer(p.map_label.size(), 1); + w.bytes.insert(w.bytes.end(), p.map_label.begin(), p.map_label.end()); } else if (p.kind == PacketKind::world_ready) { w.integer(p.world_ready ? 1 : 0, 1); } else if (p.kind == PacketKind::pose) { @@ -437,6 +492,13 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(p.appearance.card.background, 4); w.integer(p.appearance.card.emblem, 4); w.integer(p.appearance.card.title, 4); + w.integer((p.appearance.hide_tag ? 1 : 0) | (p.appearance.hide_items ? 2 : 0), 1); + for (const auto &style : p.appearance.marks) { + w.integer(static_cast(style.mode), 1); + for (const auto part : style.from) w.integer(part, 1); + for (const auto part : style.to) w.integer(part, 1); + w.integer(style.speed, 1); + } } else if (p.kind == PacketKind::audio) { w.integer(p.audio.size(), 2); AudioState previous; @@ -485,6 +547,10 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(p.tuning.size(), 2); w.bytes.insert(w.bytes.end(), p.tuning.begin(), p.tuning.end()); } + if (p.kind == PacketKind::physics_extras) { + w.integer(p.extras.size(), 2); + w.bytes.insert(w.bytes.end(), p.extras.begin(), p.extras.end()); + } if (p.kind == PacketKind::scoring) { w.integer(p.scoring, 8); w.integer(p.text.size(), 2); @@ -496,6 +562,9 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(asset.size(), 1); w.bytes.insert(w.bytes.end(), asset.begin(), asset.end()); } + w.integer(p.map_pool.size(), 2); + for (const auto index : p.map_pool) w.integer(index, 2); + w.integer(p.map_rotation, 2); } if (p.kind == PacketKind::bans) { w.integer(p.ban_total, 4); @@ -528,7 +597,11 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.integer(p.object_clears, 4); if (!valid_multiplayer_tps(p.tps)) throw std::invalid_argument("Invalid session TPS"); w.integer(p.tps, 1); + w.integer(p.chat_badge & 0xffffff, 3); // red, green, blue + w.integer(p.chat_text & 0xffffff, 3); w.integer(static_cast(p.object_placement), 1); + if (!valid_object_limit(p.object_limit)) throw std::invalid_argument("Invalid object limit"); + w.integer(p.object_limit, 2); w.integer(p.force_world_layers, 1); // One mode per world-layer catalog row; both peers read the same catalog // from the same game build. @@ -573,6 +646,13 @@ std::vector encode(const Packet &p, bool compact_pose, std::uint32 w.bytes[8 + i] = static_cast(actual_payload >> (8 * i)); return std::move(w.bytes); } +static bool read_map_label(Reader &r, std::span bytes, Packet &p) { + const auto length = r.integer(1); + if (length > max_member_name || length > bytes.size() - r.at) return false; + p.map_label.assign(reinterpret_cast(bytes.data() + r.at), static_cast(length)); + r.at += static_cast(length); + return valid_map_label(p.map_label); +} std::optional decode(std::span bytes) noexcept { try { if (bytes.size() < header_size || bytes.size() > max_packet) @@ -611,7 +691,7 @@ std::optional decode(std::span bytes) noexcept { p.destination.assign(reinterpret_cast(bytes.data() + r.at), static_cast(length)); r.at += static_cast(length); - if (!valid_map_destination(p.destination) || p.map != map_hash(p.destination)) + if (!valid_map_destination(p.destination) || p.map != map_hash(p.destination) || !read_map_label(r, bytes, p)) return {}; } if (p.kind == PacketKind::hello) { @@ -636,6 +716,7 @@ std::optional decode(std::span bytes) noexcept { if (p.destination.empty() ? (p.map != 0 || p.world_ready) : (!valid_map_destination(p.destination) || p.map != map_hash(p.destination))) return {}; + if (!read_map_label(r, bytes, p)) return {}; } } else if (p.kind == PacketKind::pose) { const auto skater = r.integer(2), board = r.integer(2); @@ -662,7 +743,18 @@ std::optional decode(std::span bytes) noexcept { p.appearance.card.background = static_cast(r.integer(4)); p.appearance.card.emblem = static_cast(r.integer(4)); p.appearance.card.title = static_cast(r.integer(4)); - if (!p.map || !valid_appearance(p.appearance)) + const auto flags = r.integer(1); + p.appearance.hide_tag = (flags & 1) != 0; + p.appearance.hide_items = (flags & 2) != 0; + bool styled = true; + for (auto &style : p.appearance.marks) { + style.mode = static_cast(r.integer(1)); + for (auto &part : style.from) part = static_cast(r.integer(1)); + for (auto &part : style.to) part = static_cast(r.integer(1)); + style.speed = static_cast(r.integer(1)); + styled = styled && valid_mark_style(style); + } + if (flags > 3 || !styled || !p.map || !valid_appearance(p.appearance)) return {}; } else if (p.kind == PacketKind::audio) { const auto count = r.integer(2); @@ -717,6 +809,12 @@ std::optional decode(std::span bytes) noexcept { return {}; p.tuning.assign(bytes.begin() + static_cast(r.at), bytes.end()); r.at = bytes.size(); + } else if (p.kind == PacketKind::physics_extras) { + const auto length = r.integer(2); + if (!p.source || length > max_physics_extras || length != bytes.size() - r.at) + return {}; + p.extras.assign(bytes.begin() + static_cast(r.at), bytes.end()); + r.at = bytes.size(); } else if (p.kind == PacketKind::scoring) { p.scoring = r.integer(8); const auto length = r.integer(2); @@ -753,6 +851,11 @@ std::optional decode(std::span bytes) noexcept { if (!valid_map_asset(asset)) return {}; p.maps.push_back(std::move(asset)); } + const auto pooled = r.integer(2); + if (pooled > p.maps.size()) return {}; + for (std::uint64_t i = 0; i < pooled; ++i) p.map_pool.push_back(static_cast(r.integer(2))); + p.map_rotation = static_cast(r.integer(2)); + if (!valid_map_pool(p.map_pool, p.maps.size()) || p.map_rotation > max_map_rotation) return {}; } else if (p.kind == PacketKind::bans) { p.ban_total = static_cast(r.integer(4)); const auto count = r.integer(2); @@ -802,9 +905,14 @@ std::optional decode(std::span bytes) noexcept { p.object_clears = static_cast(r.integer(4)); p.tps = static_cast(r.integer(1)); if (!valid_multiplayer_tps(p.tps)) return {}; + p.chat_badge = 0xff000000U | static_cast(r.integer(3)); + p.chat_text = 0xff000000U | static_cast(r.integer(3)); const auto placement = r.integer(1); if (!valid_object_placement(placement)) return {}; p.object_placement = static_cast(placement); + const auto object_limit = r.integer(2); + if (!valid_object_limit(object_limit)) return {}; + p.object_limit = static_cast(object_limit); const auto forced = r.integer(1); if (forced > 1) return {}; p.force_world_layers = forced != 0; @@ -922,6 +1030,15 @@ bool valid_map_asset(std::string_view asset) noexcept { return !asset.empty() && asset.size() <= max_map_asset && std::all_of(asset.begin(), asset.end(), [](char c) { return c > 32 && c < 127 && c != '|'; }); } +bool valid_map_label(std::string_view label) noexcept { + return label.empty() || (label.size() <= max_member_name && valid_member_name(label)); +} +bool valid_map_pool(std::span pool, std::size_t maps) noexcept { + if (pool.size() > maps) return false; + for (std::size_t i = 0; i < pool.size(); ++i) + if (pool[i] >= maps || std::find(pool.begin(), pool.begin() + i, pool[i]) != pool.begin() + i) return false; + return true; +} bool valid_admin_text(std::string_view text) noexcept { if (text.empty() || text.size() > max_admin_text) return false; for (std::size_t at = 0; at < text.size();) { @@ -950,6 +1067,15 @@ std::string clean_chat_text(std::string_view text) { if (first == std::string::npos) return {}; return result.substr(first, result.find_last_not_of(' ') - first + 1); } +std::string clean_roster_name(std::string_view text) { + auto name = clean_chat_text(text); + if (name.size() > 128) { + std::size_t cut = 128; + while (cut && (static_cast(name[cut]) & 0xC0) == 0x80) --cut; + name.resize(cut); + } + return name; +} bool newer_sequence(std::uint32_t a, std::uint32_t b) noexcept { const auto d = a - b; return d && d < 0x80000000u; diff --git a/Extension/Multiplayer/Net/protocol.h b/Extension/Multiplayer/Net/protocol.h index 5973f57..7410fbd 100644 --- a/Extension/Multiplayer/Net/protocol.h +++ b/Extension/Multiplayer/Net/protocol.h @@ -7,6 +7,7 @@ #include "Engine/Game/Multiplayer/distance_settings.h" #include "Engine/Game/Multiplayer/object_placement.h" #include "Engine/Game/Multiplayer/session_model.h" +#include "Engine/Game/Multiplayer/session_physics.h" #include "Engine/Game/Multiplayer/tick_settings.h" #include "Engine/Game/World/park_rotation.h" #include "Engine/Game/World/world_layers.h" @@ -24,7 +25,9 @@ namespace dingosdk::multiplayer { constexpr std::size_t max_skater_bones = 512, max_board_bones = 64; constexpr std::size_t max_packet = 24576; constexpr std::size_t packet_header_size = 64; -constexpr std::uint16_t protocol_version = 38; +constexpr std::uint16_t protocol_version = 45; +// A dedicated server's chat lines unless its owner says otherwise: violet (#8E5CFF) and lavender (#D9C8FF). +inline constexpr std::uint32_t default_server_chat_badge = 0xffff5c8eU, default_server_chat_text = 0xffffc8d9U; constexpr std::size_t max_throwdown_message = 4096; // Packet::tuning: the host's SkatePhysicsTuning differences (Extension/Skater/physics_tuning.h). constexpr std::size_t max_physics_tuning = 16384; @@ -64,16 +67,20 @@ enum class PacketKind : std::uint16_t { // The host's skate physics tuning, as differences from the game's own (opaque here; // empty = the game's). Only the host sends it; a dedicated server never needs to. physics_tuning = 27, - // A party request to a dedicated server, or its notice to one player (PartyAction). + // A party request to whoever hosts, or its notice to one player (PartyAction). party = 28, // A player telling the host or dedicated server how its mods change trick scoring // (Engine/Vfs/mod_scoring.h): sent once known, again whenever it changes. - scoring = 29 + scoring = 29, + // The host's physics that its tuning does not carry: the trainer's tuning-class values and + // trick multipliers (Engine/Game/Multiplayer/session_physics.h; opaque here, empty = the + // game's own). Like physics_tuning, only the host sends it and a dedicated server never does. + physics_extras = 30 }; -// Packet::party_action. Requests go from a player to the dedicated server (party_player = -// the other player involved, 0 for leave/open/close); invited and withdrawn go from the -// server to the invitee (party_player = the inviter). The server answers everything else -// with a chat line and the next roster. +// Packet::party_action. Requests go from a player to whoever hosts, a dedicated server or a +// lobby's host (party_player = the other player involved, 0 for leave/open/close); invited +// and withdrawn go from the host to the invitee (party_player = the inviter). The host +// answers everything else with a chat line and the next roster. enum class PartyAction : std::uint8_t { invite = 1, // invite party_player into the sender's party accept = 2, // accept party_player's invite @@ -84,13 +91,13 @@ enum class PartyAction : std::uint8_t { promote = 7, // the leader hands the lead to party_player open = 8, // the leader lets anyone join close = 9, // the leader makes the party invite-only - invited = 10, // server -> invitee: party_player invited you - withdrawn = 11 // server -> invitee: party_player's invite can no longer be accepted + invited = 10, // host -> invitee: party_player invited you + withdrawn = 11 // host -> invitee: party_player's invite can no longer be accepted }; bool valid_party_request(PartyAction action, std::uint64_t player) noexcept; // Steam accounts in the public universe. Players are individual accounts; a -// dedicated server signs in anonymously as a game server (type 3 or 4) and gets -// a new ID each time it starts. +// dedicated server is a game server: anonymous (type 4) with a new ID each time +// it starts, or signed in with a login token (type 3) with the same ID always. inline bool individual_steam_id(std::uint64_t id) noexcept { return (id >> 56) == 1 && ((id >> 52) & 15) == 1 && (id & 0xffffffffULL); } @@ -98,6 +105,9 @@ inline bool game_server_steam_id(std::uint64_t id) noexcept { const auto type = (id >> 52) & 15; return (id >> 56) == 1 && (type == 3 || type == 4) && (id & 0xffffffffULL); } +inline bool persistent_server_steam_id(std::uint64_t id) noexcept { + return game_server_steam_id(id) && ((id >> 52) & 15) == 3; +} // A player's own name as sent in their hello: at most this many bytes. constexpr std::size_t max_member_name = 64; constexpr std::size_t max_admin_text = 320; @@ -105,6 +115,8 @@ constexpr std::size_t max_ban_rows = 256; constexpr std::size_t max_server_maps = 128, max_map_asset = 128; // A level asset as a server's map list carries it: printable ASCII, no '|'. bool valid_map_asset(std::string_view asset) noexcept; +bool valid_map_pool(std::span pool, std::size_t maps) noexcept; // distinct indices below `maps` +bool valid_map_label(std::string_view label) noexcept; // empty, or a name like a member's struct Transform { std::array position{}; std::array rotation{0, 0, 0, 1}; @@ -122,7 +134,7 @@ struct Member { std::string name; bool admin{}; // roster: may change a dedicated server's settings // roster: the player's party (0 = none), whether they lead it, and (on the leader) whether - // anyone may join it. Each party has exactly one leader. A listen host's lobby is one party. + // anyone may join it. Each party has exactly one leader. std::uint32_t party{}; bool party_leader{}, party_open{}; // roster: a dedicated server measured the player's game running fast (a speedhack): nobody @@ -161,9 +173,14 @@ struct Packet { // lock their editor and native tools from it; the host enforces it by // freezing guest layouts (see publish_guest_objects). ObjectPlacement object_placement = ObjectPlacement::everyone; + // Objects each player may have placed (object_placement.h); 0: no limit. + unsigned object_limit{}; // Bumped each time the host deletes all guest objects. Guests delete their // own session objects when it changes after their first roster. std::uint32_t object_clears{}; + // Roster: the colours of a dedicated server's own chat lines, its badge and name and the + // text after them (IM_COL32 layout; the server's owner chooses them). + std::uint32_t chat_badge = default_server_chat_badge, chat_text = default_server_chat_text; bool force_world_layers{}; WorldLayerState layers = WorldLayerState(world_layers().size()); // all "default" ParkChoices parks; // One allowlisted selection byte per shared native park slot. @@ -182,9 +199,13 @@ struct Packet { std::vector bans; // bans: newest first std::uint32_t ban_total{}; // bans: how many the server has in all std::vector maps; // maps: level assets + std::vector map_pool; // maps: the pool as indices into `maps`, rotation order (empty: every map) + std::uint16_t map_rotation{}; // maps: minutes per map (0: off) + std::string map_label; // map_offer, world_state: the map's name for people (may be empty) std::vector throwdown; // throwdown: one encoded message (1..max_throwdown_message bytes) std::array teleport{}; // teleport: where the receiver goes (world position) std::vector tuning; // physics_tuning: 0..max_physics_tuning bytes + std::vector extras; // physics_extras: 0..max_physics_extras bytes PartyAction party_action = PartyAction::leave; // party: what is asked or told std::uint64_t party_player{}; // party: the other player (see PartyAction) // scoring: the sender's scoring fingerprint, 0 for the game's own; `text` names the mods @@ -205,8 +226,21 @@ bool valid_admin_text(std::string_view) noexcept; // The message a player typed, made valid: control characters and broken UTF-8 // dropped, surrounding blanks trimmed, cut to the byte limit on a character boundary. std::string clean_chat_text(std::string_view); +// A player's name as a roster carries it (valid_roster): the same cleaning, at most 128 bytes. +std::string clean_roster_name(std::string_view); bool valid_pose(const Pose &) noexcept; std::vector encode(const Packet &, bool compact_pose = false); +// Rounds every rotation in a pose to a multiple of 2^bits in the 16-bit form a compact pose +// packs it in (the root to at most 2^6). The pose still encodes and decodes as any other; +// what changes is that a bone turning by less than a step is the same bytes as before, so a +// difference from a reference leaves it out, and the low bits of the ones that did change are +// zero and pack away. One step is about 0.0025 degrees times 2^bits: 4 bits is 0.04 degrees, +// 7 bits a third of a degree. +void coarsen_rotations(Pose &pose, unsigned bits) noexcept; +// Keeps every bone's scale within 1/limit to limit on each axis (1: no scaling at all). A mod +// that resizes part of a skater (a head four times the size) does it with a bone's scale, +// which travels in the pose and so shows to everyone, mod or not. +void limit_bone_scale(Pose &pose, float limit) noexcept; // The same, with a pose encoded at another update interval (a recipient thinned by // distance) instead of the packet's own, so the packet need not be copied for it. std::vector encode(const Packet &, bool compact_pose, std::uint32_t pose_interval_us); @@ -304,6 +338,8 @@ class PoseBuffer { bool sample(std::uint64_t now_us, Pose &out) const; std::optional sample(std::uint64_t now_us) const; bool sample_remote(std::uint64_t now_us, Pose &out); + // Whether a pose arrived within this long: sampling gives up after a second without one. + bool heard_within(std::uint64_t now_us, std::uint64_t age_us) const; std::optional sample_remote(std::uint64_t now_us); PosePlayback playback() const { return playback_; } void clear(); diff --git a/Extension/Multiplayer/Net/sound_codec.h b/Extension/Multiplayer/Net/sound_codec.h new file mode 100644 index 0000000..68c69ef --- /dev/null +++ b/Extension/Multiplayer/Net/sound_codec.h @@ -0,0 +1,273 @@ +#pragma once +#include "Extension/Multiplayer/Remote/audio_state.h" +#include "pose_batch.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Skaters' sound between a dedicated server and its players: several players' samples in one +// message, each sample as what changed since the one before it. +// +// A sample is 58 numbers, 26 selectors and 43 switches, of which about twenty numbers change from +// one to the next and the rest seldom. Numbers are rounded to 1/1024 and sent as how far they +// moved; a switch or selector is sent only when one changed. Messages go reliably and in order +// (many samples are a pulse one sample long, which must not be lost), so each builds on the last +// one sent: the sender keeps what the receiver holds of each stream. A stream is sent whole when +// it starts and again every couple of seconds, which also puts right a receiver that missed its +// start. +namespace dingosdk::multiplayer::sound_codec { +inline constexpr std::array magic{'R', 'M', 'S', '1'}; +inline constexpr std::size_t header_size = 12; +inline constexpr std::int64_t value_scale = 1024, value_limit = 1000000LL * value_scale; +// Ages are sent in steps of 256 us. +inline constexpr unsigned age_shift = 8; +inline constexpr std::uint32_t age_limit = 1000000U >> age_shift; +inline constexpr std::uint64_t whole_every_us = 2000000; + +inline bool is_sound(std::span bytes) noexcept { + return bytes.size() >= header_size && std::equal(magic.begin(), magic.end(), bytes.begin()); +} + +struct QuantState { + std::array values{}; + std::array selectors{}; + std::uint64_t flags{}; // a bit each + bool operator==(const QuantState &) const = default; +}; +// A number that is not nothing never rounds to nothing: the game may ask only whether it is. +inline QuantState quantize(const AudioState &state) noexcept { + QuantState out; + for (std::size_t i = 0; i < audio_float_count; ++i) { + const double value = std::isfinite(state.values[i]) ? static_cast(state.values[i]) : 0.0; + auto rounded = std::llround(std::clamp(value * static_cast(value_scale), -static_cast(value_limit), static_cast(value_limit))); + if (!rounded && value != 0.0) rounded = value > 0 ? 1 : -1; + out.values[i] = static_cast(rounded); + } + out.selectors = state.selectors; + // (Never the first switch: a remote skater's sound is not the local player's mix.) + for (std::size_t i = 1; i < audio_flag_count; ++i) + if (state.flags[i]) out.flags |= std::uint64_t{1} << i; + return out; +} +inline AudioState restore(const QuantState &state) noexcept { + AudioState out; + for (std::size_t i = 0; i < audio_float_count; ++i) + out.values[i] = static_cast(static_cast(state.values[i]) / static_cast(value_scale)); + out.selectors = state.selectors; + for (std::size_t i = 0; i < audio_flag_count; ++i) out.flags[i] = (state.flags >> i) & 1; + return out; +} + +namespace detail { +using pose_batch::detail::put; +using pose_batch::detail::Reader; +using pose_batch::detail::unzigzag; +using pose_batch::detail::varint; +using pose_batch::detail::zigzag; +// Eight bytes say which numbers changed (a bit each) and, above those, whether the sample is an +// event, whether the switches follow and whether selectors do. +inline constexpr unsigned event_bit = 58, flags_bit = 59, selectors_bit = 60; +inline void put_sample(std::vector &out, const QuantState &now, const QuantState &was, std::uint32_t age_us, bool event) { + std::uint64_t changed{}; + std::uint32_t selectors{}; + for (std::size_t i = 0; i < audio_float_count; ++i) + if (now.values[i] != was.values[i]) changed |= std::uint64_t{1} << i; + for (std::size_t i = 0; i < audio_selector_count; ++i) + if (now.selectors[i] != was.selectors[i]) selectors |= 1U << i; + const bool flags = now.flags != was.flags; + put(out, changed | (std::uint64_t{event} << event_bit) | (std::uint64_t{flags} << flags_bit) | (std::uint64_t{selectors != 0} << selectors_bit), 8); + varint(out, std::min(age_us >> age_shift, age_limit)); + for (std::size_t i = 0; i < audio_float_count; ++i) + if (changed & (std::uint64_t{1} << i)) varint(out, zigzag(std::int64_t{now.values[i]} - std::int64_t{was.values[i]})); + if (flags) put(out, now.flags, 6); + if (selectors) { + put(out, selectors, 4); + for (std::size_t i = 0; i < audio_selector_count; ++i) + if (selectors & (1U << i)) put(out, now.selectors[i], 4); + } +} +// Throws on bytes that are not a sample or that make an impossible one. +inline QuantState get_sample(Reader &r, const QuantState &was, std::uint32_t &age_us, bool &event) { + const auto head = r.get(8); + if (head >> (selectors_bit + 1)) throw std::runtime_error("Invalid sound sample"); + const auto age = r.varint(); + if (age > age_limit) throw std::runtime_error("Invalid sound sample age"); + age_us = static_cast(age) << age_shift; + event = (head >> event_bit) & 1; + QuantState now = was; + for (std::size_t i = 0; i < audio_float_count; ++i) { + if (!(head & (std::uint64_t{1} << i))) continue; + const auto step = unzigzag(r.varint()); + if (step > 2 * value_limit || step < -2 * value_limit) throw std::runtime_error("Invalid sound value"); + const auto value = std::int64_t{was.values[i]} + step; + if (value > value_limit || value < -value_limit) throw std::runtime_error("Invalid sound value"); + now.values[i] = static_cast(value); + } + if ((head >> flags_bit) & 1) { + now.flags = r.get(6); + if (now.flags >> audio_flag_count || (now.flags & 1)) throw std::runtime_error("Invalid sound switches"); + } + if ((head >> selectors_bit) & 1) { + const auto selectors = r.get(4); + if (!selectors || selectors >> audio_selector_count) throw std::runtime_error("Invalid sound selectors"); + for (std::size_t i = 0; i < audio_selector_count; ++i) + if (selectors & (std::uint64_t{1} << i)) now.selectors[i] = static_cast(r.get(4)); + } + return now; +} +} // namespace detail + +// The sending side of one connection: what its receiver holds of each player's sound. begin(), +// then add() for each player with samples, then the message goes reliably and sent() is told +// whether it went: what the receiver holds moves on only when it did. +class Sender { + public: + void begin(std::uint64_t world, std::uint64_t map) { + message_.assign(magic.begin(), magic.end()); + detail::put(message_, world & 0xffffffff, 4); + detail::put(message_, map & 0xffffffff, 4); + staged_.clear(); + } + void add(std::uint64_t source, std::uint64_t epoch, std::uint32_t sequence, std::uint64_t time_us, std::span samples, std::uint64_t now) { + if (samples.empty() || samples.size() > max_audio_samples) return; + Out out; + if (const auto staged = staged_.find(source); staged != staged_.end()) out = staged->second; + else if (const auto found = streams_.find(source); found != streams_.end()) out = found->second; + if (!out.stream || out.epoch != epoch) { + out = {}; + out.epoch = epoch; + out.stream = next_stream_++; + if (!next_stream_) next_stream_ = 1; + } + const bool whole = !out.started || sequence <= out.sequence || now - out.whole_at >= whole_every_us; + detail::varint(message_, out.stream); + message_.push_back(static_cast((whole ? 1U : 0U) | (static_cast(samples.size()) << 1))); + if (whole) { + detail::put(message_, source, 8); + detail::put(message_, epoch, 8); + detail::put(message_, sequence, 4); + detail::put(message_, time_us, 8); + out.whole_at = now; + } else { + detail::varint(message_, sequence - out.sequence); + detail::varint(message_, detail::zigzag(static_cast(time_us) - static_cast(out.time_us))); + } + QuantState was = whole ? QuantState{} : out.state; + for (const auto &sample : samples) { + const auto state = quantize(sample.state); + detail::put_sample(message_, state, was, sample.age_us, sample.event); + was = state; + } + out.state = was; + out.sequence = sequence; + out.time_us = time_us; + out.started = true; + staged_[source] = out; + } + bool pending() const noexcept { return !staged_.empty(); } + std::span message() const noexcept { return message_; } + void sent(bool went) { + if (went) + for (const auto &[source, out] : staged_) streams_[source] = out; + staged_.clear(); + message_.clear(); + } + // The player left: their stream is nobody's. + void forget(std::uint64_t source) { streams_.erase(source); } + // A new world: every stream starts over. + void restart() { + streams_.clear(); + staged_.clear(); + message_.clear(); + } + + private: + struct Out { + std::uint64_t epoch{}, time_us{}, whole_at{}; + std::uint32_t sequence{}; + std::uint16_t stream{}; + bool started{}; + QuantState state; + }; + std::map streams_, staged_; + std::vector message_; + std::uint16_t next_stream_ = 1; +}; + +// The receiving side of one stream: whose it is and the last sample of it. +struct In { + std::uint64_t source{}, epoch{}, time_us{}; + std::uint32_t sequence{}; + QuantState state; +}; +// One player's samples from a message, as the packet of sound it would have been. +struct Heard { + std::uint64_t source{}, epoch{}, time_us{}; + std::uint32_t sequence{}; + std::vector samples; +}; +// What a message holds, or nothing when it is not one or is damaged. A message of another world +// holds nothing; samples of a stream this side was never told of are passed over. `streams` moves +// on with what is read. +template +std::optional> read(Streams &streams, std::span bytes, std::uint64_t world, std::uint64_t map) noexcept { + try { + if (!is_sound(bytes)) return {}; + detail::Reader r{bytes, 4}; + std::vector out; + if (r.get(4) != (world & 0xffffffff) || r.get(4) != (map & 0xffffffff)) return out; + while (r.at < bytes.size()) { + const auto stream = r.varint(); + if (stream > 0xffff) return {}; + const auto head = r.get(1); + const bool whole = head & 1; + const auto count = static_cast(head >> 1); + if (!count || count > max_audio_samples) return {}; + In in; + bool known = whole; + if (whole) { + in.source = r.get(8); + in.epoch = r.get(8); + in.sequence = static_cast(r.get(4)); + in.time_us = r.get(8); + } else { + const auto ahead = r.varint(); + const auto time_step = detail::unzigzag(r.varint()); + if (!ahead || ahead > 0xffffff) return {}; + if (const auto found = streams.find(static_cast(stream)); found != streams.end()) { + in = found->second; + known = true; + } + in.sequence += static_cast(ahead); + in.time_us = static_cast(static_cast(in.time_us) + time_step); + } + Heard heard{in.source, in.epoch, in.time_us, in.sequence, {}}; + heard.samples.reserve(count); + QuantState was = whole ? QuantState{} : in.state; + std::uint32_t last_age = 1000000; + for (std::size_t i = 0; i < count; ++i) { + std::uint32_t age{}; + bool event{}; + was = detail::get_sample(r, was, age, event); + if (age > last_age) return {}; + last_age = age; + heard.samples.push_back({age, restore(was), event}); + } + if (!known) continue; + in.state = was; + streams[static_cast(stream)] = in; + out.push_back(std::move(heard)); + if (out.size() > 512) return {}; + } + return out; + } catch (...) { + return {}; + } +} +} // namespace dingosdk::multiplayer::sound_codec diff --git a/Extension/Multiplayer/Remote/cosmetics.h b/Extension/Multiplayer/Remote/cosmetics.h index d64a4e6..c64a632 100644 --- a/Extension/Multiplayer/Remote/cosmetics.h +++ b/Extension/Multiplayer/Remote/cosmetics.h @@ -9,6 +9,9 @@ namespace dingosdk::multiplayer { inline constexpr std::uint32_t skater_recipe_key = 2759515148U, board_recipe_key = 1583459055U; inline constexpr std::size_t max_cosmetic_slots = 64, max_cosmetic_scalars = 64, max_cosmetic_parameters = 128, max_cosmetic_asset = 255; +// The largest encoded outfit packet. Well under the packet limit, so an outfit a host accepts +// always fits again with the reference marker a relayed copy carries. +inline constexpr std::size_t max_appearance_bytes = 16384; struct CosmeticSlot { std::uint32_t slot{}; std::string asset; @@ -25,9 +28,37 @@ struct PlayerCard { std::uint32_t background{}, emblem{}, title{}; bool operator==(const PlayerCard &) const = default; }; +// How one of a listed player's marked cosmetics is coloured (developer_identity.h), chosen on +// the menu's Special page and carried in their Appearance, so everyone sees the same. +enum class MarkMode : std::uint8_t { standard, off, gradient, solid }; +struct MarkStyle { + MarkMode mode{}; // standard: what the player's list gives + std::array from{}, to{}; // as picked on screen. gradient: between the two; solid: `from` + std::uint8_t speed{}; // 0 normal, 1 slow, 2 fast + bool operator==(const MarkStyle &) const = default; +}; +// The marked cosmetics, in this order: whatever the skater has in each of these slots, then the +// parts of whatever board they ride. +inline constexpr std::array mark_item_names{ + "Top", "Bottoms", "Shoes", "Socks", "Hat", "Glasses", "Outfit", "Costume", "Deck", "Grip tape", "Trucks", "Wheels"}; +inline constexpr std::size_t mark_items = mark_item_names.size(), skater_mark_items = 8; +using MarkStyles = std::array; +// The rainbow, as a style a developer or a member of staff picks (a developer's standard is +// the rainbow already). Players on builds from before it could be picked must be able to read +// the style, so it travels as one they know: a solid colour, with this in the second colour a +// solid does not use. They show the solid colour; nobody else's item takes the rainbow from it. +inline constexpr std::array rainbow_marker{0x52, 0x42, 0x57}; // "RBW" +inline bool rainbow_style(const MarkStyle &style) noexcept { return style.mode == MarkMode::solid && style.to == rainbow_marker; } +inline bool valid_mark_style(const MarkStyle &style) noexcept { + return style.mode <= MarkMode::solid && style.speed <= 2; +} struct Appearance { CosmeticRecipe skater, board; PlayerCard card; + // The player has turned off what the ReSkate backend gives them (developer_identity.h): + // nobody shows their tag, or nobody animates their hoodie and board. + bool hide_tag{}, hide_items{}; + MarkStyles marks{}; bool operator==(const Appearance &) const = default; }; // Stand-ins for a peer's cosmetic that this PC does not have (a player's diff --git a/Extension/Multiplayer/Remote/native_skater_internal.h b/Extension/Multiplayer/Remote/native_skater_internal.h index 4546fb3..07bd7b1 100644 --- a/Extension/Multiplayer/Remote/native_skater_internal.h +++ b/Extension/Multiplayer/Remote/native_skater_internal.h @@ -97,6 +97,9 @@ struct Remote { std::atomic native_evaluated{}, native_skipped{}, native_evaluation_us{}, apply_us{}; ULONGLONG next_native_animation{}, next_safety_audit{}, next_entity_audit{}, next_board_safety_audit{}, next_board_status{}, next_status{}; + // A skateboard that went away is not created again before this: one creation is a heavy + // native pass, and a player's board may come and go with every pose. + ULONGLONG next_board_create{}; bool board_failed{}; std::string board_issue; std::string board_status; diff --git a/Extension/Multiplayer/Remote/native_skater_spawn.cpp b/Extension/Multiplayer/Remote/native_skater_spawn.cpp index ef0e98e..888b870 100644 --- a/Extension/Multiplayer/Remote/native_skater_spawn.cpp +++ b/Extension/Multiplayer/Remote/native_skater_spawn.cpp @@ -252,10 +252,12 @@ std::string show_board(std::uintptr_t base, const NativeFrame &local, const Pose require(!r.board_failed, r.board_issue.c_str()); } if (pose.board.empty()) { + if (r.board_entity) r.next_board_create = GetTickCount64() + 2000; remove_board(base); return "Waiting for skateboard pose."; } if (!r.board_entity) { + if (GetTickCount64() < r.next_board_create) return "Waiting to show the skateboard again."; const auto local_board = read_native_board(readable, base, local.entity); if (!local_board.entity || local_board.entity != local.board_entity) return "Waiting for the local skateboard blueprint."; @@ -418,7 +420,7 @@ void remove_remote(std::uintptr_t base) noexcept { r.target = {}; r.pose_driven.store(false, std::memory_order_release); r.next_native_animation = r.next_safety_audit = r.next_entity_audit = r.next_board_safety_audit = - r.next_board_status = r.next_status = 0; + r.next_board_status = r.next_status = r.next_board_create = 0; r.failed = false; r.issue.clear(); r.board_failed = false; diff --git a/Extension/Multiplayer/Remote/playback_buffers.cpp b/Extension/Multiplayer/Remote/playback_buffers.cpp index 7d1ac48..79f461d 100644 --- a/Extension/Multiplayer/Remote/playback_buffers.cpp +++ b/Extension/Multiplayer/Remote/playback_buffers.cpp @@ -314,6 +314,9 @@ void PoseBuffer::correct(Pose &pose, std::uint64_t now) const { } offset_pose(pose, offset); } +bool PoseBuffer::heard_within(std::uint64_t now, std::uint64_t age) const { + return !frames_.empty() && (now < frames_.back().arrival || now - frames_.back().arrival <= age); +} bool PoseBuffer::sample_remote(std::uint64_t now, Pose &pose) { if (playback_at_ && (now < playback_at_ || now - playback_at_ > 200000)) { correction_start_ = correction_end_ = 0; diff --git a/Extension/Multiplayer/Session/object_state.h b/Extension/Multiplayer/Session/object_state.h index 20554da..822197b 100644 --- a/Extension/Multiplayer/Session/object_state.h +++ b/Extension/Multiplayer/Session/object_state.h @@ -49,6 +49,16 @@ inline bool valid_object_chunk(const ObjectChunk &chunk) { return false; return true; } +// What of an owner's uploaded `layout` everyone else is shown under a limit of `limit` objects +// (0: all of it). The objects already shown (`shared`) keep their place, so placing one too +// many never takes away one that is there; the rest fill what room is left, oldest ID first. +inline std::vector limited_layout(std::vector layout, const std::map &shared, + std::size_t limit) { + if (!limit || layout.size() <= limit) return layout; + std::stable_partition(layout.begin(), layout.end(), [&](const NetworkObject &object) { return shared.contains(object.id); }); + layout.resize(limit); + return layout; +} // Per-owner state. A multipart replacement/delta becomes visible only after all // parts validate. Reliable delivery keeps parts ordered; stale revisions cannot // resurrect deleted objects. No wire IDs are native entity handles. diff --git a/Extension/Multiplayer/Session/party_book.h b/Extension/Multiplayer/Session/party_book.h index 955786f..15a15d3 100644 --- a/Extension/Multiplayer/Session/party_book.h +++ b/Extension/Multiplayer/Session/party_book.h @@ -4,9 +4,9 @@ #include namespace dingosdk::multiplayer { -// Parties on a dedicated server (a listen host puts everyone in one party instead). The -// server owns this book: players ask it to invite, accept, leave and so on (Packet::party), -// and every roster carries each player's party (Member::party / party_leader / party_open). +// Parties in a session. Whoever hosts owns this book (a dedicated server, or a lobby's host): +// players ask it to invite, accept, leave and so on (Packet::party), and every roster carries +// each player's party (Member::party / party_leader / party_open). // Party numbers are the book's own, never 0 (0 = not in a party). A party always has at // least two members: one left alone is dissolved. class PartyBook { diff --git a/Extension/Multiplayer/Session/room.h b/Extension/Multiplayer/Session/room.h index 77f8cf2..3d95151 100644 --- a/Extension/Multiplayer/Session/room.h +++ b/Extension/Multiplayer/Session/room.h @@ -1,6 +1,9 @@ #pragma once #include "Extension/Multiplayer/Net/protocol.h" #include +#include +#include +#include #include namespace dingosdk::multiplayer { @@ -75,7 +78,47 @@ struct PoseDelivery { std::uint64_t source{}, epoch{}, last_sent{}; std::uint32_t interval_us = 50000; std::uint64_t next_source_time{}; + std::uint32_t by_distance{}; // what the distance alone asked for, before any crowd limit + std::uint8_t precision{}; // how finely the source's rotations are sent to them (a dedicated server's pose_precision) }; +// A crowd in one place: every player there is within full-rate distance of every other, and +// what one player is sent grows with the crowd until their connection cannot carry it. The +// default budget is what was seen to arrive through Steam's relays (about 1 MB/s a player, +// whatever the send rate): nobody near is slowed until about 20 players are in one place at +// 30 TPS; past that the farthest of them are. A +// player is sent at most `budget` poses a second: everyone at the low rate at least, the +// nearest at the full rate with two thirds of what is left, the next nearest at the half +// rate with the rest. These are the squared distances beyond which a player is sent at the +// half and at the low rate; `squared` (the distances to the others) is sorted here. +inline constexpr unsigned crowd_always_full = 8, crowd_always_half = 12; +struct CrowdLimits { + float half = std::numeric_limits::infinity(), low = std::numeric_limits::infinity(); +}; +inline CrowdLimits crowd_limits(std::span squared, unsigned tps, unsigned budget = crowd_pose_budget) { + const unsigned full = 1000000U / multiplayer_pose_interval(tps), half = 10, low = 5; + const auto count = static_cast(squared.size()); + if (count * full <= budget) return {}; + std::sort(squared.begin(), squared.end()); + const unsigned spare = budget > count * low ? budget - count * low : 0; + // Whatever the budget, the nearest few are sent at the full rate and the next few at the + // half rate: with enough players on, the low rate for everyone is the whole budget, and + // without this nobody at all was left at the full rate (and so nobody was heard). + const unsigned wanted_full = spare * 2 / 3 / (full - low); + const unsigned at_full = std::min(count, std::max(wanted_full, crowd_always_full)); + const unsigned left = spare > at_full * (full - low) ? spare - at_full * (full - low) : 0; + const unsigned at_half = std::min(count - at_full, std::max(left / (half - low), crowd_always_half)); + CrowdLimits limits; + // Between two players the limit falls halfway, so that neither sits on it. + const auto after = [&](unsigned sent) { + return sent >= count ? std::numeric_limits::infinity() : sent ? (squared[sent - 1] + squared[sent]) / 2 : -1.f; + }; + limits.half = after(at_full); + limits.low = after(at_full + at_half); + return limits; +} +inline std::uint32_t crowd_interval(std::uint32_t interval, float distance_squared, const CrowdLimits &limits) noexcept { + return distance_squared > limits.low ? 200000U : distance_squared > limits.half ? std::max(interval, 100000U) : interval; +} // Preserve the timer phase across variable client frames/packet arrivals. Only // the newest state is sent; elapsed slots are skipped without catch-up packets. inline void advance_pose_deadline(std::uint64_t &next, std::uint64_t now, std::uint64_t interval) { @@ -102,11 +145,74 @@ struct DirectUploadBudget { } } }; +// What one player's game sends of each relayed stream, with room to spare: a host passes on +// no more than this from any one source, whatever the connection's overall budget allows. +// Outfits: captured twice a second and sent when changed. Counted over five seconds. +inline constexpr unsigned outfit_burst = 12; +struct OutfitBudget { + std::uint64_t since{}; + unsigned count{}; + bool accept(std::uint64_t now) noexcept { + if (now < since || now - since >= 5000000) { since = now; count = 0; } + return ++count <= outfit_burst; + } +}; +// Skater sound: at most one packet per network tick, a few samples each. +struct SoundBudget { + std::uint64_t since{}; + std::size_t packets{}, samples{}; + bool accept(std::uint64_t now, std::size_t count) noexcept { + if (now < since || now - since >= 1000000) { since = now; packets = samples = 0; } + if (packets >= multiplayer_tick_rates.back() + 30U || samples + count > 400) return false; + ++packets; + samples += count; + return true; + } +}; +// A connection that never finished joining (it timed out, or had the wrong password or code) +// may try again at once the first time. After that every failure makes its Steam ID wait +// longer before a host takes its connection again, so one account cannot hold a player slot +// or try passwords over and over. +struct JoinBackoff { + struct Entry { + std::uint64_t until{}, last{}; + unsigned failures{}; + }; + static constexpr std::uint64_t forget_us = 30ULL * 60 * 1000000, longest_us = 10ULL * 60 * 1000000; + std::map entries; + // Notes a failed attempt and returns how many this ID has made lately. + unsigned failed(std::uint64_t id, std::uint64_t now) { + auto &entry = entries[id]; + if (entry.last && now >= entry.last && now - entry.last > forget_us) entry.failures = 0; + ++entry.failures; + entry.last = now; + const auto wait = entry.failures < 2 ? 0ULL + : std::min(longest_us, 5000000ULL << std::min(entry.failures - 2, 8U)); + entry.until = now + wait; + return entry.failures; + } + bool waiting(std::uint64_t id, std::uint64_t now) const { + const auto found = entries.find(id); + return found != entries.end() && now < found->second.until; + } + void joined(std::uint64_t id) { entries.erase(id); } + void prune(std::uint64_t now) { + std::erase_if(entries, [&](const auto &entry) { + return now >= entry.second.last && now - entry.second.last > forget_us; + }); + } +}; +// How much one peer may send in a second. `when` is when the packet arrived (the transport's +// TransportMessage::arrived), not when it is read: a receiver that could not read for a few +// seconds (a server whose console held it up, a game loading a map) reads everything that +// arrived meanwhile at once, and counted by reading time every peer looked like a flood and +// was dropped. Packets are not always read in arrival order (Steam's lanes are read one after +// another): one from before the second being counted is counted in it. struct ReceiveBudget { std::uint64_t since{}, bytes{}, packets{}; - bool accept(std::uint64_t now, std::size_t size, unsigned sources = 1) { - if (now - since >= 1000000) { - since = now; + bool accept(std::uint64_t when, std::size_t size, unsigned sources = 1) { + if (when > since && when - since >= 1000000) { + since = when; bytes = packets = 0; } bytes += size; diff --git a/Extension/Multiplayer/Session/session.cpp b/Extension/Multiplayer/Session/session.cpp index ada649d..ffd344d 100644 --- a/Extension/Multiplayer/Session/session.cpp +++ b/Extension/Multiplayer/Session/session.cpp @@ -3,6 +3,7 @@ #include "Extension/Multiplayer/Hud/native_indicators.h" #include "Extension/Multiplayer/Hud/native_party.h" #include "party_book.h" +#include "Engine/Game/World/world_names.h" #include "Extension/Multiplayer/Remote/native_skater.h" #include "Extension/Multiplayer/Hud/custom_nametags.h" #include "Extension/Multiplayer/Hud/follow_camera.h" @@ -19,6 +20,7 @@ #include "Extension/Skater/physics_tuning.h" #include "Extension/Multiplayer/Remote/remote_collision.h" #include "Engine/Core/Log/logging.h" +#include "Engine/Core/Text/word_filter.h" #include "Engine/Game/Multiplayer/session_tools.h" #include "Engine/Game/UI/game_view.h" #include @@ -71,10 +73,14 @@ void stop(Session &s, std::string reason) { s.voice_policy = {}; s.tps = multiplayer_default_tps; s.object_placement = ObjectPlacement::everyone; + s.object_limit = 0; + set_lobby_object_limit(0); s.server_admin = false; s.server_bans.clear(); s.server_ban_total = 0; s.server_maps.clear(); + s.server_map_pool.clear(); + s.server_map_rotation = 0; set_lobby_object_placement_allowed(true); s.guest_noclip = s.guest_no_bail = s.guest_boosts = true; s.enforce_tuning = true; @@ -86,12 +92,19 @@ void stop(Session &s, std::string reason) { s.sent_tuning.reset(); s.tuning_packet.clear(); s.next_tuning_check = 0; + s.host_extras.reset(); + s.sent_extras = 0; + s.extras_packet.clear(); + s.next_extras_check = 0; + set_session_tuning_enforced(false); + set_host_physics_extras({}); s.server_votes = 0; set_session_tools_allowed(true, true, true); s.object_clears.reset(); s.clear_pending = false; s.force_world_layers = false; s.banned.clear(); + s.join_backoff = {}; apply_host_world_layers(false, s.layers); s.layers = default_world_layers(); clear_remote_network_objects(); @@ -100,6 +113,8 @@ void stop(Session &s, std::string reason) { set_lobby_park_mode(false, false); s.parks = {}; s.next_park_update = 0; s.next_party_update = 0; + s.parties = PartyBook{}; + s.parties_revision = 0; s.client_timing = {}; s.next_publish = 0; s.last_client_log = 0; @@ -118,6 +133,7 @@ void stop(Session &s, std::string reason) { s.awaiting_map = s.join_map_authorized = s.map_load_submitted = false; s.join_started = s.last_map_request = s.last_map_load_check = 0; s.join_destination.clear(); + s.map_label.clear(); s.world = 1; s.travelling = false; s.host_world_ready = true; @@ -139,6 +155,15 @@ void stop(Session &s, std::string reason) { s.next_send = s.last_hello = s.last_roster = s.last_cosmetic_capture = s.last_routes = s.last_network_log = 0; s.sequence = s.roster_sequence = 0; + s.server_chat_badge = default_server_chat_badge; + s.server_chat_text = default_server_chat_text; + s.pose_streams.clear(); + s.pose_ack = {}; + s.pose_ack_due = false; + s.pose_upload = {}; + s.own_poses.clear(); + s.sound_streams.clear(); + s.sound_upload = {}; s.local_pose_count = s.logged_local_pose_count = 0; s.status = std::move(reason); } @@ -146,6 +171,14 @@ void stop(Session &s, std::string reason) { // admission, password challenges and stable member slots intact. void clear_world(Session &s, std::uint64_t now) { s.voice.reset(); + // The server starts its pose streams and message numbers over with the world. + s.pose_streams.clear(); + s.pose_ack = {}; + s.pose_ack_due = false; + s.pose_upload.restart(); + s.own_poses.clear(); + s.sound_streams.clear(); + s.sound_upload.restart(); clear_remote_network_objects(); s.object_owners_valid = false; s.local_objects = {}; s.next_object_update = 0; @@ -244,23 +277,40 @@ void publish_party(Session &s) { } if (!s.display_preferences_loaded) { s.display_preferences_loaded = true; - // A lobby is one party unless the player turned that off (the older PartyOverlay - // preference, off by default, is not carried over). - s.party_overlay = profile_runtime::local_preference("LobbyParty").value_or(true); s.nametags = profile_runtime::local_preference("Nametags").value_or(true); - s.custom_nametags = profile_runtime::local_preference("CustomNametags").value_or(true); + if (const auto saved = profile_runtime::local_value("NametagDistance"); saved && saved->is_number()) + s.nametag_distance = std::clamp(saved->get(), 10.f, 500.f); + s.nametag_dots = profile_runtime::local_preference("NametagDots").value_or(true); + s.prefer_direct = profile_runtime::local_preference("DirectConnections").value_or(true); + if (const auto saved = profile_runtime::local_value("PlayerDistance"); saved && saved->is_number()) + s.player_distance = std::clamp(saved->get(), player_distance_least, player_distance_unlimited); + s.nametags_friends = profile_runtime::local_preference("NametagsFriendsOnly").value_or(false); s.chat_visible = profile_runtime::local_preference("ChatVisible").value_or(true); s.chat_filter = profile_runtime::local_preference("ChatFilter").value_or(true); + s.chat_bubbles = profile_runtime::local_preference("ChatBubbles").value_or(true); + s.chat_bubbles_own = profile_runtime::local_preference("ChatBubblesOwn").value_or(false); + if (const auto saved = profile_runtime::local_value("ChatBubblesDistance"); saved && saved->is_number()) + s.chat_bubbles_distance = std::clamp(saved->get(), 5.f, 500.f); + if (const auto saved = profile_runtime::local_value("ChatBubblesDuration"); saved && saved->is_number()) + s.chat_bubbles_duration = std::clamp(saved->get(), 1.f, 30.f); + if (const auto saved = profile_runtime::local_value("ChatBubblesHistory"); saved && saved->is_number()) + s.chat_bubbles_history = std::clamp(static_cast(saved->get()), 1, 8); + show_own_tag(profile_runtime::local_preference("IdentityTag").value_or(true)); + show_own_items(profile_runtime::local_preference("IdentityItems").value_or(true)); + if (const auto saved = profile_runtime::local_value("IdentityStyles"); saved && saved->is_string()) + if (const auto styles = developer_hoodie_detail::parse_mark_styles(saved->string())) + developer_hoodie_detail::own_styles.store(*styles); apply_nametags(s); } - // The party's limit: a lobby is one party of up to its capacity; a dedicated server's - // parties hold up to eight (the game's Party panel rows). + // The party's limit: parties players form hold up to eight (the game's Party panel rows), + // in a lobby as on a dedicated server. + const bool session = s.mode == Mode::host || s.mode == Mode::join; const auto capacity = s.mode == Mode::off ? static_cast(max_players) - : dedicated_host(s) ? static_cast(PartyBook::default_limit) : s.capacity; - // A party formed on a dedicated server is always the game's party (its Social menu, Coop - // button, beacons...); a lobby is one too unless the player turned Lobby party off. - set_native_party_changes(dedicated_host(s)); - const bool shown = dedicated_host(s) ? s.local_party != 0 : s.mode != Mode::off && s.party_overlay; + : session ? static_cast(PartyBook::default_limit) : s.capacity; + // A party the player formed is the game's party (its Social menu, Coop button, beacons...). + // Nobody is in one just for being in the same lobby or on the same server. + set_native_party_changes(session); + const bool shown = session ? s.local_party != 0 : s.mode != Mode::off; update_native_party(s.base, roster, capacity, shown); } // Links the players' own throwdowns (Extension/Throwdowns/throwdown_relay.cpp). Runs @@ -341,12 +391,11 @@ Pose out_of_sight(Pose pose) { // local skater and the camera: every frame within 200 m, 20 Hz past it, 10 Hz past 350 m, and // every frame again once back inside 190 m. Between samples the skater keeps its pose, which // saves the interpolation, the pose copy and the skeleton write for players too far away to see. +// Only distance decides. Players out of the camera's view were sampled at 20 Hz as well for a +// while (it saved client-tick time on busy servers), but the game's replays record every skater +// as it stood each frame and their camera looks wherever it likes afterwards: a player who had +// been behind the camera moved at 20 Hz in the replay. constexpr int far_full_rate_return = 190, far_half_rate_start = 200, far_low_rate_start = 350; -// Out of the camera's view, players past this many metres are sampled at 20 Hz too: the whole- -// skeleton interpolation grows with the player count and was most of ReSkate's client-tick cost -// on busy servers (profiled 2026-10-03). Nearer ones stay every frame for collision and close -// shadows; one coming into view is sampled that frame. -constexpr float out_of_view_full_rate = 8.0f; // Metres from a player to the nearer of the local skater and the camera (infinite if neither // is known). float nearest_distance(const Peer &p, const NativeFrame &local, const std::optional &view) { @@ -360,23 +409,6 @@ float nearest_distance(const Peer &p, const NativeFrame &local, const std::optio if (view) consider(view->world[12], view->world[13], view->world[14]); return std::sqrt(nearest); } -// Whether a player is certainly outside the camera's view: a 2 m sphere around their body past -// the camera, or past a side, top or bottom of its view (taken as wide as a 21:9 screen). -bool out_of_view(const Peer &p, const std::optional &view) { - if (!view || !(view->vertical_fov > 1 && view->vertical_fov < 175)) return false; - const auto &m = view->world; - const auto &at = p.render_pose.root.position; - const float dx = at[0] - m[12], dy = at[1] + 1.0f - m[13], dz = at[2] - m[14]; - const float depth = -(dx * m[8] + dy * m[9] + dz * m[10]); - const float side = dx * m[0] + dy * m[1] + dz * m[2], height = dx * m[4] + dy * m[5] + dz * m[6]; - constexpr float radius = 2.0f; - if (depth < -radius) return true; - const float tan_v = std::tan(view->vertical_fov * 3.14159265f / 360.0f), tan_h = tan_v * (21.0f / 9.0f); - const auto outside = [&](float offset, float tangent) { - return std::abs(offset) - depth * tangent > radius * std::sqrt(1.0f + tangent * tangent); - }; - return outside(side, tan_h) || outside(height, tan_v); -} std::uint64_t far_sample_interval(const Peer &p, float distance) { const auto beyond = [&](int metres) { return distance > static_cast(metres); }; if (!std::isfinite(distance) || !beyond(far_full_rate_return)) return 0; @@ -387,10 +419,43 @@ std::uint64_t far_sample_interval(const Peer &p, float distance) { void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::uint64_t now) { if (!world_playing(s, local)) return; const auto view = latest_game_view(); - // ReSkate's nametags: every shown player, placed above their head each frame. + // ReSkate's nametags and chat bubbles: every shown player, placed above their head + // each frame. std::vector nametags; - const bool labels = s.nametags && s.custom_nametags; + const bool bubbles = s.chat_bubbles; + const bool labels = s.nametags || bubbles; if (labels) refresh_friends(s); + // The newest chat lines from `sender` still inside the bubble duration, oldest first, + // filtered the same way the chat panel filters them, each with the opacity it has left + // (it fades over the last half second). At most `chat_bubbles_history` lines. + const auto recent_bubbles = [&](std::uint64_t sender) { + std::vector result; + if (!bubbles) return result; + const int limit = std::max(1, s.chat_bubbles_history); + const auto duration = static_cast(std::max(0.5f, s.chat_bubbles_duration) * 1e6f); + const auto fade_from = std::min(duration, 500000); + for (auto it = s.chat.rbegin(); it != s.chat.rend() && static_cast(result.size()) < limit; ++it) { + if (it->sender != sender) continue; + // Older lines are older still: once one has expired, stop. + if (now < it->received || now - it->received >= duration) break; + NametagBubble line; + line.text = it->text; + if (s.chat_filter) { + const auto masked = s.chat_masked.find(it->sequence); + auto filtered = masked != s.chat_masked.end() ? masked->second.second : text::mask_bad_words(it->text); + if (filtered != it->text) line.raw = std::exchange(line.text, std::move(filtered)); + } + if (line.text.find_first_not_of(' ') == std::string::npos) continue; + const auto age = now - it->received; + // Pops in over the first few frames, fades over the last half second. + constexpr std::uint64_t pop_us = 220000; + line.appear = std::min(1.0f, static_cast(age) / static_cast(pop_us)); + line.fade = fade_from ? std::min(1.0f, static_cast(duration - age) / static_cast(fade_from)) : 1.0f; + result.push_back(std::move(line)); + } + std::reverse(result.begin(), result.end()); // oldest first + return result; + }; // Who is talking, as last published for the UI (10 Hz), without copying the voice model. const auto &voices = s.view.voice.players; const auto label = [&](const Peer &p) { @@ -400,7 +465,10 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui tag.name = p.member.name.empty() ? s.transport.name(p.member.id) : p.member.name; // The same role colour and badge the player's chat lines get. std::tie(tag.color, tag.tag) = player_role(s, p.member.id, false); + // Friends only: everyone else keeps their chat bubbles and loses the name and dot. + tag.nameless = s.nametags_friends && !std::binary_search(s.friend_ids.begin(), s.friend_ids.end(), p.member.id); tag.talking = std::any_of(voices.begin(), voices.end(), [&](const auto &v) { return v.id == p.member.id && v.speaking; }); + tag.bubbles = recent_bubbles(p.member.id); nametags.push_back(std::move(tag)); }; // Creating a player's actor (skater, skateboard and both recipes) or re-applying a @@ -435,6 +503,24 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui p.next_audio_update = 0; } }; + // The developer hoodie and board mark an identity: only one Steam vouches for (steam_vouched). + const auto developer_id = [&](const Peer &p) { + return steam_vouched(s, p) && shows_items(p) ? p.member.id : std::uint64_t{}; + }; + // Only players within `player_distance` get a skater: each one costs memory and frame time + // whether or not it is in view, and on a full server most are far across the map, too small + // to see. Checked twice a second; a player already shown stays until a little further + // out, so one skating along the edge is not built and taken down over and over (building + // a skater is the expensive part). Before a pose has placed them, nobody is too far. + if (now >= s.next_shown_rank) { + s.next_shown_rank = now + 500000; + const bool everyone = s.player_distance >= player_distance_unlimited; + for (auto &p : active_peers(s)) { + if (!p.member.id || (dedicated_host(s) && p.member.id == s.host_id)) continue; + p.shown_wanted = everyone || !p.placed || + nearest_distance(p, local, view) <= s.player_distance * (p.visible ? 1.15f : 1.f); + } + } each_active_peer(s, [&](Peer &p) { // A dedicated server has no skater to show. if (!p.member.id || (dedicated_host(s) && p.member.id == s.host_id)) @@ -442,10 +528,11 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui const bool was_visible = p.visible; // A far player between samples: their skater keeps its pose; sound, the label and // the spectate position are still refreshed (sound events are released one per frame). - if (was_visible && local.ready && !p.render_failed && p.far_interval && now < p.next_far_sample && - !(p.far_for_view && !out_of_view(p, view))) { - update_developer_hoodie(s.base, remote_skater_entity(), p.member.id, remote_skater_generation(), p.developer_hoodie); - update_developer_board(s.base, remote_board_entity(), p.member.id, remote_skater_generation(), p.developer_board); + if (was_visible && local.ready && !p.render_failed && p.far_interval && now < p.next_far_sample) { + update_developer_hoodie(s.base, remote_skater_entity(), developer_id(p), remote_skater_generation(), p.developer_hoodie, + mark_styles(p)); + update_developer_board(s.base, remote_board_entity(), developer_id(p), remote_skater_generation(), p.developer_board, + mark_styles(p)); present_audio(p); update_party_position(&p.render_pose); if (labels) label(p); @@ -455,11 +542,20 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui if (local.ready && !p.render_failed) { const bool sampled = s.mode == Mode::echo ? p.poses.sample(now, p.render_pose) : p.poses.sample_remote(now, p.render_pose); + // Nothing new from them for over a second (a stall here, at the host or on the way): + // their skater stays where it was for a while longer. Taking it down and building + // it again three seconds later is the most expensive thing a frame can do, and on + // a busy server one slow frame made it happen to everyone at once, which made + // the next frames slower still. + const bool held = !sampled && was_visible && p.poses.heard_within(now, 8000000); // Showing a player without an actor spawns one (native_skater_spawn.cpp). - const bool spawning = sampled && p.appearance.value() && !was_visible && !remote_skater_entity(); - if (spawning && native_pass) { + if (sampled) p.placed = true; + const bool spawning = sampled && p.shown_wanted && p.appearance.value() && !was_visible && !remote_skater_entity(); + if (spawning && now < p.next_spawn) { + p.native_status = "Waiting to show the player again."; + } else if (spawning && native_pass) { p.native_status = "Waiting for another player's skater to finish spawning."; - } else if (sampled && p.appearance.value()) { + } else if ((sampled || held) && p.shown_wanted && p.appearance.value()) { if (spawning) { native_pass = true; p.applied_cosmetics = 0; // the new actor wears no recipe yet @@ -472,9 +568,6 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui p.native_status); const auto distance = nearest_distance(p, local, view); p.far_interval = p.visible && !hidden ? far_sample_interval(p, distance) : 0; - p.far_for_view = p.visible && !hidden && !p.far_interval && std::isfinite(distance) && - distance > out_of_view_full_rate && s.mode != Mode::echo && out_of_view(p, view); - if (p.far_for_view) p.far_interval = 50000; p.next_far_sample = now + p.far_interval; // The native work a far player's skater may skip (puppet_cost.cpp). if (p.visible) note_remote_distance(distance); @@ -514,16 +607,21 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui remove_remote(s.base); } } else { - if (was_visible) + if (was_visible) { remove_remote(s.base); - p.native_status = - sampled ? "Waiting for the player's cosmetic recipe." : "Waiting for player poses."; + p.next_spawn = now + 3000000; + } + p.native_status = !p.shown_wanted ? "Further away than players are shown." + : sampled ? "Waiting for the player's cosmetic recipe." + : "Waiting for player poses."; + // No skater for them, but still their name or dot where they are. + if (!p.shown_wanted && sampled && labels) label(p); } } - update_developer_hoodie(s.base, p.visible ? remote_skater_entity() : 0, p.member.id, - remote_skater_generation(), p.developer_hoodie); - update_developer_board(s.base, p.visible ? remote_board_entity() : 0, p.member.id, - remote_skater_generation(), p.developer_board); + update_developer_hoodie(s.base, p.visible ? remote_skater_entity() : 0, developer_id(p), + remote_skater_generation(), p.developer_hoodie, mark_styles(p)); + update_developer_board(s.base, p.visible ? remote_board_entity() : 0, developer_id(p), + remote_skater_generation(), p.developer_board, mark_styles(p)); if (!p.visible) { stop_remote_audio(); p.presented_audio.reset(); @@ -535,9 +633,23 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui } } }); + // The local player's own lines, above their own skater, when asked for. + if (labels && bubbles && s.chat_bubbles_own && local.ready) { + auto own = recent_bubbles(s.transport.status().local_id); + if (!own.empty()) { + NametagPlayer tag; + tag.head = local.pose.root.position; + tag.head[1] += 1.0f; + tag.self = true; + tag.bubbles = std::move(own); + nametags.push_back(std::move(tag)); + } + } if (labels) publish_custom_nametags(s.base, std::move(nametags), - local.ready ? std::optional(local.pose.root.position) : std::nullopt); + local.ready ? std::optional(local.pose.root.position) : std::nullopt, + s.nametags, bubbles, s.chat_bubbles_distance, s.nametag_distance, + s.nametag_dots); } } // namespace MultiplayerModel model() { @@ -545,6 +657,7 @@ MultiplayerModel model() { std::lock_guard lock(s.mutex); return s.view; } +std::string take_leave_notice() { return std::exchange(session().leave_notice, {}); } MultiplayerChat chat() { auto &s = session(); std::lock_guard lock(s.mutex); @@ -637,6 +750,14 @@ bool prepare_join_map(Session &s, bool ready, std::string_view current, MapLoade s.last_map_load_check = now; std::string detail; const auto result = loader(s.join_destination, s.map_load_submitted, detail); + if (result == MapLoadResult::missing) { + const auto name = s.map_label.empty() ? world_level_name(world_destination_asset(s.join_destination)) : s.map_label; + const char *who = dedicated_host(s) ? "server" : "host"; + s.leave_notice = (s.travelling ? std::string("The ") + who + " moved to " : std::string("The ") + who + " is on ") + + name + ", which is not installed on this PC. Install its map mod and join again."; + stop(s, s.leave_notice); + return false; + } if (result == MapLoadResult::failed) { stop(s, detail.empty() ? "The host's map could not be loaded." : std::move(detail)); return false; @@ -693,6 +814,7 @@ void tick(std::uintptr_t base, std::uintptr_t client, bool ready, std::string_vi expire_remote_collision(base, now_us()); if (s.mode == Mode::off) { physics_tuning::release(base); // the player's own tuning again after a session + set_session_tuning_enforced(false); relay_throwdowns(s, ready); // Without a session the UI model follows at the same 10 Hz as in one; // commands still publish at once. @@ -785,6 +907,8 @@ void tick(std::uintptr_t base, std::uintptr_t client, bool ready, std::string_vi s.client_timing.record(ClientTiming::capture, now, receive_at); if (s.mode != Mode::echo) networking(s, local, receive_at); + for (const auto ¬e : s.transport.take_direct_notes()) + logging::log(logging::Level::info, logging::Channel::runtime, "Multiplayer: {}", note); relay_throwdowns(s, world_playing(s, local)); // A S.K.A.T.E. player waiting for their turn stays off the board. update_board_lock(client, local.entity, local.ready && throwdown_relay_waits_offboard()); diff --git a/Extension/Multiplayer/Session/session.h b/Extension/Multiplayer/Session/session.h index 06ec342..0b7bc7b 100644 --- a/Extension/Multiplayer/Session/session.h +++ b/Extension/Multiplayer/Session/session.h @@ -10,12 +10,13 @@ std::string command(std::string_view action, std::string_view argument = {}, std bool queue_command(std::string_view action, std::string_view argument, std::string_view password); // Called just before the host's validated native load is submitted, on the client thread. void host_map_change(std::string_view destination); -enum class MapLoadResult { waiting, queued, failed }; +enum class MapLoadResult { waiting, queued, failed, missing }; // missing: the map is not installed // Runs on the client thread; only catalog-registered destinations may be queued. using MapLoader = MapLoadResult (*)(std::string_view map, bool submitted, std::string &detail); void tick(std::uintptr_t base, std::uintptr_t client, bool gameplay_ready, std::string_view map, MapLoader loader = nullptr); MultiplayerModel model(); +std::string take_leave_notice(); // why the session just ended over a missing map, once; client thread // The session's chat lines for the overlay. Thread-safe and cheap: callable every frame. MultiplayerChat chat(); } // namespace dingosdk::multiplayer diff --git a/Extension/Multiplayer/Session/session_commands.cpp b/Extension/Multiplayer/Session/session_commands.cpp index bb127a8..3ea39db 100644 --- a/Extension/Multiplayer/Session/session_commands.cpp +++ b/Extension/Multiplayer/Session/session_commands.cpp @@ -1,4 +1,5 @@ #include "session_internal.h" +#include "Extension/Multiplayer/Steam/steam_social.h" #include "Extension/Multiplayer/Hud/native_indicators.h" #include "Extension/Multiplayer/Hud/native_player_ui.h" #include "Extension/Multiplayer/Hud/custom_nametags.h" @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -25,15 +27,23 @@ void apply_distances(Session &s, const MultiplayerDistances &distances) { // previous band's hysteresis or waiting for its old send deadline. for (auto &peer : active_peers(s)) peer.pose_delivery = {}; } +void apply_object_limit(Session &s, unsigned limit) { + s.object_limit = limit; + set_lobby_object_limit(s.mode == Mode::host || s.server_admin ? 0 : limit); +} void apply_object_placement(Session &s, ObjectPlacement policy) { s.object_placement = policy; // On a dedicated server "host only" means its admins. set_lobby_object_placement_allowed(object_placement_allowed(policy, s.mode == Mode::host || s.server_admin)); } void apply_nametags(const Session &s) { - set_custom_nametags_enabled(s.nametags && s.custom_nametags); - set_native_nametags_enabled(s.nametags && !s.custom_nametags); - set_native_compass_enabled(!(s.nametags && s.custom_nametags)); + // The feed also runs when only chat bubbles are on: the overlay draws whichever of the + // two is enabled from the flags it is handed. + // Nametags are ReSkate's own (Hud/custom_nametags.h); the game's are never shown. Its + // compass comes back when they are off, since they are what points at the other players. + set_custom_nametags_enabled(s.nametags || s.chat_bubbles); + set_native_nametags_enabled(false); + set_native_compass_enabled(!s.nametags); } void apply_guest_tools(Session &s, bool noclip, bool no_bail, bool boosts) { s.guest_noclip = noclip; @@ -113,9 +123,8 @@ std::string party_command(Session &s, std::string_view argument) { return p ? peer_name(s, *p) : s.transport.name(id); }; if (verb.empty() || verb == "status" || verb == "list") { - if (!s.local_party) return dedicated_host(s) ? "You're not in a party. Invite a player from their player card or with: party invite ." - : "You're the only one in this lobby."; - std::string text = std::string(dedicated_host(s) ? "Your party" : "Your lobby's party") + (s.local_party_open ? " (open):" : ":"); + if (!s.local_party) return "You're not in a party. Invite a player from their player card or with: party invite ."; + std::string text = std::string("Your party") + (s.local_party_open ? " (open):" : ":"); const auto local = s.transport.status().local_id; text += " " + s.transport.name(local) + (s.local_party_leader ? " (leader)" : ""); for (const auto &peer : active_peers(s)) @@ -123,7 +132,6 @@ std::string party_command(Session &s, std::string_view argument) { text += ", " + peer_name(s, peer) + (peer.member.party_leader ? " (leader)" : ""); return text; } - if (!dedicated_host(s)) return "Everyone in a lobby is in one party with the host."; static constexpr std::pair verbs[] = { {"invite", PartyAction::invite}, {"accept", PartyAction::accept}, {"decline", PartyAction::decline}, {"join", PartyAction::join}, {"leave", PartyAction::leave}, {"kick", PartyAction::kick}, @@ -286,6 +294,21 @@ std::string edit_object_placement(Session &s, std::string_view argument) { : *policy == ObjectPlacement::host_only ? "Only you can place objects. Guests' objects are frozen." : "Object placement is disabled for everyone. Existing objects stay."; } +std::string edit_object_limit(Session &s, std::string_view argument) { + if (s.mode != Mode::host) return "Only the session host can change the object limit."; + const auto limit = parse_object_limit(argument); + if (!limit) return "Use a number of objects from 1 to " + std::to_string(max_object_limit) + ", or off."; + // Guests' games stop them at the limit when the roster arrives; enforcement is the host + // showing everyone no more than that of each guest's layout (publish_guest_objects). + apply_object_limit(s, *limit); + for (auto &peer : active_peers(s)) peer.shared_from = 0; // look at every layout again + s.roster_dirty = true; + load_host_preferences(s); + s.host_preferences.object_limit = *limit; + save_host_preferences(s); + return *limit ? "Each guest can place up to " + std::to_string(*limit) + " objects." + : std::string("Guests can place as many objects as they like."); +} // Removes every guest's objects for everyone, whatever the placement policy. // The host's own objects are its saved park and are left alone. std::string clear_guest_objects(Session &s, std::string_view) { @@ -307,16 +330,6 @@ std::string clear_guest_objects(Session &s, std::string_view) { return removed ? "Deleted " + std::to_string(removed) + " guest object" + (removed == 1 ? "" : "s") + "." : "Guests have no placed objects to delete."; } -std::string edit_party_overlay(Session &s, std::string_view argument) { - const auto enabled = parse_switch(argument, s.party_overlay); - if (!enabled) return "Use on, off, or toggle for the lobby party."; - s.party_overlay = *enabled; - s.display_preferences_loaded = true; - profile_runtime::set_local_preference("LobbyParty", s.party_overlay); - s.roster_dirty = true; - return s.party_overlay ? "Lobby party on: everyone in a lobby is in your game's party." - : "Lobby party off: lobby players are not shown as your game's party."; -} std::string edit_nametags(Session &s, std::string_view argument) { const auto enabled = parse_switch(argument, s.nametags); if (!enabled) return "Use on, off, or toggle for peer nametags."; @@ -326,15 +339,51 @@ std::string edit_nametags(Session &s, std::string_view argument) { profile_runtime::set_local_preference("Nametags", s.nametags); return s.nametags ? "Peer nametags shown." : "Peer nametags hidden."; } -std::string edit_nametag_style(Session &s, std::string_view argument) { - const auto custom = argument == "reskate" ? std::optional(true) : argument == "game" ? std::optional(false) - : parse_switch(argument, s.custom_nametags); - if (!custom) return "Use reskate, game, or toggle for the nametag style."; - s.custom_nametags = *custom; - s.display_preferences_loaded = true; - apply_nametags(s); - profile_runtime::set_local_preference("CustomNametags", s.custom_nametags); - return s.custom_nametags ? "ReSkate nametags: names, distances and dots." : "The game's own nametags."; +// The Special page: a player on one of the backend's lists going without their tag, or +// without the animation on their items. Their next appearance packet tells everyone +// (session_send.cpp). +std::string edit_own_tag(Session &, std::string_view argument) { + const auto shown = parse_switch(argument, own_tag_shown()); + if (!shown) return "Use on, off, or toggle for your tag."; + show_own_tag(*shown); + profile_runtime::set_local_preference("IdentityTag", *shown); + return *shown ? "Your tag shows." : "Your tag is hidden, for you and everyone you skate with."; +} +std::string edit_own_items(Session &, std::string_view argument) { + const auto shown = parse_switch(argument, own_items_shown()); + if (!shown) return "Use on, off, or toggle for your items."; + show_own_items(*shown); + profile_runtime::set_local_preference("IdentityItems", *shown); + return *shown ? "Your items animate." : "Your items are plain, for you and everyone you skate with."; +} +// The Special page: how one of the player's marked cosmetics is coloured, as +// " " (MarkStyle). It goes out with their next +// appearance packet and is kept in their profile. +std::string edit_mark_style(Session &, std::string_view argument) { + std::istringstream in{std::string(argument)}; + unsigned item{}, mode{}, from{}, to{}, speed{}; + if (!(in >> item >> mode >> std::hex >> from >> to >> std::dec >> speed) || item >= mark_items || mode > 4 || + from > 0xffffff || to > 0xffffff || speed > 2) + return "That is not a cosmetic style."; + // Mode 4, the rainbow, is the staff's to pick (a developer's standard is it already). It is + // kept, and sent, as a solid colour older builds can show (rainbow_style). + const auto social = steam_social_snapshot(); + const auto mark = social ? identity_mark(social->local.id) : std::nullopt; + if (mode == 4 && mark != IdentityList::staff) return "The rainbow is not one of your styles."; + const auto colour = [](unsigned value) { + return std::array{static_cast(value >> 16), static_cast(value >> 8), + static_cast(value)}; + }; + auto styles = developer_hoodie_detail::own_styles.load(); + styles[item] = {static_cast(mode == 4 ? 3 : mode), colour(from), colour(to), static_cast(speed)}; + if (mode == 4) styles[item].from = developer_hoodie_detail::standard_picks(*mark).first, styles[item].to = rainbow_marker; + // A solid colour whose unused second colour happens to be the rainbow's marker is not one. + else if (rainbow_style(styles[item])) styles[item].to[2] ^= 1; + developer_hoodie_detail::own_styles.store(styles); + profile_runtime::set_local_values({{"IdentityStyles", Json(developer_hoodie_detail::mark_styles_text(styles))}}); + return std::string(mark_item_names[item]) + + (mode == 1 ? ": off." : mode == 2 ? ": your gradient." : mode == 3 ? ": your color." : mode == 4 ? ": rainbow." + : ": back to the usual."); } // Hidden chat still receives lines, so showing it again brings back the conversation. std::string edit_chat_visible(Session &s, std::string_view argument) { @@ -353,6 +402,114 @@ std::string edit_chat_filter(Session &s, std::string_view argument) { publish_chat(s); return s.chat_filter ? "Bad words in chat are hidden." : "Chat is shown unfiltered."; } +std::string edit_player_distance(Session &s, std::string_view argument) { + float value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || !std::isfinite(value) || + value < player_distance_least || value > player_distance_unlimited) + return "Player distance is a number of metres from 50 to 1000 (1000: every player)."; + s.player_distance = value; + s.next_shown_rank = 0; + profile_runtime::set_local_values({{"PlayerDistance", static_cast(value)}}); + return value >= player_distance_unlimited ? std::string("Every player is shown as a skater, however far.") + : "Players within " + std::to_string(static_cast(value)) + " m are shown as skaters."; +} +std::string start_pose_dump(Session &s, std::string_view argument) { + int seconds{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), seconds); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || seconds < 1 || seconds > 600) + return "pose-dump (1-600): records your own poses while in a session."; + std::error_code error; + const auto folder = std::filesystem::current_path(error) / "logs"; + std::filesystem::create_directories(folder, error); + const auto file = folder / ("poses-" + std::to_string(now_us() / 1000000) + ".bin"); + s.pose_dump.close(); + s.pose_dump.clear(); + s.pose_dump.open(file, std::ios::binary | std::ios::trunc); + if (!s.pose_dump) return "Could not write " + file.string() + "."; + s.pose_dump.write("RSPD1\n", 6); + s.pose_dump_until = now_us() + static_cast(seconds) * 1000000; + s.pose_dump_count = 0; + return "Recording your poses for " + std::to_string(seconds) + " s to " + file.string() + ". Skate as you normally would."; +} +std::string edit_direct_connections(Session &s, std::string_view argument) { + const auto enabled = parse_switch(argument, s.prefer_direct); + if (!enabled) return "Use on, off, or toggle for direct connections."; + s.prefer_direct = *enabled; + profile_runtime::set_local_preference("DirectConnections", s.prefer_direct); + return s.prefer_direct ? "Servers that offer it are connected to directly (from the next join)." + : "Every server is reached through Steam's relays (from the next join)."; +} +std::string edit_nametag_distance(Session &s, std::string_view argument) { + float value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || !std::isfinite(value) || + value < 10.f || value > 500.f) + return "Nametag distance is a number of metres from 10 to 500."; + s.nametag_distance = value; + profile_runtime::set_local_values({{"NametagDistance", static_cast(value)}}); + return "Names show within " + std::to_string(static_cast(value)) + " m."; +} +std::string edit_nametag_dots(Session &s, std::string_view argument) { + const auto enabled = parse_switch(argument, s.nametag_dots); + if (!enabled) return "Use on, off, or toggle for nametag dots."; + s.nametag_dots = *enabled; + profile_runtime::set_local_preference("NametagDots", s.nametag_dots); + return s.nametag_dots ? "Far and off-screen players show as dots." : "No dots for far and off-screen players."; +} +std::string edit_nametags_friends(Session &s, std::string_view argument) { + const auto enabled = parse_switch(argument, s.nametags_friends); + if (!enabled) return "Use on, off, or toggle for friends-only nametags."; + s.nametags_friends = *enabled; + profile_runtime::set_local_preference("NametagsFriendsOnly", s.nametags_friends); + return s.nametags_friends ? "Only your Steam friends have nametags." : "Every player has a nametag."; +} +std::string edit_chat_bubbles(Session &s, std::string_view argument) { + const auto enabled = parse_switch(argument, s.chat_bubbles); + if (!enabled) return "Use on, off, or toggle for chat bubbles."; + s.chat_bubbles = *enabled; + s.display_preferences_loaded = true; + apply_nametags(s); + profile_runtime::set_local_preference("ChatBubbles", s.chat_bubbles); + return s.chat_bubbles ? "Chat bubbles show above skaters." : "Chat bubbles hidden."; +} +std::string edit_chat_bubbles_own(Session &s, std::string_view argument) { + const auto enabled = parse_switch(argument, s.chat_bubbles_own); + if (!enabled) return "Use on, off, or toggle for your own chat bubbles."; + s.chat_bubbles_own = *enabled; + profile_runtime::set_local_preference("ChatBubblesOwn", s.chat_bubbles_own); + return s.chat_bubbles_own ? "Your own messages show as bubbles too." + : "Only other players' messages show as bubbles."; +} +std::string edit_chat_bubbles_distance(Session &s, std::string_view argument) { + float value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || !std::isfinite(value) || + value < 5.f || value > 500.f) + return "Chat bubble distance is a number of metres from 5 to 500."; + s.chat_bubbles_distance = value; + profile_runtime::set_local_values({{"ChatBubblesDistance", static_cast(value)}}); + return "Chat bubbles show within " + std::to_string(static_cast(value)) + " m."; +} +std::string edit_chat_bubbles_duration(Session &s, std::string_view argument) { + float value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || !std::isfinite(value) || + value < 1.f || value > 30.f) + return "Chat bubble duration is a number of seconds from 1 to 30."; + s.chat_bubbles_duration = value; + profile_runtime::set_local_values({{"ChatBubblesDuration", static_cast(value)}}); + return "Chat bubbles last " + std::to_string(static_cast(value)) + " seconds."; +} +std::string edit_chat_bubbles_history(Session &s, std::string_view argument) { + int value{}; + const auto parsed = std::from_chars(argument.data(), argument.data() + argument.size(), value); + if (parsed.ec != std::errc{} || parsed.ptr != argument.data() + argument.size() || value < 1 || value > 8) + return "Chat bubble history is a number of lines from 1 to 8."; + s.chat_bubbles_history = value; + profile_runtime::set_local_values({{"ChatBubblesHistory", static_cast(value)}}); + return "Chat bubbles stack up to " + std::to_string(value) + " recent line(s)."; +} std::string kick_player(Session &s, std::string_view argument) { if (s.mode != Mode::host) return "Only the session host can kick players."; const auto split = argument.find(' '); @@ -448,16 +605,25 @@ std::string send_admin(Session &s, std::string text) { if (!send_packet(s, s.host_id, request, true, false)) return "Could not reach the server."; return "Sent to the server."; } +// The Special page's commands. They are a player's own offline as well, where their hoodie and +// board still animate. +bool own_mark_command(std::string_view action) { + return action == "mark-tag" || action == "mark-items" || action == "mark-style"; +} } // namespace bool queue_command(std::string_view action, std::string_view argument, std::string_view password) { - if (launcher::offline_mode()) return false; + if (launcher::offline_mode() && !own_mark_command(action)) return false; if ((action != "host" && action != "host-config" && action != "join" && action != "join-lobby" && action != "join-friend-lobby" && action != "stop" && - action != "distances" && action != "object-placement" && action != "kick" && action != "clear-objects" && - action != "party-overlay" && action != "nametags" && action != "nametag-style" && action != "chat-visible" && action != "chat-filter" && + action != "distances" && action != "object-placement" && action != "object-limit" && action != "kick" && action != "clear-objects" && + action != "nametags" && action != "chat-visible" && action != "chat-filter" && + action != "nametag-distance" && action != "nametag-dots" && action != "nametags-friends" && action != "player-distance" && action != "direct-connections" && action != "pose-dump" && + action != "chat-bubbles" && action != "chat-bubbles-own" && action != "chat-bubbles-distance" && + action != "chat-bubbles-duration" && action != "chat-bubbles-history" && + !own_mark_command(action) && action != "voice" && action != "voice-mute" && action != "voice-volume" && action != "voice-allow" && action != "voice-range" && action != "chat" && action != "ban" && action != "unban" && action != "world-layer-sync" && action != "noclip-allow" && action != "nobail-allow" && action != "boosts-allow" && - action != "tp" && + action != "tuning-enforce" && action != "tp" && action != "tpall" && action != "tphere" && action != "browse" && action != "server" && action != "party") || argument.size() > (action == "host" || action == "host-config" ? 160U : action == "chat" ? 4 * multiplayer_chat_max_bytes @@ -477,7 +643,8 @@ bool queue_command(std::string_view action, std::string_view argument, std::stri return true; } std::string command(std::string_view action, std::string_view argument, std::string_view password) { - if (launcher::offline_mode()) return "Multiplayer is unavailable in offline mode. Start Steam and relaunch ReSkate."; + if (launcher::offline_mode() && !own_mark_command(action)) + return "Multiplayer is unavailable in offline mode. Start Steam and relaunch ReSkate."; const bool configured_host = action == "host-config"; if (configured_host) action = "host"; PrivateRequest input; @@ -488,7 +655,7 @@ std::string command(std::string_view action, std::string_view argument, std::str // their own: the same menu actions, sent to the server. "server" sends // any server console command. if (dedicated_host(s) && (action == "server" || (s.server_admin && - (action == "distances" || action == "object-placement" || action == "voice-allow" || action == "voice-range" || + (action == "distances" || action == "object-placement" || action == "object-limit" || action == "voice-allow" || action == "voice-range" || action == "clear-objects" || action == "kick" || action == "ban" || action == "unban" || action == "world-layer-sync" || action == "noclip-allow" || action == "nobail-allow" || action == "tpall" || action == "tphere" || action == "boosts-allow" || action == "tuning-enforce")))) { @@ -513,8 +680,8 @@ std::string command(std::string_view action, std::string_view argument, std::str if (!result.empty()) add_chat(s, 0, "ReSkate", result); return result; } else if (!dedicated_host(s) && verb == "p") { - // In a lobby everyone is in the one party: party chat is chat. - const auto refused = send_chat(s, rest); + // A lobby's host relays party chat, as a dedicated server does. + const auto refused = send_party_chat(s, rest); if (!refused.empty() && (s.mode == Mode::host || s.mode == Mode::join)) add_chat(s, 0, "ReSkate", refused); return refused.empty() ? "Message sent." : refused; } else if (!dedicated_host(s) && verb == "party") { @@ -623,16 +790,22 @@ std::string command(std::string_view action, std::string_view argument, std::str // Host-only settings check the mode themselves; guests get a refusal. using Setting = std::string (*)(Session &, std::string_view); static constexpr std::pair settings[] = { - {"object-placement", edit_object_placement}, {"kick", kick_player}, + {"object-placement", edit_object_placement}, {"object-limit", edit_object_limit}, {"kick", kick_player}, {"noclip-allow", edit_guest_noclip}, {"nobail-allow", edit_guest_no_bail}, {"boosts-allow", edit_guest_boosts}, {"tuning-enforce", edit_enforce_tuning}, {"score-check", edit_score_check}, {"ban", ban_player}, {"unban", unban_player}, {"clear-objects", clear_guest_objects}, {"world-layer-sync", edit_world_layer_sync}, {"distances", edit_distances}, - {"party-overlay", edit_party_overlay}, {"nametags", edit_nametags}, - {"nametag-style", edit_nametag_style}, - {"chat-visible", edit_chat_visible}, {"chat-filter", edit_chat_filter}}; + {"nametags", edit_nametags}, + {"nametag-distance", edit_nametag_distance}, {"nametag-dots", edit_nametag_dots}, + {"nametags-friends", edit_nametags_friends}, {"player-distance", edit_player_distance}, {"direct-connections", edit_direct_connections}, {"pose-dump", start_pose_dump}, + {"mark-tag", edit_own_tag}, {"mark-items", edit_own_items}, {"mark-style", edit_mark_style}, + {"chat-visible", edit_chat_visible}, {"chat-filter", edit_chat_filter}, + {"chat-bubbles", edit_chat_bubbles}, {"chat-bubbles-own", edit_chat_bubbles_own}, + {"chat-bubbles-distance", edit_chat_bubbles_distance}, + {"chat-bubbles-duration", edit_chat_bubbles_duration}, + {"chat-bubbles-history", edit_chat_bubbles_history}}; for (const auto &[name, edit] : settings) if (action == name) { s.status = edit(s, argument); @@ -671,8 +844,12 @@ std::string command(std::string_view action, std::string_view argument, std::str return s.lobbies.status().browser; } if (action == "join-lobby" || action == "join-friend-lobby") { - if (s.mode != Mode::off) { - s.status = "Leave your current session before joining another lobby."; + // A guest can hop straight to a server or lobby they pick in the browser: they leave + // this one first. A host ends everyone's session by leaving, so they end it themselves; + // and a Steam friend's join offer, which can arrive unasked, never replaces a session. + if (s.mode != Mode::off && (s.mode != Mode::join || action != "join-lobby")) { + s.status = s.mode == Mode::join ? "Leave your current session before joining another lobby." + : "End your session before joining another lobby."; publish(s); return s.status; } @@ -682,10 +859,27 @@ std::string command(std::string_view action, std::string_view argument, std::str const auto result = std::from_chars(argument.data(), argument.data() + argument.size(), id); if (result.ec != std::errc{} || result.ptr != argument.data() + argument.size() || !id) return "Invalid lobby selection. Refresh the browser."; - // A dedicated server's row carries its join code. + if (s.mode == Mode::join) { + if (id == s.joined_public_lobby) return "You are already there."; + stop(s, "Switching servers..."); + } + // A dedicated server's row carries its join code. It is a public listing like a + // lobby's: friends are told the player is there, and can follow them in. if (const auto *server = s.servers.find(id)) { const auto code = server->code; - return command("join", code, input.password); + auto reply = command("join", code, input.password); + if (s.mode == Mode::join) { + s.joined_public_lobby = id; + publish(s); + } + return reply; + } + // A friend on a server this game has not listed yet: only the list has its code. + if (game_server_steam_id(id)) { + s.servers.refresh(now_us()); + s.status = "That server is not in your server list yet. Open the server browser and join it there."; + publish(s); + return s.status; } if (!s.transport.open()) return s.transport.status().detail; @@ -717,6 +911,8 @@ std::string command(std::string_view action, std::string_view argument, std::str publish(s); return s.status; }; + // Joining is for the host or the server to refuse (a server may opt out of the bans). + if (action == "host" && reskate_banned(s.transport.status().local_id)) return refuse(std::string(banned_notice)); if (action == "host") { const auto space = argument.find(' '); if (space != std::string_view::npos) { @@ -755,6 +951,7 @@ std::string command(std::string_view action, std::string_view argument, std::str invitation = parse_invite(argument); if (!invitation) return refuse("Invalid join code. Paste the complete SteamID-session code from the host."); + if (blocked_server(invitation->steam_id)) return refuse(std::string(blocked_server_notice)); } stop(s, "Starting multiplayer..."); s.chat.clear(); // A new session starts with an empty chat. @@ -797,6 +994,7 @@ std::string command(std::string_view action, std::string_view argument, std::str apply_distances(s, remembered.distances); s.voice_range = remembered.voice_range; apply_object_placement(s, remembered.placement); + apply_object_limit(s, remembered.object_limit); apply_guest_tools(s, remembered.guest_noclip, remembered.guest_no_bail, remembered.guest_boosts); s.enforce_tuning = remembered.enforce_tuning; s.score_check = remembered.score_check; @@ -808,7 +1006,22 @@ std::string command(std::string_view action, std::string_view argument, std::str save_host_preferences(s); s.status = "Hosting for up to " + std::to_string(capacity) + " players at " + std::to_string(s.tps) + " TPS."; } else { - if (!s.transport.join(invitation->steam_id)) { + // A server that listens on its own address is connected to directly, with Steam's + // relays as the fallback (steam_transport.h). The server sees the address of a + // player who connects this way, as any dedicated server does, so the player may + // turn it off (direct-connections). Preferences load with the first session frame, + // so read this one here. + const bool direct_allowed = s.display_preferences_loaded + ? s.prefer_direct + : profile_runtime::local_preference("DirectConnections").value_or(true); + std::uint32_t direct_ip{}; + std::uint16_t direct_port{}; + if (const auto *listed = s.servers.find(invitation->steam_id); + listed && direct_allowed && listed->direct_ip && listed->direct_port > 0) { + direct_ip = listed->direct_ip; + direct_port = static_cast(listed->direct_port); + } + if (!s.transport.join(invitation->steam_id, direct_ip, direct_port)) { s.status = s.transport.status().detail; publish(s); return s.status; diff --git a/Extension/Multiplayer/Session/session_internal.h b/Extension/Multiplayer/Session/session_internal.h index 75e1c27..ad9b297 100644 --- a/Extension/Multiplayer/Session/session_internal.h +++ b/Extension/Multiplayer/Session/session_internal.h @@ -1,6 +1,7 @@ #pragma once #include "Engine/Game/Multiplayer/chat_rate.h" #include "session.h" +#include "Extension/Multiplayer/developer_identity.h" #include "Extension/Customization/developer_hoodie.h" #include "Extension/Customization/developer_board.h" #include "Extension/Multiplayer/Remote/native_skater.h" @@ -10,15 +11,20 @@ #include "Extension/Multiplayer/Steam/steam_lobbies.h" #include "Extension/Multiplayer/Steam/steam_server_browser.h" #include "room.h" +#include "party_book.h" #include "Extension/Multiplayer/Net/delta_codec.h" +#include "Extension/Multiplayer/Net/pose_batch.h" +#include "Extension/Multiplayer/Net/sound_codec.h" #include "password.h" #include "client_timing.h" #include "monotonic_clock.h" #include +#include #include #include #include #include +#include #include #include #include @@ -43,7 +49,8 @@ struct ChatBudget { }; // Throwdown messages per sender per 5 s: offers, joins, starts and ~4 scores a second. inline constexpr unsigned throwdown_burst = 60; -// The party number of a listen host's lobby, which is always one party. +// The party the echo test shows its mirrored player in (a real lobby has none until its +// players form one). inline constexpr std::uint32_t lobby_party = 1; struct Peer { // Guests: this owner's layout as relayed by the host. Host: the owner's own @@ -86,14 +93,17 @@ struct Peer { // One entry per pose source relayed or sent to this recipient (at most max_players). std::vector pose_delivery; std::uint64_t connected_at{}, last_packet{}, last_cosmetic_apply{}; + // A skater removed for lack of poses is not spawned again before this: a spawn is a heavy + // native pass, and a player's poses may stop and start again and again. + std::uint64_t next_spawn{}; + // Near enough to be shown as a skater; and whether a pose has placed them yet. + bool shown_wanted = true, placed{}; ReceiveBudget budget; PoseBuffer poses; Pose render_pose; - // Far from the local skater and the camera, or out of the camera's view, render_pose is - // sampled every far_interval (0: every frame) and the skater keeps it in between. + // Far from the local skater and the camera, render_pose is sampled every far_interval + // (0: every frame) and the skater keeps it in between. std::uint64_t far_interval{}, next_far_sample{}; - // far_interval is only for being out of view: coming into view samples at once. - bool far_for_view{}; AppearanceBuffer appearance; // Bumped per accepted outfit; render compares it with the one its actor wears // (0 after a spawn) to spread native recipe applies over frames. @@ -104,8 +114,18 @@ struct Peer { std::uint32_t voice_sequence{}; bool received_voice{}; VoiceBudget voice_budget; + OutfitBudget outfit_budget; + SoundBudget sound_budget; ChatRate chat_rate; - ChatBudget throwdown_budget; + // Chat proofs (session_receive.cpp): the copies of their lines this player sent us over + // their own Steam connection, and the lines the host passed on as theirs that wait for one. + struct ChatProof { + std::uint32_t sequence{}; + std::string text; + std::uint64_t at{}; + }; + std::vector chat_proofs, chat_waiting; + ChatBudget throwdown_budget, party_budget; // Host: how the guest's mods change trick scoring, as they reported it (Engine/Vfs/mod_scoring.h): // nothing until the report arrives, 0 for the game's own. member.scoring is the verdict. std::optional scoring; @@ -133,6 +153,7 @@ struct Session { std::uint64_t next_publish{}, last_client_log{}, next_party_update{}; MultiplayerDistances distances; ObjectPlacement object_placement = ObjectPlacement::everyone; + unsigned object_limit{}; // objects each player may have placed; 0: no limit // What guests may use: the host's choice, or the host's roster for a guest. bool guest_noclip = true, guest_no_bail = true, guest_boosts = true; // Host: guests skate with its physics tuning. Guest: the host's roster says so (a @@ -154,6 +175,13 @@ struct Session { std::optional> sent_tuning; std::vector tuning_packet; std::uint64_t next_tuning_check{}; + // The physics the tuning does not carry (Engine/Game/Multiplayer/session_physics.h). + // Guest: the host's, once it sent them. Host: which of its own were last sent, the packet + // that carried them (for players who join later) and when to look again. + std::optional> host_extras; + std::uint64_t sent_extras{}; + std::vector extras_packet; + std::uint64_t next_extras_check{}; std::uint8_t server_votes{}; // guest of a dedicated server: the votes it runs // Host: bumped per "delete all guest objects". Guest: the last value seen // (unset until the first roster) and whether a local wipe is outstanding. @@ -162,6 +190,8 @@ struct Session { bool force_world_layers{}; // Players the host kicked. They cannot reconnect until the session ends. std::set banned; + // Host: Steam IDs whose attempts to join keep failing wait longer each time (room.h). + JoinBackoff join_backoff; // Players banned for good (every session this PC hosts), from the local profile. std::vector bans; bool bans_loaded{}; @@ -182,10 +212,16 @@ struct Session { // and the server's voice range from it. bool server_admin{}; float roster_voice_range = default_voice_range; + // The colours of the dedicated server's own chat lines, as its roster gives them. + std::uint32_t server_chat_badge = default_server_chat_badge, server_chat_text = default_server_chat_text; // The server's ban list, as sent to us while we are one of its admins. std::vector server_bans; std::uint32_t server_ban_total{}; std::vector server_maps; // level assets the dedicated server allows + std::vector server_map_pool; // admins: the pool's assets in rotation order (empty: every map) + unsigned server_map_rotation{}; // minutes per map (0: off) + std::string map_label; // the host's name for join_destination (may be empty) + std::string leave_notice; // take_leave_notice() std::uint64_t joined_public_lobby{}; std::array peers; // Players take the lowest free slots, so every one sits below this mark @@ -200,9 +236,35 @@ struct Session { bool travelling{}, host_world_ready = true; unsigned capacity = max_players; // Local display preferences, loaded once from the profile. Never sent to peers. - bool party_overlay = true, nametags = true, chat_visible = true, display_preferences_loaded{}; - bool custom_nametags = true; // ReSkate's nametags instead of the game's (Hud/custom_nametags.h) + bool nametags = true, chat_visible = true, display_preferences_loaded{}; + float nametag_distance = 120.f; // names within this many metres, dots past it + // Only players within this many metres have a skater (session.cpp); checked a few times a second. + float player_distance = 120.f; + bool prefer_direct = true; // straight to servers that offer it, not through the relays + // Poses from a dedicated server (pose_batch.h): each player's stream as this game holds it, + // and which of the server's messages were read in full, which it is told once a tick. + std::unordered_map pose_streams; + pose_batch::Ack pose_ack; + bool pose_ack_due{}; + // This game's own poses go to a dedicated server the same way: what was sent and what the + // server acked, and the last poses sent, to build the next on. + pose_batch::Sender pose_upload; + std::deque own_poses; + // Skaters' sound to and from a dedicated server (sound_codec.h). + std::unordered_map sound_streams; + sound_codec::Sender sound_upload; + // pose-dump: this player's own poses, as encoded for sending, written to a file for a + // while. Real poses to measure encodings against (tools/research). + std::ofstream pose_dump; + std::uint64_t pose_dump_until{}; + std::size_t pose_dump_count{}; + std::uint64_t next_shown_rank{}; + bool nametag_dots = true, nametags_friends{}; bool chat_filter = true; // bad words in chat show as **** (Engine/Core/Text/word_filter.h) + // Chat bubbles above each skater's head (Hud/custom_nametags.h, nametag_overlay.cpp). + bool chat_bubbles = true, chat_bubbles_own{}; + float chat_bubbles_distance = 40.f, chat_bubbles_duration = 5.f; + int chat_bubbles_history = 3; bool game_menu{}; // a game menu is up or the game's UI is hidden: no chat on screen bool hooks_prepared{}; // the remote-player hooks were installed (once per process) // Host settings remembered between sessions and game restarts. @@ -212,6 +274,7 @@ struct Session { std::string lobby_name; MultiplayerDistances distances; ObjectPlacement placement = ObjectPlacement::everyone; + unsigned object_limit = default_object_limit; float voice_range = default_voice_range; bool guest_noclip = true, guest_no_bail = true, guest_boosts = true; bool enforce_tuning = true; @@ -257,8 +320,11 @@ struct Session { std::uint64_t object_owners_at{}; // Throwdown messages received since the last relay tick: sender, encoded message. std::vector>> throwdown_inbox; - // Parties (session_party.cpp): the local player's own roster entry, and the invites a - // dedicated server passed on (newest last). A listen host's lobby is always one party. + // Parties (session_party.cpp): the local player's own roster entry, and the invites the + // host passed on (newest last). Whoever hosts owns the parties: a dedicated server, or in + // a lobby the host's game, in `parties`. Nobody is in one until they form it. + PartyBook parties; + std::uint64_t parties_revision{}; // the book's revision the last roster was built from std::uint32_t local_party{}; bool local_party_leader{}, local_party_open{}; bool local_speeding{}; // the dedicated server flagged our game speed: no linked activities @@ -286,6 +352,33 @@ template void each_active_peer(Session &s, F &&fn) { } // A guest whose host is a dedicated server (a Steam game server, not a player). inline bool dedicated_host(const Session &s) { return s.mode == Mode::join && game_server_steam_id(s.host_id); } +// Whether Steam itself vouches for this player's identity to this PC: a host's guests and a +// guest's host are connected directly, and so is another guest once the direct handshake is +// done. Anyone else is known only from the host's roster, which a host can fill as it likes, +// so what rests on who a player is (the developer and friend marks) waits for this. An +// official server is the exception: Steam vouches for it to this PC and for each player to +// it, and it is ours, so its roster is taken at its word. On a busy one a player is +// connected directly to only the few nearest, and the rest would show no tag. +inline bool steam_vouched(const Session &s, const Peer &peer) { + return s.mode != Mode::join || peer.member.id == s.host_id || peer.direct_ready || official_server(s.host_id); +} +// Whether a player shows the tag the backend gives them, and whether the items that come with +// it animate: not until their appearance has arrived, and not when it says they turned that +// off (Appearance::hide_tag, Appearance::hide_items). +inline bool shows_tag(const Peer &peer) { + const auto &look = peer.appearance.value(); + return look && !look->hide_tag; +} +inline bool shows_items(const Peer &peer) { + const auto &look = peer.appearance.value(); + return look && !look->hide_items; +} +// How a player has each of their marked cosmetics animate: what their appearance says. +inline const MarkStyles &mark_styles(const Peer &peer) { + static const MarkStyles standard{}; + const auto &look = peer.appearance.value(); + return look ? look->marks : standard; +} Peer *find_peer(Session &s, std::uint64_t id); unsigned player_count(const Session &s); void reset_peer(Session &s, std::size_t slot); @@ -302,15 +395,20 @@ void save_bans(const Session &s); bool is_banned(Session &s, std::uint64_t id); // Re-reads friend_ids when the Steam social snapshot has changed. void refresh_friends(Session &s); -void add_chat(Session &s, std::uint64_t sender, std::string name, std::string text, bool local = false); -// A player's role colour and badge ("Dev", "Admin", "Host", "Friend" or none), shown in chat +// `marks`: whether the line may show the badge the backend gives its sender (player_role). +void add_chat(Session &s, std::uint64_t sender, std::string name, std::string text, bool local = false, bool marks = true); +// The colour and badge of one of the backend's categories. +std::pair mark_role(IdentityList list); +// A player's role colour and badge ("Dev", "Staff", "Creator", "Centrix", "Homie", "Admin", "Host", "Friend" or none), shown in chat // and on their nametag. `local`: the local player. -std::pair player_role(Session &s, std::uint64_t id, bool local); +std::pair player_role(Session &s, std::uint64_t id, bool local, bool marks = true); // Sends one line from this player; returns why not when it cannot. std::string send_chat(Session &s, std::string_view typed); // A "/" command for a dedicated server (votes, and any server command for its admins): sent // like chat but never shown as a line; the server answers in chat. std::string send_chat_command(Session &s, std::string_view typed); +// "/p " in a lobby: one line for the local player's party only, relayed by the host. +std::string send_party_chat(Session &s, std::string_view typed); // The "/" commands this session offers (the chat overlay lists them as the player types "/"). std::vector chat_commands(const Session &s); // One encoded throwdown message from this player to everyone else (through the host). @@ -352,8 +450,18 @@ std::uint64_t party_leader(const Session &s); // the local party's leader, 0 wit void set_local_party(Session &s, const Member &local); void receive_party(Session &s, const Packet &p, std::uint64_t now); void expire_party_invites(Session &s, std::uint64_t now); -// Sends a request to the dedicated server; returns why it can't, or empty. +// Sends a request to whoever owns the parties (a lobby's host answers its own at once); +// returns why it can't, or empty. std::string send_party_request(Session &s, PartyAction action, std::uint64_t player); +// A lobby's host: answers `from`'s party request, as a dedicated server does. +void host_party_request(Session &s, std::uint64_t from, PartyAction action, std::uint64_t player, std::uint64_t now); +// A lobby's host: `from`'s "/p" line, relayed to the rest of their party only. +void host_party_chat(Session &s, std::uint64_t from, std::string_view text, std::uint64_t now); +// A lobby's host, every network tick: players who left are out of their parties, lapsed +// invites are withdrawn, and a change sends a new roster. +void tick_host_parties(Session &s, std::uint64_t now); +// A lobby's host: each roster member's party as the book has it. +void fill_roster_parties(Session &s, std::vector &members); bool accept_data(Peer &peer, const Packet &p, std::uint64_t now); // The same, moving a pose into the playback buffer (the packet's pose is left empty). bool accept_data(Peer &peer, Packet &&p, std::uint64_t now); @@ -361,6 +469,8 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now); // session_commands.cpp void apply_distances(Session &s, const MultiplayerDistances &distances); void apply_object_placement(Session &s, ObjectPlacement policy); +// The session's limit, and this game's own share of it: none while hosting or as a server's admin. +void apply_object_limit(Session &s, unsigned limit); // Stores what guests may use and applies it to the local player (the host and a dedicated // server's admins are exempt). void apply_guest_tools(Session &s, bool noclip, bool no_bail, bool boosts); diff --git a/Extension/Multiplayer/Session/session_party.cpp b/Extension/Multiplayer/Session/session_party.cpp index 79cb2b4..759b7f0 100644 --- a/Extension/Multiplayer/Session/session_party.cpp +++ b/Extension/Multiplayer/Session/session_party.cpp @@ -2,13 +2,61 @@ #include "Extension/Multiplayer/Hud/native_party.h" #include -// Parties. A listen host's lobby is always one party, led by the host; on a dedicated server -// the server owns the parties (Server/server_party.cpp) and every roster says who is in which. -// The native party UI, map markers and coop challenges read them from here. +// Parties. Whoever hosts owns them, in a PartyBook: a dedicated server (Server/server_party.cpp) +// or, in a lobby, the host's game (the second half of this file, which answers requests the way +// the server does). Nobody is in a party until they form one, and every roster says who is in +// which. The native party UI, map markers and coop challenges read them from here. namespace dingosdk::multiplayer::session_detail { namespace { -// An invite the server passed on lapses there after a minute; drop ours a little later. +// An invite the host passed on lapses there after a minute; drop ours a little later. constexpr std::uint64_t invite_lifetime_us = 65000000; +std::string_view refusal(PartyBook::Result result) { + switch (result) { + case PartyBook::Result::ok: return {}; + case PartyBook::Result::self: return "That's you."; + case PartyBook::Result::same_party: return "You're already in a party together."; + case PartyBook::Result::full: return "That party is full."; + case PartyBook::Result::not_leader: return "Only the party leader can do that."; + case PartyBook::Result::not_member: return "You're not in a party with them."; + case PartyBook::Result::no_invite: return "That invite has expired."; + case PartyBook::Result::closed: return "That party is invite-only."; + case PartyBook::Result::no_party: return "They're not in a party."; + case PartyBook::Result::busy: return "They already have too many party invites."; + case PartyBook::Result::renewed: return "They already have your party invite; it stays open for another minute."; + } + return "That can't be done."; +} +std::string player_name(Session &s, std::uint64_t id) { + if (const auto *peer = find_peer(s, id); peer && !peer->member.name.empty()) return peer->member.name; + return s.transport.name(id); +} +// The host itself, or a guest it has admitted. +bool in_lobby(Session &s, std::uint64_t id) { + if (id == s.transport.status().local_id) return true; + const auto *peer = find_peer(s, id); + return peer && peer->handshaken; +} +// A line only `to` reads: in the host's own chat, or sent to the guest as the host's notice. +void tell(Session &s, std::uint64_t to, std::string_view text, std::uint64_t now) { + if (to == s.transport.status().local_id) return add_chat(s, 0, "ReSkate", std::string(text)); + auto notice = packet(s, PacketKind::admin, now); + notice.text = clean_chat_text(text); + send_packet(s, to, notice, true, false); +} +// An invite for `to`, or word that one can no longer be accepted. +void notify(Session &s, std::uint64_t to, PartyAction action, std::uint64_t player, std::uint64_t now) { + auto notice = packet(s, PacketKind::party, now); + notice.party_action = action; + notice.party_player = player; + if (to == s.transport.status().local_id) receive_party(s, notice, now); + else send_packet(s, to, notice, true, false); +} +void party_notice(Session &s, std::uint32_t party, std::string_view text, std::uint64_t now, std::uint64_t except = 0) { + const auto *details = s.parties.party(party); + if (!details) return; + for (const auto member : details->members) + if (member != except && in_lobby(s, member)) tell(s, member, text, now); +} } // namespace std::uint32_t party_of(const Session &s, std::uint64_t id) { @@ -68,15 +116,169 @@ void expire_party_invites(Session &s, std::uint64_t now) { if (s.party_invites.size() != before) ++s.party_revision; } std::string send_party_request(Session &s, PartyAction action, std::uint64_t player) { - if (!dedicated_host(s)) return "Everyone in a lobby is in one party with the host."; + if (s.mode != Mode::host && s.mode != Mode::join) return "Parties need a multiplayer session."; if (!valid_party_request(action, player) || action == PartyAction::invited || action == PartyAction::withdrawn) return "That party request isn't valid."; if (player && !find_peer(s, player)) return "That player is not in the session."; const auto now = now_us(); + if (s.mode == Mode::host) { + host_party_request(s, s.transport.status().local_id, action, player, now); + return {}; + } auto request = packet(s, PacketKind::party, now); request.party_action = action; request.party_player = player; - if (!send_packet(s, s.host_id, request, true, false)) return "Could not reach the server."; + if (!send_packet(s, s.host_id, request, true, false)) + return dedicated_host(s) ? "Could not reach the server." : "Could not reach the host."; return {}; } + +// --------------------------------------------------------------------------- +// A lobby's host. The same answers, word for word, as a dedicated server gives +// (Server/server_party.cpp), so a party works the same wherever it is formed. +void host_party_request(Session &s, std::uint64_t me, PartyAction action, std::uint64_t player, std::uint64_t now) { + if (s.mode != Mode::host || !in_lobby(s, me)) return; + if (player && !in_lobby(s, player)) return tell(s, me, "That player is not in the lobby.", now); + const auto my_name = player_name(s, me); + const auto their_name = player ? player_name(s, player) : std::string{}; + auto &parties = s.parties; + using R = PartyBook::Result; + auto result = R::ok; + switch (action) { + case PartyAction::invite: + result = parties.invite(me, player, now); + if (result == R::ok) { + notify(s, player, PartyAction::invited, me, now); + tell(s, me, "Invited " + their_name + " to your party.", now); + } + break; + case PartyAction::accept: + case PartyAction::join: { + const auto before = parties.party_of(me); + result = action == PartyAction::accept ? parties.accept(me, player, now) : parties.join(me, player, now); + if (result == R::ok) { + const auto party = parties.party_of(me); + if (before && before != party) party_notice(s, before, my_name + " left the party.", now); + party_notice(s, party, my_name + " joined the party.", now, me); + tell(s, me, "You joined " + their_name + "'s party.", now); + } else if (result == R::closed) { + // Ask the leader instead: they can invite. + const auto *details = parties.party(parties.party_of(player)); + if (details && in_lobby(s, details->leader)) { + tell(s, details->leader, my_name + " would like to join your party. Invite them from their player card " + "or with /party invite " + my_name, now); + return tell(s, me, "That party is invite-only; its leader was asked to invite you.", now); + } + } + break; + } + case PartyAction::decline: + result = parties.decline(me, player); + if (result == R::ok) tell(s, player, my_name + " declined your party invite.", now); + break; + case PartyAction::leave: { + const auto party = parties.party_of(me); + result = parties.leave(me); + if (result == R::ok) { + party_notice(s, party, my_name + " left the party.", now); + tell(s, me, "You left the party.", now); + } + break; + } + case PartyAction::kick: { + const auto party = parties.party_of(me); + result = parties.kick(me, player); + if (result == R::ok) { + tell(s, player, "You were removed from the party.", now); + party_notice(s, party, their_name + " was removed from the party.", now); + } + break; + } + case PartyAction::promote: + result = parties.promote(me, player); + if (result == R::ok) party_notice(s, parties.party_of(me), their_name + " now leads the party.", now); + break; + case PartyAction::open: + case PartyAction::close: { + const auto *before = parties.party(parties.party_of(me)); + const bool was_open = before && before->open; + result = parties.set_open(me, action == PartyAction::open); + if (result == R::ok && was_open != (action == PartyAction::open)) + party_notice(s, parties.party_of(me), action == PartyAction::open ? "The party is open: anyone can join." + : "The party is invite-only.", now); + break; + } + case PartyAction::invited: + case PartyAction::withdrawn: return; // the host's own notices + } + if (result != R::ok) tell(s, me, refusal(result), now); + if (parties.revision() != s.parties_revision) s.roster_dirty = true; +} +void host_party_chat(Session &s, std::uint64_t from, std::string_view text, std::uint64_t now) { + if (s.mode != Mode::host) return; + const auto party = s.parties.party_of(from); + if (!party) return tell(s, from, "You're not in a party.", now); + const auto line = clean_chat_text(text); + if (line.empty()) return tell(s, from, "/p ", now); + // Relayed as the sender's own chat line, only to the rest of their party. + const auto local = s.transport.status().local_id; + auto message = packet(s, PacketKind::chat, now); + message.text = clean_chat_text("[Party] " + line); + if (from != local) { + const auto *sender = find_peer(s, from); + if (!sender) return; + message.source = from; + message.epoch = sender->member.epoch; + } + for (const auto member : s.parties.party(party)->members) { + if (member == from) continue; + if (member == local) add_chat(s, from, player_name(s, from), message.text); + else send_packet(s, member, message, true, false); + } +} +void tick_host_parties(Session &s, std::uint64_t now) { + if (s.mode != Mode::host) return; + auto &parties = s.parties; + // A player who left the lobby is out of their party, and their invites go with them. + std::vector gone; + const auto check = [&](std::uint64_t id) { + if (!in_lobby(s, id) && std::find(gone.begin(), gone.end(), id) == gone.end()) gone.push_back(id); + }; + for (const auto &[id, details] : parties.parties()) + for (const auto member : details.members) check(member); + for (const auto &invite : parties.invites()) { check(invite.from); check(invite.to); } + for (const auto id : gone) { + const auto party = parties.party_of(id); + const auto name = player_name(s, id); + parties.remove(id); + if (party) party_notice(s, party, name + " left the lobby.", now); + } + for (const auto &lapsed : parties.expire(now)) { + if (in_lobby(s, lapsed.to)) notify(s, lapsed.to, PartyAction::withdrawn, lapsed.from, now); + if (in_lobby(s, lapsed.from) && in_lobby(s, lapsed.to)) + tell(s, lapsed.from, "Your party invite to " + player_name(s, lapsed.to) + " expired.", now); + } + for (const auto &withdrawn : parties.take_withdrawn()) + if (in_lobby(s, withdrawn.to)) notify(s, withdrawn.to, PartyAction::withdrawn, withdrawn.from, now); + if (parties.revision() != s.parties_revision) s.roster_dirty = true; +} +void fill_roster_parties(Session &s, std::vector &members) { + for (auto &m : members) { + const auto party = s.parties.party_of(m.id); + const auto *details = s.parties.party(party); + m.party = party; + m.party_leader = details && details->leader == m.id; + m.party_open = m.party_leader && details->open; + } + // The book holds only players in the lobby (tick_host_parties), so this only guards the + // roster's rules: every listed party has two members and one leader. + for (auto &m : members) + if (m.party && std::count_if(members.begin(), members.end(), [&](const Member &o) { return o.party == m.party; }) < 2) + m.party = 0, m.party_leader = m.party_open = false; + for (auto &m : members) + if (m.party && std::none_of(members.begin(), members.end(), + [&](const Member &o) { return o.party == m.party && o.party_leader; })) + m.party_leader = true; // the first listed member of a party missing its leader + s.parties_revision = s.parties.revision(); +} } // namespace dingosdk::multiplayer::session_detail diff --git a/Extension/Multiplayer/Session/session_receive.cpp b/Extension/Multiplayer/Session/session_receive.cpp index 28f85b8..ffabd9a 100644 --- a/Extension/Multiplayer/Session/session_receive.cpp +++ b/Extension/Multiplayer/Session/session_receive.cpp @@ -60,11 +60,143 @@ void receive_cosmetics(Session &s, const NativeFrame &local, std::uint64_t now) } } } +// Chat proofs. A guest takes chat from the host alone, which passes every other guest's lines +// on, and a host can pass on what it likes under any of its players' names. That is no worse +// than a name on a line, until the line shows a badge of the backend's (Dev, Creator, Homie): +// then it is a line a developer never wrote, with their badge on it. So such a guest also +// sends each line straight to the players Steam connects them to (send_chat), and here a line +// the host passed on as theirs shows their badge only when that copy says the same. One +// without it waits a moment for it, then shows as any other player's would: from an older +// build, a party line (which the host writes), or not theirs at all. +constexpr std::uint64_t chat_proof_wait = 500000, chat_proof_life = 10000000; +constexpr std::size_t chat_proofs_kept = 8; +// Whether a line the host passed on as this player's would show a badge that rests on who +// they are: another guest's, known to Steam, on one of the backend's lists and showing it. +bool owes_proof(const Session &s, const Peer &sender) { + // Their own copy only comes over a direct connection: without one there is none to wait for. + return s.mode == Mode::join && sender.member.id != s.host_id && sender.direct_ready && shows_tag(sender) && + identity_mark(sender.member.id); +} +std::string chat_name(Session &s, const Peer &sender) { + return sender.member.name.empty() ? s.transport.name(sender.member.id) : sender.member.name; +} +// Shows the lines that have waited long enough (or all of them), without the badge. +void release_chat(Session &s, Peer &sender, std::uint64_t now, bool all) { + auto &waiting = sender.chat_waiting; + auto line = waiting.begin(); + for (; line != waiting.end() && (all || now < line->at || now - line->at >= chat_proof_wait); ++line) + add_chat(s, sender.member.id, chat_name(s, sender), std::move(line->text), false, false); + waiting.erase(waiting.begin(), line); +} +// The host's copy of a line: shown at once when the sender's own copy is here, else it waits. +void relayed_chat(Session &s, Peer &sender, const Packet &p, std::uint64_t now) { + auto &proofs = sender.chat_proofs; + const auto proof = std::find_if(proofs.begin(), proofs.end(), [&](const auto ©) { + return copy.sequence == p.sequence && copy.text == p.text && now >= copy.at && now - copy.at <= chat_proof_life; + }); + const bool proven = proof != proofs.end(); + if (proven) proofs.erase(proof); + // The sender's copies come in the order of their lines: with this one's here, or none + // owed for it, nothing is on its way for the lines before it. + if (proven || p.text.starts_with("[Party] ")) { + release_chat(s, sender, now, true); + add_chat(s, sender.member.id, chat_name(s, sender), p.text, false, proven); + return; + } + if (sender.chat_waiting.size() >= chat_proofs_kept) release_chat(s, sender, now, true); + sender.chat_waiting.push_back({p.sequence, p.text, now}); +} +// The sender's own copy of a line, over their own connection. Never shown by itself. +void proven_chat(Session &s, Peer &sender, const Packet &p, std::uint64_t now) { + if (!owes_proof(s, sender)) return; + auto &waiting = sender.chat_waiting; + const auto held = std::find_if(waiting.begin(), waiting.end(), [&](const auto &line) { + return line.sequence == p.sequence && line.text == p.text; + }); + if (held != waiting.end()) { + for (auto line = waiting.begin(); line != held; ++line) + add_chat(s, sender.member.id, chat_name(s, sender), std::move(line->text), false, false); + add_chat(s, sender.member.id, chat_name(s, sender), std::move(held->text)); + waiting.erase(waiting.begin(), held + 1); + return; + } + auto &proofs = sender.chat_proofs; + std::erase_if(proofs, [&](const auto ©) { return now < copy.at || now - copy.at > chat_proof_life; }); + if (proofs.size() >= chat_proofs_kept) proofs.erase(proofs.begin()); + proofs.push_back({p.sequence, p.text, now}); +} // A hitch leaves a backlog of pose updates, all of which would be decoded in one frame. // Every pose delta references a reliable baseline, never another delta, so older deltas // of one stream in a batch can be dropped undecoded. Keep the newest ones playback may // still interpolate between (its delay plus a frame each side) among those whose baseline // is already here; baselines and every other kind are always read. +// The poses in one message from a dedicated server (pose_batch.h), each as the packet it would +// have been had it come alone. A message read in full is noted for the ack; one with a pose +// this game could not rebuild (it never got what that pose builds on) is not, so the server +// goes on building on what this game does hold. +std::vector unpack_poses(Session &s, std::span bytes) { + std::vector out; + const auto batch = pose_batch::read(bytes); + if (!batch || batch->world != static_cast(s.world) || batch->map != static_cast(s.map)) return out; + // Streams of players who left are dropped once there are many more than there are players. + if (s.pose_streams.size() > 2 * max_players) + std::erase_if(s.pose_streams, [&](const auto &stream) { return !find_peer(s, stream.second.source); }); + bool complete = true; + out.reserve(batch->entries.size()); + for (const auto &entry : batch->entries) { + auto rebuilt = pose_batch::rebuild(s.pose_streams, entry); + if (!rebuilt) { + complete = false; + continue; + } + if (rebuilt->repeat) continue; + Packet p; + p.kind = PacketKind::pose; + p.session = s.secret; + p.map = s.map; + p.world = s.world; + p.source = rebuilt->source; + p.epoch = rebuilt->epoch; + p.sequence = rebuilt->sequence; + p.time_us = rebuilt->time_us; + p.player_collision = rebuilt->collision; + p.pose_interval_us = rebuilt->rate == pose_batch::Rate::full ? multiplayer_pose_interval(s.tps) + : rebuilt->rate == pose_batch::Rate::half ? 100000U + : 200000U; + p.pose = pose_codec::restore(rebuilt->pose); + out.push_back(std::move(p)); + } + if (complete) { + s.pose_ack.note(batch->number); + s.pose_ack_due = true; + } + return out; +} +// The skaters' sound in one message from a dedicated server (sound_codec.h), each player's as +// the packet it would have been had it come alone. +std::vector unpack_sound(Session &s, std::span bytes) { + std::vector out; + if (s.sound_streams.size() > 2 * max_players) + std::erase_if(s.sound_streams, [&](const auto &stream) { return !find_peer(s, stream.second.source); }); + auto heard = sound_codec::read(s.sound_streams, bytes, s.world, s.map); + if (!heard) return out; + out.reserve(heard->size()); + for (auto &one : *heard) { + if (!valid_audio_batch(one.samples)) continue; + Packet p; + p.kind = PacketKind::audio; + p.session = s.secret; + p.map = s.map; + p.world = s.world; + p.source = one.source; + p.epoch = one.epoch; + p.sequence = one.sequence; + p.time_us = one.time_us; + p.audio = std::move(one.samples); + out.push_back(std::move(p)); + } + return out; +} std::vector stale_pose_deltas(Session &s, const std::vector &messages) { std::vector stale(messages.size()); struct Stream { @@ -74,6 +206,8 @@ std::vector stale_pose_deltas(Session &s, const std::vector streams; for (auto i = messages.size(); i-- > 0;) { const auto &bytes = messages[i].bytes; + // (A dedicated server sends none of these: what it sends that starts the same is sound.) + if (dedicated_host(s) && messages[i].peer == s.host_id) continue; // Sparse pose patches (delta_codec.cpp): "RMS1"/"RMS2", source at 4, kind at 20. if (bytes.size() <= 34 || bytes[0] != 'R' || bytes[1] != 'M' || bytes[2] != 'S' || (bytes[3] != '1' && bytes[3] != '2')) @@ -138,6 +272,7 @@ void apply_roster(Session &s, const Packet &p, std::uint64_t now) { for (auto &peer : active_peers(s)) peer.pose_delivery = {}; } apply_object_placement(s, p.object_placement); + apply_object_limit(s, p.object_limit); // after server_admin, which exempts an admin apply_guest_tools(s, p.guest_noclip, p.guest_no_bail, p.guest_boosts); s.enforce_tuning = p.enforce_tuning; s.server_votes = dedicated_host(s) ? p.server_votes : 0; @@ -157,6 +292,7 @@ void apply_roster(Session &s, const Packet &p, std::uint64_t now) { if (old[i].id != next[i].id || old[i].epoch != next[i].epoch) { if (old[i].id && old[i].id != s.host_id) s.transport.disconnect(old[i].id, "Player left or rejoined the host roster."); + release_chat(s, s.peers[i], now, true); // what they said just before leaving reset_peer(s, i); s.peers[i].last_packet = now; } @@ -171,6 +307,8 @@ void apply_roster(Session &s, const Packet &p, std::uint64_t now) { } s.capacity = p.capacity; s.roster_voice_range = p.voice_range; + s.server_chat_badge = p.chat_badge; + s.server_chat_text = p.chat_text; ++s.party_revision; // anyone's party may have changed // A dedicated server knows players only by the name each sent in their hello. for (auto &peer : active_peers(s)) @@ -183,13 +321,13 @@ void apply_roster(Session &s, const Packet &p, std::uint64_t now) { bool accept_data(Peer &peer, const Packet &p, std::uint64_t now) { bool accepted{}; if (p.kind == PacketKind::cosmetics) { - accepted = peer.appearance.push(p); + accepted = peer.outfit_budget.accept(now) && peer.appearance.push(p); if (accepted) { peer.cosmetic_packet = encode_wire(p); ++peer.cosmetic_revision; } } else if (p.kind == PacketKind::audio) - accepted = peer.audio.push(p, now); + accepted = peer.sound_budget.accept(now, p.audio.size()) && peer.audio.push(p, now); else if (p.kind == PacketKind::pose) accepted = peer.poses.push_validated(p, now); if (accepted) { @@ -222,8 +360,11 @@ bool accept_data(Peer &peer, Packet &&p, std::uint64_t now) { } void networking(Session &s, const NativeFrame &local, std::uint64_t now) { s.network_now = now; + s.join_backoff.prune(now); trim_slots(s); if (world_playing(s, local)) s.local_root = local.pose.root; + for (auto &peer : active_peers(s)) + if (!peer.chat_waiting.empty()) release_chat(s, peer, now, false); s.transport.poll(); if (s.mode == Mode::join && s.transport.status().telemetry) { const auto &t = s.transport.status(); @@ -264,6 +405,17 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { stop(s, s.transport.status().detail); return; } + // The backend's bans (reskate_banned) arrive while a session runs: a banned player stops + // hosting theirs, and nobody stays with a banned host. A banned guest is for the host or + // the server to turn away (below, and Host::tick), which a server may choose not to. + if (s.mode == Mode::host && reskate_banned(s.transport.status().local_id)) { + stop(s, std::string(banned_notice)); + return; + } + if (s.mode == Mode::join && reskate_banned(s.host_id)) { + stop(s, "This host is banned from ReSkate multiplayer."); + return; + } for (const auto &link : links) { if (s.mode == Mode::host && s.banned.contains(link.id)) { s.transport.disconnect(link.id, "You were kicked from this session."); @@ -273,7 +425,15 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { s.transport.disconnect(link.id, "You are banned from this host's lobbies."); continue; } + if (s.mode == Mode::host && reskate_banned(link.id)) { + s.transport.disconnect(link.id, banned_notice.data()); + continue; + } auto *p = find_peer(s, link.id); + if (!p && s.mode == Mode::host && s.join_backoff.waiting(link.id, now)) { + s.transport.disconnect(link.id, "Too many failed attempts to join. Wait a little and try again."); + continue; + } if (!p && s.mode == Mode::host) p = &reserve(s, link.id, now); if (!p) { @@ -323,13 +483,24 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { } const auto messages = s.transport.receive(); const auto stale = stale_pose_deltas(s, messages); - for (std::size_t index = 0; index < messages.size(); ++index) { + // A message of several poses from a dedicated server is taken apart, and the loop goes + // round once for each pose in it as if it had come alone. + std::vector batch; + std::size_t batch_at{}; + for (std::size_t index = 0; index < messages.size(); batch_at < batch.size() ? index : ++index) { const auto &message = messages[index]; auto *link = find_peer(s, message.peer); - if (!link) + if (!link) { + batch.clear(); + batch_at = 0; continue; + } const bool direct_link = s.mode == Mode::join && message.peer != s.host_id; - if (!link->budget.accept(now, message.bytes.size(), + const bool resumed = batch_at < batch.size(); + if (!resumed) { + batch.clear(); + batch_at = 0; + if (!link->budget.accept(message.arrived ? message.arrived : now, message.bytes.size(), s.mode == Mode::join && !direct_link ? max_remote_players : 1U)) { disconnect(s, message.peer, "Peer exceeded the multiplayer packet limit."); if (s.mode == Mode::off) @@ -338,8 +509,29 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { } if (stale[index]) continue; + // A dedicated server's own messages (pose_batch.h, sound_codec.h). Only what comes + // from the server is read as one: sound_codec's first four bytes are also those of + // the pose updates games send each other directly (delta_codec's "RMS1"), which + // must go on to be decoded as what they are. + if (s.mode == Mode::join && !direct_link && dedicated_host(s)) { + // The server saying which of this game's own pose messages it read. + if (const auto ack = pose_batch::Ack::read(message.bytes)) { + s.pose_upload.ack(*ack); + continue; + } + if (sound_codec::is_sound(message.bytes)) { + batch = unpack_sound(s, message.bytes); + if (batch.empty()) continue; + } else if (pose_batch::is_batch(message.bytes)) { + batch = unpack_poses(s, message.bytes); + if (batch.empty()) continue; + } + } + } bool missing_reference{}; - auto decoded = link->receiver.receive(message.bytes, missing_reference, s.world); + std::optional decoded; + if (batch_at < batch.size()) decoded = std::move(batch[batch_at++]); + else decoded = link->receiver.receive(message.bytes, missing_reference, s.world); if (missing_reference) continue; if (!decoded || decoded->session != s.secret || @@ -371,6 +563,7 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { s.travel_started = s.join_started = now; s.join_destination.clear(); s.map_name.clear(); + s.map_label.clear(); s.map = 0; s.map_load_submitted = false; s.last_map_load_check = 0; @@ -382,6 +575,7 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { s.world_state_sequence = p.sequence; if (!p.destination.empty()) { s.join_destination = s.map_name = p.destination; + s.map_label = p.map_label; s.map = p.map; } s.host_world_ready = p.world_ready; @@ -480,6 +674,7 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { link->password_challenge = p.challenge; link->map_authorized |= p.map_authorized; s.join_destination = p.destination; + s.map_label = p.map_label; s.world = p.world; s.join_map_authorized |= p.map_authorized; if (new_challenge) @@ -574,11 +769,13 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { link->world_ready = true; link->travel_since = 0; link->last_packet = now; + if (joined) s.join_backoff.joined(message.peer); if (s.mode == Mode::host) { send_required(s, message.peer, encode_wire(packet(s, PacketKind::welcome, now))); if (joined) { send_roster(s, now); if (!s.tuning_packet.empty()) send_required(s, message.peer, s.tuning_packet); + if (!s.extras_packet.empty()) send_required(s, message.peer, s.extras_packet); send_required(s, message.peer, s.cosmetic_packet); for (const auto &other : active_peers(s)) if (other.handshaken && other.member.id != message.peer) @@ -651,9 +848,18 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { s.host_tuning = std::move(p.tuning); continue; } + // And the physics its tuning does not carry (Engine/Game/Multiplayer/session_physics.h). + if (p.kind == PacketKind::physics_extras) { + if (s.mode == Mode::join && message.peer == s.host_id && p.source == s.host_id && !dedicated_host(s)) + s.host_extras = p.extras; + continue; + } if (p.kind == PacketKind::maps) { if (dedicated_host(s) && message.peer == s.host_id && p.source == s.host_id) { s.server_maps = p.maps; + s.server_map_pool.clear(); + for (const auto entry : p.map_pool) s.server_map_pool.push_back(p.maps[entry]); + s.server_map_rotation = p.map_rotation; publish(s); publish_chat(s); } @@ -667,31 +873,50 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { } continue; } - // A dedicated server telling us about a party invite. + // Whoever hosts telling us about a party invite; or, hosting a lobby, a guest asking + // for something to be done with their party. if (p.kind == PacketKind::party) { - if (dedicated_host(s) && message.peer == s.host_id && p.source == s.host_id) receive_party(s, p, now); + if (s.mode == Mode::join) { + if (message.peer == s.host_id && p.source == s.host_id) receive_party(s, p, now); + } else if (s.mode == Mode::host) { + auto *sender = find_peer(s, p.source); + if (!direct_link && sender && sender->handshaken && message.peer == p.source && + routed_source(p, sender->member, message.peer, true, s.host_id) && sender->party_budget.accept(now, 20)) + host_party_request(s, p.source, p.party_action, p.party_player, now); + } continue; } - // A dedicated server's answer to one of our admin requests. + // A dedicated server's answer to one of our admin requests, or a lobby host's to a + // party request: a line for this player only. if (p.kind == PacketKind::admin) { - if (s.mode == Mode::join && message.peer == s.host_id && p.source == s.host_id && dedicated_host(s)) - add_chat(s, 0, "Server", p.text); + if (s.mode == Mode::join && message.peer == s.host_id && p.source == s.host_id) + add_chat(s, 0, dedicated_host(s) ? "Server" : "ReSkate", p.text); continue; } // Chat is accepted while either side is still loading a map: it needs // only an authenticated sender, never the world. if (p.kind == PacketKind::chat) { auto *sender = find_peer(s, p.source); - if (direct_link || !sender || !sender->handshaken || - !routed_source(p, sender->member, message.peer, s.mode == Mode::host, s.host_id)) + if (!sender || !sender->handshaken) continue; + // Another guest's own copy of a line they said: proof of the host's, never a line. + if (direct_link) { + if (routed_source(p, sender->member, message.peer, false, s.host_id, true)) proven_chat(s, *sender, p, now); + continue; + } + if (!routed_source(p, sender->member, message.peer, s.mode == Mode::host, s.host_id)) continue; // A dedicated server speaks (welcome message, its console) as "Server". const bool server = dedicated_host(s) && sender->member.id == s.host_id; // Everyone holds everyone to the same pace, so a modified client cannot flood. if (!server && sender->chat_rate.accept(now, p.text, 1) != ChatRate::Verdict::accepted) continue; sender->last_packet = now; - add_chat(s, sender->member.id, server ? std::string("Server") - : sender->member.name.empty() ? s.transport.name(sender->member.id) : sender->member.name, p.text); + // "/p": party chat, which a lobby's host relays to the sender's party and nobody else. + if (s.mode == Mode::host && p.text.starts_with("/p ")) { + host_party_chat(s, sender->member.id, std::string_view(p.text).substr(3), now); + continue; + } + if (owes_proof(s, *sender)) relayed_chat(s, *sender, p, now); + else add_chat(s, sender->member.id, server ? std::string("Server") : chat_name(s, *sender), p.text); if (s.mode == Mode::host) broadcast(s, p, true, false, now, p.source); continue; } @@ -758,6 +983,12 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { [](const auto &sample) { return sample.event; })), p.kind != PacketKind::cosmetics, now, p.source); } + // Tell the server which of its pose messages arrived, once for everything read this tick. + if (s.pose_ack_due && s.mode == Mode::join) { + const auto ack = s.pose_ack.bytes(); + s.transport.send(s.host_id, ack, false, true, TrafficLane::gameplay); + s.pose_ack_due = false; + } receive_cosmetics(s, local, now); for (std::size_t i = 0; i < s.used_slots; ++i) { const auto &p = s.peers[i]; @@ -798,6 +1029,7 @@ void networking(Session &s, const NativeFrame &local, std::uint64_t now) { } } update_scoring(s, now); // before the roster, so a host's own flag goes out with it + tick_host_parties(s, now); // and the lobby's parties, which the roster carries if (s.mode == Mode::host && world_playing(s, local) && (s.roster_dirty || now - s.last_roster > 2000000)) send_roster(s, now); const auto peers = active_peers(s); diff --git a/Extension/Multiplayer/Session/session_send.cpp b/Extension/Multiplayer/Session/session_send.cpp index 1bf9cb6..511da74 100644 --- a/Extension/Multiplayer/Session/session_send.cpp +++ b/Extension/Multiplayer/Session/session_send.cpp @@ -26,6 +26,8 @@ void publish_guest_objects(Session &s) { if (peer.handshaken && peer.objects.revision() != peer.shared_from) { auto layout = peer.objects.layout(); std::erase_if(layout, [&](const auto &object) { return peer.cleared.contains(object.id); }); + // The host's limit on each guest's objects, which a patched client cannot place past. + layout = limited_layout(std::move(layout), peer.shared.objects(), s.object_limit); peer.shared.replace(layout); peer.shared_from = peer.objects.revision(); } @@ -120,6 +122,13 @@ void disconnect(Session &s, std::uint64_t id, const std::string &reason) { for (std::size_t i = 0; i < s.used_slots; ++i) if (s.peers[i].member.id == id) { s.roster_dirty |= s.peers[i].handshaken; + // Never admitted: it held a player slot meanwhile. An ID that keeps failing waits + // longer each time before its connection is taken again. + if (s.mode == Mode::host && !s.peers[i].handshaken) { + const auto failures = s.join_backoff.failed(id, s.network_now); + logging::log(logging::Level::info, logging::Channel::runtime, + "Multiplayer: {} did not finish joining ({}), attempt {}.", id, reason, failures); + } reset_peer(s, i); break; } @@ -131,6 +140,10 @@ bool send_packet(Session &s, std::uint64_t id, const Packet &p, bool reliable, b if (!peer) return false; auto update = raw.empty() ? peer->sender.prepare(p) : peer->sender.prepare(p, raw, wire); + // A packet that cannot be built for anyone is its source's fault, never this recipient's: + // nothing is sent, and the recipient is not treated as unreachable. + if (update.bytes.empty()) + return true; // A stream and its reliable delta references must always use the same lane. if (!s.transport.send(id, update.bytes, reliable || update.establishes_baseline(), fresh, traffic_lane(p.kind))) return false; @@ -183,6 +196,14 @@ std::string send_chat(Session &s, std::string_view typed) { message.text = text; broadcast(s, message, true, false, now); const auto local = s.transport.status().local_id; + // The host passes a guest's line on, and could pass on anything under anyone's name. So a + // guest whose line shows a badge of the backend's also sends it straight to every player + // Steam connects them to: the copy is not shown, it is what lets that player know the + // host's one is ours (chat proofs, session_receive.cpp). Older builds ignore it. + if (s.mode == Mode::join && own_tag_shown() && identity_mark(local)) + for (const auto &peer : active_peers(s)) + if (peer.handshaken && peer.direct_ready && peer.member.id != s.host_id) + send_packet(s, peer.member.id, message, true, false); add_chat(s, local, s.transport.name(local), std::move(text), true); return {}; } @@ -207,6 +228,29 @@ std::string send_chat_command(Session &s, std::string_view typed) { } return {}; } +std::string send_party_chat(Session &s, std::string_view typed) { + if (s.mode != Mode::host && s.mode != Mode::join) return "Chat needs a multiplayer session."; + if (!s.local_party) return "You're not in a party."; + const auto text = clean_chat_text(typed); + if (text.empty()) return "Type a message first."; + const auto now = now_us(); + switch (s.local_chat_rate.accept(now, text)) { + case ChatRate::Verdict::repeated: return "You just said that."; + case ChatRate::Verdict::too_fast: return "Slow down: one message every couple of seconds."; + case ChatRate::Verdict::accepted: break; + } + const auto local = s.transport.status().local_id; + if (s.mode == Mode::host) { + host_party_chat(s, local, text, now); + } else { + // The host relays it to the rest of the party, not back to us. + auto message = packet(s, PacketKind::chat, now); + message.text = clean_chat_text("/p " + text); + if (!send_packet(s, s.host_id, message, true, false)) return "Could not reach the host."; + } + add_chat(s, local, s.transport.name(local), "[Party] " + text, true); + return {}; +} void send_throwdown(Session &s, std::vector message) { if ((s.mode != Mode::host && s.mode != Mode::join) || message.empty() || message.size() > max_throwdown_message) return; const auto now = now_us(); @@ -214,6 +258,25 @@ void send_throwdown(Session &s, std::vector message) { p.throwdown = std::move(message); broadcast(s, p, true, false, now); } +// This game's pose to a dedicated server: its differences from a pose of its own the server has +// said it holds (pose_batch.h), as the server sends everyone else's. +void upload_pose(Session &s, const Packet &packet, std::uint64_t now) { + auto &kept = s.own_poses; + if (!kept.empty() && kept.back().sequence >= packet.sequence) return; + kept.push_back({packet.sequence, packet.time_us, pose_codec::quantize(packet.pose)}); + while (kept.size() > 64) kept.pop_front(); + const auto find = [&](std::uint32_t sequence) -> std::optional { + for (auto it = kept.rbegin(); it != kept.rend(); ++it) + if (it->sequence == sequence) return pose_batch::KeptView{it->sequence, it->time_us, &it->pose}; + return {}; + }; + const auto emit = [&](std::span message, bool reliable) { + return s.transport.send(s.host_id, message, reliable, !reliable, TrafficLane::gameplay); + }; + s.pose_upload.begin(s.world, s.map); + s.pose_upload.add(packet.source, packet.epoch, *find(packet.sequence), find, false, packet.player_collision, pose_batch::Rate::full, now, emit); + s.pose_upload.flush(emit); +} void broadcast(Session &s, const Packet &packet, bool reliable, bool fresh, std::uint64_t now, std::uint64_t except) { const auto *source = find_peer(s, packet.source); @@ -272,6 +335,19 @@ void broadcast(Session &s, const Packet &packet, bool reliable, bool fresh, std: (packet.kind == PacketKind::pose || packet.kind == PacketKind::audio) && !needs_relay(p.direct_routes, p.route_reported, source->member, now)) continue; + if (packet.kind == PacketKind::pose && dedicated_host(s) && p.member.id == s.host_id) { + upload_pose(s, packet, now); + continue; + } + // This game's skater's sound to a dedicated server (sound_codec.h): what changed since + // the last samples it was sent. + if (packet.kind == PacketKind::audio && dedicated_host(s) && p.member.id == s.host_id) { + s.sound_upload.begin(s.world, s.map); + s.sound_upload.add(packet.source, packet.epoch, packet.sequence, packet.time_us, packet.audio, now); + if (s.sound_upload.pending()) + s.sound_upload.sent(s.transport.send(s.host_id, s.sound_upload.message(), true, fresh, traffic_lane(packet.kind))); + continue; + } PoseDelivery *delivery{}; std::uint32_t interval = multiplayer_pose_interval(s.tps); if (packet.kind == PacketKind::pose) { @@ -311,8 +387,11 @@ void broadcast(Session &s, const Packet &packet, bool reliable, bool fresh, std: data.wire = encode_wire_bytes(data.raw); data.ready = true; } - outgoing.push_back({&p, p.sender.prepare(packet, data.raw, data.wire, data.deltas), delivery, - interval}); + auto update = p.sender.prepare(packet, data.raw, data.wire, data.deltas); + // Unbuildable for anyone (see send_packet): skipped, and no recipient is dropped for it. + if (update.bytes.empty()) + continue; + outgoing.push_back({&p, std::move(update), delivery, interval}); } if (outgoing.empty()) return; @@ -384,6 +463,7 @@ void send_roster(Session &s, std::uint64_t now) { p.voice_range = s.voice_range; p.distances = s.distances; p.object_placement = s.object_placement; + p.object_limit = s.object_limit; p.guest_noclip = s.guest_noclip; p.guest_no_bail = s.guest_no_bail; p.guest_boosts = s.guest_boosts; @@ -396,25 +476,28 @@ void send_roster(Session &s, std::uint64_t now) { p.parks = s.parks; p.capacity = s.capacity; // Names come from the transport's short-lived name cache, not a Steam call per player. - p.members.push_back({p.source, s.epoch, s.transport.name(p.source)}); + // Steam names are cleaned to what a roster may carry: one that is not would be refused + // by every guest, and by this host's own encoder. + p.members.push_back({p.source, s.epoch, clean_roster_name(s.transport.name(p.source))}); p.members.back().scoring = s.local_scoring; for (auto &peer : active_peers(s)) if (peer.handshaken) { - peer.member.name = s.transport.name(peer.member.id); + peer.member.name = clean_roster_name(s.transport.name(peer.member.id)); p.members.push_back(peer.member); } - // A lobby is one party: everyone in it, led by the host (a party needs two players). - for (auto &m : p.members) { - m.party = p.members.size() > 1 ? lobby_party : 0; - m.party_leader = m.party && m.id == p.source; - m.party_open = false; - } + // Parties are the ones the lobby's players formed (session_party.cpp), as on a dedicated + // server. The host's own copy of each player follows the roster it sends. + fill_roster_parties(s, p.members); for (auto &peer : active_peers(s)) - if (peer.handshaken) { - peer.member.party = p.members.front().party; - peer.member.party_leader = false; - } - if (p.members.front().party != s.local_party) ++s.party_revision; + if (peer.handshaken) + for (const auto &m : p.members) + if (m.id == peer.member.id) { + if (peer.member.party != m.party || peer.member.party_leader != m.party_leader || + peer.member.party_open != m.party_open) ++s.party_revision; + peer.member.party = m.party; + peer.member.party_leader = m.party_leader; + peer.member.party_open = m.party_open; + } set_local_party(s, p.members.front()); broadcast(s, p, true, false, now); s.roster_dirty = false; @@ -422,13 +505,38 @@ void send_roster(Session &s, std::uint64_t now) { publish_chat(s); // the host's "/" argument lists (players) follow its own roster } void update_physics_tuning(Session &s, const NativeFrame &local, std::uint64_t now) { + // A guest whose host sets everyone's physics: the player's own edits stand down from the + // moment of joining (the roster says otherwise, if it does) and through travel, and what + // the host shares beyond its tuning is theirs. A dedicated server shares the game's own. + const bool enforced = s.mode == Mode::join && s.enforce_tuning; + set_session_tuning_enforced(enforced); + if (enforced && s.host_extras && !dedicated_host(s)) set_host_physics_extras(*s.host_extras); + else set_host_physics_extras({}); if (s.mode == Mode::host) { physics_tuning::release(s.base); if (!s.enforce_tuning) { s.sent_tuning.reset(); s.tuning_packet.clear(); + s.sent_extras = 0; + s.extras_packet.clear(); return; } + // The host's physics beyond its tuning: small, so a look four times a second, and sent + // whenever they change (the first time even when they are the game's own, so a guest + // never keeps what an earlier spell of enforcement left it). + if (now >= s.next_extras_check) { + s.next_extras_check = now + 250000; + std::vector extras; + if (local_physics_extras(s.sent_extras, extras)) { + if (extras.size() > max_physics_extras) extras.clear(); + logging::log(logging::Level::info, logging::Channel::runtime, + "Multiplayer: sending your other physics changes to guests ({} bytes).", extras.size()); + auto p = packet(s, PacketKind::physics_extras, now); + p.extras = std::move(extras); + s.extras_packet = encode_wire(p); + broadcast(s, p, true, false, now); + } + } // Edits to the tuning are rare: a look every 5 s (a 17 KB copy) is enough. if (now < s.next_tuning_check) return; s.next_tuning_check = now + 5000000; @@ -530,7 +638,14 @@ void send_local(Session &s, const NativeFrame &local, std::uint64_t now, std::ui }; if (now - s.last_cosmetic_capture >= 500000) { s.last_cosmetic_capture = now; - const auto appearance = capture_cosmetics(s.base, local, s.cosmetic_capture_status); + auto appearance = capture_cosmetics(s.base, local, s.cosmetic_capture_status); + // The player's choices to go without their tag or their animated items travel with their + // outfit, so a change is sent like one and reaches players who join later. + if (appearance) { + appearance->hide_tag = !own_tag_shown(); + appearance->hide_items = !own_items_shown(); + appearance->marks = developer_hoodie_detail::own_styles.load(); + } if (appearance && (!s.sent_appearance || *appearance != *s.sent_appearance)) { auto p = packet(s, PacketKind::cosmetics, now); p.appearance = *appearance; @@ -552,11 +667,34 @@ void send_local(Session &s, const NativeFrame &local, std::uint64_t now, std::ui if (captured_at) p.time_us = captured_at; p.pose = local.pose; ++s.local_pose_count; + if (s.pose_dump.is_open()) { + // A record: when it was captured (8 bytes), its length (4), and the compact encoding. + const auto raw = encode(p, true); + const std::uint64_t time = p.time_us; + const auto length = static_cast(raw.size()); + s.pose_dump.write(reinterpret_cast(&time), sizeof time); + s.pose_dump.write(reinterpret_cast(&length), sizeof length); + s.pose_dump.write(reinterpret_cast(raw.data()), static_cast(raw.size())); + ++s.pose_dump_count; + if (now >= s.pose_dump_until) { + s.pose_dump.close(); + logging::log(logging::Level::info, logging::Channel::runtime, "Multiplayer: pose dump finished, {} poses.", s.pose_dump_count); + } + } deliver(p, false, true); auto samples = drain_audio_capture(now); if (!samples.empty()) { auto a = packet(s, PacketKind::audio, now); a.audio = std::move(samples); + if (s.pose_dump.is_open()) { + // The skater's sound is recorded with the poses, in the same records. + const auto raw = encode(a, true); + const std::uint64_t time = a.time_us; + const auto length = static_cast(raw.size()); + s.pose_dump.write(reinterpret_cast(&time), sizeof time); + s.pose_dump.write(reinterpret_cast(&length), sizeof length); + s.pose_dump.write(reinterpret_cast(raw.data()), static_cast(raw.size())); + } deliver(a, std::any_of(a.audio.begin(), a.audio.end(), [](const auto &sample) { return sample.event; }), true); } } diff --git a/Extension/Multiplayer/Session/session_view.cpp b/Extension/Multiplayer/Session/session_view.cpp index 310185c..2b8900c 100644 --- a/Extension/Multiplayer/Session/session_view.cpp +++ b/Extension/Multiplayer/Session/session_view.cpp @@ -50,6 +50,8 @@ void load_host_preferences(Session &s) { p.voice_range = value->get(); if (const auto value = number("Host.ObjectPlacement"); value && *value >= 0 && valid_object_placement(static_cast(*value))) p.placement = static_cast(*value); + if (const auto value = number("Host.ObjectLimit"); value && *value >= 0 && valid_object_limit(static_cast(*value))) + p.object_limit = static_cast(*value); p.guest_noclip = profile_runtime::local_preference("Host.GuestNoclip").value_or(true); p.guest_no_bail = profile_runtime::local_preference("Host.GuestNoBail").value_or(true); p.guest_boosts = profile_runtime::local_preference("Host.GuestBoosts").value_or(true); @@ -70,6 +72,7 @@ void save_host_preferences(const Session &s) { {"Host.Distance.HalfReturn", static_cast(p.distances.half_rate_return)}, {"Host.Distance.LowStart", static_cast(p.distances.low_rate_start)}, {"Host.ObjectPlacement", static_cast(p.placement)}, + {"Host.ObjectLimit", static_cast(p.object_limit)}, {"Host.GuestNoclip", p.guest_noclip}, {"Host.GuestNoBail", p.guest_no_bail}, {"Host.GuestBoosts", p.guest_boosts}, @@ -147,6 +150,9 @@ void publish(Session &s, const NativeFrame *local) { view.tps = s.tps; view.distances = s.distances; view.object_placement = s.object_placement; + view.object_limit = s.object_limit; + view.object_limit_own = s.mode == Mode::host || s.server_admin ? 0 : s.object_limit; // as apply_object_limit gives this game + view.objects_placed = s.mode == Mode::off ? 0 : static_cast(s.local_objects.objects().size()); view.guest_noclip = s.guest_noclip; view.guest_no_bail = s.guest_no_bail; view.guest_boosts = s.guest_boosts; @@ -167,12 +173,38 @@ void publish(Session &s, const NativeFrame *local) { view.lobby_name = s.lobby_name; // A dedicated server's roster capacity counts the server itself. view.capacity = static_cast(dedicated_host(s) ? s.capacity - 1 : s.capacity); - view.party_overlay = s.party_overlay; load_host_preferences(s); view.saved_host = {true, s.host_preferences.public_lobby, s.host_preferences.password_required, static_cast(s.host_preferences.capacity), s.host_preferences.tps, s.host_preferences.lobby_name}; view.nametags = s.nametags; - view.custom_nametags = s.custom_nametags; + if (const auto social = steam_social_snapshot()) + if (const auto mark = identity_mark(social->local.id)) { + std::tie(view.identity_tag_colour, view.identity_tag) = mark_role(*mark); + view.identity_animation = + *mark == IdentityList::developer ? "RAINBOW" : *mark == IdentityList::content_creator ? "RED" : + *mark == IdentityList::centrix ? "BLUE" : *mark == IdentityList::staff ? "GREEN" : "GOLD"; + // A developer's standard is the rainbow already. + view.identity_rainbow = *mark == IdentityList::staff; + const auto styles = developer_hoodie_detail::own_styles.load(); + const auto standard = developer_hoodie_detail::standard_picks(*mark); + view.identity_styles.resize(styles.size()); + for (std::size_t i = 0; i < styles.size(); ++i) { + // A cosmetic never given colours shows its list's own in the pickers. + // The rainbow has no colours of its own to show either. + const bool rainbow = view.identity_rainbow && rainbow_style(styles[i]); + const bool picked = !rainbow && (styles[i].mode == MarkMode::gradient || styles[i].mode == MarkMode::solid || + styles[i].from != styles[i].to || styles[i].from != std::array{}); + const auto &from = picked ? styles[i].from : standard.first, &to = picked ? styles[i].to : standard.second; + auto &shown = view.identity_styles[i]; + shown.name = mark_item_names[i]; + shown.mode = rainbow ? 4 : static_cast(styles[i].mode); + shown.speed = styles[i].speed; + for (std::size_t part = 0; part < 3; ++part) + shown.from[part] = static_cast(from[part]) / 255.f, shown.to[part] = static_cast(to[part]) / 255.f; + } + } + view.identity_tag_shown = own_tag_shown(); + view.identity_items_shown = own_items_shown(); load_host_preferences(s); view.voice_range = s.mode == Mode::host ? s.voice_range : dedicated_host(s) ? s.roster_voice_range : s.host_preferences.voice_range; @@ -181,7 +213,7 @@ void publish(Session &s, const NativeFrame *local) { view.party = s.local_party; view.party_leader = s.local_party_leader; view.party_open = s.local_party_open; - view.parties = view.dedicated; + view.parties = s.mode == Mode::host || s.mode == Mode::join; for (const auto &invite : s.party_invites) { const auto *from = find_peer(s, invite.from); view.party_invites.push_back({invite.from, from && !from->member.name.empty() ? from->member.name @@ -192,9 +224,22 @@ void publish(Session &s, const NativeFrame *local) { view.server_bans = true; view.server_ban_total = s.server_ban_total; view.server_maps = s.server_maps; + view.server_map_pool = s.server_map_pool; + view.server_map_rotation = s.server_map_rotation; + view.server_map_votes = (s.server_votes & server_vote_map) != 0; } + view.player_distance = s.player_distance; + view.prefer_direct = s.prefer_direct; + view.nametag_distance = s.nametag_distance; + view.nametag_dots = s.nametag_dots; + view.nametags_friends = s.nametags_friends; view.chat_visible = s.chat_visible; view.chat_filter = s.chat_filter; + view.chat_bubbles = s.chat_bubbles; + view.chat_bubbles_own = s.chat_bubbles_own; + view.chat_bubbles_distance = s.chat_bubbles_distance; + view.chat_bubbles_duration = s.chat_bubbles_duration; + view.chat_bubbles_history = s.chat_bubbles_history; view.object_status = network_object_status(); view.players = static_cast(player_count(s)); view.sent = t.sent; @@ -230,7 +275,16 @@ void publish(Session &s, const NativeFrame *local) { view.browser_status = lobby.browser; // Dedicated servers first: they are always up and never a stranger's own session. view.lobbies = s.servers.rows(); + // Looked up as the list is shown: the team's list can arrive, or change, after a server was found. + for (auto &server : view.lobbies) server.official = official_server(server.id); view.lobbies.insert(view.lobbies.end(), lobby.rows.begin(), lobby.rows.end()); + // Steam friends by the public server or lobby their game says they are in. + if (const auto social = steam_social_snapshot()) + for (const auto &player : social->friends) { + if (!player.session) continue; + const auto row = std::find_if(view.lobbies.begin(), view.lobbies.end(), [&](const auto &entry) { return entry.id == player.session; }); + if (row != view.lobbies.end() && row->friends.size() < 16) row->friends.push_back(player.name.empty() ? std::string("A friend") : player.name); + } view.status = s.status; view.native_status = s.native_status; view.audio_captured = captured_audio_frames(); @@ -395,7 +449,7 @@ std::vector chat_commands(const Session &s) { {"/tp", "/tp ", "Teleport beside a player (or /tp )", "player"}, {"/p", "/p ", "Talk to your party only"}, }; - if (dedicated_host(s)) { + if (s.mode == Mode::host || s.mode == Mode::join) { list.push_back({"/party invite", "/party invite ", "Invite a player to your party", "player"}); list.push_back({"/party accept", "/party accept [player]", "Join the party you were invited to", "player"}); list.push_back({"/party decline", "/party decline [player]", "Turn down a party invite", "player"}); @@ -407,6 +461,9 @@ std::vector chat_commands(const Session &s) { list.push_back({"/party", "/party", "Who is in your party"}); } if (dedicated_host(s)) { + // The server's own commands (server_votes.cpp, server_host.cpp): it answers them, and + // this list is only what the "/" menu offers, so one left out here still works unseen. + list.push_back({"/w", "/w ", "Send a player a private message", "player"}); if (s.server_votes & server_vote_map) list.push_back({"/vote map", "/vote map ", "Start a vote to change the map", "map"}); if (s.server_votes & server_vote_kick) list.push_back({"/vote kick", "/vote kick ", "Start a vote to kick a player", "player"}); @@ -417,6 +474,9 @@ std::vector chat_commands(const Session &s) { list.push_back({"/no", "/no", "Vote no in the running vote"}); } if (s.server_admin) { + list.push_back({"/msg", "/msg ", "Admin: message a player privately", "player"}); + list.push_back({"/msg-party", "/msg-party ", "Admin: message everyone in a player's party", "player"}); + list.push_back({"/msg-admins", "/msg-admins ", "Admin: message the admins who are on"}); list.push_back({"/kick", "/kick ", "Admin: kick a player until the server restarts", "player"}); list.push_back({"/ban", "/ban ", "Admin: ban a player", "player"}); list.push_back({"/map", "/map ", "Admin: change the server's map", "map"}); @@ -425,6 +485,8 @@ std::vector chat_commands(const Session &s) { list.push_back({"/tod", "/tod