diff --git a/Engine/Game/Multiplayer/session_model.h b/Engine/Game/Multiplayer/session_model.h index 643ce0d..d50da6d 100644 --- a/Engine/Game/Multiplayer/session_model.h +++ b/Engine/Game/Multiplayer/session_model.h @@ -253,6 +253,9 @@ struct MultiplayerModel { bool party_leader{}, party_open{}, parties{}; std::vector party_invites; // newest last bool active{}, hosting{}, connected{}, echo{}, remote_visible{}, local_ready{}; + // The session is connected but waiting on its map, which this PC does not have and is + // being asked about or downloaded: not yet a session the player is in. + bool map_fetching{}; std::uint64_t local_id{}, peer_id{}, sent{}, received{}, dropped{}, pose_updates{}, board_pose_updates{}; std::size_t skater_bones{}, board_bones{}; std::string invite, map; diff --git a/Engine/Vfs/Test/thunderstore_package_tests.cpp b/Engine/Vfs/Test/thunderstore_package_tests.cpp new file mode 100644 index 0000000..c5081d4 --- /dev/null +++ b/Engine/Vfs/Test/thunderstore_package_tests.cpp @@ -0,0 +1,90 @@ +// One Thunderstore package looked up by name (Engine/Vfs/thunderstore_package.h): the name a +// host gives, and what Thunderstore's answer has to say before the game downloads it. +#include "Engine/Vfs/thunderstore_package.h" + +#include +#include + +using namespace dingosdk; +namespace package = thunderstore_package; + +namespace { +int failures = 0; +void check(bool condition, const char* what) { + if (condition) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +// Thunderstore's answer for a package, as /api/experimental/package/// gives it. +std::string answer(std::string_view owner, std::string_view name, std::string_view version, std::string_view community, + std::string_view category, std::string_view review, bool deprecated = false) { + return std::string("{\"namespace\":\"") + std::string(owner) + "\",\"name\":\"" + std::string(name) + + "\",\"full_name\":\"x\",\"is_deprecated\":" + (deprecated ? "true" : "false") + + ",\"latest\":{\"version_number\":\"" + std::string(version) + + "\",\"icon\":\"https://gcdn.thunderstore.io/live/repository/icons/x.png\",\"description\":\"A plaza\\nby the sea\",\"download_url\":\"https://elsewhere.example/steal.zip\",\"is_active\":true},\"community_listings\":[{\"has_nsfw_content\":false," + "\"categories\":[\"" + std::string(category) + "\"],\"community\":\"" + std::string(community) + + "\",\"review_status\":\"" + std::string(review) + "\"}]}"; +} +} + +int main() { + { + const auto plain = package::parse_name("Sandos-Vancouver_Plaza"); + check(plain && plain->owner == "Sandos" && plain->name == "Vancouver_Plaza" && plain->version.empty() && + plain->folder() == "Sandos-Vancouver_Plaza", "a package name read"); + const auto versioned = package::parse_name("Sandos-Vancouver_Plaza-2.0.0"); + check(versioned && versioned->version == "2.0.0" && versioned->folder() == "Sandos-Vancouver_Plaza", "and with its version"); + for (const char* bad : {"", "Sandos", "Sandos-", "-Plaza", "San dos-Plaza", "Sandos-Plaza-2.0", "Sandos-Plaza-2.0.0.1", "Sandos-Plaza-two", + "../x-y", "Sandos-Plaza/..", "Sandos-Pla.za", "a-b-1.2.3-4"}) + check(!package::parse_name(bad), bad); + const auto name = *versioned; + check(package::details_url(name) == L"https://thunderstore.io/api/experimental/package/Sandos/Vancouver_Plaza/", "where it is asked about"); + check(package::download_url(name, "2.0.0") == L"https://thunderstore.io/package/download/Sandos/Vancouver_Plaza/2.0.0/", + "where it is downloaded from"); + } + { + const package::Name newest{"Sandos", "Vancouver_Plaza", ""}, pinned{"Sandos", "Vancouver_Plaza", "1.4.0"}; + const auto listed = answer("Sandos", "Vancouver_Plaza", "2.0.0", "reskate", "Maps", "unreviewed"); + auto choice = package::choose(listed, newest, "reskate"); + check(choice.ok && choice.version == "2.0.0" && choice.newest == "2.0.0", "a listed map is taken, its newest version"); + check(choice.description == "A plaza by the sea", "with what it says of itself, on one line"); + check(choice.icon == "https://gcdn.thunderstore.io/live/repository/icons/x.png", "and its icon, from Thunderstore's own host"); + { + auto elsewhere = listed; + const std::string host = "gcdn.thunderstore.io"; + elsewhere.replace(elsewhere.find(host), host.size(), "thunderstore.io.example.com"); + const auto other = package::choose(elsewhere, newest, "reskate"); + check(other.ok && other.icon.empty(), "an icon anywhere else is not fetched"); + } + choice = package::choose(listed, pinned, "reskate"); + check(choice.ok && choice.version == "1.4.0" && choice.newest == "2.0.0", "or the version the host has, with the newest to fall back on"); + check(package::choose(answer("sandos", "vancouver_plaza", "2.0.0", "ReSkate", "maps", "approved"), newest, "reskate").ok, + "names compare whatever their case"); + const auto refused = [&](const std::string& text, const char* what) { + const auto result = package::choose(text, newest, "reskate"); + check(!result.ok && !result.reason.empty() && result.version.empty(), what); + }; + refused(answer("Someone", "Else", "2.0.0", "reskate", "Maps", "unreviewed"), "another package's answer is refused"); + refused(answer("Sandos", "Vancouver_Plaza", "2.0.0", "reskate", "Cosmetics", "unreviewed"), "a package that is not a map is refused"); + refused(answer("Sandos", "Vancouver_Plaza", "2.0.0", "othergame", "Maps", "unreviewed"), "a package of another community is refused"); + refused(answer("Sandos", "Vancouver_Plaza", "2.0.0", "reskate", "Maps", "rejected"), "a package the moderators rejected is refused"); + refused(answer("Sandos", "Vancouver_Plaza", "2.0.0", "reskate", "Maps", "unreviewed", true), "a deprecated package is refused"); + refused(answer("Sandos", "Vancouver_Plaza", "latest", "reskate", "Maps", "unreviewed"), "a version that is not one is refused"); + { + auto marked = listed; + const std::string off = "\"has_nsfw_content\":false"; + marked.replace(marked.find(off), off.size(), "\"has_nsfw_content\":true"); + refused(marked, "a package marked NSFW is refused"); + } + refused("{\"namespace\":\"Sandos\",\"name\":\"Vancouver_Plaza\"}", "an answer with no listing is refused"); + refused("not json", "an answer that is not JSON is refused"); + refused("[]", "an answer that is not an object is refused"); + } + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "thunderstore package tests passed\n"; + return 0; +} diff --git a/Engine/Vfs/https_download.cpp b/Engine/Vfs/https_download.cpp index d655c72..4ea1975 100644 --- a/Engine/Vfs/https_download.cpp +++ b/Engine/Vfs/https_download.cpp @@ -20,7 +20,7 @@ struct Internet { } Download get(std::wstring_view url, const std::filesystem::path& destination, - std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent) { + std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent, Watch* watch) { Download result; const auto fail = [&](DWORD error) { result.error = error; return result; }; if (url.empty() || url.size() >= 4096 || url.find_first_of(L"\r\n\t #") != std::wstring_view::npos) @@ -57,18 +57,21 @@ Download get(std::wstring_view url, const std::filesystem::path& destination, DWORD length{}; size = sizeof(length); if (WinHttpQueryHeaders(request.value, WINHTTP_QUERY_CONTENT_LENGTH | WINHTTP_QUERY_FLAG_NUMBER, nullptr, &length, &size, nullptr) && length > max_bytes) return fail(ERROR_FILE_TOO_LARGE); + if (watch) watch->total.store(length), watch->received.store(0); Handle output{CreateFileW(destination.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr)}; if (output.value == INVALID_HANDLE_VALUE) return fail(GetLastError()); std::array buffer{}; std::uint64_t total{}; for (;;) { if (GetTickCount64() - started >= std::uint64_t{timeout_seconds} * 1000) return fail(ERROR_TIMEOUT); + if (watch && watch->cancel.load()) return fail(ERROR_CANCELLED); DWORD read{}, written{}; if (!WinHttpReadData(request.value, buffer.data(), static_cast(buffer.size()), &read)) return fail(GetLastError()); if (!read) break; if (total + read > max_bytes) return fail(ERROR_FILE_TOO_LARGE); if (!WriteFile(output.value, buffer.data(), read, &written, nullptr) || written != read) return fail(ERROR_WRITE_FAULT); total += read; + if (watch) watch->received.store(total); } if (!total) return fail(ERROR_BAD_LENGTH); if (!FlushFileBuffers(output.value)) return fail(GetLastError()); diff --git a/Engine/Vfs/https_download.h b/Engine/Vfs/https_download.h index 91b777a..1b9073f 100644 --- a/Engine/Vfs/https_download.h +++ b/Engine/Vfs/https_download.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -10,11 +11,17 @@ struct Download { bool ok{}; unsigned long http_status{}, error{}; }; +// A long download as another thread sees it: how far it is (`total` 0 when the server does +// not say), and a way to stop it (the download then fails with ERROR_CANCELLED). +struct Watch { + std::atomic received{}, total{}; + std::atomic cancel{}; +}; // Streams an HTTPS GET into `destination`, which must not exist yet. Follows // redirects that stay on HTTPS; refuses bodies over `max_bytes` or transfers // that take longer than `timeout_seconds` in total. Download get(std::wstring_view url, const std::filesystem::path& destination, - std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent); + std::uint64_t max_bytes, std::uint32_t timeout_seconds, const wchar_t* agent, Watch* watch = nullptr); // The body of the same GET, for an answer small enough to hold: nothing when it // fails, and `result`, when given, says how. std::optional get_text(std::wstring_view url, std::uint64_t max_bytes, diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index dc9cb04..710d1e3 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -6,6 +6,8 @@ #include "mod_store_copies.h" #include "native_db.h" +#include + #include #include #include @@ -45,12 +47,22 @@ constexpr std::array launch_superbundles{"Win32/globals.toc // more than that the mod could not be merged: what was found is not for the // mod's author to read. The catalogue is only read once a mod turns out to add // an item at all. -bool store_copy_problems(const Catalog& catalog, MergeReport& report) { +bool store_copy_problems(const Catalog& all, MergeReport& report, const std::vector& checked, + std::size_t threads, bool background) { + // (Only when some are skipped is the catalogue copied, without them.) + std::optional fewer; + if (!checked.empty()) { + fewer = all; + std::erase_if(fewer->mods, [&](const Mod& mod) { + return std::ranges::any_of(checked, [&](const std::string& name) { return lower(name) == lower(mod.name); }); + }); + } + const Catalog& catalog = fewer ? *fewer : all; std::optional store; const auto found = check_store_copies(catalog, [&store](const std::string& key) { if (!store) store = content_cache::read_catalogs(content_cache::directory()); return store->reserved(key); - }, &report.notes); + }, &report.notes, threads, background); for (const auto& source : found.mods) report.problems[source.mod].emplace_back(store_copies_problem); return !found.mods.empty(); } @@ -95,7 +107,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // anything is built: the caller merges again without it, as it does for a // mod that cannot be merged, and the patch on disk stays for that merge to // reuse or replace. - if (storeKnown && store_copy_problems(catalog, report)) return report; + // Threads for the three steps that read many files: as many as there are cores at + // launch (the player is waiting on nothing else), half of them and fewer while the game runs. + const auto cores = std::max(1U, std::thread::hardware_concurrency()); + const std::size_t readers = std::min(options.live ? std::max(1U, cores / 2) : cores, options.live ? 6U : 8U); + if (storeKnown && store_copy_problems(catalog, report, options.live ? options.checked : std::vector{}, + readers - 1, options.live)) return report; if (!options.live) fs::remove_all(output, error); lap("checking the mods"); @@ -347,7 +364,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con lap("load screens"); } advance("Collecting asset overrides"); - auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report); + auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report, readers - 1, options.live); // Carried chunks point into their mod's archives; move them to where those landed. for (auto& [name, chunks] : overrides.chunks) for (const auto* mod : mods) @@ -362,8 +379,8 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // what was worked out above, and what it writes waits in its own copy of the store // (CasStore::waiting). So they are merged on several threads, and settled here one // after another in the order they always were, which puts every byte where merging - // them in turn would have. A merge while the game runs does them in turn, on this - // thread: the game is using the others. + // them in turn would have. A merge while the game runs uses fewer threads, at a lower + // priority: the game is using the others, and a player is waiting on this one. struct Job { std::string relative; std::optional store; // waiting: what the merge wrote @@ -426,13 +443,15 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con std::atomic& flag; ~Stop() { flag = true; } } stopOnExit{stopJobs}; - const std::size_t wantedWorkers = options.live || jobs.size() < 2 ? 0 - : std::min({jobs.size(), std::max(1U, std::thread::hardware_concurrency()), 8}); + const std::size_t wantedWorkers = jobs.size() < 2 ? 0 + : std::min({jobs.size(), options.live ? std::max(1U, cores / 2) : cores, options.live ? 6U : 8U}); try { // Threads of their own, not the system's pool: the launcher holds the pool's // threads back while the game starts (see world_layer_scan.cpp). for (std::size_t index = 0; index < wantedWorkers; ++index) workers.emplace_back([&] { + // (Under the game's own threads, so a merge shows as a wait and not as stutter.) + if (options.live) SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); for (;;) { const auto mine = nextJob.fetch_add(1); if (mine >= jobs.size() || stopJobs) return; diff --git a/Engine/Vfs/mod_merge.h b/Engine/Vfs/mod_merge.h index 97e4e9c..5d516a2 100644 --- a/Engine/Vfs/mod_merge.h +++ b/Engine/Vfs/mod_merge.h @@ -62,6 +62,11 @@ struct MergeOptions { // only read at startup and stays as it is; the stamp goes, so the next // launch builds a clean patch. bool live{}; + // Mods (folder names) already loaded in the running game, by the launch's merge or an + // earlier live one: they passed the store-copies check then, so a live merge holds only + // the others against it. Reading every mod's items and meshes again is most of what + // adding one map would otherwise wait on. + std::vector checked; }; // Combines every enabled mod into one patch under Mods/.reskate: superbundle diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 20b2367..b2cb4e3 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -265,11 +265,15 @@ struct AssetOverrides { // The changes and additions asset mods (mods that add no levels) make to the // game's own EBX and Lua scripts, the highest-priority mod's version winning. Never throws: an -// unreadable mod is noted and simply changes nothing elsewhere. +// unreadable mod is noted and simply changes nothing elsewhere. The mods' files are read on up +// to `threads` threads beside this one (`background`: at a priority under the game's, for a +// merge while it runs); what they mean together is then settled in priority order, so the +// result is the same whatever the count. AssetOverrides collect_asset_overrides(const std::vector& mods, const std::map& modFiles, const CasStore& store, const fs::path& baseRoot, - const fs::path& gameRoot, MergeReport& report); + const fs::path& gameRoot, MergeReport& report, + std::size_t threads = 0, bool background = false); // A bundle's chunk metadata is a list with one record for each of its chunks: the // hash of the name of the resource the chunk belongs to ("h64") and what the diff --git a/Engine/Vfs/mod_merge_overrides.cpp b/Engine/Vfs/mod_merge_overrides.cpp index f3e8d0c..295576a 100644 --- a/Engine/Vfs/mod_merge_overrides.cpp +++ b/Engine/Vfs/mod_merge_overrides.cpp @@ -4,156 +4,289 @@ #include "Engine/Resource/ebx_merge.h" #include "Engine/Resource/ebx_writer.h" +#include + #include #include +#include #include +#include +#include #include #include +#include namespace dingosdk::mods::detail { +namespace { + +// The game's side of what every mod is held against, read once for all of them: forty +// cosmetic mods each patch the same few bundles, the largest with some 17,000 assets, and +// each used to list that bundle again for itself. +struct GameBundle { + std::optional listing; + // The game's copy of each asset: its sha1, and where in the listing it is. + std::map, std::less<>> original; + std::map> scripts; + std::set> resources; +}; +struct GameToc { + fb::TocDocument document; + std::map> bundles; + std::set chunks; +}; +// Made on first use by whichever thread asks first; the others wait for it. One that could +// not be read throws for each asker, as reading it for each did. +template class Shared { +public: + template const Value& get(const std::string& key, Make&& make) { + Entry* entry{}; + { + std::lock_guard lock(mutex_); + entry = &entries_[key]; + } + std::call_once(entry->once, [&] { make(entry->value); }); + return entry->value; + } + +private: + struct Entry { + std::once_flag once; + Value value; + }; + std::mutex mutex_; + std::map> entries_; // nodes stay where they are +}; + +// The game's assets a mod changed. They wait until its additions are settled: one that +// another mod's asset of the same name stands in for is not where a copy can find it, and a +// change must not name it there. +struct Change { + std::string bundle; // lower case: where the mod changed it + std::string name; // lower case + std::string asset; // as the bundle spells it + fb::Sha1 game; // the game's copy it replaces + fb::Sha1 sha1; + std::uint64_t originalSize{}; + std::vector encoded; + std::optional gameFile; // where the game's copy is +}; +struct Script { + std::string name; // lower case + fb::Sha1 game; // the game's copy it replaces + AssetOverride value; +}; +// What one mod's files say, read without regard to any other mod: the part of the work that +// is reading and decoding, done for several mods at once. What it means beside the mods +// above it is settled afterwards, one mod at a time in priority order. +struct ModRead { + // This mod's new assets, and the partitions its recorded changes import. + // An added asset only follows a change that refers to it: copied on its + // own, it could arrive in another mod's bundle without the resources + // and chunks it was added with. + std::vector> candidates, companions; + std::vector unreadable; // per candidate: never follows + std::map, std::less<>> addedIn; // the documents it added to each bundle, by guid + std::vector newChunks; // TOC chunks the game's TOCs do not have + std::set imported; + std::vector changes; + std::vector