diff --git a/BuiltinPacker/main.cpp b/BuiltinPacker/main.cpp new file mode 100644 index 0000000..e93853d --- /dev/null +++ b/BuiltinPacker/main.cpp @@ -0,0 +1,96 @@ +// Builds a built-in content pack (Engine/Vfs/builtin_packs.h) from its zip in assets/builtin: +// ReSkateBuiltinPacker +// The zip's files go into one plain archive (builtin_pack_format.h) with files of the same name +// next to each other (a Board Break deck pack carries much the same files once for every deck), +// which is then compressed whole with zstd and long-distance matching. Zip compresses each file +// on its own and finds none of that; this way each repeat costs next to nothing. +#include "Engine/Vfs/builtin_pack_format.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +struct File { + std::string name; + std::vector bytes; +}; +std::string base_name(const std::string& path) { + const auto slash = path.find_last_of('/'); + return slash == std::string::npos ? path : path.substr(slash + 1); +} +template void put(std::vector& out, T value) { + for (std::size_t i = 0; i < sizeof(T); ++i) out.push_back(static_cast(value >> (8 * i))); +} +int fail(const std::string& message) { + std::cerr << "ReSkateBuiltinPacker: " << message << '\n'; + return 1; +} +} + +int main(int argc, char** argv) { + if (argc != 3) return fail("usage: ReSkateBuiltinPacker "); + std::ifstream input(argv[1], std::ios::binary); + const std::vector zip_bytes{std::istreambuf_iterator(input), std::istreambuf_iterator()}; + if (zip_bytes.empty()) return fail(std::string("cannot read ") + argv[1]); + mz_zip_archive zip{}; + if (!mz_zip_reader_init_mem(&zip, zip_bytes.data(), zip_bytes.size(), 0)) return fail("not a zip"); + std::vector files; + std::uint64_t total{}; + for (mz_uint index = 0; index < mz_zip_reader_get_num_files(&zip); ++index) { + mz_zip_archive_file_stat info{}; + if (!mz_zip_reader_file_stat(&zip, index, &info)) return fail("an unreadable entry"); + if (info.m_is_directory) continue; + File file; + file.name = info.m_filename; + if (!dingosdk::mods::builtin_pack::safe_name(file.name)) return fail("an entry the game may not have: " + file.name); + file.bytes.resize(static_cast(info.m_uncomp_size)); + if (!file.bytes.empty() && !mz_zip_reader_extract_to_mem(&zip, index, file.bytes.data(), file.bytes.size(), 0)) + return fail("cannot unpack " + file.name); + total += file.bytes.size(); + files.push_back(std::move(file)); + } + mz_zip_reader_end(&zip); + if (total > dingosdk::mods::builtin_pack::max_bytes) return fail("the pack is larger than a built-in pack may be"); + // Same names together (a deck pack's decks// for every deck), then by path. + std::stable_sort(files.begin(), files.end(), [](const File& a, const File& b) { + const auto ka = base_name(a.name), kb = base_name(b.name); + return ka != kb ? ka < kb : a.name < b.name; + }); + + std::vector plain; + plain.insert(plain.end(), dingosdk::mods::builtin_pack::magic.begin(), dingosdk::mods::builtin_pack::magic.end()); + put(plain, static_cast(files.size())); + for (const auto& file : files) { + put(plain, static_cast(file.name.size())); + plain.insert(plain.end(), file.name.begin(), file.name.end()); + put(plain, file.bytes.size()); + } + for (const auto& file : files) plain.insert(plain.end(), file.bytes.begin(), file.bytes.end()); + + ZSTD_CCtx* context = ZSTD_createCCtx(); + if (!context) return fail("out of memory"); + // A window to cover the whole pack (within what any decoder takes by default, 2^27), so a + // file repeated anywhere in it is found. + int window = 10; + while (window < dingosdk::mods::builtin_pack::max_window_log && (std::size_t{1} << window) < plain.size()) ++window; + ZSTD_CCtx_setParameter(context, ZSTD_c_compressionLevel, 19); + ZSTD_CCtx_setParameter(context, ZSTD_c_enableLongDistanceMatching, 1); + ZSTD_CCtx_setParameter(context, ZSTD_c_windowLog, window); + ZSTD_CCtx_setParameter(context, ZSTD_c_checksumFlag, 1); + std::vector packed(ZSTD_compressBound(plain.size())); + const auto size = ZSTD_compress2(context, packed.data(), packed.size(), plain.data(), plain.size()); + ZSTD_freeCCtx(context); + if (ZSTD_isError(size)) return fail(std::string("zstd: ") + ZSTD_getErrorName(size)); + packed.resize(size); + std::ofstream output(argv[2], std::ios::binary | std::ios::trunc); + output.write(reinterpret_cast(packed.data()), static_cast(packed.size())); + if (!output) return fail(std::string("cannot write ") + argv[2]); + std::cout << argv[1] << ": " << files.size() << " files, " << total << " bytes -> " << packed.size() << " bytes (" + << zip_bytes.size() << " as a zip)\n"; + return 0; +} diff --git a/CMakeLists.txt b/CMakeLists.txt index 58fd620..b6b222d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -373,6 +373,11 @@ if(DINGOSDK_BUILD_HALL_OF_MEAT_TESTS) endif() get_property(project_targets DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) foreach(target IN LISTS project_targets) + # (A build step, such as packing the built-in content, compiles nothing.) + get_target_property(target_kind ${target} TYPE) + if(target_kind STREQUAL "UTILITY") + continue() + endif() if(NOT target MATCHES "^(dingosdk_imgui|dingosdk_detours|dingosdk_sqlite|dingosdk_lz4|dingosdk_zstd|dingosdk_miniz|dingosdk_spotify_device)$") dingosdk_configure_target(${target}) endif() diff --git a/Engine/Vfs/builtin_pack_format.h b/Engine/Vfs/builtin_pack_format.h new file mode 100644 index 0000000..004544f --- /dev/null +++ b/Engine/Vfs/builtin_pack_format.h @@ -0,0 +1,41 @@ +#pragma once +// The built-in pack format (Engine/Vfs/builtin_packs.h), as BuiltinPacker writes it at build time +// from a zip in assets/builtin and ReSkate unpacks it: one zstd frame (with its content size and a +// checksum) holding +// "RSKPACK1", u32 file count, +// for each file: u16 name length, the name (UTF-8, '/' between folders), u64 size, +// then every file's bytes in the same order. +// Numbers little-endian. +#include +#include +#include +#include + +namespace dingosdk::mods::builtin_pack { +inline constexpr std::array magic{'R', 'S', 'K', 'P', 'A', 'C', 'K', '1'}; +// The most a pack may unpack to, and the largest zstd window it may use (zstd's own default +// limit for decoding, 128 MiB). +inline constexpr std::uint64_t max_bytes = 512ull * 1024 * 1024; +inline constexpr int max_window_log = 27; + +// A path inside a pack: relative, forward slashes, no "..", plain characters (and spaces inside a +// name, as in a sound's "Body Impact 1.wav", but never at either end of one). +inline bool safe_name(std::string_view name) { + if (name.empty() || name.size() > 260 || name.front() == '/' || name.find(':') != std::string_view::npos || + name.find('\\') != std::string_view::npos) + return false; + std::size_t start = 0; + while (start <= name.size()) { + const auto end = std::min(name.find('/', start), name.size()); + const auto part = name.substr(start, end - start); + if (part.empty() || part == "." || part == ".." || part.front() == ' ' || part.back() == ' ') return false; + if (!std::all_of(part.begin(), part.end(), [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || + c == '-' || c == ' '; + })) + return false; + start = end + 1; + } + return true; +} +} diff --git a/Engine/Vfs/builtin_packs.cpp b/Engine/Vfs/builtin_packs.cpp index 9921308..1f44720 100644 --- a/Engine/Vfs/builtin_packs.cpp +++ b/Engine/Vfs/builtin_packs.cpp @@ -1,11 +1,18 @@ #include "builtin_packs.h" +#include "builtin_pack_format.h" #include #include +#include #include +#include +#include #include #include +#include #include #include +#include +#include namespace dingosdk::mods { namespace { @@ -15,8 +22,14 @@ struct Embedded { const wchar_t* resource; const char* name; }; -constexpr Embedded embedded[]{{L"BUILTIN_PACK_FPV_DRONE", "fpv_drone"}}; -constexpr std::uint64_t max_pack_bytes = 512ull * 1024 * 1024; +constexpr Embedded embedded[]{{L"BUILTIN_PACK_FPV_DRONE", "fpv_drone"}, + {L"BUILTIN_PACK_BOARD_BREAK", "board_break"}, + {L"BUILTIN_PACK_ROAD_RASH", "road_rash"}}; +// The mods those replace, by folder name in small letters: as Thunderstore installs them +// (Team-Name), and the bare names a mod put in by hand would have. +constexpr std::string_view superseded[]{"fourth_studio-board_break_pack", "fourth_studio-board_break", + "fourth_studio-road_rash", "board_break_pack", "road_rash"}; +constexpr std::uint64_t max_pack_bytes = builtin_pack::max_bytes; constexpr std::string_view version_file = ".builtin-version"; struct Source { @@ -30,7 +43,9 @@ struct Source { } }; bool carries_packs(HMODULE module) { - return module && FindResourceW(module, embedded[0].resource, MAKEINTRESOURCEW(10) /* RT_RCDATA */); + return module && std::any_of(std::begin(embedded), std::end(embedded), [&](const Embedded& pack) { + return FindResourceW(module, pack.resource, MAKEINTRESOURCEW(10) /* RT_RCDATA */) != nullptr; + }); } // ReSkate.dll: this module when it is the DLL (the game), else ReSkate.dll beside the program // (the launcher, which sits beside Skate.exe and the DLL), as data only. @@ -71,25 +86,79 @@ std::string read_text(const fs::path& path) { std::ifstream input(path, std::ios::binary); return {std::istreambuf_iterator(input), std::istreambuf_iterator()}; } -// A path inside the pack: relative, forward slashes, no "..", plain characters. -bool safe_entry(std::string_view name) { - if (name.empty() || name.size() > 260 || name.front() == '/' || name.find(':') != std::string_view::npos || - name.find('\\') != std::string_view::npos) +bool safe_entry(std::string_view name) { return builtin_pack::safe_name(name); } +// Writes `size` bytes to a new file at `path` (its folders made first). +bool write_file(const fs::path& path, const unsigned char* data, std::uint64_t size) { + std::error_code error; + fs::create_directories(path.parent_path(), error); + struct File { + HANDLE value; + ~File() { if (value != INVALID_HANDLE_VALUE) CloseHandle(value); } + } file{CreateFileW(path.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)}; + if (file.value == INVALID_HANDLE_VALUE) return false; + while (size) { + const auto chunk = static_cast(std::min(size, 64ull * 1024 * 1024)); + DWORD done{}; + if (!WriteFile(file.value, data, chunk, &done, nullptr) || done != chunk) return false; + data += chunk; + size -= chunk; + } + return true; +} +// A pack in ReSkate's own format (builtin_pack_format.h): one zstd frame, unpacked whole. +bool unpack_packed(std::span bytes, const fs::path& target, std::string& issue) { + const auto size = ZSTD_getFrameContentSize(bytes.data(), bytes.size()); + if (size == ZSTD_CONTENTSIZE_ERROR || size == ZSTD_CONTENTSIZE_UNKNOWN || size > max_pack_bytes + 16ull * 1024 * 1024) { + issue = "the embedded pack is not one ReSkate made"; return false; - std::size_t start = 0; - while (start <= name.size()) { - const auto end = std::min(name.find('/', start), name.size()); - const auto part = name.substr(start, end - start); - if (part.empty() || part == "." || part == "..") return false; - if (!std::all_of(part.begin(), part.end(), [](char c) { - return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '.' || c == '_' || c == '-'; - })) + } + std::vector plain(static_cast(size)); + const auto unpacked = ZSTD_decompress(plain.data(), plain.size(), bytes.data(), bytes.size()); + if (ZSTD_isError(unpacked) || unpacked != plain.size()) { + issue = std::string("the embedded pack could not be decompressed: ") + (ZSTD_isError(unpacked) ? ZSTD_getErrorName(unpacked) : "short"); + return false; + } + std::size_t at = 0; + const auto take = [&](void* out, std::size_t count) { + if (count > plain.size() - at) return false; + std::memcpy(out, plain.data() + at, count); + at += count; + return true; + }; + std::array magic{}; + std::uint32_t count{}; + if (!take(magic.data(), magic.size()) || magic != builtin_pack::magic || !take(&count, sizeof(count)) || count > 100000) { + issue = "the embedded pack is not one ReSkate made"; + return false; + } + std::vector> files(count); + std::uint64_t total{}; + for (auto& [name, length] : files) { + std::uint16_t name_length{}; + if (!take(&name_length, sizeof(name_length))) { issue = "the embedded pack's list is cut short"; return false; } + name.resize(name_length); + if (!take(name.data(), name.size()) || !take(&length, sizeof(length))) { issue = "the embedded pack's list is cut short"; return false; } + total += length; + if (!safe_entry(name) || total > max_pack_bytes) { + issue = "the embedded pack has an entry it may not: " + name; return false; - start = end + 1; + } + } + for (const auto& [name, length] : files) { + if (length > plain.size() - at) { issue = "the embedded pack is cut short at " + name; return false; } + if (!write_file(target / fs::path(name), plain.data() + at, length)) { issue = "could not unpack " + name; return false; } + at += static_cast(length); + } + if (at != plain.size()) { + issue = "the embedded pack has bytes past its last file"; + return false; } return true; } bool unpack(std::span bytes, const fs::path& target, std::string& issue) { + // ReSkate's own format (a zstd frame); else a plain zip, as a build without the packer embeds. + if (bytes.size() >= 4 && bytes[0] == 0x28 && bytes[1] == 0xb5 && bytes[2] == 0x2f && bytes[3] == 0xfd) + return unpack_packed(bytes, target, issue); struct Zip { mz_zip_archive value{}; ~Zip() { mz_zip_reader_end(&value); } @@ -140,6 +209,21 @@ bool unpack(std::span bytes, const fs::path& target, std::s } } // namespace +bool unpack_builtin_pack(std::span bytes, const fs::path& target, std::string& issue) noexcept { + try { + return unpack(bytes, target, issue); + } catch (const std::exception& failure) { + issue = failure.what(); + return false; + } +} + +bool superseded_by_builtin(std::string_view folder) noexcept { + std::string name(folder); + std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); + return std::find(std::begin(superseded), std::end(superseded), name) != std::end(superseded); +} + std::vector unpack_builtin_packs(const fs::path& mods_root, std::vector& notes) noexcept { std::vector packs; try { diff --git a/Engine/Vfs/builtin_packs.h b/Engine/Vfs/builtin_packs.h index efbac97..463818f 100644 --- a/Engine/Vfs/builtin_packs.h +++ b/Engine/Vfs/builtin_packs.h @@ -1,10 +1,12 @@ #pragma once #include +#include #include #include -// Game content ReSkate carries itself: packs of game assets built by ReSkate Studio (the FPV -// drone's model, tools/models/build_drone_pack.py), zipped into ReSkate.dll. Nobody installs or +// Game content ReSkate carries itself: packs of game assets built by ReSkate Studio, zipped into +// ReSkate.dll: the FPV drone's model (tools/models/build_drone_pack.py), and Fourth Studio's Board +// Break decks and sounds and Road Rash wounds, which were Thunderstore mods. Nobody installs or // sees them. Before a merge each is unpacked under Mods/.reskate-builtin/ (a dot folder, // which no mod list shows) when the one embedded differs from what is there, and the catalogue // merges it after every installed mod, always on (mod_catalog.cpp). @@ -19,4 +21,12 @@ struct BuiltinPack { // launcher) is opened as data. Never throws; `notes` says what was unpacked or went wrong. std::vector unpack_builtin_packs(const std::filesystem::path& mods_root, std::vector& notes) noexcept; +// One pack's bytes as ReSkate.dll carries them (builtin_pack_format.h, or a plain zip) unpacked +// into `target`; false with `issue` saying why. For tests and tools: unpack_builtin_packs uses it. +bool unpack_builtin_pack(std::span bytes, const std::filesystem::path& target, std::string& issue) noexcept; +// Whether an installed mod (its folder's name) is one a built-in pack replaces: Fourth Studio's +// Board Break pack (and its deprecated first one) and Road Rash, as Thunderstore installs them or +// put in by hand. The launcher moves such a mod to the Recycle Bin, the merge leaves it out +// should it still be there, and the launcher's Thunderstore browser neither lists nor installs it. +bool superseded_by_builtin(std::string_view folder) noexcept; } // namespace dingosdk::mods diff --git a/Engine/Vfs/mod_catalog.cpp b/Engine/Vfs/mod_catalog.cpp index f3a7f75..9447713 100644 --- a/Engine/Vfs/mod_catalog.cpp +++ b/Engine/Vfs/mod_catalog.cpp @@ -135,17 +135,29 @@ std::vector> decks_inside(const fs::path& mod) std::sort(found.begin(), found.end(), [](const auto& a, const auto& b) { return deck_before(a.first, b.first); }); return found; } +// Every mod folder a deck can come from: those in the Mods folder (but one a built-in pack +// replaces), then the built-in packs (builtin_packs.h), which the merge also puts after them. +std::vector deck_mod_folders() { + std::vector found; + const auto mods = engine_data_root() / L"Mods"; + std::error_code error; + for (fs::directory_iterator at(mods, error), end; !error && at != end; at.increment(error)) + if (at->is_directory(error) && !superseded_by_builtin(path_utf8(at->path().filename()))) found.push_back(at->path()); + error.clear(); + for (fs::directory_iterator at(mods / std::string(builtin_folder), error), end; !error && at != end; at.increment(error)) + if (at->is_directory(error) && at->path().filename().wstring().find(L".unpacking-") == std::wstring::npos) + found.push_back(at->path()); + return found; +} // Every deck in the Mods folder with the folder of the mod that brings it, in the menus' order, // whatever mods.json says of that mod: the decks inside a pack, or else the one a folder is named for. std::vector> installed_decks() { std::vector> decks; - std::error_code error; - for (fs::directory_iterator at(engine_data_root() / L"Mods", error), end; !error && at != end; at.increment(error)) { - if (!at->is_directory(error)) continue; - auto inside = decks_inside(at->path()); - for (auto& one : inside) decks.emplace_back(std::move(one.first), at->path()); + for (const auto& folder : deck_mod_folders()) { + auto inside = decks_inside(folder); + for (auto& one : inside) decks.emplace_back(std::move(one.first), folder); if (!inside.empty()) continue; - if (auto deck = board_break_deck_of(path_utf8(at->path().filename())); !deck.empty()) decks.emplace_back(std::move(deck), at->path()); + if (auto deck = board_break_deck_of(path_utf8(folder.filename())); !deck.empty()) decks.emplace_back(std::move(deck), folder); } std::stable_sort(decks.begin(), decks.end(), [](const auto& a, const auto& b) { return deck_before(a.first, b.first); }); return decks; @@ -225,13 +237,11 @@ std::vector board_break_folders() noexcept { std::filesystem::path board_break_deck_folder(std::string_view deck) noexcept { try { const auto wanted = lower(deck); - std::error_code error; - for (fs::directory_iterator at(engine_data_root() / L"Mods", error), end; !error && at != end; at.increment(error)) { - if (!at->is_directory(error)) continue; - const auto inside = decks_inside(at->path()); + for (const auto& folder : deck_mod_folders()) { + const auto inside = decks_inside(folder); for (const auto& one : inside) if (one.first == wanted) return one.second; - if (inside.empty() && board_break_deck_of(path_utf8(at->path().filename())) == wanted) return at->path(); + if (inside.empty() && board_break_deck_of(path_utf8(folder.filename())) == wanted) return folder; } } catch (...) {} return {}; @@ -326,6 +336,13 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver try { for (const auto& name : list.missing) result.notes.push_back("mods.json names a folder that is not present: " + name); + // A mod a built-in pack replaces (builtin_packs.h) is left out, should the launcher not have + // taken it away yet: its content would load twice. + std::erase_if(list.entries, [&](const ModEntry& entry) { + if (!superseded_by_builtin(entry.mod.name)) return false; + result.notes.push_back(entry.mod.name + " is built into ReSkate now; the installed copy is left out"); + return true; + }); // After every installed mod, so none of theirs is ever shadowed by it; always on. for (const auto& pack : builtin) list.entries.push_back({describe_mod(pack.directory, pack.name, result.notes), true}); diff --git a/Launcher/Test/mod_manager_tests.cpp b/Launcher/Test/mod_manager_tests.cpp index 6070274..3f75ede 100644 --- a/Launcher/Test/mod_manager_tests.cpp +++ b/Launcher/Test/mod_manager_tests.cpp @@ -1,6 +1,9 @@ // Launcher mod manager: install from .zip and folders, the Mods list and mods.json. #include "Launcher/mod_manager.h" +#include "Engine/Vfs/builtin_pack_format.h" +#include "Engine/Vfs/builtin_packs.h" +#include #include "Engine/Vfs/mod_list.h" #include @@ -344,6 +347,85 @@ int main() { check(install(store, sources / L"empty.zip") == "empty", "A plain install still takes a description-only mod"); } + // A built-in pack in ReSkate's own format unpacks to exactly its files (a name with spaces + // included), and one with a path that leaves its folder unpacks nothing. + { + const auto pack = [](const std::vector>& files) { + std::string plain(mods::builtin_pack::magic.begin(), mods::builtin_pack::magic.end()); + const auto put = [&](auto value) { plain.append(reinterpret_cast(&value), sizeof(value)); }; + put(static_cast(files.size())); + for (const auto& [name, bytes] : files) { + put(static_cast(name.size())); + plain += name; + put(static_cast(bytes.size())); + } + for (const auto& file : files) plain += file.second; + std::vector packed(ZSTD_compressBound(plain.size())); + packed.resize(ZSTD_compress(packed.data(), packed.size(), plain.data(), plain.size(), 19)); + return packed; + }; + const auto target = root / L"builtin" / L"good"; + std::string issue; + const auto good = pack({{"layout.toc", "toc"}, {"sounds/body/Body Impact 1.wav", "RIFF"}, {"decks/wide/layout.toc", "deck"}}); + check(mods::unpack_builtin_pack(good, target, issue) && issue.empty() && read(target / L"layout.toc") == "toc" && + read(target / L"sounds" / L"body" / L"Body Impact 1.wav") == "RIFF" && read(target / L"decks" / L"wide" / L"layout.toc") == "deck", + "A built-in pack unpacks to its files"); + const auto bad_target = root / L"builtin" / L"bad"; + check(!mods::unpack_builtin_pack(pack({{"../escape.txt", "x"}}), bad_target, issue) && !issue.empty() && + !fs::exists(root / L"builtin" / L"escape.txt"), + "A built-in pack may not write outside its folder"); + auto cut = good; + cut.resize(cut.size() / 2); + check(!mods::unpack_builtin_pack(cut, root / L"builtin" / L"cut", issue), "A cut-short built-in pack is refused"); + // The packs this build made (RESKATE_BUILTIN_PACKS: their folder), each unpacked whole. + std::wstring built(32768, L'\0'); + built.resize(GetEnvironmentVariableW(L"RESKATE_BUILTIN_PACKS", built.data(), static_cast(built.size()))); + if (!built.empty()) { + for (const auto& file : fs::directory_iterator(fs::path(built))) { + if (file.path().extension() != L".rskpack") continue; + std::ifstream in(file.path(), std::ios::binary); + const std::vector bytes{std::istreambuf_iterator(in), std::istreambuf_iterator()}; + const auto out = root / L"builtin" / file.path().stem(); + std::size_t count{}; + std::uintmax_t size{}; + const bool ok = mods::unpack_builtin_pack(bytes, out, issue); + for (const auto& entry : fs::recursive_directory_iterator(out)) + if (entry.is_regular_file()) { ++count; size += entry.file_size(); } + std::cout << " " << file.path().filename().string() << ": " << (ok ? "unpacked " : "FAILED ") << count << " files, " + << size << " bytes" << (ok ? "" : " (" + issue + ")") << '\n'; + check(ok, "A built-in pack this build made unpacks"); + } + } + } + + // Mods ReSkate now carries itself (Board Break, Road Rash) go, and mods.json forgets them; + // every other mod stays as it was. + { + check(mods::superseded_by_builtin("Fourth_Studio-Road_Rash") && mods::superseded_by_builtin("fourth_studio-board_break_pack") && + mods::superseded_by_builtin("Fourth_Studio-Board_Break") && mods::superseded_by_builtin("Road_Rash") && + !mods::superseded_by_builtin("Fourth_Studio-Gothic_Tights") && !mods::superseded_by_builtin("Someone-Road_Rash_Remix"), + "Only the mods built in now are taken for them"); + const auto store = root / L"retire" / L"Mods"; + fs::create_directories(store / L"Fourth_Studio-Board_Break_Pack" / L"decks"); + fs::create_directories(store / L"Fourth_Studio-Road_Rash"); + fs::create_directories(store / L"Someone-Other_Mod"); + std::ofstream(store / L"Fourth_Studio-Road_Rash" / L"layout.toc") << "toc"; + std::ofstream(store / L"Someone-Other_Mod" / L"layout.toc") << "toc"; + std::ofstream(store / L"mods.json") + << R"({"schema":1,"mods":[{"name":"Fourth_Studio-Road_Rash","enabled":true},{"name":"Someone-Other_Mod","enabled":false}]})"; + auto retired = launcher_mods::retire_superseded(store); + std::sort(retired.begin(), retired.end()); + check(retired == std::vector{"Fourth_Studio-Board_Break_Pack", "Fourth_Studio-Road_Rash"} && + !fs::exists(store / L"Fourth_Studio-Road_Rash") && !fs::exists(store / L"Fourth_Studio-Board_Break_Pack") && + fs::exists(store / L"Someone-Other_Mod" / L"layout.toc"), + "The mods built in now are taken away, the others left"); + const auto remaining = mods::scan_mods(store.parent_path()); + check(remaining.issue.empty() && remaining.missing.empty() && remaining.entries.size() == 1 && + remaining.entries[0].mod.name == "Someone-Other_Mod" && !remaining.entries[0].enabled, + "mods.json forgets them and keeps the rest as it was"); + check(launcher_mods::retire_superseded(store).empty(), "Nothing is taken away twice"); + } + std::error_code ignored; fs::remove_all(root, ignored); if (failures) { diff --git a/Launcher/gui_mods_browse.cpp b/Launcher/gui_mods_browse.cpp index 1ed18c8..b30cdf7 100644 --- a/Launcher/gui_mods_browse.cpp +++ b/Launcher/gui_mods_browse.cpp @@ -1,5 +1,6 @@ #include "gui_internal.h" #include "gui_renderer.h" +#include "Engine/Vfs/builtin_packs.h" #include "mod_manager.h" #include "updater.h" @@ -215,6 +216,8 @@ std::vector visible_packages(const Store& store, const ts::I // or a category of that name. A deprecated one only shows once installed. if (nsfw(package)) continue; if (package.deprecated && !have) continue; + // ReSkate carries these itself now (Engine/Vfs/builtin_packs.h). + if (mods::superseded_by_builtin(ts::folder_for(package.full_name))) continue; if (!store.category.empty() && !package.in_category(store.category)) continue; if (!query.empty() && lower(package.title()).find(query) == std::string::npos && lower(package.owner).find(query) == std::string::npos && @@ -321,7 +324,8 @@ const thunderstore::Package* package_for(const Store& store, std::string_view fo std::vector updates(const Store& store, const thunderstore::Installed& installed) { std::vector result; for (const auto& package : store.packages) - if (ts::update_available(package, installed)) result.push_back(&package); + if (ts::update_available(package, installed) && !mods::superseded_by_builtin(ts::folder_for(package.full_name))) + result.push_back(&package); return result; } @@ -333,7 +337,9 @@ void start_store_install(ModsPanel& panel, std::vector pa std::vector expanded; for (const auto& package : packages) for (auto& needed : thunderstore::with_dependencies(package, panel.store.packages, have)) - if (std::none_of(expanded.begin(), expanded.end(), + // (A mod that needs Board Break or Road Rash has them built in: they are not installed.) + if (!mods::superseded_by_builtin(thunderstore::folder_for(needed.full_name)) && + std::none_of(expanded.begin(), expanded.end(), [&](const auto& queued) { return queued.full_name == needed.full_name; })) expanded.push_back(std::move(needed)); packages = std::move(expanded); diff --git a/Launcher/launch.cpp b/Launcher/launch.cpp index e4a28cc..bf8a344 100644 --- a/Launcher/launch.cpp +++ b/Launcher/launch.cpp @@ -5,6 +5,7 @@ #include "Engine/Vfs/content_cache_install.h" #include "Engine/Vfs/world_layer_scan.h" #include "launch.h" +#include "mod_manager.h" #include "text_encoding.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Game/Build/20260929/runtime.h" @@ -467,6 +468,10 @@ Session open_session(const std::string& log_level) { logging::Channel::launcher, "File logging is unavailable; debugger output remains active."); logging::write(logging::Level::info, logging::Channel::launcher, L"ReSkate launcher starting"); logging::write(logging::Level::info, logging::Channel::launcher, L"Current launch log: " + (session.paths.logs / L"ReSkate.log").wstring()); + // Mods ReSkate now carries itself are taken away before anything reads the Mods folder. + for (const auto& name : launcher_mods::retire_superseded(launcher_mods::mods_root(session.paths.directory))) + logging::write(logging::Level::info, logging::Channel::launcher, + "Mod removed: " + name + " is built into ReSkate now (it is in the Recycle Bin)"); return session; } diff --git a/Launcher/mod_manager.cpp b/Launcher/mod_manager.cpp index aee3c34..d6c4e6a 100644 --- a/Launcher/mod_manager.cpp +++ b/Launcher/mod_manager.cpp @@ -1,6 +1,7 @@ #include "mod_manager.h" #include "text_encoding.h" +#include "Engine/Vfs/builtin_packs.h" #include "Engine/Vfs/mod_list.h" #include "Engine/Core/Json/json.h" #include "Engine/Core/Platform/launcher_support.h" @@ -403,6 +404,31 @@ std::string install(const fs::path& mods, const fs::path& source, bool replace, } } +std::vector retire_superseded(const fs::path& mods) noexcept { + std::vector retired; + try { + std::vector> found; + std::error_code error; + for (fs::directory_iterator at(mods, error), end; !error && at != end; at.increment(error)) { + if (!at->is_directory(error)) continue; + const auto name = utf8(at->path().filename().wstring()); + if (mods::superseded_by_builtin(name)) found.emplace_back(name, at->path()); + } + for (const auto& [name, path] : found) { + try { + recycle(path); + retired.push_back(name); + } catch (...) {} + } + // mods.json keeps the rest as it was; a malformed one is left for the player to see. + if (!retired.empty()) { + const auto list = mods::scan_mods(mods.parent_path()); + if (list.issue.empty()) mods::save_mod_order(mods, list.entries); + } + } catch (...) {} + return retired; +} + void remove(const fs::path& mods, const std::string& name) { if (!mods::valid_mod_name(name)) fail("\"" + name + "\" is not a mod folder."); const auto target = mods / from_utf8(name); diff --git a/Launcher/mod_manager.h b/Launcher/mod_manager.h index 6e78f0e..596b598 100644 --- a/Launcher/mod_manager.h +++ b/Launcher/mod_manager.h @@ -5,6 +5,7 @@ #include #include #include +#include // The launcher's side of the Mods folder: installing mods from a .zip or a // folder, and removing them. Order and enabled state live in mods.json @@ -52,4 +53,10 @@ std::string install(const std::filesystem::path& mods_root, const std::filesyste // Moves Mods/ to the Recycle Bin. Throws a user-facing message. void remove(const std::filesystem::path& mods_root, const std::string& name); +// The mods ReSkate now carries itself (Engine/Vfs/builtin_packs.h: Fourth Studio's Board Break and +// Road Rash) go to the Recycle Bin and out of mods.json, so the built-in copy is the one that +// loads. Returns the folders taken away; one that will not go stays (the merge leaves it out). +// Never throws. +std::vector retire_superseded(const std::filesystem::path& mods_root) noexcept; + } // namespace dingosdk::launcher_mods diff --git a/assets/builtin/board_break.zip b/assets/builtin/board_break.zip new file mode 100644 index 0000000..b5b9af1 Binary files /dev/null and b/assets/builtin/board_break.zip differ diff --git a/assets/builtin/road_rash.zip b/assets/builtin/road_rash.zip new file mode 100644 index 0000000..09807d4 Binary files /dev/null and b/assets/builtin/road_rash.zip differ diff --git a/cmake/Apps.cmake b/cmake/Apps.cmake index b1ce263..46d5327 100644 --- a/cmake/Apps.cmake +++ b/cmake/Apps.cmake @@ -22,8 +22,9 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32) target_include_directories(dingosdk_oodle_path_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_dependencies(dingosdk_oodle_path_tests dingosdk_test_oodle_codec) add_test(NAME oodle_path COMMAND dingosdk_oodle_path_tests "$") - add_executable(dingosdk_mod_manager_tests Launcher/Test/mod_manager_tests.cpp Launcher/mod_manager.cpp) - target_link_libraries(dingosdk_mod_manager_tests PRIVATE dingosdk_mod_list dingosdk_launcher_support dingosdk_miniz shell32) + add_executable(dingosdk_mod_manager_tests Launcher/Test/mod_manager_tests.cpp Launcher/mod_manager.cpp + Engine/Vfs/builtin_packs.cpp) + target_link_libraries(dingosdk_mod_manager_tests PRIVATE dingosdk_mod_list dingosdk_launcher_support dingosdk_miniz dingosdk_zstd shell32) add_test(NAME launcher_mod_manager COMMAND dingosdk_mod_manager_tests) add_executable(dingosdk_thunderstore_tests Launcher/Test/thunderstore_tests.cpp Launcher/thunderstore.cpp) target_link_libraries(dingosdk_thunderstore_tests PRIVATE dingosdk_json dingosdk_miniz) @@ -189,6 +190,12 @@ if(WIN32) add_executable(dingosdk_emote_packer EmotePacker/main.cpp) target_link_libraries(dingosdk_emote_packer PRIVATE dingosdk_json windowscodecs ole32) set_target_properties(dingosdk_emote_packer PROPERTIES OUTPUT_NAME "ReSkateEmotePacker") + # ReSkateBuiltinPacker: a built-in content pack's zip (assets/builtin) -> the zstd pack + # ReSkate.dll carries (Engine/Vfs/builtin_pack_format.h). Run by the build (cmake/Runtime.cmake). + add_executable(dingosdk_builtin_packer BuiltinPacker/main.cpp) + target_include_directories(dingosdk_builtin_packer PRIVATE "${PROJECT_SOURCE_DIR}") + target_link_libraries(dingosdk_builtin_packer PRIVATE dingosdk_miniz dingosdk_zstd) + set_target_properties(dingosdk_builtin_packer PROPERTIES OUTPUT_NAME "ReSkateBuiltinPacker") endif() # ReSkate dedicated server: 1-1 native desktop GUI on Windows (with --headless support), headless on Linux. diff --git a/cmake/Core.cmake b/cmake/Core.cmake index b7e6850..c9fc748 100644 --- a/cmake/Core.cmake +++ b/cmake/Core.cmake @@ -107,7 +107,7 @@ if(WIN32) Engine/Vfs/mod_scoring.cpp Engine/Vfs/mod_store_copies.cpp Engine/Vfs/builtin_packs.cpp) - target_link_libraries(dingosdk_mods PUBLIC dingosdk_json dingosdk_mod_list PRIVATE dingosdk_game_archives dingosdk_native_db dingosdk_frostbite dingosdk_content_cache dingosdk_miniz shell32 bcrypt) + target_link_libraries(dingosdk_mods PUBLIC dingosdk_json dingosdk_mod_list PRIVATE dingosdk_game_archives dingosdk_native_db dingosdk_frostbite dingosdk_content_cache dingosdk_miniz dingosdk_zstd shell32 bcrypt) add_library(dingosdk_custom_scripts STATIC Engine/Scripting/custom_scripts.cpp) target_link_libraries(dingosdk_custom_scripts PUBLIC dingosdk_json PRIVATE dingosdk_initfs) diff --git a/cmake/Runtime.cmake b/cmake/Runtime.cmake index 3f4ef9d..7a7f1ba 100644 --- a/cmake/Runtime.cmake +++ b/cmake/Runtime.cmake @@ -318,16 +318,39 @@ if(EXISTS "${dingosdk_emotes}/emotes.json" AND EXISTS "${dingosdk_emotes}/emotes "EMOTES_PNG RCDATA \"${dingosdk_emotes_native}\\\\emotes.png\"\n") target_sources(dingosdk_runtime PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/generated/emotes.rc") endif() -# Game content ReSkate carries itself (Engine/Vfs/builtin_packs.h): the FPV drone's model, built -# by tools/models/build_drone_pack.py. Without the zip, other players' drones use a stand-in prop. -set(dingosdk_drone_pack "${PROJECT_SOURCE_DIR}/assets/builtin/fpv_drone.zip") -if(EXISTS "${dingosdk_drone_pack}") - file(TO_NATIVE_PATH "${dingosdk_drone_pack}" dingosdk_drone_pack_native) - string(REPLACE "\\" "\\\\" dingosdk_drone_pack_native "${dingosdk_drone_pack_native}") - file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin_packs.rc" - "BUILTIN_PACK_FPV_DRONE RCDATA \"${dingosdk_drone_pack_native}\"\n") +# Game content ReSkate carries itself (Engine/Vfs/builtin_packs.h), one zip each in assets/builtin: +# the FPV drone's model (tools/models/build_drone_pack.py; without it other players' drones use a +# stand-in prop), and Fourth Studio's Board Break and Road Rash packs. A missing zip is left out. +# Each zip is made into a zstd pack by ReSkateBuiltinPacker as the build goes (again only when the +# zip changes), which is what the DLL carries: compressed whole, a pack's repeated files cost +# next to nothing (Board Break's ten decks). +set(dingosdk_builtin_rc "") +set(dingosdk_builtin_packs "") +foreach(pack IN ITEMS "FPV_DRONE fpv_drone" "BOARD_BREAK board_break" "ROAD_RASH road_rash") + separate_arguments(pack) + list(GET pack 0 pack_resource) + list(GET pack 1 pack_file) + set(pack_zip "${PROJECT_SOURCE_DIR}/assets/builtin/${pack_file}.zip") + if(EXISTS "${pack_zip}") + set(pack_out "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin/${pack_file}.rskpack") + add_custom_command(OUTPUT "${pack_out}" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin" + COMMAND dingosdk_builtin_packer "${pack_zip}" "${pack_out}" + DEPENDS "${pack_zip}" dingosdk_builtin_packer + COMMENT "Packing built-in content ${pack_file}" + VERBATIM) + file(TO_NATIVE_PATH "${pack_out}" pack_out_native) + string(REPLACE "\\" "\\\\" pack_out_native "${pack_out_native}") + string(APPEND dingosdk_builtin_rc "BUILTIN_PACK_${pack_resource} RCDATA \"${pack_out_native}\"\n") + list(APPEND dingosdk_builtin_packs "${pack_out}") + endif() +endforeach() +if(dingosdk_builtin_rc) + add_custom_target(dingosdk_builtin_content DEPENDS ${dingosdk_builtin_packs}) + add_dependencies(dingosdk_runtime dingosdk_builtin_content) + file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin_packs.rc" "${dingosdk_builtin_rc}") target_sources(dingosdk_runtime PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin_packs.rc") - set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin_packs.rc" APPEND PROPERTY OBJECT_DEPENDS "${dingosdk_drone_pack}") + set_property(SOURCE "${CMAKE_CURRENT_BINARY_DIR}/generated/builtin_packs.rc" APPEND PROPERTY OBJECT_DEPENDS "${dingosdk_builtin_packs}") endif() # The music sources' setup page (Extension/Music/spotify_dashboard.h). file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/Extension/Music/setup_page.html" dingosdk_music_page_native)