diff --git a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp new file mode 100644 index 00000000..414c0f15 --- /dev/null +++ b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp @@ -0,0 +1,345 @@ +// Assets a mod ADDS to a bundle (new songs, say) while another mod ships its own copy of that +// bundle (a map does, for its levels). The merged bundle is built from one mod's copy, so +// without the carrying step the added assets are missing from it whenever the map's copy wins. +// +// The fixture is a small made-up game and two mods in a temp folder, written with the repo's +// own writers, so it needs no game files: +// game bundle "win32/test/shared": test/playlist, test/other +// map a level mod: its own level superbundle (a new TOC) with a copy of that bundle, as a +// custom map carries the core assets for its level; it edits nothing +// music an asset mod that changes test/playlist to name a new song, and adds the song, the +// wave the song names, the wave's sound-bank resource, and an asset nothing names. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/native_db.h" + +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +namespace db = dingosdk::native_db; +using namespace dingosdk; + +namespace { +using Bytes = std::vector; +constexpr std::uint32_t package = 0x1234; +constexpr char bundle_name[] = "win32/test/shared"; + +int failures = 0; +void expect(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +// ---- Bytes --------------------------------------------------------------------------------------- +void put32(Bytes& out, std::uint32_t value) { + for (int shift = 0; shift < 32; shift += 8) out.push_back(static_cast(value >> shift)); +} +void put_bytes(Bytes& out, std::string_view text) { + for (const char c : text) out.push_back(static_cast(c)); +} +void put_guid(Bytes& out, const fb::Guid& guid) { out.insert(out.end(), guid.bytes.begin(), guid.bytes.end()); } +fb::Guid guid(std::uint8_t seed) { + fb::Guid value; + value.bytes.fill(static_cast(seed)); + return value; +} +void write(const fs::path& path, std::span bytes) { + fs::create_directories(path.parent_path()); + std::ofstream(path, std::ios::binary).write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); +} +Bytes read(const fs::path& path) { + std::ifstream in(path, std::ios::binary | std::ios::ate); + Bytes bytes(static_cast(in.tellg())); + in.seekg(0); + in.read(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + return bytes; +} + +// ---- An EBX document with no instances: a file GUID and the files it imports. ----------------- +// That is all the merge reads from an asset to follow what it refers to. +void chunk(Bytes& out, const char (&id)[5], const Bytes& body) { + put_bytes(out, std::string_view(id, 4)); + put32(out, static_cast(body.size())); + out.insert(out.end(), body.begin(), body.end()); + if (body.size() & 1) out.push_back(std::byte{0}); +} +Bytes ebx_document(const fb::Guid& file, const std::vector& imports) { + Bytes fixup; + put_guid(fixup, file); + for (int i = 0; i < 6; ++i) put32(fixup, 0); // types, signatures, exported, instances, pointers, resource refs + put32(fixup, static_cast(imports.size())); + for (const auto& import : imports) { + put_guid(fixup, import); + put_guid(fixup, guid(0xEE)); // the imported class + } + for (int i = 0; i < 2; ++i) put32(fixup, 0); // import relocations, type-info references + for (int i = 0; i < 3; ++i) put32(fixup, 0); // array, boxed-value and string sections + const Bytes data(16); + Bytes extra; + put32(extra, 0); // arrays + put32(extra, 0); // boxed values + Bytes reflection; + for (int i = 0; i < 5; ++i) put32(reflection, 0); // signatures, types, fields, groups, mappings + Bytes body; + put_bytes(body, std::string_view("EBX\0", 4)); + chunk(body, "EFIX", fixup); + chunk(body, "EBXD", data); + chunk(body, "EBXX", extra); + chunk(body, "REFL", reflection); + Bytes riff; + put_bytes(riff, "RIFF"); + put32(riff, static_cast(body.size())); + riff.insert(riff.end(), body.begin(), body.end()); + return riff; +} + +// ---- The fixture's files ----------------------------------------------------------------------- +// layout.toc with one install chunk, `package`, whose archives live in Win32/pkg. +Bytes layout_toc(const std::vector& extra = {}) { + const auto leaf = [](unsigned type, const char* name, std::vector payload) { + db::Node node; + node.type = type; + node.named = true; + node.name = name; + node.owned = std::move(payload); + return node; + }; + const auto container = [](unsigned type, const char* name, std::vector children) { + db::Node node; + node.type = type; + node.named = *name != '\0'; + node.name = name; + node.terminated = true; + node.children = std::move(children); + return node; + }; + const auto records = [](const std::vector& names) { + std::vector rows; + for (const auto& name : names) rows.push_back(db::make_named_record("name", name)); + return rows; + }; + const auto low = static_cast(package & 0xFF), high = static_cast(package >> 8); + const auto chunk_entry = container(2, "", {db::make_named_record("name", "pkg").children.front(), + leaf(8, "persistentIndex", {low, high, 0, 0}), + container(1, "superbundles", records(extra))}); + const auto root = container(2, "", { + container(1, "superBundles", [&] { auto rows = records({"Win32/globals"}); for (auto& row : records(extra)) rows.push_back(std::move(row)); return rows; }()), + container(2, "installManifest", {container(1, "installChunks", {chunk_entry})}), + leaf(19, "layeredInstallChunkFiles", {1, 0, low, high, 0, 0, 0, 0})}); + const auto tree = db::write(root); + Bytes file(db::envelope_size + tree.size()); + std::memcpy(file.data(), db::magic, sizeof(db::magic)); + std::memcpy(file.data() + db::envelope_size, tree.data(), tree.size()); + return file; +} + +Bytes encoded(const Bytes& payload) { return fb::encode_cas(payload, {.compression = fb::CasCompression::raw}); } + +fb::Sha1 sha(const std::string& name, unsigned version) { + fb::Sha1 value; + for (std::size_t i = 0; i < name.size(); ++i) value.bytes[i % 19] ^= static_cast(name[i]); + value.bytes[19] = static_cast(version); + return value; +} + +struct Ebx { std::string name; unsigned version; Bytes payload; }; +struct Resource { std::string name; Bytes payload; }; + +fb::BinaryBundle manifest_of(const std::vector& assets, const std::vector& resources) { + fb::BinaryBundle manifest; + for (const auto& asset : assets) { + fb::BundleAsset entry; + entry.kind = fb::AssetKind::ebx; + entry.name = asset.name; + entry.sha1 = sha(asset.name, asset.version); + entry.originalSize = asset.payload.size(); + manifest.ebx.push_back(std::move(entry)); + } + for (const auto& resource : resources) { + fb::BundleAsset entry; + entry.kind = fb::AssetKind::resource; + entry.name = resource.name; + entry.sha1 = sha(resource.name, 0); + entry.resourceId = 7; + entry.resourceType = 0xb2c465f6; + entry.resourceMeta.assign(16, std::byte{0}); + entry.originalSize = resource.payload.size(); + manifest.resources.push_back(std::move(entry)); + } + return manifest; +} + +struct Fixture { + fs::path root; + mods::Catalog catalog; + + explicit Fixture(const std::string& name) { + wchar_t temp[MAX_PATH]{}; + GetTempPathW(MAX_PATH, temp); + root = fs::path(temp) / ("reskate-merge-added-" + std::to_string(GetCurrentProcessId()) + "-" + name); + fs::remove_all(root); + catalog.data_root = root / "game"; + catalog.root = catalog.data_root / "Mods"; + catalog.present = true; + // The game: two assets in one bundle, stored in the game's own archive. + const std::vector assets{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}}; + Bytes archive; + std::vector files; + for (const auto& asset : assets) { + const auto payload = encoded(asset.payload); + files.push_back({{false, package, 1}, static_cast(archive.size()), static_cast(payload.size())}); + archive.insert(archive.end(), payload.begin(), payload.end()); + } + const std::vector bundles{{bundle_name, fb::write_bundle_region(files, fb::write_binary_bundle(manifest_of(assets, {}))), 1}}; + write(catalog.data_root / "Data" / "layout.toc", layout_toc()); + write(catalog.data_root / "Data" / "Win32" / "test_shared.toc", fb::write_patch_toc(bundles)); + write(catalog.data_root / "Data" / "Win32" / "pkg" / "cas_01.cas", archive); + } + ~Fixture() { + std::error_code ignored; + fs::remove_all(root, ignored); + } + + // A mod with its own copy of the bundle: the manifest first (raw), then every payload. + void add(const std::string& name, bool levels, const std::string& toc, const std::vector& superbundles, + const std::vector& assets, const std::vector& resources = {}) { + const auto listing = fb::write_binary_bundle(manifest_of(assets, resources)); + Bytes archive(listing); + std::vector files{{{true, package, 1}, 0, static_cast(listing.size())}}; + const auto store = [&](const Bytes& payload) { + const auto bytes = encoded(payload); + files.push_back({{true, package, 1}, static_cast(archive.size()), static_cast(bytes.size())}); + archive.insert(archive.end(), bytes.begin(), bytes.end()); + }; + for (const auto& asset : assets) store(asset.payload); + for (const auto& resource : resources) store(resource.payload); + const std::vector bundles{{bundle_name, fb::write_bundle_region(files), 1}}; + const auto directory = catalog.root / name; + write(directory / "layout.toc", layout_toc(superbundles)); + write(directory / "Win32" / fs::path(toc), fb::write_patch_toc(bundles)); + write(directory / "Win32" / "pkg" / "cas_01.cas", archive); + mods::Mod mod; + mod.name = name; + mod.directory = directory; + mod.provides_layout = true; + mod.provides_levels = levels; + catalog.mods.push_back(std::move(mod)); + } + + // How many files the merged copy of the bundle in `toc` lists, or -1 when the merge left none. + int merged_files(const std::string& relative) const { + const auto toc = catalog.root / mods::generated_folder / "Win32" / fs::path(relative); + if (!fs::exists(toc)) return -1; + for (const auto& bundle : fb::read_toc(read(toc)).bundles) + if (bundle.name == bundle_name) return static_cast(fb::read_bundle_region(bundle.region).files.size()); + return -1; + } +}; + +// The game's copy of the bundle, as a level mod carries it. +std::vector game_copy() { + return {{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}}; +} +// The music mod's bundle: the playlist now names the song (a change), the song names its wave, +// and one more asset nothing names. +std::vector music_assets() { + return {{"test/playlist", 1, ebx_document(guid(1), {guid(10)})}, + {"test/other", 0, ebx_document(guid(2), {})}, + {"test/song", 0, ebx_document(guid(10), {guid(11)})}, + {"test/wave", 0, ebx_document(guid(11), {})}, + {"test/unrelated", 0, ebx_document(guid(12), {})}}; +} +std::vector music_resources() { return {{"test/wave", {std::byte{1}, std::byte{2}, std::byte{3}}}}; } + +bool noted(const mods::MergeReport& report, const std::string& text) { + for (const auto& note : report.notes) + if (note == text) return true; + return false; +} +std::string describe(const mods::MergeReport& report) { + std::string text = "issue: '" + report.issue + "'"; + for (const auto& note : report.notes) text += "\n note: " + note; + for (const auto& [mod, problems] : report.problems) + for (const auto& problem : problems) text += "\n " + mod + ": " + problem; + return text; +} + +constexpr char shared_toc[] = "test_shared.toc"; +constexpr char map_toc[] = "levels/test/map.toc"; +constexpr char map_superbundle[] = "Win32/levels/test/map"; + +// The level mod alone: what its copy of the bundle holds with nothing carried into it. +int map_only() { + Fixture fixture("map-only"); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "map only: the merge builds the patch\n" + describe(report)); + return fixture.merged_files(map_toc); +} + +// The music mod edits the game's TOC; the map's copy of the bundle is in its own TOC, so nothing +// of the music mod reaches it unless the merge carries it. The playlist change always did (it is a +// change to an asset the map's copy has); the song, its wave and the wave's resource are what +// the changed playlist leads to. The asset nothing names must stay out. Either mod may come first. +void carried_into_the_maps_copy(bool map_first, int baseline) { + const std::string order = map_first ? "map first" : "music first"; + Fixture fixture(map_first ? "map-first" : "music-first"); + const auto add_map = [&] { fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); }; + const auto add_music = [&] { fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); }; + if (map_first) { add_map(); add_music(); } else { add_music(); add_map(); } + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, order + ": the merge builds the patch\n" + describe(report)); + expect(noted(report, std::string("map: ") + bundle_name + ": 3 asset(s) added by other mods, e.g. test/song"), + order + ": the map's copy receives them\n" + describe(report)); + const auto files = fixture.merged_files(map_toc); + expect(files == baseline + 3, order + ": the map's copy gained those three files and not the unnamed asset (" + + std::to_string(files) + " files, " + std::to_string(baseline) + " without the music mod)"); +} + +// Another asset mod ships the game's copy of the bundle in the TOC the music mod ships, the way +// cosmetic mods share one. That copy merges with the music mod's bundle, which already has the +// song, so nothing is carried into it (no "added by other mods" note), and the merged bundle is +// exactly the union: the manifest, the music mod's five EBX and its resource. +void copy_in_the_adders_toc_is_left_alone() { + Fixture fixture("same-toc"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("other", false, shared_toc, {}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "same TOC: the merge builds the patch\n" + describe(report)); + for (const auto& note : report.notes) + expect(note.find("added by other mods") == std::string::npos, + "same TOC: nothing is carried into a copy in the adder's own TOC: " + note); + const auto expected = static_cast(1 + music_assets().size() + music_resources().size()); + const auto files = fixture.merged_files(shared_toc); + expect(files == expected, "same TOC: the merged bundle is the union, " + std::to_string(expected) + " files (" + + std::to_string(files) + " files)"); +} +} // namespace + +int main() try { + const auto baseline = map_only(); + expect(baseline > 0, "map only: the map's copy exists (" + std::to_string(baseline) + ")"); + carried_into_the_maps_copy(true, baseline); + carried_into_the_maps_copy(false, baseline); + copy_in_the_adders_toc_is_left_alone(); + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "mod merge added assets: ok\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/cmake/Apps.cmake b/cmake/Apps.cmake index be7f09f2..0ce1c0c4 100644 --- a/cmake/Apps.cmake +++ b/cmake/Apps.cmake @@ -48,6 +48,10 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32) target_link_libraries(dingosdk_mod_merge_unshift_tests PRIVATE dingosdk_mods) target_include_directories(dingosdk_mod_merge_unshift_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME mod_merge_unshift COMMAND dingosdk_mod_merge_unshift_tests) + add_executable(dingosdk_mod_merge_added_assets_tests Engine/Vfs/Test/mod_merge_added_assets_tests.cpp) + target_link_libraries(dingosdk_mod_merge_added_assets_tests PRIVATE dingosdk_mods) + target_include_directories(dingosdk_mod_merge_added_assets_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME mod_merge_added_assets COMMAND dingosdk_mod_merge_added_assets_tests) endif() if(WIN32)