From 89927bf90f346dbbf2ffef1dd5799f2c7ab8f062 Mon Sep 17 00:00:00 2001 From: worthlessnorms <110003169+worthlessnorms@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:55:25 -0500 Subject: [PATCH] Mod merge: an asset carried into many maps is written once A carried asset was written again for every superbundle that carried it: 1.4 GB of 2.2 GB with 96 mods, which left no room for a live apply. Each one is now written once per install chunk, when its first superbundle settles. --- .../Vfs/Test/mod_merge_added_assets_tests.cpp | 96 +++++++++++++++++++ Engine/Vfs/mod_merge.cpp | 4 +- Engine/Vfs/mod_merge_cas_store.cpp | 32 ++++++- Engine/Vfs/mod_merge_combine.cpp | 2 +- Engine/Vfs/mod_merge_internal.h | 25 ++++- cmake/Apps.cmake | 2 +- 6 files changed, 152 insertions(+), 9 deletions(-) diff --git a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp index 2a450d3..38f08f7 100644 --- a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp +++ b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp @@ -14,6 +14,7 @@ #include "Engine/Resource/cas_codec.h" #include "Engine/Resource/ebx_document.h" #include "Engine/Resource/toc.h" +#include "Engine/Vfs/game_archives.h" #include "Engine/Vfs/mod_catalog.h" #include "Engine/Vfs/native_db.h" @@ -23,6 +24,7 @@ #include #include #include +#include #include #include @@ -291,6 +293,8 @@ std::string describe(const mods::MergeReport& report) { constexpr char shared_toc[] = "test_shared.toc"; constexpr char map_toc[] = "levels/test/map.toc"; constexpr char map_superbundle[] = "Win32/levels/test/map"; +constexpr char second_map_toc[] = "levels/test/map2.toc"; +constexpr char second_map_superbundle[] = "Win32/levels/test/map2"; // The level mod alone: what its copy of the bundle holds with nothing carried into it. int map_only() { @@ -453,6 +457,95 @@ void replaced_while_the_game_runs() { report = mods::merge_mods(fixture.catalog, {}, {.live = true}); expect(report.built && report.issue.empty() && !noted("placed again"), "replaced: placed again without a change\n" + describe(report)); } +// Where the merged copy of the bundle in `toc` keeps `asset`: its manifest is the region's first +// file, stored raw in the patch's own archive, and the assets follow in manifest order. +std::optional merged_file(const Fixture& fixture, const std::string& toc, const std::string& asset) { + const auto patch = fixture.catalog.root / mods::generated_folder; + const auto path = patch / "Win32" / fs::path(toc); + if (!fs::exists(path)) return std::nullopt; + for (const auto& bundle : fb::read_toc(read(path)).bundles) { + if (bundle.name != bundle_name) continue; + const auto region = fb::read_bundle_region(bundle.region); + if (region.files.empty() || !region.inlineManifest.empty()) return std::nullopt; + const auto& first = region.files.front(); + const auto archive = read((first.location.patch ? patch : fixture.catalog.data_root / "Data") / "Win32" / "pkg" / + vfs::archive_file(first.location.archive)); + const auto manifest = fb::read_binary_bundle(std::span(archive).subspan(first.offset, first.size)); + for (std::size_t index = 0; index < manifest.ebx.size(); ++index) + if (manifest.ebx[index].name == asset && 1 + index < region.files.size()) return region.files[1 + index]; + } + return std::nullopt; +} + +// Two maps each carry the music mod's song. Both copies read the one payload. +void carried_payload_written_once() { + Fixture fixture("carried-once"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "carried once: the merge builds the patch\n" + describe(report)); + const auto first = merged_file(fixture, map_toc, "test/song"); + const auto second = merged_file(fixture, second_map_toc, "test/song"); + expect(first && second, "carried once: both maps' copies hold test/song"); + if (first && second) + expect(first->location == second->location && first->offset == second->offset, + "carried once: the two copies read different payloads (byte " + std::to_string(first->offset) + " and byte " + + std::to_string(second->offset) + ")"); +} + +// Every placement in every merged TOC lies inside an archive that exists, and the merged layout +// declares only real archives. +void every_placement_lands_in_an_archive() { + Fixture fixture("placements"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "placements: the merge builds the patch\n" + describe(report)); + const auto patch = fixture.catalog.root / mods::generated_folder; + std::size_t checked{}; + for (const auto& entry : fs::recursive_directory_iterator(patch / "Win32")) { + if (entry.path().extension() != ".toc") continue; + for (const auto& bundle : fb::read_toc(read(entry.path())).bundles) + for (const auto& file : fb::read_bundle_region(bundle.region).files) { + const auto archive = (file.location.patch ? patch : fixture.catalog.data_root / "Data") / "Win32" / "pkg" / + vfs::archive_file(file.location.archive); + std::error_code error; + const auto size = fs::file_size(archive, error); + expect(!error && std::uint64_t{file.offset} + file.size <= size, + "placements: " + bundle.name + " points at byte " + std::to_string(file.offset) + " of " + + archive.filename().string() + (error ? ", which is not there" : ", past its end")); + ++checked; + } + } + expect(checked > 0, "placements: the merged TOCs hold placements"); + const auto layout = vfs::read_layout(patch / "layout.toc"); + const auto* table = layout.root.field("layeredInstallChunkFiles"); + expect(table != nullptr, "placements: the merged layout declares archives"); + if (table) + vfs::for_each_install_chunk_file(table->payload(), [&](std::uint32_t, std::uint16_t archive) { + expect(archive != 0xFFFE && archive != 0xFFFF, "placements: the layout declares marker archive " + std::to_string(archive)); + }); +} + +// A merge built again from the same mods writes the same bytes. +void two_merges_write_the_same_bytes() { + Fixture fixture("same-bytes"); + fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); + fixture.add("map", true, map_toc, {map_superbundle}, game_copy()); + fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy()); + const auto patch = fixture.catalog.root / mods::generated_folder; + auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, "same bytes: the first merge\n" + describe(report)); + const auto archive = read(patch / "Win32" / "pkg" / "cas_01.cas"); + const auto toc = read(patch / "Win32" / fs::path(second_map_toc)); + fs::remove(patch / "reskate-merge.stamp"); + report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built && !report.reused, "same bytes: the second merge\n" + describe(report)); + expect(read(patch / "Win32" / "pkg" / "cas_01.cas") == archive, "same bytes: the patch's own archive differs"); + expect(read(patch / "Win32" / fs::path(second_map_toc)) == toc, "same bytes: the second map's TOC differs"); +} } // namespace int main() try { @@ -467,6 +560,9 @@ int main() try { two_music_mods_both_carried_into_maps_copy(baseline); carried_into_another_bundle_with_the_list(true); carried_into_another_bundle_with_the_list(false); + carried_payload_written_once(); + every_placement_lands_in_an_archive(); + two_merges_write_the_same_bytes(); if (failures) { std::cerr << failures << " failure(s)\n"; return 1; diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index 219f505..4a96e09 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -442,7 +442,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con for (std::size_t index = 0; index < job.merged.bundles.size(); ++index) { auto region = fb::read_bundle_region(job.merged.bundles[index].region); if (std::ranges::any_of(region.files, [](const fb::BundleFileInfo& file) { - return CasStore::waits(file.location); })) + return CasStore::placeholder(file.location); })) job.held.emplace_back(index, std::move(region)); } } catch (...) { @@ -509,7 +509,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con } for (auto& chunk : job.merged.chunks) job.store->settled(chunk.location, chunk.offset); for (const auto& [installChunk, archive] : job.used) - used.emplace(installChunk, archive == CasStore::waiting_archive ? manifestArchive : archive); + used.emplace(installChunk, CasStore::placeholder(archive) ? manifestArchive : archive); report.mergedBundles += job.report.mergedBundles; report.mergedAssets += job.report.mergedAssets; report.notes.insert(report.notes.end(), std::make_move_iterator(job.report.notes.begin()), diff --git a/Engine/Vfs/mod_merge_cas_store.cpp b/Engine/Vfs/mod_merge_cas_store.cpp index c7e0228..ab55c4e 100644 --- a/Engine/Vfs/mod_merge_cas_store.cpp +++ b/Engine/Vfs/mod_merge_cas_store.cpp @@ -61,7 +61,7 @@ fb::BundleFileInfo CasStore::write(std::uint32_t installChunk, std::uint16_t arc const auto path = archive_path(installChunk, archive); if (waiting_) { // A placement that waits names no archive, so there can be only the one to settle into. - if (archive == waiting_archive || (archive_ && *archive_ != archive)) + if (placeholder(archive) || (archive_ && *archive_ != archive)) throw std::logic_error("A waiting store writes to one archive index"); archive_ = archive; auto& held = held_[path.wstring()].bytes; @@ -83,19 +83,47 @@ CasStore CasStore::waiting() const { copy.archive_.reset(); copy.held_.clear(); copy.offsets_.clear(); + copy.carried_.clear(); + copy.carriedIndex_.clear(); + copy.carriedAt_.clear(); return copy; } +fb::BundleFileInfo CasStore::carry(std::uint32_t installChunk, std::uint16_t archive, std::span encoded) { + if (!waiting_) throw std::logic_error("Only a waiting store carries payloads"); + const std::pair key{installChunk, sha1_of(encoded)}; + const auto [found, fresh] = carriedIndex_.try_emplace(key, carried_.size()); + if (fresh) carried_.push_back({installChunk, archive, key.second, encoded, {}}); + return {{true, installChunk, carried_archive}, static_cast(found->second), + static_cast(encoded.size())}; +} + void CasStore::settle(CasStore& waiting) { // In path order: which file comes first makes no difference to where anything lands. for (auto& [path, held] : waiting.held_) { held.at = append(fs::path(path), held.bytes); held.bytes = {}; } + // Then the carried assets, in the order the superbundle first named them. An asset an + // earlier superbundle settled in this install chunk is not written again. + for (auto& carried : waiting.carried_) { + const std::pair key{carried.installChunk, carried.sha1}; + auto found = carriedAt_.find(key); + if (found == carriedAt_.end()) + found = carriedAt_.emplace(key, write(carried.installChunk, carried.archive, carried.bytes)).first; + carried.at = found->second; + } } bool CasStore::settled(fb::CasIdentifier& location, std::uint32_t& offset) const { - if (!waits(location)) return false; + if (!placeholder(location)) return false; + if (location.archive == carried_archive) { + if (offset >= carried_.size()) throw std::logic_error("A carried placement names no asset its store holds"); + const auto& at = carried_[offset].at; + location = at.location; + offset = at.offset; + return true; + } const auto found = archive_ ? held_.find(archive_path(location.installChunk, *archive_).wstring()) : held_.end(); if (found == held_.end()) throw std::logic_error("A placement waits in a store that holds nothing for it"); const auto moved = found->second.at + offset; diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 0ff70ca..9c82a79 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -615,7 +615,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, auto placed = carriedAt.find({installChunk, addition}); if (placed == carriedAt.end()) placed = carriedAt.emplace(std::pair{installChunk, addition}, - store.write(installChunk, manifestArchive, addition->encoded)).first; + store.carry(installChunk, manifestArchive, addition->encoded)).first; additions.push_back({addition->asset, placed->second}); addedFrom.insert(addition->mod); } catch (const std::exception& error) { diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index df9cbf5..b34f9b5 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -132,11 +132,20 @@ public: // own has been settled, settle() appends what it holds to the real archives, and // settled() then moves a placement that points into it to where that went. static constexpr std::uint16_t waiting_archive = 0xFFFF; + // An asset carried into copies in several superbundles. A waiting store names it by its + // place in the store's carried list, and settle() writes it once per install chunk. + static constexpr std::uint16_t carried_archive = 0xFFFE; [[nodiscard]] CasStore waiting() const; - [[nodiscard]] bool holding() const noexcept { return !held_.empty(); } - [[nodiscard]] static bool waits(const fb::CasIdentifier& location) noexcept { - return location.patch && location.archive == waiting_archive; + [[nodiscard]] bool holding() const noexcept { return !held_.empty() || !carried_.empty(); } + [[nodiscard]] static bool placeholder(std::uint16_t archive) noexcept { + return archive == waiting_archive || archive == carried_archive; } + [[nodiscard]] static bool placeholder(const fb::CasIdentifier& location) noexcept { + return location.patch && placeholder(location.archive); + } + // A waiting store only. The bytes must stay valid until settle(). + [[nodiscard]] fb::BundleFileInfo carry(std::uint32_t installChunk, std::uint16_t archive, + std::span encoded); void settle(CasStore& waiting); // True when the placement was one of this waiting store's; it now names the archive. bool settled(fb::CasIdentifier& location, std::uint32_t& offset) const; @@ -160,6 +169,16 @@ private: bool waiting_{}; std::optional archive_; std::map held_; + struct Carried { + std::uint32_t installChunk{}; + std::uint16_t archive{}; + fb::Sha1 sha1; + std::span bytes; + fb::BundleFileInfo at; // where settle() put it + }; + std::vector carried_; // waiting: first-reference order + std::map, std::size_t> carriedIndex_; // waiting: into carried_ + std::map, fb::BundleFileInfo> carriedAt_; // settled: one per install chunk }; using ArchiveUse = std::set>; diff --git a/cmake/Apps.cmake b/cmake/Apps.cmake index b1ce263..7fdb017 100644 --- a/cmake/Apps.cmake +++ b/cmake/Apps.cmake @@ -101,7 +101,7 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32) target_include_directories(dingosdk_chunk_metadata_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME chunk_metadata_merge COMMAND dingosdk_chunk_metadata_merge_tests) add_executable(dingosdk_mod_merge_added_assets_tests Engine/Vfs/Test/mod_merge_added_assets_tests.cpp) - target_link_libraries(dingosdk_mod_merge_added_assets_tests PRIVATE dingosdk_mods) + target_link_libraries(dingosdk_mod_merge_added_assets_tests PRIVATE dingosdk_mods dingosdk_game_archives) target_include_directories(dingosdk_mod_merge_added_assets_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME mod_merge_added_assets COMMAND dingosdk_mod_merge_added_assets_tests) add_executable(dingosdk_item_list_merge_tests Engine/Vfs/Test/item_list_merge_tests.cpp)