Mod merge: an asset carried into many maps is written once

A carried asset was written again for every superbundle that carried it: 1.4 GB of 2.2 GB with 96 mods, which left no room for a live apply. Each one is now written once per install chunk, when its first superbundle settles.

Engine/Vfs/Test/mod_merge_added_assets_tests.cpp

@@ -14,6 +14,7 @@
 #include "Engine/Resource/cas_codec.h"
 #include "Engine/Resource/ebx_document.h"
 #include "Engine/Resource/toc.h"
+#include "Engine/Vfs/game_archives.h"
 #include "Engine/Vfs/mod_catalog.h"
 #include "Engine/Vfs/native_db.h"
 
@@ -23,6 +24,7 @@
 #include <filesystem>
 #include <fstream>
 #include <iostream>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -291,6 +293,8 @@ std::string describe(const mods::MergeReport& report) {
 constexpr char shared_toc[] = "test_shared.toc";
 constexpr char map_toc[] = "levels/test/map.toc";
 constexpr char map_superbundle[] = "Win32/levels/test/map";
+constexpr char second_map_toc[] = "levels/test/map2.toc";
+constexpr char second_map_superbundle[] = "Win32/levels/test/map2";
 
 // The level mod alone: what its copy of the bundle holds with nothing carried into it.
 int map_only() {
@@ -453,6 +457,95 @@ void replaced_while_the_game_runs() {
     report = mods::merge_mods(fixture.catalog, {}, {.live = true});
     expect(report.built && report.issue.empty() && !noted("placed again"), "replaced: placed again without a change\n" + describe(report));
 }
+// Where the merged copy of the bundle in `toc` keeps `asset`: its manifest is the region's first
+// file, stored raw in the patch's own archive, and the assets follow in manifest order.
+std::optional<fb::BundleFileInfo> merged_file(const Fixture& fixture, const std::string& toc, const std::string& asset) {
+    const auto patch = fixture.catalog.root / mods::generated_folder;
+    const auto path = patch / "Win32" / fs::path(toc);
+    if (!fs::exists(path)) return std::nullopt;
+    for (const auto& bundle : fb::read_toc(read(path)).bundles) {
+        if (bundle.name != bundle_name) continue;
+        const auto region = fb::read_bundle_region(bundle.region);
+        if (region.files.empty() || !region.inlineManifest.empty()) return std::nullopt;
+        const auto& first = region.files.front();
+        const auto archive = read((first.location.patch ? patch : fixture.catalog.data_root / "Data") / "Win32" / "pkg" /
+                                  vfs::archive_file(first.location.archive));
+        const auto manifest = fb::read_binary_bundle(std::span(archive).subspan(first.offset, first.size));
+        for (std::size_t index = 0; index < manifest.ebx.size(); ++index)
+            if (manifest.ebx[index].name == asset && 1 + index < region.files.size()) return region.files[1 + index];
+    }
+    return std::nullopt;
+}
+
+// Two maps each carry the music mod's song. Both copies read the one payload.
+void carried_payload_written_once() {
+    Fixture fixture("carried-once");
+    fixture.add("music", false, shared_toc, {}, music_assets(), music_resources());
+    fixture.add("map", true, map_toc, {map_superbundle}, game_copy());
+    fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy());
+    const auto report = mods::merge_mods(fixture.catalog);
+    expect(report.issue.empty() && report.built, "carried once: the merge builds the patch\n" + describe(report));
+    const auto first = merged_file(fixture, map_toc, "test/song");
+    const auto second = merged_file(fixture, second_map_toc, "test/song");
+    expect(first && second, "carried once: both maps' copies hold test/song");
+    if (first && second)
+        expect(first->location == second->location && first->offset == second->offset,
+               "carried once: the two copies read different payloads (byte " + std::to_string(first->offset) + " and byte " +
+                   std::to_string(second->offset) + ")");
+}
+
+// Every placement in every merged TOC lies inside an archive that exists, and the merged layout
+// declares only real archives.
+void every_placement_lands_in_an_archive() {
+    Fixture fixture("placements");
+    fixture.add("music", false, shared_toc, {}, music_assets(), music_resources());
+    fixture.add("map", true, map_toc, {map_superbundle}, game_copy());
+    fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy());
+    const auto report = mods::merge_mods(fixture.catalog);
+    expect(report.issue.empty() && report.built, "placements: the merge builds the patch\n" + describe(report));
+    const auto patch = fixture.catalog.root / mods::generated_folder;
+    std::size_t checked{};
+    for (const auto& entry : fs::recursive_directory_iterator(patch / "Win32")) {
+        if (entry.path().extension() != ".toc") continue;
+        for (const auto& bundle : fb::read_toc(read(entry.path())).bundles)
+            for (const auto& file : fb::read_bundle_region(bundle.region).files) {
+                const auto archive = (file.location.patch ? patch : fixture.catalog.data_root / "Data") / "Win32" / "pkg" /
+                                     vfs::archive_file(file.location.archive);
+                std::error_code error;
+                const auto size = fs::file_size(archive, error);
+                expect(!error && std::uint64_t{file.offset} + file.size <= size,
+                       "placements: " + bundle.name + " points at byte " + std::to_string(file.offset) + " of " +
+                           archive.filename().string() + (error ? ", which is not there" : ", past its end"));
+                ++checked;
+            }
+    }
+    expect(checked > 0, "placements: the merged TOCs hold placements");
+    const auto layout = vfs::read_layout(patch / "layout.toc");
+    const auto* table = layout.root.field("layeredInstallChunkFiles");
+    expect(table != nullptr, "placements: the merged layout declares archives");
+    if (table)
+        vfs::for_each_install_chunk_file(table->payload(), [&](std::uint32_t, std::uint16_t archive) {
+            expect(archive != 0xFFFE && archive != 0xFFFF, "placements: the layout declares marker archive " + std::to_string(archive));
+        });
+}
+
+// A merge built again from the same mods writes the same bytes.
+void two_merges_write_the_same_bytes() {
+    Fixture fixture("same-bytes");
+    fixture.add("music", false, shared_toc, {}, music_assets(), music_resources());
+    fixture.add("map", true, map_toc, {map_superbundle}, game_copy());
+    fixture.add("map2", true, second_map_toc, {second_map_superbundle}, game_copy());
+    const auto patch = fixture.catalog.root / mods::generated_folder;
+    auto report = mods::merge_mods(fixture.catalog);
+    expect(report.issue.empty() && report.built, "same bytes: the first merge\n" + describe(report));
+    const auto archive = read(patch / "Win32" / "pkg" / "cas_01.cas");
+    const auto toc = read(patch / "Win32" / fs::path(second_map_toc));
+    fs::remove(patch / "reskate-merge.stamp");
+    report = mods::merge_mods(fixture.catalog);
+    expect(report.issue.empty() && report.built && !report.reused, "same bytes: the second merge\n" + describe(report));
+    expect(read(patch / "Win32" / "pkg" / "cas_01.cas") == archive, "same bytes: the patch's own archive differs");
+    expect(read(patch / "Win32" / fs::path(second_map_toc)) == toc, "same bytes: the second map's TOC differs");
+}
 } // namespace
 
 int main() try {
@@ -467,6 +560,9 @@ int main() try {
     two_music_mods_both_carried_into_maps_copy(baseline);
     carried_into_another_bundle_with_the_list(true);
     carried_into_another_bundle_with_the_list(false);
+    carried_payload_written_once();
+    every_placement_lands_in_an_archive();
+    two_merges_write_the_same_bytes();
     if (failures) {
         std::cerr << failures << " failure(s)\n";
         return 1;

Engine/Vfs/mod_merge.cpp

@@ -442,7 +442,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con
                     for (std::size_t index = 0; index < job.merged.bundles.size(); ++index) {
                         auto region = fb::read_bundle_region(job.merged.bundles[index].region);
                         if (std::ranges::any_of(region.files, [](const fb::BundleFileInfo& file) {
-                                return CasStore::waits(file.location); }))
+                                return CasStore::placeholder(file.location); }))
                             job.held.emplace_back(index, std::move(region));
                     }
             } catch (...) {
@@ -509,7 +509,7 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con
             }
             for (auto& chunk : job.merged.chunks) job.store->settled(chunk.location, chunk.offset);
             for (const auto& [installChunk, archive] : job.used)
-                used.emplace(installChunk, archive == CasStore::waiting_archive ? manifestArchive : archive);
+                used.emplace(installChunk, CasStore::placeholder(archive) ? manifestArchive : archive);
             report.mergedBundles += job.report.mergedBundles;
             report.mergedAssets += job.report.mergedAssets;
             report.notes.insert(report.notes.end(), std::make_move_iterator(job.report.notes.begin()),

Engine/Vfs/mod_merge_cas_store.cpp

@@ -61,7 +61,7 @@ fb::BundleFileInfo CasStore::write(std::uint32_t installChunk, std::uint16_t arc
     const auto path = archive_path(installChunk, archive);
     if (waiting_) {
         // A placement that waits names no archive, so there can be only the one to settle into.
-        if (archive == waiting_archive || (archive_ && *archive_ != archive))
+        if (placeholder(archive) || (archive_ && *archive_ != archive))
             throw std::logic_error("A waiting store writes to one archive index");
         archive_ = archive;
         auto& held = held_[path.wstring()].bytes;
@@ -83,19 +83,47 @@ CasStore CasStore::waiting() const {
     copy.archive_.reset();
     copy.held_.clear();
     copy.offsets_.clear();
+    copy.carried_.clear();
+    copy.carriedIndex_.clear();
+    copy.carriedAt_.clear();
     return copy;
 }
 
+fb::BundleFileInfo CasStore::carry(std::uint32_t installChunk, std::uint16_t archive, std::span<const std::byte> encoded) {
+    if (!waiting_) throw std::logic_error("Only a waiting store carries payloads");
+    const std::pair key{installChunk, sha1_of(encoded)};
+    const auto [found, fresh] = carriedIndex_.try_emplace(key, carried_.size());
+    if (fresh) carried_.push_back({installChunk, archive, key.second, encoded, {}});
+    return {{true, installChunk, carried_archive}, static_cast<std::uint32_t>(found->second),
+            static_cast<std::uint32_t>(encoded.size())};
+}
+
 void CasStore::settle(CasStore& waiting) {
     // In path order: which file comes first makes no difference to where anything lands.
     for (auto& [path, held] : waiting.held_) {
         held.at = append(fs::path(path), held.bytes);
         held.bytes = {};
     }
+    // Then the carried assets, in the order the superbundle first named them. An asset an
+    // earlier superbundle settled in this install chunk is not written again.
+    for (auto& carried : waiting.carried_) {
+        const std::pair key{carried.installChunk, carried.sha1};
+        auto found = carriedAt_.find(key);
+        if (found == carriedAt_.end())
+            found = carriedAt_.emplace(key, write(carried.installChunk, carried.archive, carried.bytes)).first;
+        carried.at = found->second;
+    }
 }
 
 bool CasStore::settled(fb::CasIdentifier& location, std::uint32_t& offset) const {
-    if (!waits(location)) return false;
+    if (!placeholder(location)) return false;
+    if (location.archive == carried_archive) {
+        if (offset >= carried_.size()) throw std::logic_error("A carried placement names no asset its store holds");
+        const auto& at = carried_[offset].at;
+        location = at.location;
+        offset = at.offset;
+        return true;
+    }
     const auto found = archive_ ? held_.find(archive_path(location.installChunk, *archive_).wstring()) : held_.end();
     if (found == held_.end()) throw std::logic_error("A placement waits in a store that holds nothing for it");
     const auto moved = found->second.at + offset;

Engine/Vfs/mod_merge_combine.cpp

@@ -615,7 +615,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
                         auto placed = carriedAt.find({installChunk, addition});
                         if (placed == carriedAt.end())
                             placed = carriedAt.emplace(std::pair{installChunk, addition},
-                                store.write(installChunk, manifestArchive, addition->encoded)).first;
+                                store.carry(installChunk, manifestArchive, addition->encoded)).first;
                         additions.push_back({addition->asset, placed->second});
                         addedFrom.insert(addition->mod);
                     } catch (const std::exception& error) {

Engine/Vfs/mod_merge_internal.h

@@ -132,11 +132,20 @@ public:
     // own has been settled, settle() appends what it holds to the real archives, and
     // settled() then moves a placement that points into it to where that went.
     static constexpr std::uint16_t waiting_archive = 0xFFFF;
+    // An asset carried into copies in several superbundles. A waiting store names it by its
+    // place in the store's carried list, and settle() writes it once per install chunk.
+    static constexpr std::uint16_t carried_archive = 0xFFFE;
     [[nodiscard]] CasStore waiting() const;
-    [[nodiscard]] bool holding() const noexcept { return !held_.empty(); }
-    [[nodiscard]] static bool waits(const fb::CasIdentifier& location) noexcept {
-        return location.patch && location.archive == waiting_archive;
+    [[nodiscard]] bool holding() const noexcept { return !held_.empty() || !carried_.empty(); }
+    [[nodiscard]] static bool placeholder(std::uint16_t archive) noexcept {
+        return archive == waiting_archive || archive == carried_archive;
     }
+    [[nodiscard]] static bool placeholder(const fb::CasIdentifier& location) noexcept {
+        return location.patch && placeholder(location.archive);
+    }
+    // A waiting store only. The bytes must stay valid until settle().
+    [[nodiscard]] fb::BundleFileInfo carry(std::uint32_t installChunk, std::uint16_t archive,
+                                           std::span<const std::byte> encoded);
     void settle(CasStore& waiting);
     // True when the placement was one of this waiting store's; it now names the archive.
     bool settled(fb::CasIdentifier& location, std::uint32_t& offset) const;
@@ -160,6 +169,16 @@ private:
     bool waiting_{};
     std::optional<std::uint16_t> archive_;
     std::map<std::wstring, Held> held_;
+    struct Carried {
+        std::uint32_t installChunk{};
+        std::uint16_t archive{};
+        fb::Sha1 sha1;
+        std::span<const std::byte> bytes;
+        fb::BundleFileInfo at;   // where settle() put it
+    };
+    std::vector<Carried> carried_;                                                // waiting: first-reference order
+    std::map<std::pair<std::uint32_t, fb::Sha1>, std::size_t> carriedIndex_;     // waiting: into carried_
+    std::map<std::pair<std::uint32_t, fb::Sha1>, fb::BundleFileInfo> carriedAt_; // settled: one per install chunk
 };
 
 using ArchiveUse = std::set<std::pair<std::uint32_t, std::uint16_t>>;

cmake/Apps.cmake

@@ -101,7 +101,7 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32)
     target_include_directories(dingosdk_chunk_metadata_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}")
     add_test(NAME chunk_metadata_merge COMMAND dingosdk_chunk_metadata_merge_tests)
     add_executable(dingosdk_mod_merge_added_assets_tests Engine/Vfs/Test/mod_merge_added_assets_tests.cpp)
-    target_link_libraries(dingosdk_mod_merge_added_assets_tests PRIVATE dingosdk_mods)
+    target_link_libraries(dingosdk_mod_merge_added_assets_tests PRIVATE dingosdk_mods dingosdk_game_archives)
     target_include_directories(dingosdk_mod_merge_added_assets_tests PRIVATE "${PROJECT_SOURCE_DIR}")
     add_test(NAME mod_merge_added_assets COMMAND dingosdk_mod_merge_added_assets_tests)
     add_executable(dingosdk_item_list_merge_tests Engine/Vfs/Test/item_list_merge_tests.cpp)