Mod merge: carry the assets a mod adds into the copies maps carry

Maps ship their own copy of game bundles such as
bam_levelroot/bam_coregameassets, and that copy loads instead of the game's
on their levels. Changes an asset mod makes to the game's EBX already
propagate into those copies; assets it adds did not. A mod that appends a new
song to the master music playlist therefore left the playlist naming a song
the map's copy never contained, so the song was missing on custom maps.

collect_asset_overrides now also records what an asset mod adds to a game
bundle, and combine inserts it into other mods' copies of the same bundle,
after their own assets, unless the copy already has that name:
- an added EBX is carried only when one of the mod's recorded changes imports
  it, or another carried addition does (the playlist names the song, the song
  imports its wave), so an addition never travels without the change that
  needs it;
- a resource the mod adds under the same name as a carried EBX goes with it
  (a wave's sound-bank resource registers the wave; without it the game
  crashes instantiating the wave);
- TOC chunks the mod adds that its carried EBX name are listed in each
  superbundle those assets go into, pointing at where the mod's archive
  landed (a map's superbundle resolves chunks from its own TOC).

A copy in the TOC the adding mod ships its own bundle in (cosmetic mods share one)
merges with that mod's bundle and already has the asset, so only copies in other
superbundles, i.e. maps' level TOCs, receive additions.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Engine/Vfs/mod_merge.cpp

@@ -291,7 +291,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con
             std::erase_if(superbundles, [](const std::string& relative) { return lower(relative) == root_level; });
         }
         if (options.live) report.load_screens = read_load_screens(mods, modFiles, store, baseRoot, gameRoot, report);
-        const auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report);
+        auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report);
+        // Carried chunks point into their mod's archives; move them to where those landed.
+        for (auto& [name, chunks] : overrides.chunks)
+            for (const auto* mod : mods)
+                if (mod->name == name)
+                    for (auto& chunk : chunks) store.shift(chunk.location, chunk.offset, &placements[mod]);
 
         for (const auto& relative : superbundles) {
             if (keepRoot && lower(relative) == root_level) {

Engine/Vfs/mod_merge_combine.cpp

@@ -147,6 +147,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
         std::string example;
     };
     std::map<std::string, ArchiveFault> archiveFaults;
+    // Mods whose added assets went into a copy here: their added chunks come too.
+    std::set<std::string> addedFrom;
 
     const auto absorb = [&](const fb::TocBundle& bundle, const ArchivePlacement* placement,
                             bool isBase, const fs::path& root) {
@@ -300,8 +302,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
             std::string example;
             for (std::size_t index = 0; index < casManifest.ebx.size(); ++index) {
                 auto& asset = casManifest.ebx[index];
-                const auto named = overrides.find(lower(asset.name));
-                if (named == overrides.end()) continue;
+                const auto named = overrides.changed.find(lower(asset.name));
+                if (named == overrides.changed.end()) continue;
                 const auto change = named->second.find(asset.sha1);
                 const auto at = 1 + index;
                 if (change == named->second.end() || change->second.mod == modName || at >= region.files.size())
@@ -321,8 +323,37 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
                 report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(overridden.size()) +
                     " asset(s) take another mod's change, e.g. " + example);
         }
+        // Assets another mod added to the game's copy of this bundle: this copy
+        // loads instead of the game's on this mod's levels, so it gets them too.
+        std::vector<Asset> additions;
+        if (propagate && casBacked) {
+            if (const auto found = overrides.added.find(key); found != overrides.added.end()) {
+                std::set<std::string, std::less<>> present;
+                for (const auto& asset : casManifest.ebx) present.insert(asset_key(asset));
+                for (const auto& asset : casManifest.resources) present.insert(asset_key(asset));
+                // A copy in the TOC the adder ships itself merges with the adder's bundle and has
+                // its assets already; only copies in other superbundles (maps) need them carried.
+                const auto here = lower(relative);
+                for (const auto& addition : found->second) {
+                    if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk ||
+                        addition.toc == here || present.contains(asset_key(addition.asset))) continue;
+                    try {
+                        additions.push_back({addition.asset, store.write(region.files.front().location.installChunk,
+                                                                         manifestArchive, addition.encoded)});
+                        addedFrom.insert(addition.mod);
+                    } catch (const std::exception& error) {
+                        report.notes.push_back(modName + ": " + bundle.name + ": " + addition.asset.name +
+                            " from " + addition.mod + " could not be added (" + error.what() + ")");
+                    }
+                }
+                if (!additions.empty())
+                    report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(additions.size()) +
+                        " asset(s) added by other mods, e.g. " + additions.front().asset.name);
+            }
+        }
         // Nothing moved in this bundle: it passes through whole, as before.
-        if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty()) casBacked = false;
+        if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty() && additions.empty())
+            casBacked = false;
 
         for (auto& file : region.files) {
             store.shift(file.location, file.offset, placement);
@@ -333,6 +364,16 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
                 region.files[index] = file;
                 used.emplace(file.location.installChunk, file.location.archive);
             }
+        // Added EBX go after this copy's own EBX and added resources after its own
+        // resources, ahead of its chunks; after the indexed moves above so their
+        // positions still hold.
+        for (auto& addition : additions) {
+            const auto ebx = addition.asset.kind == fb::AssetKind::ebx;
+            const auto at = 1 + casManifest.ebx.size() + (ebx ? 0 : casManifest.resources.size());
+            (ebx ? casManifest.ebx : casManifest.resources).push_back(std::move(addition.asset));
+            region.files.insert(region.files.begin() + static_cast<std::ptrdiff_t>(at), addition.file);
+            used.emplace(addition.file.location.installChunk, addition.file.location.archive);
+        }
         const auto fresh = !states.contains(key);
         if (fresh) { order.push_back(bundle.name); states.emplace(key, BundleState{}); }
         auto& state = states.at(key);
@@ -637,6 +678,20 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
             merged.chunks.push_back(chunk);
         }
     }
+    // This superbundle resolves chunks from its own TOC: the audio of a song another
+    // mod added to a bundle copied here has to be listed here as well.
+    for (const auto& name : addedFrom) {
+        const auto found = overrides.chunks.find(name);
+        if (found == overrides.chunks.end()) continue;
+        std::size_t listed{};
+        for (const auto& chunk : found->second) {
+            if (!chunkAt.emplace(chunk.guid, merged.chunks.size()).second) continue;
+            merged.chunks.push_back(chunk);
+            used.emplace(chunk.location.installChunk, chunk.location.archive);
+            ++listed;
+        }
+        if (listed) report.notes.push_back(relative + ": " + std::to_string(listed) + " chunk(s) from " + name);
+    }
 
     if (sources.size() > 1) {
         report.notes.push_back(relative + ": combined " + std::to_string(sources.size()) +

Engine/Vfs/mod_merge_internal.h

@@ -175,12 +175,37 @@ struct AssetOverride {
     std::uint64_t originalSize{};
     std::vector<std::byte> encoded;   // its payload, as stored in cas
 };
-// By lower-case asset name, then the game's sha1 the change replaces.
-using AssetOverrides = std::map<std::string, std::map<fb::Sha1, AssetOverride>, std::less<>>;
+// An EBX asset a mod added to one of the game's bundles, or a resource it added
+// under the same name as such an asset (a wave's sound-bank). Maps carry their own
+// copy of a bundle like bam_coregameassets, and that copy loads instead of the
+// game's on their levels; an edit to a list there (the master music playlist)
+// propagates as a change, so the asset it now names has to come along too.
+struct AssetAddition {
+    std::string mod;
+    fb::BundleAsset asset;
+    std::vector<std::byte> encoded;   // its payload, as stored in cas
+    // The lower-case relative path of the TOC the mod ships this bundle in. A copy of the
+    // bundle in that same TOC merges with the mod's own bundle and already has the asset;
+    // only copies in other superbundles (a map's level TOC) need it carried.
+    std::string toc;
+};
+struct AssetOverrides {
+    // By lower-case asset name, then the game's sha1 the change replaces.
+    std::map<std::string, std::map<fb::Sha1, AssetOverride>, std::less<>> changed;
+    // By lower-case bundle name, in priority order.
+    std::map<std::string, std::vector<AssetAddition>, std::less<>> added;
+    // By mod folder name: TOC chunks the mod adds that its carried additions name
+    // (a new wave's audio). A map's superbundle resolves chunks from its own TOC,
+    // so wherever the additions go these entries go too. Collected pointing into
+    // the mod's own archives; the merge shifts them to where those archives landed.
+    std::map<std::string, std::vector<fb::TocChunk>, std::less<>> chunks;
 
-// The changes asset mods (mods that add no levels) make to the game's own EBX,
-// the highest-priority mod's change winning. Never throws: an unreadable mod
-// is noted and simply changes nothing elsewhere.
+    [[nodiscard]] bool empty() const noexcept { return changed.empty() && added.empty(); }
+};
+
+// The changes and additions asset mods (mods that add no levels) make to the
+// game's own EBX, the highest-priority mod's version winning. Never throws: an
+// unreadable mod is noted and simply changes nothing elsewhere.
 AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
                                        const std::map<const Mod*, RelativeFiles>& modFiles,
                                        const CasStore& store, const fs::path& baseRoot,

Engine/Vfs/mod_merge_overrides.cpp

@@ -1,5 +1,9 @@
 #include "mod_merge_internal.h"
+#include "Engine/Resource/cas_codec.h"
+#include "Engine/Resource/ebx_document.h"
 
+#include <algorithm>
+#include <set>
 #include <stdexcept>
 
 namespace dingosdk::mods::detail {
@@ -17,6 +21,16 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
         const auto files = modFiles.find(mod);
         if (files == modFiles.end()) continue;
         std::size_t changed{};
+        // This mod's new assets, and the partitions its recorded changes import.
+        // An added asset only follows a change that refers to it: copied on its
+        // own, it could arrive in another mod's bundle without the resources
+        // and chunks it was added with.
+        std::vector<std::pair<std::string, AssetAddition>> candidates, companions;
+        std::vector<fb::TocChunk> newChunks;   // TOC chunks the game's TOCs do not have
+        std::set<fb::Guid> imported;
+        const auto decoded = [&](std::span<const std::byte> encoded) {
+            return fb::ebx::read_document(fb::decode_cas(encoded, {gameRoot}));
+        };
         for (const auto& relative : files->second.tocs) {
             std::error_code error;
             const auto baseToc = baseRoot / fs::path(relative);
@@ -26,6 +40,10 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
                 const auto game = fb::read_toc(read_file(baseToc));
                 std::map<std::string, const fb::TocBundle*, std::less<>> gameBundles;
                 for (const auto& bundle : game.bundles) gameBundles.emplace(lower(bundle.name), &bundle);
+                std::set<fb::Guid> gameChunks;
+                for (const auto& chunk : game.chunks) gameChunks.insert(chunk.guid);
+                for (const auto& chunk : own.chunks)
+                    if (!chunk.removed && chunk.location.patch && !gameChunks.contains(chunk.guid)) newChunks.push_back(chunk);
                 for (const auto& bundle : own.bundles) {
                     const auto shipped = gameBundles.find(lower(bundle.name));
                     if (shipped == gameBundles.end()) continue;
@@ -38,17 +56,39 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
                         const auto& asset = modListing->manifest.ebx[index];
                         const auto name = lower(asset.name);
                         const auto game_copy = original.find(name);
-                        if (game_copy == original.end() || game_copy->second == asset.sha1) continue;
-                        auto& versions = out[name];
-                        if (versions.contains(game_copy->second)) continue;
+                        if (game_copy != original.end() && game_copy->second == asset.sha1) continue;
                         const auto at = modListing->first + index;
                         if (at >= modListing->files.size()) continue;
                         const auto& file = modListing->files[at];
-                        versions.emplace(game_copy->second,
-                            AssetOverride{mod->name, asset.sha1, asset.originalSize,
-                                          store.read(file.location.patch ? mod->directory : baseRoot,
-                                                     file.location, file.offset, file.size)});
+                        const auto payload = [&] {
+                            return store.read(file.location.patch ? mod->directory : baseRoot,
+                                              file.location, file.offset, file.size);
+                        };
+                        if (game_copy == original.end()) {
+                            candidates.push_back({lower(bundle.name), {mod->name, asset, payload(), lower(relative)}});
+                            continue;
+                        }
+                        auto& versions = out.changed[name];
+                        if (versions.contains(game_copy->second)) continue;
+                        const auto& change = versions.emplace(game_copy->second,
+                            AssetOverride{mod->name, asset.sha1, asset.originalSize, payload()}).first->second;
                         ++changed;
+                        try {
+                            for (const auto& reference : decoded(change.encoded).imports)
+                                imported.insert(reference.fileGuid);
+                        } catch (const std::exception&) {}
+                    }
+                    // Resources the mod adds, kept to go with an added EBX of the same name
+                    // (a wave's sound-bank resource registers the wave with the audio system).
+                    std::set<std::string, std::less<>> shippedResources;
+                    for (const auto& asset : gameListing->manifest.resources) shippedResources.insert(lower(asset.name));
+                    for (std::size_t index = 0; index < modListing->manifest.resources.size(); ++index) {
+                        const auto& asset = modListing->manifest.resources[index];
+                        const auto at = modListing->first + modListing->manifest.ebx.size() + index;
+                        if (shippedResources.contains(lower(asset.name)) || at >= modListing->files.size()) continue;
+                        const auto& file = modListing->files[at];
+                        companions.push_back({lower(bundle.name), {mod->name, asset,
+                            store.read(file.location.patch ? mod->directory : baseRoot, file.location, file.offset, file.size), lower(relative)}});
                     }
                 }
             } catch (const std::exception& failure) {
@@ -56,6 +96,68 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
                     ": its changes could not be read for other mods' copies (" + failure.what() + ")");
             }
         }
+        // One addition per kind and name in a bundle, the highest-priority mod's.
+        const auto keep = [&](const std::string& bundle, AssetAddition addition) {
+            auto& list = out.added[bundle];
+            const auto name = lower(addition.asset.name);
+            if (std::ranges::any_of(list, [&](const AssetAddition& other) {
+                    return other.asset.kind == addition.asset.kind && lower(other.asset.name) == name; }))
+                return false;
+            list.push_back(std::move(addition));
+            return true;
+        };
+        // Transitively: an addition a following addition imports follows too (a new song
+        // imports its new wave, and only the song is named by the changed playlist).
+        struct Candidate { fb::Guid file; std::vector<fb::Guid> imports; bool taken{}; };
+        std::vector<Candidate> parsed(candidates.size());
+        for (std::size_t index = 0; index < candidates.size(); ++index) {
+            try {
+                const auto document = decoded(candidates[index].second.encoded);
+                parsed[index].file = document.fileGuid;
+                for (const auto& reference : document.imports) parsed[index].imports.push_back(reference.fileGuid);
+            } catch (const std::exception&) { parsed[index].taken = true; }   // unreadable: never follows
+        }
+        for (bool grew = true; grew;) {
+            grew = false;
+            for (std::size_t index = 0; index < candidates.size(); ++index) {
+                auto& candidate = parsed[index];
+                if (candidate.taken || !imported.contains(candidate.file)) continue;
+                candidate.taken = grew = true;
+                imported.insert(candidate.imports.begin(), candidate.imports.end());
+                auto& [bundle, addition] = candidates[index];
+                keep(bundle, std::move(addition));
+            }
+        }
+        for (auto& [bundle, companion] : companions) {
+            const auto& list = out.added[bundle];
+            const auto name = lower(companion.asset.name);
+            if (std::ranges::any_of(list, [&](const AssetAddition& addition) {
+                    return addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx &&
+                           lower(addition.asset.name) == name; }))
+                keep(bundle, std::move(companion));
+        }
+        // The new TOC chunks this mod's carried EBX name (as raw GUID bytes, the way an
+        // EBX stores a ChunkId), so they can follow those assets into other superbundles.
+        if (!newChunks.empty()) {
+            std::vector<std::vector<std::byte>> carried;
+            for (const auto& [bundle, list] : out.added)
+                for (const auto& addition : list)
+                    if (addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx) try {
+                        carried.push_back(fb::decode_cas(addition.encoded, {gameRoot}));
+                    } catch (const std::exception&) {}
+            std::set<fb::Guid> taken;
+            for (const auto& chunk : newChunks) {
+                const auto& id = chunk.guid.bytes;
+                if (!taken.contains(chunk.guid) && std::ranges::any_of(carried, [&](const std::vector<std::byte>& bytes) {
+                        return std::search(bytes.begin(), bytes.end(), id.begin(), id.end()) != bytes.end(); })) {
+                    out.chunks[mod->name].push_back(chunk);
+                    taken.insert(chunk.guid);
+                }
+            }
+            if (!taken.empty())
+                report.notes.push_back(mod->name + ": " + std::to_string(taken.size()) +
+                    " added chunk(s) follow its added assets into other mods' superbundles");
+        }
         if (changed)
             report.notes.push_back(mod->name + ": " + std::to_string(changed) +
                 " changed asset(s) also apply to the copies other mods carry");