diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index 4180511..f39c89a 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -291,7 +291,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con std::erase_if(superbundles, [](const std::string& relative) { return lower(relative) == root_level; }); } if (options.live) report.load_screens = read_load_screens(mods, modFiles, store, baseRoot, gameRoot, report); - const auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report); + auto overrides = collect_asset_overrides(mods, modFiles, store, baseRoot, gameRoot, report); + // Carried chunks point into their mod's archives; move them to where those landed. + for (auto& [name, chunks] : overrides.chunks) + for (const auto* mod : mods) + if (mod->name == name) + for (auto& chunk : chunks) store.shift(chunk.location, chunk.offset, &placements[mod]); for (const auto& relative : superbundles) { if (keepRoot && lower(relative) == root_level) { diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 5de90c4..db4d412 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -147,6 +147,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, std::string example; }; std::map archiveFaults; + // Mods whose added assets went into a copy here: their added chunks come too. + std::set addedFrom; const auto absorb = [&](const fb::TocBundle& bundle, const ArchivePlacement* placement, bool isBase, const fs::path& root) { @@ -300,8 +302,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, std::string example; for (std::size_t index = 0; index < casManifest.ebx.size(); ++index) { auto& asset = casManifest.ebx[index]; - const auto named = overrides.find(lower(asset.name)); - if (named == overrides.end()) continue; + const auto named = overrides.changed.find(lower(asset.name)); + if (named == overrides.changed.end()) continue; const auto change = named->second.find(asset.sha1); const auto at = 1 + index; if (change == named->second.end() || change->second.mod == modName || at >= region.files.size()) @@ -321,8 +323,37 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(overridden.size()) + " asset(s) take another mod's change, e.g. " + example); } + // Assets another mod added to the game's copy of this bundle: this copy + // loads instead of the game's on this mod's levels, so it gets them too. + std::vector additions; + if (propagate && casBacked) { + if (const auto found = overrides.added.find(key); found != overrides.added.end()) { + std::set> present; + for (const auto& asset : casManifest.ebx) present.insert(asset_key(asset)); + for (const auto& asset : casManifest.resources) present.insert(asset_key(asset)); + // A copy in the TOC the adder ships itself merges with the adder's bundle and has + // its assets already; only copies in other superbundles (maps) need them carried. + const auto here = lower(relative); + for (const auto& addition : found->second) { + if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk || + addition.toc == here || present.contains(asset_key(addition.asset))) continue; + try { + additions.push_back({addition.asset, store.write(region.files.front().location.installChunk, + manifestArchive, addition.encoded)}); + addedFrom.insert(addition.mod); + } catch (const std::exception& error) { + report.notes.push_back(modName + ": " + bundle.name + ": " + addition.asset.name + + " from " + addition.mod + " could not be added (" + error.what() + ")"); + } + } + if (!additions.empty()) + report.notes.push_back(modName + ": " + bundle.name + ": " + std::to_string(additions.size()) + + " asset(s) added by other mods, e.g. " + additions.front().asset.name); + } + } // Nothing moved in this bundle: it passes through whole, as before. - if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty()) casBacked = false; + if (casBacked && !shared && !gridBundle && renumbered.empty() && overridden.empty() && additions.empty()) + casBacked = false; for (auto& file : region.files) { store.shift(file.location, file.offset, placement); @@ -333,6 +364,16 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, region.files[index] = file; used.emplace(file.location.installChunk, file.location.archive); } + // Added EBX go after this copy's own EBX and added resources after its own + // resources, ahead of its chunks; after the indexed moves above so their + // positions still hold. + for (auto& addition : additions) { + const auto ebx = addition.asset.kind == fb::AssetKind::ebx; + const auto at = 1 + casManifest.ebx.size() + (ebx ? 0 : casManifest.resources.size()); + (ebx ? casManifest.ebx : casManifest.resources).push_back(std::move(addition.asset)); + region.files.insert(region.files.begin() + static_cast(at), addition.file); + used.emplace(addition.file.location.installChunk, addition.file.location.archive); + } const auto fresh = !states.contains(key); if (fresh) { order.push_back(bundle.name); states.emplace(key, BundleState{}); } auto& state = states.at(key); @@ -637,6 +678,20 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, merged.chunks.push_back(chunk); } } + // This superbundle resolves chunks from its own TOC: the audio of a song another + // mod added to a bundle copied here has to be listed here as well. + for (const auto& name : addedFrom) { + const auto found = overrides.chunks.find(name); + if (found == overrides.chunks.end()) continue; + std::size_t listed{}; + for (const auto& chunk : found->second) { + if (!chunkAt.emplace(chunk.guid, merged.chunks.size()).second) continue; + merged.chunks.push_back(chunk); + used.emplace(chunk.location.installChunk, chunk.location.archive); + ++listed; + } + if (listed) report.notes.push_back(relative + ": " + std::to_string(listed) + " chunk(s) from " + name); + } if (sources.size() > 1) { report.notes.push_back(relative + ": combined " + std::to_string(sources.size()) + diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 8f04fb1..af6dc05 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -175,12 +175,37 @@ struct AssetOverride { std::uint64_t originalSize{}; std::vector encoded; // its payload, as stored in cas }; -// By lower-case asset name, then the game's sha1 the change replaces. -using AssetOverrides = std::map, std::less<>>; +// An EBX asset a mod added to one of the game's bundles, or a resource it added +// under the same name as such an asset (a wave's sound-bank). Maps carry their own +// copy of a bundle like bam_coregameassets, and that copy loads instead of the +// game's on their levels; an edit to a list there (the master music playlist) +// propagates as a change, so the asset it now names has to come along too. +struct AssetAddition { + std::string mod; + fb::BundleAsset asset; + std::vector encoded; // its payload, as stored in cas + // The lower-case relative path of the TOC the mod ships this bundle in. A copy of the + // bundle in that same TOC merges with the mod's own bundle and already has the asset; + // only copies in other superbundles (a map's level TOC) need it carried. + std::string toc; +}; +struct AssetOverrides { + // By lower-case asset name, then the game's sha1 the change replaces. + std::map, std::less<>> changed; + // By lower-case bundle name, in priority order. + std::map, std::less<>> added; + // By mod folder name: TOC chunks the mod adds that its carried additions name + // (a new wave's audio). A map's superbundle resolves chunks from its own TOC, + // so wherever the additions go these entries go too. Collected pointing into + // the mod's own archives; the merge shifts them to where those archives landed. + std::map, std::less<>> chunks; -// The changes asset mods (mods that add no levels) make to the game's own EBX, -// the highest-priority mod's change winning. Never throws: an unreadable mod -// is noted and simply changes nothing elsewhere. + [[nodiscard]] bool empty() const noexcept { return changed.empty() && added.empty(); } +}; + +// The changes and additions asset mods (mods that add no levels) make to the +// game's own EBX, the highest-priority mod's version winning. Never throws: an +// unreadable mod is noted and simply changes nothing elsewhere. AssetOverrides collect_asset_overrides(const std::vector& mods, const std::map& modFiles, const CasStore& store, const fs::path& baseRoot, diff --git a/Engine/Vfs/mod_merge_overrides.cpp b/Engine/Vfs/mod_merge_overrides.cpp index 0bb612d..17ac178 100644 --- a/Engine/Vfs/mod_merge_overrides.cpp +++ b/Engine/Vfs/mod_merge_overrides.cpp @@ -1,5 +1,9 @@ #include "mod_merge_internal.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_document.h" +#include +#include #include namespace dingosdk::mods::detail { @@ -17,6 +21,16 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto files = modFiles.find(mod); if (files == modFiles.end()) continue; std::size_t changed{}; + // This mod's new assets, and the partitions its recorded changes import. + // An added asset only follows a change that refers to it: copied on its + // own, it could arrive in another mod's bundle without the resources + // and chunks it was added with. + std::vector> candidates, companions; + std::vector newChunks; // TOC chunks the game's TOCs do not have + std::set imported; + const auto decoded = [&](std::span encoded) { + return fb::ebx::read_document(fb::decode_cas(encoded, {gameRoot})); + }; for (const auto& relative : files->second.tocs) { std::error_code error; const auto baseToc = baseRoot / fs::path(relative); @@ -26,6 +40,10 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto game = fb::read_toc(read_file(baseToc)); std::map> gameBundles; for (const auto& bundle : game.bundles) gameBundles.emplace(lower(bundle.name), &bundle); + std::set gameChunks; + for (const auto& chunk : game.chunks) gameChunks.insert(chunk.guid); + for (const auto& chunk : own.chunks) + if (!chunk.removed && chunk.location.patch && !gameChunks.contains(chunk.guid)) newChunks.push_back(chunk); for (const auto& bundle : own.bundles) { const auto shipped = gameBundles.find(lower(bundle.name)); if (shipped == gameBundles.end()) continue; @@ -38,17 +56,39 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto& asset = modListing->manifest.ebx[index]; const auto name = lower(asset.name); const auto game_copy = original.find(name); - if (game_copy == original.end() || game_copy->second == asset.sha1) continue; - auto& versions = out[name]; - if (versions.contains(game_copy->second)) continue; + if (game_copy != original.end() && game_copy->second == asset.sha1) continue; const auto at = modListing->first + index; if (at >= modListing->files.size()) continue; const auto& file = modListing->files[at]; - versions.emplace(game_copy->second, - AssetOverride{mod->name, asset.sha1, asset.originalSize, - store.read(file.location.patch ? mod->directory : baseRoot, - file.location, file.offset, file.size)}); + const auto payload = [&] { + return store.read(file.location.patch ? mod->directory : baseRoot, + file.location, file.offset, file.size); + }; + if (game_copy == original.end()) { + candidates.push_back({lower(bundle.name), {mod->name, asset, payload(), lower(relative)}}); + continue; + } + auto& versions = out.changed[name]; + if (versions.contains(game_copy->second)) continue; + const auto& change = versions.emplace(game_copy->second, + AssetOverride{mod->name, asset.sha1, asset.originalSize, payload()}).first->second; ++changed; + try { + for (const auto& reference : decoded(change.encoded).imports) + imported.insert(reference.fileGuid); + } catch (const std::exception&) {} + } + // Resources the mod adds, kept to go with an added EBX of the same name + // (a wave's sound-bank resource registers the wave with the audio system). + std::set> shippedResources; + for (const auto& asset : gameListing->manifest.resources) shippedResources.insert(lower(asset.name)); + for (std::size_t index = 0; index < modListing->manifest.resources.size(); ++index) { + const auto& asset = modListing->manifest.resources[index]; + const auto at = modListing->first + modListing->manifest.ebx.size() + index; + if (shippedResources.contains(lower(asset.name)) || at >= modListing->files.size()) continue; + const auto& file = modListing->files[at]; + companions.push_back({lower(bundle.name), {mod->name, asset, + store.read(file.location.patch ? mod->directory : baseRoot, file.location, file.offset, file.size), lower(relative)}}); } } } catch (const std::exception& failure) { @@ -56,6 +96,68 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, ": its changes could not be read for other mods' copies (" + failure.what() + ")"); } } + // One addition per kind and name in a bundle, the highest-priority mod's. + const auto keep = [&](const std::string& bundle, AssetAddition addition) { + auto& list = out.added[bundle]; + const auto name = lower(addition.asset.name); + if (std::ranges::any_of(list, [&](const AssetAddition& other) { + return other.asset.kind == addition.asset.kind && lower(other.asset.name) == name; })) + return false; + list.push_back(std::move(addition)); + return true; + }; + // Transitively: an addition a following addition imports follows too (a new song + // imports its new wave, and only the song is named by the changed playlist). + struct Candidate { fb::Guid file; std::vector imports; bool taken{}; }; + std::vector parsed(candidates.size()); + for (std::size_t index = 0; index < candidates.size(); ++index) { + try { + const auto document = decoded(candidates[index].second.encoded); + parsed[index].file = document.fileGuid; + for (const auto& reference : document.imports) parsed[index].imports.push_back(reference.fileGuid); + } catch (const std::exception&) { parsed[index].taken = true; } // unreadable: never follows + } + for (bool grew = true; grew;) { + grew = false; + for (std::size_t index = 0; index < candidates.size(); ++index) { + auto& candidate = parsed[index]; + if (candidate.taken || !imported.contains(candidate.file)) continue; + candidate.taken = grew = true; + imported.insert(candidate.imports.begin(), candidate.imports.end()); + auto& [bundle, addition] = candidates[index]; + keep(bundle, std::move(addition)); + } + } + for (auto& [bundle, companion] : companions) { + const auto& list = out.added[bundle]; + const auto name = lower(companion.asset.name); + if (std::ranges::any_of(list, [&](const AssetAddition& addition) { + return addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx && + lower(addition.asset.name) == name; })) + keep(bundle, std::move(companion)); + } + // The new TOC chunks this mod's carried EBX name (as raw GUID bytes, the way an + // EBX stores a ChunkId), so they can follow those assets into other superbundles. + if (!newChunks.empty()) { + std::vector> carried; + for (const auto& [bundle, list] : out.added) + for (const auto& addition : list) + if (addition.mod == mod->name && addition.asset.kind == fb::AssetKind::ebx) try { + carried.push_back(fb::decode_cas(addition.encoded, {gameRoot})); + } catch (const std::exception&) {} + std::set taken; + for (const auto& chunk : newChunks) { + const auto& id = chunk.guid.bytes; + if (!taken.contains(chunk.guid) && std::ranges::any_of(carried, [&](const std::vector& bytes) { + return std::search(bytes.begin(), bytes.end(), id.begin(), id.end()) != bytes.end(); })) { + out.chunks[mod->name].push_back(chunk); + taken.insert(chunk.guid); + } + } + if (!taken.empty()) + report.notes.push_back(mod->name + ": " + std::to_string(taken.size()) + + " added chunk(s) follow its added assets into other mods' superbundles"); + } if (changed) report.notes.push_back(mod->name + ": " + std::to_string(changed) + " changed asset(s) also apply to the copies other mods carry");