diff --git a/Engine/Vfs/mod_catalog.cpp b/Engine/Vfs/mod_catalog.cpp index fad31d7a..cde2bc3e 100644 --- a/Engine/Vfs/mod_catalog.cpp +++ b/Engine/Vfs/mod_catalog.cpp @@ -448,8 +448,10 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver auto exclusions = read_exclusions(result.root); const auto sdk = sdk_identity(); std::map> still_excluded; - // `remembered`: it stays out on later launches without a retry, until its files or ReSkate change. - const auto leave_out = [&](std::size_t index, std::vector problems, const std::string& fingerprint, bool remembered = true) { + // `remembered`: it stays out on later launches without a retry, until its files or ReSkate change + // (or, left out for a conflict, the mod it conflicts with: `conflict`, that mod's name and files). + const auto leave_out = [&](std::size_t index, std::vector problems, const std::string& fingerprint, bool remembered = true, + std::pair conflict = {}) { auto mod = std::move(result.mods[index]); result.mods.erase(result.mods.begin() + static_cast(index)); // Level bundles decide whether a map loads at all, so their problems lead. @@ -457,7 +459,7 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver return problem.find("levels/game/") != std::string::npos; }); mod.problems = std::move(problems); - if (remembered) still_excluded[mod.name] = {fingerprint, sdk, mod.problems}; + if (remembered) still_excluded[mod.name] = {fingerprint, sdk, mod.problems, conflict.first, conflict.second}; result.excluded.push_back(std::move(mod)); }; for (std::size_t i = result.mods.size(); i-- > 0;) { @@ -469,6 +471,18 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver // game may have been put right since (its files verified), with the mod untouched. // Remembered, it stayed out for good with the game fixed (seen 2026-10-13). if (!found->second.empty() && found->second.front().starts_with("made for game build ")) continue; + // Left out for a conflict: only while the mod it lost to still loads above it, as it was. + if (!previous.conflictWith.empty()) { + const auto winner = std::find_if(result.mods.begin(), result.mods.begin() + static_cast(i), + [&](const Mod& mod) { return mod.name == previous.conflictWith; }); + if (winner == result.mods.begin() + static_cast(i) || + mod_fingerprint(winner->directory) != previous.conflictFingerprint) + continue; + result.notes.push_back("Leaving out " + result.mods[i].name + ": it conflicts with " + previous.conflictWith + + ", which loads above it, and neither has changed since"); + leave_out(i, found->second, previous.fingerprint, true, {previous.conflictWith, previous.conflictFingerprint}); + continue; + } result.notes.push_back("Leaving out " + result.mods[i].name + ": it could not be merged cleanly before and its files have not changed since"); leave_out(i, found->second, previous.fingerprint); @@ -519,18 +533,69 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver // without it. Removing one mod cannot break another, but a few rounds // are allowed in case two damaged mods hid each other's problems. for (int round = 0;; ++round) { - merge = merge_mods(result, observe); + // The first looks for conflicts before building anything (find_conflicts). + merge = merge_mods(result, observe, {.conflicts_first = round == 0}); if (!merge.issue.empty() || round == 4) break; // a fifth merge only rebuilds without round four's removals - bool removed = false; - for (std::size_t i = result.mods.size(); i-- > 0;) { - const auto found = merge.problems.find(result.mods[i].name); - if (found == merge.problems.end() || found->second.empty()) continue; - result.notes.push_back("Leaving out " + result.mods[i].name + - ": it could not be merged cleanly; merging the other mods again without it"); - leave_out(i, found->second, mod_fingerprint(result.mods[i].directory)); - removed = true; + // (What the check before the merge noted, which the merge's own notes below would not carry.) + if (merge.stopped_for_conflicts) + for (auto& note : merge.notes) result.notes.push_back(std::move(note)); + // Everything this merge found is settled at once, so one more merge is enough: the + // mods that could not be merged cleanly, and, of two mods that change one asset in + // ways that cannot both be in the game, the one lower in the load order, whole (so + // neither loads half its changes). Mods are looked at from the top of the load + // order down, so one that lost only to a mod that is itself left out stays in. + // ReSkate's own packs, which load below every mod, are never left out for this. + const auto official = official_packs(data_root / mods_folder, true); + const auto ours = [&](const std::string& name) { + return name.starts_with("reskate-builtin-") || name.find('#') != std::string::npos || + std::ranges::any_of(official, [&](const BuiltinPack& pack) { return pack.name == name; }); + }; + struct Out { + std::vector problems; + std::string winner; // left out for a conflict with this mod + }; + std::map out; + std::set leaving; + for (std::size_t i = 0; i < result.mods.size(); ++i) { + const auto& name = result.mods[i].name; + if (const auto found = merge.problems.find(name); found != merge.problems.end() && !found->second.empty()) { + out[i].problems = found->second; + leaving.insert(name); + continue; + } + if (ours(name)) continue; + for (const auto& row : merge.conflicts) { + if (row.lost != name || leaving.contains(row.kept)) continue; + auto& entry = out[i]; + if (entry.winner.empty()) entry.winner = row.kept; + entry.problems.push_back("conflicts with " + row.kept + ": both change " + row.asset + + (row.assets > 1 ? " and " + std::to_string(row.assets - 1) + " other asset(s)" : std::string{}) + + ", and " + row.kept + " is higher in the load order"); + } + if (out.contains(i)) leaving.insert(name); } - if (!removed) break; + const bool removed = !out.empty(); + if (removed) + result.notes.push_back((merge.stopped_for_conflicts ? std::string("The check before merging") + : "Merge " + std::to_string(round + 1)) + + " left " + std::to_string(out.size()) + " mod(s) out; merging once more without them"); + for (auto at = out.rbegin(); at != out.rend(); ++at) { + const auto i = at->first; + const auto& [problems, winner] = at->second; + if (winner.empty()) { + result.notes.push_back("Leaving out " + result.mods[i].name + + ": it could not be merged cleanly; merging the other mods again without it"); + leave_out(i, problems, mod_fingerprint(result.mods[i].directory)); + continue; + } + const auto kept = std::ranges::find_if(result.mods, [&](const Mod& mod) { return mod.name == winner; }); + result.notes.push_back("Leaving out " + result.mods[i].name + ": it conflicts with " + winner + + ", which is higher in the load order; merging the other mods again without it"); + leave_out(i, problems, mod_fingerprint(result.mods[i].directory), true, + {winner, kept != result.mods.end() ? mod_fingerprint(kept->directory) : std::string{}}); + } + // (Conflicts only of ReSkate's own packs leave nothing out: then the merge itself runs.) + if (!removed && !merge.stopped_for_conflicts) break; } for (auto& note : merge.notes) result.notes.push_back(std::move(note)); result.merged = merge.built; @@ -555,6 +620,13 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver "). Update it for this game version."); continue; } + if (!mod.problems.empty() && mod.problems.front().starts_with("conflicts with ")) { + const auto& problem = mod.problems.front(); + const auto other = problem.substr(15, problem.find(':') - 15); + result.warnings.push_back("Mod " + mod.name + " was not loaded: it " + problem + ". Only " + other + + " is loaded. To use " + mod.name + " instead, move it above " + other + " in the mod list, or turn one of them off."); + continue; + } if (!mod.problems.empty() && mod.problems.front().starts_with("made for game build ")) { result.warnings.push_back("Mod " + mod.name + " was not loaded: it was " + mod.problems.front() + (mod.levels.empty() ? std::string{} : " (its maps are not listed: " + join(mod.levels, ", ") + ")") + "."); diff --git a/Engine/Vfs/mod_list.cpp b/Engine/Vfs/mod_list.cpp index e2e3bd65..b9a28268 100644 --- a/Engine/Vfs/mod_list.cpp +++ b/Engine/Vfs/mod_list.cpp @@ -261,6 +261,8 @@ std::map> read_exclusions(const std::filesys Exclusion exclusion; exclusion.fingerprint = row.value("fingerprint", std::string{}); exclusion.sdk = row.value("sdk", std::string{}); + exclusion.conflictWith = row.value("conflict_with", std::string{}); + exclusion.conflictFingerprint = row.value("conflict_fingerprint", std::string{}); if (row.contains("problems") && row.at("problems").is_array()) for (const auto& problem : row.at("problems")) if (problem.is_string() && exclusion.problems.size() < 16) @@ -282,6 +284,10 @@ void write_exclusions(const std::filesystem::path& mods_root, auto row = Json::object(); row["fingerprint"] = exclusion.fingerprint; row["sdk"] = exclusion.sdk; + if (!exclusion.conflictWith.empty()) { + row["conflict_with"] = exclusion.conflictWith; + row["conflict_fingerprint"] = exclusion.conflictFingerprint; + } auto problems = Json::array(); for (std::size_t i = 0; i < exclusion.problems.size() && i < 16; ++i) problems.push_back(exclusion.problems[i].substr(0, 600)); diff --git a/Engine/Vfs/mod_list.h b/Engine/Vfs/mod_list.h index 5237ca52..2ccdf9de 100644 --- a/Engine/Vfs/mod_list.h +++ b/Engine/Vfs/mod_list.h @@ -67,6 +67,10 @@ struct Exclusion { std::string fingerprint; // mod_fingerprint() when it was left out std::string sdk; // the ReSkate.dll that left it out; a new one retries std::vector problems; // why, most important first + // Left out for a conflict with this mod, higher in the load order, as its files were then + // (its mod_fingerprint()). It stays out only while that mod still loads above it unchanged. + std::string conflictWith; + std::string conflictFingerprint; }; // Names, sizes and write times of every file in the mod folder. std::string mod_fingerprint(const std::filesystem::path& directory); diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index e9ed70f4..3008ae75 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -118,9 +118,28 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // launch (the player is waiting on nothing else), half of them and fewer while the game runs. const auto cores = std::max(1U, std::thread::hardware_concurrency()); const std::size_t readers = std::min(options.live ? std::max(1U, cores / 2) : cores, options.live ? 6U : 8U); + // The checks before the merge say what they are doing, the bar still at its start. + const auto checking = [&](std::string step) { + if (observe) try { observe({0, 1, mods.size(), std::move(step)}); } catch (...) {} + }; std::optional> itemsRead; + if (storeKnown) checking("Checking the mods"); if (storeKnown && store_copy_problems(catalog, report, options.live ? options.checked : std::vector{}, readers - 1, options.live, itemsRead)) return report; + // Mods that conflict, found the same way before anything is built: the catalog leaves the + // losing ones out and merges once, where finding them in the merge took a second merge. + if (options.conflicts_first && !options.live) { + checking("Checking the mods for conflicts"); + const auto started = Clock::now(); + report.conflicts = find_conflicts(mods, modFiles, catalog.data_root, output, readers - 1); + const auto tenths = (std::chrono::duration_cast(Clock::now() - started).count() + 50) / 100; + report.notes.push_back("Conflict check: " + std::to_string(report.conflicts.size()) + " conflicting pair(s) in " + + std::to_string(tenths / 10) + '.' + std::to_string(tenths % 10) + " s"); + if (!report.conflicts.empty()) { + report.stopped_for_conflicts = true; + return report; + } + } // Which mod each item is from (mod_item_sources.h): what was just read, and for a mod a // live merge did not read again, what the launch's merge found. Read before the patch // folder is cleared. @@ -547,6 +566,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con auto& list = report.problems[mod]; list.insert(list.end(), std::make_move_iterator(problems.begin()), std::make_move_iterator(problems.end())); } + for (auto& row : job.report.conflicts) { + const auto same = std::ranges::find_if(report.conflicts, [&](const MergeReport::Conflict& held) { + return held.kept == row.kept && held.lost == row.lost; }); + if (same != report.conflicts.end()) same->assets += row.assets; + else report.conflicts.push_back(std::move(row)); + } auto bytes = fb::write_patch_toc(job.merged.bundles, job.merged.chunks, job.merged.flags); // Read the result back before publishing it: a TOC whose perfect // hash does not resolve would take the game down at load time. diff --git a/Engine/Vfs/mod_merge.h b/Engine/Vfs/mod_merge.h index cff48198..229e38a5 100644 --- a/Engine/Vfs/mod_merge.h +++ b/Engine/Vfs/mod_merge.h @@ -27,6 +27,19 @@ struct MergeReport { // be decoded, a copy that had to be left out. Each mod gets a summary line // from these, and a level that never finishes loading can name its mod. std::map> problems; + // Two mods changing one asset in ways that cannot both be in the game: the copy of the mod + // higher in the load order is kept and the other's changes are not in it. One row per pair of + // mods (folder names), with one of the assets and how many. The catalog leaves the losing mod + // out whole and merges again, so no mod loads half its changes. + struct Conflict { + std::string kept; + std::string lost; + std::string asset; + std::size_t assets{}; + }; + std::vector conflicts; + // Nothing was built: the check before the merge found `conflicts` (MergeOptions::conflicts_first). + bool stopped_for_conflicts{}; // Every superbundle the merged mods' layouts assign to an install chunk, // as (superbundle, install chunk name). A live merge registers the ones // the game did not know at launch. @@ -79,6 +92,9 @@ struct MergeOptions { // the others against it. Reading every mod's items and meshes again is most of what // adding one map would otherwise wait on. std::vector checked; + // Look for mods that conflict (find_conflicts) before building anything, and when some do, + // stop there (MergeReport::stopped_for_conflicts) so the caller merges once without them. + bool conflicts_first{}; }; // Combines every enabled mod into one patch under Mods/.reskate: superbundle diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index c0233b0b..4bfe2bfa 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -121,6 +121,14 @@ struct BundleState { // The base's own sha1 for each asset, so a mod that merely carries an // unchanged copy can be told apart from one that changed it. std::unordered_map baseSha; + // Of each asset, the last version a mod shipped as its own change (not the game's copy, not + // one rewritten to carry another mod's change), and every mod that shipped that same version. + // A later mod shipping its own different version means theirs is lost: a conflict. + struct OwnChange { + fb::Sha1 sha1; + std::vector mods; + }; + std::unordered_map ownChange; // Assets a mod added that a higher-priority mod's different asset of the // same name then replaced: the key, and the sha1 of the copy that went. std::vector> displaced; @@ -223,6 +231,17 @@ std::span unsigned_bytes(std::span data) { return {reinterpret_cast(data.data()), data.size()}; } +// `lost`'s change to `asset` is not in the game, `kept`'s copy being there (MergeReport::conflicts). +void conflict(MergeReport& report, const std::string& kept, const std::string& lost, const std::string& asset) { + if (kept.empty() || lost.empty() || kept == lost) return; + for (auto& row : report.conflicts) + if (row.kept == kept && row.lost == lost) { + ++row.assets; + return; + } + report.conflicts.push_back({kept, lost, asset, 1}); +} + } // namespace std::vector merge_chunk_metadata(std::span> lists, @@ -823,6 +842,26 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, const auto* owner = isBase ? nullptr : &*owners.insert(modName).first; // By the mod whose added assets this copy's replace: how many, and one of them. std::map> replaced; + // This mod's own version of an asset the merge does not combine (EBX the game has is combined + // further down, and so are the shader tables and the preset table): a mod lower in the load + // order whose own version differs loses it, a conflict. find_conflicts applies the same rule + // before the merge. + const auto own_change = [&](const std::string& key, const Asset& asset, bool rewritten) { + if (isBase || rewritten) return; + if (asset.asset.kind == fb::AssetKind::resource && + (asset.asset.resourceType == fb::shader::programLookupType || + asset.asset.resourceType == fb::shader::textureLookupType || fb::bundle_ref::is_table(asset.asset.name))) + return; + const auto shipped = state.baseSha.find(key); + if (shipped != state.baseSha.end() && (asset.asset.kind == fb::AssetKind::ebx || shipped->second == asset.asset.sha1)) return; + auto& last = state.ownChange[key]; + if (!last.mods.empty() && last.sha1 == asset.asset.sha1) { + if (std::ranges::find(last.mods, modName) == last.mods.end()) last.mods.push_back(modName); + return; + } + for (const auto& lost : last.mods) conflict(report, modName, lost, asset.asset.name); + last = {asset.asset.sha1, {modName}}; + }; for (std::size_t position = 0; position < flat.size(); ++position) { auto& entry = flat[position]; entry.base = isBase; @@ -856,6 +895,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, } if (isBase) state.baseSha.insert_or_assign(id, entry.asset.sha1); const auto at = state.seen.find(id); + own_change(id, entry, rewritten); if (at == state.seen.end()) { state.seen.emplace(std::move(id), state.assets.size()); state.assets.push_back(std::move(entry)); @@ -876,9 +916,10 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // Two mods rewriting one non-EBX asset cannot both be honoured; // say so, because the loser silently loses whatever it shipped. if (entry.asset.kind == fb::AssetKind::chunk && !previous.base && - previous.asset.sha1 != entry.asset.sha1) + previous.asset.sha1 != entry.asset.sha1) { report.notes.push_back("contested chunk " + entry.asset.name + ": kept the highest-priority copy"); + } // Two mods each adding a different asset under one name: a bundle // holds one asset per name, so the lower-priority mod's is gone, // and whatever of that mod's still names it by guid finds nothing @@ -1067,6 +1108,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, if (entry.mod) others.erase(*entry.mod); std::string names; for (const auto& other : others) names += (names.empty() ? "" : ", ") + other; + if (entry.mod) + for (const auto& other : others) conflict(report, *entry.mod, other, entry.asset.name); if (!others.empty()) report.notes.push_back(entry.asset.name + ": kept " + (entry.mod ? *entry.mod + "'s copy" : std::string("the highest-priority copy")) + @@ -1128,6 +1171,10 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, report.notes.push_back(entry.asset.name + ": kept the highest-priority copy, the edits could not be combined (" + failure.what() + ")"); + if (entry.mod) + for (const auto& contribution : found->second) + if (!contribution.base && contribution.sha1 != baseCopy->sha1 && contribution.sha1 != entry.asset.sha1) + conflict(report, *entry.mod, path_utf8(contribution.root->filename()), entry.asset.name); } } diff --git a/Engine/Vfs/mod_merge_conflicts.cpp b/Engine/Vfs/mod_merge_conflicts.cpp new file mode 100644 index 00000000..df7c5a0a --- /dev/null +++ b/Engine/Vfs/mod_merge_conflicts.cpp @@ -0,0 +1,280 @@ +#include "mod_merge_internal.h" +#include "Engine/Resource/bundle_ref_table.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_merge.h" +#include "Engine/Resource/shader_lookup.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::mods::detail { +namespace { + +// The merge keys an asset within a bundle by its kind and, for a chunk its guid, for the rest its +// name in any case (asset_key in mod_merge_combine.cpp). The same, hashed: a big bundle has some +// 17,000 assets and a hundred mods' copies of it, too many to make a string for each. +std::uint64_t key_hash(const fb::BundleAsset& asset) { + std::uint64_t value = 0xCBF29CE484222325ULL; + if (asset.kind == fb::AssetKind::chunk) + for (const auto byte : asset.guid.bytes) value = (value ^ static_cast(byte)) * 0x100000001B3ULL; + else + value = name_hash(asset.name); + return value ^ (static_cast(asset.kind) + 1) * 0x9E3779B97F4A7C15ULL; +} + +// Assets the merge combines whatever the mods did to them, so two versions are no conflict: +// the shader tables, the preset table and the collision the material grid renumbers. +bool combined_anyway(const fb::BundleAsset& asset) { + return asset.kind == fb::AssetKind::resource && + (asset.resourceType == fb::shader::programLookupType || asset.resourceType == fb::shader::textureLookupType || + asset.resourceType == fb::material_grid::physicsResourceType || fb::bundle_ref::is_table(asset.name)); +} + +using Rows = std::vector; +void conflict(Rows& rows, const std::string& kept, const std::string& lost, const std::string& asset) { + if (kept.empty() || lost.empty() || kept == lost) return; + for (auto& row : rows) + if (row.kept == kept && row.lost == lost) { + ++row.assets; + return; + } + rows.push_back({kept, lost, asset, 1}); +} + +// One mod's version of an asset in a bundle that is not the game's. +struct Change { + std::uint64_t key{}; + std::size_t copy{}; // in Shared::copies + std::size_t at{}; // the asset's place in the manifest's EBX, for reading it + const fb::BundleAsset* asset{}; +}; + +// One bundle that two or more mods ship in one superbundle, and what is read of it. +struct Shared { + const fb::TocBundle* game{}; + std::vector> copies; // (mod, its copy), lowest priority first + std::optional gameListing; + // The game's version of each asset: its sha1, and its place among the EBX. + std::unordered_map> original; + std::vector> listings; // by copy + std::vector> changes; // by copy: what it ships that is not the game's +}; + +// `count` pieces of work on up to `threads` threads beside this one, taken in turn. +void in_parallel(std::size_t count, std::size_t threads, bool background, const std::function& work) { + std::atomic next{0}; + const auto run = [&] { + for (;;) { + const auto mine = next.fetch_add(1); + if (mine >= count) return; + try { + work(mine); + } catch (const std::exception&) {} // a part that fails finds nothing: the merge still checks for itself + } + }; + std::vector workers; + try { + // Threads of their own, not the system's pool (see the superbundles in mod_merge.cpp). + for (std::size_t index = 0; index + 1 < count && index < threads; ++index) + workers.emplace_back([&] { + if (background) SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); + run(); + }); + } catch (const std::system_error&) {} // fewer threads, or none: this one does the rest + run(); +} + +// The game's copy of a bundle, keyed, to tell a mod's change from a copy of the game's. +void read_game(Shared& shared, const CasStore& store, const fs::path& baseRoot, const fs::path& gameRoot) { + if (!shared.game) return; + shared.gameListing = list_bundle(store, baseRoot, baseRoot, *shared.game, gameRoot); + if (!shared.gameListing) return; + const auto& manifest = shared.gameListing->manifest; + for (std::size_t index = 0; index < manifest.ebx.size(); ++index) + shared.original.emplace(key_hash(manifest.ebx[index]), std::pair{manifest.ebx[index].sha1, index}); + for (const auto* list : {&manifest.resources, &manifest.chunks}) + for (const auto& asset : *list) shared.original.emplace(key_hash(asset), std::pair{asset.sha1, std::size_t{}}); +} + +// One mod's copy: its asset list, and of that what is not the game's. +void read_copy(Shared& shared, std::size_t copy, const std::vector& mods, const CasStore& store, + const fs::path& baseRoot, const fs::path& gameRoot) { + auto& listing = shared.listings[copy]; + listing = list_bundle(store, mods[shared.copies[copy].first]->directory, baseRoot, *shared.copies[copy].second, gameRoot); + if (!listing) return; // (the merge notes what it cannot read) + auto& changes = shared.changes[copy]; + const auto take = [&](const std::vector& list, bool ebx) { + for (std::size_t index = 0; index < list.size(); ++index) { + const auto& asset = list[index]; + if (combined_anyway(asset)) continue; + const auto key = key_hash(asset); + if (const auto shipped = shared.original.find(key); shipped != shared.original.end() && shipped->second.first == asset.sha1) + continue; + changes.push_back({key, copy, ebx ? index : 0, &asset}); + } + }; + take(listing->manifest.ebx, true); + take(listing->manifest.resources, false); + take(listing->manifest.chunks, false); +} + +// The conflicts in one bundle, from the changes its copies were found to ship. +Rows decide(const Shared& shared, const std::vector& mods, const CasStore& store, + const fs::path& baseRoot, const fs::path& gameRoot) { + Rows rows; + // Each asset's changes, lowest priority first, the assets in the order they were first met. + std::unordered_map> byAsset; + std::vector order; + for (const auto& copy : shared.changes) + for (const auto& change : copy) { + auto& list = byAsset[change.key]; + if (list.empty()) order.push_back(change.key); + list.push_back(&change); + } + const auto mod_of = [&](const Change& change) -> const std::string& { return mods[shared.copies[change.copy].first]->name; }; + for (const auto key : order) { + const auto& changes = byAsset.at(key); + const auto& kept = *changes.back(); + // Two mods' different changes, or nothing to decide. + if (std::ranges::all_of(changes, [&](const Change* change) { + return change->asset->sha1 == kept.asset->sha1 || shared.copies[change->copy].first == shared.copies[kept.copy].first; })) + continue; + const auto shipped = shared.original.find(key); + + if (kept.asset->kind != fb::AssetKind::ebx || shipped == shared.original.end()) { + // Not combined: of each asset the highest-priority mod's version is the one in the game, and a + // mod lower down that changed it differently loses its change. (The rule of absorb's own_change.) + fb::Sha1 last{}; + std::vector holders; + for (const auto* change : changes) { + const auto& name = mod_of(*change); + if (!holders.empty() && change->asset->sha1 == last) { + if (std::ranges::find(holders, &name) == holders.end()) holders.push_back(&name); + continue; + } + for (const auto* lost : holders) conflict(rows, name, *lost, change->asset->name); + last = change->asset->sha1; + holders = {&name}; + } + continue; + } + + // EBX the game has is combined, the changes of every mod onto the game's copy: a conflict only + // when that cannot be done (the kept-whole and could-not-combine cases of the merge's combine). + const auto lose_to_kept = [&] { + for (const auto* change : changes) + if (change->asset->sha1 != kept.asset->sha1) conflict(rows, mod_of(kept), mod_of(*change), kept.asset->name); + }; + try { + const auto base = fb::ebx::read_document( + read_asset(store, baseRoot, baseRoot, *shared.gameListing, shipped->second.second, gameRoot)); + // The combined material grid is worked out separately (mod_merge_grid.cpp). + if (lower(base.rootType).find("materialgrid") != std::string::npos) continue; + std::map documents; + std::vector edits; + for (const auto* change : changes) { + auto [at, fresh] = documents.try_emplace(change->asset->sha1); + if (fresh) + at->second = fb::ebx::read_document(read_asset(store, mods[shared.copies[change->copy].first]->directory, + baseRoot, *shared.listings[change->copy], change->at, gameRoot)); + edits.push_back(&at->second); + } + fb::ebx::MergeSummary summary; + (void)fb::ebx::merge_documents(base, edits, &summary); + if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries && (!summary.values || !summary.exact())) + lose_to_kept(); + } catch (const std::exception&) { + lose_to_kept(); + } + } + return rows; +} + +} // namespace + +std::vector find_conflicts(const std::vector& mods, + const std::map& modFiles, + const fs::path& dataRoot, const fs::path& output, + std::size_t threads, bool background) { + const auto baseRoot = dataRoot / L"Data"; + const auto gameRoot = dataRoot; + // The superbundles two or more mods ship: only there can two mods' versions of an asset meet. + std::map> shippedBy; // lower case relative path, mods highest priority first + std::map spelled; + for (std::size_t mod = 0; mod < mods.size(); ++mod) + if (const auto files = modFiles.find(mods[mod]); files != modFiles.end()) + for (const auto& relative : files->second.tocs) { + auto key = lower(relative); + spelled.try_emplace(key, relative); + shippedBy[std::move(key)].push_back(mod); + } + std::erase_if(shippedBy, [](const auto& entry) { return entry.second.size() < 2; }); + if (shippedBy.empty()) return {}; + + // Their TOCs, the game's and the mods', kept while the bundles they list are read. + std::vector tocs; + tocs.reserve([&] { + std::size_t count{}; + for (const auto& [relative, by] : shippedBy) count += by.size() + 1; + return count; + }()); + std::vector bundles; + for (const auto& [relative, by] : shippedBy) { + std::map named; + for (auto mod = by.rbegin(); mod != by.rend(); ++mod) { + try { + const auto& toc = tocs.emplace_back(fb::read_toc(read_file(mods[*mod]->directory / fs::path(spelled.at(relative))))); + for (const auto& bundle : toc.bundles) named[lower(bundle.name)].copies.emplace_back(*mod, &bundle); + } catch (const std::exception&) {} // (the merge leaves a mod out whose TOC cannot be read) + } + std::erase_if(named, [](const auto& entry) { return entry.second.copies.size() < 2; }); + if (named.empty()) continue; + std::error_code error; + if (const auto game = baseRoot / fs::path(spelled.at(relative)); fs::is_regular_file(game, error)) try { + const auto& toc = tocs.emplace_back(fb::read_toc(read_file(game))); + for (const auto& bundle : toc.bundles) + if (const auto found = named.find(lower(bundle.name)); found != named.end()) found->second.game = &bundle; + } catch (const std::exception&) {} + for (auto& [name, shared] : named) { + shared.listings.resize(shared.copies.size()); + shared.changes.resize(shared.copies.size()); + bundles.push_back(std::move(shared)); + } + } + if (bundles.empty()) return {}; + + // On several threads, in three steps, so that one bundle a hundred mods ship is not all one + // thread's: the game's copy of each bundle, then each mod's copy of each by itself, then what + // the copies' changes mean in each bundle. The rows are then put together in bundle order. + const auto layout = vfs::read_layout(baseRoot / L"layout.toc"); + const CasStore store(baseRoot, output, layout.root); + in_parallel(bundles.size(), threads, background, [&](std::size_t index) { read_game(bundles[index], store, baseRoot, gameRoot); }); + std::vector> copies; // (bundle, copy) + for (std::size_t index = 0; index < bundles.size(); ++index) + for (std::size_t copy = 0; copy < bundles[index].copies.size(); ++copy) copies.emplace_back(index, copy); + in_parallel(copies.size(), threads, background, [&](std::size_t index) { + read_copy(bundles[copies[index].first], copies[index].second, mods, store, baseRoot, gameRoot); + }); + std::vector found(bundles.size()); + in_parallel(bundles.size(), threads, background, [&](std::size_t index) { + found[index] = decide(bundles[index], mods, store, baseRoot, gameRoot); + }); + Rows rows; + for (const auto& bundle : found) + for (const auto& row : bundle) { + const auto same = std::ranges::find_if(rows, [&](const MergeReport::Conflict& held) { + return held.kept == row.kept && held.lost == row.lost; }); + if (same != rows.end()) same->assets += row.assets; + else rows.push_back(row); + } + return rows; +} + +} // namespace dingosdk::mods::detail diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 28cf4144..6e1110c4 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -320,6 +320,17 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const fs::path& gameRoot, MergeReport& report, std::size_t threads = 0, bool background = false); +// Of two mods that change one of the game's assets in ways that cannot both be in the game, the +// one lower in the load order, found from the mods' asset lists before anything is merged: the +// rule the merge applies as it combines (MergeReport::conflicts), read off the bundles two or +// more mods ship, so the catalog can leave the losing mods out before the one merge that builds +// the patch. `mods` highest priority first; `output` is only where the patch would go. Never +// throws: what cannot be read is left to the merge, which still checks for itself. +std::vector find_conflicts(const std::vector& mods, + const std::map& modFiles, + const fs::path& dataRoot, const fs::path& output, + std::size_t threads = 0, bool background = false); + // A bundle's chunk metadata is a list with one record for each of its chunks: the // hash of the name of the resource the chunk belongs to ("h64") and what the // engine needs before it streams the chunk (a texture's first mip). In the game's diff --git a/Launcher/gui_internal.h b/Launcher/gui_internal.h index eecfbe76..cbfc9dc6 100644 --- a/Launcher/gui_internal.h +++ b/Launcher/gui_internal.h @@ -227,6 +227,7 @@ struct ModProblem { std::string name; // folder under Mods/, so it can be switched off std::string title; std::string reason; + bool conflict{}; // left out for another mod above it that changes the same game files }; struct State { diff --git a/Launcher/gui_launcher.cpp b/Launcher/gui_launcher.cpp index 557b5682..d644c653 100644 --- a/Launcher/gui_launcher.cpp +++ b/Launcher/gui_launcher.cpp @@ -503,14 +503,16 @@ bool Launcher::run_mod_merge() { // Mods built for another game version say so for themselves, in game // and in the mod manager; they do not hold up a launch. if (!mod.outdated.empty()) continue; - problems.push_back({mod.name, mod.title.empty() ? mod.name : mod.title, - mod.problems.empty() ? std::string("it could not be merged cleanly") : mod.problems.front()}); + const auto reason = mod.problems.empty() ? std::string("it could not be merged cleanly") : mod.problems.front(); + problems.push_back({mod.name, mod.title.empty() ? mod.name : mod.title, reason, reason.starts_with("conflicts with ")}); } if (problems.empty()) return true; std::lock_guard lock(mutex_); state_.phase = Phase::mods_broken; - state_.status = problems.size() == 1 ? "A mod could not be merged" - : std::format("{} mods could not be merged", problems.size()); + const bool conflicts = std::ranges::all_of(problems, &ModProblem::conflict); + state_.status = problems.size() == 1 ? (conflicts ? "A mod conflicts with another" : "A mod could not be merged") + : conflicts ? std::format("{} mods conflict with others", problems.size()) + : std::format("{} mods could not be merged", problems.size()); state_.detail.clear(); state_.progress = -1; state_.mod_problems = std::move(problems); diff --git a/Launcher/gui_mods.cpp b/Launcher/gui_mods.cpp index f3c3b413..407351ff 100644 --- a/Launcher/gui_mods.cpp +++ b/Launcher/gui_mods.cpp @@ -206,8 +206,8 @@ bool rail(const Fonts& fonts, ModsPanel& panel, HWND window, float width, // ---------------------------------------------------------------- MY MODS -constexpr std::array filter_names{"All mods", "Enabled", "Disabled", "Maps", "Game data", "Updates", - "Problems"}; +constexpr std::array filter_names{"All mods", "Enabled", "Disabled", "Maps", "Game data", "Updates", + "Problems", "Conflicts"}; constexpr std::array order_names{"Load order", "Name", "Largest first"}; // Something the page was asked to do to one mod or to all the ticked ones. @@ -229,6 +229,22 @@ bool update_waiting(const ModsPanel& panel, const thunderstore::Installed& insta return package && thunderstore::update_available(*package, installed); } +// Left out because it conflicts with a mod higher in the load order (mod_catalog.cpp): that +// mod's folder name, else empty. +std::string conflict_winner(const ModsPanel& panel, const std::string& name) { + constexpr std::string_view prefix = "conflicts with "; + const auto found = panel.list.excluded.find(name); + if (found == panel.list.excluded.end() || found->second.empty() || !found->second.front().starts_with(prefix)) return {}; + const auto& problem = found->second.front(); + return problem.substr(prefix.size(), problem.find(':') - prefix.size()); +} +// A mod's title, by its folder name. +std::string title_of(const ModsPanel& panel, const std::string& name) { + for (const auto& entry : panel.list.entries) + if (entry.mod.name == name) return entry.mod.title.empty() ? name : entry.mod.title; + return name; +} + // The rows the list shows, as places in the load order: what the search and the // filter leave, in the order asked for. std::vector visible_mods(const ModsPanel& panel, const thunderstore::Installed& installed) { @@ -245,7 +261,8 @@ std::vector visible_mods(const ModsPanel& panel, const thunderstore::Instal case 3: keep = !mod.levels.empty(); break; case 4: keep = mod.levels.empty() && mod.provides_layout; break; case 5: keep = update_waiting(panel, installed, mod.name); break; - case 6: keep = !mod.outdated.empty() || panel.list.excluded.contains(mod.name); break; + case 6: keep = !mod.outdated.empty() || (panel.list.excluded.contains(mod.name) && conflict_winner(panel, mod.name).empty()); break; + case 7: keep = mod.outdated.empty() && !conflict_winner(panel, mod.name).empty(); break; default: break; } if (keep && !query.empty()) @@ -379,6 +396,7 @@ void installed_row(const Fonts& fonts, ModsPanel& panel, const thunderstore::Ins } const float right = start.x + width; const bool left_out = panel.list.excluded.contains(mod.name); + const bool conflicted = left_out && !conflict_winner(panel, mod.name).empty(); const auto* package = package_for(panel.store, mod.name); const bool update = package && thunderstore::update_available(*package, installed); @@ -436,6 +454,7 @@ void installed_row(const Fonts& fonts, ModsPanel& panel, const thunderstore::Ins next -= S(12); }; if (!mod.outdated.empty()) pill("OUTDATED", color::warning); + else if (conflicted) pill("CONFLICT", color::warning); else if (left_out) pill("NOT LOADED", color::danger); // The same icon the store shows, so a mod looks like itself on both pages. @@ -498,9 +517,13 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co const float top = ImGui::GetCursorPosY(); { const auto enabled = std::count_if(entries.begin(), entries.end(), [](const auto& entry) { return entry.enabled; }); + // A mod left out for a conflict is no fault of its own: counted apart from the problems. + const auto conflicts = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { + return entry.enabled && entry.mod.outdated.empty() && !conflict_winner(panel, entry.mod.name).empty(); + }); const auto problems = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { return !entry.mod.outdated.empty() || (entry.enabled && panel.list.excluded.contains(entry.mod.name)); - }); + }) - conflicts; const auto waiting = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { return update_waiting(panel, installed, entry.mod.name); }); std::uint64_t bytes{}; @@ -511,9 +534,11 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co "Your installed mods. They load top to bottom: where two change the same thing, the higher one wins. " "Changes apply the next time Skate starts.", problems ? std::format("{} {}", problems, problems == 1 ? "PROBLEM" : "PROBLEMS") + : conflicts ? std::format("{} {}", conflicts, conflicts == 1 ? "CONFLICT" : "CONFLICTS") : waiting ? std::format("{} {} WAITING", waiting, waiting == 1 ? "UPDATE" : "UPDATES") : entries.empty() ? std::string("NONE YET") : std::string("ALL GOOD"), - problems ? color::danger : waiting ? color::blue : entries.empty() ? color::tile_grey : color::good, caption); + problems ? color::danger : conflicts ? color::warning : waiting ? color::blue + : entries.empty() ? color::tile_grey : color::good, caption); } // ------------------------------------------------ search, filter, order, refresh @@ -621,10 +646,29 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "mods.json could not be read (%s), so the game loads no mods. Any change here rewrites it.", panel.list.issue.c_str()); - std::vector dropped; - for (const auto& entry : entries) - if (entry.enabled && panel.list.excluded.contains(entry.mod.name) && entry.mod.outdated.empty()) + std::vector dropped, conflicted; + std::string winner; // with one conflicted mod: the mod it lost to + for (const auto& entry : entries) { + if (!entry.enabled || !panel.list.excluded.contains(entry.mod.name) || !entry.mod.outdated.empty()) continue; + if (auto other = conflict_winner(panel, entry.mod.name); !other.empty()) { + conflicted.push_back(entry.mod.title); + winner = title_of(panel, other); + } else { dropped.push_back(entry.mod.title); + } + } + if (!conflicted.empty()) { + std::string names; + for (const auto& title : conflicted) names += (names.empty() ? "" : ", ") + title; + if (conflicted.size() == 1) + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "%s did not load: it conflicts with %s, which is higher in the load order, so only that one loads. " + "Open it to see more.", names.c_str(), winner.c_str()); + else + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "%s did not load: each conflicts with a mod higher in the load order, so only that one loads. " + "Open one to see which.", names.c_str()); + } if (!dropped.empty()) { std::string names; for (const auto& title : dropped) names += (names.empty() ? "" : ", ") + title; @@ -760,7 +804,15 @@ void mod_overview(const Fonts& fonts, ModsPanel& panel, const thunderstore::Inst ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "Outdated: this mod does not load. It was made for another version of Skate (%s). Get an updated " "version, or rebuild it with the latest ReSkate Studio.", mod.outdated.c_str()); - if (const auto missing = panel.list.excluded.find(mod.name); missing != panel.list.excluded.end()) { + if (const auto other = conflict_winner(panel, mod.name); !other.empty()) { + const auto winner = title_of(panel, other); + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "Not loaded: this mod changes the same parts of the game as %s, which is higher in the load order, in a " + "way the two cannot share, so only %s loads. To use this mod instead, move it above %s, or switch one " + "of them off.", winner.c_str(), winner.c_str(), winner.c_str()); + if (const auto& problems = panel.list.excluded.at(mod.name); !problems.empty()) + ImGui::TextDisabled("%s", problems.front().c_str()); + } else if (const auto missing = panel.list.excluded.find(mod.name); missing != panel.list.excluded.end()) { ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "Not loaded: the game could not merge this mod cleanly, so none of it is used. Reinstall the whole " "mod folder, or rebuild it with a current ReSkate Studio."); @@ -943,16 +995,25 @@ void start_install(ModsPanel& panel, const fs::path& source, bool replace) { void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ui, ModsPanel& panel, const std::vector& problems) { const bool one = problems.size() == 1; + // A conflict is two mods changing the same game files: only the one higher in the load order loads. + const bool conflicts = std::all_of(problems.begin(), problems.end(), [](const ModProblem& problem) { return problem.conflict; }); const float rows = static_cast(problems.size()) * S(52); const auto frame = begin_panel("##mods_broken_panel", size, ImVec2(S(620), std::min(size.y - S(80), S(300) + rows))); - panel_title(fonts, one ? "A MOD COULD NOT BE MERGED" : "MODS COULD NOT BE MERGED"); + panel_title(fonts, conflicts ? (one ? "MOD CONFLICT" : "MOD CONFLICTS") : one ? "A MOD COULD NOT BE MERGED" : "MODS COULD NOT BE MERGED"); ImGui::PushTextWrapPos(0); - ImGui::TextDisabled(one - ? "This mod cannot be combined with the game's files, so none of its content would load. Skate would " - "start without it and never say why." - : "These mods cannot be combined with the game's files, so none of their content would load. Skate would " - "start without them and never say why."); + if (conflicts) + ImGui::TextDisabled(one + ? "This mod changes the same parts of the game as a mod above it in your load order, in a way the two " + "cannot share. Only the higher one loads, so this one is left out." + : "These mods change the same parts of the game as mods above them in your load order, in ways they " + "cannot share. Only the higher ones load, so these are left out."); + else + ImGui::TextDisabled(one + ? "This mod cannot be combined with the game's files, so none of its content would load. Skate would " + "start without it and never say why." + : "These mods cannot be combined with the game's files, so none of their content would load. Skate would " + "start without them and never say why."); ImGui::PopTextWrapPos(); ImGui::Spacing(); @@ -961,7 +1022,7 @@ void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ImGuiChildFlags_Borders | ImGuiChildFlags_NavFlattened); for (const auto& problem : problems) { ImGui::PushFont(fonts.bold); - ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(color::danger)); + ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(problem.conflict ? color::warning : color::danger)); ImGui::TextUnformatted(problem.title.c_str()); ImGui::PopStyleColor(); ImGui::PopFont(); @@ -973,7 +1034,9 @@ void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ImGui::EndChild(); ImGui::Spacing(); ImGui::PushFont(fonts.caption); - ImGui::TextDisabled("Reinstall the whole mod folder, or rebuild it with a current ReSkate Studio."); + ImGui::TextDisabled("%s", conflicts + ? "To use a left-out mod instead, move it above the other in the Mod Manager, or switch one of them off." + : "Reinstall the whole mod folder, or rebuild it with a current ReSkate Studio."); ImGui::PopFont(); ImGui::SetCursorPosY(frame.y - S(24) - ImGui::GetFrameHeight()); diff --git a/cmake/Core.cmake b/cmake/Core.cmake index 7615b836..1d8df8e7 100644 --- a/cmake/Core.cmake +++ b/cmake/Core.cmake @@ -102,6 +102,7 @@ if(WIN32) Engine/Vfs/mod_merge_cas_store.cpp Engine/Vfs/mod_merge_combine.cpp Engine/Vfs/mod_merge_overrides.cpp + Engine/Vfs/mod_merge_conflicts.cpp Engine/Vfs/mod_merge_textures.cpp Engine/Vfs/mod_merge_grid.cpp Engine/Vfs/mod_merge_load_screens.cpp