From db99ed4d724dd77edd2abf62f3de0dbc5bb6717e Mon Sep 17 00:00:00 2001 From: AyeZeeBB Date: Sun, 18 Oct 2026 01:35:23 -0500 Subject: [PATCH] Mods: two mods that conflict load only the higher one, found before the merge Where two mods change one of the game's assets in ways that cannot both be in the game (a resource or chunk changed differently, an EBX whose changes cannot be combined, different assets added under one name), the one lower in the load order is left out whole, so neither loads half its changes. The merge used to keep the higher copy of each asset and lose the rest of the lower mod's change silently, which broke Skaterize beside deckorate. A check before the merge (find_conflicts) reads only the bundles two or more mods ship and applies the merge's rule: about 2 s for 296 mods, on several threads, each mod's copy of a bundle by itself. When it finds conflicts nothing is built, the catalog leaves the losing mods out and merges once. The merge keeps its own check behind it, tracking per asset the last change a mod made itself, so a change carried through other mods' copies is still caught. A conflict is remembered in .reskate-excluded.json with the mod it lost to, and stays out while both are unchanged and the winner still loads above it. The launcher says "Checking the mods" and "Checking the mods for conflicts" during those steps. Its mod problems screen says MOD CONFLICT when that is all there is, and the Mod Manager shows a CONFLICT badge, an N CONFLICTS banner, a Conflicts filter and, on the mod, which mod it lost to and how to load it instead. --- Engine/Vfs/mod_catalog.cpp | 98 ++++++++-- Engine/Vfs/mod_list.cpp | 6 + Engine/Vfs/mod_list.h | 4 + Engine/Vfs/mod_merge.cpp | 25 +++ Engine/Vfs/mod_merge.h | 16 ++ Engine/Vfs/mod_merge_combine.cpp | 49 ++++- Engine/Vfs/mod_merge_conflicts.cpp | 280 +++++++++++++++++++++++++++++ Engine/Vfs/mod_merge_internal.h | 11 ++ Launcher/gui_internal.h | 1 + Launcher/gui_launcher.cpp | 10 +- Launcher/gui_mods.cpp | 97 ++++++++-- cmake/Core.cmake | 1 + 12 files changed, 563 insertions(+), 35 deletions(-) create mode 100644 Engine/Vfs/mod_merge_conflicts.cpp diff --git a/Engine/Vfs/mod_catalog.cpp b/Engine/Vfs/mod_catalog.cpp index fad31d7a..cde2bc3e 100644 --- a/Engine/Vfs/mod_catalog.cpp +++ b/Engine/Vfs/mod_catalog.cpp @@ -448,8 +448,10 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver auto exclusions = read_exclusions(result.root); const auto sdk = sdk_identity(); std::map> still_excluded; - // `remembered`: it stays out on later launches without a retry, until its files or ReSkate change. - const auto leave_out = [&](std::size_t index, std::vector problems, const std::string& fingerprint, bool remembered = true) { + // `remembered`: it stays out on later launches without a retry, until its files or ReSkate change + // (or, left out for a conflict, the mod it conflicts with: `conflict`, that mod's name and files). + const auto leave_out = [&](std::size_t index, std::vector problems, const std::string& fingerprint, bool remembered = true, + std::pair conflict = {}) { auto mod = std::move(result.mods[index]); result.mods.erase(result.mods.begin() + static_cast(index)); // Level bundles decide whether a map loads at all, so their problems lead. @@ -457,7 +459,7 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver return problem.find("levels/game/") != std::string::npos; }); mod.problems = std::move(problems); - if (remembered) still_excluded[mod.name] = {fingerprint, sdk, mod.problems}; + if (remembered) still_excluded[mod.name] = {fingerprint, sdk, mod.problems, conflict.first, conflict.second}; result.excluded.push_back(std::move(mod)); }; for (std::size_t i = result.mods.size(); i-- > 0;) { @@ -469,6 +471,18 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver // game may have been put right since (its files verified), with the mod untouched. // Remembered, it stayed out for good with the game fixed (seen 2026-10-13). if (!found->second.empty() && found->second.front().starts_with("made for game build ")) continue; + // Left out for a conflict: only while the mod it lost to still loads above it, as it was. + if (!previous.conflictWith.empty()) { + const auto winner = std::find_if(result.mods.begin(), result.mods.begin() + static_cast(i), + [&](const Mod& mod) { return mod.name == previous.conflictWith; }); + if (winner == result.mods.begin() + static_cast(i) || + mod_fingerprint(winner->directory) != previous.conflictFingerprint) + continue; + result.notes.push_back("Leaving out " + result.mods[i].name + ": it conflicts with " + previous.conflictWith + + ", which loads above it, and neither has changed since"); + leave_out(i, found->second, previous.fingerprint, true, {previous.conflictWith, previous.conflictFingerprint}); + continue; + } result.notes.push_back("Leaving out " + result.mods[i].name + ": it could not be merged cleanly before and its files have not changed since"); leave_out(i, found->second, previous.fingerprint); @@ -519,18 +533,69 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver // without it. Removing one mod cannot break another, but a few rounds // are allowed in case two damaged mods hid each other's problems. for (int round = 0;; ++round) { - merge = merge_mods(result, observe); + // The first looks for conflicts before building anything (find_conflicts). + merge = merge_mods(result, observe, {.conflicts_first = round == 0}); if (!merge.issue.empty() || round == 4) break; // a fifth merge only rebuilds without round four's removals - bool removed = false; - for (std::size_t i = result.mods.size(); i-- > 0;) { - const auto found = merge.problems.find(result.mods[i].name); - if (found == merge.problems.end() || found->second.empty()) continue; - result.notes.push_back("Leaving out " + result.mods[i].name + - ": it could not be merged cleanly; merging the other mods again without it"); - leave_out(i, found->second, mod_fingerprint(result.mods[i].directory)); - removed = true; + // (What the check before the merge noted, which the merge's own notes below would not carry.) + if (merge.stopped_for_conflicts) + for (auto& note : merge.notes) result.notes.push_back(std::move(note)); + // Everything this merge found is settled at once, so one more merge is enough: the + // mods that could not be merged cleanly, and, of two mods that change one asset in + // ways that cannot both be in the game, the one lower in the load order, whole (so + // neither loads half its changes). Mods are looked at from the top of the load + // order down, so one that lost only to a mod that is itself left out stays in. + // ReSkate's own packs, which load below every mod, are never left out for this. + const auto official = official_packs(data_root / mods_folder, true); + const auto ours = [&](const std::string& name) { + return name.starts_with("reskate-builtin-") || name.find('#') != std::string::npos || + std::ranges::any_of(official, [&](const BuiltinPack& pack) { return pack.name == name; }); + }; + struct Out { + std::vector problems; + std::string winner; // left out for a conflict with this mod + }; + std::map out; + std::set leaving; + for (std::size_t i = 0; i < result.mods.size(); ++i) { + const auto& name = result.mods[i].name; + if (const auto found = merge.problems.find(name); found != merge.problems.end() && !found->second.empty()) { + out[i].problems = found->second; + leaving.insert(name); + continue; + } + if (ours(name)) continue; + for (const auto& row : merge.conflicts) { + if (row.lost != name || leaving.contains(row.kept)) continue; + auto& entry = out[i]; + if (entry.winner.empty()) entry.winner = row.kept; + entry.problems.push_back("conflicts with " + row.kept + ": both change " + row.asset + + (row.assets > 1 ? " and " + std::to_string(row.assets - 1) + " other asset(s)" : std::string{}) + + ", and " + row.kept + " is higher in the load order"); + } + if (out.contains(i)) leaving.insert(name); } - if (!removed) break; + const bool removed = !out.empty(); + if (removed) + result.notes.push_back((merge.stopped_for_conflicts ? std::string("The check before merging") + : "Merge " + std::to_string(round + 1)) + + " left " + std::to_string(out.size()) + " mod(s) out; merging once more without them"); + for (auto at = out.rbegin(); at != out.rend(); ++at) { + const auto i = at->first; + const auto& [problems, winner] = at->second; + if (winner.empty()) { + result.notes.push_back("Leaving out " + result.mods[i].name + + ": it could not be merged cleanly; merging the other mods again without it"); + leave_out(i, problems, mod_fingerprint(result.mods[i].directory)); + continue; + } + const auto kept = std::ranges::find_if(result.mods, [&](const Mod& mod) { return mod.name == winner; }); + result.notes.push_back("Leaving out " + result.mods[i].name + ": it conflicts with " + winner + + ", which is higher in the load order; merging the other mods again without it"); + leave_out(i, problems, mod_fingerprint(result.mods[i].directory), true, + {winner, kept != result.mods.end() ? mod_fingerprint(kept->directory) : std::string{}}); + } + // (Conflicts only of ReSkate's own packs leave nothing out: then the merge itself runs.) + if (!removed && !merge.stopped_for_conflicts) break; } for (auto& note : merge.notes) result.notes.push_back(std::move(note)); result.merged = merge.built; @@ -555,6 +620,13 @@ Catalog load_catalog(const std::filesystem::path& data_root, const MergeObserver "). Update it for this game version."); continue; } + if (!mod.problems.empty() && mod.problems.front().starts_with("conflicts with ")) { + const auto& problem = mod.problems.front(); + const auto other = problem.substr(15, problem.find(':') - 15); + result.warnings.push_back("Mod " + mod.name + " was not loaded: it " + problem + ". Only " + other + + " is loaded. To use " + mod.name + " instead, move it above " + other + " in the mod list, or turn one of them off."); + continue; + } if (!mod.problems.empty() && mod.problems.front().starts_with("made for game build ")) { result.warnings.push_back("Mod " + mod.name + " was not loaded: it was " + mod.problems.front() + (mod.levels.empty() ? std::string{} : " (its maps are not listed: " + join(mod.levels, ", ") + ")") + "."); diff --git a/Engine/Vfs/mod_list.cpp b/Engine/Vfs/mod_list.cpp index e2e3bd65..b9a28268 100644 --- a/Engine/Vfs/mod_list.cpp +++ b/Engine/Vfs/mod_list.cpp @@ -261,6 +261,8 @@ std::map> read_exclusions(const std::filesys Exclusion exclusion; exclusion.fingerprint = row.value("fingerprint", std::string{}); exclusion.sdk = row.value("sdk", std::string{}); + exclusion.conflictWith = row.value("conflict_with", std::string{}); + exclusion.conflictFingerprint = row.value("conflict_fingerprint", std::string{}); if (row.contains("problems") && row.at("problems").is_array()) for (const auto& problem : row.at("problems")) if (problem.is_string() && exclusion.problems.size() < 16) @@ -282,6 +284,10 @@ void write_exclusions(const std::filesystem::path& mods_root, auto row = Json::object(); row["fingerprint"] = exclusion.fingerprint; row["sdk"] = exclusion.sdk; + if (!exclusion.conflictWith.empty()) { + row["conflict_with"] = exclusion.conflictWith; + row["conflict_fingerprint"] = exclusion.conflictFingerprint; + } auto problems = Json::array(); for (std::size_t i = 0; i < exclusion.problems.size() && i < 16; ++i) problems.push_back(exclusion.problems[i].substr(0, 600)); diff --git a/Engine/Vfs/mod_list.h b/Engine/Vfs/mod_list.h index 5237ca52..2ccdf9de 100644 --- a/Engine/Vfs/mod_list.h +++ b/Engine/Vfs/mod_list.h @@ -67,6 +67,10 @@ struct Exclusion { std::string fingerprint; // mod_fingerprint() when it was left out std::string sdk; // the ReSkate.dll that left it out; a new one retries std::vector problems; // why, most important first + // Left out for a conflict with this mod, higher in the load order, as its files were then + // (its mod_fingerprint()). It stays out only while that mod still loads above it unchanged. + std::string conflictWith; + std::string conflictFingerprint; }; // Names, sizes and write times of every file in the mod folder. std::string mod_fingerprint(const std::filesystem::path& directory); diff --git a/Engine/Vfs/mod_merge.cpp b/Engine/Vfs/mod_merge.cpp index e9ed70f4..3008ae75 100644 --- a/Engine/Vfs/mod_merge.cpp +++ b/Engine/Vfs/mod_merge.cpp @@ -118,9 +118,28 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con // launch (the player is waiting on nothing else), half of them and fewer while the game runs. const auto cores = std::max(1U, std::thread::hardware_concurrency()); const std::size_t readers = std::min(options.live ? std::max(1U, cores / 2) : cores, options.live ? 6U : 8U); + // The checks before the merge say what they are doing, the bar still at its start. + const auto checking = [&](std::string step) { + if (observe) try { observe({0, 1, mods.size(), std::move(step)}); } catch (...) {} + }; std::optional> itemsRead; + if (storeKnown) checking("Checking the mods"); if (storeKnown && store_copy_problems(catalog, report, options.live ? options.checked : std::vector{}, readers - 1, options.live, itemsRead)) return report; + // Mods that conflict, found the same way before anything is built: the catalog leaves the + // losing ones out and merges once, where finding them in the merge took a second merge. + if (options.conflicts_first && !options.live) { + checking("Checking the mods for conflicts"); + const auto started = Clock::now(); + report.conflicts = find_conflicts(mods, modFiles, catalog.data_root, output, readers - 1); + const auto tenths = (std::chrono::duration_cast(Clock::now() - started).count() + 50) / 100; + report.notes.push_back("Conflict check: " + std::to_string(report.conflicts.size()) + " conflicting pair(s) in " + + std::to_string(tenths / 10) + '.' + std::to_string(tenths % 10) + " s"); + if (!report.conflicts.empty()) { + report.stopped_for_conflicts = true; + return report; + } + } // Which mod each item is from (mod_item_sources.h): what was just read, and for a mod a // live merge did not read again, what the launch's merge found. Read before the patch // folder is cleared. @@ -547,6 +566,12 @@ MergeReport merge_mods(const Catalog& catalog, const MergeObserver& observe, con auto& list = report.problems[mod]; list.insert(list.end(), std::make_move_iterator(problems.begin()), std::make_move_iterator(problems.end())); } + for (auto& row : job.report.conflicts) { + const auto same = std::ranges::find_if(report.conflicts, [&](const MergeReport::Conflict& held) { + return held.kept == row.kept && held.lost == row.lost; }); + if (same != report.conflicts.end()) same->assets += row.assets; + else report.conflicts.push_back(std::move(row)); + } auto bytes = fb::write_patch_toc(job.merged.bundles, job.merged.chunks, job.merged.flags); // Read the result back before publishing it: a TOC whose perfect // hash does not resolve would take the game down at load time. diff --git a/Engine/Vfs/mod_merge.h b/Engine/Vfs/mod_merge.h index cff48198..229e38a5 100644 --- a/Engine/Vfs/mod_merge.h +++ b/Engine/Vfs/mod_merge.h @@ -27,6 +27,19 @@ struct MergeReport { // be decoded, a copy that had to be left out. Each mod gets a summary line // from these, and a level that never finishes loading can name its mod. std::map> problems; + // Two mods changing one asset in ways that cannot both be in the game: the copy of the mod + // higher in the load order is kept and the other's changes are not in it. One row per pair of + // mods (folder names), with one of the assets and how many. The catalog leaves the losing mod + // out whole and merges again, so no mod loads half its changes. + struct Conflict { + std::string kept; + std::string lost; + std::string asset; + std::size_t assets{}; + }; + std::vector conflicts; + // Nothing was built: the check before the merge found `conflicts` (MergeOptions::conflicts_first). + bool stopped_for_conflicts{}; // Every superbundle the merged mods' layouts assign to an install chunk, // as (superbundle, install chunk name). A live merge registers the ones // the game did not know at launch. @@ -79,6 +92,9 @@ struct MergeOptions { // the others against it. Reading every mod's items and meshes again is most of what // adding one map would otherwise wait on. std::vector checked; + // Look for mods that conflict (find_conflicts) before building anything, and when some do, + // stop there (MergeReport::stopped_for_conflicts) so the caller merges once without them. + bool conflicts_first{}; }; // Combines every enabled mod into one patch under Mods/.reskate: superbundle diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index c0233b0b..4bfe2bfa 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -121,6 +121,14 @@ struct BundleState { // The base's own sha1 for each asset, so a mod that merely carries an // unchanged copy can be told apart from one that changed it. std::unordered_map baseSha; + // Of each asset, the last version a mod shipped as its own change (not the game's copy, not + // one rewritten to carry another mod's change), and every mod that shipped that same version. + // A later mod shipping its own different version means theirs is lost: a conflict. + struct OwnChange { + fb::Sha1 sha1; + std::vector mods; + }; + std::unordered_map ownChange; // Assets a mod added that a higher-priority mod's different asset of the // same name then replaced: the key, and the sha1 of the copy that went. std::vector> displaced; @@ -223,6 +231,17 @@ std::span unsigned_bytes(std::span data) { return {reinterpret_cast(data.data()), data.size()}; } +// `lost`'s change to `asset` is not in the game, `kept`'s copy being there (MergeReport::conflicts). +void conflict(MergeReport& report, const std::string& kept, const std::string& lost, const std::string& asset) { + if (kept.empty() || lost.empty() || kept == lost) return; + for (auto& row : report.conflicts) + if (row.kept == kept && row.lost == lost) { + ++row.assets; + return; + } + report.conflicts.push_back({kept, lost, asset, 1}); +} + } // namespace std::vector merge_chunk_metadata(std::span> lists, @@ -823,6 +842,26 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, const auto* owner = isBase ? nullptr : &*owners.insert(modName).first; // By the mod whose added assets this copy's replace: how many, and one of them. std::map> replaced; + // This mod's own version of an asset the merge does not combine (EBX the game has is combined + // further down, and so are the shader tables and the preset table): a mod lower in the load + // order whose own version differs loses it, a conflict. find_conflicts applies the same rule + // before the merge. + const auto own_change = [&](const std::string& key, const Asset& asset, bool rewritten) { + if (isBase || rewritten) return; + if (asset.asset.kind == fb::AssetKind::resource && + (asset.asset.resourceType == fb::shader::programLookupType || + asset.asset.resourceType == fb::shader::textureLookupType || fb::bundle_ref::is_table(asset.asset.name))) + return; + const auto shipped = state.baseSha.find(key); + if (shipped != state.baseSha.end() && (asset.asset.kind == fb::AssetKind::ebx || shipped->second == asset.asset.sha1)) return; + auto& last = state.ownChange[key]; + if (!last.mods.empty() && last.sha1 == asset.asset.sha1) { + if (std::ranges::find(last.mods, modName) == last.mods.end()) last.mods.push_back(modName); + return; + } + for (const auto& lost : last.mods) conflict(report, modName, lost, asset.asset.name); + last = {asset.asset.sha1, {modName}}; + }; for (std::size_t position = 0; position < flat.size(); ++position) { auto& entry = flat[position]; entry.base = isBase; @@ -856,6 +895,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, } if (isBase) state.baseSha.insert_or_assign(id, entry.asset.sha1); const auto at = state.seen.find(id); + own_change(id, entry, rewritten); if (at == state.seen.end()) { state.seen.emplace(std::move(id), state.assets.size()); state.assets.push_back(std::move(entry)); @@ -876,9 +916,10 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // Two mods rewriting one non-EBX asset cannot both be honoured; // say so, because the loser silently loses whatever it shipped. if (entry.asset.kind == fb::AssetKind::chunk && !previous.base && - previous.asset.sha1 != entry.asset.sha1) + previous.asset.sha1 != entry.asset.sha1) { report.notes.push_back("contested chunk " + entry.asset.name + ": kept the highest-priority copy"); + } // Two mods each adding a different asset under one name: a bundle // holds one asset per name, so the lower-priority mod's is gone, // and whatever of that mod's still names it by guid finds nothing @@ -1067,6 +1108,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, if (entry.mod) others.erase(*entry.mod); std::string names; for (const auto& other : others) names += (names.empty() ? "" : ", ") + other; + if (entry.mod) + for (const auto& other : others) conflict(report, *entry.mod, other, entry.asset.name); if (!others.empty()) report.notes.push_back(entry.asset.name + ": kept " + (entry.mod ? *entry.mod + "'s copy" : std::string("the highest-priority copy")) + @@ -1128,6 +1171,10 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, report.notes.push_back(entry.asset.name + ": kept the highest-priority copy, the edits could not be combined (" + failure.what() + ")"); + if (entry.mod) + for (const auto& contribution : found->second) + if (!contribution.base && contribution.sha1 != baseCopy->sha1 && contribution.sha1 != entry.asset.sha1) + conflict(report, *entry.mod, path_utf8(contribution.root->filename()), entry.asset.name); } } diff --git a/Engine/Vfs/mod_merge_conflicts.cpp b/Engine/Vfs/mod_merge_conflicts.cpp new file mode 100644 index 00000000..df7c5a0a --- /dev/null +++ b/Engine/Vfs/mod_merge_conflicts.cpp @@ -0,0 +1,280 @@ +#include "mod_merge_internal.h" +#include "Engine/Resource/bundle_ref_table.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/ebx_merge.h" +#include "Engine/Resource/shader_lookup.h" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace dingosdk::mods::detail { +namespace { + +// The merge keys an asset within a bundle by its kind and, for a chunk its guid, for the rest its +// name in any case (asset_key in mod_merge_combine.cpp). The same, hashed: a big bundle has some +// 17,000 assets and a hundred mods' copies of it, too many to make a string for each. +std::uint64_t key_hash(const fb::BundleAsset& asset) { + std::uint64_t value = 0xCBF29CE484222325ULL; + if (asset.kind == fb::AssetKind::chunk) + for (const auto byte : asset.guid.bytes) value = (value ^ static_cast(byte)) * 0x100000001B3ULL; + else + value = name_hash(asset.name); + return value ^ (static_cast(asset.kind) + 1) * 0x9E3779B97F4A7C15ULL; +} + +// Assets the merge combines whatever the mods did to them, so two versions are no conflict: +// the shader tables, the preset table and the collision the material grid renumbers. +bool combined_anyway(const fb::BundleAsset& asset) { + return asset.kind == fb::AssetKind::resource && + (asset.resourceType == fb::shader::programLookupType || asset.resourceType == fb::shader::textureLookupType || + asset.resourceType == fb::material_grid::physicsResourceType || fb::bundle_ref::is_table(asset.name)); +} + +using Rows = std::vector; +void conflict(Rows& rows, const std::string& kept, const std::string& lost, const std::string& asset) { + if (kept.empty() || lost.empty() || kept == lost) return; + for (auto& row : rows) + if (row.kept == kept && row.lost == lost) { + ++row.assets; + return; + } + rows.push_back({kept, lost, asset, 1}); +} + +// One mod's version of an asset in a bundle that is not the game's. +struct Change { + std::uint64_t key{}; + std::size_t copy{}; // in Shared::copies + std::size_t at{}; // the asset's place in the manifest's EBX, for reading it + const fb::BundleAsset* asset{}; +}; + +// One bundle that two or more mods ship in one superbundle, and what is read of it. +struct Shared { + const fb::TocBundle* game{}; + std::vector> copies; // (mod, its copy), lowest priority first + std::optional gameListing; + // The game's version of each asset: its sha1, and its place among the EBX. + std::unordered_map> original; + std::vector> listings; // by copy + std::vector> changes; // by copy: what it ships that is not the game's +}; + +// `count` pieces of work on up to `threads` threads beside this one, taken in turn. +void in_parallel(std::size_t count, std::size_t threads, bool background, const std::function& work) { + std::atomic next{0}; + const auto run = [&] { + for (;;) { + const auto mine = next.fetch_add(1); + if (mine >= count) return; + try { + work(mine); + } catch (const std::exception&) {} // a part that fails finds nothing: the merge still checks for itself + } + }; + std::vector workers; + try { + // Threads of their own, not the system's pool (see the superbundles in mod_merge.cpp). + for (std::size_t index = 0; index + 1 < count && index < threads; ++index) + workers.emplace_back([&] { + if (background) SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL); + run(); + }); + } catch (const std::system_error&) {} // fewer threads, or none: this one does the rest + run(); +} + +// The game's copy of a bundle, keyed, to tell a mod's change from a copy of the game's. +void read_game(Shared& shared, const CasStore& store, const fs::path& baseRoot, const fs::path& gameRoot) { + if (!shared.game) return; + shared.gameListing = list_bundle(store, baseRoot, baseRoot, *shared.game, gameRoot); + if (!shared.gameListing) return; + const auto& manifest = shared.gameListing->manifest; + for (std::size_t index = 0; index < manifest.ebx.size(); ++index) + shared.original.emplace(key_hash(manifest.ebx[index]), std::pair{manifest.ebx[index].sha1, index}); + for (const auto* list : {&manifest.resources, &manifest.chunks}) + for (const auto& asset : *list) shared.original.emplace(key_hash(asset), std::pair{asset.sha1, std::size_t{}}); +} + +// One mod's copy: its asset list, and of that what is not the game's. +void read_copy(Shared& shared, std::size_t copy, const std::vector& mods, const CasStore& store, + const fs::path& baseRoot, const fs::path& gameRoot) { + auto& listing = shared.listings[copy]; + listing = list_bundle(store, mods[shared.copies[copy].first]->directory, baseRoot, *shared.copies[copy].second, gameRoot); + if (!listing) return; // (the merge notes what it cannot read) + auto& changes = shared.changes[copy]; + const auto take = [&](const std::vector& list, bool ebx) { + for (std::size_t index = 0; index < list.size(); ++index) { + const auto& asset = list[index]; + if (combined_anyway(asset)) continue; + const auto key = key_hash(asset); + if (const auto shipped = shared.original.find(key); shipped != shared.original.end() && shipped->second.first == asset.sha1) + continue; + changes.push_back({key, copy, ebx ? index : 0, &asset}); + } + }; + take(listing->manifest.ebx, true); + take(listing->manifest.resources, false); + take(listing->manifest.chunks, false); +} + +// The conflicts in one bundle, from the changes its copies were found to ship. +Rows decide(const Shared& shared, const std::vector& mods, const CasStore& store, + const fs::path& baseRoot, const fs::path& gameRoot) { + Rows rows; + // Each asset's changes, lowest priority first, the assets in the order they were first met. + std::unordered_map> byAsset; + std::vector order; + for (const auto& copy : shared.changes) + for (const auto& change : copy) { + auto& list = byAsset[change.key]; + if (list.empty()) order.push_back(change.key); + list.push_back(&change); + } + const auto mod_of = [&](const Change& change) -> const std::string& { return mods[shared.copies[change.copy].first]->name; }; + for (const auto key : order) { + const auto& changes = byAsset.at(key); + const auto& kept = *changes.back(); + // Two mods' different changes, or nothing to decide. + if (std::ranges::all_of(changes, [&](const Change* change) { + return change->asset->sha1 == kept.asset->sha1 || shared.copies[change->copy].first == shared.copies[kept.copy].first; })) + continue; + const auto shipped = shared.original.find(key); + + if (kept.asset->kind != fb::AssetKind::ebx || shipped == shared.original.end()) { + // Not combined: of each asset the highest-priority mod's version is the one in the game, and a + // mod lower down that changed it differently loses its change. (The rule of absorb's own_change.) + fb::Sha1 last{}; + std::vector holders; + for (const auto* change : changes) { + const auto& name = mod_of(*change); + if (!holders.empty() && change->asset->sha1 == last) { + if (std::ranges::find(holders, &name) == holders.end()) holders.push_back(&name); + continue; + } + for (const auto* lost : holders) conflict(rows, name, *lost, change->asset->name); + last = change->asset->sha1; + holders = {&name}; + } + continue; + } + + // EBX the game has is combined, the changes of every mod onto the game's copy: a conflict only + // when that cannot be done (the kept-whole and could-not-combine cases of the merge's combine). + const auto lose_to_kept = [&] { + for (const auto* change : changes) + if (change->asset->sha1 != kept.asset->sha1) conflict(rows, mod_of(kept), mod_of(*change), kept.asset->name); + }; + try { + const auto base = fb::ebx::read_document( + read_asset(store, baseRoot, baseRoot, *shared.gameListing, shipped->second.second, gameRoot)); + // The combined material grid is worked out separately (mod_merge_grid.cpp). + if (lower(base.rootType).find("materialgrid") != std::string::npos) continue; + std::map documents; + std::vector edits; + for (const auto* change : changes) { + auto [at, fresh] = documents.try_emplace(change->asset->sha1); + if (fresh) + at->second = fb::ebx::read_document(read_asset(store, mods[shared.copies[change->copy].first]->directory, + baseRoot, *shared.listings[change->copy], change->at, gameRoot)); + edits.push_back(&at->second); + } + fb::ebx::MergeSummary summary; + (void)fb::ebx::merge_documents(base, edits, &summary); + if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries && (!summary.values || !summary.exact())) + lose_to_kept(); + } catch (const std::exception&) { + lose_to_kept(); + } + } + return rows; +} + +} // namespace + +std::vector find_conflicts(const std::vector& mods, + const std::map& modFiles, + const fs::path& dataRoot, const fs::path& output, + std::size_t threads, bool background) { + const auto baseRoot = dataRoot / L"Data"; + const auto gameRoot = dataRoot; + // The superbundles two or more mods ship: only there can two mods' versions of an asset meet. + std::map> shippedBy; // lower case relative path, mods highest priority first + std::map spelled; + for (std::size_t mod = 0; mod < mods.size(); ++mod) + if (const auto files = modFiles.find(mods[mod]); files != modFiles.end()) + for (const auto& relative : files->second.tocs) { + auto key = lower(relative); + spelled.try_emplace(key, relative); + shippedBy[std::move(key)].push_back(mod); + } + std::erase_if(shippedBy, [](const auto& entry) { return entry.second.size() < 2; }); + if (shippedBy.empty()) return {}; + + // Their TOCs, the game's and the mods', kept while the bundles they list are read. + std::vector tocs; + tocs.reserve([&] { + std::size_t count{}; + for (const auto& [relative, by] : shippedBy) count += by.size() + 1; + return count; + }()); + std::vector bundles; + for (const auto& [relative, by] : shippedBy) { + std::map named; + for (auto mod = by.rbegin(); mod != by.rend(); ++mod) { + try { + const auto& toc = tocs.emplace_back(fb::read_toc(read_file(mods[*mod]->directory / fs::path(spelled.at(relative))))); + for (const auto& bundle : toc.bundles) named[lower(bundle.name)].copies.emplace_back(*mod, &bundle); + } catch (const std::exception&) {} // (the merge leaves a mod out whose TOC cannot be read) + } + std::erase_if(named, [](const auto& entry) { return entry.second.copies.size() < 2; }); + if (named.empty()) continue; + std::error_code error; + if (const auto game = baseRoot / fs::path(spelled.at(relative)); fs::is_regular_file(game, error)) try { + const auto& toc = tocs.emplace_back(fb::read_toc(read_file(game))); + for (const auto& bundle : toc.bundles) + if (const auto found = named.find(lower(bundle.name)); found != named.end()) found->second.game = &bundle; + } catch (const std::exception&) {} + for (auto& [name, shared] : named) { + shared.listings.resize(shared.copies.size()); + shared.changes.resize(shared.copies.size()); + bundles.push_back(std::move(shared)); + } + } + if (bundles.empty()) return {}; + + // On several threads, in three steps, so that one bundle a hundred mods ship is not all one + // thread's: the game's copy of each bundle, then each mod's copy of each by itself, then what + // the copies' changes mean in each bundle. The rows are then put together in bundle order. + const auto layout = vfs::read_layout(baseRoot / L"layout.toc"); + const CasStore store(baseRoot, output, layout.root); + in_parallel(bundles.size(), threads, background, [&](std::size_t index) { read_game(bundles[index], store, baseRoot, gameRoot); }); + std::vector> copies; // (bundle, copy) + for (std::size_t index = 0; index < bundles.size(); ++index) + for (std::size_t copy = 0; copy < bundles[index].copies.size(); ++copy) copies.emplace_back(index, copy); + in_parallel(copies.size(), threads, background, [&](std::size_t index) { + read_copy(bundles[copies[index].first], copies[index].second, mods, store, baseRoot, gameRoot); + }); + std::vector found(bundles.size()); + in_parallel(bundles.size(), threads, background, [&](std::size_t index) { + found[index] = decide(bundles[index], mods, store, baseRoot, gameRoot); + }); + Rows rows; + for (const auto& bundle : found) + for (const auto& row : bundle) { + const auto same = std::ranges::find_if(rows, [&](const MergeReport::Conflict& held) { + return held.kept == row.kept && held.lost == row.lost; }); + if (same != rows.end()) same->assets += row.assets; + else rows.push_back(row); + } + return rows; +} + +} // namespace dingosdk::mods::detail diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 28cf4144..6e1110c4 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -320,6 +320,17 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const fs::path& gameRoot, MergeReport& report, std::size_t threads = 0, bool background = false); +// Of two mods that change one of the game's assets in ways that cannot both be in the game, the +// one lower in the load order, found from the mods' asset lists before anything is merged: the +// rule the merge applies as it combines (MergeReport::conflicts), read off the bundles two or +// more mods ship, so the catalog can leave the losing mods out before the one merge that builds +// the patch. `mods` highest priority first; `output` is only where the patch would go. Never +// throws: what cannot be read is left to the merge, which still checks for itself. +std::vector find_conflicts(const std::vector& mods, + const std::map& modFiles, + const fs::path& dataRoot, const fs::path& output, + std::size_t threads = 0, bool background = false); + // A bundle's chunk metadata is a list with one record for each of its chunks: the // hash of the name of the resource the chunk belongs to ("h64") and what the // engine needs before it streams the chunk (a texture's first mip). In the game's diff --git a/Launcher/gui_internal.h b/Launcher/gui_internal.h index eecfbe76..cbfc9dc6 100644 --- a/Launcher/gui_internal.h +++ b/Launcher/gui_internal.h @@ -227,6 +227,7 @@ struct ModProblem { std::string name; // folder under Mods/, so it can be switched off std::string title; std::string reason; + bool conflict{}; // left out for another mod above it that changes the same game files }; struct State { diff --git a/Launcher/gui_launcher.cpp b/Launcher/gui_launcher.cpp index 557b5682..d644c653 100644 --- a/Launcher/gui_launcher.cpp +++ b/Launcher/gui_launcher.cpp @@ -503,14 +503,16 @@ bool Launcher::run_mod_merge() { // Mods built for another game version say so for themselves, in game // and in the mod manager; they do not hold up a launch. if (!mod.outdated.empty()) continue; - problems.push_back({mod.name, mod.title.empty() ? mod.name : mod.title, - mod.problems.empty() ? std::string("it could not be merged cleanly") : mod.problems.front()}); + const auto reason = mod.problems.empty() ? std::string("it could not be merged cleanly") : mod.problems.front(); + problems.push_back({mod.name, mod.title.empty() ? mod.name : mod.title, reason, reason.starts_with("conflicts with ")}); } if (problems.empty()) return true; std::lock_guard lock(mutex_); state_.phase = Phase::mods_broken; - state_.status = problems.size() == 1 ? "A mod could not be merged" - : std::format("{} mods could not be merged", problems.size()); + const bool conflicts = std::ranges::all_of(problems, &ModProblem::conflict); + state_.status = problems.size() == 1 ? (conflicts ? "A mod conflicts with another" : "A mod could not be merged") + : conflicts ? std::format("{} mods conflict with others", problems.size()) + : std::format("{} mods could not be merged", problems.size()); state_.detail.clear(); state_.progress = -1; state_.mod_problems = std::move(problems); diff --git a/Launcher/gui_mods.cpp b/Launcher/gui_mods.cpp index f3c3b413..407351ff 100644 --- a/Launcher/gui_mods.cpp +++ b/Launcher/gui_mods.cpp @@ -206,8 +206,8 @@ bool rail(const Fonts& fonts, ModsPanel& panel, HWND window, float width, // ---------------------------------------------------------------- MY MODS -constexpr std::array filter_names{"All mods", "Enabled", "Disabled", "Maps", "Game data", "Updates", - "Problems"}; +constexpr std::array filter_names{"All mods", "Enabled", "Disabled", "Maps", "Game data", "Updates", + "Problems", "Conflicts"}; constexpr std::array order_names{"Load order", "Name", "Largest first"}; // Something the page was asked to do to one mod or to all the ticked ones. @@ -229,6 +229,22 @@ bool update_waiting(const ModsPanel& panel, const thunderstore::Installed& insta return package && thunderstore::update_available(*package, installed); } +// Left out because it conflicts with a mod higher in the load order (mod_catalog.cpp): that +// mod's folder name, else empty. +std::string conflict_winner(const ModsPanel& panel, const std::string& name) { + constexpr std::string_view prefix = "conflicts with "; + const auto found = panel.list.excluded.find(name); + if (found == panel.list.excluded.end() || found->second.empty() || !found->second.front().starts_with(prefix)) return {}; + const auto& problem = found->second.front(); + return problem.substr(prefix.size(), problem.find(':') - prefix.size()); +} +// A mod's title, by its folder name. +std::string title_of(const ModsPanel& panel, const std::string& name) { + for (const auto& entry : panel.list.entries) + if (entry.mod.name == name) return entry.mod.title.empty() ? name : entry.mod.title; + return name; +} + // The rows the list shows, as places in the load order: what the search and the // filter leave, in the order asked for. std::vector visible_mods(const ModsPanel& panel, const thunderstore::Installed& installed) { @@ -245,7 +261,8 @@ std::vector visible_mods(const ModsPanel& panel, const thunderstore::Instal case 3: keep = !mod.levels.empty(); break; case 4: keep = mod.levels.empty() && mod.provides_layout; break; case 5: keep = update_waiting(panel, installed, mod.name); break; - case 6: keep = !mod.outdated.empty() || panel.list.excluded.contains(mod.name); break; + case 6: keep = !mod.outdated.empty() || (panel.list.excluded.contains(mod.name) && conflict_winner(panel, mod.name).empty()); break; + case 7: keep = mod.outdated.empty() && !conflict_winner(panel, mod.name).empty(); break; default: break; } if (keep && !query.empty()) @@ -379,6 +396,7 @@ void installed_row(const Fonts& fonts, ModsPanel& panel, const thunderstore::Ins } const float right = start.x + width; const bool left_out = panel.list.excluded.contains(mod.name); + const bool conflicted = left_out && !conflict_winner(panel, mod.name).empty(); const auto* package = package_for(panel.store, mod.name); const bool update = package && thunderstore::update_available(*package, installed); @@ -436,6 +454,7 @@ void installed_row(const Fonts& fonts, ModsPanel& panel, const thunderstore::Ins next -= S(12); }; if (!mod.outdated.empty()) pill("OUTDATED", color::warning); + else if (conflicted) pill("CONFLICT", color::warning); else if (left_out) pill("NOT LOADED", color::danger); // The same icon the store shows, so a mod looks like itself on both pages. @@ -498,9 +517,13 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co const float top = ImGui::GetCursorPosY(); { const auto enabled = std::count_if(entries.begin(), entries.end(), [](const auto& entry) { return entry.enabled; }); + // A mod left out for a conflict is no fault of its own: counted apart from the problems. + const auto conflicts = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { + return entry.enabled && entry.mod.outdated.empty() && !conflict_winner(panel, entry.mod.name).empty(); + }); const auto problems = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { return !entry.mod.outdated.empty() || (entry.enabled && panel.list.excluded.contains(entry.mod.name)); - }); + }) - conflicts; const auto waiting = std::count_if(entries.begin(), entries.end(), [&](const auto& entry) { return update_waiting(panel, installed, entry.mod.name); }); std::uint64_t bytes{}; @@ -511,9 +534,11 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co "Your installed mods. They load top to bottom: where two change the same thing, the higher one wins. " "Changes apply the next time Skate starts.", problems ? std::format("{} {}", problems, problems == 1 ? "PROBLEM" : "PROBLEMS") + : conflicts ? std::format("{} {}", conflicts, conflicts == 1 ? "CONFLICT" : "CONFLICTS") : waiting ? std::format("{} {} WAITING", waiting, waiting == 1 ? "UPDATE" : "UPDATES") : entries.empty() ? std::string("NONE YET") : std::string("ALL GOOD"), - problems ? color::danger : waiting ? color::blue : entries.empty() ? color::tile_grey : color::good, caption); + problems ? color::danger : conflicts ? color::warning : waiting ? color::blue + : entries.empty() ? color::tile_grey : color::good, caption); } // ------------------------------------------------ search, filter, order, refresh @@ -621,10 +646,29 @@ void installed_page(Launcher& launcher, const Fonts& fonts, ModsPanel& panel, co ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "mods.json could not be read (%s), so the game loads no mods. Any change here rewrites it.", panel.list.issue.c_str()); - std::vector dropped; - for (const auto& entry : entries) - if (entry.enabled && panel.list.excluded.contains(entry.mod.name) && entry.mod.outdated.empty()) + std::vector dropped, conflicted; + std::string winner; // with one conflicted mod: the mod it lost to + for (const auto& entry : entries) { + if (!entry.enabled || !panel.list.excluded.contains(entry.mod.name) || !entry.mod.outdated.empty()) continue; + if (auto other = conflict_winner(panel, entry.mod.name); !other.empty()) { + conflicted.push_back(entry.mod.title); + winner = title_of(panel, other); + } else { dropped.push_back(entry.mod.title); + } + } + if (!conflicted.empty()) { + std::string names; + for (const auto& title : conflicted) names += (names.empty() ? "" : ", ") + title; + if (conflicted.size() == 1) + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "%s did not load: it conflicts with %s, which is higher in the load order, so only that one loads. " + "Open it to see more.", names.c_str(), winner.c_str()); + else + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "%s did not load: each conflicts with a mod higher in the load order, so only that one loads. " + "Open one to see which.", names.c_str()); + } if (!dropped.empty()) { std::string names; for (const auto& title : dropped) names += (names.empty() ? "" : ", ") + title; @@ -760,7 +804,15 @@ void mod_overview(const Fonts& fonts, ModsPanel& panel, const thunderstore::Inst ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "Outdated: this mod does not load. It was made for another version of Skate (%s). Get an updated " "version, or rebuild it with the latest ReSkate Studio.", mod.outdated.c_str()); - if (const auto missing = panel.list.excluded.find(mod.name); missing != panel.list.excluded.end()) { + if (const auto other = conflict_winner(panel, mod.name); !other.empty()) { + const auto winner = title_of(panel, other); + ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::warning), + "Not loaded: this mod changes the same parts of the game as %s, which is higher in the load order, in a " + "way the two cannot share, so only %s loads. To use this mod instead, move it above %s, or switch one " + "of them off.", winner.c_str(), winner.c_str(), winner.c_str()); + if (const auto& problems = panel.list.excluded.at(mod.name); !problems.empty()) + ImGui::TextDisabled("%s", problems.front().c_str()); + } else if (const auto missing = panel.list.excluded.find(mod.name); missing != panel.list.excluded.end()) { ImGui::TextColored(ImGui::ColorConvertU32ToFloat4(color::danger), "Not loaded: the game could not merge this mod cleanly, so none of it is used. Reinstall the whole " "mod folder, or rebuild it with a current ReSkate Studio."); @@ -943,16 +995,25 @@ void start_install(ModsPanel& panel, const fs::path& source, bool replace) { void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ui, ModsPanel& panel, const std::vector& problems) { const bool one = problems.size() == 1; + // A conflict is two mods changing the same game files: only the one higher in the load order loads. + const bool conflicts = std::all_of(problems.begin(), problems.end(), [](const ModProblem& problem) { return problem.conflict; }); const float rows = static_cast(problems.size()) * S(52); const auto frame = begin_panel("##mods_broken_panel", size, ImVec2(S(620), std::min(size.y - S(80), S(300) + rows))); - panel_title(fonts, one ? "A MOD COULD NOT BE MERGED" : "MODS COULD NOT BE MERGED"); + panel_title(fonts, conflicts ? (one ? "MOD CONFLICT" : "MOD CONFLICTS") : one ? "A MOD COULD NOT BE MERGED" : "MODS COULD NOT BE MERGED"); ImGui::PushTextWrapPos(0); - ImGui::TextDisabled(one - ? "This mod cannot be combined with the game's files, so none of its content would load. Skate would " - "start without it and never say why." - : "These mods cannot be combined with the game's files, so none of their content would load. Skate would " - "start without them and never say why."); + if (conflicts) + ImGui::TextDisabled(one + ? "This mod changes the same parts of the game as a mod above it in your load order, in a way the two " + "cannot share. Only the higher one loads, so this one is left out." + : "These mods change the same parts of the game as mods above them in your load order, in ways they " + "cannot share. Only the higher ones load, so these are left out."); + else + ImGui::TextDisabled(one + ? "This mod cannot be combined with the game's files, so none of its content would load. Skate would " + "start without it and never say why." + : "These mods cannot be combined with the game's files, so none of their content would load. Skate would " + "start without them and never say why."); ImGui::PopTextWrapPos(); ImGui::Spacing(); @@ -961,7 +1022,7 @@ void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ImGuiChildFlags_Borders | ImGuiChildFlags_NavFlattened); for (const auto& problem : problems) { ImGui::PushFont(fonts.bold); - ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(color::danger)); + ImGui::PushStyleColor(ImGuiCol_Text, ImGui::ColorConvertU32ToFloat4(problem.conflict ? color::warning : color::danger)); ImGui::TextUnformatted(problem.title.c_str()); ImGui::PopStyleColor(); ImGui::PopFont(); @@ -973,7 +1034,9 @@ void mods_broken_window(Launcher& launcher, const Fonts& fonts, ImVec2 size, Ui& ImGui::EndChild(); ImGui::Spacing(); ImGui::PushFont(fonts.caption); - ImGui::TextDisabled("Reinstall the whole mod folder, or rebuild it with a current ReSkate Studio."); + ImGui::TextDisabled("%s", conflicts + ? "To use a left-out mod instead, move it above the other in the Mod Manager, or switch one of them off." + : "Reinstall the whole mod folder, or rebuild it with a current ReSkate Studio."); ImGui::PopFont(); ImGui::SetCursorPosY(frame.y - S(24) - ImGui::GetFrameHeight()); diff --git a/cmake/Core.cmake b/cmake/Core.cmake index 7615b836..1d8df8e7 100644 --- a/cmake/Core.cmake +++ b/cmake/Core.cmake @@ -102,6 +102,7 @@ if(WIN32) Engine/Vfs/mod_merge_cas_store.cpp Engine/Vfs/mod_merge_combine.cpp Engine/Vfs/mod_merge_overrides.cpp + Engine/Vfs/mod_merge_conflicts.cpp Engine/Vfs/mod_merge_textures.cpp Engine/Vfs/mod_merge_grid.cpp Engine/Vfs/mod_merge_load_screens.cpp