diff --git a/Engine/Resource/ebx_merge.cpp b/Engine/Resource/ebx_merge.cpp index da307d4..224698a 100644 --- a/Engine/Resource/ebx_merge.cpp +++ b/Engine/Resource/ebx_merge.cpp @@ -5,12 +5,16 @@ #include #include +#include #include #include #include +#include #include #include +#include #include +#include namespace dingosdk::frostbite::ebx { namespace { @@ -81,6 +85,170 @@ const Value::Array* root_array(const Document& document, std::string_view name) return nullptr; } +// A type's name, which is what two documents that number their types +// independently have in common. Empty for a descriptor that names none. +std::string type_or_none(const Document& document, std::int32_t descriptor) { + return descriptor >= 0 && static_cast(descriptor) < document.types.size() + ? document.types[static_cast(descriptor)].name : std::string{}; +} + +constexpr auto unmatched = static_cast(-1); + +// Which of an edit's instances each of the base's is: the root is the root, an +// exported one is found by its guid, and an internal one, which has nothing +// else to go by, by its place among the internal ones. An edit that added or +// dropped an internal instance has moved the rest along, so then none of them +// is matched. Two that are not of one type are not a match either. +std::vector match_instances(const Document& base, const Document& edit) { + std::vector match(base.instances.size(), unmatched); + if (base.instances.empty() || edit.instances.empty()) return match; + match[0] = 0; + std::map exported; + std::vector internal, baseInternal; + for (std::size_t index = 1; index < edit.instances.size(); ++index) { + if (edit.instances[index].exported) exported.emplace(edit.instances[index].instanceGuid, index); + else internal.push_back(index); + } + for (std::size_t index = 1; index < base.instances.size(); ++index) { + if (!base.instances[index].exported) { baseInternal.push_back(index); continue; } + if (const auto found = exported.find(base.instances[index].instanceGuid); found != exported.end()) + match[index] = found->second; + } + if (internal.size() == baseInternal.size()) + for (std::size_t at = 0; at < internal.size(); ++at) match[baseInternal[at]] = internal[at]; + for (std::size_t index = 0; index < match.size(); ++index) + if (match[index] != unmatched && type_or_none(base, base.instances[index].descriptor) != + type_or_none(edit, edit.instances[match[index]].descriptor)) + match[index] = unmatched; + return match; +} + +bool same(double left, double right) { return std::bit_cast(left) == std::bit_cast(right); } +bool same(const ResourceReference& left, const ResourceReference& right) { return left.id == right.id; } +template bool same(const Type& left, const Type& right) { return left == right; } + +// Finds what one edit changed in place. The base and the edit are walked side +// by side, along with the result: it began as a copy of the base, so it has the +// same shape, and may already hold what an earlier edit changed. Nothing is +// written until the whole edit has been seen, so an edit either has all its +// values taken or none of them. +class InPlace final { +public: + InPlace(const Document& base, const Document& edit, const std::vector& match) + : base_(base), edit_(edit), match_(match) {} + + void fields(const Object& base, const Object& edit, Object& result, bool root) { + if (base.fields.size() != edit.fields.size() || base.fields.size() != result.fields.size()) { uncarried = true; return; } + for (std::size_t index = 0; index < base.fields.size(); ++index) { + if (base.fields[index].name != edit.fields[index].name) { uncarried = true; return; } + value(base.fields[index].value, edit.fields[index].value, result.fields[index].value, root); + } + } + + void apply() { + for (const auto& [to, from] : changes_) to->data = from->data; + } + + std::size_t values{}; // values the edit changed from the base + std::size_t contested{}; // of those, ones the result already had changed to something else + bool uncarried{}; // a change was seen that cannot be taken + bool unread{}; // somewhere a change could not be looked for + +private: + template bool plain(const Value& base, const Value& edit, Value& result) { + const auto* before = std::get_if(&base.data); + if (!before) return false; + const auto& after = std::get(edit.data); + if (same(*before, after)) return true; + ++values; + const auto& current = std::get(result.data); + if (same(current, after)) return true; + if (!same(current, *before)) ++contested; + changes_.emplace_back(&result, &edit); + return true; + } + + void value(const Value& base, const Value& edit, Value& result, bool rootList) { + if (base.data.index() != edit.data.index() || base.data.index() != result.data.index()) { uncarried = true; return; } + if (std::holds_alternative(base.data)) return; + if (plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result) || + plain(base, edit, result) || plain(base, edit, result)) return; + if (const auto* before = std::get_if(&base.data)) { + reference(*before, std::get(edit.data)); + return; + } + if (const auto* before = std::get_if>(&base.data)) { + const auto& after = std::get>(edit.data); + const auto& mine = std::get>(result.data); + if (!*before && !after) return; + if (!*before || !after || !mine) { uncarried = true; return; } + fields(**before, *after, *mine, false); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + auto& mine = std::get(result.data); + // Only a list on the root grows at its end, and what was appended + // comes across on its own. A list that shrank, or one deeper in + // that changed length, has entries no longer told apart by place. + if (rootList ? after.size() < before->size() : after.size() != before->size()) { uncarried = true; return; } + if (mine.size() < before->size()) { uncarried = true; return; } + for (std::size_t index = 0; index < before->size(); ++index) + value((*before)[index], after[index], mine[index], false); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + if (before->primitive != after.primitive) uncarried = true; + else if (before->primitive) uncarried |= before->primitiveType != after.primitiveType; + else uncarried |= type_or_none(base_, before->descriptor) != type_or_none(edit_, after.descriptor); + return; + } + if (const auto* before = std::get_if(&base.data)) { + const auto& after = std::get(edit.data); + // What a boxed value holds is not read, so a change inside one is not seen. + if ((before->encodedType == 0) != (after.encodedType == 0)) uncarried = true; + else if (before->encodedType != 0) unread = true; + } + } + + // A reference still has to lead to the same place; one pointed elsewhere + // cannot be taken, because what it points at may not be in the result. + void reference(const PointerReference& before, const PointerReference& after) { + if (before.kind != after.kind) { uncarried = true; return; } + if (before.kind == PointerKind::null) return; + if (before.index < 0 || after.index < 0) { uncarried |= before.index != after.index; return; } + const auto from = static_cast(before.index), to = static_cast(after.index); + if (before.kind == PointerKind::external) { + uncarried |= from >= base_.imports.size() || to >= edit_.imports.size() || + base_.imports[from].fileGuid != edit_.imports[to].fileGuid || + base_.imports[from].classGuid != edit_.imports[to].classGuid; + return; + } + if (from >= match_.size()) { uncarried = true; return; } + // An internal instance that could not be matched is not known to have moved. + if (match_[from] == unmatched) (base_.instances[from].exported ? uncarried : unread) = true; + else uncarried |= match_[from] != to; + } + + const Document& base_; + const Document& edit_; + const std::vector& match_; + std::vector> changes_; +}; + +template void each_pointer(Value& value, const Visit& visit) { + if (auto* pointer = std::get_if(&value.data)) { visit(*pointer); return; } + if (auto* nested = std::get_if>(&value.data)) { + if (*nested) for (auto& field : (*nested)->fields) each_pointer(field.value, visit); + return; + } + if (auto* list = std::get_if(&value.data)) + for (auto& element : *list) each_pointer(element, visit); +} + } // namespace namespace detail { @@ -302,12 +470,44 @@ Document merge_documents(const Document& base, const std::spansize()); MergeSummary totals; + // The references the edits so far appended to each root array. + std::map>, std::less<>> appended; for (std::size_t index = 0; index < edits.size(); ++index) { const auto* edit = edits[index]; if (!edit) continue; const auto* editRoot = edit->root(); if (!editRoot || !editRoot->object) throw std::runtime_error("EBX edit has no root object"); + + // What the edit changed in place. The result's first instances are still + // the base's, in the base's order: carried ones are added behind them, + // and nothing is sorted until every edit is in. + const auto match = match_instances(base, *edit); + InPlace changes(base, *edit, match); + for (std::size_t at = 0; at < base.instances.size(); ++at) { + const auto& before = base.instances[at]; + if (match[at] == unmatched) { + // An exported instance is found whenever the edit still has it. + (before.exported ? changes.uncarried : changes.unread) = true; + continue; + } + const auto& after = edit->instances[match[at]]; + if (!before.object || !after.object || !result.instances[at].object) { + changes.unread |= before.object || after.object; + continue; + } + changes.fields(*before.object, *after.object, *result.instances[at].object, at == 0); + } + if (changes.uncarried) { + totals.uncarried.push_back(index); + } else { + changes.apply(); + totals.values += changes.values; + totals.contested += changes.contested; + } + if (changes.unread) totals.unread.push_back(index); + detail::Carrier mapper(*edit, result); + std::map>, std::less<>> mine; // Whatever an edit appended to one of the root's arrays comes across, // pulling in the instances those entries point at. @@ -331,10 +531,24 @@ Document merge_documents(const Document& base, const std::spanpush_back(mapper.value(entry)); + auto carried = mapper.value(entry); + // Two edits that append the same thing mean it once. What one edit + // lists twice on its own is left as its author wrote it. + if (const auto* reference = std::get_if(&carried.data); + reference && reference->kind != PointerKind::null) { + const std::pair key{reference->kind, reference->index}; + if (const auto earlier = appended.find(field.name); + earlier != appended.end() && earlier->second.contains(key)) { + ++totals.repeated; + continue; + } + mine[field.name].insert(key); + } + target->push_back(std::move(carried)); ++totals.arrayEntries; } } + for (auto& [name, references] : mine) appended[name].merge(references); // An instance an edit added but never linked from the root still has to // exist, or the asset it belongs to goes missing. @@ -357,4 +571,51 @@ Document merge_documents(const Document& base, const std::span(document.root()); + if (!keep || !root || !root->object) return 0; + std::size_t dropped{}; + for (auto& field : root->object->fields) { + auto* array = std::get_if(&field.value.data); + if (!array) continue; + dropped += std::erase_if(*array, [&](const Value& entry) { + const auto* pointer = std::get_if(&entry.data); + return pointer && pointer->kind == PointerKind::external && pointer->index >= 0 && + static_cast(pointer->index) < document.imports.size() && + !keep(document.imports[static_cast(pointer->index)]); + }); + } + // A boxed value is kept as the bytes it was read as, import numbers and + // all, so a document with one keeps its imports where they are. + if (!dropped || !document.boxedValues.empty()) return dropped; + + std::vector used(document.imports.size()); + for (auto& instance : document.instances) + if (instance.object) + for (auto& field : instance.object->fields) + each_pointer(field.value, [&](const PointerReference& pointer) { + if (pointer.kind == PointerKind::external && pointer.index >= 0 && + static_cast(pointer.index) < used.size()) + used[static_cast(pointer.index)] = true; + }); + std::vector moveTo(used.size(), -1); + std::vector kept; + for (std::size_t index = 0; index < used.size(); ++index) { + if (!used[index]) continue; + moveTo[index] = static_cast(kept.size()); + kept.push_back(document.imports[index]); + } + if (kept.size() == document.imports.size()) return dropped; + for (auto& instance : document.instances) + if (instance.object) + for (auto& field : instance.object->fields) + each_pointer(field.value, [&](PointerReference& pointer) { + if (pointer.kind == PointerKind::external && pointer.index >= 0 && + static_cast(pointer.index) < moveTo.size()) + pointer.index = moveTo[static_cast(pointer.index)]; + }); + document.imports = std::move(kept); + return dropped; +} + } // namespace dingosdk::frostbite::ebx diff --git a/Engine/Resource/ebx_merge.h b/Engine/Resource/ebx_merge.h index 5440e76..294ed25 100644 --- a/Engine/Resource/ebx_merge.h +++ b/Engine/Resource/ebx_merge.h @@ -19,6 +19,18 @@ struct MergeSummary { ImportReference target; }; std::vector dropped; + std::size_t repeated{}; // appended references an earlier entry had already added, left out + std::size_t values{}; // plain values an edit changed in place, taken from it + std::size_t contested{}; // of those, values an earlier edit had changed to something else + // Edits (indices into `edits`) with a change that did not come across: a reference + // pointed somewhere else, a list resized anywhere but at the root's end, an instance gone. + std::vector uncarried; + // Edits whose in-place changes could not all be looked for, because their internal + // instances do not line up with the base's or the document holds boxed values. + std::vector unread; + + // Every edit's in-place changes are in the result, as far as anyone can tell. + [[nodiscard]] bool exact() const noexcept { return uncarried.empty() && unread.empty(); } }; // Whether a root array entry an edit appended comes across, asked for an entry @@ -27,13 +39,20 @@ struct MergeSummary { using CarryImport = std::function; // Combines several independently edited copies of one EBX asset with the base -// they were all derived from. Each edit contributes the instances it added and -// the entries it appended to the root's arrays; anything an edit merely changed -// in place stays as the base had it, because two edits to one value cannot both -// be honoured. Edits are applied in the order given, so the last one wins a tie. +// they were all derived from. Each edit contributes the instances it added, the +// entries it appended to the root's arrays (a reference two edits both append is +// listed once), and the plain values it changed in place: a number, a string, a +// guid. Two edits to one value cannot both be honoured, so edits are applied in +// the order given and the last one wins. A change of any other shape (a +// reference pointed elsewhere, a list resized in the middle of the document) +// stays as the base had it, and the summary names the edit it was in. [[nodiscard]] Document merge_documents(const Document& base, std::span edits, MergeSummary* summary = nullptr, const CarryImport& carry = {}); +// Takes out the root array entries that refer to a document `keep` turns down, +// and the imports nothing refers to afterwards. Returns how many entries went. +std::size_t drop_root_references(Document& document, const CarryImport& keep); + } // namespace dingosdk::frostbite::ebx diff --git a/Engine/Vfs/Test/ebx_merge_tests.cpp b/Engine/Vfs/Test/ebx_merge_tests.cpp new file mode 100644 index 0000000..a821183 --- /dev/null +++ b/Engine/Vfs/Test/ebx_merge_tests.cpp @@ -0,0 +1,280 @@ +// The document merge, on documents made in memory, so it needs no game files. +// +// Several mods each ship their own copy of one of the game's assets. merge_documents puts the +// copies together over the game's: what each added, and what each changed in place. The cases: +// * values two edits changed in place both arrive, and one both changed is the later edit's; +// * an edit that also changed something that cannot be taken (a reference pointed elsewhere, a +// list resized) has none of its values taken, and the summary names it; +// * an exported instance is found by its guid wherever the edit keeps it, and internal +// instances that do not line up are reported as not looked at; +// * a reference two edits both append is listed once, and one edit's own repeat is left alone; +// * drop_root_references takes entries out of the root's lists with the imports they used. +#include "Engine/Resource/ebx_carry.h" +#include "Engine/Resource/ebx_merge.h" + +#include +#include +#include +#include +#include + +namespace ebx = dingosdk::frostbite::ebx; +using dingosdk::frostbite::Guid; + +namespace { +int failures = 0; +void expect(bool ok, const std::string& what) { + if (ok) return; + std::cerr << "FAIL: " << what << '\n'; + ++failures; +} + +Guid guid(std::uint8_t seed) { + Guid value; + value.bytes.fill(static_cast(seed)); + return value; +} + +template ebx::Value value_of(Type data) { + ebx::Value value; + value.data = std::move(data); + return value; +} +ebx::Value number(std::int64_t value) { return value_of(value); } +ebx::Value real(double value) { return value_of(value); } +ebx::Value text(std::string value) { return value_of(std::move(value)); } +// A reference to another document, by the place of its import, and one to an instance of this one. +ebx::Value outside(std::int32_t import) { return value_of(ebx::PointerReference{ebx::PointerKind::external, import}); } +ebx::Value inside(std::int32_t instance) { return value_of(ebx::PointerReference{ebx::PointerKind::internal, instance}); } +ebx::Value list(ebx::Value::Array entries) { return value_of(std::move(entries)); } + +std::shared_ptr object(std::int32_t type, std::vector> fields) { + auto result = std::make_shared(); + result->descriptor = type; + for (auto& [name, value] : fields) result->fields.push_back({result->fields.size(), std::move(name), std::move(value)}); + return result; +} +ebx::InstanceRecord instance(std::int32_t type, std::shared_ptr fields, const Guid* exported = nullptr) { + ebx::InstanceRecord record; + record.descriptor = type; + record.exported = exported != nullptr; + if (exported) record.instanceGuid = *exported; + record.object = std::move(fields); + return record; +} + +constexpr std::int32_t list_type = 0, entry_type = 1, detail_type = 2; +const Guid entry_guid = guid(0xA1); + +// The game's asset: a root with plain values, a list of two other documents and a nested +// structure; an exported instance the root points at; and an internal one that one points at. +ebx::Document game() { + ebx::Document document; + document.fileGuid = guid(1); + document.rootType = "ListAsset"; + document.types.resize(3); + document.types[list_type].name = "ListAsset"; + document.types[entry_type].name = "Entry"; + document.types[detail_type].name = "Detail"; + document.imports = {{guid(0x10), guid(0xC0)}, {guid(0x11), guid(0xC0)}}; + document.instances.push_back(instance(list_type, object(list_type, { + {"Volume", number(5)}, {"Gain", real(1.0)}, {"Title", text("game")}, + {"Items", list({outside(0), outside(1)})}, {"First", inside(1)}, + {"Tuning", value_of(object(detail_type, {{"Weights", list({number(1), number(2)})}, {"Scale", real(2.0)}}))}}))); + document.instances.push_back(instance(entry_type, object(entry_type, {{"Cost", number(10)}, {"Detail", inside(2)}}), &entry_guid)); + document.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(3)}}))); + return document; +} +ebx::Document copy(const ebx::Document& document) { return ebx::detail::clone_document(document); } + +ebx::Value& field(ebx::Object& from, std::string_view name) { + for (auto& entry : from.fields) + if (entry.name == name) return entry.value; + throw std::runtime_error("no field " + std::string(name)); +} +ebx::Value& field(ebx::Document& document, std::size_t at, std::string_view name) { return field(*document.instances.at(at).object, name); } +template const Type& read(const ebx::Document& document, std::size_t at, std::string_view name) { + return std::get(field(*document.instances.at(at).object, name).data); +} +ebx::Object& tuning(ebx::Document& document) { return *std::get>(field(document, 0, "Tuning").data); } +ebx::Value::Array& items(ebx::Document& document) { return std::get(field(document, 0, "Items").data); } +// The documents a list names, in order. +std::vector named(const ebx::Document& document) { + std::vector result; + for (const auto& entry : std::get(field(*document.instances.front().object, "Items").data)) + result.push_back(document.imports.at(static_cast(std::get(entry.data).index)).fileGuid); + return result; +} +void append(ebx::Document& document, const Guid& target) { + document.imports.push_back({target, guid(0xC0)}); + items(document).push_back(outside(static_cast(document.imports.size() - 1))); +} +// Where the exported instance with this guid is. +std::size_t find(const ebx::Document& document, const Guid& wanted) { + for (std::size_t index = 0; index < document.instances.size(); ++index) + if (document.instances[index].exported && document.instances[index].instanceGuid == wanted) return index; + throw std::runtime_error("no such exported instance"); +} + +ebx::Document merge(const ebx::Document& base, std::initializer_list edits, ebx::MergeSummary& summary) { + const std::vector documents(edits); + return ebx::merge_documents(base, documents, &summary); +} + +void values_from_every_edit() { + const auto base = game(); + auto first = copy(base), second = copy(base); + field(first, 0, "Volume") = number(9); + field(first, 0, "Title") = text("first"); + field(second, 0, "Gain") = real(0.5); + field(tuning(second), "Scale") = real(4.0); + std::get(field(tuning(second), "Weights").data)[1] = number(7); + field(second, 1, "Cost") = number(20); + field(second, 2, "Depth") = number(8); + ebx::MergeSummary summary; + auto merged = merge(base, {&first, &second}, summary); + expect(read(merged, 0, "Volume") == 9 && read(merged, 0, "Title") == "first", + "values: the first edit's values are in the result"); + expect(read(merged, 0, "Gain") == 0.5 && std::get(field(tuning(merged), "Scale").data) == 4.0 && + std::get(std::get(field(tuning(merged), "Weights").data)[1].data) == 7, + "values: the second edit's values are in the result, the nested ones too"); + expect(read(merged, 1, "Cost") == 20 && read(merged, 2, "Depth") == 8, + "values: values in an exported and in an internal instance are in the result"); + expect(summary.values == 7 && summary.contested == 0 && summary.exact() && !summary.instances && !summary.arrayEntries, + "values: seven values taken, none disagreed, nothing added (" + std::to_string(summary.values) + " taken)"); + expect(named(merged) == named(base) && merged.instances.size() == base.instances.size(), "values: nothing else moved"); +} + +void one_value_from_two_edits(bool same) { + const std::string label = same ? "same value twice" : "contested value"; + const auto base = game(); + auto lower = copy(base), higher = copy(base); + field(lower, 0, "Volume") = number(9); + field(higher, 0, "Volume") = number(same ? 9 : 11); + ebx::MergeSummary summary; + const auto merged = merge(base, {&lower, &higher}, summary); + expect(read(merged, 0, "Volume") == (same ? 9 : 11), label + ": the later edit's value is the one kept"); + expect(summary.values == 2 && summary.contested == (same ? 0u : 1u) && summary.exact(), + label + ": " + (same ? "no" : "one") + " disagreement is counted"); +} + +// What cannot be taken: the edit that made such a change keeps all its values out, so the result +// never has half of what a mod did to the game's own content. +void a_change_that_cannot_be_taken(int kind) { + static constexpr std::array labels{"retargeted reference", "shorter root list", "resized nested list", "instance gone"}; + const std::string label = labels[static_cast(kind)]; + const auto base = game(); + auto reshaped = copy(base), plain = copy(base); + field(reshaped, 0, "Volume") = number(9); + if (kind == 0) items(reshaped)[0] = outside(1); + else if (kind == 1) items(reshaped).pop_back(); + else if (kind == 2) std::get(field(tuning(reshaped), "Weights").data).push_back(number(3)); + else { + // The exported instance is not in the edit: the root points at the internal one instead. + reshaped.instances.erase(reshaped.instances.begin() + 1); + field(reshaped, 0, "First") = inside(1); + } + field(plain, 0, "Title") = text("plain"); + ebx::MergeSummary summary; + const auto merged = merge(base, {&reshaped, &plain}, summary); + expect(read(merged, 0, "Volume") == 5, label + ": none of that edit's values are taken"); + expect(read(merged, 0, "Title") == "plain", label + ": the other edit's value is"); + expect(summary.uncarried == std::vector{0} && !summary.exact() && summary.values == 1, + label + ": the summary names the edit"); + expect(named(merged) == named(base), label + ": the game's list is as it was"); +} + +// An edit that added an exported instance has moved the game's along; they are still found. +void instances_found_by_guid() { + const auto base = game(); + auto edit = copy(base); + const auto added = guid(0x05); // sorts ahead of the game's + edit.instances.insert(edit.instances.begin() + 1, + instance(entry_type, object(entry_type, {{"Cost", number(1)}, {"Detail", value_of(ebx::PointerReference{})}}), &added)); + field(edit, 0, "First") = inside(2); + field(edit, 2, "Detail") = inside(3); + field(edit, 2, "Cost") = number(30); + ebx::MergeSummary summary; + const auto merged = merge(base, {&edit}, summary); + expect(summary.uncarried.empty() && summary.unread.empty() && summary.values == 1 && summary.instances == 1, + "by guid: one value taken and one instance added, nothing left behind"); + expect(read(merged, find(merged, entry_guid), "Cost") == 30, "by guid: the game's instance has the edit's value"); + expect(read(merged, find(merged, added), "Cost") == 1, "by guid: the added instance came across"); + expect(find(merged, added) < find(merged, entry_guid), "by guid: exported instances are in guid order"); + expect(static_cast(read(merged, 0, "First").index) == find(merged, entry_guid), + "by guid: the root still points at the game's instance"); +} + +// An internal instance has no guid. With one more of them in the edit the places say nothing, +// so they are not compared, and the summary says the edit was not looked at in full. +void internal_instances_that_do_not_line_up() { + const auto base = game(); + auto edit = copy(base); + edit.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(4)}}))); + field(edit, 0, "Volume") = number(9); + ebx::MergeSummary summary; + const auto merged = merge(base, {&edit}, summary); + expect(read(merged, 0, "Volume") == 9, "internal: the value on the root is taken"); + expect(summary.uncarried.empty() && summary.unread == std::vector{0} && !summary.exact(), + "internal: the edit is reported as not looked at in full"); +} + +void appended_references() { + const auto base = game(); + auto first = copy(base), second = copy(base), third = copy(base); + append(first, guid(0x20)); + append(first, guid(0x21)); + append(second, guid(0x21)); // the first edit's as well + append(second, guid(0x22)); + append(third, guid(0x23)); // twice in one edit: its author's doing + append(third, guid(0x23)); + ebx::MergeSummary summary; + const auto merged = merge(base, {&first, &second, &third}, summary); + const std::vector expected{guid(0x10), guid(0x11), guid(0x20), guid(0x21), guid(0x22), guid(0x23), guid(0x23)}; + expect(named(merged) == expected, "appended: each reference once, in the order of the edits, one edit's own repeat kept (" + + std::to_string(named(merged).size()) + " entries)"); + expect(summary.arrayEntries == 5 && summary.repeated == 1 && summary.exact(), "appended: five entries taken, one repeat left out"); +} + +void dropped_references() { + auto document = game(); + append(document, guid(0x20)); + append(document, guid(0x21)); + const auto gone = ebx::drop_root_references(document, [](const ebx::ImportReference& reference) { + return reference.fileGuid != guid(0x11) && reference.fileGuid != guid(0x20); }); + expect(gone == 2, "dropped: two entries went (" + std::to_string(gone) + ")"); + expect((named(document) == std::vector{guid(0x10), guid(0x21)}), "dropped: the others are left, in order"); + expect(document.imports.size() == 2, "dropped: the imports nothing uses went with them (" + + std::to_string(document.imports.size()) + " left)"); + + // A boxed value is kept as bytes with its import numbers in them, so the imports stay put. + auto boxed = game(); + boxed.boxedValues.emplace_back(); + expect(ebx::drop_root_references(boxed, [](const ebx::ImportReference& reference) { return reference.fileGuid != guid(0x10); }) == 1 && + boxed.imports.size() == 2 && named(boxed) == std::vector{guid(0x11)}, + "dropped: a document with a boxed value keeps its imports where they are"); + auto untouched = game(); + expect(ebx::drop_root_references(untouched, [](const ebx::ImportReference&) { return true; }) == 0 && + untouched.imports.size() == 2, "dropped: nothing goes when every entry is kept"); +} +} // namespace + +int main() try { + values_from_every_edit(); + one_value_from_two_edits(false); + one_value_from_two_edits(true); + for (int kind = 0; kind < 4; ++kind) a_change_that_cannot_be_taken(kind); + instances_found_by_guid(); + internal_instances_that_do_not_line_up(); + appended_references(); + dropped_references(); + if (failures) { + std::cerr << failures << " failure(s)\n"; + return 1; + } + std::cout << "ebx merge: ok\n"; + return 0; +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/Test/item_list_merge_tests.cpp b/Engine/Vfs/Test/item_list_merge_tests.cpp index 4d5317b..41d6209 100644 --- a/Engine/Vfs/Test/item_list_merge_tests.cpp +++ b/Engine/Vfs/Test/item_list_merge_tests.cpp @@ -11,6 +11,12 @@ // one mod ships is not merged at all, it passes through as that mod built it, so every case // here has two mods shipping the shared bundle.) // +// The same mods also show what else the merge does with two copies of one of the game's assets: +// * an item two mods both list is in the merged list once; +// * values two mods changed in place in one asset are both in the merged asset, and where that +// cannot be done the merge keeps one mod's copy whole and says whose changes are missing; +// * the list other mods' copies of the bundle are given leaves out an item that is not in them. +// // The mods are made here from the installed game's own list and one of its items. // Arguments: []; skipped when the game is not there. With a second // argument the two mods of the first case are also left in that folder, stamped for this game, @@ -28,7 +34,9 @@ #include #include +#include #include +#include #include namespace fs = std::filesystem; @@ -140,6 +148,8 @@ struct Game { std::string listName; fb::ebx::Document list; std::size_t listed{}; + std::size_t itemIndex{}; + std::string itemName; fb::ebx::Document item; }; Game read_game(const fs::path& root, const CasStore& store) { @@ -162,6 +172,8 @@ Game read_game(const fs::path& root, const CasStore& store) { game.list = std::move(document); } else if (!item && type.find("itemasset") != std::string::npos) { item = true; + game.itemIndex = index; + game.itemName = assets[index].name; game.item = std::move(document); } } @@ -172,8 +184,9 @@ Game read_game(const fs::path& root, const CasStore& store) { // A cosmetic mod's folder: its copy of the shared bundle in items.toc (the game's assets, the // item list replaced and its items added) and one archive holding the manifest and those payloads. // With no list the mod leaves the game's as it is, as a mod does whose items go in another one. +// `changed` replaces assets of the game's, by their place in the bundle. void write_mod(const fs::path& directory, const CasStore& store, const Game& game, const Bytes* list, - std::initializer_list items) { + std::initializer_list items, const std::map& changed = {}) { auto manifest = game.shared.listing.manifest; const auto& shipped = game.shared.listing.files; std::vector files(shipped.begin() + static_cast(game.shared.listing.first), shipped.end()); @@ -191,6 +204,11 @@ void write_mod(const fs::path& directory, const CasStore& store, const Game& gam manifest.ebx[game.listIndex].originalSize = list->size(); files[game.listIndex] = stored(encoded(*list)); } + for (const auto& [index, payload] : changed) { + manifest.ebx[index].sha1 = sha1_of(payload); + manifest.ebx[index].originalSize = payload.size(); + files[index] = stored(encoded(payload)); + } for (const auto& item : items) { fb::BundleAsset asset; asset.kind = fb::AssetKind::ebx; @@ -365,6 +383,167 @@ void one_mod_changes_the_list(const Game& game, const CasStore& store, bool miss expect(said == missing, label + (missing ? ": the merge says which item was left out\n" : ": nothing is reported\n") + describe(report)); } + +// Two packs that both carry one item, each with an item of its own besides. Both list the shared +// one, and it is one item: the merged list has it once. +void an_item_two_mods_both_list(const Game& game, const CasStore& store) { + Merge merge(game, "listed-twice"); + const auto both = item_from(game.item, "items/reskate_test/own_in_both", 0x10); + const auto first = item_from(game.item, "items/reskate_test/own_first", 0x20); + const auto second = item_from(game.item, "items/reskate_test/own_second", 0x30); + const auto firstList = list_with(game.list, {both.identity, first.identity}); + const auto secondList = list_with(game.list, {both.identity, second.identity}); + write_mod(merge.add("first"), store, game, &firstList, {both, first}); + write_mod(merge.add("second"), store, game, &secondList, {both, second}); + const auto report = mods::merge_mods(merge.catalog); + const auto list = merged_list(merge, game, report, "listed by both", 3); + expect(names(list, both) && names(list, first) && names(list, second), "listed by both: the list has all three items"); + expect(std::ranges::count(list, both.identity.fileGuid) == 1, "listed by both: the item both mods list is in it once"); + expect(noted(report, game.listName + ": combined 2 edits (0 instance(s), 3 list entries, 1 repeated entry(ies) listed once)"), + "listed by both: the merge says one entry was a repeat\n" + describe(report)); +} + +// The game's item as a mod that retunes it ships it: the same document, changed in place. +Bytes item_with(const fb::ebx::Document& game, const std::function& change) { + auto item = fb::ebx::detail::clone_document(game); + change(*item.instances.front().object); + return fb::ebx::write_document(item); +} +fb::ebx::Value& field(fb::ebx::Object& object, std::string_view name) { + for (auto& entry : object.fields) + if (entry.name == name) return entry.value; + throw std::runtime_error("the game's item has no " + std::string(name)); +} +void set_hash(fb::ebx::Object& object, std::uint32_t hash) { + auto& value = field(object, "HashedAssetKey"); + if (std::holds_alternative(value.data)) value.data = static_cast(hash); + else value.data = static_cast(hash); +} +std::uint32_t hash_of(const fb::ebx::Object& object) { + const auto& value = field(const_cast(object), "HashedAssetKey"); + const auto* number = std::get_if(&value.data); + return number ? static_cast(*number) : static_cast(std::get(value.data)); +} +// A change the merge cannot take value by value: the first list of the item made one shorter, +// or failing that its first reference cleared. False when the item has neither. +bool reshape(fb::ebx::Object& object) { + for (auto& entry : object.fields) + if (auto* list = std::get_if(&entry.value.data); list && !list->empty()) { + list->pop_back(); + return true; + } + for (auto& entry : object.fields) + if (auto* pointer = std::get_if(&entry.value.data); + pointer && pointer->kind != fb::ebx::PointerKind::null) { + *pointer = {}; + return true; + } + return false; +} + +// The merged shared bundle's copy of the game's item, and the sha1 the bundle lists it under. +std::pair merged_item(const Merge& merge, const Game& game) { + const auto output = merge.catalog.root / mods::generated_folder; + const auto layout = vfs::read_layout(output / L"layout.toc"); + const CasStore store(game.base, merge.root / L"unused", layout.root); + const auto shared = shared_in(store, output, game.base, game.root); + const auto& assets = shared.listing.manifest.ebx; + for (std::size_t index = 0; index < assets.size(); ++index) + if (assets[index].name == game.itemName) + return {fb::ebx::read_document(read_asset(store, output, game.base, shared.listing, index, game.root)), assets[index].sha1}; + throw std::runtime_error("the merged bundle has no " + game.itemName); +} + +// Two mods each ship the game's item with values of their own changed in place. Nothing was added +// to it, so the merge has no list to put together: it has the values. Each mod's are in the +// merged item, and where both changed one the higher mod's is. +void values_two_mods_changed(const Game& game, const CasStore& store, bool contested) { + const std::string label = contested ? "one value from two mods" : "values from two mods"; + Merge merge(game, contested ? "one-value" : "two-values"); + const auto upper = item_with(game.item, [&](fb::ebx::Object& root) { + if (contested) set_hash(root, 0x1111); + else field(root, "Key").data = std::string("reskate_upper"); + }); + const auto under = item_with(game.item, [&](fb::ebx::Object& root) { + set_hash(root, 0x2222); + if (contested) field(root, "Key").data = std::string("reskate_under"); + }); + write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}}); + write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}}); + const auto report = mods::merge_mods(merge.catalog); + expect(report.issue.empty() && report.built && report.problems.empty(), + label + ": the merge builds the patch and keeps every mod\n" + describe(report)); + if (!report.built) return; + const auto item = merged_item(merge, game).first; + auto& root = *item.instances.front().object; + expect(std::get(field(root, "Key").data) == (contested ? "reskate_under" : "reskate_upper"), + label + ": the merged item has the key one mod changed"); + expect(hash_of(root) == (contested ? 0x1111u : 0x2222u), + label + (contested ? ": the value both changed is the higher mod's" : ": and the value the other mod changed")); + expect(noted(report, game.itemName + ": combined 2 edits (0 instance(s), 0 list entries, " + + (contested ? "3 changed value(s), 1 disagreed)" : "2 changed value(s))")), + label + ": the merge says what it combined\n" + describe(report)); +} + +// One of the two also changed the item in a way that is not a value (a list made shorter). That +// cannot be put together with the other mod's values, so the higher mod's copy stays whole, as +// it always did, and the merge now says whose changes are not in the game. +void a_change_that_does_not_combine(const Game& game, const CasStore& store) { + Merge merge(game, "not-combined"); + bool reshaped{}; + const auto upper = item_with(game.item, [&](fb::ebx::Object& root) { + field(root, "Key").data = std::string("reskate_upper"); + reshaped = reshape(root); + }); + if (!reshaped) { + std::cout << "The game's item has no list or reference to change; that case is skipped.\n"; + return; + } + const auto under = item_with(game.item, [&](fb::ebx::Object& root) { set_hash(root, 0x2222); }); + write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}}); + write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}}); + const auto report = mods::merge_mods(merge.catalog); + expect(report.issue.empty() && report.built && report.problems.empty(), + "not combined: the merge builds the patch and keeps every mod\n" + describe(report)); + if (!report.built) return; + expect(merged_item(merge, game).second == sha1_of(upper), "not combined: the bundle has the higher mod's copy as that mod built it"); + expect(noted(report, game.itemName + ": kept upper's copy whole; the changes under made to it are of a kind that cannot " + "be combined with it and are not in the game"), + "not combined: the merge says whose changes are missing\n" + describe(report)); +} + +// What other mods' copies of the bundle are given (a map carries one for its levels): one list +// for all of them, put together from every mod's. Under a name two mods share, such a copy gets +// the higher mod's asset, so the lower mod's entry for its own has to be out of that list too. +void the_list_other_copies_are_given(const Game& game, const CasStore& store) { + Merge merge(game, "carried-list"); + const auto upper = item_from(game.item, same_name, 0x10); + const auto under = item_from(game.item, same_name, 0x20); + const auto upperList = list_with(game.list, {upper.identity}), underList = list_with(game.list, {under.identity}); + write_mod(merge.add("upper"), store, game, &upperList, {upper}); + write_mod(merge.add("under"), store, game, &underList, {under}); + std::vector order; + std::map files; + for (const auto& mod : merge.catalog.mods) { + order.push_back(&mod); + files.emplace(&mod, scan(mod.directory)); + } + mods::MergeReport report; + const auto overrides = collect_asset_overrides(order, files, store, game.base, game.root, report); + const auto versions = overrides.changed.find(lower(game.listName)); + expect(versions != overrides.changed.end() && versions->second.size() == 1, + "carried list: there is one changed list for other mods' copies\n" + describe(report)); + if (versions == overrides.changed.end() || versions->second.empty()) return; + const auto& change = versions->second.begin()->second; + const auto list = listed(fb::ebx::read_document(fb::decode_cas(change.encoded, {game.root}))); + expect(list.size() == game.listed + 1 && names(list, upper) && !names(list, under), + "carried list: it has the game's items and the higher mod's, not the one no copy is given (" + + std::to_string(list.size()) + " entries)"); + expect(change.names == std::set{upper.identity.fileGuid}, "carried list: the item that goes with it is the higher mod's"); + expect(noted(report, "under: " + game.listName + ": 1 entry(ies) left out of the copies other mods carry, where another " + "mod's asset of the same name stands in for what they name"), + "carried list: the merge says which mod's entry was left out\n" + describe(report)); +} } // namespace int main(int argc, char** argv) try { @@ -381,6 +560,11 @@ int main(int argc, char** argv) try { different_names_from_two_mods(game, store); one_mod_changes_the_list(game, store, false); one_mod_changes_the_list(game, store, true); + an_item_two_mods_both_list(game, store); + values_two_mods_changed(game, store, false); + values_two_mods_changed(game, store, true); + a_change_that_does_not_combine(game, store); + the_list_other_copies_are_given(game, store); if (failures) { std::cerr << failures << " failure(s)\n"; return 1; diff --git a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp index b80b681..2923651 100644 --- a/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp +++ b/Engine/Vfs/Test/mod_merge_added_assets_tests.cpp @@ -5,6 +5,7 @@ // The fixture is a small made-up game and two mods in a temp folder, written with the repo's // own writers, so it needs no game files: // game bundle "win32/test/shared": test/playlist, test/other +// bundle "win32/test/second": test/playlist again, test/extra // map a level mod: its own level superbundle (a new TOC) with a copy of that bundle, as a // custom map carries the core assets for its level; it edits nothing // music an asset mod that changes test/playlist to name a new song, and adds the song, the @@ -34,6 +35,7 @@ namespace { using Bytes = std::vector; constexpr std::uint32_t package = 0x1234; constexpr char bundle_name[] = "win32/test/shared"; +constexpr char second_bundle[] = "win32/test/second"; int failures = 0; void expect(bool ok, const std::string& what) { @@ -193,16 +195,26 @@ struct Fixture { catalog.data_root = root / "game"; catalog.root = catalog.data_root / "Mods"; catalog.present = true; - // The game: two assets in one bundle, stored in the game's own archive. + // The game: two assets in one bundle, and a second bundle that has the playlist as well + // (the game keeps a list in several bundles), stored in the game's own archive. const std::vector assets{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}}; + const std::vector second{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 0, ebx_document(guid(3), {})}}; Bytes archive; - std::vector files; - for (const auto& asset : assets) { - const auto payload = encoded(asset.payload); - files.push_back({{false, package, 1}, static_cast(archive.size()), static_cast(payload.size())}); - archive.insert(archive.end(), payload.begin(), payload.end()); - } - const std::vector bundles{{bundle_name, fb::write_bundle_region(files, fb::write_binary_bundle(manifest_of(assets, {}))), 1}}; + const auto region = [&](const std::vector& list) { + // The asset list is the region's first file, as it is in every bundle of the game: + // a bundle one mod alone ships then passes through as that mod built it. + const auto listing = fb::write_binary_bundle(manifest_of(list, {})); + std::vector files{{{false, package, 1}, static_cast(archive.size()), + static_cast(listing.size())}}; + archive.insert(archive.end(), listing.begin(), listing.end()); + for (const auto& asset : list) { + const auto payload = encoded(asset.payload); + files.push_back({{false, package, 1}, static_cast(archive.size()), static_cast(payload.size())}); + archive.insert(archive.end(), payload.begin(), payload.end()); + } + return fb::write_bundle_region(files); + }; + const std::vector bundles{{bundle_name, region(assets), 1}, {second_bundle, region(second), 1}}; write(catalog.data_root / "Data" / "layout.toc", layout_toc()); write(catalog.data_root / "Data" / "Win32" / "test_shared.toc", fb::write_patch_toc(bundles)); write(catalog.data_root / "Data" / "Win32" / "pkg" / "cas_01.cas", archive); @@ -212,9 +224,9 @@ struct Fixture { fs::remove_all(root, ignored); } - // A mod with its own copy of the bundle: the manifest first (raw), then every payload. + // A mod with its own copy of a bundle: the manifest first (raw), then every payload. void add(const std::string& name, bool levels, const std::string& toc, const std::vector& superbundles, - const std::vector& assets, const std::vector& resources = {}) { + const std::vector& assets, const std::vector& resources = {}, const char* bundle = bundle_name) { const auto listing = fb::write_binary_bundle(manifest_of(assets, resources)); Bytes archive(listing); std::vector files{{{true, package, 1}, 0, static_cast(listing.size())}}; @@ -225,7 +237,7 @@ struct Fixture { }; for (const auto& asset : assets) store(asset.payload); for (const auto& resource : resources) store(resource.payload); - const std::vector bundles{{bundle_name, fb::write_bundle_region(files), 1}}; + const std::vector bundles{{bundle, fb::write_bundle_region(files), 1}}; const auto directory = catalog.root / name; write(directory / "layout.toc", layout_toc(superbundles)); write(directory / "Win32" / fs::path(toc), fb::write_patch_toc(bundles)); @@ -238,12 +250,12 @@ struct Fixture { catalog.mods.push_back(std::move(mod)); } - // How many files the merged copy of the bundle in `toc` lists, or -1 when the merge left none. - int merged_files(const std::string& relative) const { + // How many files the merged copy of a bundle in `toc` lists, or -1 when the merge left none. + int merged_files(const std::string& relative, const std::string& name = bundle_name) const { const auto toc = catalog.root / mods::generated_folder / "Win32" / fs::path(relative); if (!fs::exists(toc)) return -1; for (const auto& bundle : fb::read_toc(read(toc)).bundles) - if (bundle.name == bundle_name) return static_cast(fb::read_bundle_region(bundle.region).files.size()); + if (bundle.name == name) return static_cast(fb::read_bundle_region(bundle.region).files.size()); return -1; } }; @@ -382,6 +394,35 @@ void two_music_mods_both_carried_into_maps_copy(int baseline) { expect(files == baseline + 6, "two music mods: the map's copy gained 6 files (3 per music mod): " + std::to_string(files) + " vs " + std::to_string(baseline + 6)); } + +// The game keeps the playlist in a second bundle too, as it keeps the music playlist in eight. A +// mod that ships its own copy of that one (it changed something else in it) takes the music mod's +// playlist there. The song, its wave and the wave's resource were added beside the playlist in +// the first bundle; they have to be beside it in this one as well, or the list names a song its +// bundle does not hold. The asset nothing names stays out, and the music mod's own bundle is as +// that mod built it. Either mod may come first. +void carried_into_another_bundle_with_the_list(bool music_first) { + const std::string order = music_first ? "second bundle, music first" : "second bundle, music last"; + Fixture fixture(music_first ? "second-bundle-music-first" : "second-bundle-music-last"); + const auto add_music = [&] { fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); }; + const auto add_tweak = [&] { + fixture.add("tweak", false, shared_toc, {}, + {{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 1, ebx_document(guid(3), {})}}, {}, second_bundle); + }; + if (music_first) { add_music(); add_tweak(); } else { add_tweak(); add_music(); } + const auto report = mods::merge_mods(fixture.catalog); + expect(report.issue.empty() && report.built, order + ": the merge builds the patch\n" + describe(report)); + expect(noted(report, std::string("tweak: ") + second_bundle + ": 1 asset(s) take another mod's change, e.g. test/playlist from music"), + order + ": the copy of the second bundle takes the changed playlist\n" + describe(report)); + expect(noted(report, std::string("tweak: ") + second_bundle + ": 3 asset(s) added by other mods, e.g. test/song"), + order + ": and with it the song, its wave and the wave's resource\n" + describe(report)); + const auto files = fixture.merged_files(shared_toc, second_bundle); + expect(files == 1 + 2 + 3, order + ": that copy lists its manifest, its two assets and those three (" + std::to_string(files) + " files)"); + const auto own = fixture.merged_files(shared_toc); + const auto built = static_cast(1 + music_assets().size() + music_resources().size()); + expect(own == built, order + ": the music mod's own bundle is as it built it (" + std::to_string(own) + " files, " + + std::to_string(built) + " built)"); +} } // namespace int main() try { @@ -393,6 +434,8 @@ int main() try { same_name_from_two_mods(true); same_name_from_two_mods(false); two_music_mods_both_carried_into_maps_copy(baseline); + carried_into_another_bundle_with_the_list(true); + carried_into_another_bundle_with_the_list(false); if (failures) { std::cerr << failures << " failure(s)\n"; return 1; diff --git a/Engine/Vfs/Test/mod_merge_audit_tests.cpp b/Engine/Vfs/Test/mod_merge_audit_tests.cpp new file mode 100644 index 0000000..692688c --- /dev/null +++ b/Engine/Vfs/Test/mod_merge_audit_tests.cpp @@ -0,0 +1,915 @@ +// Runs the whole mod merge on a data root and audits the patch it wrote, against the game and +// against every mod, so a Mods folder of real mods can be checked without starting the game. +// +// What has to hold of any merge: +// * every superbundle TOC reads back, and every placement lies inside an archive that is there +// and that the merged layout declares; +// * every payload the patch stores decodes to the size its asset list gives, and every EBX in +// it parses; +// * every bundle, asset and TOC chunk a mod ships is in the patch; +// * a document two mods' assets fought over by name is named by nothing that is left; +// * a document the merge wrote keeps its instances in the order the engine searches them, has +// every reference a mod appended to one of its lists, and has every value a mod changed in +// place changed: no value in it is the merge's own invention; +// * a list whose entries the game keeps beside it (an item list, the music playlist) names +// nothing its own bundle does not hold, in every copy of that bundle the patch carries. +// What is only reported: assets two mods add under one name, and assets of the game two mods +// change where the patch has one mod's copy whole. +// +// Arguments: [], the root being a folder holding the game's Data/ and +// a Mods/ folder; skipped when none is given. The merge runs there as the game would run it and +// writes Mods/.reskate. With a second argument, every asset the patch itself stores is written +// to that file, one line each, to compare two merges of the same mods. +#include "Engine/Resource/binary_bundle.h" +#include "Engine/Resource/cas_codec.h" +#include "Engine/Resource/ebx_document.h" +#include "Engine/Resource/toc.h" +#include "Engine/Vfs/mod_catalog.h" +#include "Engine/Vfs/mod_merge_internal.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +namespace fb = dingosdk::frostbite; +using namespace dingosdk; +using namespace dingosdk::mods::detail; + +namespace { +using Bytes = std::vector; + +std::map failures; +// The first of each kind in full, the rest counted: one broken mod can fail ten thousand times. +void fail(const std::string& kind, const std::string& text) { + const auto count = ++failures[kind]; + if (count <= 25) std::cout << "FAIL [" << kind << "] " << text << '\n'; + else if (count == 26) std::cout << "FAIL [" << kind << "] ... more of these, counted below\n"; +} +void section(const std::string& title) { std::cout << "\n=== " << title << '\n'; } + +std::string hex(const fb::Sha1& sha) { + static constexpr char digits[] = "0123456789abcdef"; + std::string text; + for (const auto value : sha.bytes) { + text += digits[static_cast(value) >> 4]; + text += digits[static_cast(value) & 15]; + } + return text; +} +std::string key_of(const fb::BundleAsset& asset) { + return std::to_string(static_cast(asset.kind)) + ':' + + (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : lower(asset.name)); +} +const char* kind_name(fb::AssetKind kind) { + return kind == fb::AssetKind::ebx ? "ebx" : kind == fb::AssetKind::resource ? "res" : "chunk"; +} +std::string name_of(const fb::BundleAsset& asset) { + return std::string(kind_name(asset.kind)) + " " + (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : asset.name); +} + +struct Entry { + const fb::BundleAsset* asset{}; + fb::BundleFileInfo file; +}; +// One copy of a bundle: its asset list, and the assets in region order (ebx, resources, chunks). +struct Bundle { + std::string name; + std::optional listing; + std::vector assets; + std::map at; // asset key -> index into assets +}; +struct Toc { + std::string relative; // Win32/..., generic separators + fb::TocDocument document; + std::vector bundles; + std::map bundleAt; // lower-case name + std::set chunks; +}; + +// Where a payload really lives: 0 the game, 1 the merged patch, 2 and up a mod. +struct Place { + int layer{}; + std::uint32_t chunk{}; + std::uint16_t archive{}; + std::uint32_t offset{}, size{}; + auto operator<=>(const Place&) const = default; +}; + +struct EbxFacts { + bool parsed{}; + fb::Guid file; + std::string rootType; + std::vector imports; +}; + +// One mod's own copy of an asset that is not the game's copy of it. +struct Copy { + std::size_t mod{}; + fb::Sha1 sha1; + fb::BundleFileInfo file; +}; +// An asset at least one mod ships a copy of its own of, and what the patch has for it. +struct Contested { + std::string bundle, name; + fb::AssetKind kind{}; + bool shipped{}; // the game has this asset in this bundle + fb::Sha1 game, merged; + Place gamePlace, mergedPlace; + std::vector copies; +}; + +void walk(const fb::ebx::Value& value, const std::function& visit) { + if (const auto* pointer = std::get_if(&value.data)) { visit(*pointer); return; } + if (const auto* object = std::get_if>(&value.data)) { + if (*object) for (const auto& field : (*object)->fields) walk(field.value, visit); + return; + } + if (const auto* array = std::get_if(&value.data)) + for (const auto& element : *array) walk(element, visit); +} + +// What a list entry refers to, so the same entry can be found in another copy of the document; +// empty for an entry that is not a reference to something with a guid. +std::string identity(const fb::ebx::Document& document, const fb::ebx::Value& value) { + const auto* pointer = std::get_if(&value.data); + if (!pointer || pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) return {}; + const auto index = static_cast(pointer->index); + if (pointer->kind == fb::ebx::PointerKind::external) + return index < document.imports.size() + ? "x:" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string{}; + return index < document.instances.size() && document.instances[index].exported + ? "i:" + document.instances[index].instanceGuid.string() : std::string{}; +} +// The document a list entry names, when it names one in another file. +const fb::Guid* named(const fb::ebx::Document& document, const fb::ebx::Value& value) { + const auto* pointer = std::get_if(&value.data); + if (!pointer || pointer->kind != fb::ebx::PointerKind::external || pointer->index < 0 || + static_cast(pointer->index) >= document.imports.size()) return nullptr; + return &document.imports[static_cast(pointer->index)].fileGuid; +} + +// Every value of a document by where it is: its instance (the root, an exported one's guid, an +// internal one's place among the internal ones), then the fields and list positions down to it. +// A plain value is spelled with a lower-case letter in front; a reference (what it leads to) and +// a list's length are spelled with an upper-case one, and are the document's shape. +struct Flat { + std::map values; + std::size_t internal{}; // internal instances: a place is all one is known by +}; +bool plain(const std::string& spelled) { return !spelled.empty() && spelled.front() >= 'a' && spelled.front() <= 'z'; } +void flatten(const fb::ebx::Document& document, const std::vector& instances, + const fb::ebx::Value& value, const std::string& path, Flat& out) { + const auto& data = value.data; + if (const auto* flag = std::get_if(&data)) out.values[path] = *flag ? "b1" : "b0"; + else if (const auto* number = std::get_if(&data)) out.values[path] = "i" + std::to_string(*number); + else if (const auto* count = std::get_if(&data)) out.values[path] = "u" + std::to_string(*count); + else if (const auto* real = std::get_if(&data)) out.values[path] = "d" + std::to_string(std::bit_cast(*real)); + else if (const auto* text = std::get_if(&data)) out.values[path] = "s" + *text; + else if (const auto* guid = std::get_if(&data)) out.values[path] = "g" + guid->string(); + else if (const auto* sha = std::get_if(&data)) out.values[path] = "h" + hex(*sha); + else if (const auto* resource = std::get_if(&data)) out.values[path] = "r" + std::to_string(resource->id); + else if (const auto* pointer = std::get_if(&data)) { + const auto index = pointer->index < 0 ? std::size_t{} : static_cast(pointer->index); + if (pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) out.values[path] = "P"; + else if (pointer->kind == fb::ebx::PointerKind::external) + out.values[path] = index < document.imports.size() + ? "P" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string("P?"); + else out.values[path] = index < instances.size() ? "P" + instances[index] : std::string("P?"); + } else if (const auto* object = std::get_if>(&data)) { + if (*object) for (const auto& field : (*object)->fields) flatten(document, instances, field.value, path + "/" + field.name, out); + } else if (const auto* array = std::get_if(&data)) { + out.values[path + "/#"] = "N" + std::to_string(array->size()); + for (std::size_t index = 0; index < array->size(); ++index) + flatten(document, instances, (*array)[index], path + "/" + std::to_string(index), out); + } +} +Flat flatten(const fb::ebx::Document& document) { + Flat out; + std::vector instances; + for (std::size_t index = 0; index < document.instances.size(); ++index) { + const auto& instance = document.instances[index]; + instances.push_back(index == 0 ? std::string("root") + : instance.exported ? "x:" + instance.instanceGuid.string() : "n:" + std::to_string(out.internal++)); + } + for (std::size_t index = 0; index < document.instances.size(); ++index) + if (const auto& object = document.instances[index].object) + for (const auto& field : object->fields) flatten(document, instances, field.value, instances[index] + "/" + field.name, out); + return out; +} + +struct Audit { + fs::path root, base, out; + mods::Catalog catalog; + vfs::Layout baseLayout, mergedLayout; + std::unique_ptr store; + // A deque: bundles hold pointers into their own asset lists, which must not move. + std::deque tocs; + std::map tocAt; // lower-case relative path + // The game's bundles by name, wherever they are, to tell a mod's own content from the game's + // copy of something it carries. + std::map gameBundles; + std::map> gameListings; + std::map facts; + std::map guids; + std::vector contested; + std::chrono::steady_clock::time_point started = std::chrono::steady_clock::now(); + + void progress(const std::string& what) const { + const auto seconds = std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count(); + std::cerr << "[" << seconds << "s] " << what << std::endl; + } + const fs::path& layer(int index) const { + return index == 0 ? base : index == 1 ? out : catalog.mods.at(static_cast(index - 2)).directory; + } + // A placement that is not a patch one is in the game, whichever layer's bundle names it. + Place place(int from, const fb::BundleFileInfo& file) const { + return {file.location.patch ? from : 0, file.location.installChunk, file.location.archive, file.offset, file.size}; + } + Bytes payload(const Place& where) const { + fb::CasDecodeOptions options{root}; + options.maximumOutputSize = 2048ULL * 1024ULL * 1024ULL; + return fb::decode_cas(store->read(layer(where.layer), {where.layer != 0, where.chunk, where.archive}, where.offset, where.size), options); + } + static EbxFacts facts_of(const Bytes& bytes) { + EbxFacts result; + try { + const auto document = fb::ebx::read_document(bytes); + result.parsed = true; + result.file = document.fileGuid; + result.rootType = document.rootType; + result.imports = document.imports; + } catch (const std::exception&) {} + return result; + } + const EbxFacts& ebx(const Place& where) { + if (const auto found = facts.find(where); found != facts.end()) return found->second; + EbxFacts result; + try { result = facts_of(payload(where)); } catch (const std::exception&) {} + return facts.emplace(where, std::move(result)).first->second; + } + const fb::Guid* guid(const Place& where) { + if (const auto found = guids.find(where); found != guids.end()) return &found->second; + if (const auto known = facts.find(where); known != facts.end() && known->second.parsed) + return &guids.emplace(where, known->second.file).first->second; + try { return &guids.emplace(where, fb::ebx::read_file_guid(payload(where))).first->second; } + catch (const std::exception&) { return nullptr; } + } + + Bundle read_bundle(int from, const fb::TocBundle& bundle) const { + Bundle result; + result.name = bundle.name; + try { + result.listing = list_bundle(*store, layer(from), base, bundle, root); + if (!result.listing) return result; + std::size_t at = result.listing->first; + for (const auto* list : {&result.listing->manifest.ebx, &result.listing->manifest.resources, &result.listing->manifest.chunks}) + for (const auto& asset : *list) { + if (at >= result.listing->files.size()) throw std::runtime_error("more assets than files"); + result.at.emplace(key_of(asset), result.assets.size()); + result.assets.push_back({&asset, result.listing->files[at++]}); + } + } catch (const std::exception&) { + result.listing.reset(); + result.assets.clear(); + result.at.clear(); + } + return result; + } + const Bundle* game_bundle(const std::string& name) { + const auto key = lower(name); + if (const auto found = gameListings.find(key); found != gameListings.end()) return found->second ? &*found->second : nullptr; + auto& slot = gameListings[key]; + if (const auto shipped = gameBundles.find(key); shipped != gameBundles.end()) + if (auto bundle = read_bundle(0, shipped->second); bundle.listing) slot = std::move(bundle); + return slot ? &*slot : nullptr; + } + // The guid of every document a copy of a bundle holds. + std::set held(int from, const Bundle& bundle, bool& unsure) { + std::set result; + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + if (const auto* id = guid(place(from, entry.file))) result.insert(*id); else unsure = true; + } + return result; + } + std::string mod_names(const std::vector& list) const { + std::set unique; + for (const auto index : list) unique.insert(catalog.mods[index].name); + std::string text; + for (const auto& name : unique) text += (text.empty() ? "" : ", ") + name; + return text; + } + + // ---------------------------------------------------------------- the merge, and reading what it wrote + bool merge() { + section("Merge"); + catalog = mods::load_catalog(root); + std::cout << catalog.mods.size() << " mod(s) merged, " << catalog.excluded.size() << " left out, " + << catalog.disabled.size() << " disabled; " << catalog.notes.size() << " note(s)\n"; + if (!catalog.issue.empty()) fail("merge", catalog.issue); + for (const auto& warning : catalog.warnings) fail("left out", warning); + if (!catalog.merged) fail("merge", "no merged patch was produced"); + return catalog.merged; + } + + void load() { + base = root / L"Data"; + out = root / L"Mods" / mods::generated_folder; + baseLayout = vfs::read_layout(base / L"layout.toc"); + mergedLayout = vfs::read_layout(out / L"layout.toc"); + store = std::make_unique(base, fs::temp_directory_path() / L"reskate-merge-audit-unused", mergedLayout.root); + + section("Superbundles"); + const auto toc_files = [](const fs::path& under) { + std::vector files; + std::error_code error; + for (fs::recursive_directory_iterator it(under / L"Win32", error), end; it != end && !error; it.increment(error)) + if (it->is_regular_file(error) && lower(it->path().extension().string()) == ".toc") files.push_back(it->path()); + std::ranges::sort(files); + return files; + }; + std::error_code error; + std::size_t bundles{}, chunks{}, unreadable{}; + for (const auto& file : toc_files(out)) { + Toc toc; + toc.relative = fs::relative(file, out, error).generic_string(); + try { + toc.document = fb::read_toc(read_file(file)); + fb::verify_toc(toc.document); + } catch (const std::exception& failure) { + fail("toc", toc.relative + ": " + failure.what()); + continue; + } + toc.bundles.reserve(toc.document.bundles.size()); + for (const auto& bundle : toc.document.bundles) { + if (!toc.bundleAt.emplace(lower(bundle.name), toc.bundles.size()).second) + fail("toc", toc.relative + " lists " + bundle.name + " twice"); + toc.bundles.push_back(read_bundle(1, bundle)); + unreadable += !toc.bundles.back().listing; + ++bundles; + } + for (const auto& chunk : toc.document.chunks) + if (!chunk.removed) toc.chunks.insert(chunk.guid); + chunks += toc.document.chunks.size(); + tocAt.emplace(lower(toc.relative), tocs.size()); + tocs.push_back(std::move(toc)); + } + std::cout << "the patch: " << tocs.size() << " superbundle TOC(s), " << bundles << " bundle(s) (" << unreadable + << " without a readable asset list), " << chunks << " TOC chunk(s)\n"; + std::size_t shipped{}; + for (const auto& file : toc_files(base)) { + ++shipped; + try { + for (auto& bundle : fb::read_toc(read_file(file)).bundles) gameBundles.try_emplace(lower(bundle.name), std::move(bundle)); + } catch (const std::exception&) {} + } + std::cout << "the game: " << shipped << " superbundle TOC(s), " << gameBundles.size() << " bundle name(s)\n"; + } + + // ---------------------------------------------------------------- placements and the layout + void placements() { + section("Placements and layout"); + using Archive = std::pair; + const auto declared_in = [](const native_db::Node& layout) { + std::set result; + for (const auto* field : {"layeredInstallChunkFiles", "unlayeredInstallChunkFiles"}) + if (const auto* node = layout.field(field); node && node->type == 19) + vfs::for_each_install_chunk_file(node->payload(), [&](std::uint32_t id, std::uint16_t archive) { result.emplace(id, archive); }); + return result; + }; + const auto declared = declared_in(mergedLayout.root), shipped = declared_in(baseLayout.root); + std::set used; + std::size_t checked{}; + const auto check = [&](const std::string& what, const fb::CasIdentifier& location, std::uint32_t offset, std::uint32_t size) { + ++checked; + const auto on_disk = store->archive_size(location.patch ? out : base, location); + if (!on_disk) { + fail("placement", what + " is in " + store->describe(location) + (location.patch ? " of the patch" : " of the game") + ", which is not there"); + return; + } + if (static_cast(offset) + size > *on_disk) + fail("placement", what + ": " + std::to_string(size) + " bytes at " + std::to_string(offset) + " run past the end of " + + store->describe(location) + " (" + std::to_string(*on_disk) + " bytes)"); + if (location.patch) used.emplace(location.installChunk, location.archive); + }; + for (const auto& toc : tocs) { + for (const auto& bundle : toc.document.bundles) { + const auto region = fb::read_bundle_region(bundle.region); + for (std::size_t index = 0; index < region.files.size(); ++index) + check(toc.relative + ": " + bundle.name + " file " + std::to_string(index), region.files[index].location, + region.files[index].offset, region.files[index].size); + } + for (const auto& chunk : toc.document.chunks) + if (!chunk.removed) check(toc.relative + ": chunk " + chunk.guid.string(), chunk.location, chunk.offset, chunk.size); + } + // An archive the layout does not declare is never opened; one it declares and the patch + // lacks sends the engine looking for a file that is absent. + for (const auto& archive : used) + if (!declared.contains(archive)) + fail("layout", "the patch reads archive " + std::to_string(archive.second) + " of install chunk " + + std::to_string(archive.first) + ", which the merged layout does not declare"); + std::size_t added{}; + for (const auto& archive : declared) { + if (shipped.contains(archive)) continue; + ++added; + if (!store->archive_size(out, {true, archive.first, archive.second})) + fail("layout", "the merged layout declares archive " + std::to_string(archive.second) + " of install chunk " + + std::to_string(archive.first) + ", and the patch has no such file"); + } + std::cout << checked << " placement(s) checked; " << used.size() << " patch archive(s) in use, " << added << " declared beyond the game's\n"; + + // The superbundle list is searched by name, so it stays in order; a mod's own superbundles + // have to be in it, in an install chunk, and have a TOC. + const auto names = [](const native_db::Node& layout) { + std::vector result; + if (const auto* list = layout.field("superBundles")) + for (const auto& row : list->children) + if (const auto* name = row.field("name"); name && name->type == 7) result.push_back(lower(name->text)); + return result; + }; + const auto merged = names(mergedLayout.root), game = names(baseLayout.root); + const std::set known(merged.begin(), merged.end()), stock(game.begin(), game.end()); + if (!std::ranges::is_sorted(merged)) fail("layout", "the merged superbundle list is not in name order"); + if (known.size() != merged.size()) fail("layout", "the merged superbundle list names one twice"); + std::set placed; + if (const auto* manifest = mergedLayout.root.field("installManifest")) + if (const auto* chunks = manifest->field("installChunks")) + for (const auto& chunk : chunks->children) + if (const auto* list = chunk.field("superbundles")) + for (const auto& entry : list->children) + if (entry.type == 7) placed.insert(lower(entry.text)); + std::size_t own{}; + for (const auto& mod : catalog.mods) { + if (!mod.provides_layout) continue; + try { + const auto layout = vfs::read_layout(mod.directory / L"layout.toc"); + for (const auto& name : names(layout.root)) { + if (stock.contains(name)) continue; + ++own; + if (!known.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is not in the merged layout"); + if (!placed.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is in no install chunk of the merged layout"); + if (!tocAt.contains(name + ".toc")) fail("layout", mod.name + ": its superbundle " + name + " has no TOC in the patch"); + } + } catch (const std::exception& failure) { + fail("layout", mod.name + ": layout.toc could not be read (" + failure.what() + ")"); + } + } + std::cout << merged.size() << " superbundle(s) in the merged layout, " << own << " listed by mods beyond the game's\n"; + } + + // ---------------------------------------------------------------- asset lists and payloads + void payloads(const fs::path& listing) { + section("Asset lists and payloads"); + // What a listed size means is learnt from the game's own assets first: only a kind whose + // every sample agrees is held to it. + std::array sampled{}, agreed{}; + std::set seen; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) + for (const auto& entry : bundle.assets) { + const auto kind = static_cast(entry.asset->kind); + const auto where = place(1, entry.file); + if (where.layer != 0 || sampled[kind] >= 150 || entry.file.size > (8u << 20) || !seen.insert(where).second) continue; + ++sampled[kind]; + try { agreed[kind] += payload(where).size() == entry.asset->originalSize; } catch (const std::exception&) {} + } + std::array sized{}; + for (std::size_t kind = 0; kind < 3; ++kind) sized[kind] = sampled[kind] >= 20 && agreed[kind] == sampled[kind]; + + std::map> stored; + std::size_t listed{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) { + if (bundle.at.size() != bundle.assets.size()) { + std::set names; + std::string example; + for (const auto& entry : bundle.assets) + if (!names.insert(key_of(*entry.asset)).second && example.empty()) example = name_of(*entry.asset); + fail("asset list", toc.relative + ": " + bundle.name + " lists an asset twice, e.g. " + example); + } + for (const auto& entry : bundle.assets) { + ++listed; + if (const auto where = place(1, entry.file); where.layer == 1) stored.try_emplace(where, &entry, toc.relative + ": " + bundle.name); + } + } + std::cout << listed << " asset(s) listed across the patch; " << stored.size() << " payload(s) are stored in the patch's own archives\n"; + progress("decoding " + std::to_string(stored.size()) + " payloads"); + std::ofstream written; + if (!listing.empty()) written.open(listing, std::ios::binary); + std::array decoded{}; + std::size_t bytesIn{}, bytesOut{}, unparsed{}; + for (const auto& [where, what] : stored) { + const auto& [entry, from] = what; + const auto kind = static_cast(entry->asset->kind); + try { + const auto bytes = payload(where); + ++decoded[kind]; + bytesIn += where.size; + bytesOut += bytes.size(); + if (sized[kind] && bytes.size() != entry->asset->originalSize) + fail("payload", from + ": " + name_of(*entry->asset) + " decodes to " + std::to_string(bytes.size()) + + " bytes, its asset list says " + std::to_string(entry->asset->originalSize)); + if (entry->asset->kind == fb::AssetKind::ebx) { + auto result = facts_of(bytes); + if (!result.parsed) { ++unparsed; fail("payload", from + ": " + name_of(*entry->asset) + " is not a document the merge can read"); } + facts.emplace(where, std::move(result)); + } + if (written.is_open()) written << from << '\t' << name_of(*entry->asset) << '\t' << bytes.size() << '\t' << hex(sha1_of(bytes)) << '\n'; + } catch (const std::exception& failure) { + fail("payload", from + ": " + name_of(*entry->asset) + " at " + store->describe({true, where.chunk, where.archive}) + "+" + + std::to_string(where.offset) + " cannot be decoded (" + failure.what() + ")"); + } + } + std::cout << decoded[0] << " ebx, " << decoded[1] << " res and " << decoded[2] << " chunk payload(s) decoded: " << bytesIn / (1024 * 1024) + << " MB stored, " << bytesOut / (1024 * 1024) << " MB decoded; " << unparsed << " EBX do not parse\n"; + } + + // ---------------------------------------------------------------- every mod against the patch + void mods_against_patch() { + section("Every mod against the patch"); + std::map found; // by bundle name, asset and what the patch has + std::size_t bundles{}, assets{}, chunks{}, byName{}; + for (std::size_t index = 0; index < catalog.mods.size(); ++index) { + const auto& mod = catalog.mods[index]; + if (!mod.provides_layout) continue; + if (index % 20 == 0) progress("mod " + std::to_string(index + 1) + " of " + std::to_string(catalog.mods.size())); + for (const auto& relative : scan(mod.directory).tocs) { + const auto merged = tocAt.find(lower(relative)); + if (merged == tocAt.end()) { fail("mod content", mod.name + ": the patch has no " + relative); continue; } + const auto& toc = tocs[merged->second]; + fb::TocDocument own; + try { own = fb::read_toc(read_file(mod.directory / fs::path(relative))); } + catch (const std::exception& failure) { fail("mod content", mod.name + ": " + relative + " cannot be read (" + failure.what() + ")"); continue; } + for (const auto& chunk : own.chunks) { + if (chunk.removed) continue; + ++chunks; + if (!toc.chunks.contains(chunk.guid)) + fail("mod content", mod.name + ": " + relative + ": its chunk " + chunk.guid.string() + " is not in the patch's TOC"); + } + for (const auto& bundle : own.bundles) { + ++bundles; + const auto at = toc.bundleAt.find(lower(bundle.name)); + if (at == toc.bundleAt.end()) { fail("mod content", mod.name + ": " + relative + ": its bundle " + bundle.name + " is not in the patch"); continue; } + const auto& patched = toc.bundles[at->second]; + const auto mine = read_bundle(static_cast(index + 2), bundle); + if (!mine.listing || !patched.listing) { ++byName; continue; } + const auto* game = game_bundle(bundle.name); + for (const auto& entry : mine.assets) { + ++assets; + const auto key = key_of(*entry.asset); + const auto there = patched.at.find(key); + if (there == patched.at.end()) { + fail("mod content", mod.name + ": " + relative + ": " + bundle.name + ": its " + name_of(*entry.asset) + " is not in the patch's copy"); + continue; + } + const auto& kept = patched.assets[there->second]; + const Entry* shipped{}; + if (game) if (const auto it = game->at.find(key); it != game->at.end()) shipped = &game->assets[it->second]; + // The game's copy carried along is not this mod's content. + if (shipped && shipped->asset->sha1 == entry.asset->sha1) continue; + auto& record = found[lower(bundle.name) + '|' + key + '|' + hex(kept.asset->sha1)]; + if (record.copies.empty()) { + record.bundle = bundle.name; + record.name = entry.asset->name; + record.kind = entry.asset->kind; + record.shipped = shipped != nullptr; + if (shipped) { record.game = shipped->asset->sha1; record.gamePlace = place(0, shipped->file); } + record.merged = kept.asset->sha1; + record.mergedPlace = place(1, kept.file); + } + if (std::ranges::none_of(record.copies, [&](const Copy& copy) { return copy.mod == index && copy.sha1 == entry.asset->sha1; })) + record.copies.push_back({index, entry.asset->sha1, entry.file}); + } + } + } + } + std::cout << bundles << " mod bundle(s), " << assets << " listed asset(s) and " << chunks << " TOC chunk(s) looked up in the patch; " + << byName << " bundle(s) could only be checked by name\n"; + for (auto& [key, record] : found) contested.push_back(std::move(record)); + } + + // ---------------------------------------------------------------- what became of each mod's own content + void outcomes() { + section("What became of each mod's own content"); + struct Clash { const Contested* asset; std::vector kept, lost; }; + std::vector clashes, overridden; + std::vector written; + std::size_t asShipped{}, rewritten{}; + for (const auto& record : contested) { + std::set versions; + for (const auto& copy : record.copies) versions.insert(copy.sha1); + const bool one = versions.contains(record.merged); + if (one && versions.size() == 1) { ++asShipped; continue; } + if (record.shipped && record.merged == record.game) { + fail("lost edit", record.bundle + ": " + record.name + ": " + std::to_string(record.copies.size()) + " mod(s) change it (e.g. " + + catalog.mods[record.copies.front().mod].name + ") and the patch has the game's own copy"); + continue; + } + if (!one) { + // No one mod's copy: the merge wrote it (edits combined, a change carried in). + if (record.kind == fb::AssetKind::ebx) written.push_back(&record); else ++rewritten; + continue; + } + Clash clash{&record, {}, {}}; + for (const auto& copy : record.copies) (copy.sha1 == record.merged ? clash.kept : clash.lost).push_back(copy.mod); + (record.shipped ? overridden : clashes).push_back(std::move(clash)); + } + std::cout << asShipped << " asset(s) are in the patch as their one mod ships them; the merge wrote " << written.size() + << " EBX document(s) and " << rewritten << " other asset(s) itself\n"; + + std::cout << "\nassets two mods add under one name, with different content: " << clashes.size() << '\n'; + std::map> pairs; + for (const auto& clash : clashes) pairs[mod_names(clash.lost) + " lost them to " + mod_names(clash.kept)].push_back(&clash); + for (const auto& [pair, list] : pairs) + std::cout << " " << pair << ": " << list.size() << ", e.g. " << list.front()->asset->name << " in " << list.front()->asset->bundle << '\n'; + + std::cout << "assets of the game two mods change, where the patch has one mod's copy whole: " << overridden.size() << '\n'; + std::size_t printed{}; + for (const auto& clash : overridden) { + if (++printed > 40) { std::cout << " ... and " << overridden.size() - 40 << " more\n"; break; } + std::cout << " " << kind_name(clash.asset->kind) << ' ' << clash.asset->name << " (" << clash.asset->bundle << "): kept " + << mod_names(clash.kept) << "; not in it: " << mod_names(clash.lost) << '\n'; + } + + // A reference is by the guid of the document it names. Where two mods' assets shared a + // name and one went, whatever still names the one that went finds nothing. + section("References to documents that lost their name to another mod"); + struct Gone { std::string mod, name, keptBy; }; + std::map gone; + for (const auto& clash : clashes) { + if (clash.asset->kind != fb::AssetKind::ebx) continue; + const auto& kept = ebx(clash.asset->mergedPlace); + for (const auto& copy : clash.asset->copies) { + if (copy.sha1 == clash.asset->merged) continue; + const auto& lost = ebx(place(static_cast(copy.mod + 2), copy.file)); + // The same guid under both copies still resolves, to the copy that stayed. + if (lost.parsed && kept.parsed && lost.file != kept.file) + gone.emplace(lost.file, Gone{catalog.mods[copy.mod].name, clash.asset->name, mod_names(clash.kept)}); + } + } + std::map> holders; + std::size_t references{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + const auto where = place(1, entry.file); + if (where.layer != 1) continue; + for (const auto& import : ebx(where).imports) { + const auto lost = gone.find(import.fileGuid); + if (lost == gone.end()) continue; + ++references; + auto& [count, example] = holders[bundle.name + ": " + entry.asset->name + " names " + lost->second.mod + "'s " + + lost->second.name + ", which " + lost->second.keptBy + "'s replaced"]; + if (!count++) example = toc.relative; + } + } + for (const auto& [text, where] : holders) + fail("dangling reference", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + std::cout << gone.size() << " document(s) went with a guid nothing in the patch has; " << references << " reference(s) in the patch still name one\n"; + + section("Documents the merge wrote"); + std::size_t checked{}, lists{}, kept{}, inPlace{}, valueEdits{}, reshaped{}; + for (const auto* record : written) { + fb::ebx::Document merged; + try { merged = fb::ebx::read_document(payload(record->mergedPlace)); } + catch (const std::exception& failure) { fail("merged document", record->bundle + ": " + record->name + " cannot be read back (" + failure.what() + ")"); continue; } + ++checked; + const auto label = record->bundle + ": " + record->name + " (" + merged.rootType + ")"; + // The engine binary-searches the exported instances after the root by guid, and + // internal ones have to follow them. + bool internal{}; + const fb::Guid* previous{}; + for (std::size_t index = 1; index < merged.instances.size(); ++index) { + const auto& instance = merged.instances[index]; + if (!instance.exported) { internal = true; continue; } + if (internal) { fail("merged document", label + ": an exported instance follows an internal one"); break; } + if (previous && !(*previous < instance.instanceGuid)) { fail("merged document", label + ": its exported instances are not in guid order"); break; } + previous = &instance.instanceGuid; + } + std::size_t stray{}; + for (const auto& instance : merged.instances) + if (instance.object) + for (const auto& field : instance.object->fields) + walk(field.value, [&](const fb::ebx::PointerReference& pointer) { + const auto limit = pointer.kind == fb::ebx::PointerKind::internal ? merged.instances.size() + : pointer.kind == fb::ebx::PointerKind::external ? merged.imports.size() : std::size_t{}; + stray += pointer.kind != fb::ebx::PointerKind::null && (pointer.index < 0 || static_cast(pointer.index) >= limit); + }); + if (stray) fail("merged document", label + ": " + std::to_string(stray) + " reference(s) index past the instance or import table"); + if (!record->shipped) continue; + fb::ebx::Document game; + try { game = fb::ebx::read_document(payload(record->gamePlace)); } catch (const std::exception&) { continue; } + if (!merged.root() || !merged.root()->object || !game.root() || !game.root()->object) continue; + // Every reference a mod appended to one of the root's lists is in the merged list, + // unless what it names lost its name to another mod's asset. + std::set done; + for (const auto& copy : record->copies) { + if (!done.insert(copy.sha1).second) continue; + fb::ebx::Document edit; + try { edit = fb::ebx::read_document(payload(place(static_cast(copy.mod + 2), copy.file))); } catch (const std::exception&) { continue; } + if (!edit.root() || !edit.root()->object) continue; + for (const auto& field : edit.root()->object->fields) { + const auto* added = std::get_if(&field.value.data); + const auto* before = game.root()->object->find(field.name); + const auto* after = merged.root()->object->find(field.name); + const auto* beforeList = before ? std::get_if(&before->value.data) : nullptr; + const auto* afterList = after ? std::get_if(&after->value.data) : nullptr; + if (!added || !beforeList || !afterList || added->size() <= beforeList->size()) continue; + ++lists; + std::set present; + for (const auto& value : *afterList) present.insert(identity(merged, value)); + for (auto at = beforeList->size(); at < added->size(); ++at) { + const auto id = identity(edit, (*added)[at]); + if (id.empty()) continue; + if (present.contains(id)) { ++kept; continue; } + if (const auto* target = named(edit, (*added)[at]); target && gone.contains(*target)) continue; + fail("merged list", label + ": " + field.name + " lacks an entry " + catalog.mods[copy.mod].name + " added (" + id + ")"); + } + } + } + + // Values changed in place. What the merged document has that the game's does not is + // some mod's value, and a mod that changed values and nothing else has every one of + // them changed in the merged document: to its own, or to another mod's that changed + // the same one. + const auto before = flatten(game), after = flatten(merged); + // An internal instance is known by its place alone, which says nothing once one was + // added or taken away. + const auto told = [&](const Flat& other, const std::string& key) { + return !key.starts_with("n:") || other.internal == before.internal; + }; + std::vector> edits; + done.clear(); + for (const auto& copy : record->copies) { + if (!done.insert(copy.sha1).second) continue; + try { edits.emplace_back(copy.mod, flatten(fb::ebx::read_document(payload(place(static_cast(copy.mod + 2), copy.file))))); } + catch (const std::exception&) {} + } + std::size_t invented{}; + std::string example; + for (const auto& [key, value] : before.values) { + if (!plain(value) || !told(after, key)) continue; + const auto now = after.values.find(key); + if (now == after.values.end() || now->second == value) continue; + ++inPlace; + if (std::ranges::none_of(edits, [&](const std::pair& edit) { + const auto theirs = edit.second.values.find(key); + return told(edit.second, key) && theirs != edit.second.values.end() && theirs->second == now->second; })) + if (!invented++) example = key; + } + if (invented) + fail("merged value", label + ": " + std::to_string(invented) + " value(s) are neither the game's nor any mod's, e.g. " + example); + for (const auto& [mod, edit] : edits) { + bool shape{}; // a reference moved, a list resized, something of the game's gone + std::vector changed; + for (const auto& [key, value] : before.values) { + if (!told(edit, key)) continue; + const auto theirs = edit.values.find(key); + if (theirs == edit.values.end()) { shape = true; continue; } + if (theirs->second == value) continue; + if (plain(value) && plain(theirs->second)) { changed.push_back(&key); continue; } + // A list on the root that grew at its end is the one change of shape that + // is combined, and is checked above. + const bool rootList = key.starts_with("root/") && key.ends_with("/#") && std::ranges::count(key, '/') == 2; + if (!(rootList && std::stoull(theirs->second.substr(1)) > std::stoull(value.substr(1)))) shape = true; + } + if (changed.empty()) continue; + if (shape) { ++reshaped; continue; } + std::size_t lost{}; + for (const auto* key : changed) { + const auto now = after.values.find(*key); + if (told(after, *key) && (now == after.values.end() || now->second == before.values.at(*key)) && !lost++) example = *key; + } + if (lost) + fail("lost value", label + ": " + std::to_string(lost) + " of the " + std::to_string(changed.size()) + " value(s) " + + catalog.mods[mod].name + " changed are the game's again, e.g. " + example); + ++valueEdits; + } + } + std::cout << checked << " document(s) read back; " << lists << " list(s) a mod appended to, " << kept << " appended reference(s) found in them\n"; + std::cout << inPlace << " value(s) in them differ from the game's, each one some mod's; " << valueEdits + << " mod copy(ies) that changed values alone have them all in; " << reshaped + << " that also moved a reference or resized a list are not combined that way\n"; + } + + // ---------------------------------------------------------------- lists and what they name + // The game keeps what some lists name in the list's own bundle (an item list and its items, + // the music playlist and its songs), and reads such a list without checking an entry. So in + // every copy of that bundle the patch carries, the entries mods added have to be there too. + void lists() { + section("Lists whose entries the game keeps beside them"); + struct Shipped { fb::ebx::Document document; std::set held; bool unsure{}; }; + std::map> shipped; // by bundle and asset key + std::map> missing, twice; + std::size_t copies{}, entries{}; + for (const auto& toc : tocs) + for (const auto& bundle : toc.bundles) { + const auto* game = game_bundle(bundle.name); + if (!game) continue; + std::optional> here; + bool unsure{}; + for (const auto& entry : bundle.assets) { + if (entry.asset->kind != fb::AssetKind::ebx) continue; + const auto where = place(1, entry.file); + const auto key = key_of(*entry.asset); + const auto original = game->at.find(key); + if (where.layer != 1 || original == game->at.end()) continue; + auto& known = shipped[lower(bundle.name) + '|' + key]; + if (!known) { + Shipped value; + try { value.document = fb::ebx::read_document(payload(place(0, game->assets[original->second].file))); } catch (const std::exception&) {} + known = std::move(value); + } + const auto* gameRoot = known->document.root(); + if (!gameRoot || !gameRoot->object) continue; + // Only a document with a list of references to other documents is read at all. + const auto lists_references = std::ranges::any_of(gameRoot->object->fields, [&](const fb::ebx::FieldValue& field) { + const auto* list = std::get_if(&field.value.data); + return list && !list->empty() && named(known->document, list->front()); + }); + if (!lists_references) continue; + fb::ebx::Document merged; + try { merged = fb::ebx::read_document(payload(where)); } catch (const std::exception&) { continue; } + if (!merged.root() || !merged.root()->object) continue; + for (const auto& field : merged.root()->object->fields) { + const auto* list = std::get_if(&field.value.data); + const auto* before = gameRoot->object->find(field.name); + const auto* beforeList = before ? std::get_if(&before->value.data) : nullptr; + if (!list || !beforeList || beforeList->empty() || list->size() <= beforeList->size()) continue; + // Does the game keep every one of its own entries in this bundle? + if (known->held.empty() && !known->unsure) known->held = held(0, *game, known->unsure); + if (known->unsure || !std::ranges::all_of(*beforeList, [&](const fb::ebx::Value& value) { + const auto* target = named(known->document, value); + return target && known->held.contains(*target); })) continue; + if (!here) here = held(1, bundle, unsure); + if (unsure) continue; + ++copies; + std::set seen; + std::size_t absent{}, repeated{}; + for (std::size_t index = 0; index < list->size(); ++index) { + const auto* target = named(merged, (*list)[index]); + if (!target) { if (index >= beforeList->size()) ++absent; continue; } + if (index < beforeList->size()) continue; + ++entries; + absent += !here->contains(*target); + repeated += !seen.insert(*target).second; + } + const auto what = bundle.name + ": " + field.name + " of " + entry.asset->name; + if (absent) { auto& [count, example] = missing[what + ": " + std::to_string(absent) + " of " + std::to_string(list->size() - beforeList->size()) + + " added entr" + (absent == 1 ? "y names" : "ies name") + " a document that bundle does not hold"]; + if (!count++) example = toc.relative; } + if (repeated) { auto& [count, example] = twice[what + ": " + std::to_string(repeated) + " added entr" + (repeated == 1 ? "y repeats" : "ies repeat") + " one a mod already added"]; + if (!count++) example = toc.relative; } + } + } + } + for (const auto& [text, where] : missing) + fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + for (const auto& [text, where] : twice) + fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")"); + std::cout << copies << " such list(s) with entries mods added, across every copy in the patch; " << entries << " added entries checked\n"; + } + + int summary() const { + section("Summary"); + std::size_t total{}; + for (const auto& [kind, count] : failures) { std::cout << count << " x " << kind << '\n'; total += count; } + std::cout << (total ? "mod merge audit: " + std::to_string(total) + " problem(s)" : std::string("mod merge audit: ok")) << '\n'; + return total ? 1 : 0; + } +}; +} // namespace + +int main(int argc, char** argv) try { + if (argc < 2 || !fs::exists(fs::path(argv[1]) / "Data") || !fs::exists(fs::path(argv[1]) / "Mods")) { + std::cout << "No data root with Data and Mods given; skipped.\n"; + return 0; + } + Audit audit; + audit.root = argv[1]; + if (!audit.merge()) return audit.summary(); + audit.load(); + audit.placements(); + audit.payloads(argc > 2 ? fs::path(argv[2]) : fs::path{}); + audit.mods_against_patch(); + audit.outcomes(); + audit.lists(); + return audit.summary(); +} catch (const std::exception& error) { + std::cerr << "FAIL: " << error.what() << '\n'; + return 1; +} diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 5311512..3695e4c 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace dingosdk::mods::detail { namespace { @@ -236,6 +237,22 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, std::set addedFrom; // The mods absorbed so far, so each asset can say whose copy it is. std::set owners; + // Every EBX a mod added to one of the game's bundles, by its document's guid + // and with the bundle it was added to: what a changed list names is looked + // up here, whichever bundle the mod put it in. + std::multimap> addedByGuid; + // The resource a mod added to a bundle under a name, by bundle, mod and name. + std::map, const AssetAddition*> companions; + for (const auto& [bundleName, list] : overrides.added) + for (const auto& addition : list) { + if (addition.asset.kind == fb::AssetKind::ebx) + addedByGuid.emplace(addition.file, std::pair{&bundleName, &addition}); + else if (addition.asset.kind == fb::AssetKind::resource) + companions.emplace(std::tuple{bundleName, addition.mod, lower(addition.asset.name)}, &addition); + } + // Where an addition's payload already went in this superbundle's patch + // archive, by install chunk: every copy that is given it reads the one. + std::map, fb::BundleFileInfo> carriedAt; const auto absorb = [&](const fb::TocBundle& bundle, const ArchivePlacement* placement, bool isBase, const fs::path& root) { @@ -385,6 +402,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // The mod's unchanged copies of assets another mod changed: the change // goes into the merged patch, next to this bundle's other files. std::map overridden; + // The documents those changes refer to: the ones a mod added come along below. + std::set wanted; if (propagate && (casBacked || !region.inlineManifest.empty())) { std::string example; auto manifest = casBacked ? std::move(casManifest) : fb::read_binary_bundle(region.inlineManifest); @@ -407,6 +426,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, asset.sha1 = change->second.sha1; asset.originalSize = change->second.originalSize; if (resource) asset.resourceMeta = resource->resourceMeta; + wanted.insert(change->second.names.begin(), change->second.names.end()); if (example.empty()) example = asset.name + " from " + change->second.mod; } catch (const std::exception& error) { report.notes.push_back(modName + ": " + bundle.name + ": " + asset.name + @@ -426,23 +446,55 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, // loads instead of the game's on this mod's levels, so it gets them too. std::vector additions; if (propagate && casBacked) { - if (const auto found = overrides.added.find(key); found != overrides.added.end()) { + std::vector take; + std::set taking; + // A copy in the TOC the adder ships itself merges with the adder's bundle and has + // its assets already; only copies in other superbundles (maps) need them carried. + const auto here = lower(relative); + if (const auto found = overrides.added.find(key); found != overrides.added.end()) + for (const auto& addition : found->second) { + if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk || + addition.toc == here) continue; + take.push_back(&addition); + taking.insert(&addition); + } + // What the changes this copy took name. The game keeps a list in more + // bundles than the one a mod added to (the music playlist is in eight), + // and a copy of any of them takes the changed list: it needs what the + // mod added next to that list, and what those refer to in turn. + for (std::vector open(wanted.begin(), wanted.end()); !open.empty();) { + const auto guid = open.back(); + open.pop_back(); + const auto [first, last] = addedByGuid.equal_range(guid); + for (auto at = first; at != last; ++at) { + const auto& [from, addition] = at->second; + if ((*from == key && addition->toc == here) || !taking.insert(addition).second) continue; + take.push_back(addition); + for (const auto& name : addition->names) + if (wanted.insert(name).second) open.push_back(name); + // The resource added under the same name goes where the asset goes. + if (const auto companion = companions.find({*from, addition->mod, lower(addition->asset.name)}); + companion != companions.end() && taking.insert(companion->second).second) + take.push_back(companion->second); + } + } + if (!take.empty()) { std::set> present; for (const auto& asset : casManifest.ebx) present.insert(asset_key(asset)); for (const auto& asset : casManifest.resources) present.insert(asset_key(asset)); - // A copy in the TOC the adder ships itself merges with the adder's bundle and has - // its assets already; only copies in other superbundles (maps) need them carried. - const auto here = lower(relative); - for (const auto& addition : found->second) { - if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk || - addition.toc == here || present.contains(asset_key(addition.asset))) continue; + const auto installChunk = region.files.front().location.installChunk; + for (const auto* addition : take) { + if (!present.insert(asset_key(addition->asset)).second) continue; try { - additions.push_back({addition.asset, store.write(region.files.front().location.installChunk, - manifestArchive, addition.encoded)}); - addedFrom.insert(addition.mod); + auto placed = carriedAt.find({installChunk, addition}); + if (placed == carriedAt.end()) + placed = carriedAt.emplace(std::pair{installChunk, addition}, + store.write(installChunk, manifestArchive, addition->encoded)).first; + additions.push_back({addition->asset, placed->second}); + addedFrom.insert(addition->mod); } catch (const std::exception& error) { - report.notes.push_back(modName + ": " + bundle.name + ": " + addition.asset.name + - " from " + addition.mod + " could not be added (" + error.what() + ")"); + report.notes.push_back(modName + ": " + bundle.name + ": " + addition->asset.name + + " from " + addition->mod + " could not be added (" + error.what() + ")"); } } if (!additions.empty()) @@ -733,15 +785,55 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, return known.fileGuid == item.fileGuid; }) || held.holds(item.fileGuid); }; + // Any other list loses an entry only when what it names was in + // this bundle and another mod's asset of the same name took its + // place: nothing else is known about where its entries live. + const fb::ebx::CarryImport notReplaced = [&](const fb::ebx::ImportReference& named) { + return !held.replacement(named.fileGuid) || held.holds(named.fileGuid); + }; fb::ebx::MergeSummary summary; auto combined = fb::ebx::merge_documents(baseDocument, pointers, &summary, - itemList ? inBundle : fb::ebx::CarryImport{}); - // One mod's copy with nothing to leave out stays as that mod wrote it. - if (summary.dropped.empty() && - (edits.size() < 2 || (!summary.instances && !summary.arrayEntries))) continue; + itemList ? inBundle : notReplaced); + const auto mod_of = [&](std::size_t edit) { return path_utf8(edits[edit]->root.filename()); }; + if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries) { + // One mod's copy with nothing to leave out stays as that mod wrote it. + if (edits.size() < 2) continue; + // The same change from each of them is one change. + if (std::ranges::all_of(edits, [&](const Contribution* edit) { + return edit->sha1 == edits.front()->sha1; })) continue; + // Nothing was added to it, so each mod changed what was there. + // Values changed in place combine; when that is not all of + // some mod's change, the copy already kept stays whole, and + // the others' changes to it are not in the game. + if (!summary.values || !summary.exact()) { + std::set others; + for (std::size_t index = 0; index < edits.size(); ++index) + if (edits[index]->sha1 != entry.asset.sha1) others.insert(mod_of(index)); + if (entry.mod) others.erase(*entry.mod); + std::string names; + for (const auto& other : others) names += (names.empty() ? "" : ", ") + other; + if (!others.empty()) + report.notes.push_back(entry.asset.name + ": kept " + + (entry.mod ? *entry.mod + "'s copy" : std::string("the highest-priority copy")) + + " whole; the changes " + names + " made to it are of a kind that cannot be " + "combined with it and are not in the game"); + continue; + } + } rebuilt = fb::ebx::write_document(combined); what = std::to_string(summary.instances) + " instance(s), " + std::to_string(summary.arrayEntries) + " list entries"; + if (summary.values) what += ", " + std::to_string(summary.values) + " changed value(s)"; + if (summary.contested) what += ", " + std::to_string(summary.contested) + " disagreed"; + if (summary.repeated) + what += ", " + std::to_string(summary.repeated) + " repeated entry(ies) listed once"; + // The merged copy is the game's with what the mods added: a mod's + // change to what was already there is in it only when it is one + // that could be taken. + for (const auto index : summary.uncarried) + report.notes.push_back(mod_of(index) + ": " + entry.asset.name + + ": the merged copy has what this mod added to it, but not its changes to what " + "the game's copy already held"); std::map> left; // by edit for (const auto& gone : summary.dropped) left[gone.edit].push_back(gone.target.fileGuid); for (const auto& [index, guids] : left) { @@ -749,8 +841,9 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, const Asset* taken{}; for (const auto& guid : guids) if ((taken = held.replacement(guid))) break; - report.notes.push_back(path_utf8(edits[index]->root.filename()) + ": " + entry.asset.name + - ": " + std::to_string(guids.size()) + " item(s) left out of the list because the merged " + report.notes.push_back(mod_of(index) + ": " + entry.asset.name + + ": " + std::to_string(guids.size()) + (itemList ? " item(s)" : " entry(ies)") + + " left out of the list because the merged " "bundle does not hold them, e.g. " + (taken ? taken->asset.name + ", replaced by " + (taken->mod ? *taken->mod : "another mod") + "'s asset of the same name" diff --git a/Engine/Vfs/mod_merge_internal.h b/Engine/Vfs/mod_merge_internal.h index 52c483e..c2dac61 100644 --- a/Engine/Vfs/mod_merge_internal.h +++ b/Engine/Vfs/mod_merge_internal.h @@ -180,13 +180,19 @@ struct AssetOverride { // Present for a Lua resource replacement: its identity and metadata must // travel with the script bytes, including source and bytecode sizes. std::optional resource; + // The documents the changed EBX refers to that its mod added to the bundle it + // made the change in, by their guids. They go wherever the change goes: the + // game keeps a list in more bundles than one, each with what the list names. + std::set names; }; inline constexpr std::uint32_t luaScriptResourceType = 0xEC383B87; // An EBX asset a mod added to one of the game's bundles, or a resource it added // under the same name as such an asset (a wave's sound-bank). Maps carry their own // copy of a bundle like bam_coregameassets, and that copy loads instead of the // game's on their levels; an edit to a list there (the master music playlist) -// propagates as a change, so the asset it now names has to come along too. +// propagates as a change, so the asset it now names has to come along too. The +// game also keeps such a list in more bundles than the one the mod put its +// assets in, and a copy of any of those takes the change the same way. struct AssetAddition { std::string mod; fb::BundleAsset asset; @@ -195,6 +201,10 @@ struct AssetAddition { // bundle in that same TOC merges with the mod's own bundle and already has the asset; // only copies in other superbundles (a map's level TOC) need it carried. std::string toc; + // For an EBX: its document's guid, which is what a list names it by, and the + // guids of the documents it refers to in turn (a song names its wave). + fb::Guid file; + std::vector names; }; struct AssetOverrides { // By lower-case asset name, then the game's sha1 the change replaces. diff --git a/Engine/Vfs/mod_merge_overrides.cpp b/Engine/Vfs/mod_merge_overrides.cpp index 23cbbe1..049e7c7 100644 --- a/Engine/Vfs/mod_merge_overrides.cpp +++ b/Engine/Vfs/mod_merge_overrides.cpp @@ -31,6 +31,20 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, std::vector> candidates, companions; std::vector newChunks; // TOC chunks the game's TOCs do not have std::set imported; + // The game's assets this mod changed. They wait until its additions are + // settled: one that another mod's asset of the same name stands in for + // is not where a copy can find it, and a change must not name it there. + struct Change { + std::string bundle; // lower case: where the mod changed it + std::string name; // lower case + std::string asset; // as the bundle spells it + fb::Sha1 game; // the game's copy it replaces + fb::Sha1 sha1; + std::uint64_t originalSize{}; + std::vector encoded; + std::optional gameFile; // where the game's copy is + }; + std::vector changes; const auto decoded = [&](std::span encoded) { return fb::ebx::read_document(fb::decode_cas(encoded, {gameRoot})); }; @@ -53,13 +67,17 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, const auto modListing = list_bundle(store, mod->directory, baseRoot, bundle, gameRoot); const auto gameListing = list_bundle(store, baseRoot, baseRoot, *shipped->second, gameRoot); if (!modListing || !gameListing) continue; - std::map> original; - for (const auto& asset : gameListing->manifest.ebx) original.emplace(lower(asset.name), asset.sha1); + // The game's copy of each asset: its sha1, and where in the listing it is. + std::map, std::less<>> original; + for (std::size_t index = 0; index < gameListing->manifest.ebx.size(); ++index) { + const auto& asset = gameListing->manifest.ebx[index]; + original.emplace(lower(asset.name), std::pair{asset.sha1, index}); + } for (std::size_t index = 0; index < modListing->manifest.ebx.size(); ++index) { const auto& asset = modListing->manifest.ebx[index]; const auto name = lower(asset.name); const auto game_copy = original.find(name); - if (game_copy != original.end() && game_copy->second == asset.sha1) continue; + if (game_copy != original.end() && game_copy->second.first == asset.sha1) continue; const auto at = modListing->first + index; if (at >= modListing->files.size()) continue; const auto& file = modListing->files[at]; @@ -71,44 +89,25 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, candidates.push_back({lower(bundle.name), {mod->name, asset, payload(), lower(relative)}}); continue; } - const auto encodedPayload = payload(); + auto encodedPayload = payload(); try { for (const auto& reference : decoded(encodedPayload).imports) imported.insert(reference.fileGuid); } catch (const std::exception&) {} - auto& versions = out.changed[name]; - const auto existing = versions.find(game_copy->second); - if (existing == versions.end()) { - versions.emplace(game_copy->second, - AssetOverride{mod->name, asset.sha1, asset.originalSize, encodedPayload}); - ++changed; - } else { - try { - std::size_t baseIndex = 0; - while (baseIndex < gameListing->manifest.ebx.size() && - lower(gameListing->manifest.ebx[baseIndex].name) != name) - ++baseIndex; - if (gameListing->first + baseIndex < gameListing->files.size()) { - const auto& baseFile = gameListing->files[gameListing->first + baseIndex]; - const auto baseBytes = store.read(baseRoot, baseFile.location, baseFile.offset, baseFile.size); - const auto baseDoc = decoded(baseBytes); - const auto existingDoc = decoded(existing->second.encoded); - const auto thisDoc = decoded(encodedPayload); - const std::array editDocs{&existingDoc, &thisDoc}; - fb::ebx::MergeSummary summary; - auto combined = fb::ebx::merge_documents(baseDoc, editDocs, &summary); - if (summary.instances || summary.arrayEntries) { - const auto rebuilt = fb::ebx::write_document(combined); - existing->second.sha1 = sha1_of(rebuilt); - existing->second.originalSize = rebuilt.size(); - existing->second.encoded = fb::encode_cas(rebuilt, {gameRoot}); - existing->second.mod += " + " + mod->name; - for (const auto& reference : combined.imports) - imported.insert(reference.fileGuid); - } - } + // A higher mod's version of the asset, which this change is + // combined with below, may name what this mod adds. + if (const auto versions = out.changed.find(name); versions != out.changed.end()) + if (const auto existing = versions->second.find(game_copy->second.first); + existing != versions->second.end()) try { + for (const auto& reference : decoded(existing->second.encoded).imports) + imported.insert(reference.fileGuid); } catch (const std::exception&) {} - } + std::optional gameFile; + if (const auto gameAt = gameListing->first + game_copy->second.second; + gameAt < gameListing->files.size()) + gameFile = gameListing->files[gameAt]; + changes.push_back({lower(bundle.name), name, asset.name, game_copy->second.first, asset.sha1, + asset.originalSize, std::move(encodedPayload), gameFile}); } // Maps carry stock SkaterLoader too. Replacing it only in the game's // bundles loses custom cosmetic material targets on a map transition. @@ -157,6 +156,9 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, // is, and only one of them can be what a copy gets: say whose, so a song or an // item that goes missing on a map can be traced to the mod that took its name. std::map, std::less<>> shadowed; // by the mod kept + // This mod's added EBX that another mod's different document stands in for: what + // names one by its guid finds nothing in a copy that was given the other's. + std::set lost; const auto keep = [&](const std::string& bundle, AssetAddition addition) { auto& list = out.added[bundle]; const auto name = lower(addition.asset.name); @@ -166,6 +168,8 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, if (holder->mod != addition.mod && holder->asset.sha1 != addition.asset.sha1) { auto& [count, example] = shadowed[holder->mod]; if (!count++) example = addition.asset.name; + if (addition.asset.kind == fb::AssetKind::ebx && holder->file != addition.file) + lost.insert(addition.file); } return false; } @@ -174,23 +178,25 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, }; // Transitively: an addition a following addition imports follows too (a new song // imports its new wave, and only the song is named by the changed playlist). - struct Candidate { fb::Guid file; std::vector imports; bool taken{}; }; - std::vector parsed(candidates.size()); + std::vector followed(candidates.size()); + // The documents this mod added to each bundle, by guid. + std::map, std::less<>> addedIn; for (std::size_t index = 0; index < candidates.size(); ++index) { + auto& addition = candidates[index].second; try { - const auto document = decoded(candidates[index].second.encoded); - parsed[index].file = document.fileGuid; - for (const auto& reference : document.imports) parsed[index].imports.push_back(reference.fileGuid); - } catch (const std::exception&) { parsed[index].taken = true; } // unreadable: never follows + const auto document = decoded(addition.encoded); + addition.file = document.fileGuid; + for (const auto& reference : document.imports) addition.names.push_back(reference.fileGuid); + addedIn[candidates[index].first].insert(addition.file); + } catch (const std::exception&) { followed[index] = true; } // unreadable: never follows } for (bool grew = true; grew;) { grew = false; for (std::size_t index = 0; index < candidates.size(); ++index) { - auto& candidate = parsed[index]; - if (candidate.taken || !imported.contains(candidate.file)) continue; - candidate.taken = grew = true; - imported.insert(candidate.imports.begin(), candidate.imports.end()); auto& [bundle, addition] = candidates[index]; + if (followed[index] || !imported.contains(addition.file)) continue; + followed[index] = grew = true; + imported.insert(addition.names.begin(), addition.names.end()); keep(bundle, std::move(addition)); } } @@ -204,6 +210,65 @@ AssetOverrides collect_asset_overrides(const std::vector& mods, lower(addition.asset.name) == name; })) keep(bundle, std::move(companion)); } + for (auto& change : changes) { + // What the change names of the assets the mod added beside it. The mod + // put them in one bundle with what names them, so they stay together. + std::set names; + const auto beside = addedIn.find(change.bundle); + try { + auto document = decoded(change.encoded); + const auto stays = [&](const fb::ebx::ImportReference& reference) { + return !lost.contains(reference.fileGuid); }; + if (!std::ranges::all_of(document.imports, stays)) + if (const auto gone = fb::ebx::drop_root_references(document, stays)) { + const auto rebuilt = fb::ebx::write_document(document); + change.sha1 = sha1_of(rebuilt); + change.originalSize = rebuilt.size(); + change.encoded = fb::encode_cas(rebuilt, {gameRoot}); + report.notes.push_back(mod->name + ": " + change.asset + ": " + std::to_string(gone) + + " entry(ies) left out of the copies other mods carry, where another mod's asset of " + "the same name stands in for what they name"); + } + if (beside != addedIn.end()) + for (const auto& reference : document.imports) + if (beside->second.contains(reference.fileGuid) && stays(reference)) + names.insert(reference.fileGuid); + } catch (const std::exception&) {} + auto& versions = out.changed[change.name]; + const auto existing = versions.find(change.game); + if (existing == versions.end()) { + versions.emplace(change.game, AssetOverride{mod->name, change.sha1, change.originalSize, + std::move(change.encoded), std::nullopt, std::move(names)}); + ++changed; + continue; + } + // The same change again, from another of the mod's bundles or another mod. + if (existing->second.sha1 == change.sha1) { + existing->second.names.insert(names.begin(), names.end()); + continue; + } + if (!change.gameFile) continue; + try { + const auto baseDoc = decoded(store.read(baseRoot, change.gameFile->location, + change.gameFile->offset, change.gameFile->size)); + const auto existingDoc = decoded(existing->second.encoded); + const auto thisDoc = decoded(change.encoded); + // Lowest priority first, the order a bundle's own copies are combined + // in: where two mods changed one value, the higher mod's is kept. + const std::array editDocs{&thisDoc, &existingDoc}; + fb::ebx::MergeSummary summary; + auto combined = fb::ebx::merge_documents(baseDoc, editDocs, &summary); + // Changed values alone are only worth a copy that has all of them. + if (!summary.instances && !summary.arrayEntries && !(summary.values && summary.exact())) continue; + const auto rebuilt = fb::ebx::write_document(combined); + existing->second.sha1 = sha1_of(rebuilt); + existing->second.originalSize = rebuilt.size(); + existing->second.encoded = fb::encode_cas(rebuilt, {gameRoot}); + if (existing->second.mod != mod->name && !existing->second.mod.ends_with(" + " + mod->name)) + existing->second.mod += " + " + mod->name; + existing->second.names.insert(names.begin(), names.end()); + } catch (const std::exception&) {} + } for (const auto& [other, clash] : shadowed) report.notes.push_back(mod->name + ": " + std::to_string(clash.first) + " added asset(s) share a name with ones " + other + " adds, e.g. " + clash.second + diff --git a/cmake/Apps.cmake b/cmake/Apps.cmake index 355270b..7c77a21 100644 --- a/cmake/Apps.cmake +++ b/cmake/Apps.cmake @@ -68,6 +68,14 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32) target_link_libraries(dingosdk_item_list_merge_tests PRIVATE dingosdk_mods) target_include_directories(dingosdk_item_list_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}") add_test(NAME item_list_merge COMMAND dingosdk_item_list_merge_tests "${DINGOSDK_TEST_GAME_ROOT}") + add_executable(dingosdk_ebx_merge_tests Engine/Vfs/Test/ebx_merge_tests.cpp) + target_link_libraries(dingosdk_ebx_merge_tests PRIVATE dingosdk_mods) + target_include_directories(dingosdk_ebx_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME ebx_merge COMMAND dingosdk_ebx_merge_tests) + add_executable(dingosdk_mod_merge_audit_tests Engine/Vfs/Test/mod_merge_audit_tests.cpp) + target_link_libraries(dingosdk_mod_merge_audit_tests PRIVATE dingosdk_mods) + target_include_directories(dingosdk_mod_merge_audit_tests PRIVATE "${PROJECT_SOURCE_DIR}") + add_test(NAME mod_merge_audit COMMAND dingosdk_mod_merge_audit_tests "${DINGOSDK_TEST_GAME_ROOT}") add_executable(dingosdk_store_copies_tests Engine/Vfs/Test/store_copies_tests.cpp) target_link_libraries(dingosdk_store_copies_tests PRIVATE dingosdk_mods dingosdk_content_cache) target_include_directories(dingosdk_store_copies_tests PRIVATE "${PROJECT_SOURCE_DIR}")