DieMitchell/ReSkate
1
0 Fork 0
mirror of https://github.com/Dingo-Shenanigans/ReSkate.git synced 2026-10-17 17:27:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

carry added assets with a changed list, combine in-place values, list repeats once

Browse Source 
Merge: carry added assets with a changed list, combine in-place values, list repeats once

Three merge bugs found by auditing 104 installed mods:

- A changed list (the music playlist) reached every bundle copy that holds
  it, but the songs a mod added beside it only reached copies of the bundle
  the mod added them to. The game keeps the playlist in eight bundles, so
  the other seven named songs they did not hold. Added assets now follow
  the change into whichever bundle copy takes it.
- An entry naming an asset that lost its name to another mod's asset is
  now dropped from the list, both in the merged bundle and in the list
  other mods' copies are given.
- Two mods changing values in place in one asset (lighting) kept one mod's
  copy whole and silently dropped the other's. Plain values are now
  combined, the higher-priority mod winning where both changed one. An
  edit that also moved a reference or resized a list is taken whole or not
  at all, and the merge notes whose changes are missing.
- A reference two mods both append to a list is listed once.
This commit is contained in:
AyeZeeBB committed 2026-10-05 18:53:51 -05:00
1 parent a9742673da
commit 137273e274
10 files changed
+1963 -85
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Engine/Resource/ebx_merge.cpp
+262 -1
View File
@@ -5,12 +5,16 @@
#include <cstdint>
#include <algorithm>
#include <bit>
#include <cstring>
#include <map>
#include <memory>
#include <set>
#include <stdexcept>
#include <string>
#include <utility>
#include <variant>
#include <vector>
namespace dingosdk::frostbite::ebx {
namespace {
@@ -81,6 +85,170 @@ const Value::Array* root_array(const Document& document, std::string_view name)
return nullptr;
}
// A type's name, which is what two documents that number their types
// independently have in common. Empty for a descriptor that names none.
std::string type_or_none(const Document& document, std::int32_t descriptor) {
return descriptor >= 0 && static_cast<std::size_t>(descriptor) < document.types.size()
? document.types[static_cast<std::size_t>(descriptor)].name : std::string{};
}
constexpr auto unmatched = static_cast<std::size_t>(-1);
// Which of an edit's instances each of the base's is: the root is the root, an
// exported one is found by its guid, and an internal one, which has nothing
// else to go by, by its place among the internal ones. An edit that added or
// dropped an internal instance has moved the rest along, so then none of them
// is matched. Two that are not of one type are not a match either.
std::vector<std::size_t> match_instances(const Document& base, const Document& edit) {
std::vector<std::size_t> match(base.instances.size(), unmatched);
if (base.instances.empty() || edit.instances.empty()) return match;
match[0] = 0;
std::map<Guid, std::size_t> exported;
std::vector<std::size_t> internal, baseInternal;
for (std::size_t index = 1; index < edit.instances.size(); ++index) {
if (edit.instances[index].exported) exported.emplace(edit.instances[index].instanceGuid, index);
else internal.push_back(index);
}
for (std::size_t index = 1; index < base.instances.size(); ++index) {
if (!base.instances[index].exported) { baseInternal.push_back(index); continue; }
if (const auto found = exported.find(base.instances[index].instanceGuid); found != exported.end())
match[index] = found->second;
}
if (internal.size() == baseInternal.size())
for (std::size_t at = 0; at < internal.size(); ++at) match[baseInternal[at]] = internal[at];
for (std::size_t index = 0; index < match.size(); ++index)
if (match[index] != unmatched && type_or_none(base, base.instances[index].descriptor) !=
type_or_none(edit, edit.instances[match[index]].descriptor))
match[index] = unmatched;
return match;
}
bool same(double left, double right) { return std::bit_cast<std::uint64_t>(left) == std::bit_cast<std::uint64_t>(right); }
bool same(const ResourceReference& left, const ResourceReference& right) { return left.id == right.id; }
template <typename Type> bool same(const Type& left, const Type& right) { return left == right; }
// Finds what one edit changed in place. The base and the edit are walked side
// by side, along with the result: it began as a copy of the base, so it has the
// same shape, and may already hold what an earlier edit changed. Nothing is
// written until the whole edit has been seen, so an edit either has all its
// values taken or none of them.
class InPlace final {
public:
InPlace(const Document& base, const Document& edit, const std::vector<std::size_t>& match)
: base_(base), edit_(edit), match_(match) {}
void fields(const Object& base, const Object& edit, Object& result, bool root) {
if (base.fields.size() != edit.fields.size() || base.fields.size() != result.fields.size()) { uncarried = true; return; }
for (std::size_t index = 0; index < base.fields.size(); ++index) {
if (base.fields[index].name != edit.fields[index].name) { uncarried = true; return; }
value(base.fields[index].value, edit.fields[index].value, result.fields[index].value, root);
}
}
void apply() {
for (const auto& [to, from] : changes_) to->data = from->data;
}
std::size_t values{}; // values the edit changed from the base
std::size_t contested{}; // of those, ones the result already had changed to something else
bool uncarried{}; // a change was seen that cannot be taken
bool unread{}; // somewhere a change could not be looked for
private:
template <typename Type> bool plain(const Value& base, const Value& edit, Value& result) {
const auto* before = std::get_if<Type>(&base.data);
if (!before) return false;
const auto& after = std::get<Type>(edit.data);
if (same(*before, after)) return true;
++values;
const auto& current = std::get<Type>(result.data);
if (same(current, after)) return true;
if (!same(current, *before)) ++contested;
changes_.emplace_back(&result, &edit);
return true;
}
void value(const Value& base, const Value& edit, Value& result, bool rootList) {
if (base.data.index() != edit.data.index() || base.data.index() != result.data.index()) { uncarried = true; return; }
if (std::holds_alternative<std::monostate>(base.data)) return;
if (plain<bool>(base, edit, result) || plain<std::int64_t>(base, edit, result) ||
plain<std::uint64_t>(base, edit, result) || plain<double>(base, edit, result) ||
plain<std::string>(base, edit, result) || plain<Guid>(base, edit, result) ||
plain<Sha1>(base, edit, result) || plain<ResourceReference>(base, edit, result)) return;
if (const auto* before = std::get_if<PointerReference>(&base.data)) {
reference(*before, std::get<PointerReference>(edit.data));
return;
}
if (const auto* before = std::get_if<std::shared_ptr<Object>>(&base.data)) {
const auto& after = std::get<std::shared_ptr<Object>>(edit.data);
const auto& mine = std::get<std::shared_ptr<Object>>(result.data);
if (!*before && !after) return;
if (!*before || !after || !mine) { uncarried = true; return; }
fields(**before, *after, *mine, false);
return;
}
if (const auto* before = std::get_if<Value::Array>(&base.data)) {
const auto& after = std::get<Value::Array>(edit.data);
auto& mine = std::get<Value::Array>(result.data);
// Only a list on the root grows at its end, and what was appended
// comes across on its own. A list that shrank, or one deeper in
// that changed length, has entries no longer told apart by place.
if (rootList ? after.size() < before->size() : after.size() != before->size()) { uncarried = true; return; }
if (mine.size() < before->size()) { uncarried = true; return; }
for (std::size_t index = 0; index < before->size(); ++index)
value((*before)[index], after[index], mine[index], false);
return;
}
if (const auto* before = std::get_if<TypeReference>(&base.data)) {
const auto& after = std::get<TypeReference>(edit.data);
if (before->primitive != after.primitive) uncarried = true;
else if (before->primitive) uncarried |= before->primitiveType != after.primitiveType;
else uncarried |= type_or_none(base_, before->descriptor) != type_or_none(edit_, after.descriptor);
return;
}
if (const auto* before = std::get_if<BoxedReference>(&base.data)) {
const auto& after = std::get<BoxedReference>(edit.data);
// What a boxed value holds is not read, so a change inside one is not seen.
if ((before->encodedType == 0) != (after.encodedType == 0)) uncarried = true;
else if (before->encodedType != 0) unread = true;
}
}
// A reference still has to lead to the same place; one pointed elsewhere
// cannot be taken, because what it points at may not be in the result.
void reference(const PointerReference& before, const PointerReference& after) {
if (before.kind != after.kind) { uncarried = true; return; }
if (before.kind == PointerKind::null) return;
if (before.index < 0 || after.index < 0) { uncarried |= before.index != after.index; return; }
const auto from = static_cast<std::size_t>(before.index), to = static_cast<std::size_t>(after.index);
if (before.kind == PointerKind::external) {
uncarried |= from >= base_.imports.size() || to >= edit_.imports.size() ||
base_.imports[from].fileGuid != edit_.imports[to].fileGuid ||
base_.imports[from].classGuid != edit_.imports[to].classGuid;
return;
}
if (from >= match_.size()) { uncarried = true; return; }
// An internal instance that could not be matched is not known to have moved.
if (match_[from] == unmatched) (base_.instances[from].exported ? uncarried : unread) = true;
else uncarried |= match_[from] != to;
}
const Document& base_;
const Document& edit_;
const std::vector<std::size_t>& match_;
std::vector<std::pair<Value*, const Value*>> changes_;
};
template <typename Visit> void each_pointer(Value& value, const Visit& visit) {
if (auto* pointer = std::get_if<PointerReference>(&value.data)) { visit(*pointer); return; }
if (auto* nested = std::get_if<std::shared_ptr<Object>>(&value.data)) {
if (*nested) for (auto& field : (*nested)->fields) each_pointer(field.value, visit);
return;
}
if (auto* list = std::get_if<Value::Array>(&value.data))
for (auto& element : *list) each_pointer(element, visit);
}
} // namespace
namespace detail {
@@ -302,12 +470,44 @@ Document merge_documents(const Document& base, const std::span<const Document* c
baseLengths.emplace(field.name, array->size());
MergeSummary totals;
// The references the edits so far appended to each root array.
std::map<std::string, std::set<std::pair<PointerKind, std::int32_t>>, std::less<>> appended;
for (std::size_t index = 0; index < edits.size(); ++index) {
const auto* edit = edits[index];
if (!edit) continue;
const auto* editRoot = edit->root();
if (!editRoot || !editRoot->object) throw std::runtime_error("EBX edit has no root object");
// What the edit changed in place. The result's first instances are still
// the base's, in the base's order: carried ones are added behind them,
// and nothing is sorted until every edit is in.
const auto match = match_instances(base, *edit);
InPlace changes(base, *edit, match);
for (std::size_t at = 0; at < base.instances.size(); ++at) {
const auto& before = base.instances[at];
if (match[at] == unmatched) {
// An exported instance is found whenever the edit still has it.
(before.exported ? changes.uncarried : changes.unread) = true;
continue;
}
const auto& after = edit->instances[match[at]];
if (!before.object || !after.object || !result.instances[at].object) {
changes.unread |= before.object || after.object;
continue;
}
changes.fields(*before.object, *after.object, *result.instances[at].object, at == 0);
}
if (changes.uncarried) {
totals.uncarried.push_back(index);
} else {
changes.apply();
totals.values += changes.values;
totals.contested += changes.contested;
}
if (changes.unread) totals.unread.push_back(index);
detail::Carrier mapper(*edit, result);
std::map<std::string, std::set<std::pair<PointerKind, std::int32_t>>, std::less<>> mine;
// Whatever an edit appended to one of the root's arrays comes across,
// pulling in the instances those entries point at.
@@ -331,10 +531,24 @@ Document merge_documents(const Document& base, const std::span<const Document* c
continue;
}
}
target->push_back(mapper.value(entry));
auto carried = mapper.value(entry);
// Two edits that append the same thing mean it once. What one edit
// lists twice on its own is left as its author wrote it.
if (const auto* reference = std::get_if<PointerReference>(&carried.data);
reference && reference->kind != PointerKind::null) {
const std::pair key{reference->kind, reference->index};
if (const auto earlier = appended.find(field.name);
earlier != appended.end() && earlier->second.contains(key)) {
++totals.repeated;
continue;
}
mine[field.name].insert(key);
}
target->push_back(std::move(carried));
++totals.arrayEntries;
}
}
for (auto& [name, references] : mine) appended[name].merge(references);
// An instance an edit added but never linked from the root still has to
// exist, or the asset it belongs to goes missing.
@@ -357,4 +571,51 @@ Document merge_documents(const Document& base, const std::span<const Document* c
return result;
}
std::size_t drop_root_references(Document& document, const CarryImport& keep) {
auto* root = const_cast<InstanceRecord*>(document.root());
if (!keep || !root || !root->object) return 0;
std::size_t dropped{};
for (auto& field : root->object->fields) {
auto* array = std::get_if<Value::Array>(&field.value.data);
if (!array) continue;
dropped += std::erase_if(*array, [&](const Value& entry) {
const auto* pointer = std::get_if<PointerReference>(&entry.data);
return pointer && pointer->kind == PointerKind::external && pointer->index >= 0 &&
static_cast<std::size_t>(pointer->index) < document.imports.size() &&
!keep(document.imports[static_cast<std::size_t>(pointer->index)]);
});
}
// A boxed value is kept as the bytes it was read as, import numbers and
// all, so a document with one keeps its imports where they are.
if (!dropped || !document.boxedValues.empty()) return dropped;
std::vector<bool> used(document.imports.size());
for (auto& instance : document.instances)
if (instance.object)
for (auto& field : instance.object->fields)
each_pointer(field.value, [&](const PointerReference& pointer) {
if (pointer.kind == PointerKind::external && pointer.index >= 0 &&
static_cast<std::size_t>(pointer.index) < used.size())
used[static_cast<std::size_t>(pointer.index)] = true;
});
std::vector<std::int32_t> moveTo(used.size(), -1);
std::vector<ImportReference> kept;
for (std::size_t index = 0; index < used.size(); ++index) {
if (!used[index]) continue;
moveTo[index] = static_cast<std::int32_t>(kept.size());
kept.push_back(document.imports[index]);
}
if (kept.size() == document.imports.size()) return dropped;
for (auto& instance : document.instances)
if (instance.object)
for (auto& field : instance.object->fields)
each_pointer(field.value, [&](PointerReference& pointer) {
if (pointer.kind == PointerKind::external && pointer.index >= 0 &&
static_cast<std::size_t>(pointer.index) < moveTo.size())
pointer.index = moveTo[static_cast<std::size_t>(pointer.index)];
});
document.imports = std::move(kept);
return dropped;
}
} // namespace dingosdk::frostbite::ebx
Engine/Resource/ebx_merge.h
+23 -4
View File
@@ -19,6 +19,18 @@ struct MergeSummary {
ImportReference target;
};
std::vector<Dropped> dropped;
std::size_t repeated{}; // appended references an earlier entry had already added, left out
std::size_t values{}; // plain values an edit changed in place, taken from it
std::size_t contested{}; // of those, values an earlier edit had changed to something else
// Edits (indices into `edits`) with a change that did not come across: a reference
// pointed somewhere else, a list resized anywhere but at the root's end, an instance gone.
std::vector<std::size_t> uncarried;
// Edits whose in-place changes could not all be looked for, because their internal
// instances do not line up with the base's or the document holds boxed values.
std::vector<std::size_t> unread;
// Every edit's in-place changes are in the result, as far as anyone can tell.
[[nodiscard]] bool exact() const noexcept { return uncarried.empty() && unread.empty(); }
};
// Whether a root array entry an edit appended comes across, asked for an entry
@@ -27,13 +39,20 @@ struct MergeSummary {
using CarryImport = std::function<bool(const ImportReference&)>;
// Combines several independently edited copies of one EBX asset with the base
// they were all derived from. Each edit contributes the instances it added and
// the entries it appended to the root's arrays; anything an edit merely changed
// in place stays as the base had it, because two edits to one value cannot both
// be honoured. Edits are applied in the order given, so the last one wins a tie.
// they were all derived from. Each edit contributes the instances it added, the
// entries it appended to the root's arrays (a reference two edits both append is
// listed once), and the plain values it changed in place: a number, a string, a
// guid. Two edits to one value cannot both be honoured, so edits are applied in
// the order given and the last one wins. A change of any other shape (a
// reference pointed elsewhere, a list resized in the middle of the document)
// stays as the base had it, and the summary names the edit it was in.
[[nodiscard]] Document merge_documents(const Document& base,
std::span<const Document* const> edits,
MergeSummary* summary = nullptr,
const CarryImport& carry = {});
// Takes out the root array entries that refer to a document `keep` turns down,
// and the imports nothing refers to afterwards. Returns how many entries went.
std::size_t drop_root_references(Document& document, const CarryImport& keep);
} // namespace dingosdk::frostbite::ebx
Engine/Vfs/Test/ebx_merge_tests.cpp
+280
View File
@@ -0,0 +1,280 @@
// The document merge, on documents made in memory, so it needs no game files.
//
// Several mods each ship their own copy of one of the game's assets. merge_documents puts the
// copies together over the game's: what each added, and what each changed in place. The cases:
// * values two edits changed in place both arrive, and one both changed is the later edit's;
// * an edit that also changed something that cannot be taken (a reference pointed elsewhere, a
// list resized) has none of its values taken, and the summary names it;
// * an exported instance is found by its guid wherever the edit keeps it, and internal
// instances that do not line up are reported as not looked at;
// * a reference two edits both append is listed once, and one edit's own repeat is left alone;
// * drop_root_references takes entries out of the root's lists with the imports they used.
#include "Engine/Resource/ebx_carry.h"
#include "Engine/Resource/ebx_merge.h"
#include <array>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
namespace ebx = dingosdk::frostbite::ebx;
using dingosdk::frostbite::Guid;
namespace {
int failures = 0;
void expect(bool ok, const std::string& what) {
if (ok) return;
std::cerr << "FAIL: " << what << '\n';
++failures;
}
Guid guid(std::uint8_t seed) {
Guid value;
value.bytes.fill(static_cast<std::byte>(seed));
return value;
}
template <typename Type> ebx::Value value_of(Type data) {
ebx::Value value;
value.data = std::move(data);
return value;
}
ebx::Value number(std::int64_t value) { return value_of(value); }
ebx::Value real(double value) { return value_of(value); }
ebx::Value text(std::string value) { return value_of(std::move(value)); }
// A reference to another document, by the place of its import, and one to an instance of this one.
ebx::Value outside(std::int32_t import) { return value_of(ebx::PointerReference{ebx::PointerKind::external, import}); }
ebx::Value inside(std::int32_t instance) { return value_of(ebx::PointerReference{ebx::PointerKind::internal, instance}); }
ebx::Value list(ebx::Value::Array entries) { return value_of(std::move(entries)); }
std::shared_ptr<ebx::Object> object(std::int32_t type, std::vector<std::pair<std::string, ebx::Value>> fields) {
auto result = std::make_shared<ebx::Object>();
result->descriptor = type;
for (auto& [name, value] : fields) result->fields.push_back({result->fields.size(), std::move(name), std::move(value)});
return result;
}
ebx::InstanceRecord instance(std::int32_t type, std::shared_ptr<ebx::Object> fields, const Guid* exported = nullptr) {
ebx::InstanceRecord record;
record.descriptor = type;
record.exported = exported != nullptr;
if (exported) record.instanceGuid = *exported;
record.object = std::move(fields);
return record;
}
constexpr std::int32_t list_type = 0, entry_type = 1, detail_type = 2;
const Guid entry_guid = guid(0xA1);
// The game's asset: a root with plain values, a list of two other documents and a nested
// structure; an exported instance the root points at; and an internal one that one points at.
ebx::Document game() {
ebx::Document document;
document.fileGuid = guid(1);
document.rootType = "ListAsset";
document.types.resize(3);
document.types[list_type].name = "ListAsset";
document.types[entry_type].name = "Entry";
document.types[detail_type].name = "Detail";
document.imports = {{guid(0x10), guid(0xC0)}, {guid(0x11), guid(0xC0)}};
document.instances.push_back(instance(list_type, object(list_type, {
{"Volume", number(5)}, {"Gain", real(1.0)}, {"Title", text("game")},
{"Items", list({outside(0), outside(1)})}, {"First", inside(1)},
{"Tuning", value_of(object(detail_type, {{"Weights", list({number(1), number(2)})}, {"Scale", real(2.0)}}))}})));
document.instances.push_back(instance(entry_type, object(entry_type, {{"Cost", number(10)}, {"Detail", inside(2)}}), &entry_guid));
document.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(3)}})));
return document;
}
ebx::Document copy(const ebx::Document& document) { return ebx::detail::clone_document(document); }
ebx::Value& field(ebx::Object& from, std::string_view name) {
for (auto& entry : from.fields)
if (entry.name == name) return entry.value;
throw std::runtime_error("no field " + std::string(name));
}
ebx::Value& field(ebx::Document& document, std::size_t at, std::string_view name) { return field(*document.instances.at(at).object, name); }
template <typename Type> const Type& read(const ebx::Document& document, std::size_t at, std::string_view name) {
return std::get<Type>(field(*document.instances.at(at).object, name).data);
}
ebx::Object& tuning(ebx::Document& document) { return *std::get<std::shared_ptr<ebx::Object>>(field(document, 0, "Tuning").data); }
ebx::Value::Array& items(ebx::Document& document) { return std::get<ebx::Value::Array>(field(document, 0, "Items").data); }
// The documents a list names, in order.
std::vector<Guid> named(const ebx::Document& document) {
std::vector<Guid> result;
for (const auto& entry : std::get<ebx::Value::Array>(field(*document.instances.front().object, "Items").data))
result.push_back(document.imports.at(static_cast<std::size_t>(std::get<ebx::PointerReference>(entry.data).index)).fileGuid);
return result;
}
void append(ebx::Document& document, const Guid& target) {
document.imports.push_back({target, guid(0xC0)});
items(document).push_back(outside(static_cast<std::int32_t>(document.imports.size() - 1)));
}
// Where the exported instance with this guid is.
std::size_t find(const ebx::Document& document, const Guid& wanted) {
for (std::size_t index = 0; index < document.instances.size(); ++index)
if (document.instances[index].exported && document.instances[index].instanceGuid == wanted) return index;
throw std::runtime_error("no such exported instance");
}
ebx::Document merge(const ebx::Document& base, std::initializer_list<const ebx::Document*> edits, ebx::MergeSummary& summary) {
const std::vector<const ebx::Document*> documents(edits);
return ebx::merge_documents(base, documents, &summary);
}
void values_from_every_edit() {
const auto base = game();
auto first = copy(base), second = copy(base);
field(first, 0, "Volume") = number(9);
field(first, 0, "Title") = text("first");
field(second, 0, "Gain") = real(0.5);
field(tuning(second), "Scale") = real(4.0);
std::get<ebx::Value::Array>(field(tuning(second), "Weights").data)[1] = number(7);
field(second, 1, "Cost") = number(20);
field(second, 2, "Depth") = number(8);
ebx::MergeSummary summary;
auto merged = merge(base, {&first, &second}, summary);
expect(read<std::int64_t>(merged, 0, "Volume") == 9 && read<std::string>(merged, 0, "Title") == "first",
"values: the first edit's values are in the result");
expect(read<double>(merged, 0, "Gain") == 0.5 && std::get<double>(field(tuning(merged), "Scale").data) == 4.0 &&
std::get<std::int64_t>(std::get<ebx::Value::Array>(field(tuning(merged), "Weights").data)[1].data) == 7,
"values: the second edit's values are in the result, the nested ones too");
expect(read<std::int64_t>(merged, 1, "Cost") == 20 && read<std::int64_t>(merged, 2, "Depth") == 8,
"values: values in an exported and in an internal instance are in the result");
expect(summary.values == 7 && summary.contested == 0 && summary.exact() && !summary.instances && !summary.arrayEntries,
"values: seven values taken, none disagreed, nothing added (" + std::to_string(summary.values) + " taken)");
expect(named(merged) == named(base) && merged.instances.size() == base.instances.size(), "values: nothing else moved");
}
void one_value_from_two_edits(bool same) {
const std::string label = same ? "same value twice" : "contested value";
const auto base = game();
auto lower = copy(base), higher = copy(base);
field(lower, 0, "Volume") = number(9);
field(higher, 0, "Volume") = number(same ? 9 : 11);
ebx::MergeSummary summary;
const auto merged = merge(base, {&lower, &higher}, summary);
expect(read<std::int64_t>(merged, 0, "Volume") == (same ? 9 : 11), label + ": the later edit's value is the one kept");
expect(summary.values == 2 && summary.contested == (same ? 0u : 1u) && summary.exact(),
label + ": " + (same ? "no" : "one") + " disagreement is counted");
}
// What cannot be taken: the edit that made such a change keeps all its values out, so the result
// never has half of what a mod did to the game's own content.
void a_change_that_cannot_be_taken(int kind) {
static constexpr std::array labels{"retargeted reference", "shorter root list", "resized nested list", "instance gone"};
const std::string label = labels[static_cast<std::size_t>(kind)];
const auto base = game();
auto reshaped = copy(base), plain = copy(base);
field(reshaped, 0, "Volume") = number(9);
if (kind == 0) items(reshaped)[0] = outside(1);
else if (kind == 1) items(reshaped).pop_back();
else if (kind == 2) std::get<ebx::Value::Array>(field(tuning(reshaped), "Weights").data).push_back(number(3));
else {
// The exported instance is not in the edit: the root points at the internal one instead.
reshaped.instances.erase(reshaped.instances.begin() + 1);
field(reshaped, 0, "First") = inside(1);
}
field(plain, 0, "Title") = text("plain");
ebx::MergeSummary summary;
const auto merged = merge(base, {&reshaped, &plain}, summary);
expect(read<std::int64_t>(merged, 0, "Volume") == 5, label + ": none of that edit's values are taken");
expect(read<std::string>(merged, 0, "Title") == "plain", label + ": the other edit's value is");
expect(summary.uncarried == std::vector<std::size_t>{0} && !summary.exact() && summary.values == 1,
label + ": the summary names the edit");
expect(named(merged) == named(base), label + ": the game's list is as it was");
}
// An edit that added an exported instance has moved the game's along; they are still found.
void instances_found_by_guid() {
const auto base = game();
auto edit = copy(base);
const auto added = guid(0x05); // sorts ahead of the game's
edit.instances.insert(edit.instances.begin() + 1,
instance(entry_type, object(entry_type, {{"Cost", number(1)}, {"Detail", value_of(ebx::PointerReference{})}}), &added));
field(edit, 0, "First") = inside(2);
field(edit, 2, "Detail") = inside(3);
field(edit, 2, "Cost") = number(30);
ebx::MergeSummary summary;
const auto merged = merge(base, {&edit}, summary);
expect(summary.uncarried.empty() && summary.unread.empty() && summary.values == 1 && summary.instances == 1,
"by guid: one value taken and one instance added, nothing left behind");
expect(read<std::int64_t>(merged, find(merged, entry_guid), "Cost") == 30, "by guid: the game's instance has the edit's value");
expect(read<std::int64_t>(merged, find(merged, added), "Cost") == 1, "by guid: the added instance came across");
expect(find(merged, added) < find(merged, entry_guid), "by guid: exported instances are in guid order");
expect(static_cast<std::size_t>(read<ebx::PointerReference>(merged, 0, "First").index) == find(merged, entry_guid),
"by guid: the root still points at the game's instance");
}
// An internal instance has no guid. With one more of them in the edit the places say nothing,
// so they are not compared, and the summary says the edit was not looked at in full.
void internal_instances_that_do_not_line_up() {
const auto base = game();
auto edit = copy(base);
edit.instances.push_back(instance(detail_type, object(detail_type, {{"Depth", number(4)}})));
field(edit, 0, "Volume") = number(9);
ebx::MergeSummary summary;
const auto merged = merge(base, {&edit}, summary);
expect(read<std::int64_t>(merged, 0, "Volume") == 9, "internal: the value on the root is taken");
expect(summary.uncarried.empty() && summary.unread == std::vector<std::size_t>{0} && !summary.exact(),
"internal: the edit is reported as not looked at in full");
}
void appended_references() {
const auto base = game();
auto first = copy(base), second = copy(base), third = copy(base);
append(first, guid(0x20));
append(first, guid(0x21));
append(second, guid(0x21)); // the first edit's as well
append(second, guid(0x22));
append(third, guid(0x23)); // twice in one edit: its author's doing
append(third, guid(0x23));
ebx::MergeSummary summary;
const auto merged = merge(base, {&first, &second, &third}, summary);
const std::vector<Guid> expected{guid(0x10), guid(0x11), guid(0x20), guid(0x21), guid(0x22), guid(0x23), guid(0x23)};
expect(named(merged) == expected, "appended: each reference once, in the order of the edits, one edit's own repeat kept (" +
std::to_string(named(merged).size()) + " entries)");
expect(summary.arrayEntries == 5 && summary.repeated == 1 && summary.exact(), "appended: five entries taken, one repeat left out");
}
void dropped_references() {
auto document = game();
append(document, guid(0x20));
append(document, guid(0x21));
const auto gone = ebx::drop_root_references(document, [](const ebx::ImportReference& reference) {
return reference.fileGuid != guid(0x11) && reference.fileGuid != guid(0x20); });
expect(gone == 2, "dropped: two entries went (" + std::to_string(gone) + ")");
expect((named(document) == std::vector<Guid>{guid(0x10), guid(0x21)}), "dropped: the others are left, in order");
expect(document.imports.size() == 2, "dropped: the imports nothing uses went with them (" +
std::to_string(document.imports.size()) + " left)");
// A boxed value is kept as bytes with its import numbers in them, so the imports stay put.
auto boxed = game();
boxed.boxedValues.emplace_back();
expect(ebx::drop_root_references(boxed, [](const ebx::ImportReference& reference) { return reference.fileGuid != guid(0x10); }) == 1 &&
boxed.imports.size() == 2 && named(boxed) == std::vector<Guid>{guid(0x11)},
"dropped: a document with a boxed value keeps its imports where they are");
auto untouched = game();
expect(ebx::drop_root_references(untouched, [](const ebx::ImportReference&) { return true; }) == 0 &&
untouched.imports.size() == 2, "dropped: nothing goes when every entry is kept");
}
} // namespace
int main() try {
values_from_every_edit();
one_value_from_two_edits(false);
one_value_from_two_edits(true);
for (int kind = 0; kind < 4; ++kind) a_change_that_cannot_be_taken(kind);
instances_found_by_guid();
internal_instances_that_do_not_line_up();
appended_references();
dropped_references();
if (failures) {
std::cerr << failures << " failure(s)\n";
return 1;
}
std::cout << "ebx merge: ok\n";
return 0;
} catch (const std::exception& error) {
std::cerr << "FAIL: " << error.what() << '\n';
return 1;
}
Engine/Vfs/Test/item_list_merge_tests.cpp
+185 -1
View File
@@ -11,6 +11,12 @@
// one mod ships is not merged at all, it passes through as that mod built it, so every case
// here has two mods shipping the shared bundle.)
//
// The same mods also show what else the merge does with two copies of one of the game's assets:
// * an item two mods both list is in the merged list once;
// * values two mods changed in place in one asset are both in the merged asset, and where that
// cannot be done the merge keeps one mod's copy whole and says whose changes are missing;
// * the list other mods' copies of the bundle are given leaves out an item that is not in them.
//
// The mods are made here from the installed game's own list and one of its items.
// Arguments: <Skate folder> [<folder>]; skipped when the game is not there. With a second
// argument the two mods of the first case are also left in that folder, stamped for this game,
@@ -28,7 +34,9 @@
#include <Windows.h>
#include <algorithm>
#include <functional>
#include <iostream>
#include <map>
#include <stdexcept>
namespace fs = std::filesystem;
@@ -140,6 +148,8 @@ struct Game {
std::string listName;
fb::ebx::Document list;
std::size_t listed{};
std::size_t itemIndex{};
std::string itemName;
fb::ebx::Document item;
};
Game read_game(const fs::path& root, const CasStore& store) {
@@ -162,6 +172,8 @@ Game read_game(const fs::path& root, const CasStore& store) {
game.list = std::move(document);
} else if (!item && type.find("itemasset") != std::string::npos) {
item = true;
game.itemIndex = index;
game.itemName = assets[index].name;
game.item = std::move(document);
}
}
@@ -172,8 +184,9 @@ Game read_game(const fs::path& root, const CasStore& store) {
// A cosmetic mod's folder: its copy of the shared bundle in items.toc (the game's assets, the
// item list replaced and its items added) and one archive holding the manifest and those payloads.
// With no list the mod leaves the game's as it is, as a mod does whose items go in another one.
// `changed` replaces assets of the game's, by their place in the bundle.
void write_mod(const fs::path& directory, const CasStore& store, const Game& game, const Bytes* list,
std::initializer_list<Item> items) {
std::initializer_list<Item> items, const std::map<std::size_t, Bytes>& changed = {}) {
auto manifest = game.shared.listing.manifest;
const auto& shipped = game.shared.listing.files;
std::vector<fb::BundleFileInfo> files(shipped.begin() + static_cast<std::ptrdiff_t>(game.shared.listing.first), shipped.end());
@@ -191,6 +204,11 @@ void write_mod(const fs::path& directory, const CasStore& store, const Game& gam
manifest.ebx[game.listIndex].originalSize = list->size();
files[game.listIndex] = stored(encoded(*list));
}
for (const auto& [index, payload] : changed) {
manifest.ebx[index].sha1 = sha1_of(payload);
manifest.ebx[index].originalSize = payload.size();
files[index] = stored(encoded(payload));
}
for (const auto& item : items) {
fb::BundleAsset asset;
asset.kind = fb::AssetKind::ebx;
@@ -365,6 +383,167 @@ void one_mod_changes_the_list(const Game& game, const CasStore& store, bool miss
expect(said == missing, label + (missing ? ": the merge says which item was left out\n" : ": nothing is reported\n") +
describe(report));
}
// Two packs that both carry one item, each with an item of its own besides. Both list the shared
// one, and it is one item: the merged list has it once.
void an_item_two_mods_both_list(const Game& game, const CasStore& store) {
Merge merge(game, "listed-twice");
const auto both = item_from(game.item, "items/reskate_test/own_in_both", 0x10);
const auto first = item_from(game.item, "items/reskate_test/own_first", 0x20);
const auto second = item_from(game.item, "items/reskate_test/own_second", 0x30);
const auto firstList = list_with(game.list, {both.identity, first.identity});
const auto secondList = list_with(game.list, {both.identity, second.identity});
write_mod(merge.add("first"), store, game, &firstList, {both, first});
write_mod(merge.add("second"), store, game, &secondList, {both, second});
const auto report = mods::merge_mods(merge.catalog);
const auto list = merged_list(merge, game, report, "listed by both", 3);
expect(names(list, both) && names(list, first) && names(list, second), "listed by both: the list has all three items");
expect(std::ranges::count(list, both.identity.fileGuid) == 1, "listed by both: the item both mods list is in it once");
expect(noted(report, game.listName + ": combined 2 edits (0 instance(s), 3 list entries, 1 repeated entry(ies) listed once)"),
"listed by both: the merge says one entry was a repeat\n" + describe(report));
}
// The game's item as a mod that retunes it ships it: the same document, changed in place.
Bytes item_with(const fb::ebx::Document& game, const std::function<void(fb::ebx::Object&)>& change) {
auto item = fb::ebx::detail::clone_document(game);
change(*item.instances.front().object);
return fb::ebx::write_document(item);
}
fb::ebx::Value& field(fb::ebx::Object& object, std::string_view name) {
for (auto& entry : object.fields)
if (entry.name == name) return entry.value;
throw std::runtime_error("the game's item has no " + std::string(name));
}
void set_hash(fb::ebx::Object& object, std::uint32_t hash) {
auto& value = field(object, "HashedAssetKey");
if (std::holds_alternative<std::int64_t>(value.data)) value.data = static_cast<std::int64_t>(hash);
else value.data = static_cast<std::uint64_t>(hash);
}
std::uint32_t hash_of(const fb::ebx::Object& object) {
const auto& value = field(const_cast<fb::ebx::Object&>(object), "HashedAssetKey");
const auto* number = std::get_if<std::int64_t>(&value.data);
return number ? static_cast<std::uint32_t>(*number) : static_cast<std::uint32_t>(std::get<std::uint64_t>(value.data));
}
// A change the merge cannot take value by value: the first list of the item made one shorter,
// or failing that its first reference cleared. False when the item has neither.
bool reshape(fb::ebx::Object& object) {
for (auto& entry : object.fields)
if (auto* list = std::get_if<fb::ebx::Value::Array>(&entry.value.data); list && !list->empty()) {
list->pop_back();
return true;
}
for (auto& entry : object.fields)
if (auto* pointer = std::get_if<fb::ebx::PointerReference>(&entry.value.data);
pointer && pointer->kind != fb::ebx::PointerKind::null) {
*pointer = {};
return true;
}
return false;
}
// The merged shared bundle's copy of the game's item, and the sha1 the bundle lists it under.
std::pair<fb::ebx::Document, fb::Sha1> merged_item(const Merge& merge, const Game& game) {
const auto output = merge.catalog.root / mods::generated_folder;
const auto layout = vfs::read_layout(output / L"layout.toc");
const CasStore store(game.base, merge.root / L"unused", layout.root);
const auto shared = shared_in(store, output, game.base, game.root);
const auto& assets = shared.listing.manifest.ebx;
for (std::size_t index = 0; index < assets.size(); ++index)
if (assets[index].name == game.itemName)
return {fb::ebx::read_document(read_asset(store, output, game.base, shared.listing, index, game.root)), assets[index].sha1};
throw std::runtime_error("the merged bundle has no " + game.itemName);
}
// Two mods each ship the game's item with values of their own changed in place. Nothing was added
// to it, so the merge has no list to put together: it has the values. Each mod's are in the
// merged item, and where both changed one the higher mod's is.
void values_two_mods_changed(const Game& game, const CasStore& store, bool contested) {
const std::string label = contested ? "one value from two mods" : "values from two mods";
Merge merge(game, contested ? "one-value" : "two-values");
const auto upper = item_with(game.item, [&](fb::ebx::Object& root) {
if (contested) set_hash(root, 0x1111);
else field(root, "Key").data = std::string("reskate_upper");
});
const auto under = item_with(game.item, [&](fb::ebx::Object& root) {
set_hash(root, 0x2222);
if (contested) field(root, "Key").data = std::string("reskate_under");
});
write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}});
write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}});
const auto report = mods::merge_mods(merge.catalog);
expect(report.issue.empty() && report.built && report.problems.empty(),
label + ": the merge builds the patch and keeps every mod\n" + describe(report));
if (!report.built) return;
const auto item = merged_item(merge, game).first;
auto& root = *item.instances.front().object;
expect(std::get<std::string>(field(root, "Key").data) == (contested ? "reskate_under" : "reskate_upper"),
label + ": the merged item has the key one mod changed");
expect(hash_of(root) == (contested ? 0x1111u : 0x2222u),
label + (contested ? ": the value both changed is the higher mod's" : ": and the value the other mod changed"));
expect(noted(report, game.itemName + ": combined 2 edits (0 instance(s), 0 list entries, " +
(contested ? "3 changed value(s), 1 disagreed)" : "2 changed value(s))")),
label + ": the merge says what it combined\n" + describe(report));
}
// One of the two also changed the item in a way that is not a value (a list made shorter). That
// cannot be put together with the other mod's values, so the higher mod's copy stays whole, as
// it always did, and the merge now says whose changes are not in the game.
void a_change_that_does_not_combine(const Game& game, const CasStore& store) {
Merge merge(game, "not-combined");
bool reshaped{};
const auto upper = item_with(game.item, [&](fb::ebx::Object& root) {
field(root, "Key").data = std::string("reskate_upper");
reshaped = reshape(root);
});
if (!reshaped) {
std::cout << "The game's item has no list or reference to change; that case is skipped.\n";
return;
}
const auto under = item_with(game.item, [&](fb::ebx::Object& root) { set_hash(root, 0x2222); });
write_mod(merge.add("upper"), store, game, nullptr, {}, {{game.itemIndex, upper}});
write_mod(merge.add("under"), store, game, nullptr, {}, {{game.itemIndex, under}});
const auto report = mods::merge_mods(merge.catalog);
expect(report.issue.empty() && report.built && report.problems.empty(),
"not combined: the merge builds the patch and keeps every mod\n" + describe(report));
if (!report.built) return;
expect(merged_item(merge, game).second == sha1_of(upper), "not combined: the bundle has the higher mod's copy as that mod built it");
expect(noted(report, game.itemName + ": kept upper's copy whole; the changes under made to it are of a kind that cannot "
"be combined with it and are not in the game"),
"not combined: the merge says whose changes are missing\n" + describe(report));
}
// What other mods' copies of the bundle are given (a map carries one for its levels): one list
// for all of them, put together from every mod's. Under a name two mods share, such a copy gets
// the higher mod's asset, so the lower mod's entry for its own has to be out of that list too.
void the_list_other_copies_are_given(const Game& game, const CasStore& store) {
Merge merge(game, "carried-list");
const auto upper = item_from(game.item, same_name, 0x10);
const auto under = item_from(game.item, same_name, 0x20);
const auto upperList = list_with(game.list, {upper.identity}), underList = list_with(game.list, {under.identity});
write_mod(merge.add("upper"), store, game, &upperList, {upper});
write_mod(merge.add("under"), store, game, &underList, {under});
std::vector<const mods::Mod*> order;
std::map<const mods::Mod*, RelativeFiles> files;
for (const auto& mod : merge.catalog.mods) {
order.push_back(&mod);
files.emplace(&mod, scan(mod.directory));
}
mods::MergeReport report;
const auto overrides = collect_asset_overrides(order, files, store, game.base, game.root, report);
const auto versions = overrides.changed.find(lower(game.listName));
expect(versions != overrides.changed.end() && versions->second.size() == 1,
"carried list: there is one changed list for other mods' copies\n" + describe(report));
if (versions == overrides.changed.end() || versions->second.empty()) return;
const auto& change = versions->second.begin()->second;
const auto list = listed(fb::ebx::read_document(fb::decode_cas(change.encoded, {game.root})));
expect(list.size() == game.listed + 1 && names(list, upper) && !names(list, under),
"carried list: it has the game's items and the higher mod's, not the one no copy is given (" +
std::to_string(list.size()) + " entries)");
expect(change.names == std::set<fb::Guid>{upper.identity.fileGuid}, "carried list: the item that goes with it is the higher mod's");
expect(noted(report, "under: " + game.listName + ": 1 entry(ies) left out of the copies other mods carry, where another "
"mod's asset of the same name stands in for what they name"),
"carried list: the merge says which mod's entry was left out\n" + describe(report));
}
} // namespace
int main(int argc, char** argv) try {
@@ -381,6 +560,11 @@ int main(int argc, char** argv) try {
different_names_from_two_mods(game, store);
one_mod_changes_the_list(game, store, false);
one_mod_changes_the_list(game, store, true);
an_item_two_mods_both_list(game, store);
values_two_mods_changed(game, store, false);
values_two_mods_changed(game, store, true);
a_change_that_does_not_combine(game, store);
the_list_other_copies_are_given(game, store);
if (failures) {
std::cerr << failures << " failure(s)\n";
return 1;
Engine/Vfs/Test/mod_merge_added_assets_tests.cpp
+57 -14
View File
@@ -5,6 +5,7 @@
// The fixture is a small made-up game and two mods in a temp folder, written with the repo's
// own writers, so it needs no game files:
// game bundle "win32/test/shared": test/playlist, test/other
// bundle "win32/test/second": test/playlist again, test/extra
// map a level mod: its own level superbundle (a new TOC) with a copy of that bundle, as a
// custom map carries the core assets for its level; it edits nothing
// music an asset mod that changes test/playlist to name a new song, and adds the song, the
@@ -34,6 +35,7 @@ namespace {
using Bytes = std::vector<std::byte>;
constexpr std::uint32_t package = 0x1234;
constexpr char bundle_name[] = "win32/test/shared";
constexpr char second_bundle[] = "win32/test/second";
int failures = 0;
void expect(bool ok, const std::string& what) {
@@ -193,16 +195,26 @@ struct Fixture {
catalog.data_root = root / "game";
catalog.root = catalog.data_root / "Mods";
catalog.present = true;
// The game: two assets in one bundle, stored in the game's own archive.
// The game: two assets in one bundle, and a second bundle that has the playlist as well
// (the game keeps a list in several bundles), stored in the game's own archive.
const std::vector<Ebx> assets{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/other", 0, ebx_document(guid(2), {})}};
const std::vector<Ebx> second{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 0, ebx_document(guid(3), {})}};
Bytes archive;
std::vector<fb::BundleFileInfo> files;
for (const auto& asset : assets) {
const auto payload = encoded(asset.payload);
files.push_back({{false, package, 1}, static_cast<std::uint32_t>(archive.size()), static_cast<std::uint32_t>(payload.size())});
archive.insert(archive.end(), payload.begin(), payload.end());
}
const std::vector<fb::TocBundle> bundles{{bundle_name, fb::write_bundle_region(files, fb::write_binary_bundle(manifest_of(assets, {}))), 1}};
const auto region = [&](const std::vector<Ebx>& list) {
// The asset list is the region's first file, as it is in every bundle of the game:
// a bundle one mod alone ships then passes through as that mod built it.
const auto listing = fb::write_binary_bundle(manifest_of(list, {}));
std::vector<fb::BundleFileInfo> files{{{false, package, 1}, static_cast<std::uint32_t>(archive.size()),
static_cast<std::uint32_t>(listing.size())}};
archive.insert(archive.end(), listing.begin(), listing.end());
for (const auto& asset : list) {
const auto payload = encoded(asset.payload);
files.push_back({{false, package, 1}, static_cast<std::uint32_t>(archive.size()), static_cast<std::uint32_t>(payload.size())});
archive.insert(archive.end(), payload.begin(), payload.end());
}
return fb::write_bundle_region(files);
};
const std::vector<fb::TocBundle> bundles{{bundle_name, region(assets), 1}, {second_bundle, region(second), 1}};
write(catalog.data_root / "Data" / "layout.toc", layout_toc());
write(catalog.data_root / "Data" / "Win32" / "test_shared.toc", fb::write_patch_toc(bundles));
write(catalog.data_root / "Data" / "Win32" / "pkg" / "cas_01.cas", archive);
@@ -212,9 +224,9 @@ struct Fixture {
fs::remove_all(root, ignored);
}
// A mod with its own copy of the bundle: the manifest first (raw), then every payload.
// A mod with its own copy of a bundle: the manifest first (raw), then every payload.
void add(const std::string& name, bool levels, const std::string& toc, const std::vector<std::string>& superbundles,
const std::vector<Ebx>& assets, const std::vector<Resource>& resources = {}) {
const std::vector<Ebx>& assets, const std::vector<Resource>& resources = {}, const char* bundle = bundle_name) {
const auto listing = fb::write_binary_bundle(manifest_of(assets, resources));
Bytes archive(listing);
std::vector<fb::BundleFileInfo> files{{{true, package, 1}, 0, static_cast<std::uint32_t>(listing.size())}};
@@ -225,7 +237,7 @@ struct Fixture {
};
for (const auto& asset : assets) store(asset.payload);
for (const auto& resource : resources) store(resource.payload);
const std::vector<fb::TocBundle> bundles{{bundle_name, fb::write_bundle_region(files), 1}};
const std::vector<fb::TocBundle> bundles{{bundle, fb::write_bundle_region(files), 1}};
const auto directory = catalog.root / name;
write(directory / "layout.toc", layout_toc(superbundles));
write(directory / "Win32" / fs::path(toc), fb::write_patch_toc(bundles));
@@ -238,12 +250,12 @@ struct Fixture {
catalog.mods.push_back(std::move(mod));
}
// How many files the merged copy of the bundle in `toc` lists, or -1 when the merge left none.
int merged_files(const std::string& relative) const {
// How many files the merged copy of a bundle in `toc` lists, or -1 when the merge left none.
int merged_files(const std::string& relative, const std::string& name = bundle_name) const {
const auto toc = catalog.root / mods::generated_folder / "Win32" / fs::path(relative);
if (!fs::exists(toc)) return -1;
for (const auto& bundle : fb::read_toc(read(toc)).bundles)
if (bundle.name == bundle_name) return static_cast<int>(fb::read_bundle_region(bundle.region).files.size());
if (bundle.name == name) return static_cast<int>(fb::read_bundle_region(bundle.region).files.size());
return -1;
}
};
@@ -382,6 +394,35 @@ void two_music_mods_both_carried_into_maps_copy(int baseline) {
expect(files == baseline + 6, "two music mods: the map's copy gained 6 files (3 per music mod): " +
std::to_string(files) + " vs " + std::to_string(baseline + 6));
}
// The game keeps the playlist in a second bundle too, as it keeps the music playlist in eight. A
// mod that ships its own copy of that one (it changed something else in it) takes the music mod's
// playlist there. The song, its wave and the wave's resource were added beside the playlist in
// the first bundle; they have to be beside it in this one as well, or the list names a song its
// bundle does not hold. The asset nothing names stays out, and the music mod's own bundle is as
// that mod built it. Either mod may come first.
void carried_into_another_bundle_with_the_list(bool music_first) {
const std::string order = music_first ? "second bundle, music first" : "second bundle, music last";
Fixture fixture(music_first ? "second-bundle-music-first" : "second-bundle-music-last");
const auto add_music = [&] { fixture.add("music", false, shared_toc, {}, music_assets(), music_resources()); };
const auto add_tweak = [&] {
fixture.add("tweak", false, shared_toc, {},
{{"test/playlist", 0, ebx_document(guid(1), {})}, {"test/extra", 1, ebx_document(guid(3), {})}}, {}, second_bundle);
};
if (music_first) { add_music(); add_tweak(); } else { add_tweak(); add_music(); }
const auto report = mods::merge_mods(fixture.catalog);
expect(report.issue.empty() && report.built, order + ": the merge builds the patch\n" + describe(report));
expect(noted(report, std::string("tweak: ") + second_bundle + ": 1 asset(s) take another mod's change, e.g. test/playlist from music"),
order + ": the copy of the second bundle takes the changed playlist\n" + describe(report));
expect(noted(report, std::string("tweak: ") + second_bundle + ": 3 asset(s) added by other mods, e.g. test/song"),
order + ": and with it the song, its wave and the wave's resource\n" + describe(report));
const auto files = fixture.merged_files(shared_toc, second_bundle);
expect(files == 1 + 2 + 3, order + ": that copy lists its manifest, its two assets and those three (" + std::to_string(files) + " files)");
const auto own = fixture.merged_files(shared_toc);
const auto built = static_cast<int>(1 + music_assets().size() + music_resources().size());
expect(own == built, order + ": the music mod's own bundle is as it built it (" + std::to_string(own) + " files, " +
std::to_string(built) + " built)");
}
} // namespace
int main() try {
@@ -393,6 +434,8 @@ int main() try {
same_name_from_two_mods(true);
same_name_from_two_mods(false);
two_music_mods_both_carried_into_maps_copy(baseline);
carried_into_another_bundle_with_the_list(true);
carried_into_another_bundle_with_the_list(false);
if (failures) {
std::cerr << failures << " failure(s)\n";
return 1;
Engine/Vfs/Test/mod_merge_audit_tests.cpp
+915
View File
@@ -0,0 +1,915 @@
// Runs the whole mod merge on a data root and audits the patch it wrote, against the game and
// against every mod, so a Mods folder of real mods can be checked without starting the game.
//
// What has to hold of any merge:
// * every superbundle TOC reads back, and every placement lies inside an archive that is there
// and that the merged layout declares;
// * every payload the patch stores decodes to the size its asset list gives, and every EBX in
// it parses;
// * every bundle, asset and TOC chunk a mod ships is in the patch;
// * a document two mods' assets fought over by name is named by nothing that is left;
// * a document the merge wrote keeps its instances in the order the engine searches them, has
// every reference a mod appended to one of its lists, and has every value a mod changed in
// place changed: no value in it is the merge's own invention;
// * a list whose entries the game keeps beside it (an item list, the music playlist) names
// nothing its own bundle does not hold, in every copy of that bundle the patch carries.
// What is only reported: assets two mods add under one name, and assets of the game two mods
// change where the patch has one mod's copy whole.
//
// Arguments: <data root> [<listing file>], the root being a folder holding the game's Data/ and
// a Mods/ folder; skipped when none is given. The merge runs there as the game would run it and
// writes Mods/.reskate. With a second argument, every asset the patch itself stores is written
// to that file, one line each, to compare two merges of the same mods.
#include "Engine/Resource/binary_bundle.h"
#include "Engine/Resource/cas_codec.h"
#include "Engine/Resource/ebx_document.h"
#include "Engine/Resource/toc.h"
#include "Engine/Vfs/mod_catalog.h"
#include "Engine/Vfs/mod_merge_internal.h"
#include <algorithm>
#include <array>
#include <bit>
#include <chrono>
#include <deque>
#include <fstream>
#include <functional>
#include <iostream>
#include <map>
#include <memory>
#include <optional>
#include <set>
#include <stdexcept>
#include <string>
#include <vector>
namespace fs = std::filesystem;
namespace fb = dingosdk::frostbite;
using namespace dingosdk;
using namespace dingosdk::mods::detail;
namespace {
using Bytes = std::vector<std::byte>;
std::map<std::string, std::size_t> failures;
// The first of each kind in full, the rest counted: one broken mod can fail ten thousand times.
void fail(const std::string& kind, const std::string& text) {
const auto count = ++failures[kind];
if (count <= 25) std::cout << "FAIL [" << kind << "] " << text << '\n';
else if (count == 26) std::cout << "FAIL [" << kind << "] ... more of these, counted below\n";
}
void section(const std::string& title) { std::cout << "\n=== " << title << '\n'; }
std::string hex(const fb::Sha1& sha) {
static constexpr char digits[] = "0123456789abcdef";
std::string text;
for (const auto value : sha.bytes) {
text += digits[static_cast<unsigned>(value) >> 4];
text += digits[static_cast<unsigned>(value) & 15];
}
return text;
}
std::string key_of(const fb::BundleAsset& asset) {
return std::to_string(static_cast<int>(asset.kind)) + ':' +
(asset.kind == fb::AssetKind::chunk ? asset.guid.string() : lower(asset.name));
}
const char* kind_name(fb::AssetKind kind) {
return kind == fb::AssetKind::ebx ? "ebx" : kind == fb::AssetKind::resource ? "res" : "chunk";
}
std::string name_of(const fb::BundleAsset& asset) {
return std::string(kind_name(asset.kind)) + " " + (asset.kind == fb::AssetKind::chunk ? asset.guid.string() : asset.name);
}
struct Entry {
const fb::BundleAsset* asset{};
fb::BundleFileInfo file;
};
// One copy of a bundle: its asset list, and the assets in region order (ebx, resources, chunks).
struct Bundle {
std::string name;
std::optional<Listing> listing;
std::vector<Entry> assets;
std::map<std::string, std::size_t> at; // asset key -> index into assets
};
struct Toc {
std::string relative; // Win32/..., generic separators
fb::TocDocument document;
std::vector<Bundle> bundles;
std::map<std::string, std::size_t> bundleAt; // lower-case name
std::set<fb::Guid> chunks;
};
// Where a payload really lives: 0 the game, 1 the merged patch, 2 and up a mod.
struct Place {
int layer{};
std::uint32_t chunk{};
std::uint16_t archive{};
std::uint32_t offset{}, size{};
auto operator<=>(const Place&) const = default;
};
struct EbxFacts {
bool parsed{};
fb::Guid file;
std::string rootType;
std::vector<fb::ebx::ImportReference> imports;
};
// One mod's own copy of an asset that is not the game's copy of it.
struct Copy {
std::size_t mod{};
fb::Sha1 sha1;
fb::BundleFileInfo file;
};
// An asset at least one mod ships a copy of its own of, and what the patch has for it.
struct Contested {
std::string bundle, name;
fb::AssetKind kind{};
bool shipped{}; // the game has this asset in this bundle
fb::Sha1 game, merged;
Place gamePlace, mergedPlace;
std::vector<Copy> copies;
};
void walk(const fb::ebx::Value& value, const std::function<void(const fb::ebx::PointerReference&)>& visit) {
if (const auto* pointer = std::get_if<fb::ebx::PointerReference>(&value.data)) { visit(*pointer); return; }
if (const auto* object = std::get_if<std::shared_ptr<fb::ebx::Object>>(&value.data)) {
if (*object) for (const auto& field : (*object)->fields) walk(field.value, visit);
return;
}
if (const auto* array = std::get_if<fb::ebx::Value::Array>(&value.data))
for (const auto& element : *array) walk(element, visit);
}
// What a list entry refers to, so the same entry can be found in another copy of the document;
// empty for an entry that is not a reference to something with a guid.
std::string identity(const fb::ebx::Document& document, const fb::ebx::Value& value) {
const auto* pointer = std::get_if<fb::ebx::PointerReference>(&value.data);
if (!pointer || pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) return {};
const auto index = static_cast<std::size_t>(pointer->index);
if (pointer->kind == fb::ebx::PointerKind::external)
return index < document.imports.size()
? "x:" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string{};
return index < document.instances.size() && document.instances[index].exported
? "i:" + document.instances[index].instanceGuid.string() : std::string{};
}
// The document a list entry names, when it names one in another file.
const fb::Guid* named(const fb::ebx::Document& document, const fb::ebx::Value& value) {
const auto* pointer = std::get_if<fb::ebx::PointerReference>(&value.data);
if (!pointer || pointer->kind != fb::ebx::PointerKind::external || pointer->index < 0 ||
static_cast<std::size_t>(pointer->index) >= document.imports.size()) return nullptr;
return &document.imports[static_cast<std::size_t>(pointer->index)].fileGuid;
}
// Every value of a document by where it is: its instance (the root, an exported one's guid, an
// internal one's place among the internal ones), then the fields and list positions down to it.
// A plain value is spelled with a lower-case letter in front; a reference (what it leads to) and
// a list's length are spelled with an upper-case one, and are the document's shape.
struct Flat {
std::map<std::string, std::string> values;
std::size_t internal{}; // internal instances: a place is all one is known by
};
bool plain(const std::string& spelled) { return !spelled.empty() && spelled.front() >= 'a' && spelled.front() <= 'z'; }
void flatten(const fb::ebx::Document& document, const std::vector<std::string>& instances,
const fb::ebx::Value& value, const std::string& path, Flat& out) {
const auto& data = value.data;
if (const auto* flag = std::get_if<bool>(&data)) out.values[path] = *flag ? "b1" : "b0";
else if (const auto* number = std::get_if<std::int64_t>(&data)) out.values[path] = "i" + std::to_string(*number);
else if (const auto* count = std::get_if<std::uint64_t>(&data)) out.values[path] = "u" + std::to_string(*count);
else if (const auto* real = std::get_if<double>(&data)) out.values[path] = "d" + std::to_string(std::bit_cast<std::uint64_t>(*real));
else if (const auto* text = std::get_if<std::string>(&data)) out.values[path] = "s" + *text;
else if (const auto* guid = std::get_if<fb::Guid>(&data)) out.values[path] = "g" + guid->string();
else if (const auto* sha = std::get_if<fb::Sha1>(&data)) out.values[path] = "h" + hex(*sha);
else if (const auto* resource = std::get_if<fb::ebx::ResourceReference>(&data)) out.values[path] = "r" + std::to_string(resource->id);
else if (const auto* pointer = std::get_if<fb::ebx::PointerReference>(&data)) {
const auto index = pointer->index < 0 ? std::size_t{} : static_cast<std::size_t>(pointer->index);
if (pointer->kind == fb::ebx::PointerKind::null || pointer->index < 0) out.values[path] = "P";
else if (pointer->kind == fb::ebx::PointerKind::external)
out.values[path] = index < document.imports.size()
? "P" + document.imports[index].fileGuid.string() + "/" + document.imports[index].classGuid.string() : std::string("P?");
else out.values[path] = index < instances.size() ? "P" + instances[index] : std::string("P?");
} else if (const auto* object = std::get_if<std::shared_ptr<fb::ebx::Object>>(&data)) {
if (*object) for (const auto& field : (*object)->fields) flatten(document, instances, field.value, path + "/" + field.name, out);
} else if (const auto* array = std::get_if<fb::ebx::Value::Array>(&data)) {
out.values[path + "/#"] = "N" + std::to_string(array->size());
for (std::size_t index = 0; index < array->size(); ++index)
flatten(document, instances, (*array)[index], path + "/" + std::to_string(index), out);
}
}
Flat flatten(const fb::ebx::Document& document) {
Flat out;
std::vector<std::string> instances;
for (std::size_t index = 0; index < document.instances.size(); ++index) {
const auto& instance = document.instances[index];
instances.push_back(index == 0 ? std::string("root")
: instance.exported ? "x:" + instance.instanceGuid.string() : "n:" + std::to_string(out.internal++));
}
for (std::size_t index = 0; index < document.instances.size(); ++index)
if (const auto& object = document.instances[index].object)
for (const auto& field : object->fields) flatten(document, instances, field.value, instances[index] + "/" + field.name, out);
return out;
}
struct Audit {
fs::path root, base, out;
mods::Catalog catalog;
vfs::Layout baseLayout, mergedLayout;
std::unique_ptr<CasStore> store;
// A deque: bundles hold pointers into their own asset lists, which must not move.
std::deque<Toc> tocs;
std::map<std::string, std::size_t> tocAt; // lower-case relative path
// The game's bundles by name, wherever they are, to tell a mod's own content from the game's
// copy of something it carries.
std::map<std::string, fb::TocBundle> gameBundles;
std::map<std::string, std::optional<Bundle>> gameListings;
std::map<Place, EbxFacts> facts;
std::map<Place, fb::Guid> guids;
std::vector<Contested> contested;
std::chrono::steady_clock::time_point started = std::chrono::steady_clock::now();
void progress(const std::string& what) const {
const auto seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - started).count();
std::cerr << "[" << seconds << "s] " << what << std::endl;
}
const fs::path& layer(int index) const {
return index == 0 ? base : index == 1 ? out : catalog.mods.at(static_cast<std::size_t>(index - 2)).directory;
}
// A placement that is not a patch one is in the game, whichever layer's bundle names it.
Place place(int from, const fb::BundleFileInfo& file) const {
return {file.location.patch ? from : 0, file.location.installChunk, file.location.archive, file.offset, file.size};
}
Bytes payload(const Place& where) const {
fb::CasDecodeOptions options{root};
options.maximumOutputSize = 2048ULL * 1024ULL * 1024ULL;
return fb::decode_cas(store->read(layer(where.layer), {where.layer != 0, where.chunk, where.archive}, where.offset, where.size), options);
}
static EbxFacts facts_of(const Bytes& bytes) {
EbxFacts result;
try {
const auto document = fb::ebx::read_document(bytes);
result.parsed = true;
result.file = document.fileGuid;
result.rootType = document.rootType;
result.imports = document.imports;
} catch (const std::exception&) {}
return result;
}
const EbxFacts& ebx(const Place& where) {
if (const auto found = facts.find(where); found != facts.end()) return found->second;
EbxFacts result;
try { result = facts_of(payload(where)); } catch (const std::exception&) {}
return facts.emplace(where, std::move(result)).first->second;
}
const fb::Guid* guid(const Place& where) {
if (const auto found = guids.find(where); found != guids.end()) return &found->second;
if (const auto known = facts.find(where); known != facts.end() && known->second.parsed)
return &guids.emplace(where, known->second.file).first->second;
try { return &guids.emplace(where, fb::ebx::read_file_guid(payload(where))).first->second; }
catch (const std::exception&) { return nullptr; }
}
Bundle read_bundle(int from, const fb::TocBundle& bundle) const {
Bundle result;
result.name = bundle.name;
try {
result.listing = list_bundle(*store, layer(from), base, bundle, root);
if (!result.listing) return result;
std::size_t at = result.listing->first;
for (const auto* list : {&result.listing->manifest.ebx, &result.listing->manifest.resources, &result.listing->manifest.chunks})
for (const auto& asset : *list) {
if (at >= result.listing->files.size()) throw std::runtime_error("more assets than files");
result.at.emplace(key_of(asset), result.assets.size());
result.assets.push_back({&asset, result.listing->files[at++]});
}
} catch (const std::exception&) {
result.listing.reset();
result.assets.clear();
result.at.clear();
}
return result;
}
const Bundle* game_bundle(const std::string& name) {
const auto key = lower(name);
if (const auto found = gameListings.find(key); found != gameListings.end()) return found->second ? &*found->second : nullptr;
auto& slot = gameListings[key];
if (const auto shipped = gameBundles.find(key); shipped != gameBundles.end())
if (auto bundle = read_bundle(0, shipped->second); bundle.listing) slot = std::move(bundle);
return slot ? &*slot : nullptr;
}
// The guid of every document a copy of a bundle holds.
std::set<fb::Guid> held(int from, const Bundle& bundle, bool& unsure) {
std::set<fb::Guid> result;
for (const auto& entry : bundle.assets) {
if (entry.asset->kind != fb::AssetKind::ebx) continue;
if (const auto* id = guid(place(from, entry.file))) result.insert(*id); else unsure = true;
}
return result;
}
std::string mod_names(const std::vector<std::size_t>& list) const {
std::set<std::string> unique;
for (const auto index : list) unique.insert(catalog.mods[index].name);
std::string text;
for (const auto& name : unique) text += (text.empty() ? "" : ", ") + name;
return text;
}
// ---------------------------------------------------------------- the merge, and reading what it wrote
bool merge() {
section("Merge");
catalog = mods::load_catalog(root);
std::cout << catalog.mods.size() << " mod(s) merged, " << catalog.excluded.size() << " left out, "
<< catalog.disabled.size() << " disabled; " << catalog.notes.size() << " note(s)\n";
if (!catalog.issue.empty()) fail("merge", catalog.issue);
for (const auto& warning : catalog.warnings) fail("left out", warning);
if (!catalog.merged) fail("merge", "no merged patch was produced");
return catalog.merged;
}
void load() {
base = root / L"Data";
out = root / L"Mods" / mods::generated_folder;
baseLayout = vfs::read_layout(base / L"layout.toc");
mergedLayout = vfs::read_layout(out / L"layout.toc");
store = std::make_unique<CasStore>(base, fs::temp_directory_path() / L"reskate-merge-audit-unused", mergedLayout.root);
section("Superbundles");
const auto toc_files = [](const fs::path& under) {
std::vector<fs::path> files;
std::error_code error;
for (fs::recursive_directory_iterator it(under / L"Win32", error), end; it != end && !error; it.increment(error))
if (it->is_regular_file(error) && lower(it->path().extension().string()) == ".toc") files.push_back(it->path());
std::ranges::sort(files);
return files;
};
std::error_code error;
std::size_t bundles{}, chunks{}, unreadable{};
for (const auto& file : toc_files(out)) {
Toc toc;
toc.relative = fs::relative(file, out, error).generic_string();
try {
toc.document = fb::read_toc(read_file(file));
fb::verify_toc(toc.document);
} catch (const std::exception& failure) {
fail("toc", toc.relative + ": " + failure.what());
continue;
}
toc.bundles.reserve(toc.document.bundles.size());
for (const auto& bundle : toc.document.bundles) {
if (!toc.bundleAt.emplace(lower(bundle.name), toc.bundles.size()).second)
fail("toc", toc.relative + " lists " + bundle.name + " twice");
toc.bundles.push_back(read_bundle(1, bundle));
unreadable += !toc.bundles.back().listing;
++bundles;
}
for (const auto& chunk : toc.document.chunks)
if (!chunk.removed) toc.chunks.insert(chunk.guid);
chunks += toc.document.chunks.size();
tocAt.emplace(lower(toc.relative), tocs.size());
tocs.push_back(std::move(toc));
}
std::cout << "the patch: " << tocs.size() << " superbundle TOC(s), " << bundles << " bundle(s) (" << unreadable
<< " without a readable asset list), " << chunks << " TOC chunk(s)\n";
std::size_t shipped{};
for (const auto& file : toc_files(base)) {
++shipped;
try {
for (auto& bundle : fb::read_toc(read_file(file)).bundles) gameBundles.try_emplace(lower(bundle.name), std::move(bundle));
} catch (const std::exception&) {}
}
std::cout << "the game: " << shipped << " superbundle TOC(s), " << gameBundles.size() << " bundle name(s)\n";
}
// ---------------------------------------------------------------- placements and the layout
void placements() {
section("Placements and layout");
using Archive = std::pair<std::uint32_t, std::uint16_t>;
const auto declared_in = [](const native_db::Node& layout) {
std::set<Archive> result;
for (const auto* field : {"layeredInstallChunkFiles", "unlayeredInstallChunkFiles"})
if (const auto* node = layout.field(field); node && node->type == 19)
vfs::for_each_install_chunk_file(node->payload(), [&](std::uint32_t id, std::uint16_t archive) { result.emplace(id, archive); });
return result;
};
const auto declared = declared_in(mergedLayout.root), shipped = declared_in(baseLayout.root);
std::set<Archive> used;
std::size_t checked{};
const auto check = [&](const std::string& what, const fb::CasIdentifier& location, std::uint32_t offset, std::uint32_t size) {
++checked;
const auto on_disk = store->archive_size(location.patch ? out : base, location);
if (!on_disk) {
fail("placement", what + " is in " + store->describe(location) + (location.patch ? " of the patch" : " of the game") + ", which is not there");
return;
}
if (static_cast<std::uint64_t>(offset) + size > *on_disk)
fail("placement", what + ": " + std::to_string(size) + " bytes at " + std::to_string(offset) + " run past the end of " +
store->describe(location) + " (" + std::to_string(*on_disk) + " bytes)");
if (location.patch) used.emplace(location.installChunk, location.archive);
};
for (const auto& toc : tocs) {
for (const auto& bundle : toc.document.bundles) {
const auto region = fb::read_bundle_region(bundle.region);
for (std::size_t index = 0; index < region.files.size(); ++index)
check(toc.relative + ": " + bundle.name + " file " + std::to_string(index), region.files[index].location,
region.files[index].offset, region.files[index].size);
}
for (const auto& chunk : toc.document.chunks)
if (!chunk.removed) check(toc.relative + ": chunk " + chunk.guid.string(), chunk.location, chunk.offset, chunk.size);
}
// An archive the layout does not declare is never opened; one it declares and the patch
// lacks sends the engine looking for a file that is absent.
for (const auto& archive : used)
if (!declared.contains(archive))
fail("layout", "the patch reads archive " + std::to_string(archive.second) + " of install chunk " +
std::to_string(archive.first) + ", which the merged layout does not declare");
std::size_t added{};
for (const auto& archive : declared) {
if (shipped.contains(archive)) continue;
++added;
if (!store->archive_size(out, {true, archive.first, archive.second}))
fail("layout", "the merged layout declares archive " + std::to_string(archive.second) + " of install chunk " +
std::to_string(archive.first) + ", and the patch has no such file");
}
std::cout << checked << " placement(s) checked; " << used.size() << " patch archive(s) in use, " << added << " declared beyond the game's\n";
// The superbundle list is searched by name, so it stays in order; a mod's own superbundles
// have to be in it, in an install chunk, and have a TOC.
const auto names = [](const native_db::Node& layout) {
std::vector<std::string> result;
if (const auto* list = layout.field("superBundles"))
for (const auto& row : list->children)
if (const auto* name = row.field("name"); name && name->type == 7) result.push_back(lower(name->text));
return result;
};
const auto merged = names(mergedLayout.root), game = names(baseLayout.root);
const std::set<std::string> known(merged.begin(), merged.end()), stock(game.begin(), game.end());
if (!std::ranges::is_sorted(merged)) fail("layout", "the merged superbundle list is not in name order");
if (known.size() != merged.size()) fail("layout", "the merged superbundle list names one twice");
std::set<std::string> placed;
if (const auto* manifest = mergedLayout.root.field("installManifest"))
if (const auto* chunks = manifest->field("installChunks"))
for (const auto& chunk : chunks->children)
if (const auto* list = chunk.field("superbundles"))
for (const auto& entry : list->children)
if (entry.type == 7) placed.insert(lower(entry.text));
std::size_t own{};
for (const auto& mod : catalog.mods) {
if (!mod.provides_layout) continue;
try {
const auto layout = vfs::read_layout(mod.directory / L"layout.toc");
for (const auto& name : names(layout.root)) {
if (stock.contains(name)) continue;
++own;
if (!known.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is not in the merged layout");
if (!placed.contains(name)) fail("layout", mod.name + ": its superbundle " + name + " is in no install chunk of the merged layout");
if (!tocAt.contains(name + ".toc")) fail("layout", mod.name + ": its superbundle " + name + " has no TOC in the patch");
}
} catch (const std::exception& failure) {
fail("layout", mod.name + ": layout.toc could not be read (" + failure.what() + ")");
}
}
std::cout << merged.size() << " superbundle(s) in the merged layout, " << own << " listed by mods beyond the game's\n";
}
// ---------------------------------------------------------------- asset lists and payloads
void payloads(const fs::path& listing) {
section("Asset lists and payloads");
// What a listed size means is learnt from the game's own assets first: only a kind whose
// every sample agrees is held to it.
std::array<std::size_t, 3> sampled{}, agreed{};
std::set<Place> seen;
for (const auto& toc : tocs)
for (const auto& bundle : toc.bundles)
for (const auto& entry : bundle.assets) {
const auto kind = static_cast<std::size_t>(entry.asset->kind);
const auto where = place(1, entry.file);
if (where.layer != 0 || sampled[kind] >= 150 || entry.file.size > (8u << 20) || !seen.insert(where).second) continue;
++sampled[kind];
try { agreed[kind] += payload(where).size() == entry.asset->originalSize; } catch (const std::exception&) {}
}
std::array<bool, 3> sized{};
for (std::size_t kind = 0; kind < 3; ++kind) sized[kind] = sampled[kind] >= 20 && agreed[kind] == sampled[kind];
std::map<Place, std::pair<const Entry*, std::string>> stored;
std::size_t listed{};
for (const auto& toc : tocs)
for (const auto& bundle : toc.bundles) {
if (bundle.at.size() != bundle.assets.size()) {
std::set<std::string> names;
std::string example;
for (const auto& entry : bundle.assets)
if (!names.insert(key_of(*entry.asset)).second && example.empty()) example = name_of(*entry.asset);
fail("asset list", toc.relative + ": " + bundle.name + " lists an asset twice, e.g. " + example);
}
for (const auto& entry : bundle.assets) {
++listed;
if (const auto where = place(1, entry.file); where.layer == 1) stored.try_emplace(where, &entry, toc.relative + ": " + bundle.name);
}
}
std::cout << listed << " asset(s) listed across the patch; " << stored.size() << " payload(s) are stored in the patch's own archives\n";
progress("decoding " + std::to_string(stored.size()) + " payloads");
std::ofstream written;
if (!listing.empty()) written.open(listing, std::ios::binary);
std::array<std::size_t, 3> decoded{};
std::size_t bytesIn{}, bytesOut{}, unparsed{};
for (const auto& [where, what] : stored) {
const auto& [entry, from] = what;
const auto kind = static_cast<std::size_t>(entry->asset->kind);
try {
const auto bytes = payload(where);
++decoded[kind];
bytesIn += where.size;
bytesOut += bytes.size();
if (sized[kind] && bytes.size() != entry->asset->originalSize)
fail("payload", from + ": " + name_of(*entry->asset) + " decodes to " + std::to_string(bytes.size()) +
" bytes, its asset list says " + std::to_string(entry->asset->originalSize));
if (entry->asset->kind == fb::AssetKind::ebx) {
auto result = facts_of(bytes);
if (!result.parsed) { ++unparsed; fail("payload", from + ": " + name_of(*entry->asset) + " is not a document the merge can read"); }
facts.emplace(where, std::move(result));
}
if (written.is_open()) written << from << '\t' << name_of(*entry->asset) << '\t' << bytes.size() << '\t' << hex(sha1_of(bytes)) << '\n';
} catch (const std::exception& failure) {
fail("payload", from + ": " + name_of(*entry->asset) + " at " + store->describe({true, where.chunk, where.archive}) + "+" +
std::to_string(where.offset) + " cannot be decoded (" + failure.what() + ")");
}
}
std::cout << decoded[0] << " ebx, " << decoded[1] << " res and " << decoded[2] << " chunk payload(s) decoded: " << bytesIn / (1024 * 1024)
<< " MB stored, " << bytesOut / (1024 * 1024) << " MB decoded; " << unparsed << " EBX do not parse\n";
}
// ---------------------------------------------------------------- every mod against the patch
void mods_against_patch() {
section("Every mod against the patch");
std::map<std::string, Contested> found; // by bundle name, asset and what the patch has
std::size_t bundles{}, assets{}, chunks{}, byName{};
for (std::size_t index = 0; index < catalog.mods.size(); ++index) {
const auto& mod = catalog.mods[index];
if (!mod.provides_layout) continue;
if (index % 20 == 0) progress("mod " + std::to_string(index + 1) + " of " + std::to_string(catalog.mods.size()));
for (const auto& relative : scan(mod.directory).tocs) {
const auto merged = tocAt.find(lower(relative));
if (merged == tocAt.end()) { fail("mod content", mod.name + ": the patch has no " + relative); continue; }
const auto& toc = tocs[merged->second];
fb::TocDocument own;
try { own = fb::read_toc(read_file(mod.directory / fs::path(relative))); }
catch (const std::exception& failure) { fail("mod content", mod.name + ": " + relative + " cannot be read (" + failure.what() + ")"); continue; }
for (const auto& chunk : own.chunks) {
if (chunk.removed) continue;
++chunks;
if (!toc.chunks.contains(chunk.guid))
fail("mod content", mod.name + ": " + relative + ": its chunk " + chunk.guid.string() + " is not in the patch's TOC");
}
for (const auto& bundle : own.bundles) {
++bundles;
const auto at = toc.bundleAt.find(lower(bundle.name));
if (at == toc.bundleAt.end()) { fail("mod content", mod.name + ": " + relative + ": its bundle " + bundle.name + " is not in the patch"); continue; }
const auto& patched = toc.bundles[at->second];
const auto mine = read_bundle(static_cast<int>(index + 2), bundle);
if (!mine.listing || !patched.listing) { ++byName; continue; }
const auto* game = game_bundle(bundle.name);
for (const auto& entry : mine.assets) {
++assets;
const auto key = key_of(*entry.asset);
const auto there = patched.at.find(key);
if (there == patched.at.end()) {
fail("mod content", mod.name + ": " + relative + ": " + bundle.name + ": its " + name_of(*entry.asset) + " is not in the patch's copy");
continue;
}
const auto& kept = patched.assets[there->second];
const Entry* shipped{};
if (game) if (const auto it = game->at.find(key); it != game->at.end()) shipped = &game->assets[it->second];
// The game's copy carried along is not this mod's content.
if (shipped && shipped->asset->sha1 == entry.asset->sha1) continue;
auto& record = found[lower(bundle.name) + '|' + key + '|' + hex(kept.asset->sha1)];
if (record.copies.empty()) {
record.bundle = bundle.name;
record.name = entry.asset->name;
record.kind = entry.asset->kind;
record.shipped = shipped != nullptr;
if (shipped) { record.game = shipped->asset->sha1; record.gamePlace = place(0, shipped->file); }
record.merged = kept.asset->sha1;
record.mergedPlace = place(1, kept.file);
}
if (std::ranges::none_of(record.copies, [&](const Copy& copy) { return copy.mod == index && copy.sha1 == entry.asset->sha1; }))
record.copies.push_back({index, entry.asset->sha1, entry.file});
}
}
}
}
std::cout << bundles << " mod bundle(s), " << assets << " listed asset(s) and " << chunks << " TOC chunk(s) looked up in the patch; "
<< byName << " bundle(s) could only be checked by name\n";
for (auto& [key, record] : found) contested.push_back(std::move(record));
}
// ---------------------------------------------------------------- what became of each mod's own content
void outcomes() {
section("What became of each mod's own content");
struct Clash { const Contested* asset; std::vector<std::size_t> kept, lost; };
std::vector<Clash> clashes, overridden;
std::vector<const Contested*> written;
std::size_t asShipped{}, rewritten{};
for (const auto& record : contested) {
std::set<fb::Sha1> versions;
for (const auto& copy : record.copies) versions.insert(copy.sha1);
const bool one = versions.contains(record.merged);
if (one && versions.size() == 1) { ++asShipped; continue; }
if (record.shipped && record.merged == record.game) {
fail("lost edit", record.bundle + ": " + record.name + ": " + std::to_string(record.copies.size()) + " mod(s) change it (e.g. " +
catalog.mods[record.copies.front().mod].name + ") and the patch has the game's own copy");
continue;
}
if (!one) {
// No one mod's copy: the merge wrote it (edits combined, a change carried in).
if (record.kind == fb::AssetKind::ebx) written.push_back(&record); else ++rewritten;
continue;
}
Clash clash{&record, {}, {}};
for (const auto& copy : record.copies) (copy.sha1 == record.merged ? clash.kept : clash.lost).push_back(copy.mod);
(record.shipped ? overridden : clashes).push_back(std::move(clash));
}
std::cout << asShipped << " asset(s) are in the patch as their one mod ships them; the merge wrote " << written.size()
<< " EBX document(s) and " << rewritten << " other asset(s) itself\n";
std::cout << "\nassets two mods add under one name, with different content: " << clashes.size() << '\n';
std::map<std::string, std::vector<const Clash*>> pairs;
for (const auto& clash : clashes) pairs[mod_names(clash.lost) + " lost them to " + mod_names(clash.kept)].push_back(&clash);
for (const auto& [pair, list] : pairs)
std::cout << " " << pair << ": " << list.size() << ", e.g. " << list.front()->asset->name << " in " << list.front()->asset->bundle << '\n';
std::cout << "assets of the game two mods change, where the patch has one mod's copy whole: " << overridden.size() << '\n';
std::size_t printed{};
for (const auto& clash : overridden) {
if (++printed > 40) { std::cout << " ... and " << overridden.size() - 40 << " more\n"; break; }
std::cout << " " << kind_name(clash.asset->kind) << ' ' << clash.asset->name << " (" << clash.asset->bundle << "): kept "
<< mod_names(clash.kept) << "; not in it: " << mod_names(clash.lost) << '\n';
}
// A reference is by the guid of the document it names. Where two mods' assets shared a
// name and one went, whatever still names the one that went finds nothing.
section("References to documents that lost their name to another mod");
struct Gone { std::string mod, name, keptBy; };
std::map<fb::Guid, Gone> gone;
for (const auto& clash : clashes) {
if (clash.asset->kind != fb::AssetKind::ebx) continue;
const auto& kept = ebx(clash.asset->mergedPlace);
for (const auto& copy : clash.asset->copies) {
if (copy.sha1 == clash.asset->merged) continue;
const auto& lost = ebx(place(static_cast<int>(copy.mod + 2), copy.file));
// The same guid under both copies still resolves, to the copy that stayed.
if (lost.parsed && kept.parsed && lost.file != kept.file)
gone.emplace(lost.file, Gone{catalog.mods[copy.mod].name, clash.asset->name, mod_names(clash.kept)});
}
}
std::map<std::string, std::pair<std::size_t, std::string>> holders;
std::size_t references{};
for (const auto& toc : tocs)
for (const auto& bundle : toc.bundles)
for (const auto& entry : bundle.assets) {
if (entry.asset->kind != fb::AssetKind::ebx) continue;
const auto where = place(1, entry.file);
if (where.layer != 1) continue;
for (const auto& import : ebx(where).imports) {
const auto lost = gone.find(import.fileGuid);
if (lost == gone.end()) continue;
++references;
auto& [count, example] = holders[bundle.name + ": " + entry.asset->name + " names " + lost->second.mod + "'s " +
lost->second.name + ", which " + lost->second.keptBy + "'s replaced"];
if (!count++) example = toc.relative;
}
}
for (const auto& [text, where] : holders)
fail("dangling reference", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")");
std::cout << gone.size() << " document(s) went with a guid nothing in the patch has; " << references << " reference(s) in the patch still name one\n";
section("Documents the merge wrote");
std::size_t checked{}, lists{}, kept{}, inPlace{}, valueEdits{}, reshaped{};
for (const auto* record : written) {
fb::ebx::Document merged;
try { merged = fb::ebx::read_document(payload(record->mergedPlace)); }
catch (const std::exception& failure) { fail("merged document", record->bundle + ": " + record->name + " cannot be read back (" + failure.what() + ")"); continue; }
++checked;
const auto label = record->bundle + ": " + record->name + " (" + merged.rootType + ")";
// The engine binary-searches the exported instances after the root by guid, and
// internal ones have to follow them.
bool internal{};
const fb::Guid* previous{};
for (std::size_t index = 1; index < merged.instances.size(); ++index) {
const auto& instance = merged.instances[index];
if (!instance.exported) { internal = true; continue; }
if (internal) { fail("merged document", label + ": an exported instance follows an internal one"); break; }
if (previous && !(*previous < instance.instanceGuid)) { fail("merged document", label + ": its exported instances are not in guid order"); break; }
previous = &instance.instanceGuid;
}
std::size_t stray{};
for (const auto& instance : merged.instances)
if (instance.object)
for (const auto& field : instance.object->fields)
walk(field.value, [&](const fb::ebx::PointerReference& pointer) {
const auto limit = pointer.kind == fb::ebx::PointerKind::internal ? merged.instances.size()
: pointer.kind == fb::ebx::PointerKind::external ? merged.imports.size() : std::size_t{};
stray += pointer.kind != fb::ebx::PointerKind::null && (pointer.index < 0 || static_cast<std::size_t>(pointer.index) >= limit);
});
if (stray) fail("merged document", label + ": " + std::to_string(stray) + " reference(s) index past the instance or import table");
if (!record->shipped) continue;
fb::ebx::Document game;
try { game = fb::ebx::read_document(payload(record->gamePlace)); } catch (const std::exception&) { continue; }
if (!merged.root() || !merged.root()->object || !game.root() || !game.root()->object) continue;
// Every reference a mod appended to one of the root's lists is in the merged list,
// unless what it names lost its name to another mod's asset.
std::set<fb::Sha1> done;
for (const auto& copy : record->copies) {
if (!done.insert(copy.sha1).second) continue;
fb::ebx::Document edit;
try { edit = fb::ebx::read_document(payload(place(static_cast<int>(copy.mod + 2), copy.file))); } catch (const std::exception&) { continue; }
if (!edit.root() || !edit.root()->object) continue;
for (const auto& field : edit.root()->object->fields) {
const auto* added = std::get_if<fb::ebx::Value::Array>(&field.value.data);
const auto* before = game.root()->object->find(field.name);
const auto* after = merged.root()->object->find(field.name);
const auto* beforeList = before ? std::get_if<fb::ebx::Value::Array>(&before->value.data) : nullptr;
const auto* afterList = after ? std::get_if<fb::ebx::Value::Array>(&after->value.data) : nullptr;
if (!added || !beforeList || !afterList || added->size() <= beforeList->size()) continue;
++lists;
std::set<std::string> present;
for (const auto& value : *afterList) present.insert(identity(merged, value));
for (auto at = beforeList->size(); at < added->size(); ++at) {
const auto id = identity(edit, (*added)[at]);
if (id.empty()) continue;
if (present.contains(id)) { ++kept; continue; }
if (const auto* target = named(edit, (*added)[at]); target && gone.contains(*target)) continue;
fail("merged list", label + ": " + field.name + " lacks an entry " + catalog.mods[copy.mod].name + " added (" + id + ")");
}
}
}
// Values changed in place. What the merged document has that the game's does not is
// some mod's value, and a mod that changed values and nothing else has every one of
// them changed in the merged document: to its own, or to another mod's that changed
// the same one.
const auto before = flatten(game), after = flatten(merged);
// An internal instance is known by its place alone, which says nothing once one was
// added or taken away.
const auto told = [&](const Flat& other, const std::string& key) {
return !key.starts_with("n:") || other.internal == before.internal;
};
std::vector<std::pair<std::size_t, Flat>> edits;
done.clear();
for (const auto& copy : record->copies) {
if (!done.insert(copy.sha1).second) continue;
try { edits.emplace_back(copy.mod, flatten(fb::ebx::read_document(payload(place(static_cast<int>(copy.mod + 2), copy.file))))); }
catch (const std::exception&) {}
}
std::size_t invented{};
std::string example;
for (const auto& [key, value] : before.values) {
if (!plain(value) || !told(after, key)) continue;
const auto now = after.values.find(key);
if (now == after.values.end() || now->second == value) continue;
++inPlace;
if (std::ranges::none_of(edits, [&](const std::pair<std::size_t, Flat>& edit) {
const auto theirs = edit.second.values.find(key);
return told(edit.second, key) && theirs != edit.second.values.end() && theirs->second == now->second; }))
if (!invented++) example = key;
}
if (invented)
fail("merged value", label + ": " + std::to_string(invented) + " value(s) are neither the game's nor any mod's, e.g. " + example);
for (const auto& [mod, edit] : edits) {
bool shape{}; // a reference moved, a list resized, something of the game's gone
std::vector<const std::string*> changed;
for (const auto& [key, value] : before.values) {
if (!told(edit, key)) continue;
const auto theirs = edit.values.find(key);
if (theirs == edit.values.end()) { shape = true; continue; }
if (theirs->second == value) continue;
if (plain(value) && plain(theirs->second)) { changed.push_back(&key); continue; }
// A list on the root that grew at its end is the one change of shape that
// is combined, and is checked above.
const bool rootList = key.starts_with("root/") && key.ends_with("/#") && std::ranges::count(key, '/') == 2;
if (!(rootList && std::stoull(theirs->second.substr(1)) > std::stoull(value.substr(1)))) shape = true;
}
if (changed.empty()) continue;
if (shape) { ++reshaped; continue; }
std::size_t lost{};
for (const auto* key : changed) {
const auto now = after.values.find(*key);
if (told(after, *key) && (now == after.values.end() || now->second == before.values.at(*key)) && !lost++) example = *key;
}
if (lost)
fail("lost value", label + ": " + std::to_string(lost) + " of the " + std::to_string(changed.size()) + " value(s) " +
catalog.mods[mod].name + " changed are the game's again, e.g. " + example);
++valueEdits;
}
}
std::cout << checked << " document(s) read back; " << lists << " list(s) a mod appended to, " << kept << " appended reference(s) found in them\n";
std::cout << inPlace << " value(s) in them differ from the game's, each one some mod's; " << valueEdits
<< " mod copy(ies) that changed values alone have them all in; " << reshaped
<< " that also moved a reference or resized a list are not combined that way\n";
}
// ---------------------------------------------------------------- lists and what they name
// The game keeps what some lists name in the list's own bundle (an item list and its items,
// the music playlist and its songs), and reads such a list without checking an entry. So in
// every copy of that bundle the patch carries, the entries mods added have to be there too.
void lists() {
section("Lists whose entries the game keeps beside them");
struct Shipped { fb::ebx::Document document; std::set<fb::Guid> held; bool unsure{}; };
std::map<std::string, std::optional<Shipped>> shipped; // by bundle and asset key
std::map<std::string, std::pair<std::size_t, std::string>> missing, twice;
std::size_t copies{}, entries{};
for (const auto& toc : tocs)
for (const auto& bundle : toc.bundles) {
const auto* game = game_bundle(bundle.name);
if (!game) continue;
std::optional<std::set<fb::Guid>> here;
bool unsure{};
for (const auto& entry : bundle.assets) {
if (entry.asset->kind != fb::AssetKind::ebx) continue;
const auto where = place(1, entry.file);
const auto key = key_of(*entry.asset);
const auto original = game->at.find(key);
if (where.layer != 1 || original == game->at.end()) continue;
auto& known = shipped[lower(bundle.name) + '|' + key];
if (!known) {
Shipped value;
try { value.document = fb::ebx::read_document(payload(place(0, game->assets[original->second].file))); } catch (const std::exception&) {}
known = std::move(value);
}
const auto* gameRoot = known->document.root();
if (!gameRoot || !gameRoot->object) continue;
// Only a document with a list of references to other documents is read at all.
const auto lists_references = std::ranges::any_of(gameRoot->object->fields, [&](const fb::ebx::FieldValue& field) {
const auto* list = std::get_if<fb::ebx::Value::Array>(&field.value.data);
return list && !list->empty() && named(known->document, list->front());
});
if (!lists_references) continue;
fb::ebx::Document merged;
try { merged = fb::ebx::read_document(payload(where)); } catch (const std::exception&) { continue; }
if (!merged.root() || !merged.root()->object) continue;
for (const auto& field : merged.root()->object->fields) {
const auto* list = std::get_if<fb::ebx::Value::Array>(&field.value.data);
const auto* before = gameRoot->object->find(field.name);
const auto* beforeList = before ? std::get_if<fb::ebx::Value::Array>(&before->value.data) : nullptr;
if (!list || !beforeList || beforeList->empty() || list->size() <= beforeList->size()) continue;
// Does the game keep every one of its own entries in this bundle?
if (known->held.empty() && !known->unsure) known->held = held(0, *game, known->unsure);
if (known->unsure || !std::ranges::all_of(*beforeList, [&](const fb::ebx::Value& value) {
const auto* target = named(known->document, value);
return target && known->held.contains(*target); })) continue;
if (!here) here = held(1, bundle, unsure);
if (unsure) continue;
++copies;
std::set<fb::Guid> seen;
std::size_t absent{}, repeated{};
for (std::size_t index = 0; index < list->size(); ++index) {
const auto* target = named(merged, (*list)[index]);
if (!target) { if (index >= beforeList->size()) ++absent; continue; }
if (index < beforeList->size()) continue;
++entries;
absent += !here->contains(*target);
repeated += !seen.insert(*target).second;
}
const auto what = bundle.name + ": " + field.name + " of " + entry.asset->name;
if (absent) { auto& [count, example] = missing[what + ": " + std::to_string(absent) + " of " + std::to_string(list->size() - beforeList->size()) +
" added entr" + (absent == 1 ? "y names" : "ies name") + " a document that bundle does not hold"];
if (!count++) example = toc.relative; }
if (repeated) { auto& [count, example] = twice[what + ": " + std::to_string(repeated) + " added entr" + (repeated == 1 ? "y repeats" : "ies repeat") + " one a mod already added"];
if (!count++) example = toc.relative; }
}
}
}
for (const auto& [text, where] : missing)
fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")");
for (const auto& [text, where] : twice)
fail("list entry", text + " (" + std::to_string(where.first) + " cop" + (where.first == 1 ? "y" : "ies") + ", e.g. in " + where.second + ")");
std::cout << copies << " such list(s) with entries mods added, across every copy in the patch; " << entries << " added entries checked\n";
}
int summary() const {
section("Summary");
std::size_t total{};
for (const auto& [kind, count] : failures) { std::cout << count << " x " << kind << '\n'; total += count; }
std::cout << (total ? "mod merge audit: " + std::to_string(total) + " problem(s)" : std::string("mod merge audit: ok")) << '\n';
return total ? 1 : 0;
}
};
} // namespace
int main(int argc, char** argv) try {
if (argc < 2 || !fs::exists(fs::path(argv[1]) / "Data") || !fs::exists(fs::path(argv[1]) / "Mods")) {
std::cout << "No data root with Data and Mods given; skipped.\n";
return 0;
}
Audit audit;
audit.root = argv[1];
if (!audit.merge()) return audit.summary();
audit.load();
audit.placements();
audit.payloads(argc > 2 ? fs::path(argv[2]) : fs::path{});
audit.mods_against_patch();
audit.outcomes();
audit.lists();
return audit.summary();
} catch (const std::exception& error) {
std::cerr << "FAIL: " << error.what() << '\n';
return 1;
}
Engine/Vfs/mod_merge_combine.cpp
+111 -18
View File
@@ -12,6 +12,7 @@
#include <map>
#include <set>
#include <stdexcept>
#include <tuple>
namespace dingosdk::mods::detail {
namespace {
@@ -236,6 +237,22 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
std::set<std::string> addedFrom;
// The mods absorbed so far, so each asset can say whose copy it is.
std::set<std::string> owners;
// Every EBX a mod added to one of the game's bundles, by its document's guid
// and with the bundle it was added to: what a changed list names is looked
// up here, whichever bundle the mod put it in.
std::multimap<fb::Guid, std::pair<const std::string*, const AssetAddition*>> addedByGuid;
// The resource a mod added to a bundle under a name, by bundle, mod and name.
std::map<std::tuple<std::string, std::string, std::string>, const AssetAddition*> companions;
for (const auto& [bundleName, list] : overrides.added)
for (const auto& addition : list) {
if (addition.asset.kind == fb::AssetKind::ebx)
addedByGuid.emplace(addition.file, std::pair{&bundleName, &addition});
else if (addition.asset.kind == fb::AssetKind::resource)
companions.emplace(std::tuple{bundleName, addition.mod, lower(addition.asset.name)}, &addition);
}
// Where an addition's payload already went in this superbundle's patch
// archive, by install chunk: every copy that is given it reads the one.
std::map<std::pair<std::uint32_t, const AssetAddition*>, fb::BundleFileInfo> carriedAt;
const auto absorb = [&](const fb::TocBundle& bundle, const ArchivePlacement* placement,
bool isBase, const fs::path& root) {
@@ -385,6 +402,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
// The mod's unchanged copies of assets another mod changed: the change
// goes into the merged patch, next to this bundle's other files.
std::map<std::size_t, fb::BundleFileInfo> overridden;
// The documents those changes refer to: the ones a mod added come along below.
std::set<fb::Guid> wanted;
if (propagate && (casBacked || !region.inlineManifest.empty())) {
std::string example;
auto manifest = casBacked ? std::move(casManifest) : fb::read_binary_bundle(region.inlineManifest);
@@ -407,6 +426,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
asset.sha1 = change->second.sha1;
asset.originalSize = change->second.originalSize;
if (resource) asset.resourceMeta = resource->resourceMeta;
wanted.insert(change->second.names.begin(), change->second.names.end());
if (example.empty()) example = asset.name + " from " + change->second.mod;
} catch (const std::exception& error) {
report.notes.push_back(modName + ": " + bundle.name + ": " + asset.name +
@@ -426,23 +446,55 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
// loads instead of the game's on this mod's levels, so it gets them too.
std::vector<Asset> additions;
if (propagate && casBacked) {
if (const auto found = overrides.added.find(key); found != overrides.added.end()) {
std::vector<const AssetAddition*> take;
std::set<const AssetAddition*> taking;
// A copy in the TOC the adder ships itself merges with the adder's bundle and has
// its assets already; only copies in other superbundles (maps) need them carried.
const auto here = lower(relative);
if (const auto found = overrides.added.find(key); found != overrides.added.end())
for (const auto& addition : found->second) {
if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk ||
addition.toc == here) continue;
take.push_back(&addition);
taking.insert(&addition);
}
// What the changes this copy took name. The game keeps a list in more
// bundles than the one a mod added to (the music playlist is in eight),
// and a copy of any of them takes the changed list: it needs what the
// mod added next to that list, and what those refer to in turn.
for (std::vector<fb::Guid> open(wanted.begin(), wanted.end()); !open.empty();) {
const auto guid = open.back();
open.pop_back();
const auto [first, last] = addedByGuid.equal_range(guid);
for (auto at = first; at != last; ++at) {
const auto& [from, addition] = at->second;
if ((*from == key && addition->toc == here) || !taking.insert(addition).second) continue;
take.push_back(addition);
for (const auto& name : addition->names)
if (wanted.insert(name).second) open.push_back(name);
// The resource added under the same name goes where the asset goes.
if (const auto companion = companions.find({*from, addition->mod, lower(addition->asset.name)});
companion != companions.end() && taking.insert(companion->second).second)
take.push_back(companion->second);
}
}
if (!take.empty()) {
std::set<std::string, std::less<>> present;
for (const auto& asset : casManifest.ebx) present.insert(asset_key(asset));
for (const auto& asset : casManifest.resources) present.insert(asset_key(asset));
// A copy in the TOC the adder ships itself merges with the adder's bundle and has
// its assets already; only copies in other superbundles (maps) need them carried.
const auto here = lower(relative);
for (const auto& addition : found->second) {
if (addition.mod == modName || addition.asset.kind == fb::AssetKind::chunk ||
addition.toc == here || present.contains(asset_key(addition.asset))) continue;
const auto installChunk = region.files.front().location.installChunk;
for (const auto* addition : take) {
if (!present.insert(asset_key(addition->asset)).second) continue;
try {
additions.push_back({addition.asset, store.write(region.files.front().location.installChunk,
manifestArchive, addition.encoded)});
addedFrom.insert(addition.mod);
auto placed = carriedAt.find({installChunk, addition});
if (placed == carriedAt.end())
placed = carriedAt.emplace(std::pair{installChunk, addition},
store.write(installChunk, manifestArchive, addition->encoded)).first;
additions.push_back({addition->asset, placed->second});
addedFrom.insert(addition->mod);
} catch (const std::exception& error) {
report.notes.push_back(modName + ": " + bundle.name + ": " + addition.asset.name +
" from " + addition.mod + " could not be added (" + error.what() + ")");
report.notes.push_back(modName + ": " + bundle.name + ": " + addition->asset.name +
" from " + addition->mod + " could not be added (" + error.what() + ")");
}
}
if (!additions.empty())
@@ -733,15 +785,55 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
return known.fileGuid == item.fileGuid; }) ||
held.holds(item.fileGuid);
};
// Any other list loses an entry only when what it names was in
// this bundle and another mod's asset of the same name took its
// place: nothing else is known about where its entries live.
const fb::ebx::CarryImport notReplaced = [&](const fb::ebx::ImportReference& named) {
return !held.replacement(named.fileGuid) || held.holds(named.fileGuid);
};
fb::ebx::MergeSummary summary;
auto combined = fb::ebx::merge_documents(baseDocument, pointers, &summary,
itemList ? inBundle : fb::ebx::CarryImport{});
// One mod's copy with nothing to leave out stays as that mod wrote it.
if (summary.dropped.empty() &&
(edits.size() < 2 || (!summary.instances && !summary.arrayEntries))) continue;
itemList ? inBundle : notReplaced);
const auto mod_of = [&](std::size_t edit) { return path_utf8(edits[edit]->root.filename()); };
if (summary.dropped.empty() && !summary.instances && !summary.arrayEntries) {
// One mod's copy with nothing to leave out stays as that mod wrote it.
if (edits.size() < 2) continue;
// The same change from each of them is one change.
if (std::ranges::all_of(edits, [&](const Contribution* edit) {
return edit->sha1 == edits.front()->sha1; })) continue;
// Nothing was added to it, so each mod changed what was there.
// Values changed in place combine; when that is not all of
// some mod's change, the copy already kept stays whole, and
// the others' changes to it are not in the game.
if (!summary.values || !summary.exact()) {
std::set<std::string> others;
for (std::size_t index = 0; index < edits.size(); ++index)
if (edits[index]->sha1 != entry.asset.sha1) others.insert(mod_of(index));
if (entry.mod) others.erase(*entry.mod);
std::string names;
for (const auto& other : others) names += (names.empty() ? "" : ", ") + other;
if (!others.empty())
report.notes.push_back(entry.asset.name + ": kept " +
(entry.mod ? *entry.mod + "'s copy" : std::string("the highest-priority copy")) +
" whole; the changes " + names + " made to it are of a kind that cannot be "
"combined with it and are not in the game");
continue;
}
}
rebuilt = fb::ebx::write_document(combined);
what = std::to_string(summary.instances) + " instance(s), " +
std::to_string(summary.arrayEntries) + " list entries";
if (summary.values) what += ", " + std::to_string(summary.values) + " changed value(s)";
if (summary.contested) what += ", " + std::to_string(summary.contested) + " disagreed";
if (summary.repeated)
what += ", " + std::to_string(summary.repeated) + " repeated entry(ies) listed once";
// The merged copy is the game's with what the mods added: a mod's
// change to what was already there is in it only when it is one
// that could be taken.
for (const auto index : summary.uncarried)
report.notes.push_back(mod_of(index) + ": " + entry.asset.name +
": the merged copy has what this mod added to it, but not its changes to what "
"the game's copy already held");
std::map<std::size_t, std::vector<fb::Guid>> left; // by edit
for (const auto& gone : summary.dropped) left[gone.edit].push_back(gone.target.fileGuid);
for (const auto& [index, guids] : left) {
@@ -749,8 +841,9 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
const Asset* taken{};
for (const auto& guid : guids)
if ((taken = held.replacement(guid))) break;
report.notes.push_back(path_utf8(edits[index]->root.filename()) + ": " + entry.asset.name +
": " + std::to_string(guids.size()) + " item(s) left out of the list because the merged "
report.notes.push_back(mod_of(index) + ": " + entry.asset.name +
": " + std::to_string(guids.size()) + (itemList ? " item(s)" : " entry(ies)") +
" left out of the list because the merged "
"bundle does not hold them, e.g. " +
(taken ? taken->asset.name + ", replaced by " + (taken->mod ? *taken->mod : "another mod") +
"'s asset of the same name"
Engine/Vfs/mod_merge_internal.h
+11 -1
View File
@@ -180,13 +180,19 @@ struct AssetOverride {
// Present for a Lua resource replacement: its identity and metadata must
// travel with the script bytes, including source and bytecode sizes.
std::optional<fb::BundleAsset> resource;
// The documents the changed EBX refers to that its mod added to the bundle it
// made the change in, by their guids. They go wherever the change goes: the
// game keeps a list in more bundles than one, each with what the list names.
std::set<fb::Guid> names;
};
inline constexpr std::uint32_t luaScriptResourceType = 0xEC383B87;
// An EBX asset a mod added to one of the game's bundles, or a resource it added
// under the same name as such an asset (a wave's sound-bank). Maps carry their own
// copy of a bundle like bam_coregameassets, and that copy loads instead of the
// game's on their levels; an edit to a list there (the master music playlist)
// propagates as a change, so the asset it now names has to come along too.
// propagates as a change, so the asset it now names has to come along too. The
// game also keeps such a list in more bundles than the one the mod put its
// assets in, and a copy of any of those takes the change the same way.
struct AssetAddition {
std::string mod;
fb::BundleAsset asset;
@@ -195,6 +201,10 @@ struct AssetAddition {
// bundle in that same TOC merges with the mod's own bundle and already has the asset;
// only copies in other superbundles (a map's level TOC) need it carried.
std::string toc;
// For an EBX: its document's guid, which is what a list names it by, and the
// guids of the documents it refers to in turn (a song names its wave).
fb::Guid file;
std::vector<fb::Guid> names;
};
struct AssetOverrides {
// By lower-case asset name, then the game's sha1 the change replaces.
Engine/Vfs/mod_merge_overrides.cpp
+111 -46
View File
@@ -31,6 +31,20 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
std::vector<std::pair<std::string, AssetAddition>> candidates, companions;
std::vector<fb::TocChunk> newChunks; // TOC chunks the game's TOCs do not have
std::set<fb::Guid> imported;
// The game's assets this mod changed. They wait until its additions are
// settled: one that another mod's asset of the same name stands in for
// is not where a copy can find it, and a change must not name it there.
struct Change {
std::string bundle; // lower case: where the mod changed it
std::string name; // lower case
std::string asset; // as the bundle spells it
fb::Sha1 game; // the game's copy it replaces
fb::Sha1 sha1;
std::uint64_t originalSize{};
std::vector<std::byte> encoded;
std::optional<fb::BundleFileInfo> gameFile; // where the game's copy is
};
std::vector<Change> changes;
const auto decoded = [&](std::span<const std::byte> encoded) {
return fb::ebx::read_document(fb::decode_cas(encoded, {gameRoot}));
};
@@ -53,13 +67,17 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
const auto modListing = list_bundle(store, mod->directory, baseRoot, bundle, gameRoot);
const auto gameListing = list_bundle(store, baseRoot, baseRoot, *shipped->second, gameRoot);
if (!modListing || !gameListing) continue;
std::map<std::string, fb::Sha1, std::less<>> original;
for (const auto& asset : gameListing->manifest.ebx) original.emplace(lower(asset.name), asset.sha1);
// The game's copy of each asset: its sha1, and where in the listing it is.
std::map<std::string, std::pair<fb::Sha1, std::size_t>, std::less<>> original;
for (std::size_t index = 0; index < gameListing->manifest.ebx.size(); ++index) {
const auto& asset = gameListing->manifest.ebx[index];
original.emplace(lower(asset.name), std::pair{asset.sha1, index});
}
for (std::size_t index = 0; index < modListing->manifest.ebx.size(); ++index) {
const auto& asset = modListing->manifest.ebx[index];
const auto name = lower(asset.name);
const auto game_copy = original.find(name);
if (game_copy != original.end() && game_copy->second == asset.sha1) continue;
if (game_copy != original.end() && game_copy->second.first == asset.sha1) continue;
const auto at = modListing->first + index;
if (at >= modListing->files.size()) continue;
const auto& file = modListing->files[at];
@@ -71,44 +89,25 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
candidates.push_back({lower(bundle.name), {mod->name, asset, payload(), lower(relative)}});
continue;
}
const auto encodedPayload = payload();
auto encodedPayload = payload();
try {
for (const auto& reference : decoded(encodedPayload).imports)
imported.insert(reference.fileGuid);
} catch (const std::exception&) {}
auto& versions = out.changed[name];
const auto existing = versions.find(game_copy->second);
if (existing == versions.end()) {
versions.emplace(game_copy->second,
AssetOverride{mod->name, asset.sha1, asset.originalSize, encodedPayload});
++changed;
} else {
try {
std::size_t baseIndex = 0;
while (baseIndex < gameListing->manifest.ebx.size() &&
lower(gameListing->manifest.ebx[baseIndex].name) != name)
++baseIndex;
if (gameListing->first + baseIndex < gameListing->files.size()) {
const auto& baseFile = gameListing->files[gameListing->first + baseIndex];
const auto baseBytes = store.read(baseRoot, baseFile.location, baseFile.offset, baseFile.size);
const auto baseDoc = decoded(baseBytes);
const auto existingDoc = decoded(existing->second.encoded);
const auto thisDoc = decoded(encodedPayload);
const std::array<const fb::ebx::Document*, 2> editDocs{&existingDoc, &thisDoc};
fb::ebx::MergeSummary summary;
auto combined = fb::ebx::merge_documents(baseDoc, editDocs, &summary);
if (summary.instances || summary.arrayEntries) {
const auto rebuilt = fb::ebx::write_document(combined);
existing->second.sha1 = sha1_of(rebuilt);
existing->second.originalSize = rebuilt.size();
existing->second.encoded = fb::encode_cas(rebuilt, {gameRoot});
existing->second.mod += " + " + mod->name;
for (const auto& reference : combined.imports)
imported.insert(reference.fileGuid);
}
}
// A higher mod's version of the asset, which this change is
// combined with below, may name what this mod adds.
if (const auto versions = out.changed.find(name); versions != out.changed.end())
if (const auto existing = versions->second.find(game_copy->second.first);
existing != versions->second.end()) try {
for (const auto& reference : decoded(existing->second.encoded).imports)
imported.insert(reference.fileGuid);
} catch (const std::exception&) {}
}
std::optional<fb::BundleFileInfo> gameFile;
if (const auto gameAt = gameListing->first + game_copy->second.second;
gameAt < gameListing->files.size())
gameFile = gameListing->files[gameAt];
changes.push_back({lower(bundle.name), name, asset.name, game_copy->second.first, asset.sha1,
asset.originalSize, std::move(encodedPayload), gameFile});
}
// Maps carry stock SkaterLoader too. Replacing it only in the game's
// bundles loses custom cosmetic material targets on a map transition.
@@ -157,6 +156,9 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
// is, and only one of them can be what a copy gets: say whose, so a song or an
// item that goes missing on a map can be traced to the mod that took its name.
std::map<std::string, std::pair<std::size_t, std::string>, std::less<>> shadowed; // by the mod kept
// This mod's added EBX that another mod's different document stands in for: what
// names one by its guid finds nothing in a copy that was given the other's.
std::set<fb::Guid> lost;
const auto keep = [&](const std::string& bundle, AssetAddition addition) {
auto& list = out.added[bundle];
const auto name = lower(addition.asset.name);
@@ -166,6 +168,8 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
if (holder->mod != addition.mod && holder->asset.sha1 != addition.asset.sha1) {
auto& [count, example] = shadowed[holder->mod];
if (!count++) example = addition.asset.name;
if (addition.asset.kind == fb::AssetKind::ebx && holder->file != addition.file)
lost.insert(addition.file);
}
return false;
}
@@ -174,23 +178,25 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
};
// Transitively: an addition a following addition imports follows too (a new song
// imports its new wave, and only the song is named by the changed playlist).
struct Candidate { fb::Guid file; std::vector<fb::Guid> imports; bool taken{}; };
std::vector<Candidate> parsed(candidates.size());
std::vector<bool> followed(candidates.size());
// The documents this mod added to each bundle, by guid.
std::map<std::string, std::set<fb::Guid>, std::less<>> addedIn;
for (std::size_t index = 0; index < candidates.size(); ++index) {
auto& addition = candidates[index].second;
try {
const auto document = decoded(candidates[index].second.encoded);
parsed[index].file = document.fileGuid;
for (const auto& reference : document.imports) parsed[index].imports.push_back(reference.fileGuid);
} catch (const std::exception&) { parsed[index].taken = true; } // unreadable: never follows
const auto document = decoded(addition.encoded);
addition.file = document.fileGuid;
for (const auto& reference : document.imports) addition.names.push_back(reference.fileGuid);
addedIn[candidates[index].first].insert(addition.file);
} catch (const std::exception&) { followed[index] = true; } // unreadable: never follows
}
for (bool grew = true; grew;) {
grew = false;
for (std::size_t index = 0; index < candidates.size(); ++index) {
auto& candidate = parsed[index];
if (candidate.taken || !imported.contains(candidate.file)) continue;
candidate.taken = grew = true;
imported.insert(candidate.imports.begin(), candidate.imports.end());
auto& [bundle, addition] = candidates[index];
if (followed[index] || !imported.contains(addition.file)) continue;
followed[index] = grew = true;
imported.insert(addition.names.begin(), addition.names.end());
keep(bundle, std::move(addition));
}
}
@@ -204,6 +210,65 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
lower(addition.asset.name) == name; }))
keep(bundle, std::move(companion));
}
for (auto& change : changes) {
// What the change names of the assets the mod added beside it. The mod
// put them in one bundle with what names them, so they stay together.
std::set<fb::Guid> names;
const auto beside = addedIn.find(change.bundle);
try {
auto document = decoded(change.encoded);
const auto stays = [&](const fb::ebx::ImportReference& reference) {
return !lost.contains(reference.fileGuid); };
if (!std::ranges::all_of(document.imports, stays))
if (const auto gone = fb::ebx::drop_root_references(document, stays)) {
const auto rebuilt = fb::ebx::write_document(document);
change.sha1 = sha1_of(rebuilt);
change.originalSize = rebuilt.size();
change.encoded = fb::encode_cas(rebuilt, {gameRoot});
report.notes.push_back(mod->name + ": " + change.asset + ": " + std::to_string(gone) +
" entry(ies) left out of the copies other mods carry, where another mod's asset of "
"the same name stands in for what they name");
}
if (beside != addedIn.end())
for (const auto& reference : document.imports)
if (beside->second.contains(reference.fileGuid) && stays(reference))
names.insert(reference.fileGuid);
} catch (const std::exception&) {}
auto& versions = out.changed[change.name];
const auto existing = versions.find(change.game);
if (existing == versions.end()) {
versions.emplace(change.game, AssetOverride{mod->name, change.sha1, change.originalSize,
std::move(change.encoded), std::nullopt, std::move(names)});
++changed;
continue;
}
// The same change again, from another of the mod's bundles or another mod.
if (existing->second.sha1 == change.sha1) {
existing->second.names.insert(names.begin(), names.end());
continue;
}
if (!change.gameFile) continue;
try {
const auto baseDoc = decoded(store.read(baseRoot, change.gameFile->location,
change.gameFile->offset, change.gameFile->size));
const auto existingDoc = decoded(existing->second.encoded);
const auto thisDoc = decoded(change.encoded);
// Lowest priority first, the order a bundle's own copies are combined
// in: where two mods changed one value, the higher mod's is kept.
const std::array<const fb::ebx::Document*, 2> editDocs{&thisDoc, &existingDoc};
fb::ebx::MergeSummary summary;
auto combined = fb::ebx::merge_documents(baseDoc, editDocs, &summary);
// Changed values alone are only worth a copy that has all of them.
if (!summary.instances && !summary.arrayEntries && !(summary.values && summary.exact())) continue;
const auto rebuilt = fb::ebx::write_document(combined);
existing->second.sha1 = sha1_of(rebuilt);
existing->second.originalSize = rebuilt.size();
existing->second.encoded = fb::encode_cas(rebuilt, {gameRoot});
if (existing->second.mod != mod->name && !existing->second.mod.ends_with(" + " + mod->name))
existing->second.mod += " + " + mod->name;
existing->second.names.insert(names.begin(), names.end());
} catch (const std::exception&) {}
}
for (const auto& [other, clash] : shadowed)
report.notes.push_back(mod->name + ": " + std::to_string(clash.first) +
" added asset(s) share a name with ones " + other + " adds, e.g. " + clash.second +
cmake/Apps.cmake
+8
View File
@@ -68,6 +68,14 @@ if(DINGOSDK_BUILD_LAUNCHER_TESTS AND WIN32)
target_link_libraries(dingosdk_item_list_merge_tests PRIVATE dingosdk_mods)
target_include_directories(dingosdk_item_list_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}")
add_test(NAME item_list_merge COMMAND dingosdk_item_list_merge_tests "${DINGOSDK_TEST_GAME_ROOT}")
add_executable(dingosdk_ebx_merge_tests Engine/Vfs/Test/ebx_merge_tests.cpp)
target_link_libraries(dingosdk_ebx_merge_tests PRIVATE dingosdk_mods)
target_include_directories(dingosdk_ebx_merge_tests PRIVATE "${PROJECT_SOURCE_DIR}")
add_test(NAME ebx_merge COMMAND dingosdk_ebx_merge_tests)
add_executable(dingosdk_mod_merge_audit_tests Engine/Vfs/Test/mod_merge_audit_tests.cpp)
target_link_libraries(dingosdk_mod_merge_audit_tests PRIVATE dingosdk_mods)
target_include_directories(dingosdk_mod_merge_audit_tests PRIVATE "${PROJECT_SOURCE_DIR}")
add_test(NAME mod_merge_audit COMMAND dingosdk_mod_merge_audit_tests "${DINGOSDK_TEST_GAME_ROOT}")
add_executable(dingosdk_store_copies_tests Engine/Vfs/Test/store_copies_tests.cpp)
target_link_libraries(dingosdk_store_copies_tests PRIVATE dingosdk_mods dingosdk_content_cache)
target_include_directories(dingosdk_store_copies_tests PRIVATE "${PROJECT_SOURCE_DIR}")