From f22adf2348b8a07bb8abd0e41643094002029a36 Mon Sep 17 00:00:00 2001 From: Dave Johnson Date: Fri, 16 Oct 2026 17:25:36 +0000 Subject: [PATCH 1/3] EBX writer: keep the arrays and strings boxed values point at A rewrite copied a boxed value's bytes, pointers and all, but wrote the arrays and strings those pointers reach only when a field also reached them. Rewritten documents lost that data and the boxed pointers aimed at whatever moved into its place; the mod merge rewrites real documents this way. Read documents now keep their EBXD bytes, and the writer carries every array, string, instance and boxed value a boxed pointer reaches, repointed. Found by a round-trip check on bam's level root (one string and about 1,120 bytes of arrays lost). --- Engine/Resource/ebx_document.cpp | 2 + Engine/Resource/ebx_document.h | 3 + Engine/Resource/ebx_writer.cpp | 137 ++++++++++++++++++++++++++++--- 3 files changed, 130 insertions(+), 12 deletions(-) diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index 3f439739..efadfaa6 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -615,6 +615,8 @@ Document read_document(const std::span bytes) { result.hashedReflectionNames = reflection2 != nullptr; result.reflectionData.assign(bytes.begin() + static_cast(reflectionChunk.begin), bytes.begin() + static_cast(reflectionChunk.end)); + result.payload.assign(bytes.begin() + static_cast(result.dataStart), + bytes.begin() + static_cast(result.dataEnd)); map_instances(result, fixup, reflectionIds); ValueReader(bytes, result).read_instances(); std::ranges::sort(result.boxedValues, {}, &BoxedValueRecord::offset); diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index 4ae98098..ce63e81a 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -189,6 +189,9 @@ struct Document { std::vector importOffsets; std::vector typeInfoOffsets; std::vector boxedValueSlots; + // The EBXD bytes the document was read from (offsets as in the fixup tables): what boxed + // values point at (arrays, strings) is carried from here when the document is written. + std::vector payload; [[nodiscard]] const InstanceRecord* root() const noexcept; }; diff --git a/Engine/Resource/ebx_writer.cpp b/Engine/Resource/ebx_writer.cpp index a1d7bcf6..1924d662 100644 --- a/Engine/Resource/ebx_writer.cpp +++ b/Engine/Resource/ebx_writer.cpp @@ -359,6 +359,124 @@ private: for (std::size_t index = 0; index < arraysPending_.size(); ++index) { if (!arraysPending_[index].emitted) emit_array(index); } + carry_boxed_arrays(); + } + + // What a boxed value points at is not in the document's values: its bytes are carried as they + // were, so the arrays and strings its pointers reach are carried with them and the pointers + // moved to the new places. Without this a rewrite dropped them (seen on the game's own + // level roots: a boxed debug string and about 1 KB of arrays). + + enum class Target { array, string, instance, boxed, empty, unknown }; + + std::pair old_target(const std::uint32_t slot) const { + const auto& old = document_.payload; + std::int32_t displacement{}; + std::memcpy(&displacement, old.data() + slot, sizeof(displacement)); + const auto target = static_cast(static_cast(slot) + displacement); + if (oldArrays_.contains(target)) return {Target::array, target}; + if (document_.stringsOffset <= target && target < old.size()) return {Target::string, target}; + if (const auto found = oldInstances_.find(target); found != oldInstances_.end()) + return {Target::instance, found->second}; + if (document_.boxedValuesOffset <= target && target < document_.stringsOffset) return {Target::boxed, target}; + if (target >= 4 && target <= old.size()) { + std::uint32_t count{}; + std::memcpy(&count, old.data() + target - 4, sizeof(count)); + if (count == 0) return {Target::empty, target}; + } + return {Target::unknown, target}; + } + + static std::vector old_slots(const std::uint32_t start, const std::size_t length, + std::vector offsets) { + std::ranges::sort(offsets); + offsets.erase(std::unique(offsets.begin(), offsets.end()), offsets.end()); + std::vector out; + for (const auto value : offsets) + if (value >= start && value - start < length) out.push_back(value); + return out; + } + + void carry_boxed_arrays() { + if (document_.payload.empty()) return; + for (const auto& record : document_.arrays) oldArrays_.emplace(record.offset, record); + for (std::size_t index = 0; index < document_.instances.size(); ++index) + oldInstances_.emplace(document_.instances[index].dataOffset, static_cast(index)); + auto records = document_.boxedValues; + std::ranges::sort(records, {}, &BoxedValueRecord::offset); + for (const auto& source : records) + for (const auto slot : old_slots(source.offset, source.rawBytes.size(), document_.pointerOffsets)) { + const auto [kind, key] = old_target(slot); + if (kind == Target::array) carry_array(oldArrays_.at(key)); + } + } + + std::pair array_element(const ArrayRecord& record) const { + FieldDescriptor probe; + probe.encodedType = static_cast(record.encodedType >> 1); + if (probe.type() == FieldType::structure) { + const auto slot = record.classRef; + if (slot < document_.fixupTypeGuids.size() && slot < document_.fixupTypeSignatures.size()) + for (const auto& type : document_.types) + if (type.guid == document_.fixupTypeGuids[slot] && type.signature == document_.fixupTypeSignatures[slot]) + return {std::max(type.size, 1), std::max(type.alignment, 1)}; + throw std::runtime_error("EBX array record has an unknown struct type"); + } + const auto stride = scalar_size(probe.type()); + return {stride, std::max(std::min(stride, 8), 1)}; + } + + void carry_array(const ArrayRecord& record) { + if (carried_.contains(record.offset)) return; + const auto [stride, alignment] = array_element(record); + const auto length = stride * record.count; + if (record.offset + length > document_.payload.size()) throw std::runtime_error("EBX carried array exceeds EBXD"); + const auto data = align_up(payload_.size() + 4, alignment); + put_number(data - 4, record.count); + put(data, std::span(document_.payload).subspan(record.offset, length)); + const auto placed = checked_u32(data, "EBX array data"); + carried_[record.offset] = placed; + arrays_.push_back({placed, record.count, record.hash, record.encodedType, record.classRef}); + relocate(record.offset, length, data); + } + + void relocate(const std::uint32_t oldStart, const std::size_t length, const std::size_t newStart) { + const auto moved = [&](const std::uint32_t old) { + return checked_u32(newStart + (old - oldStart), "EBX carried relocation"); + }; + for (const auto old : old_slots(oldStart, length, document_.resourceRefOffsets)) resourceOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.importOffsets)) importOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.typeInfoOffsets)) typeInfoOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.pointerOffsets)) { + const auto slot = moved(old); + pointerOffsets_.push_back(slot); + if (document_.payload.empty()) continue; + const auto [kind, key] = old_target(old); + switch (kind) { + case Target::array: + carry_array(oldArrays_.at(key)); + put_number(slot, static_cast(carried_.at(key) - slot)); + break; + case Target::string: { + const auto& source = document_.payload; + auto end = static_cast(key); + while (end < source.size() && source[end] != std::byte{0}) ++end; + stringsPending_.push_back({slot, std::string(reinterpret_cast(source.data()) + key, end - key)}); + break; + } + case Target::instance: + put_number(slot, static_cast(instanceOffsets_.at(key) - slot)); + break; + case Target::empty: + put_number(slot, static_cast(emptyArrayTarget_ - slot)); + break; + case Target::boxed: + boxedTargets_.emplace_back(slot, key); + break; + case Target::unknown: + break; + } + } } void emit_array(const std::size_t index) { @@ -407,23 +525,14 @@ private: if (source.offset < document_.boxedValuesOffset) throw std::runtime_error("EBX boxed record precedes its section"); const auto offset = static_cast(boxedOffset_) + source.offset - document_.boxedValuesOffset; put(offset, source.rawBytes); - const auto relocate = [&](const std::vector& sourceOffsets, - std::vector& destinationOffsets) { - for (const auto old : sourceOffsets) { - if (old < source.offset || old - source.offset >= source.rawBytes.size()) continue; - destinationOffsets.push_back(checked_u32(offset + old - source.offset, - "EBX boxed relocation")); - } - }; - relocate(document_.pointerOffsets, pointerOffsets_); - relocate(document_.resourceRefOffsets, resourceOffsets_); - relocate(document_.importOffsets, importOffsets_); - relocate(document_.typeInfoOffsets, typeInfoOffsets_); + relocate(source.offset, source.rawBytes.size(), offset); moved[source.offset] = checked_u32(offset, "EBX boxed value"); auto authored = source; authored.offset = checked_u32(offset, "EBX boxed value"); boxed_.push_back(std::move(authored)); } + for (const auto& [slot, target] : boxedTargets_) + put_number(slot, static_cast(boxedOffset_ + (target - document_.boxedValuesOffset) - slot)); for (const auto& pending : boxesPending_) { put_number(pending.slot, pending.value.encodedType); put_number(static_cast(pending.slot) + 4, std::uint32_t{}); @@ -588,6 +697,10 @@ private: std::uint32_t arraysOffset_{}; std::uint32_t boxedOffset_{}; std::uint32_t emptyArrayTarget_{}; + std::unordered_map oldArrays_; + std::unordered_map oldInstances_; + std::unordered_map carried_; + std::vector> boxedTargets_; }; } From db0d1f348c3db6c69bb385979742a2e5cea9f649 Mon Sep 17 00:00:00 2001 From: Dave Johnson Date: Fri, 16 Oct 2026 17:46:37 +0000 Subject: [PATCH 2/3] EBX writer: rewritten arrays keep their own record hash Each array record carries a hash the original files make specific to the field or element type; the writer used to give every array the hash of the first record of the same element type, so a rewrite changed hundreds of them in bam's level root. Arrays read from a document now keep their record's hash; only arrays with no source record borrow as before. --- Engine/Resource/ebx_document.cpp | 20 ++++++++++++++++++-- Engine/Resource/ebx_document.h | 4 ++++ Engine/Resource/ebx_writer.cpp | 8 ++++++-- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index efadfaa6..a81ef1d8 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -1,6 +1,7 @@ #include "ebx_document.h" #include +#include #include #include #include @@ -356,7 +357,9 @@ void map_instances(Document& document, class ValueReader final { public: ValueReader(const std::span bytes, Document& document) - : bytes_(bytes), reader_(bytes), document_(document) {} + : bytes_(bytes), reader_(bytes), document_(document) { + for (const auto& record : document.arrays) arrayHashes_.emplace(record.offset, record.hash); + } void read_instances() { for (auto& instance : document_.instances) { @@ -423,7 +426,19 @@ private: } const auto position = start + field.dataOffset; if (position > document_.dataEnd) throw std::runtime_error("EBX field offset exceeds EBXD"); - output.fields.push_back({descriptorIndex, field.name, read_field(field, position)}); + auto value = read_field(field, position); + if (field.category() == FieldCategory::array) { + const auto* values = std::get_if(&value.data); + if (values && !values->empty()) { + std::int32_t displacement{}; + std::memcpy(&displacement, bytes_.data() + position, sizeof(displacement)); + const auto target = static_cast(position) + displacement - + static_cast(document_.dataStart); + if (const auto found = arrayHashes_.find(target); found != arrayHashes_.end()) + value.arrayHash = found->second; + } + } + output.fields.push_back({descriptorIndex, field.name, std::move(value)}); } } @@ -552,6 +567,7 @@ private: } std::span bytes_; + std::unordered_map arrayHashes_; // record offset -> its hash BinaryReader reader_; Document& document_; unsigned depth_{}; diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index ce63e81a..947558dc 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -149,6 +150,9 @@ struct Value { PointerReference, TypeReference, BoxedReference, std::shared_ptr, Array>; Storage data; + // An array's own record hash, kept when the document is written again; a new array has none + // and borrows one (ebx_writer.cpp, array_hash). + std::optional arrayHash; }; struct FieldValue { diff --git a/Engine/Resource/ebx_writer.cpp b/Engine/Resource/ebx_writer.cpp index 1924d662..23b4ea7c 100644 --- a/Engine/Resource/ebx_writer.cpp +++ b/Engine/Resource/ebx_writer.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +107,7 @@ private: std::uint32_t slot{}; const Value::Array* values{}; std::size_t descriptor{}; + std::optional hash; bool emitted{}; }; @@ -208,7 +210,7 @@ private: static const Value::Array empty; const auto* array = value ? std::get_if(&value->data) : nullptr; arraysPending_.push_back({checked_u32(position, "EBX array slot"), array ? array : &empty, - descriptorIndex}); + descriptorIndex, value ? value->arrayHash : std::nullopt}); } else if (field.type() == FieldType::structure) { if (field.classRef >= document_.types.size()) throw std::runtime_error("EBX struct type is invalid"); const auto nestedStart = align_up(position, std::max(document_.types[field.classRef].alignment, 1)); @@ -505,8 +507,10 @@ private: write_scalar(field, elementStart, &value); } } + // An array read from a record keeps that record's hash; a new one borrows (array_hash). + const auto hash = arraysPending_[index].hash ? *arraysPending_[index].hash : array_hash(field); arrays_.push_back({checked_u32(data, "EBX array data"), checked_u32(values->size(), "EBX array count"), - array_hash(field), static_cast(field.encodedType << 1), + hash, static_cast(field.encodedType << 1), field.type() == FieldType::structure ? fixup_slot_for_descriptor(field.classRef) : static_cast(0xFFFF)}); From 9dbcef7557865c762688fc640058aa7801a91694 Mon Sep 17 00:00:00 2001 From: Dave Johnson Date: Fri, 16 Oct 2026 18:00:32 +0000 Subject: [PATCH 3/3] EBX writer: a new array takes the hash of its own field An array record's hash is one per field type in a document (every array the base game's files hold follows this, and each such hash is a key of the document's reflection group table, now read). A new array takes the hash of an existing array of the same holding type and field; callers can ask the writer to refuse when there is none, and the mod merge keeps the old fallback. Checked on the game's own level documents. --- Engine/Resource/ebx_document.cpp | 10 +++++- Engine/Resource/ebx_document.h | 9 +++++ Engine/Resource/ebx_writer.cpp | 58 ++++++++++++++++++++++++++++---- Engine/Resource/ebx_writer.h | 6 +++- 4 files changed, 75 insertions(+), 8 deletions(-) diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index a81ef1d8..c0960a9a 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -293,7 +293,14 @@ std::vector read_reflection(BinaryReader& reader, const auto groupCount = static_cast(nonnegative(reader, "reflection group")); ensure_records(reader, chunk.end, groupCount, 12, "reflection group"); - reader.skip(static_cast(groupCount * 12)); + document.reflectionGroups.reserve(groupCount); + for (std::size_t index = 0; index < groupCount; ++index) { + ReflectionGroup group; + group.hash = reader.u32(); + group.count = reader.u32(); + group.first = reader.u32(); + document.reflectionGroups.push_back(group); + } const auto mappingCount = static_cast(nonnegative(reader, "reflection mapping")); ensure_records(reader, chunk.end, mappingCount, 8, "reflection mapping"); reader.skip(static_cast(mappingCount * 8)); @@ -568,6 +575,7 @@ private: std::span bytes_; std::unordered_map arrayHashes_; // record offset -> its hash + BinaryReader reader_; Document& document_; unsigned depth_{}; diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index 947558dc..e99b2f45 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -96,6 +96,14 @@ struct ArrayRecord { std::uint16_t classRef{}; }; +// A 12-byte record of the RFL2 group table: one per array type the document uses, sorted by +// hash (first[i + 1] = first[i] + count[i]). Every array record's hash is one of these keys. +struct ReflectionGroup { + std::uint32_t hash{}; + std::uint32_t count{}; + std::uint32_t first{}; +}; + struct BoxedValueRecord { std::uint32_t offset{}; std::uint32_t count{}; @@ -196,6 +204,7 @@ struct Document { // The EBXD bytes the document was read from (offsets as in the fixup tables): what boxed // values point at (arrays, strings) is carried from here when the document is written. std::vector payload; + std::vector reflectionGroups; [[nodiscard]] const InstanceRecord* root() const noexcept; }; diff --git a/Engine/Resource/ebx_writer.cpp b/Engine/Resource/ebx_writer.cpp index 23b4ea7c..395b6455 100644 --- a/Engine/Resource/ebx_writer.cpp +++ b/Engine/Resource/ebx_writer.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -86,7 +87,8 @@ std::size_t scalar_size(const FieldType type) { class Builder final { public: - explicit Builder(const Document& document) : document_(document) {} + Builder(const Document& document, const bool refuseNewArrayHashes) + : document_(document), refuse_(refuseNewArrayHashes) {} std::vector run() { validate(); @@ -108,6 +110,7 @@ private: const Value::Array* values{}; std::size_t descriptor{}; std::optional hash; + std::int32_t holder{-1}; // type of the object holding the field bool emitted{}; }; @@ -210,7 +213,8 @@ private: static const Value::Array empty; const auto* array = value ? std::get_if(&value->data) : nullptr; arraysPending_.push_back({checked_u32(position, "EBX array slot"), array ? array : &empty, - descriptorIndex, value ? value->arrayHash : std::nullopt}); + descriptorIndex, value ? value->arrayHash : std::nullopt, + object ? object->descriptor : static_cast(typeIndex)}); } else if (field.type() == FieldType::structure) { if (field.classRef >= document_.types.size()) throw std::runtime_error("EBX struct type is invalid"); const auto nestedStart = align_up(position, std::max(document_.types[field.classRef].alignment, 1)); @@ -354,6 +358,44 @@ private: return field.nameHash; } + // A new array takes the hash of an array the document read for the same field of the same type; + // failing that it borrows (array_hash), or is refused (see write_document). + std::uint32_t new_array_hash(const PendingArray& pending, const FieldDescriptor& field) { + if (!knownWalked_) { + knownWalked_ = true; + for (const auto& instance : document_.instances) + if (instance.object) collect_hashes(*instance.object); + } + const auto found = knownHashes_.find({pending.holder, pending.descriptor}); + if (refuse_) { + if (found == knownHashes_.end()) + throw std::runtime_error("EBX new array " + std::string(field.name) + + " has no array of its field to take a hash from"); + if (document_.hashedReflectionNames && + std::ranges::none_of(document_.reflectionGroups, + [&](const ReflectionGroup& group) { return group.hash == found->second; })) + throw std::runtime_error("EBX array hash of " + std::string(field.name) + + " is not in the reflection's group table"); + } + return found != knownHashes_.end() ? found->second : array_hash(field); + } + + void collect_hashes(const Object& object) { + for (const auto& field : object.fields) { + if (field.value.arrayHash) + knownHashes_.emplace(std::pair{object.descriptor, field.descriptor}, *field.value.arrayHash); + collect_value(field.value); + } + } + + void collect_value(const Value& value) { + if (const auto* nested = std::get_if>(&value.data)) { + if (*nested) collect_hashes(**nested); + } else if (const auto* array = std::get_if(&value.data)) { + for (const auto& element : *array) collect_value(element); + } + } + void write_arrays() { arraysOffset_ = checked_u32(align_up(payload_.size(), 16), "EBX array section"); grow(static_cast(arraysOffset_) + 32); @@ -508,9 +550,10 @@ private: } } // An array read from a record keeps that record's hash; a new one borrows (array_hash). - const auto hash = arraysPending_[index].hash ? *arraysPending_[index].hash : array_hash(field); + auto hash = arraysPending_[index].hash; + if (!hash) hash = new_array_hash(arraysPending_[index], field); arrays_.push_back({checked_u32(data, "EBX array data"), checked_u32(values->size(), "EBX array count"), - hash, static_cast(field.encodedType << 1), + *hash, static_cast(field.encodedType << 1), field.type() == FieldType::structure ? fixup_slot_for_descriptor(field.classRef) : static_cast(0xFFFF)}); @@ -705,6 +748,9 @@ private: std::unordered_map oldInstances_; std::unordered_map carried_; std::vector> boxedTargets_; + bool refuse_{}; + bool knownWalked_{}; + std::map, std::uint32_t> knownHashes_; // (holder, field) -> hash }; } @@ -728,8 +774,8 @@ std::uint16_t ensure_fixup_type(Document& document, const std::int32_t descripto return static_cast(count); } -std::vector write_document(const Document& document) { - return Builder(document).run(); +std::vector write_document(const Document& document, const bool refuseNewArrayHashes) { + return Builder(document, refuseNewArrayHashes).run(); } } diff --git a/Engine/Resource/ebx_writer.h b/Engine/Resource/ebx_writer.h index c042f681..96179a0d 100644 --- a/Engine/Resource/ebx_writer.h +++ b/Engine/Resource/ebx_writer.h @@ -9,6 +9,10 @@ namespace dingosdk::frostbite::ebx { [[nodiscard]] std::uint16_t ensure_fixup_type(Document& document, std::int32_t descriptor); -[[nodiscard]] std::vector write_document(const Document& document); +// A new array (one no record was read for) takes the hash of an array the document read for the +// same field of the same type. Without one it borrows another array's hash (the merge's way), or, +// with refuseNewArrayHashes (the map builder's), the write is refused: the RFL2 group table would +// need a new entry. A hash that is not a key of that table is refused the same way. +[[nodiscard]] std::vector write_document(const Document& document, bool refuseNewArrayHashes = false); }