diff --git a/Engine/Resource/ebx_document.cpp b/Engine/Resource/ebx_document.cpp index 3f439739..efadfaa6 100644 --- a/Engine/Resource/ebx_document.cpp +++ b/Engine/Resource/ebx_document.cpp @@ -615,6 +615,8 @@ Document read_document(const std::span bytes) { result.hashedReflectionNames = reflection2 != nullptr; result.reflectionData.assign(bytes.begin() + static_cast(reflectionChunk.begin), bytes.begin() + static_cast(reflectionChunk.end)); + result.payload.assign(bytes.begin() + static_cast(result.dataStart), + bytes.begin() + static_cast(result.dataEnd)); map_instances(result, fixup, reflectionIds); ValueReader(bytes, result).read_instances(); std::ranges::sort(result.boxedValues, {}, &BoxedValueRecord::offset); diff --git a/Engine/Resource/ebx_document.h b/Engine/Resource/ebx_document.h index 4ae98098..ce63e81a 100644 --- a/Engine/Resource/ebx_document.h +++ b/Engine/Resource/ebx_document.h @@ -189,6 +189,9 @@ struct Document { std::vector importOffsets; std::vector typeInfoOffsets; std::vector boxedValueSlots; + // The EBXD bytes the document was read from (offsets as in the fixup tables): what boxed + // values point at (arrays, strings) is carried from here when the document is written. + std::vector payload; [[nodiscard]] const InstanceRecord* root() const noexcept; }; diff --git a/Engine/Resource/ebx_writer.cpp b/Engine/Resource/ebx_writer.cpp index a1d7bcf6..1924d662 100644 --- a/Engine/Resource/ebx_writer.cpp +++ b/Engine/Resource/ebx_writer.cpp @@ -359,6 +359,124 @@ private: for (std::size_t index = 0; index < arraysPending_.size(); ++index) { if (!arraysPending_[index].emitted) emit_array(index); } + carry_boxed_arrays(); + } + + // What a boxed value points at is not in the document's values: its bytes are carried as they + // were, so the arrays and strings its pointers reach are carried with them and the pointers + // moved to the new places. Without this a rewrite dropped them (seen on the game's own + // level roots: a boxed debug string and about 1 KB of arrays). + + enum class Target { array, string, instance, boxed, empty, unknown }; + + std::pair old_target(const std::uint32_t slot) const { + const auto& old = document_.payload; + std::int32_t displacement{}; + std::memcpy(&displacement, old.data() + slot, sizeof(displacement)); + const auto target = static_cast(static_cast(slot) + displacement); + if (oldArrays_.contains(target)) return {Target::array, target}; + if (document_.stringsOffset <= target && target < old.size()) return {Target::string, target}; + if (const auto found = oldInstances_.find(target); found != oldInstances_.end()) + return {Target::instance, found->second}; + if (document_.boxedValuesOffset <= target && target < document_.stringsOffset) return {Target::boxed, target}; + if (target >= 4 && target <= old.size()) { + std::uint32_t count{}; + std::memcpy(&count, old.data() + target - 4, sizeof(count)); + if (count == 0) return {Target::empty, target}; + } + return {Target::unknown, target}; + } + + static std::vector old_slots(const std::uint32_t start, const std::size_t length, + std::vector offsets) { + std::ranges::sort(offsets); + offsets.erase(std::unique(offsets.begin(), offsets.end()), offsets.end()); + std::vector out; + for (const auto value : offsets) + if (value >= start && value - start < length) out.push_back(value); + return out; + } + + void carry_boxed_arrays() { + if (document_.payload.empty()) return; + for (const auto& record : document_.arrays) oldArrays_.emplace(record.offset, record); + for (std::size_t index = 0; index < document_.instances.size(); ++index) + oldInstances_.emplace(document_.instances[index].dataOffset, static_cast(index)); + auto records = document_.boxedValues; + std::ranges::sort(records, {}, &BoxedValueRecord::offset); + for (const auto& source : records) + for (const auto slot : old_slots(source.offset, source.rawBytes.size(), document_.pointerOffsets)) { + const auto [kind, key] = old_target(slot); + if (kind == Target::array) carry_array(oldArrays_.at(key)); + } + } + + std::pair array_element(const ArrayRecord& record) const { + FieldDescriptor probe; + probe.encodedType = static_cast(record.encodedType >> 1); + if (probe.type() == FieldType::structure) { + const auto slot = record.classRef; + if (slot < document_.fixupTypeGuids.size() && slot < document_.fixupTypeSignatures.size()) + for (const auto& type : document_.types) + if (type.guid == document_.fixupTypeGuids[slot] && type.signature == document_.fixupTypeSignatures[slot]) + return {std::max(type.size, 1), std::max(type.alignment, 1)}; + throw std::runtime_error("EBX array record has an unknown struct type"); + } + const auto stride = scalar_size(probe.type()); + return {stride, std::max(std::min(stride, 8), 1)}; + } + + void carry_array(const ArrayRecord& record) { + if (carried_.contains(record.offset)) return; + const auto [stride, alignment] = array_element(record); + const auto length = stride * record.count; + if (record.offset + length > document_.payload.size()) throw std::runtime_error("EBX carried array exceeds EBXD"); + const auto data = align_up(payload_.size() + 4, alignment); + put_number(data - 4, record.count); + put(data, std::span(document_.payload).subspan(record.offset, length)); + const auto placed = checked_u32(data, "EBX array data"); + carried_[record.offset] = placed; + arrays_.push_back({placed, record.count, record.hash, record.encodedType, record.classRef}); + relocate(record.offset, length, data); + } + + void relocate(const std::uint32_t oldStart, const std::size_t length, const std::size_t newStart) { + const auto moved = [&](const std::uint32_t old) { + return checked_u32(newStart + (old - oldStart), "EBX carried relocation"); + }; + for (const auto old : old_slots(oldStart, length, document_.resourceRefOffsets)) resourceOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.importOffsets)) importOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.typeInfoOffsets)) typeInfoOffsets_.push_back(moved(old)); + for (const auto old : old_slots(oldStart, length, document_.pointerOffsets)) { + const auto slot = moved(old); + pointerOffsets_.push_back(slot); + if (document_.payload.empty()) continue; + const auto [kind, key] = old_target(old); + switch (kind) { + case Target::array: + carry_array(oldArrays_.at(key)); + put_number(slot, static_cast(carried_.at(key) - slot)); + break; + case Target::string: { + const auto& source = document_.payload; + auto end = static_cast(key); + while (end < source.size() && source[end] != std::byte{0}) ++end; + stringsPending_.push_back({slot, std::string(reinterpret_cast(source.data()) + key, end - key)}); + break; + } + case Target::instance: + put_number(slot, static_cast(instanceOffsets_.at(key) - slot)); + break; + case Target::empty: + put_number(slot, static_cast(emptyArrayTarget_ - slot)); + break; + case Target::boxed: + boxedTargets_.emplace_back(slot, key); + break; + case Target::unknown: + break; + } + } } void emit_array(const std::size_t index) { @@ -407,23 +525,14 @@ private: if (source.offset < document_.boxedValuesOffset) throw std::runtime_error("EBX boxed record precedes its section"); const auto offset = static_cast(boxedOffset_) + source.offset - document_.boxedValuesOffset; put(offset, source.rawBytes); - const auto relocate = [&](const std::vector& sourceOffsets, - std::vector& destinationOffsets) { - for (const auto old : sourceOffsets) { - if (old < source.offset || old - source.offset >= source.rawBytes.size()) continue; - destinationOffsets.push_back(checked_u32(offset + old - source.offset, - "EBX boxed relocation")); - } - }; - relocate(document_.pointerOffsets, pointerOffsets_); - relocate(document_.resourceRefOffsets, resourceOffsets_); - relocate(document_.importOffsets, importOffsets_); - relocate(document_.typeInfoOffsets, typeInfoOffsets_); + relocate(source.offset, source.rawBytes.size(), offset); moved[source.offset] = checked_u32(offset, "EBX boxed value"); auto authored = source; authored.offset = checked_u32(offset, "EBX boxed value"); boxed_.push_back(std::move(authored)); } + for (const auto& [slot, target] : boxedTargets_) + put_number(slot, static_cast(boxedOffset_ + (target - document_.boxedValuesOffset) - slot)); for (const auto& pending : boxesPending_) { put_number(pending.slot, pending.value.encodedType); put_number(static_cast(pending.slot) + 4, std::uint32_t{}); @@ -588,6 +697,10 @@ private: std::uint32_t arraysOffset_{}; std::uint32_t boxedOffset_{}; std::uint32_t emptyArrayTarget_{}; + std::unordered_map oldArrays_; + std::unordered_map oldInstances_; + std::unordered_map carried_; + std::vector> boxedTargets_; }; }