EBX writer: keep the arrays and strings boxed values point at

A rewrite copied a boxed value's bytes, pointers and all, but wrote the
arrays and strings those pointers reach only when a field also reached
them. Rewritten documents lost that data and the boxed pointers aimed
at whatever moved into its place; the mod merge rewrites real documents
this way. Read documents now keep their EBXD bytes, and the writer
carries every array, string, instance and boxed value a boxed pointer
reaches, repointed. Found by a round-trip check on bam's level root (one string and
about 1,120 bytes of arrays lost).

Engine/Resource/ebx_document.cpp

@@ -615,6 +615,8 @@ Document read_document(const std::span<const std::byte> bytes) {
     result.hashedReflectionNames = reflection2 != nullptr;
     result.reflectionData.assign(bytes.begin() + static_cast<std::ptrdiff_t>(reflectionChunk.begin),
                                  bytes.begin() + static_cast<std::ptrdiff_t>(reflectionChunk.end));
+    result.payload.assign(bytes.begin() + static_cast<std::ptrdiff_t>(result.dataStart),
+                          bytes.begin() + static_cast<std::ptrdiff_t>(result.dataEnd));
     map_instances(result, fixup, reflectionIds);
     ValueReader(bytes, result).read_instances();
     std::ranges::sort(result.boxedValues, {}, &BoxedValueRecord::offset);

Engine/Resource/ebx_document.h

@@ -189,6 +189,9 @@ struct Document {
     std::vector<std::uint32_t> importOffsets;
     std::vector<std::uint32_t> typeInfoOffsets;
     std::vector<std::uint32_t> boxedValueSlots;
+    // The EBXD bytes the document was read from (offsets as in the fixup tables): what boxed
+    // values point at (arrays, strings) is carried from here when the document is written.
+    std::vector<std::byte> payload;
 
     [[nodiscard]] const InstanceRecord* root() const noexcept;
 };

Engine/Resource/ebx_writer.cpp

@@ -359,6 +359,124 @@ private:
         for (std::size_t index = 0; index < arraysPending_.size(); ++index) {
             if (!arraysPending_[index].emitted) emit_array(index);
         }
+        carry_boxed_arrays();
+    }
+
+    // What a boxed value points at is not in the document's values: its bytes are carried as they
+    // were, so the arrays and strings its pointers reach are carried with them and the pointers
+    // moved to the new places. Without this a rewrite dropped them (seen on the game's own
+    // level roots: a boxed debug string and about 1 KB of arrays).
+
+    enum class Target { array, string, instance, boxed, empty, unknown };
+
+    std::pair<Target, std::uint32_t> old_target(const std::uint32_t slot) const {
+        const auto& old = document_.payload;
+        std::int32_t displacement{};
+        std::memcpy(&displacement, old.data() + slot, sizeof(displacement));
+        const auto target = static_cast<std::uint32_t>(static_cast<std::int64_t>(slot) + displacement);
+        if (oldArrays_.contains(target)) return {Target::array, target};
+        if (document_.stringsOffset <= target && target < old.size()) return {Target::string, target};
+        if (const auto found = oldInstances_.find(target); found != oldInstances_.end())
+            return {Target::instance, found->second};
+        if (document_.boxedValuesOffset <= target && target < document_.stringsOffset) return {Target::boxed, target};
+        if (target >= 4 && target <= old.size()) {
+            std::uint32_t count{};
+            std::memcpy(&count, old.data() + target - 4, sizeof(count));
+            if (count == 0) return {Target::empty, target};
+        }
+        return {Target::unknown, target};
+    }
+
+    static std::vector<std::uint32_t> old_slots(const std::uint32_t start, const std::size_t length,
+                                                std::vector<std::uint32_t> offsets) {
+        std::ranges::sort(offsets);
+        offsets.erase(std::unique(offsets.begin(), offsets.end()), offsets.end());
+        std::vector<std::uint32_t> out;
+        for (const auto value : offsets)
+            if (value >= start && value - start < length) out.push_back(value);
+        return out;
+    }
+
+    void carry_boxed_arrays() {
+        if (document_.payload.empty()) return;
+        for (const auto& record : document_.arrays) oldArrays_.emplace(record.offset, record);
+        for (std::size_t index = 0; index < document_.instances.size(); ++index)
+            oldInstances_.emplace(document_.instances[index].dataOffset, static_cast<std::uint32_t>(index));
+        auto records = document_.boxedValues;
+        std::ranges::sort(records, {}, &BoxedValueRecord::offset);
+        for (const auto& source : records)
+            for (const auto slot : old_slots(source.offset, source.rawBytes.size(), document_.pointerOffsets)) {
+                const auto [kind, key] = old_target(slot);
+                if (kind == Target::array) carry_array(oldArrays_.at(key));
+            }
+    }
+
+    std::pair<std::size_t, std::size_t> array_element(const ArrayRecord& record) const {
+        FieldDescriptor probe;
+        probe.encodedType = static_cast<std::uint16_t>(record.encodedType >> 1);
+        if (probe.type() == FieldType::structure) {
+            const auto slot = record.classRef;
+            if (slot < document_.fixupTypeGuids.size() && slot < document_.fixupTypeSignatures.size())
+                for (const auto& type : document_.types)
+                    if (type.guid == document_.fixupTypeGuids[slot] && type.signature == document_.fixupTypeSignatures[slot])
+                        return {std::max<std::size_t>(type.size, 1), std::max<std::size_t>(type.alignment, 1)};
+            throw std::runtime_error("EBX array record has an unknown struct type");
+        }
+        const auto stride = scalar_size(probe.type());
+        return {stride, std::max<std::size_t>(std::min<std::size_t>(stride, 8), 1)};
+    }
+
+    void carry_array(const ArrayRecord& record) {
+        if (carried_.contains(record.offset)) return;
+        const auto [stride, alignment] = array_element(record);
+        const auto length = stride * record.count;
+        if (record.offset + length > document_.payload.size()) throw std::runtime_error("EBX carried array exceeds EBXD");
+        const auto data = align_up(payload_.size() + 4, alignment);
+        put_number(data - 4, record.count);
+        put(data, std::span(document_.payload).subspan(record.offset, length));
+        const auto placed = checked_u32(data, "EBX array data");
+        carried_[record.offset] = placed;
+        arrays_.push_back({placed, record.count, record.hash, record.encodedType, record.classRef});
+        relocate(record.offset, length, data);
+    }
+
+    void relocate(const std::uint32_t oldStart, const std::size_t length, const std::size_t newStart) {
+        const auto moved = [&](const std::uint32_t old) {
+            return checked_u32(newStart + (old - oldStart), "EBX carried relocation");
+        };
+        for (const auto old : old_slots(oldStart, length, document_.resourceRefOffsets)) resourceOffsets_.push_back(moved(old));
+        for (const auto old : old_slots(oldStart, length, document_.importOffsets)) importOffsets_.push_back(moved(old));
+        for (const auto old : old_slots(oldStart, length, document_.typeInfoOffsets)) typeInfoOffsets_.push_back(moved(old));
+        for (const auto old : old_slots(oldStart, length, document_.pointerOffsets)) {
+            const auto slot = moved(old);
+            pointerOffsets_.push_back(slot);
+            if (document_.payload.empty()) continue;
+            const auto [kind, key] = old_target(old);
+            switch (kind) {
+            case Target::array:
+                carry_array(oldArrays_.at(key));
+                put_number(slot, static_cast<std::uint32_t>(carried_.at(key) - slot));
+                break;
+            case Target::string: {
+                const auto& source = document_.payload;
+                auto end = static_cast<std::size_t>(key);
+                while (end < source.size() && source[end] != std::byte{0}) ++end;
+                stringsPending_.push_back({slot, std::string(reinterpret_cast<const char*>(source.data()) + key, end - key)});
+                break;
+            }
+            case Target::instance:
+                put_number(slot, static_cast<std::uint32_t>(instanceOffsets_.at(key) - slot));
+                break;
+            case Target::empty:
+                put_number(slot, static_cast<std::uint32_t>(emptyArrayTarget_ - slot));
+                break;
+            case Target::boxed:
+                boxedTargets_.emplace_back(slot, key);
+                break;
+            case Target::unknown:
+                break;
+            }
+        }
     }
 
     void emit_array(const std::size_t index) {
@@ -407,23 +525,14 @@ private:
             if (source.offset < document_.boxedValuesOffset) throw std::runtime_error("EBX boxed record precedes its section");
             const auto offset = static_cast<std::size_t>(boxedOffset_) + source.offset - document_.boxedValuesOffset;
             put(offset, source.rawBytes);
-            const auto relocate = [&](const std::vector<std::uint32_t>& sourceOffsets,
-                                      std::vector<std::uint32_t>& destinationOffsets) {
-                for (const auto old : sourceOffsets) {
-                    if (old < source.offset || old - source.offset >= source.rawBytes.size()) continue;
-                    destinationOffsets.push_back(checked_u32(offset + old - source.offset,
-                                                             "EBX boxed relocation"));
-                }
-            };
-            relocate(document_.pointerOffsets, pointerOffsets_);
-            relocate(document_.resourceRefOffsets, resourceOffsets_);
-            relocate(document_.importOffsets, importOffsets_);
-            relocate(document_.typeInfoOffsets, typeInfoOffsets_);
+            relocate(source.offset, source.rawBytes.size(), offset);
             moved[source.offset] = checked_u32(offset, "EBX boxed value");
             auto authored = source;
             authored.offset = checked_u32(offset, "EBX boxed value");
             boxed_.push_back(std::move(authored));
         }
+        for (const auto& [slot, target] : boxedTargets_)
+            put_number(slot, static_cast<std::uint32_t>(boxedOffset_ + (target - document_.boxedValuesOffset) - slot));
         for (const auto& pending : boxesPending_) {
             put_number(pending.slot, pending.value.encodedType);
             put_number(static_cast<std::size_t>(pending.slot) + 4, std::uint32_t{});
@@ -588,6 +697,10 @@ private:
     std::uint32_t arraysOffset_{};
     std::uint32_t boxedOffset_{};
     std::uint32_t emptyArrayTarget_{};
+    std::unordered_map<std::uint32_t, ArrayRecord> oldArrays_;
+    std::unordered_map<std::uint32_t, std::uint32_t> oldInstances_;
+    std::unordered_map<std::uint32_t, std::uint32_t> carried_;
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> boxedTargets_;
 };
 
 }