DieMitchell/ReSkate
1
0 Fork 0
mirror of https://github.com/Dingo-Shenanigans/ReSkate.git synced 2026-10-23 16:25:58 +02:00
Code Issues Packages Projects Releases Wiki Activity

Fix costume merging, restrict server names, add Centrix tag

Browse Source 
- Mod merge: rebuild a merged bundle's chunk metadata the way the game
  writes it (one record per chunk, in chunk GUID order), so costumes
  from different mods built on the same game costume keep their
  textures and meshes
- Merge audit checks that layout, and there is a new unit test for it
- Server names: only letters, numbers, spaces and - _ [ ] ( ),
  1 to 64 characters; the server browser hides servers with other names
- New Centrix tag from the backend's "centrix" list: blue and white
  name, badge and item animation, ranked after Dev and before Creator
This commit is contained in:
AyeZeeBB committed 2026-10-05 23:39:16 -05:00
1 parent 0d5dfda243
commit 2cdc5ee8df
24 files changed
+645 -29
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Engine/Game/Multiplayer/session_model.h
+18 -3
View File
@@ -8,10 +8,25 @@
#include <cstddef>
#include <cstdint>
#include <string>
#include <string_view>
#include <vector>
namespace dingosdk {
constexpr unsigned max_map_rotation = 1440; // minutes a server's rotation keeps one map, at most
// A dedicated server's name: 1 to 64 letters, digits, spaces and - _ [ ] ( ), with a
// letter or digit among them and no space at either end. A server refuses any other
// name, and the server browser does not show one.
inline constexpr char server_name_rule[] = "1 to 64 letters, numbers, spaces and - _ [ ] ( )";
[[nodiscard]] constexpr bool valid_server_name(std::string_view name) noexcept {
if (name.empty() || name.size() > 64 || name.front() == ' ' || name.back() == ' ') return false;
bool named{};
for (const auto c : name) {
const bool word = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
if (!word && std::string_view(" -_[]()").find(c) == std::string_view::npos) return false;
named |= word;
}
return named;
}
struct MultiplayerLobby {
std::uint64_t id{}, owner{};
std::string name, map, code;
@@ -71,7 +86,7 @@ struct MultiplayerChatLine {
std::string name, text;
bool local{}; // sent by this player
// The sender's role, as their nametag shows it: its colour (IM_COL32 layout, 0 = none)
// and a tag shown in a box before the name ("Dev", "Creator", "Homie", "Admin", "Host", "Friend" or empty).
// and a tag shown in a box before the name ("Dev", "Creator", "Centrix", "Homie", "Admin", "Host", "Friend" or empty).
std::uint32_t color{};
std::string tag;
// With the chat filter on, `text` is masked and this is the line as sent (same length), so
@@ -127,7 +142,7 @@ struct MultiplayerModel {
float chat_bubbles_distance{40.f};
float chat_bubbles_duration{5.f};
int chat_bubbles_history{3};
// Local: the tag the ReSkate backend gives this player ("Dev", "Creator" or "Homie"; empty
// Local: the tag the ReSkate backend gives this player ("Dev", "Creator", "Centrix" or "Homie"; empty
// for most players) and its role colour, and whether they show it, and the animated items
// that come with it, to everyone.
std::string identity_tag;
@@ -135,7 +150,7 @@ struct MultiplayerModel {
bool identity_tag_shown{true}, identity_items_shown{true};
// Local: how each of this player's marked cosmetics is coloured, for the Special page: what
// they wear in each slot ("Top", "Shoes", ...), then the parts of their board. mode: 0 what
// their list gives (`identity_animation`: "RAINBOW", "RED" or "GOLD"), 1 off, 2 a gradient
// their list gives (`identity_animation`: "RAINBOW", "RED", "BLUE" or "GOLD"), 1 off, 2 a gradient
// between the two colours they picked, 3 the first of them alone. speed: 0 normal, 1 slow,
// 2 fast.
struct IdentityStyle {
Engine/Vfs/Test/chunk_metadata_merge_tests.cpp
+203
View File
@@ -0,0 +1,203 @@
// A bundle's chunk metadata has one record for each of its chunks: whose the chunk is (the hash of
// the resource's name) and, for a texture, the first mip the bundle holds. In the game's bundles
// the records are in the order of the chunks' guids, which is not the order the chunks are listed
// in. A mod's tool starts from the game's list and writes a record where each chunk it adds or
// replaces is LISTED: right for a chunk it adds at the end, but a replaced chunk's record lands
// on another chunk's, which is then gone from the mod's list.
//
// When two mods each ship a copy of one game bundle with chunks of their own (two costumes built
// on the same game costume), the merged bundle needs a list that describes every chunk and is in
// the game's form. With the game's list alone, or with records taken from where the mods' lists
// have them, the costumes' meshes and textures never streamed in.
#include "Engine/Vfs/mod_merge_internal.h"
#include <algorithm>
#include <cstring>
#include <iostream>
#include <optional>
namespace db = dingosdk::native_db;
namespace fb = dingosdk::frostbite;
using namespace dingosdk::mods::detail;
namespace {
using Bytes = std::vector<std::byte>;
int failures = 0;
void check(bool condition, const char* what) {
if (!condition) { std::cerr << "FAILED: " << what << "\n"; ++failures; }
}
// A chunk's guid: the first byte decides where it sorts.
fb::Guid guid(unsigned char first) {
fb::Guid value;
value.bytes.fill(std::byte{0x11});
value.bytes[0] = static_cast<std::byte>(first);
return value;
}
// One chunk's record: the hash of what it belongs to, and for a texture its first mip.
struct Record {
std::uint64_t owner{};
std::optional<std::uint32_t> firstMip;
bool operator==(const Record&) const = default;
};
db::Node number(unsigned type, const char* name, std::uint64_t value, std::size_t size) {
db::Node node;
node.type = type;
node.named = true;
node.name = name;
for (std::size_t byte = 0; byte < size; ++byte) node.owned.push_back(static_cast<unsigned char>(value >> (8 * byte)));
return node;
}
Bytes list_of(const std::vector<Record>& records) {
db::Node list;
list.type = 1;
list.named = true;
list.name = "chunkMeta";
list.terminated = true;
for (const auto& record : records) {
db::Node meta;
meta.type = 2;
meta.named = true;
meta.name = "meta";
meta.terminated = true;
if (record.firstMip) meta.children.push_back(number(8, "firstMip", *record.firstMip, 4));
db::Node row;
row.type = 2;
row.terminated = true;
row.children.push_back(number(9, "h64", record.owner, 8));
row.children.push_back(std::move(meta));
list.children.push_back(std::move(row));
}
const auto written = db::write(list);
const auto* bytes = reinterpret_cast<const std::byte*>(written.data());
return {bytes, bytes + written.size()};
}
// What a list says, record by record; a record that says nothing reads as owner 0.
std::vector<Record> read_list(const Bytes& list) {
const std::span<const unsigned char> bytes(reinterpret_cast<const unsigned char*>(list.data()), list.size());
std::size_t used{};
const auto root = db::read(bytes, "chunk metadata", &used);
check(used == bytes.size() && root.type == 1 && root.name == "chunkMeta", "the merged list is one list, named as the game names it");
std::vector<Record> records;
for (const auto& row : root.children) {
Record record;
if (const auto* owner = row.field("h64"); owner && owner->payload().size() == 8)
std::memcpy(&record.owner, owner->payload().data(), 8);
if (const auto* meta = row.field("meta"))
if (const auto* mip = meta->field("firstMip"); mip && mip->payload().size() == 4) {
std::uint32_t value{};
std::memcpy(&value, mip->payload().data(), 4);
record.firstMip = value;
}
records.push_back(record);
}
return records;
}
ChunkRecord chunk(unsigned char first, std::size_t copy, std::size_t index, std::size_t shipped = ChunkRecord::none) {
return {guid(first), copy, index, shipped};
}
} // namespace
int main() {
// The game's bundle lists five chunks: something of its own, a texture, a mesh in two chunks
// around another texture. Its list has their records by guid: 10, 30, 50, 60, 70.
const Record mesh{0x1111}, textureA{0xAAAA, 2}, textureB{0xBBBB, 3}, other{0x9999};
const std::vector<fb::Guid> shipped{guid(0x50), guid(0x30), guid(0x70), guid(0x10), guid(0x60)};
const std::vector<Record> game{mesh, textureA, other, textureB, mesh};
constexpr std::size_t gameList = 0;
// One costume adds a texture (guid 20) and a mesh chunk (guid 80), listed after the game's five,
// with their records at those two places.
const Record firstTexture{0xF001, 0}, firstMesh{0xF002};
auto first = game;
first.insert(first.end(), {firstTexture, firstMesh});
// Another, made with an older tool, replaces the game's second texture (guid 60, listed fifth)
// with one whose first mip is 1: its record went over the fifth record, the mesh's, with no hash.
// It also adds a texture (guid 40).
const Record secondTexture{0x5001, 0};
auto second = game;
second[4] = Record{0, 1};
second.push_back(secondTexture);
const std::vector<Bytes> lists{list_of(game), list_of(first), list_of(second)};
// The merged bundle: the game's five as the higher-priority mod has them, then each mod's own.
const std::vector<ChunkRecord> merged{chunk(0x50, 2, 0, 0), chunk(0x30, 2, 1, 1), chunk(0x70, 2, 2, 2), chunk(0x10, 2, 3, 3),
chunk(0x60, 2, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6), chunk(0x40, 2, 5)};
const auto replaced = Record{textureB.owner, 1};
const std::vector<Record> expected{mesh, firstTexture, textureA, secondTexture, other, replaced, mesh, firstMesh};
const auto list = read_list(merge_chunk_metadata(lists, merged, gameList, shipped));
check(list == expected, "a record for every chunk, in the order of the chunks' guids");
check(list.size() == merged.size(), "as many records as chunks");
check(std::ranges::count(list, mesh) == 2, "the record the older tool wrote over is back: the mesh has both of its");
check(std::ranges::find(list, replaced) != list.end() && std::ranges::find(list, textureB) == list.end(),
"the replaced texture keeps its own name and takes the first mip the mod gave it");
check(std::ranges::none_of(list, [](const Record& record) { return record.owner == 0; }), "no record is left without a hash");
// The fault this is for: the game's own list is three records short, and the mods' lists have
// their records where the chunks are listed, which is not where the game looks.
check(read_list(lists[gameList]).size() + 3 == merged.size(), "the game's own list is three records short of the merged bundle");
// No mod's chunk in it: the game's list is not written again.
const std::vector<ChunkRecord> untouched{chunk(0x50, 0, 0, 0), chunk(0x30, 0, 1, 1), chunk(0x70, 0, 2, 2), chunk(0x10, 0, 3, 3), chunk(0x60, 0, 4, 4)};
check(merge_chunk_metadata(lists, untouched, gameList, shipped) == lists[gameList], "a bundle that is all the game's keeps the game's list as it is");
// One mod's copy alone is still written in the game's form: its additions go where their guids put them.
const std::vector<ChunkRecord> oneMod{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3),
chunk(0x60, 1, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6)};
check(read_list(merge_chunk_metadata(lists, oneMod, gameList, shipped)) ==
std::vector<Record>{mesh, firstTexture, textureA, other, textureB, mesh, firstMesh},
"one mod's additions are put in guid order with the game's");
// A newer tool writes the replaced texture's own hash with its first mip; the first mip is what counts.
auto newer = game;
newer[4] = Record{textureB.owner, 0};
const std::vector<Bytes> withNewer{list_of(game), list_of(newer)};
const std::vector<ChunkRecord> keepsNewer{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3), chunk(0x60, 1, 4, 4)};
check(read_list(merge_chunk_metadata(withNewer, keepsNewer, gameList, shipped)) ==
std::vector<Record>{mesh, textureA, other, Record{textureB.owner, 0}, mesh},
"a replaced texture's new first mip goes into its own record, and the record written over is back");
// A mod that only carries the game's version of a chunk another mod replaced changes nothing of it.
const std::vector<ChunkRecord> carried{chunk(0x50, 1, 0, 0), chunk(0x30, 1, 1, 1), chunk(0x70, 1, 2, 2), chunk(0x10, 1, 3, 3),
chunk(0x60, 1, 4, 4), chunk(0x20, 1, 5), chunk(0x80, 1, 6)};
check(read_list(merge_chunk_metadata(lists, carried, gameList, shipped))[4] == textureB, "a chunk a mod only carries keeps the game's record");
// A chunk whose copy has no list, or a shorter one, still takes up a place.
const std::vector<ChunkRecord> unknown{chunk(0x50, 0, 0, 0), chunk(0x30, 0, 1, 1), chunk(0x70, 0, 2, 2), chunk(0x10, 0, 3, 3),
chunk(0x60, 0, 4, 4), chunk(0x20, ChunkRecord::none, 5), chunk(0x80, 1, 9)};
check(read_list(merge_chunk_metadata(lists, unknown, gameList, shipped)) ==
std::vector<Record>{mesh, Record{}, textureA, other, textureB, mesh, Record{}},
"a chunk no list describes gets a record that says nothing, and the others stay in step");
// A bundle the game has no list for (it has no chunks there, or does not ship the bundle): every
// chunk is a mod's, with the record of the copy it came in. Maps each add their loading screen
// to one such bundle.
const std::vector<Bytes> modsOnly{list_of({firstTexture, firstMesh}), list_of({firstTexture, firstMesh, secondTexture})};
const std::vector<ChunkRecord> theirs{chunk(0x90, 1, 0), chunk(0x20, 1, 1), chunk(0x40, 1, 2)};
check(merge_chunk_metadata(modsOnly, theirs) == modsOnly[1], "a bundle that is one mod's copy alone keeps that copy's list");
const std::vector<ChunkRecord> both{chunk(0x90, 0, 0), chunk(0x20, 0, 1), chunk(0x40, 1, 2)};
check(read_list(merge_chunk_metadata(modsOnly, both)) == std::vector<Record>{firstMesh, secondTexture, firstTexture},
"and otherwise each chunk's record from the copy it came in, by guid like any other");
// A game list that is not one record a chunk is of some other kind: nothing is put in order by it.
const std::vector<Bytes> shortGame{list_of({mesh, textureA}), list_of(first)};
check(read_list(merge_chunk_metadata(shortGame, oneMod, gameList, shipped)) == first,
"a game list that is not a record a chunk leaves the records where the copies have them");
// No chunks, no list to write: what the first copy has stays.
check(merge_chunk_metadata({}, {}).empty(), "no list from no lists");
check(merge_chunk_metadata(lists, {}, gameList, shipped) == lists[gameList], "a bundle without chunks keeps the first copy's list");
const std::vector<Bytes> none{Bytes{}, Bytes{}};
const std::vector<ChunkRecord> bare{chunk(0x10, ChunkRecord::none, 0), chunk(0x20, ChunkRecord::none, 1)};
check(merge_chunk_metadata(none, bare).empty(), "nor from copies that have none");
// A list that is needed and cannot be read is the caller's to deal with.
const std::vector<Bytes> broken{lists[0], Bytes(7, std::byte{0x41})};
bool threw{};
try { (void)merge_chunk_metadata(broken, oneMod, gameList, shipped); }
catch (const std::exception&) { threw = true; }
check(threw, "an unreadable list throws");
if (failures) {
std::cerr << "chunk metadata merge: " << failures << " check(s) failed\n";
return 1;
}
std::cout << "chunk metadata merge: ok\n";
return 0;
}
Engine/Vfs/Test/mod_merge_audit_tests.cpp
+152 -1
View File
@@ -12,7 +12,10 @@
// every reference a mod appended to one of its lists, and has every value a mod changed in
// place changed: no value in it is the merge's own invention;
// * a list whose entries the game keeps beside it (an item list, the music playlist) names
// nothing its own bundle does not hold, in every copy of that bundle the patch carries.
// nothing its own bundle does not hold, in every copy of that bundle the patch carries;
// * a bundle's chunk metadata has one record for each of its chunks, in the order of the
// chunks' guids as the game writes it: every texture's own record, with the first mip its
// header gives, is at its chunk's place.
// What is only reported: assets two mods add under one name, and assets of the game two mods
// change where the patch has one mod's copy whole.
//
@@ -30,7 +33,9 @@
#include <algorithm>
#include <array>
#include <bit>
#include <cctype>
#include <chrono>
#include <cstring>
#include <deque>
#include <fstream>
#include <functional>
@@ -225,6 +230,12 @@ struct Audit {
std::map<Place, EbxFacts> facts;
std::map<Place, fb::Guid> guids;
std::vector<Contested> contested;
// The patch's bundles (by TOC and bundle) that a mod ships a readable copy of: the ones whose
// chunk metadata the merge may have written.
std::set<std::pair<std::size_t, std::size_t>> chunkBundles;
// Those of them the patch has exactly as one mod ships them, chunks and list: the merge wrote
// nothing there.
std::set<std::pair<std::size_t, std::size_t>> chunkBundlesAsShipped;
std::chrono::steady_clock::time_point started = std::chrono::steady_clock::now();
void progress(const std::string& what) const {
@@ -305,6 +316,84 @@ struct Audit {
}
return result;
}
// One record of a bundle's chunk metadata: whose the chunk is, and a texture's first mip.
struct ChunkMeta {
std::optional<std::uint64_t> owner;
std::optional<std::int32_t> firstMip;
};
// A copy's chunk metadata, record by record; nothing when the list cannot be read.
static std::optional<std::vector<ChunkMeta>> records_of(const Bundle& bundle) {
std::vector<ChunkMeta> records;
const auto& list = bundle.listing->manifest.chunkMetadata;
if (list.empty()) return records;
try {
const std::span<const unsigned char> bytes(reinterpret_cast<const unsigned char*>(list.data()), list.size());
const auto root = native_db::read(bytes, "chunk metadata", nullptr, {.unique_fields = false, .max_entries = bytes.size()});
for (const auto& row : root.children) {
ChunkMeta record;
if (const auto* hash = row.field("h64"); hash && hash->payload().size() == 8)
std::memcpy(&record.owner.emplace(), hash->payload().data(), 8);
if (const auto* meta = row.field("meta"))
if (const auto* mip = meta->field("firstMip"); mip && mip->payload().size() == 4)
std::memcpy(&record.firstMip.emplace(), mip->payload().data(), 4);
records.push_back(record);
}
} catch (const std::exception&) {
return std::nullopt;
}
return records;
}
// The hash a record names a resource by: of its lower-case name.
static std::uint64_t owner_hash(std::string_view name) {
std::uint64_t value = 5381;
for (const auto c : name) value = value * 33U ^ static_cast<unsigned char>(std::tolower(static_cast<unsigned char>(c)));
return value;
}
// The textures of a copy of a bundle whose pixels are one of the bundle's chunks, each against
// the record at its chunk's place when the chunks are in the order of their guids: how many
// there are, how many have another resource's record (or none) there, how many their own with
// another first mip than the texture's header gives, and one of those that are wrong.
struct TextureRecords {
std::size_t textures{}, misplaced{}, mips{};
std::string example;
};
TextureRecords texture_records(int from, const Bundle& bundle, const std::vector<ChunkMeta>& records) const {
constexpr std::uint32_t textureType = 0x6BDE20BA;
const auto& manifest = bundle.listing->manifest;
std::vector<std::size_t> order(manifest.chunks.size());
for (std::size_t index = 0; index < order.size(); ++index) order[index] = index;
std::ranges::sort(order, [&](std::size_t left, std::size_t right) { return manifest.chunks[left].guid < manifest.chunks[right].guid; });
std::map<fb::Guid, std::size_t> guidPlace;
for (std::size_t at = 0; at < order.size(); ++at) guidPlace.emplace(manifest.chunks[order[at]].guid, at);
TextureRecords result;
for (std::size_t index = 0; index < manifest.resources.size(); ++index) {
const auto& resource = manifest.resources[index];
if (resource.resourceType != textureType || resource.resourceMeta.size() < 4) continue;
Bytes header;
try { header = payload(place(from, bundle.assets[manifest.ebx.size() + index].file)); } catch (const std::exception&) { continue; }
if (header.size() != 144 && header.size() != 180) continue;
// The header: type and format, a version's extra field, then flags and sizes, the mip
// count and first mip, padding, and the chunk the pixels are in.
std::uint32_t version{};
std::memcpy(&version, resource.resourceMeta.data(), 4);
const std::size_t mips = 8 + 4 + 4 + (version >= 12 ? 4 : 0) + 10;
const auto firstMip = static_cast<std::int32_t>(header[mips + 1]);
fb::Guid chunk;
std::memcpy(chunk.bytes.data(), header.data() + mips + 2 + (header.size() == 180 ? 8 : 4), 16);
const auto at = guidPlace.find(chunk);
if (at == guidPlace.end()) continue; // its pixels are not in this bundle
++result.textures;
const auto* record = at->second < records.size() ? &records[at->second] : nullptr;
if (!record || record->owner != owner_hash(resource.name)) {
if (!result.misplaced++ && result.example.empty()) result.example = resource.name + " has " + (record ? "another resource's record" : "no record") + " at its chunk's place";
} else if (record->firstMip != firstMip) {
if (!result.mips++ && result.example.empty())
result.example = resource.name + " is told first mip " + (record->firstMip ? std::to_string(*record->firstMip) : std::string("none")) +
", its header says " + std::to_string(firstMip);
}
}
return result;
}
std::string mod_names(const std::vector<std::size_t>& list) const {
std::set<std::string> unique;
for (const auto index : list) unique.insert(catalog.mods[index].name);
@@ -567,6 +656,11 @@ struct Audit {
const auto mine = read_bundle(static_cast<int>(index + 2), bundle);
if (!mine.listing || !patched.listing) { ++byName; continue; }
const auto* game = game_bundle(bundle.name);
chunkBundles.emplace(merged->second, at->second);
if (mine.listing->manifest.chunkMetadata == patched.listing->manifest.chunkMetadata &&
std::ranges::equal(mine.listing->manifest.chunks, patched.listing->manifest.chunks, {},
&fb::BundleAsset::guid, &fb::BundleAsset::guid))
chunkBundlesAsShipped.emplace(merged->second, at->second);
for (const auto& entry : mine.assets) {
++assets;
const auto key = key_of(*entry.asset);
@@ -884,6 +978,62 @@ struct Audit {
std::cout << copies << " such list(s) with entries mods added, across every copy in the patch; " << entries << " added entries checked\n";
}
// ---------------------------------------------------------------- chunk metadata
// The engine reads a chunk's record before it streams the chunk. The game writes a bundle's
// list with a record for each chunk, in the order of the chunks' guids, so a texture's record
// (the hash of its name, and its first mip) is at its chunk's place in that order. A bundle
// the patch has from more than the game's copy (two mods' costumes built on one game costume,
// each with chunks of its own) has to be in that form as well: with fewer records than
// chunks, or with records where the mods' own lists have them, textures and meshes get
// another's record or none and never appear. Checked wherever the game's own copy of the
// bundle is in that form, which is everywhere it has been looked at.
void chunk_metadata() {
section("Chunk metadata");
std::size_t bundles{}, untouched{}, asShipped{}, checked{}, textures{}, unknownForm{};
for (const auto& where : chunkBundles) {
const auto& toc = tocs[where.first];
const auto& patched = toc.bundles[where.second];
const auto& kept = patched.listing->manifest.chunks;
if (kept.empty()) continue;
++bundles;
const auto what = toc.relative + ": " + patched.name;
const auto* game = game_bundle(patched.name);
// The game's own list on the game's own chunks is what the game shipped.
if (game && game->listing->manifest.chunkMetadata == patched.listing->manifest.chunkMetadata &&
game->listing->manifest.chunks.size() == kept.size() &&
std::ranges::equal(game->listing->manifest.chunks, kept, {}, &fb::BundleAsset::guid, &fb::BundleAsset::guid)) {
++untouched;
continue;
}
// One mod's copy passed through whole is that mod's to have right: its tool's list is
// not in the game's form, and the merge does not rewrite what only one mod ships.
if (chunkBundlesAsShipped.contains(where)) { ++asShipped; continue; }
const auto records = records_of(patched);
if (!records) { fail("chunk metadata", what + ": the patch's list cannot be read"); continue; }
if (records->size() != kept.size() && !(records->empty() && (!game || game->listing->manifest.chunkMetadata.empty()))) {
fail("chunk metadata", what + ": " + std::to_string(records->size()) + " record(s) for " +
std::to_string(kept.size()) + " chunk(s)");
continue;
}
// Only a bundle whose own copy in the game is in guid order says what the patch's has to be.
if (!game) { ++unknownForm; continue; }
const auto shipped = records_of(*game);
if (!shipped) { ++unknownForm; continue; }
const auto original = texture_records(0, *game, *shipped);
if (original.misplaced || original.mips) { ++unknownForm; continue; }
++checked;
const auto merged = texture_records(1, patched, *records);
textures += merged.textures;
if (merged.misplaced || merged.mips)
fail("chunk metadata", what + ": of " + std::to_string(merged.textures) + " texture(s) with a chunk in the bundle, " +
std::to_string(merged.misplaced) + " do not have their own record at their chunk's place and " +
std::to_string(merged.mips) + " have another first mip than their header, e.g. " + merged.example);
}
std::cout << bundles << " bundle(s) mods ship that hold chunks: " << untouched << " with the game's own list on the game's own chunks, "
<< asShipped << " exactly as one mod ships them, " << checked << " others checked, " << textures
<< " texture record(s) in them; " << unknownForm << " whose own copy in the game gives nothing to check against\n";
}
int summary() const {
section("Summary");
std::size_t total{};
@@ -908,6 +1058,7 @@ int main(int argc, char** argv) try {
audit.mods_against_patch();
audit.outcomes();
audit.lists();
audit.chunk_metadata();
return audit.summary();
} catch (const std::exception& error) {
std::cerr << "FAIL: " << error.what() << '\n';
Engine/Vfs/mod_merge_combine.cpp
+172 -3
View File
@@ -36,6 +36,9 @@ struct Asset {
fb::BundleFileInfo file;
bool base{};
const std::string* mod{}; // whose copy of the bundle it came in; null for the game's
// For a chunk: where its record is in the chunk metadata of the copy this version
// of it came in, and where the game's copy lists the chunk (merge_chunk_metadata).
ChunkRecord record;
};
// Where one contributor's copy of an asset lives, so conflicting copies can be
@@ -95,7 +98,13 @@ struct BundleState {
std::uint32_t manifestChunk{};
std::vector<Asset> assets;
std::vector<fb::BundleFileInfo> files;
std::vector<std::byte> metadata;
// Each copy's chunk metadata; copies that carry the same list share an entry.
// Every chunk remembers its record in one of these (Asset::record), and the
// list is written again when the manifest is rebuilt (merge_chunk_metadata).
std::vector<std::vector<std::byte>> metadata;
// Which of those is the game's own, and the chunks the game's copy lists.
std::size_t gameMetadata{ChunkRecord::none};
std::vector<fb::Guid> shippedChunks;
// Keyed position of each asset, so replacing one stays constant time: these
// bundles hold tens of thousands of entries.
std::map<std::string, std::size_t, std::less<>> seen;
@@ -196,8 +205,115 @@ void build_manifest(const std::vector<Asset>& assets, std::span<const std::byte>
emit(fb::AssetKind::chunk, manifest.chunks);
}
std::span<const unsigned char> unsigned_bytes(std::span<const std::byte> data) {
return {reinterpret_cast<const unsigned char*>(data.data()), data.size()};
}
} // namespace
std::vector<std::byte> merge_chunk_metadata(std::span<const std::vector<std::byte>> lists,
std::span<const ChunkRecord> chunks, std::size_t game,
std::span<const fb::Guid> shipped) {
// A bundle with no chunks has nothing to describe; what the first copy has stays.
if (chunks.empty()) return lists.empty() ? std::vector<std::byte>{} : lists.front();
const bool shippedList = game < lists.size() && !lists[game].empty();
// Nothing moved: nearly every bundle is this, and its list is not read at all.
// With the game's list, every chunk is the game's own where the game lists it;
// without one, every chunk keeps one copy's record at its own place.
const auto first = shippedList ? game : chunks.front().copy;
bool whole = first < lists.size() && (!shippedList || chunks.size() == shipped.size());
for (std::size_t index = 0; whole && index < chunks.size(); ++index)
whole = chunks[index].copy == first && chunks[index].index == index &&
(!shippedList || chunks[index].shipped == index);
if (whole) return lists[first];
std::vector<std::optional<native_db::Node>> read(lists.size());
const auto list = [&](std::size_t copy) -> const native_db::Node* {
if (copy >= lists.size() || lists[copy].empty()) return nullptr;
// A level's root bundle has thousands of chunks; every value takes a byte at least.
if (!read[copy])
read[copy] = native_db::read(unsigned_bytes(lists[copy]), "chunk metadata", nullptr,
{.unique_fields = false, .max_entries = lists[copy].size()});
return &*read[copy];
};
const auto row = [&](std::size_t copy, std::size_t index) -> const native_db::Node* {
const auto* from = list(copy);
return from && index < from->children.size() ? &from->children[index] : nullptr;
};
// The game's list, when it is the kind this is written as: a record for each of
// the game's chunks, in the order of their guids.
const auto* original = shippedList ? list(game) : nullptr;
if (original && original->children.size() != shipped.size()) original = nullptr;
std::vector<std::size_t> place(shipped.size()); // where each of the game's chunks has its record
if (original) {
std::vector<std::size_t> order(shipped.size());
for (std::size_t index = 0; index < order.size(); ++index) order[index] = index;
std::ranges::sort(order, [&](std::size_t left, std::size_t right) { return shipped[left] < shipped[right]; });
for (std::size_t at = 0; at < order.size(); ++at) place[order[at]] = at;
}
std::vector<native_db::Node> records;
std::vector<bool> described(chunks.size());
records.reserve(chunks.size());
for (std::size_t index = 0; index < chunks.size(); ++index) {
const auto& chunk = chunks[index];
const auto* written = row(chunk.copy, chunk.index);
const native_db::Node* kept = written;
std::optional<native_db::Node> changed;
if (original && chunk.shipped < shipped.size()) {
kept = &original->children[place[chunk.shipped]];
// A mod's version of the chunk: its tool started from the game's list and wrote a
// record where the chunk is listed, in place of whatever the game has there. The
// first mip it gave is the one the mod's texture has; the rest of it is not the
// chunk's (the hash is another chunk's, or empty).
const auto* before = chunk.index < original->children.size() ? &original->children[chunk.index] : nullptr;
if (chunk.copy != game && written && (!before || native_db::write(*written) != native_db::write(*before)))
if (const auto* meta = written->field("meta"))
if (const auto* mip = meta->field("firstMip")) {
changed = *kept;
if (auto* own = changed->field("meta")) {
if (auto* current = own->field("firstMip")) *current = *mip;
else own->children.push_back(*mip);
kept = &*changed;
}
}
}
described[index] = kept != nullptr;
records.push_back(kept ? *kept : native_db::Node{});
}
const native_db::Node* shape = original;
for (std::size_t copy = 0; !shape && copy < read.size(); ++copy)
if (read[copy]) shape = &*read[copy];
if (!shape) return {}; // no copy has a list at all
// A chunk no list describes still takes up its place, with a record that says nothing.
native_db::Node blank;
blank.type = 2;
for (std::size_t index = 0; index < records.size(); ++index)
if (described[index]) {
blank = records[index];
blank.children.clear();
break;
}
for (std::size_t index = 0; index < records.size(); ++index)
if (!described[index]) records[index] = blank;
// As the game writes it: by the chunks' guids. Only a bundle whose own list in the
// game is of another kind keeps the order its chunks are listed in.
std::vector<std::size_t> order(chunks.size());
for (std::size_t index = 0; index < order.size(); ++index) order[index] = index;
if (original || !shippedList)
std::ranges::stable_sort(order, [&](std::size_t left, std::size_t right) { return chunks[left].guid < chunks[right].guid; });
auto merged = *shape;
merged.children.clear();
merged.children.reserve(records.size());
for (const auto index : order) merged.children.push_back(std::move(records[index]));
const auto bytes = native_db::write(merged);
const auto* data = reinterpret_cast<const std::byte*>(bytes.data());
return {data, data + bytes.size()};
}
// One superbundle, combined across every mod that ships it. `used` collects the
// patch archives the result actually references, keyed by install chunk, so the
// layout declares those and only those.
@@ -561,6 +677,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
state.opaqueWhy = manifestFailure;
state.files = std::move(region.files);
state.assets.clear(); state.seen.clear();
state.metadata.clear(); state.shippedChunks.clear();
state.gameMetadata = ChunkRecord::none;
}
return;
}
@@ -580,7 +698,29 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
}
if (casBacked) state.casBacked = true;
if (!region.files.empty()) state.manifestChunk = region.files.front().location.installChunk;
if (fresh || state.metadata.empty()) state.metadata = manifest.chunkMetadata;
// Each chunk remembers where this copy lists it, and with that its record
// in this copy's chunk metadata, whichever version of the chunk is kept.
{
auto copy = ChunkRecord::none;
if (!manifest.chunkMetadata.empty()) {
auto list = std::find(state.metadata.begin(), state.metadata.end(), manifest.chunkMetadata);
if (list == state.metadata.end()) list = state.metadata.insert(state.metadata.end(), manifest.chunkMetadata);
copy = static_cast<std::size_t>(list - state.metadata.begin());
}
if (isBase) {
state.gameMetadata = copy;
state.shippedChunks.clear();
for (const auto& chunk : manifest.chunks) state.shippedChunks.push_back(chunk.guid);
}
const auto firstChunk = manifest.ebx.size() + manifest.resources.size();
for (std::size_t index = 0; index < manifest.chunks.size() && firstChunk + index < flat.size(); ++index) {
auto& record = flat[firstChunk + index].record;
record.guid = manifest.chunks[index].guid;
record.copy = copy;
record.index = index;
if (isBase) record.shipped = index;
}
}
const std::size_t firstFile = casBacked ? 1 : 0;
const auto* owner = isBase ? nullptr : &*owners.insert(modName).first;
// By the mod whose added assets this copy's replace: how many, and one of them.
@@ -645,6 +785,8 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
if (!count++) example = entry.asset.name;
state.displaced.emplace_back(at->first, previous.asset.sha1);
}
// Whichever version is kept, it is still the chunk the game lists there.
entry.record.shipped = previous.record.shipped;
state.assets[at->second] = std::move(entry);
}
}
@@ -884,9 +1026,36 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
}
}
// The merged bundle holds every mod's chunks, so its chunk metadata has
// to describe them all, and in the game's own form: a record for each
// chunk, in the order of the chunks' guids. With the game's list alone,
// or with the lists the mods ship (which are not in that order, and
// have lost the records their tools wrote over), the meshes and textures
// of costumes built on one game costume never stream in.
std::vector<std::byte> metadata;
{
std::vector<ChunkRecord> records;
std::size_t fromMods{};
for (const auto& entry : state.assets) {
if (entry.asset.kind != fb::AssetKind::chunk) continue;
records.push_back(entry.record);
fromMods += entry.record.shipped == ChunkRecord::none || entry.record.copy != state.gameMetadata;
}
try {
metadata = merge_chunk_metadata(state.metadata, records, state.gameMetadata, state.shippedChunks);
if (fromMods && state.gameMetadata != ChunkRecord::none &&
metadata != state.metadata[state.gameMetadata])
report.notes.push_back(name + ": chunk metadata written again for " + std::to_string(records.size()) +
" chunks, " + std::to_string(fromMods) + " of them added, changed or carried by mods");
} catch (const std::exception& failure) {
if (!state.metadata.empty()) metadata = state.metadata.front();
report.notes.push_back(name + ": the copies' chunk metadata could not be put together (" + failure.what() +
"); kept the first copy's, so chunks mods add to this bundle may not load");
}
}
fb::BinaryBundle manifest;
std::vector<fb::BundleFileInfo> files;
build_manifest(state.assets, state.metadata, manifest, files);
build_manifest(state.assets, metadata, manifest, files);
if (!state.casBacked) {
region = fb::write_bundle_region(files, fb::write_binary_bundle(manifest));
} else {
Engine/Vfs/mod_merge_internal.h
+34
View File
@@ -230,6 +230,40 @@ AssetOverrides collect_asset_overrides(const std::vector<const Mod*>& mods,
const CasStore& store, const fs::path& baseRoot,
const fs::path& gameRoot, MergeReport& report);
// A bundle's chunk metadata is a list with one record for each of its chunks: the
// hash of the name of the resource the chunk belongs to ("h64") and what the
// engine needs before it streams the chunk (a texture's first mip). In the game's
// own bundles the records are in the order of the chunks' guids, not the order
// the chunks are listed in. A mod's copy starts from the game's list as it is and
// has, at the place each chunk it adds or replaces is listed at, a record its
// tool wrote: for an added chunk that is the chunk's own record, but for a
// replaced one it has taken the place of another chunk's (and older tools left
// its hash empty), so the game's record for that other chunk is gone from the
// mod's list.
struct ChunkRecord {
static constexpr std::size_t none = static_cast<std::size_t>(-1);
fb::Guid guid;
std::size_t copy{none}; // the list of the copy this version of the chunk came in; none when that copy has no list
std::size_t index{}; // where that copy lists the chunk
std::size_t shipped{none}; // where the game's copy lists it; none for a chunk mods add
};
// The list for a merged bundle, written as the game writes one: a record for each
// of `chunks` (the bundle's chunks, as listed), in the order of their guids. A
// chunk of the game's keeps the game's record for it, whichever copy carries it
// (`game` is the game's list among `lists`, `shipped` the chunks the game's copy
// lists); when the kept version is one a mod replaced it with, the first mip that
// mod's record gives goes into it. A chunk a mod adds has that mod's record, and
// one nothing describes gets a record that says nothing. In a bundle the game
// does not ship, or ships without a list, every chunk is one a mod adds. When
// every chunk is the game's own where the game lists it, the game's list comes
// back as it is; so does one mod's, for a bundle without a list in the game that
// is that mod's copy alone.
// Throws when a list that is needed cannot be read.
[[nodiscard]] std::vector<std::byte> merge_chunk_metadata(std::span<const std::vector<std::byte>> lists,
std::span<const ChunkRecord> chunks,
std::size_t game = ChunkRecord::none,
std::span<const fb::Guid> shipped = {});
fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
std::vector<Source>& sources, MergeReport& report,
const std::string& relative, ArchiveUse& used, CasStore& store,
Engine/Vfs/native_db.cpp
+1 -1
View File
@@ -12,7 +12,7 @@ public:
: input_(input), context_(context), options_(options) {}
Node read(unsigned depth = 0) {
if (depth > 32 || ++nodes_ > 32768) fail("DB nesting/entry limit exceeded");
if (depth > 32 || ++nodes_ > options_.max_entries) fail("DB nesting/entry limit exceeded");
const auto tag = take(1)[0];
Node node;
node.type = tag & 31;
Engine/Vfs/native_db.h
+4
View File
@@ -41,6 +41,10 @@ struct Options {
// an entry. layout.toc's meta section repeats keys by design, so readers of
// that file turn the rule off.
bool unique_fields = true;
// How many values a file may hold: more than any the game reads whole at
// startup. A reader of something longer by nature (the chunk list of a
// level's root bundle) raises it to what its input can hold.
std::size_t max_entries = 32768;
};
// Throws std::runtime_error prefixed with `context`, for example "InitFS".