#!/usr/bin/env python3 """Apply symbol-derived and configured fixes to a compiled object.""" import argparse import bisect import functools import json import os import re import struct import subprocess import sys from collections import defaultdict from pathlib import Path ROOT = Path(__file__).parents[2] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / "tools" / "kazarov" / "python")) from kazarov_elf import Elf # noqa: E402 from scripts.build import disassemble # noqa: E402 from scripts.build import region # noqa: E402 from scripts.build import retail # noqa: E402 @functools.lru_cache(maxsize=None) def retail_symbols(image): """{name: address} of one image's own symbols, and their addresses sorted. Each image's list also names the symbols of the others it refers to, as absolute addresses; those are left out. """ own = {} path = ROOT / region.CONFIG / f"{image}.symbols.txt" for line in path.read_text(encoding="utf-8").splitlines(): match = re.match(r"(\S+) = (0x[0-9a-fA-F]+);(.*)$", line.strip()) if match and "absolute:True" not in match.group(3): own.setdefault(match.group(1), int(match.group(2), 16)) return own, sorted(set(own.values())) def retail_constant(name, image): """The bytes retail holds for one named constant, up to the next symbol.""" own, addresses = retail_symbols(image) address = own.get(name) if address is None: return None index = bisect.bisect_right(addresses, address) if index == len(addresses): return None return retail.read(image, address, addresses[index] - address) def compiled_constants(path): """Every local `@N` constant the object defines, with its bytes.""" data = path.read_bytes() e_shoff, = struct.unpack_from("> 4 != 0 or not st_name: # STB_LOCAL only continue end = names.index(b"\0", st_name) symbol = names[st_name:end].decode() if symbol.startswith("@") and st_shndx < len(sections): out.setdefault(symbol, contents(st_shndx)) return out def export_constants(path, names, parser, image, reserved_constants=()): """Rename the object's own constants to the names retail's other units use. A translation unit that is only half decompiled has its string constants generated by the compiled half under invented `@N` names, while the assembly half still refers to them by the name splat took from retail. Matching the two by content is what lets the dump stop supplying a second copy -- which is what moves the small data after it. """ if not names: return # A constant the source defines under a name of its own already carries # its retail name once the `symbols` fixup has run, and needs no alias. elf = Elf(path.read_bytes()) defined = { symbol.name for symbol in elf.symtab.symbols if symbol.st_shndx != 0 } compiled_symbols = { symbol.name: symbol for symbol in elf.symtab.symbols if symbol.bind == 0 and symbol.name.startswith("@") and 0 < symbol.st_shndx < len(elf.sections) } compiled = { name: elf.sections[symbol.st_shndx].data for name, symbol in compiled_symbols.items() } # A configured symbol rename may already have assigned a constant its # retail name. Reserve such constants so an identical byte sequence cannot # also satisfy a later content-based export. reserved = set(reserved_constants) & defined arguments = [] padded = False # Both halves of the unit emit their constants in source order, so taking # retail's in address order and ours in emission order pairs duplicates -- # a template a second function initialises identically -- with the copy # retail's assembly actually names. addresses = retail_addresses() order = {name: index for index, name in enumerate(names)} emitted = {name: symbol.st_shndx for name, symbol in compiled_symbols.items()} for name in sorted(names, key=lambda n: (addresses.get(n, 0), order[n])): if name in defined: continue wanted = retail_constant(name, image) if wanted is None: parser.error(f"retail's {image} defines no {name!r}") matches = [symbol for symbol, body in compiled.items() if symbol not in reserved and body and wanted.startswith(body)] # A shorter constant is a prefix of every longer one that begins with # the same bytes -- the empty string is a prefix of a zero vector -- so # a constant of exactly retail's length is the one meant, and every # other candidate belongs to a name still to come. exact = [symbol for symbol in matches if compiled[symbol] == wanted] if exact: matches = sorted(exact, key=lambda symbol: emitted[symbol])[:1] if len(matches) != 1: parser.error(f"{name!r} matches {len(matches)} constants in {path}") # MWLD recognizes a local `@N` as a compiler constant and rounds its # contribution to eight bytes, ignoring a larger section alignment. # Preserve the explicit zero tail in retail's dump inside the section # itself so the following constant still begins at its retail address. old = matches[0] section = elf.sections[compiled_symbols[old].st_shndx] if len(wanted) > len(section.data): tail = wanted[len(section.data):] if any(tail): parser.error(f"{name!r} has nonzero bytes beyond its compiler " f"constant in {path}") section.data = wanted section.sh_size = len(wanted) compiled[old] = wanted padded = True # One compiled constant stands for one of retail's, so it is spent. reserved.add(old) arguments += ["--redefine-sym", f"{old}={name}", "--globalize-symbol", name] if padded: path.write_bytes(elf.pack()) objcopy = os.environ.get("MIPS_TOOL_PREFIX", "mips-ps2-decompals-") + "objcopy" subprocess.run([objcopy] + arguments + [str(path), str(path)], check=True) def rename_symbols(path, mappings): """Spell the unit's own symbols the way the reference assembly does. Retail gives two functions in different images the same mangled name, and the disassembler tells them apart with a `__N` suffix that the source cannot write. Renaming the object's definition and its references together keeps the call inside the unit, rather than letting it resolve against the other image's copy. """ if not mappings: return arguments = [] for old, new in mappings.items(): arguments += ["--redefine-sym", f"{old}={new}"] objcopy = os.environ.get("MIPS_TOOL_PREFIX", "mips-ps2-decompals-") + "objcopy" subprocess.run([objcopy] + arguments + [str(path), str(path)], check=True) def globalize_symbols(path, names): """Expose local functions referenced by retained cross-unit data tables.""" if not names: return objcopy = os.environ.get("MIPS_TOOL_PREFIX", "mips-ps2-decompals-") + "objcopy" arguments = [value for name in names for value in ("--globalize-symbol", name)] subprocess.run([objcopy] + arguments + [str(path), str(path)], check=True) # Every datum retail places in .sdata or .sbss starts on a four-byte boundary, # whatever its own type needs: the two-byte `rpad$180` is followed by two bytes # of padding, and each one-byte flag by three. MWCC gives each static a section # of its own with the alignment of its type, so a `char` lands wherever the # previous datum ended and every gp-relative offset after it is short. SMALL_DATA_ALIGNMENT = 4 SMALL_DATA_SECTIONS = (".sdata", ".sbss") # Retail starts every constant on an eight-byte boundary, whatever its own # length asks for: `"IMG"` follows an eighteen-byte name at 0x299028 rather than # at 0x299024. MWCC gives a constant of four bytes or fewer a section of # four-byte alignment, so without this the short ones pull everything after them # forward. RODATA_ALIGNMENT = 8 # A constant the reference assembly still supplies arrives asking for sixteen, # because that is what splat writes at the head of every dump. Retail granted # that to some and not to others -- 121 of the map editor's 283 constants sit # where eight divides and sixteen does not -- so where the address is known it # is the address, rather than either compiler's habit, that says how wide the # slot is. RODATA_ALIGNMENTS = (16, 8, 4) @functools.lru_cache(maxsize=None) def retail_addresses(): """Every symbol the configuration gives an address, by name.""" out = {} for path in sorted((ROOT / region.CONFIG).glob("*.symbols.txt")): for line in path.read_text(encoding="utf-8").splitlines(): match = re.match(r"(\S+) = (0x[0-9a-fA-F]+);", line.strip()) if match: out.setdefault(match.group(1), int(match.group(2), 16)) return out def rodata_alignment(names, retail_names=None): """The slot retail gave a constant, or none where it names no address. A constant the source defines under a name of its own and the unit's `symbols` fixup renames to retail's `@N` is looked up by that retail name. """ retail_names = retail_names or {} for name in names: known = retail_names.get(name) if known: address = retail_addresses().get(known) if address: return next(a for a in RODATA_ALIGNMENTS if address % a == 0) # MWCC assigns @N independently in every translation unit. A compiled # @N can share a spelling with an unrelated retail constant, so its # number cannot be used as an address-stable identity. if name.startswith("@"): continue address = retail_addresses().get(name) if address: return next(a for a in RODATA_ALIGNMENTS if address % a == 0) return None # A datum defined for a function the source writes is named for the retail # datum it stands for: `pal_` + retail's name, `@` spelled `at` and # a static's `$` spelled `_S`. GENERATED_DATUM = re.compile(r"^pal_(.+)$") def generated_retail_name(name): """The retail name a generated datum's identifier spells, or None.""" match = GENERATED_DATUM.match(name) if not match: return None body = match.group(1) if re.match(r"^at\d", body): body = "@" + body[2:] return re.sub(r"_S(\d+)((?:__\d+)?)$", r"$\1\2", body) def align_generated_data(elf): """Give each generated datum's section the alignment retail's address implies. The compiler gives an array of sixteen bytes or more a sixteen-byte slot whatever its place, so a datum retail kept on an eight-byte boundary would move everything after it. """ for symbol in elf.symtab.symbols: if not (0 < symbol.st_shndx < len(elf.sections)): continue name = generated_retail_name(symbol.name) address = retail_addresses().get(name) if name else None if address is None: continue section = elf.sections[symbol.st_shndx] floor = SMALL_DATA_ALIGNMENT if section.name in SMALL_DATA_SECTIONS else 1 section.sh_addralign = max(floor, min(16, address & -address)) def align_small_data(elf, retail_names=None): """Give each small-data section retail's four-byte slot.""" defined = defaultdict(list) for symbol in elf.symtab.symbols: if symbol.st_shndx and symbol.st_shndx < len(elf.sections): defined[symbol.st_shndx].append(symbol.name) for index, section in enumerate(elf.sections): if (section.name in SMALL_DATA_SECTIONS and section.sh_addralign < SMALL_DATA_ALIGNMENT): section.sh_addralign = SMALL_DATA_ALIGNMENT elif section.name in (".rodata", ".rdata", ".vutext", ".vudata"): wanted = rodata_alignment(defined.get(index, ()), retail_names) if wanted is not None: section.sh_addralign = wanted elif section.sh_addralign < RODATA_ALIGNMENT: section.sh_addralign = RODATA_ALIGNMENT def rename_sections(elf, mappings, parser): """Rename the sections that define configured symbols. A compiler-numbered constant only carries its retail name once export_constants has bound it, so an entry naming one is handed back for a second pass over the written object. """ # A compiler-numbered constant can share its name with a retail constant # spliced in from assembly; the section to rename is the writable one. symbols = {} for symbol in elf.symtab.symbols: held = symbols.get(symbol.name) if held is None or (0 < held.st_shndx < len(elf.sections) and elf.sections[held.st_shndx].name == ".rodata"): symbols[symbol.name] = symbol renamed = {} deferred = {} for section_name, names in mappings.items(): name_index = elf.add_sh_symbol(section_name) for symbol_name in names: symbol = symbols.get(symbol_name) if symbol is None: if NUMBERED_DATUM.match(symbol_name): deferred.setdefault(section_name, []).append(symbol_name) continue parser.error(f"{symbol_name!r} is not defined") index = symbol.st_shndx if index in renamed and renamed[index] != section_name: parser.error(f"section {index} has conflicting names") elf.sections[index].sh_name = name_index elf.sections[index].name = section_name renamed[index] = section_name return deferred # A datum the compiler invented a name for: `@N`, or a function-local static # named `name$N`. NUMBERED_DATUM = re.compile(r"^(?:@\d+|[A-Za-z_]\w*\$\d+)(?:__\d+)?$") DISCARD_SECTION = ".discard" MWCATS_PREFIX = ".mwcats_" def drop_functions(elf, names, parser): """Keep a function out of the link without keeping it out of the compile. A few of the movie's units carry a function nothing calls and retail's link removed. They cannot simply be deleted: the compiler carries state from one definition to the next, and taking one out moves the constants of the definitions after it. MWLD lays a section out whatever its flags say, so the section is given a name of its own here and objcopy takes it away below; the compiler still saw the definition. """ if not names: return wanted = set(names) # MWCC names a function the linker may merge in a `.mwcats` section of its # own, under a section symbol spelling the function. Only the entry for a # function going away may be taken with it: the rest are what let MWLD keep # one copy of a constructor the compiler wrote itself. markers = {MWCATS_PREFIX + name for name in names} name_index = elf.add_sh_symbol(DISCARD_SECTION) for symbol in elf.symtab.symbols: if symbol.name in wanted | markers and symbol.st_shndx: section = elf.sections[symbol.st_shndx] section.sh_name = name_index section.name = DISCARD_SECTION wanted.discard(symbol.name) if wanted: parser.error("the object defines no %s" % ", ".join(sorted(wanted))) # A constructor or destructor MWCC writes itself, which every unit that needs # its address emits a copy of. Retail's overlay carries none of them: its units # reach the one copy main holds. GENERATED_MEMBER = re.compile(r"^__(?:ct|dt)__") def extern_functions(elf, names, parser): """Let a unit reach main's copy instead of laying out its own. MWLD keeps one copy of a constructor the compiler wrote itself only when every copy is one the compiler wrote: retail's is assembled, at a fixed address in main, so a compiled copy beside it is a second definition and the link fails. The definition goes in the section objcopy takes away below and the symbol is left undefined, which is what the relocation needs to bind to main's. """ if not names: return wanted = set(names) name_index = elf.add_sh_symbol(DISCARD_SECTION) for symbol in elf.symtab.symbols: if symbol.name in wanted and symbol.st_shndx: section = elf.sections[symbol.st_shndx] section.sh_name = name_index section.name = DISCARD_SECTION symbol.st_shndx = 0 symbol.st_value = 0 symbol.st_size = 0 wanted.discard(symbol.name) if wanted: parser.error("the object defines no %s" % ", ".join(sorted(wanted))) def shared_constant_name(text): """The name two units reach one constant by.""" return "_S_" + re.sub(r"\W", "_", text) def constant_sections(elf, wanted): """{text: symbol} for each wanted string the object holds its own copy of. The empty string is a prefix of every constant that begins with a zero byte, so where several sections fit, the shortest is the one that holds the string and nothing more. """ found = {} for symbol in elf.symtab.symbols: if not symbol.st_shndx or symbol.st_shndx >= len(elf.sections): continue section = elf.sections[symbol.st_shndx] data = section.data for text in wanted: if data is None or not section.name.startswith(".rodata"): continue if data.startswith(text.encode() + b"\0") \ and len(data) - len(text) - 1 < 16: held = found.get(text) if held is None or len(data) < len(elf.sections[held.st_shndx].data): found[text] = symbol return found def share_constants(elf, exported, imported, parser, source_only=False): """Keep one copy of a constant two units use, the way retail's link has it. MWCC gives every unit its own copy of a string it names, and MWLD merges none of them: retail's own image holds `opdat/chara/01p19a1a.chr` once, in op_a, with op_b's table pointing at it. Whichever unit retail leaves it in names it here and exports it; the others drop their copy and reach that one. """ if not exported and not imported: return # An entry is the constant's text, or `[text, name]` where the name is what # both units reach it by -- which is how a constant retail's dump already # names, such as one the source defines under a `symbols` rename, keeps # that name. exported = [(e, None) if isinstance(e, str) else tuple(e) for e in exported] imported = [(e, None) if isinstance(e, str) else tuple(e) for e in imported] entries = exported + imported by_name = {symbol.name: symbol for symbol in elf.symtab.symbols} found = constant_sections(elf, [text for text, _name in entries]) # The unit retail leaves the constant in already carries the shared name, # so name it outright rather than looking for it by content. for text, name in entries: if name is not None and name in by_name: found[text] = by_name[name] missing = [text for text, _name in entries if text not in found] if missing: # A named entry can refer to a constant a source-only object does not # hold; it then has neither the name nor the text, and nothing to share. if not source_only: parser.error("the object holds no %s" % ", ".join(repr(m) for m in missing)) exported = [e for e in exported if e[0] in found] imported = [e for e in imported if e[0] in found] for text, name in exported: symbol = found[text] symbol.name = name or shared_constant_name(text) symbol.st_name = elf.strtab.add_symbol(symbol.name) symbol.st_info = (1 << 4) | (symbol.st_info & 0xF) # STB_GLOBAL if imported: name_index = elf.add_sh_symbol(DISCARD_SECTION) for text, name in imported: symbol = found[text] section = elf.sections[symbol.st_shndx] section.sh_name = name_index section.name = DISCARD_SECTION symbol.name = name or shared_constant_name(text) symbol.st_name = elf.strtab.add_symbol(symbol.name) symbol.st_info = (1 << 4) | (symbol.st_info & 0xF) symbol.st_shndx = 0 symbol.st_value = 0 symbol.st_size = 0 def coalesced_functions(elf): """The constructors MWCC wrote itself, which have to stay global. MWCC emits one of these global but places it among the local symbols, and every objcopy pass rebuilds the table from `sh_info` and so demotes it. A local copy is a second definition, which MWLD lays out in full rather than binding the reference to the one main already holds. Nothing else in the catalogue needs this: a function the source wrote is global already, and globalizing one is what it already is. """ return sorted( name[len(MWCATS_PREFIX):] for name in {symbol.name for symbol in elf.symtab.symbols} if name.startswith(MWCATS_PREFIX) and GENERATED_MEMBER.match(name[len(MWCATS_PREFIX):]) ) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("object", type=Path) parser.add_argument("source") parser.add_argument("--config", type=Path, default=ROOT / region.CONFIG / "object_fixups.json") # objdiff's base object is a plain compile of the source alone, with no # spliced assembly beside it. The fixups that reach a constant or a static # the splice supplies have nothing to act on there, and saying so is not an # error -- the object is only ever read by objdiff, never linked. parser.add_argument("--source-only", action="store_true") args = parser.parse_args() elf = Elf(args.object.read_bytes()) config = json.loads(args.config.read_text(encoding="utf-8")) fixups = config.get(args.source, {}) # A function-local static is `name$N` in the object and `name$M` in retail, # both numbered off the compiler's counter; `statics` pairs the two. aliases = fixups.get("statics", {}) for symbol in elf.symtab.symbols: replacement = aliases.get(symbol.name) if replacement: symbol.name = replacement symbol.st_name = elf.strtab.add_symbol(replacement) # The retail name each of the unit's own names stands for: a `symbols` # entry renames the source's name to retail's, or retail's `@N` to the name # the other units reach it by. retail_names = {} for old_name, new_name in fixups.get("symbols", {}).items(): if old_name.startswith("@"): retail_names[new_name] = old_name else: retail_names[old_name] = new_name align_small_data(elf, retail_names) align_generated_data(elf) drop_functions(elf, fixups.get("drop_functions", []), parser) extern_functions(elf, fixups.get("extern_functions", []), parser) share_constants(elf, fixups.get("export_constants_shared", []), fixups.get("import_constants_shared", []), parser, args.source_only) # Every objcopy pass rebuilds the symbol table from `sh_info`, and MWCC # puts a global that belongs after it among the locals, so each pass # demotes it. fixup_sections.sh is the last one over the object: it reads # this list back and restores them there, after nothing else can undo it. Path(str(args.object) + ".coal").write_text( "".join(name + "\n" for name in coalesced_functions(elf) + [shared_constant_name(entry) if isinstance(entry, str) else entry[1] for entry in fixups.get("export_constants_shared", [])]), encoding="utf-8") deferred_sections = rename_sections(elf, fixups.get("sections", {}), parser) args.object.write_bytes(elf.pack()) # After the rewrite: objcopy reads the file, so these have to be last. dropped = fixups.get("drop_functions", []) if dropped or fixups.get("extern_functions") \ or fixups.get("import_constants_shared"): objcopy = os.environ.get("MIPS_TOOL_PREFIX", "mips-ps2-decompals-") + "objcopy" arguments = ["--wildcard", "--remove-section", DISCARD_SECTION] # the symbol has to go with the section it stood in, or objcopy calls # it required but not present for name in dropped: arguments += ["--strip-symbol", name] subprocess.run([objcopy] + arguments + [str(args.object), str(args.object)], check=True) # Sections retail's link never kept -- the exception tables of the runtime # functions compiled with exceptions on: retail's table is empty. removed = fixups.get("remove_sections", []) if removed: objcopy = os.environ.get("MIPS_TOOL_PREFIX", "mips-ps2-decompals-") + "objcopy" arguments = [] for name in removed: arguments += ["--remove-section", name] subprocess.run([objcopy] + arguments + [str(args.object), str(args.object)], check=True) rename_symbols(args.object, fixups.get("symbols", {})) # GNU assembly cannot spell MWCC's angle-bracket template names. Splat # uses the same deterministic safe spelling for definitions and callers. native_symbols = { symbol.name for symbol in Elf(args.object.read_bytes()).symtab.symbols if "<" in symbol.name } template_aliases = { name: name.replace("<", "_").replace(",", "_") .replace(">", "_").replace(" ", "") for name in native_symbols } rename_symbols(args.object, template_aliases) globalize_symbols(args.object, fixups.get("globalize_symbols", [])) export_constants(args.object, fixups.get("rodata_exports", []), parser, disassemble.image_of_unit(args.source[len("ps2/src/"):]), fixups.get("symbols", {}).values()) if deferred_sections: elf = Elf(args.object.read_bytes()) if rename_sections(elf, deferred_sections, parser) and not args.source_only: parser.error( "a numbered constant in `sections` is not exported; if the " "edit renamed it, empty the entries and re-key them with " "scripts/build/rekey_sections.py") args.object.write_bytes(elf.pack()) if __name__ == "__main__": main()