diff --git a/include/compat/host_abi_profile.h b/include/compat/host_abi_profile.h index c015e8a..413b768 100644 --- a/include/compat/host_abi_profile.h +++ b/include/compat/host_abi_profile.h @@ -109,6 +109,15 @@ struct HostAbiProfile { size_t NativeMimallocConstructorRangeEndExclusive() const noexcept; bool ShouldInitializeNativeMimallocThreadAfterConstructor( size_t index) const noexcept; + // True when both replays have the one verified shape. The native replay is + // one range that runs to the end of .init_array. Its first four + // constructors are the process-load constructor at the thread-initializer + // checkpoint, the JNI singleton initializer, and two wiring entries. The + // host-bridge replay runs the same constructors without the two wiring + // entries. `jni_singleton_constructor` is the .init_array index of the JNI + // singleton initializer. + bool HasVerifiedConstructorPolicy( + size_t jni_singleton_constructor) const noexcept; bool HasValidNativePreJniBootstrap() const noexcept; HostAllocatorStrategy ResolveAllocatorStrategy( bool host_bridges_override_present, diff --git a/scripts/derive_roblox_host_abi_profile.py b/scripts/derive_roblox_host_abi_profile.py index affd557..d1a8389 100755 --- a/scripts/derive_roblox_host_abi_profile.py +++ b/scripts/derive_roblox_host_abi_profile.py @@ -161,6 +161,10 @@ class SignatureSpec: allow_registry_object_size_change: bool = False minimum_anchor_bytes: int = 6 registry_size_indices: tuple[int, int] = (8, 12) + # Instructions whose register operands the compiler may have chosen + # differently. Each reference register must stand for one candidate + # register across all of them, and the reverse. + renamed_register_indices: tuple[int, ...] = () @dataclass(frozen=True) @@ -1075,6 +1079,51 @@ def semantic_shape( ) +def register_family(name: str) -> str: + """Return the 64-bit register that name is all or part of.""" + if len(name) >= 2 and name[0] == "r" and name[1].isdigit(): + return name.rstrip("bwd") + for family in ("ax", "bx", "cx", "dx", "si", "di", "bp", "sp"): + if name in (family, "e" + family, "r" + family): + return "r" + family + return name + + +def renamed_shapes_match( + reference_shape: tuple[Any, ...], + candidate_shape: tuple[Any, ...], + forward: dict[str, str], + backward: dict[str, str], +) -> bool: + """Compare two semantic shapes, letting register operands be renamed. + + forward and backward hold the renames accepted so far and gain new ones. + """ + reference_mnemonic, reference_operands = reference_shape + candidate_mnemonic, candidate_operands = candidate_shape + if reference_mnemonic != candidate_mnemonic or len(reference_operands) != len( + candidate_operands + ): + return False + for left, right in zip(reference_operands, candidate_operands): + if left[0] != "reg" or right[0] != "reg": + if left != right: + return False + continue + if left[2] != right[2]: + return False + reference_family = register_family(left[1]) + candidate_family = register_family(right[1]) + if ( + forward.setdefault(reference_family, candidate_family) + != candidate_family + or backward.setdefault(candidate_family, reference_family) + != reference_family + ): + return False + return True + + def validate_semantic_match( reference: Sequence[Any], candidate: Sequence[Any], @@ -1082,6 +1131,8 @@ def validate_semantic_match( ) -> None: if len(reference) != len(candidate): raise AnalyzerError("normalized signature instruction count changed") + forward: dict[str, str] = {} + backward: dict[str, str] = {} for index, (reference_instruction, candidate_instruction) in enumerate( zip(reference, candidate) ): @@ -1089,8 +1140,14 @@ def validate_semantic_match( spec.allow_registry_object_size_change and index in spec.registry_size_indices ) - if semantic_shape(reference_instruction, wildcard_size) != semantic_shape( - candidate_instruction, wildcard_size + reference_shape = semantic_shape(reference_instruction, wildcard_size) + candidate_shape = semantic_shape(candidate_instruction, wildcard_size) + if ( + not renamed_shapes_match( + reference_shape, candidate_shape, forward, backward + ) + if index in spec.renamed_register_indices + else reference_shape != candidate_shape ): raise AnalyzerError( f"normalized instruction semantics changed at signature index {index}" @@ -1613,6 +1670,73 @@ def find_registry_function_entry( return entries[0] +def find_renamed_registry_body_match( + reference: ElfImage, + candidate: ElfImage, + reference_body: Sequence[Any], + minimum_anchor_bytes: int, +) -> SignatureMatch: + """Find the registry initializer body under other temporary registers. + + The body allocates the registry, clears it, and starts filling it: + + 0 mov edi, SIZE 6 xor esi, esi + 1 call allocate 7 call memset + 2 mov rbx, rax 8 mov ONE, 0x3f800000 + 3 xor ZERO, ZERO 9 mov dword ptr [rbx + 0x20], ONE + 4 mov edx, SIZE 10 lea FIELD, [rbx + 0x28] + 5 mov rdi, rax + + ZERO, ONE and FIELD are the compiler's choice. Instructions 4 to 7 use + registers the calling convention fixes, so they are searched for by bytes + and every hit is then checked against all eleven instructions. + """ + clear_index = 4 + clear_matches = find_signature_matches( + reference, + candidate, + SignatureSpec( + "registry-initializer-clear", + reference_body[clear_index].address, + 4, + True, + minimum_anchor_bytes, + (0, 0), + ), + allow_multiple_candidates=True, + ) + body_spec = SignatureSpec( + "registry-initializer-body", + reference_body[0].address, + len(reference_body), + True, + minimum_anchor_bytes, + (0, 4), + (3, 8, 9, 10), + ) + # Instructions 0 to 3 keep their lengths as long as ZERO stays one of r8 + # to r15, so the body starts this far before the clear sequence. + lead = reference_body[clear_index].address - reference_body[0].address + found = [] + for clear in clear_matches: + if clear.rva < lead: + continue + try: + body = disassemble(candidate, clear.rva - lead, len(reference_body)) + if body[clear_index].address != clear.rva: + continue + validate_semantic_match(reference_body, body, body_spec) + except AnalyzerError: + continue + found.append(SignatureMatch(clear.rva - lead, tuple(reference_body), body)) + if len(found) != 1: + raise AnalyzerError( + f"signature registry-initializer-body matched {len(found)} " + "candidate locations with renamed registers" + ) + return found[0] + + def find_registry_signature_match( reference: ElfImage, candidate: ElfImage, @@ -1633,7 +1757,18 @@ def find_registry_signature_match( spec.minimum_anchor_bytes, (0, 4), ) - body_match = find_signature_match(reference, candidate, body_spec) + try: + body_match = find_signature_match(reference, candidate, body_spec) + except AnalyzerError: + body_match = find_renamed_registry_body_match( + reference, + candidate, + reference_full[ + body_instruction_index : body_instruction_index + + body_instruction_count + ], + spec.minimum_anchor_bytes, + ) candidate_entry = find_registry_function_entry( candidate, body_match.rva, @@ -1671,14 +1806,14 @@ def find_constructor_boundary_match( reference: ElfImage, candidate: ElfImage, spec: SignatureSpec, - reference_entry: int, - candidate_entry: int, + candidate_entries: Sequence[int], ) -> SignatureMatch: - if reference_entry != spec.reference_rva: - raise AnalyzerError( - "reference constructor boundary moved outside init-array index 4" - ) - reference_wrapper = disassemble(reference, reference_entry, 1) + """Match a constructor that is one jump to its body. + + The body is found by signature, and the candidate constructor is the one + of candidate_entries that jumps to it. + """ + reference_wrapper = disassemble(reference, spec.reference_rva, 1) reference_target = direct_jump_target( reference_wrapper[0], "reference constructor boundary" ) @@ -1686,20 +1821,29 @@ def find_constructor_boundary_match( reference, candidate, SignatureSpec( - "constructor-4-target", + f"{spec.name}-target", reference_target, spec.instruction_count, ), ) - candidate_wrapper = disassemble(candidate, candidate_entry, 1) - candidate_target = direct_jump_target( - candidate_wrapper[0], "candidate constructor boundary" - ) - if candidate_target != target_match.rva: + found = [] + for candidate_entry in candidate_entries: + try: + candidate_wrapper = disassemble(candidate, candidate_entry, 1) + candidate_target = direct_jump_target( + candidate_wrapper[0], "candidate constructor boundary" + ) + except AnalyzerError: + continue + if candidate_target == target_match.rva: + found.append( + SignatureMatch(candidate_entry, reference_wrapper, candidate_wrapper) + ) + if len(found) != 1: raise AnalyzerError( - "constructor index 4 no longer targets its verified boundary" + f"{spec.name} no longer targets its verified boundary" ) - return SignatureMatch(candidate_entry, reference_wrapper, candidate_wrapper) + return found[0] def rip_targets(instruction: Any) -> tuple[int, ...]: @@ -1713,6 +1857,161 @@ def rip_targets(instruction: Any) -> tuple[int, ...]: return tuple(targets) +def is_register_operand(operand: Any, register: int) -> bool: + return operand.type == capstone_x86.X86_OP_REG and operand.reg == register + + +def jni_singleton_publication(image: ElfImage, entry: int) -> tuple[Any, ...]: + """Return the instructions that publish the JNI singleton. + + They open its initializer at entry, after a stack frame if it has one: + + lea rax, [rip + OBJECT] + xchg qword ptr [rip + SLOT], rax + mov qword ptr [rip + OBJECT], rax + + OBJECT is the eight bytes after SLOT. + """ + for frame in (0, 2): + try: + decoded = disassemble(image, entry, frame + 3) + except AnalyzerError: + continue + if frame: + push, link = decoded[:2] + if ( + push.mnemonic != "push" + or len(push.operands) != 1 + or not is_register_operand(push.operands[0], capstone_x86.X86_REG_RBP) + or link.mnemonic != "mov" + or len(link.operands) != 2 + or not is_register_operand(link.operands[0], capstone_x86.X86_REG_RBP) + or not is_register_operand(link.operands[1], capstone_x86.X86_REG_RSP) + ): + continue + load, exchange, store = decoded[frame:] + object_targets = rip_targets(load) + slot_targets = rip_targets(exchange) + stored_targets = rip_targets(store) + if ( + load.mnemonic == "lea" + and len(load.operands) == 2 + and is_register_operand(load.operands[0], capstone_x86.X86_REG_RAX) + and exchange.mnemonic == "xchg" + and len(exchange.operands) == 2 + and exchange.operands[0].size == 8 + and is_register_operand(exchange.operands[1], capstone_x86.X86_REG_RAX) + and store.mnemonic == "mov" + and len(store.operands) == 2 + and store.operands[0].size == 8 + and is_register_operand(store.operands[1], capstone_x86.X86_REG_RAX) + and len(object_targets) == 1 + and len(slot_targets) == 1 + and len(stored_targets) == 1 + and object_targets[0] == stored_targets[0] + and object_targets[0] == slot_targets[0] + 8 + ): + return decoded[frame:] + raise AnalyzerError( + f"JNI singleton initializer at {format_rva(entry)} " + "does not publish its singleton" + ) + + +def find_jni_singleton_match( + reference: ElfImage, + candidate: ElfImage, + spec: SignatureSpec, + candidate_entries: Sequence[int], +) -> SignatureMatch: + """Find the JNI singleton initializer among candidate_entries. + + The instructions after the publication follow the singleton's layout, so + when a release changes that layout only the publication is matched. The + initializer itself runs from the init-array either way. + """ + try: + return find_signature_match(reference, candidate, spec) + except AnalyzerError: + pass + reference_publication = jni_singleton_publication(reference, spec.reference_rva) + found = find_signature_match( + reference, + candidate, + SignatureSpec( + "jni-singleton-publication", + reference_publication[0].address, + len(reference_publication), + minimum_anchor_bytes=3, + ), + ) + for entry in candidate_entries: + try: + publication = jni_singleton_publication(candidate, entry) + except AnalyzerError: + continue + if publication[0].address == found.rva: + return SignatureMatch(entry, reference_publication, publication) + raise AnalyzerError("JNI singleton initializer is not a leading constructor") + + +# The native replay starts at this init-array index on x86_64. +LEADING_CONSTRUCTOR_BEGIN = 2 +# The process-load constructor, the JNI singleton initializer and two wiring +# entries. A release may put them in any order. +LEADING_CONSTRUCTOR_COUNT = 4 + + +def match_leading_constructors( + reference: ElfImage, + candidate: ElfImage, + constructor_anchors: dict[str, Any], + jni_spec: SignatureSpec, + reference_init: Sequence[int], + candidate_init: Sequence[int], +) -> tuple[tuple[int, ...], SignatureMatch]: + """Find each reference leading constructor in the candidate by its code. + + Returns, for each reference leading constructor, the candidate init-array + index of the same constructor, both counted from + LEADING_CONSTRUCTOR_BEGIN, and the JNI singleton initializer match. + """ + moved = "constructor moved outside verified init-array boundary" + end = LEADING_CONSTRUCTOR_BEGIN + LEADING_CONSTRUCTOR_COUNT + if len(reference_init) < end or len(candidate_init) < end: + raise AnalyzerError("init-array shape is not derivable") + candidate_entries = tuple(candidate_init[LEADING_CONSTRUCTOR_BEGIN:end]) + candidate_offsets = [] + jni_match = None + for index in range(LEADING_CONSTRUCTOR_BEGIN, end): + reference_entry = reference_init[index] + name = f"constructor-{index}" + if parse_rva(constructor_anchors.get(str(index)), name) != reference_entry: + raise AnalyzerError(moved) + if reference_entry == jni_spec.reference_rva: + match = find_jni_singleton_match( + reference, candidate, jni_spec, candidate_entries + ) + jni_match = match + else: + spec = SignatureSpec(name, reference_entry, 24) + if disassemble(reference, reference_entry, 1)[0].mnemonic == "jmp": + match = find_constructor_boundary_match( + reference, candidate, spec, candidate_entries + ) + else: + match = find_signature_match(reference, candidate, spec) + if match.rva not in candidate_entries: + raise AnalyzerError(moved) + candidate_offset = candidate_entries.index(match.rva) + if candidate_offset in candidate_offsets: + raise AnalyzerError(moved) + candidate_offsets.append(candidate_offset) + if jni_match is None: + raise AnalyzerError("JNI singleton initializer is not a leading constructor") + return tuple(candidate_offsets), jni_match + + def mapped_rip_target( match: SignatureMatch, reference_target: int, description: str ) -> int: @@ -1805,7 +2104,6 @@ def create_signature_specs( bridges = bridge_map(profile) data_seeds = profile["data_seeds"] bootstrap = profile["native_pre_jni_bootstrap"] - constructor_rvas = anchors["constructor_rvas"] return [ SignatureSpec( "small-allocate", @@ -1870,36 +2168,51 @@ def create_signature_specs( parse_rva(anchors["jni_singleton_initializer_rva"], "JNI singleton anchor"), 11, ), - SignatureSpec( - "constructor-2", parse_rva(constructor_rvas["2"], "constructor 2"), 24 - ), - SignatureSpec( - "constructor-3", parse_rva(constructor_rvas["3"], "constructor 3"), 24 - ), - SignatureSpec( - "constructor-4", - parse_rva(constructor_rvas["4"], "constructor 4"), - 24, - minimum_anchor_bytes=3, - ), ] -def adjusted_ranges( - ranges: list[dict[str, int]], reference_count: int, candidate_count: int +def remapped_constructor_ranges( + ranges: list[dict[str, int]], + candidate_offsets: Sequence[int], + reference_count: int, + candidate_count: int, ) -> list[dict[str, int]]: - result = [] + """Rewrite the reference constructor ranges for the candidate init-array. + + Each leading constructor keeps its place in or out of the replay under + its candidate index, and every later constructor runs. + """ + cannot_fit = "reference constructor range cannot fit candidate init-array" + leading_end = LEADING_CONSTRUCTOR_BEGIN + LEADING_CONSTRUCTOR_COUNT + ranges = validated_ranges(ranges, "reference constructor ranges", reference_count) + if ( + ranges[0]["begin"] < LEADING_CONSTRUCTOR_BEGIN + or ranges[-1]["begin"] > leading_end + or ranges[-1]["end_exclusive"] != reference_count + or reference_count <= leading_end + or candidate_count <= leading_end + ): + raise AnalyzerError(cannot_fit) + runs = [False] * LEADING_CONSTRUCTOR_COUNT for item in ranges: - end = ( - candidate_count - if item["end_exclusive"] == reference_count - else item["end_exclusive"] - ) - if end > candidate_count: - raise AnalyzerError( - "reference constructor range cannot fit candidate init-array" - ) - result.append({"begin": item["begin"], "end_exclusive": end}) + for index in range(item["begin"], min(item["end_exclusive"], leading_end)): + runs[candidate_offsets[index - LEADING_CONSTRUCTOR_BEGIN]] = True + result = [] + run_begin = None + for offset, run in enumerate(runs): + index = LEADING_CONSTRUCTOR_BEGIN + offset + if run: + if run_begin is None: + run_begin = index + elif run_begin is not None: + result.append({"begin": run_begin, "end_exclusive": index}) + run_begin = None + result.append( + { + "begin": leading_end if run_begin is None else run_begin, + "end_exclusive": candidate_count, + } + ) return validated_ranges(result, "derived constructor ranges", candidate_count) @@ -1948,13 +2261,14 @@ def derive_profile( matches[spec.name] = find_registry_signature_match( reference, candidate, spec ) - elif spec.name == "constructor-4": - matches[spec.name] = find_constructor_boundary_match( + elif spec.name == "jni-singleton-initializer": + leading_offsets, matches[spec.name] = match_leading_constructors( reference, candidate, + anchors["constructor_rvas"], spec, - reference_init[4], - candidate_init[4], + reference_init, + candidate_init, ) else: matches[spec.name] = find_signature_match(reference, candidate, spec) @@ -1990,21 +2304,7 @@ def derive_profile( "registry allocator call" ) matches["allocate"] = selected_allocate[0] - for index in (2, 3, 4): - match = matches[f"constructor-{index}"] - if ( - reference_init[index] != match.reference_instructions[0].address - or candidate_init[index] != match.rva - ): - raise AnalyzerError( - f"constructor {index} moved outside its verified init-array boundary" - ) singleton_match = matches["jni-singleton-initializer"] - if ( - reference_init[5] != singleton_match.reference_instructions[0].address - or candidate_init[5] != singleton_match.rva - ): - raise AnalyzerError("JNI singleton initializer is not constructor index 5") candidate_bridge_entries = [] for entry in reference_profile["bridge_entries"]: @@ -2075,28 +2375,34 @@ def derive_profile( init_section = candidate.sections[".init_array"] reference_count = reference_profile["init_array_count"] candidate_count = len(candidate_init) - constructor_ranges = adjusted_ranges( - validated_ranges( - reference_profile["constructor_run_ranges"], - "reference constructor ranges", - reference_count, - ), + constructor_ranges = remapped_constructor_ranges( + reference_profile["constructor_run_ranges"], + leading_offsets, reference_count, candidate_count, ) - native_ranges = adjusted_ranges( - validated_ranges( - reference_profile["native_mimalloc_constructor_run_ranges"], - "reference native constructor ranges", - reference_count, - ), + native_ranges = remapped_constructor_ranges( + reference_profile["native_mimalloc_constructor_run_ranges"], + leading_offsets, reference_count, candidate_count, ) - thread_boundary = reference_profile[ + reference_thread_boundary = reference_profile[ "native_mimalloc_thread_initializer_after_constructor" ] - if thread_boundary != 2 or not any( + if not ( + LEADING_CONSTRUCTOR_BEGIN + <= reference_thread_boundary + < LEADING_CONSTRUCTOR_BEGIN + LEADING_CONSTRUCTOR_COUNT + ): + raise AnalyzerError("native allocator TLS boundary is no longer verified") + # The thread initializer follows the process-load constructor to its + # candidate index. + thread_boundary = ( + LEADING_CONSTRUCTOR_BEGIN + + leading_offsets[reference_thread_boundary - LEADING_CONSTRUCTOR_BEGIN] + ) + if not any( item["begin"] <= thread_boundary < item["end_exclusive"] for item in native_ranges ): diff --git a/src/compat/host_abi_profile.cc b/src/compat/host_abi_profile.cc index fa5d9a4..3a39602 100644 --- a/src/compat/host_abi_profile.cc +++ b/src/compat/host_abi_profile.cc @@ -270,6 +270,42 @@ bool HostAbiProfile::ShouldInitializeNativeMimallocThreadAfterConstructor( index == native_mimalloc_thread_initializer_after_constructor; } +bool HostAbiProfile::HasVerifiedConstructorPolicy( + size_t jni_singleton_constructor) const noexcept { + constexpr size_t kLeadingConstructors = 4; + constexpr size_t kWiringConstructors = 2; + if (!HasValidConstructorRanges() || + !HasValidNativeMimallocConstructorRanges() || + native_mimalloc_constructor_run_range_count != 1) { + return false; + } + const size_t begin = native_mimalloc_constructor_run_ranges[0].begin; + const size_t checkpoint = + native_mimalloc_thread_initializer_after_constructor; + const ConstructorRange &tail = + constructor_run_ranges[constructor_run_range_count - 1]; + if (init_array_count < begin + kLeadingConstructors || + native_mimalloc_constructor_run_ranges[0].end_exclusive != + init_array_count || + checkpoint < begin || checkpoint >= begin + kLeadingConstructors || + !AllowsConstructor(checkpoint) || jni_singleton_constructor < begin || + jni_singleton_constructor >= begin + kLeadingConstructors || + jni_singleton_constructor == checkpoint || + !AllowsConstructor(jni_singleton_constructor) || + ConstructorRangeBegin() < begin || + tail.begin > begin + kLeadingConstructors || + tail.end_exclusive != init_array_count) { + return false; + } + size_t excluded = 0; + for (size_t index = begin; index < begin + kLeadingConstructors; ++index) { + if (!AllowsConstructor(index)) { + ++excluded; + } + } + return excluded == kWiringConstructors; +} + bool HostAbiProfile::HasValidNativePreJniBootstrap() const noexcept { return native_pre_jni_bootstrap.IsValid(); } diff --git a/src/compat/host_abi_profile_loader.cc b/src/compat/host_abi_profile_loader.cc index 2467ab5..7f050e5 100644 --- a/src/compat/host_abi_profile_loader.cc +++ b/src/compat/host_abi_profile_loader.cc @@ -612,26 +612,26 @@ std::unique_ptr ParseProfile(const Json& root, *error = "external host ABI profile has inconsistent constructor policy"; return nullptr; } - // The verified constructor layout is the same on both guest ABIs: a single - // process-load constructor at the thread-initializer checkpoint, two wiring - // entries excluded under host bridges, and one contiguous native-mimalloc - // replay from the checkpoint to the end of .init_array. Only the checkpoint - // index itself differs between payloads. - const std::size_t checkpoint = - owned->profile.native_mimalloc_thread_initializer_after_constructor; - if (owned->profile.init_array_count <= checkpoint + 3 || - owned->profile.constructor_run_range_count != 2 || - owned->profile.constructor_run_ranges[0].begin != checkpoint || - owned->profile.constructor_run_ranges[0].end_exclusive != checkpoint + 1 || - owned->profile.constructor_run_ranges[1].begin != checkpoint + 3 || - owned->profile.constructor_run_ranges[1].end_exclusive != - owned->profile.init_array_count || - owned->profile.native_mimalloc_constructor_run_range_count != 1 || - owned->profile.native_mimalloc_constructor_run_ranges[0].begin != - checkpoint || - owned->profile.native_mimalloc_constructor_run_ranges[0].end_exclusive != - owned->profile.init_array_count || - (kGuestElfMachine == EM_X86_64 && checkpoint != 2)) { + // The verified constructor layout is the same on both guest ABIs. Payloads + // differ in the order of the four leading constructors, and the guest ABIs + // differ in the index the native replay starts at. + // derivation_code_rvas holds three initializer anchors, then the anchors of + // the four leading constructors in .init_array order. + constexpr size_t kJniSingletonAnchor = 2; + constexpr size_t kFirstConstructorAnchor = 3; + constexpr size_t kConstructorAnchors = 4; + size_t jni_singleton_constructor = owned->profile.init_array_count; + for (size_t anchor = 0; anchor < kConstructorAnchors; ++anchor) { + if (owned->derivation_code_rvas[kFirstConstructorAnchor + anchor] == + owned->derivation_code_rvas[kJniSingletonAnchor]) { + jni_singleton_constructor = + owned->profile.NativeMimallocConstructorRangeBegin() + anchor; + break; + } + } + if (!owned->profile.HasVerifiedConstructorPolicy(jni_singleton_constructor) || + (kGuestElfMachine == EM_X86_64 && + owned->profile.NativeMimallocConstructorRangeBegin() != 2)) { *error = "external host ABI profile changed the verified constructor policy"; return nullptr; diff --git a/src/update/host_abi_deriver.cc b/src/update/host_abi_deriver.cc index 0039689..b421685 100644 --- a/src/update/host_abi_deriver.cc +++ b/src/update/host_abi_deriver.cc @@ -887,6 +887,10 @@ struct SignatureSpec { bool registry_size_may_change = false; std::size_t minimum_anchor_bytes = 6; std::array registry_size_indices = {8, 12}; + // Instructions whose register operands the compiler may have chosen + // differently. Each reference register must stand for one candidate + // register across all of them, and the reverse. + std::vector renamed_register_indices = {}; }; struct SignatureMatch { @@ -895,10 +899,46 @@ struct SignatureMatch { std::vector candidate; }; +// The 64-bit register that `name` is all or part of, such as rax for eax. +std::string RegisterFamily(std::string_view name) { + if (name.size() >= 2 && name[0] == 'r' && name[1] >= '0' && name[1] <= '9') { + while (!name.empty() && (name.back() < '0' || name.back() > '9')) { + name.remove_suffix(1); + } + return std::string(name); + } + for (const std::string_view family : + {"ax", "bx", "cx", "dx", "si", "di", "bp", "sp"}) { + if (name == family || (name.size() == 3 && name.substr(1) == family && + (name[0] == 'e' || name[0] == 'r'))) { + return "r" + std::string(family); + } + } + return std::string(name); +} + +// Reference register families and the candidate families that stand for them. +struct RegisterRenames { + std::map forward; + std::map backward; + + bool Accept(std::string_view reference, std::string_view candidate) { + const std::string left = RegisterFamily(reference); + const std::string right = RegisterFamily(candidate); + const auto forward_found = forward.emplace(left, right).first; + const auto backward_found = backward.emplace(right, left).first; + return forward_found->second == right && backward_found->second == left; + } +}; + bool SameSemanticOperand(const Operand& left, const Operand& right, - bool control_flow, bool wildcard_size) { + bool control_flow, bool wildcard_size, + RegisterRenames* renames = nullptr) { if (left.type != right.type || left.size != right.size) return false; if (left.type == Operand::Type::kRegister) { + if (renames != nullptr) { + return renames->Accept(left.register_name, right.register_name); + } return left.register_name == right.register_name; } if (left.type == Operand::Type::kImmediate) { @@ -918,6 +958,7 @@ bool ValidateSemantics(const std::vector& reference, *error = "normalized signature instruction count changed"; return false; } + RegisterRenames renames; for (std::size_t index = 0; index < reference.size(); ++index) { const Instruction& left = reference[index]; const Instruction& right = candidate[index]; @@ -925,6 +966,10 @@ bool ValidateSemantics(const std::vector& reference, spec.registry_size_may_change && (index == spec.registry_size_indices[0] || index == spec.registry_size_indices[1]); + const bool renamed = + std::find(spec.renamed_register_indices.begin(), + spec.renamed_register_indices.end(), + index) != spec.renamed_register_indices.end(); if (left.mnemonic != right.mnemonic || left.operands.size() != right.operands.size()) { *error = "normalized instruction semantics changed at index " + @@ -933,7 +978,8 @@ bool ValidateSemantics(const std::vector& reference, } for (std::size_t operand = 0; operand < left.operands.size(); ++operand) { if (!SameSemanticOperand(left.operands[operand], right.operands[operand], - left.control_flow, wildcard)) { + left.control_flow, wildcard, + renamed ? &renames : nullptr)) { *error = "normalized instruction operand changed at index " + std::to_string(index); return false; @@ -1569,6 +1615,74 @@ std::optional FindRegistryFunctionEntry( return entries.front(); } +// Finds the registry initializer body when a rebuild holds its temporaries in +// other registers, which changes the bytes the exact search needs. +// +// The body allocates the registry, clears it, and starts filling it: +// +// 0 mov edi, SIZE 6 xor esi, esi +// 1 call allocate 7 call memset +// 2 mov rbx, rax 8 mov ONE, 0x3f800000 +// 3 xor ZERO, ZERO 9 mov dword ptr [rbx + 0x20], ONE +// 4 mov edx, SIZE 10 lea FIELD, [rbx + 0x28] +// 5 mov rdi, rax +// +// ZERO, ONE and FIELD are the compiler's choice. Instructions 4 to 7 use +// registers the calling convention fixes, so they are searched for by bytes +// and every hit is then checked against all eleven instructions. +std::optional FindRenamedRegistryBodyMatch( + const ElfImage& reference, const ElfImage& candidate, + const Disassembler& disassembler, + const std::vector& reference_body, + std::size_t minimum_anchor_bytes, std::string* error) { + constexpr std::size_t kClearIndex = 4; + constexpr std::size_t kClearInstructionCount = 4; + const SignatureSpec clear_spec{ + "registry-initializer-clear", + reference_body[kClearIndex].address, + kClearInstructionCount, + true, + minimum_anchor_bytes, + {0, 0}, + }; + const auto clear_matches = FindSignatureMatches( + reference, candidate, disassembler, clear_spec, true, error); + if (!clear_matches.has_value()) return std::nullopt; + SignatureSpec body_spec{ + "registry-initializer-body", + reference_body.front().address, + reference_body.size(), + true, + minimum_anchor_bytes, + {0, 4}, + }; + body_spec.renamed_register_indices = {3, 8, 9, 10}; + // Instructions 0 to 3 keep their lengths as long as ZERO stays one of r8 + // to r15, so the body starts this far before the clear sequence. + const std::uint64_t lead = + reference_body[kClearIndex].address - reference_body.front().address; + std::vector found; + for (const SignatureMatch& clear : *clear_matches) { + if (clear.rva < lead) continue; + std::string ignored; + const auto body = disassembler.Decode(candidate, clear.rva - lead, + reference_body.size(), 4096, + &ignored); + if (!body.has_value() || (*body)[kClearIndex].address != clear.rva || + !ValidateSemantics(reference_body, *body, body_spec, &ignored)) { + continue; + } + found.push_back(SignatureMatch{clear.rva - lead, reference_body, *body}); + } + if (found.size() != 1U) { + *error = "signature registry-initializer-body matched " + + std::to_string(found.size()) + + " candidate locations with renamed registers"; + return std::nullopt; + } + return std::move(found.front()); +} + std::optional FindRegistrySignatureMatch( const ElfImage& reference, const ElfImage& candidate, const Disassembler& disassembler, const SignatureSpec& spec, @@ -1590,8 +1704,18 @@ std::optional FindRegistrySignatureMatch( spec.minimum_anchor_bytes, {0, 4}, }; - const auto body_match = + auto body_match = FindSignatureMatch(reference, candidate, disassembler, body_spec, error); + if (!body_match.has_value()) { + error->clear(); + body_match = FindRenamedRegistryBodyMatch( + reference, candidate, disassembler, + std::vector( + std::next(reference_full->begin(), kBodyInstructionIndex), + std::next(reference_full->begin(), + kBodyInstructionIndex + kBodyInstructionCount)), + spec.minimum_anchor_bytes, error); + } if (!body_match.has_value()) return std::nullopt; const std::vector reference_prologue( reference_full->begin(), @@ -1633,39 +1757,48 @@ std::optional DirectJumpTarget( return static_cast(instruction.operands[0].value); } +// Matches a constructor that is one jump to its body. The body is found by +// signature, and the candidate constructor is the one of `candidate_entries` +// that jumps to it. std::optional FindConstructorBoundaryMatch( const ElfImage& reference, const ElfImage& candidate, const Disassembler& disassembler, const SignatureSpec& spec, - std::uint64_t reference_entry, std::uint64_t candidate_entry, - std::string* error) { - if (reference_entry != spec.reference_rva) { - *error = "reference constructor boundary moved outside init-array index 4"; - return std::nullopt; - } + const std::vector& candidate_entries, std::string* error) { const auto reference_wrapper = - disassembler.Decode(reference, reference_entry, 1, 16, error); + disassembler.Decode(reference, spec.reference_rva, 1, 16, error); if (!reference_wrapper.has_value()) return std::nullopt; const auto reference_target = DirectJumpTarget( reference_wrapper->front(), "reference constructor boundary", error); if (!reference_target.has_value()) return std::nullopt; - const SignatureSpec target_spec{"constructor-4-target", *reference_target, + const SignatureSpec target_spec{spec.name + "-target", *reference_target, spec.instruction_count}; const auto target_match = FindSignatureMatch( reference, candidate, disassembler, target_spec, error); if (!target_match.has_value()) return std::nullopt; - const auto candidate_wrapper = - disassembler.Decode(candidate, candidate_entry, 1, 16, error); - if (!candidate_wrapper.has_value()) return std::nullopt; - const auto candidate_target = DirectJumpTarget( - candidate_wrapper->front(), "candidate constructor boundary", error); - if (!candidate_target.has_value() || *candidate_target != target_match->rva) { - if (error->empty()) { - *error = "constructor index 4 no longer targets its verified boundary"; + std::optional found; + for (const std::uint64_t candidate_entry : candidate_entries) { + std::string ignored; + const auto candidate_wrapper = + disassembler.Decode(candidate, candidate_entry, 1, 16, &ignored); + if (!candidate_wrapper.has_value()) continue; + const auto candidate_target = DirectJumpTarget( + candidate_wrapper->front(), "candidate constructor boundary", + &ignored); + if (!candidate_target.has_value() || + *candidate_target != target_match->rva) { + continue; } - return std::nullopt; + if (found.has_value()) { + found.reset(); + break; + } + found = SignatureMatch{candidate_entry, *reference_wrapper, + *candidate_wrapper}; } - return SignatureMatch{candidate_entry, *reference_wrapper, - *candidate_wrapper}; + if (!found.has_value()) { + *error = spec.name + " no longer targets its verified boundary"; + } + return found; } std::vector RipTargets(const Instruction& instruction) { @@ -1678,6 +1811,187 @@ std::vector RipTargets(const Instruction& instruction) { return result; } +bool IsRegisterOperand(const Operand& operand, std::string_view name) { + return operand.type == Operand::Type::kRegister && + operand.register_name == name; +} + +// The instructions that publish the JNI singleton, which open its +// initializer at `entry`, after a stack frame if it has one: +// +// lea rax, [rip + OBJECT] +// xchg qword ptr [rip + SLOT], rax +// mov qword ptr [rip + OBJECT], rax +// +// OBJECT is the eight bytes after SLOT. +std::optional> JniSingletonPublication( + const ElfImage& image, const Disassembler& disassembler, + std::uint64_t entry, std::string* error) { + for (const std::size_t frame : {std::size_t{0}, std::size_t{2}}) { + std::string ignored; + const auto decoded = + disassembler.Decode(image, entry, frame + 3, 64, &ignored); + if (!decoded.has_value()) continue; + if (frame != 0) { + const Instruction& push = (*decoded)[0]; + const Instruction& link = (*decoded)[1]; + if (push.mnemonic != "push" || push.operands.size() != 1 || + !IsRegisterOperand(push.operands[0], "rbp") || + link.mnemonic != "mov" || link.operands.size() != 2 || + !IsRegisterOperand(link.operands[0], "rbp") || + !IsRegisterOperand(link.operands[1], "rsp")) { + continue; + } + } + const Instruction& load = (*decoded)[frame]; + const Instruction& exchange = (*decoded)[frame + 1]; + const Instruction& store = (*decoded)[frame + 2]; + const auto object = RipTargets(load); + const auto slot = RipTargets(exchange); + const auto stored = RipTargets(store); + if (load.mnemonic != "lea" || load.operands.size() != 2 || + !IsRegisterOperand(load.operands[0], "rax") || + exchange.mnemonic != "xchg" || exchange.operands.size() != 2 || + exchange.operands[0].size != 8 || + !IsRegisterOperand(exchange.operands[1], "rax") || + store.mnemonic != "mov" || store.operands.size() != 2 || + store.operands[0].size != 8 || + !IsRegisterOperand(store.operands[1], "rax") || object.size() != 1 || + slot.size() != 1 || stored.size() != 1 || object[0] != stored[0] || + object[0] != slot[0] + 8) { + continue; + } + return std::vector(std::next(decoded->begin(), frame), + decoded->end()); + } + *error = "JNI singleton initializer at " + FormatRva(entry) + + " does not publish its singleton"; + return std::nullopt; +} + +// Finds the JNI singleton initializer among `candidate_entries`. The +// instructions after the publication follow the singleton's layout, so when +// a release changes that layout only the publication is matched. The +// initializer itself runs from the init-array either way. +std::optional FindJniSingletonMatch( + const ElfImage& reference, const ElfImage& candidate, + const Disassembler& disassembler, const SignatureSpec& spec, + const std::vector& candidate_entries, std::string* error) { + auto whole = + FindSignatureMatch(reference, candidate, disassembler, spec, error); + if (whole.has_value()) return whole; + error->clear(); + const auto reference_publication = JniSingletonPublication( + reference, disassembler, spec.reference_rva, error); + if (!reference_publication.has_value()) return std::nullopt; + const SignatureSpec publication_spec{ + "jni-singleton-publication", reference_publication->front().address, + reference_publication->size(), false, 3}; + const auto found = FindSignatureMatch(reference, candidate, disassembler, + publication_spec, error); + if (!found.has_value()) return std::nullopt; + for (const std::uint64_t entry : candidate_entries) { + std::string ignored; + const auto publication = + JniSingletonPublication(candidate, disassembler, entry, &ignored); + if (publication.has_value() && + publication->front().address == found->rva) { + return SignatureMatch{entry, *reference_publication, *publication}; + } + } + *error = "JNI singleton initializer is not a leading constructor"; + return std::nullopt; +} + +// The native replay starts at this init-array index on x86_64. +constexpr std::size_t kLeadingConstructorBegin = 2; +// The process-load constructor, the JNI singleton initializer and two wiring +// entries. A release may put them in any order. +constexpr std::size_t kLeadingConstructorCount = 4; + +struct LeadingConstructors { + // For each reference leading constructor, the candidate init-array index + // of the same constructor. Both count from kLeadingConstructorBegin. + std::array candidate_offset{}; + SignatureMatch jni_singleton; +}; + +// Finds each reference leading constructor among the candidate's leading +// constructors by its code. +std::optional MatchLeadingConstructors( + const ElfImage& reference, const ElfImage& candidate, + const Disassembler& disassembler, const Json& constructor_anchors, + const SignatureSpec& jni_spec, + const std::vector& reference_init, + const std::vector& candidate_init, std::string* error) { + constexpr std::size_t kConstructorInstructionCount = 24; + constexpr char kMoved[] = + "constructor moved outside verified init-array boundary"; + const std::size_t end = kLeadingConstructorBegin + kLeadingConstructorCount; + if (reference_init.size() < end || candidate_init.size() < end) { + *error = "init-array shape is not derivable"; + return std::nullopt; + } + const std::vector candidate_entries( + std::next(candidate_init.begin(), kLeadingConstructorBegin), + std::next(candidate_init.begin(), end)); + LeadingConstructors result; + std::array taken{}; + bool found_jni = false; + for (std::size_t offset = 0; offset < kLeadingConstructorCount; ++offset) { + const std::size_t index = kLeadingConstructorBegin + offset; + const std::uint64_t reference_entry = reference_init[index]; + const std::string name = "constructor-" + std::to_string(index); + const auto anchor = ParseRva( + constructor_anchors.value(std::to_string(index), Json()), name, error); + if (!anchor.has_value()) return std::nullopt; + if (*anchor != reference_entry) { + *error = kMoved; + return std::nullopt; + } + std::optional match; + if (reference_entry == jni_spec.reference_rva) { + match = FindJniSingletonMatch(reference, candidate, disassembler, + jni_spec, candidate_entries, error); + if (!match.has_value()) return std::nullopt; + result.jni_singleton = *match; + found_jni = true; + } else { + const auto first = + disassembler.Decode(reference, reference_entry, 1, 16, error); + if (!first.has_value()) return std::nullopt; + const SignatureSpec spec{name, reference_entry, + kConstructorInstructionCount}; + match = first->front().mnemonic == "jmp" + ? FindConstructorBoundaryMatch(reference, candidate, + disassembler, spec, + candidate_entries, error) + : FindSignatureMatch(reference, candidate, disassembler, + spec, error); + if (!match.has_value()) return std::nullopt; + } + const auto found = std::find(candidate_entries.begin(), + candidate_entries.end(), match->rva); + if (found == candidate_entries.end()) { + *error = kMoved; + return std::nullopt; + } + const std::size_t candidate_offset = + static_cast(found - candidate_entries.begin()); + if (taken[candidate_offset]) { + *error = kMoved; + return std::nullopt; + } + taken[candidate_offset] = true; + result.candidate_offset[offset] = candidate_offset; + } + if (!found_jni) { + *error = "JNI singleton initializer is not a leading constructor"; + return std::nullopt; + } + return result; +} + std::optional MappedRipTarget(const SignatureMatch& match, std::uint64_t reference_target, std::string_view description, @@ -1755,16 +2069,23 @@ std::optional PositiveSize(const Json& value, return static_cast(number); } -std::optional AdjustRanges(const Json& ranges, - std::size_t reference_count, - std::size_t candidate_count, - std::string* error) { +// Rewrites the reference constructor ranges for the candidate init-array. +// Each leading constructor keeps its place in or out of the replay under its +// candidate index, and every later constructor runs. +std::optional RemapConstructorRanges( + const Json& ranges, const LeadingConstructors& leading, + std::size_t reference_count, std::size_t candidate_count, + std::string* error) { if (!ranges.is_array() || ranges.empty()) { *error = "constructor ranges are missing"; return std::nullopt; } - Json result = Json::array(); + const std::size_t leading_end = + kLeadingConstructorBegin + kLeadingConstructorCount; + std::array runs{}; std::size_t previous_end = 0; + std::size_t tail_begin = SIZE_MAX; + std::size_t tail_end = 0; for (const Json& range : ranges) { if (!range.is_object() || !range.contains("begin") || !range.contains("end_exclusive") || @@ -1774,15 +2095,39 @@ std::optional AdjustRanges(const Json& ranges, return std::nullopt; } const std::size_t begin = range["begin"].get(); - std::size_t end = range["end_exclusive"].get(); - if (end == reference_count) end = candidate_count; - if (begin >= end || begin < previous_end || end > candidate_count) { + const std::size_t end = range["end_exclusive"].get(); + if (begin >= end || begin < previous_end || end > reference_count || + begin < kLeadingConstructorBegin) { *error = "constructor range cannot fit candidate init-array"; return std::nullopt; } - result.push_back({{"begin", begin}, {"end_exclusive", end}}); + for (std::size_t index = begin; index < std::min(end, leading_end); + ++index) { + runs[leading.candidate_offset[index - kLeadingConstructorBegin]] = true; + } previous_end = end; + tail_begin = begin; + tail_end = end; } + if (tail_begin > leading_end || tail_end != reference_count || + reference_count <= leading_end || candidate_count <= leading_end) { + *error = "constructor range cannot fit candidate init-array"; + return std::nullopt; + } + Json result = Json::array(); + std::size_t run_begin = SIZE_MAX; + for (std::size_t offset = 0; offset < kLeadingConstructorCount; ++offset) { + const std::size_t index = kLeadingConstructorBegin + offset; + if (runs[offset]) { + if (run_begin == SIZE_MAX) run_begin = index; + } else if (run_begin != SIZE_MAX) { + result.push_back({{"begin", run_begin}, {"end_exclusive", index}}); + run_begin = SIZE_MAX; + } + } + result.push_back( + {{"begin", run_begin == SIZE_MAX ? leading_end : run_begin}, + {"end_exclusive", candidate_count}}); return result; } @@ -1826,7 +2171,6 @@ std::optional> SignatureSpecs(const Json& sidecar, }; const Json& seeds = profile["data_seeds"]; const Json& bootstrap = profile["native_pre_jni_bootstrap"]; - const Json& constructors = anchors["constructor_rvas"]; std::vector result; const auto add = [&](std::string name, const Json& encoded, std::size_t count, bool registry = false, std::size_t minimum = 6) -> bool { @@ -1859,10 +2203,7 @@ std::optional> SignatureSpecs(const Json& sidecar, !add("empty-string-initializer", anchors["empty_string_initializer_rva"], 23) || !add("jni-singleton-initializer", - anchors["jni_singleton_initializer_rva"], 11) || - !add("constructor-2", constructors["2"], 24) || - !add("constructor-3", constructors["3"], 24) || - !add("constructor-4", constructors["4"], 24, false, 3)) { + anchors["jni_singleton_initializer_rva"], 11)) { return std::nullopt; } return result; @@ -1966,7 +2307,17 @@ std::optional> DeriveDocuments(const ElfImage& reference, } std::map matches; std::vector allocate_matches; + std::optional leading; for (const SignatureSpec& spec : *specs) { + if (spec.name == "jni-singleton-initializer") { + leading = MatchLeadingConstructors( + reference, candidate, disassembler, + sidecar["derivation_anchors"]["constructor_rvas"], spec, + *reference_init, *candidate_init, error); + if (!leading.has_value()) return std::nullopt; + matches.emplace(spec.name, leading->jni_singleton); + continue; + } if (spec.name == "allocate") { auto candidates = FindSignatureMatches(reference, candidate, disassembler, spec, true, error); @@ -1979,11 +2330,6 @@ std::optional> DeriveDocuments(const ElfImage& reference, return FindRegistrySignatureMatch(reference, candidate, disassembler, spec, error); } - if (spec.name == "constructor-4") { - return FindConstructorBoundaryMatch( - reference, candidate, disassembler, spec, (*reference_init)[4], - (*candidate_init)[4], error); - } return FindSignatureMatch(reference, candidate, disassembler, spec, error); }(); @@ -2028,21 +2374,7 @@ std::optional> DeriveDocuments(const ElfImage& reference, return std::nullopt; } matches.emplace("allocate", std::move(*selected_allocate)); - for (const std::size_t index : {2U, 3U, 4U}) { - const SignatureMatch& match = - matches["constructor-" + std::to_string(index)]; - if ((*reference_init)[index] != match.reference[0].address || - (*candidate_init)[index] != match.rva) { - *error = "constructor moved outside verified init-array boundary"; - return std::nullopt; - } - } const SignatureMatch& singleton = matches["jni-singleton-initializer"]; - if ((*reference_init)[5] != singleton.reference[0].address || - (*candidate_init)[5] != singleton.rva) { - *error = "JNI singleton initializer is not constructor index 5"; - return std::nullopt; - } Json candidate_bridges = Json::array(); std::map bridge_rvas; @@ -2112,21 +2444,29 @@ std::optional> DeriveDocuments(const ElfImage& reference, return std::nullopt; } - const auto constructor_ranges = - AdjustRanges(reference_profile["constructor_run_ranges"], - *reference_count, candidate_init->size(), error); - const auto native_ranges = - AdjustRanges(reference_profile["native_mimalloc_constructor_run_ranges"], - *reference_count, candidate_init->size(), error); + const auto constructor_ranges = RemapConstructorRanges( + reference_profile["constructor_run_ranges"], *leading, *reference_count, + candidate_init->size(), error); + const auto native_ranges = RemapConstructorRanges( + reference_profile["native_mimalloc_constructor_run_ranges"], *leading, + *reference_count, candidate_init->size(), error); if (!constructor_ranges.has_value() || !native_ranges.has_value()) { return std::nullopt; } - const std::size_t thread_boundary = reference_profile.value( + const std::size_t reference_thread_boundary = reference_profile.value( "native_mimalloc_thread_initializer_after_constructor", SIZE_MAX); - if (thread_boundary != 2) { + if (reference_thread_boundary < kLeadingConstructorBegin || + reference_thread_boundary >= + kLeadingConstructorBegin + kLeadingConstructorCount) { *error = "native allocator TLS boundary is no longer verified"; return std::nullopt; } + // The thread initializer follows the process-load constructor to its + // candidate index. + const std::size_t thread_boundary = + kLeadingConstructorBegin + + leading->candidate_offset[reference_thread_boundary - + kLeadingConstructorBegin]; const auto singleton_bytes = PositiveSize(seeds["jni_singleton_bytes"], "JNI singleton size", error); const auto arena_slots = PositiveSize(seeds["arena_table_slot_count"], diff --git a/tests/derive_roblox_host_abi_profile_test.py b/tests/derive_roblox_host_abi_profile_test.py index 64d99bd..7765b9d 100644 --- a/tests/derive_roblox_host_abi_profile_test.py +++ b/tests/derive_roblox_host_abi_profile_test.py @@ -356,6 +356,61 @@ class SignatureSemanticsTest(unittest.TestCase): ANALYZER.validate_semantic_match(reference, candidate, spec) + REGISTRY_BODY = ( + "bf38030000 e800000000 4889c3 4531f6 ba38030000 " + "4889c7 31f6 e800000000 b80000803f 894320 488d4b28" + ) + + def renamed_registry_body_spec(self, renamed: tuple[int, ...]): + return ANALYZER.SignatureSpec( + "registry-body", 0x1000, 11, True, 6, (0, 4), renamed + ) + + def test_registry_body_registers_may_be_renamed(self) -> None: + reference = decode_x86(bytes.fromhex(self.REGISTRY_BODY)) + # r15d for r14d, r13d for eax, r12 for rcx, and a smaller object. + candidate = decode_x86( + bytes.fromhex( + "bff8020000 e800000000 4889c3 4531ff baf8020000 " + "4889c7 31f6 e800000000 41bd0000803f 44896b20 4c8d6328" + ), + 0x2000, + ) + + ANALYZER.validate_semantic_match( + reference, candidate, self.renamed_registry_body_spec((3, 8, 9, 10)) + ) + with self.assertRaisesRegex(ANALYZER.AnalyzerError, "semantics changed"): + ANALYZER.validate_semantic_match( + reference, candidate, self.renamed_registry_body_spec(()) + ) + + def test_renamed_registers_must_stay_consistent(self) -> None: + reference = decode_x86(bytes.fromhex(self.REGISTRY_BODY)) + # The constant goes to r13d but r14d is stored. + two_for_one = decode_x86( + bytes.fromhex( + "bff8020000 e800000000 4889c3 4531ff baf8020000 " + "4889c7 31f6 e800000000 41bd0000803f 44897320 4c8d6328" + ), + 0x2000, + ) + # r13 stands for both r14 and rax. + one_for_two = decode_x86( + bytes.fromhex( + "bff8020000 e800000000 4889c3 4531ed baf8020000 " + "4889c7 31f6 e800000000 41bd0000803f 44896b20 4c8d6328" + ), + 0x2000, + ) + spec = self.renamed_registry_body_spec((3, 8, 9, 10)) + + for candidate in (two_for_one, one_for_two): + with self.assertRaisesRegex( + ANALYZER.AnalyzerError, "semantics changed" + ): + ANALYZER.validate_semantic_match(reference, candidate, spec) + def test_short_anchor_is_opt_in(self) -> None: encoded = b"abc\0def" mask = b"\x01\x01\x01\0\x01\x01\x01" @@ -364,6 +419,116 @@ class SignatureSemanticsTest(unittest.TestCase): self.assertEqual(ANALYZER.longest_fixed_anchor(encoded, mask, 3), (0, b"abc")) +class ConstructorRangeRemapTest(unittest.TestCase): + HOST_BRIDGE = [ + {"begin": 2, "end_exclusive": 3}, + {"begin": 5, "end_exclusive": 100}, + ] + NATIVE = [{"begin": 2, "end_exclusive": 100}] + # The 2.742 order: reference constructors 2, 3, 4, 5 are candidate + # constructors 5, 2, 4, 3. + REORDERED = (3, 0, 2, 1) + + def test_same_order_only_moves_the_end(self) -> None: + self.assertEqual( + ANALYZER.remapped_constructor_ranges( + self.HOST_BRIDGE, (0, 1, 2, 3), 100, 120 + ), + [ + {"begin": 2, "end_exclusive": 3}, + {"begin": 5, "end_exclusive": 120}, + ], + ) + + def test_leading_constructors_follow_their_new_index(self) -> None: + self.assertEqual( + ANALYZER.remapped_constructor_ranges( + self.HOST_BRIDGE, self.REORDERED, 100, 120 + ), + [ + {"begin": 3, "end_exclusive": 4}, + {"begin": 5, "end_exclusive": 120}, + ], + ) + self.assertEqual( + ANALYZER.remapped_constructor_ranges( + self.NATIVE, self.REORDERED, 100, 120 + ), + [{"begin": 2, "end_exclusive": 120}], + ) + + def test_rejects_ranges_outside_the_verified_shape(self) -> None: + for ranges in ( + # Runs a constructor before the leading four. + [{"begin": 1, "end_exclusive": 100}], + # Stops before the end of the init-array. + [{"begin": 2, "end_exclusive": 99}], + # Skips a constructor after the leading four. + [{"begin": 2, "end_exclusive": 6}, {"begin": 7, "end_exclusive": 100}], + ): + with self.assertRaisesRegex(ANALYZER.AnalyzerError, "cannot fit"): + ANALYZER.remapped_constructor_ranges( + ranges, self.REORDERED, 100, 120 + ) + + +@unittest.skipIf(ANALYZER.capstone is None, "Python capstone is unavailable") +class JniSingletonPublicationTest(unittest.TestCase): + class Image: + def __init__(self, encoded: bytes): + self.data = encoded + self.file_size = len(encoded) + + def require_code_rva(self, rva: int, size: int = 1) -> None: + if rva != 0x1000 or size > len(self.data): + raise ANALYZER.AnalyzerError("unexpected code RVA") + + def rva_to_offset(self, rva: int, size: int = 1) -> int: + return rva - 0x1000 + + def bytes_at(self, offset: int, size: int, _description: str) -> bytes: + return self.data[offset : offset + size] + + @staticmethod + def publication(rva: int, slot: int, stored: int) -> bytes: + def rip(opcode: str, instruction_rva: int, target: int) -> bytes: + return bytes.fromhex(opcode) + struct.pack( + " None: + bare = self.Image(self.publication(0x1000, 0x2000, 0x2008) + b"\xc3") + framed = self.Image( + bytes.fromhex("554889e5") + + self.publication(0x1004, 0x2000, 0x2008) + + b"\x5d\xc3" + ) + + self.assertEqual( + ANALYZER.jni_singleton_publication(bare, 0x1000)[0].address, 0x1000 + ) + publication = ANALYZER.jni_singleton_publication(framed, 0x1000) + self.assertEqual(len(publication), 3) + self.assertEqual(publication[0].address, 0x1004) + self.assertEqual(ANALYZER.rip_targets(publication[1]), (0x2000,)) + + def test_rejects_code_that_publishes_nothing(self) -> None: + other_object = self.Image( + self.publication(0x1000, 0x2000, 0x2010) + b"\xc3" + ) + unrelated = self.Image(bytes.fromhex("554889e5 4157 4156 4155 c3")) + + for image in (other_object, unrelated): + with self.assertRaisesRegex(ANALYZER.AnalyzerError, "does not publish"): + ANALYZER.jni_singleton_publication(image, 0x1000) + + @unittest.skipIf(ANALYZER.capstone is None, "Python capstone is unavailable") class RegistrySlotDerivationTest(unittest.TestCase): class Image: diff --git a/tests/host_abi_profile_test.cc b/tests/host_abi_profile_test.cc index e132de6..06e01c5 100644 --- a/tests/host_abi_profile_test.cc +++ b/tests/host_abi_profile_test.cc @@ -154,6 +154,84 @@ TEST(HostAbiProfileTest, ResearchedBaselineKeepsHostBridgeDefault) { HostAllocatorStrategy::kNativeMimalloc); } +// A profile of 100 constructors whose native replay starts at index 2. +HostAbiProfile ConstructorPolicyProfile( + size_t checkpoint, std::initializer_list host_ranges) { + HostAbiProfile profile; + profile.init_array_count = 100; + for (const ConstructorRange& range : host_ranges) { + profile.constructor_run_ranges[profile.constructor_run_range_count++] = + range; + } + profile.native_mimalloc_constructor_run_ranges[0] = {2, 100}; + profile.native_mimalloc_constructor_run_range_count = 1; + profile.native_mimalloc_thread_initializer_after_constructor = checkpoint; + return profile; +} + +TEST(HostAbiProfileTest, ConstructorPolicyAcceptsProcessLoadConstructorFirst) { + EXPECT_TRUE(ConstructorPolicyProfile(2, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(5)); +} + +TEST(HostAbiProfileTest, ConstructorPolicyAcceptsProcessLoadConstructorLast) { + // The 2.742 order: wiring, JNI singleton, wiring, process-load constructor. + EXPECT_TRUE(ConstructorPolicyProfile(5, {{3, 4}, {5, 100}}) + .HasVerifiedConstructorPolicy(3)); +} + +TEST(HostAbiProfileTest, ConstructorPolicyRequiresJniSingletonConstructor) { + // Host bridges run a wiring constructor and skip the JNI singleton. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 4}, {6, 100}}) + .HasVerifiedConstructorPolicy(5)); + EXPECT_FALSE(ConstructorPolicyProfile(5, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(3)); + // The JNI singleton constructor is the checkpoint constructor. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(2)); + // The JNI singleton constructor is outside the four leading constructors. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(6)); +} + +TEST(HostAbiProfileTest, ConstructorPolicyRejectsOtherShapes) { + // The checkpoint is outside the four leading constructors. + EXPECT_FALSE(ConstructorPolicyProfile(6, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(5)); + // The checkpoint constructor does not run under host bridges. + EXPECT_FALSE(ConstructorPolicyProfile(3, {{2, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(5)); + // Only one wiring constructor is excluded. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 4}, {5, 100}}) + .HasVerifiedConstructorPolicy(5)); + // A constructor after the leading four is excluded. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 3}, {5, 6}, {7, 100}}) + .HasVerifiedConstructorPolicy(5)); + // Host bridges run a constructor the native replay skips. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{1, 3}, {5, 100}}) + .HasVerifiedConstructorPolicy(5)); + // The replay stops before the end of the init-array. + EXPECT_FALSE(ConstructorPolicyProfile(2, {{2, 3}, {5, 99}}) + .HasVerifiedConstructorPolicy(5)); + + HostAbiProfile split_native = ConstructorPolicyProfile(2, {{2, 3}, {5, 100}}); + split_native.native_mimalloc_constructor_run_ranges[0] = {2, 50}; + split_native.native_mimalloc_constructor_run_ranges[1] = {50, 100}; + split_native.native_mimalloc_constructor_run_range_count = 2; + EXPECT_FALSE(split_native.HasVerifiedConstructorPolicy(5)); + + // The init-array ends inside the four leading constructors. + HostAbiProfile too_short = ConstructorPolicyProfile(2, {{2, 3}}); + too_short.init_array_count = 5; + too_short.native_mimalloc_constructor_run_ranges[0] = {2, 5}; + EXPECT_FALSE(too_short.HasVerifiedConstructorPolicy(4)); + + HostAbiProfile shortest = ConstructorPolicyProfile(2, {{2, 3}, {5, 6}}); + shortest.init_array_count = 6; + shortest.native_mimalloc_constructor_run_ranges[0] = {2, 6}; + EXPECT_TRUE(shortest.HasVerifiedConstructorPolicy(5)); +} + TEST(HostAbiProfileTest, UnknownBuildHasNoNativeOffsets) { EXPECT_EQ(FindHostAbiProfile("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), nullptr); @@ -428,7 +506,7 @@ class ExternalHostAbiProfileLoaderTest : public ::testing::Test { {"empty_string_initializer_rva", Hex(SymbolRva("MocktailFixtureThreadInitialize"))}, {"jni_singleton_initializer_rva", - Hex(SymbolRva("MocktailFixtureRegistryInitialize"))}, + Hex(SymbolRva("MocktailFixtureConstructorFive"))}, {"constructor_rvas", {{"2", Hex(SymbolRva("MocktailFixtureConstructorTwo"))}, {"3", Hex(SymbolRva("MocktailFixtureConstructorThree"))},