# ninja file and solution generator for csplit import io import json import math import os import platform import sys from enum import Enum from pathlib import Path from typing import ( IO, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypedDict, Union, cast, ) from . import ninja_syntax from .ninja_syntax import serialize_path from .vsgen.configuration import BuildParams from .vsgen.configuration import Configuration from .vsgen.sln import SolutionFile from .vsgen.vcxproj import VcxProj class ObjectStatus(Enum): Missing = 0 NonMatching = 1 Matching = 2 class Object: def __init__(self, completed: ObjectStatus, file_path: Path, **options: Any) -> None: self.file_path = file_path self.status = completed self.options: Dict[str, Any] = { "cflags": None, "include_dirs": None, "defines": None, } self.options.update(options) # Internal self.base_obj_path: Optional[Path] = None self.split_obj_path: Optional[Path] = None def resolve(self, sln: "SolutionConfig", proj: "ProjectConfig") -> "Object": # Use object options, then library options obj = Object(self.status, self.file_path, **proj.options) for key, value in self.options.items(): if value is not None or key not in obj.options: obj.options[key] = value # Resolve paths base_name = self.file_path.with_suffix("") obj.base_obj_path = sln.build_dir / "base" / f"{base_name}.obj" obj.split_obj_path = sln.build_dir / "split" / f"{base_name}.obj" return obj class ProjectConfig: def __init__(self, **options: Any) -> None: self.name: Optional[str] = None self.guid: Optional[str] = None self.objects: Optional[List[Object]] = None self.options: Dict[str, Any] = { "cflags": None, "include_dirs": None, "headers": None, "defines": None, } self.options.update(options) def validate(self) -> None: required_attrs = [ "name", "guid", "objects", ] for attr in required_attrs: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") # Creates a map of object names to Object instances def resolve(self, sln: "SolutionConfig") -> Dict[str, Object]: out = {} for obj in self.objects: name = str(obj.file_path.with_suffix("")).replace(os.sep, "/") if name in out: sys.exit(f"Duplicate object name {name}") out[name] = obj.resolve(sln, self) return out class SolutionConfig: def __init__(self) -> None: self.build_dir: Optional[Path] = None self.config_dir: Optional[Path] = None self.project_dir: Optional[Path] = None self.tools_dir: Optional[Path] = None self.name: Optional[str] = None self.guid: Optional[str] = None self.projects: Optional[List[ProjectConfig]] = None # Tooling self.ninja_path: Optional[Path] = None # If None, use system PATH self.csplit_tag: Optional[str] = None # Git tag self.csplit_path: Optional[Path] = None # If None, download self.objdiff_tag: Optional[str] = None # Git tag self.objdiff_path: Optional[Path] = None # If None, download self.wibo_tag: Optional[str] = None # Git tag self.wrapper: Optional[Path] = None # If None, download wibo on Linux # Project config self.baserom: Optional[Path] = None # Progress output and report.json config self.progress = True # Enable report.json generation and CLI progress output self.print_progress_categories: bool = True self.progress_report_args: Optional[List[str]] = ( None # Flags to `objdiff-cli report generate` ) # Progress fancy printing self.progress_use_fancy: bool = False self.progress_code_fancy_frac: int = 0 self.progress_code_fancy_item: str = "" self.progress_data_fancy_frac: int = 0 self.progress_data_fancy_item: str = "0" def validate(self) -> None: required_attrs = [ "build_dir", "config_dir", "project_dir", "tools_dir", "name", "guid", "projects", "baserom", ] for attr in required_attrs: if getattr(self, attr) is None: sys.exit(f"SolutionConfig.{attr} missing") for proj in self.projects: proj.validate() # Gets the wrapper to use for compiler commands, if set. def compiler_wrapper(self) -> Optional[Path]: wrapper = self.wrapper if self.use_wibo(): wrapper = self.build_dir / "tools" / "wibo" if not is_windows() and wrapper is None: wrapper = Path("wine") return wrapper # Determines whether or not to use wibo as the compiler wrapper. def use_wibo(self) -> bool: return ( self.wibo_tag is not None and (sys.platform == "linux" or sys.platform == "darwin") and platform.machine() in ("i386", "x86_64", "aarch64", "arm64") and self.wrapper is None ) def is_windows() -> bool: return os.name == "nt" # On Windows, we need this to use && in commands CHAIN = "cmd /c " if is_windows() else "" # Native executable extension EXE = ".exe" if is_windows() else "" # Generate all build files def generate_build(sln: SolutionConfig) -> None: sln.validate() generate_build_ninja(sln) generate_objdiff_config(sln) generate_compile_commands(sln) if is_windows(): generate_solution(sln) # Generate build.ninja def generate_build_ninja(sln: SolutionConfig) -> None: out = io.StringIO() n = ninja_syntax.Writer(out) n.variable("ninja_required_version", "1.3") n.newline() configure_script = Path(os.path.relpath(os.path.abspath(sys.argv[0]))) python_lib = Path(os.path.relpath(__file__)) python_lib_dir = python_lib.parent n.comment("The arguments passed to configure.py, for rerunning it.") n.variable("configure_args", sys.argv[1:]) n.variable("python", f'"{sys.executable}"') n.newline() ### # Tooling ### n.comment("Tooling") report_path = sln.build_dir / "report.json" build_tools_path = sln.build_dir / "tools" download_tool = sln.tools_dir / "download_tool.py" n.rule( name="download_tool", command=f"$python {download_tool} $tool $out --tag $tag", description="TOOL $out", ) if sln.objdiff_path: objdiff = sln.objdiff_path elif sln.objdiff_tag: objdiff = build_tools_path / f"objdiff-cli{EXE}" n.build( outputs=objdiff, rule="download_tool", implicit=download_tool, variables={ "tool": "objdiff-cli", "tag": sln.objdiff_tag, }, ) else: sys.exit("SolutionConfig.objdiff_tag missing") if sln.csplit_path: csplit = sln.csplit_path elif sln.csplit_tag: csplit = build_tools_path / f"csplit{EXE}" n.build( outputs=csplit, rule="download_tool", implicit=download_tool, variables={ "tool": "csplit", "tag": sln.csplit_tag, }, ) else: sys.exit("SolutionConfig.csplit_tag missing") wrapper = sln.compiler_wrapper() if wrapper is not None and sln.use_wibo(): n.build( outputs=wrapper, rule="download_tool", implicit=download_tool, variables={ "tool": "wibo", "tag": sln.wibo_tag, }, ) wrapper_implicit: Optional[Path] = None if wrapper is not None and (wrapper.exists() or sln.use_wibo()): wrapper_implicit = wrapper wrapper_cmd = f"{wrapper} " if wrapper else "" n.newline() ### # Helper rule for downloading all tools ### n.comment("Download all tools") n.build( outputs="tools", rule="phony", inputs=[objdiff, csplit], ) n.newline() ### # Build rules ### n.rule( name="cl", command=f"{wrapper_cmd}xbox/bin/vc7/CL.Exe /nologo /c $cflags /Fo$out $in", description="CL $out", ) n.newline() ### # Source files / projects ### n.comment("Projects") n.newline() base_object_targets: List[Path] = [] split_object_targets: List[Path] = [] for proj in sln.projects: objects: Dict[str, Object] = proj.resolve(sln) n.comment(proj.name) proj_base_object_targets: List[Path] = [] for obj_name, obj in objects.items(): split_object_targets.append(obj.split_obj_path) if obj.status != ObjectStatus.Missing: proj_base_object_targets.append(obj.base_obj_path) cflags: List[str] = [] cflags.extend(obj.options["cflags"]) cflags.extend([f"/D{define}" for define in obj.options["defines"]]) cflags.extend([f"/I\"{path}\"" for path in obj.options["include_dirs"]]) n.build( outputs=obj.base_obj_path, rule="cl", variables={"cflags": cflags}, inputs=obj.file_path, implicit=wrapper_implicit ) base_object_targets.extend(proj_base_object_targets) n.build( outputs=f"{proj.name}_build", rule="phony", inputs=proj_base_object_targets, ) n.newline() ### # Helper rule for building all source files ### n.comment("Build all source files") n.build( outputs="all_source", rule="phony", inputs=base_object_targets, ) n.newline() ### # Build all split objects ### n.comment("Build all split files") n.build( outputs="all_split", rule="phony", inputs=split_object_targets, ) n.newline() ### # Split EXE ### n.rule( name="csplit", command=f"{csplit} -i $in -p $config_dir -o $out_dir", ) n.build( outputs=split_object_targets, rule="csplit", variables={ "out_dir": sln.build_dir / "split", "config_dir": sln.config_dir }, inputs=sln.baserom, implicit=[ csplit, sln.config_dir / "config.json", sln.config_dir / "contribs.json", sln.config_dir / "relocs.json", sln.config_dir / "splits.json", sln.config_dir / "symbols.json", ], ) n.newline() ### # Calculate progress ### n.comment("Calculate progress") n.rule( name="progress", command=f"$python {configure_script} $configure_args progress", description="PROGRESS", ) n.build( outputs="progress", rule="progress", implicit=[ configure_script, python_lib, report_path, ], ) n.newline() ### # Generate progress report ### n.comment("Generate progress report") n.rule( name="report", command=f"{objdiff} report generate $objdiff_report_args -o $out", description="REPORT", ) n.build( outputs=report_path, rule="report", implicit=[objdiff, "objdiff.json", "all_source", "all_split"], ) n.newline() ### # Regenerate on change ### n.comment("Reconfigure on change") n.rule( name="configure", command=f"$python {configure_script} $configure_args", generator=True, description=f"RUN {configure_script}", ) n.build( outputs=["build.ninja", "objdiff.json"], rule="configure", implicit=[ sln.config_dir / "config.json", configure_script, python_lib, python_lib_dir / "ninja_syntax.py", ], ) n.newline() ### # Default rule ### n.comment("Default rule") if sln.progress: n.default("progress") else: n.default("all_source") # Write build.ninja with open("build.ninja", "w", encoding="utf-8") as f: f.write(out.getvalue()) out.close() # Generate objdiff.json def generate_objdiff_config(sln: SolutionConfig) -> None: if sln.ninja_path: ninja = str(sln.ninja_path.absolute()) else: ninja = "ninja" objdiff_config: Dict[str, Any] = { "min_version": "2.0.0-beta.5", "custom_make": "ninja", "build_target": True, "watch_patterns": [ "*.c", "*.cc", "*.cp", "*.cpp", "*.cxx", "*.c++", "*.h", "*.hh", "*.hp", "*.hpp", "*.hxx", "*.h++", "*.pch", "*.pch++", "*.inc", "*.py", "*.yml", "*.txt", "*.json", ], "units": [], "progress_categories": [], } for proj in sln.projects: objdiff_config["progress_categories"].append( { "id": proj.name, "name": proj.name, } ) for obj_name, obj in proj.resolve(sln).items(): unit = { "name": obj_name, "target_path": obj.split_obj_path, "metadata": { "progress_categories": [ proj.name ], } } if obj.status != ObjectStatus.Missing: unit["base_path"] = obj.base_obj_path unit["metadata"]["source_path"] = obj.file_path if obj.status == ObjectStatus.Matching: unit["metadata"]["complete"] = True objdiff_config["units"].append(unit) def cleandict(d): if isinstance(d, dict): return {k: cleandict(v) for k, v in d.items() if v is not None} elif isinstance(d, list): return [cleandict(v) for v in d] else: return d # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: def unix_path(input: Any) -> str: return str(input).replace(os.sep, "/") if input else "" json.dump(cleandict(objdiff_config), w, indent=2, default=unix_path) # Generate compile_commands.json def generate_compile_commands(sln: SolutionConfig) -> None: pass # Generate solution and projects def generate_solution(sln: SolutionConfig) -> None: relative_root = Path("..") vc_config = Configuration("Debug", "Win32", False, "", relative_root) vc_projects: List[VcxProj] = [] for proj in sln.projects: vc_proj = VcxProj(proj.name, sln.project_dir) vc_proj.uuid = proj.guid vc_proj.add_config(vc_config) vc_macros: List[str] = proj.options["defines"] vc_include_dirs: List[Path] = [relative_root / path for path in proj.options["include_dirs"] or []] vc_params = BuildParams(macros=vc_macros, include_directories=vc_include_dirs, output=f"{proj.name}_build") vc_proj.add_build_params(str(vc_config), vc_params) vc_sources: List[Path] = [] for obj in proj.resolve(sln).values(): if obj.status != ObjectStatus.Missing: vc_sources.append(relative_root / obj.file_path) vc_proj.add_sources(vc_sources) vc_headers: List[Path] = [relative_root / header for header in proj.options["headers"] or []] vc_proj.add_headers(vc_headers) vc_extras: List[Path] = [ relative_root / sln.config_dir / "config.json", relative_root / sln.config_dir / "symbols.json", ] vc_proj.add_extra_files(vc_extras) vc_proj.write() vc_projects.append(vc_proj) vc_sln = SolutionFile(sln.project_dir / sln.name) vc_sln.uuid = sln.guid for vc_proj in vc_projects: vc_sln.add_project(vc_proj, build_solution_target=True) vc_sln.write() # Print progress information from objdiff report def calculate_progress(sln: SolutionConfig) -> None: sln.validate() report_path = sln.build_dir / "report.json" if not report_path.is_file(): sys.exit(f"Report file {report_path} does not exist") report_data: Dict[str, Any] = {} with open(report_path, "r", encoding="utf-8") as f: report_data = json.load(f) # Convert string numbers (u64) to int def convert_numbers(data: Dict[str, Any]) -> None: for key, value in data.items(): if isinstance(value, str) and value.isdigit(): data[key] = int(value) convert_numbers(report_data["measures"]) for category in report_data.get("categories", []): convert_numbers(category["measures"]) # Output to GitHub Actions job summary, if available summary_path = os.getenv("GITHUB_STEP_SUMMARY") summary_file: Optional[IO[str]] = None if summary_path: summary_file = open(summary_path, "a", encoding="utf-8") summary_file.write("```\n") def progress_print(s: str) -> None: print(s) if summary_file: summary_file.write(s + "\n") # Print human-readable progress progress_print("Progress:") def print_category(name: str, measures: Dict[str, Any]) -> None: total_code = measures.get("total_code", 0) matched_code = measures.get("matched_code", 0) matched_code_percent = measures.get("matched_code_percent", 0) total_data = measures.get("total_data", 0) matched_data = measures.get("matched_data", 0) matched_data_percent = measures.get("matched_data_percent", 0) total_functions = measures.get("total_functions", 0) matched_functions = measures.get("matched_functions", 0) complete_code_percent = measures.get("complete_code_percent", 0) total_units = measures.get("total_units", 0) complete_units = measures.get("complete_units", 0) progress_print( f" {name}: {matched_code_percent:.2f}% matched, {complete_code_percent:.2f}% linked ({complete_units} / {total_units} files)" ) progress_print( f" Code: {matched_code} / {total_code} bytes ({matched_functions} / {total_functions} functions)" ) progress_print( f" Data: {matched_data} / {total_data} bytes ({matched_data_percent:.2f}%)" ) print_category("All", report_data["measures"]) for category in report_data.get("categories", []): if sln.print_progress_categories is True or ( isinstance(sln.print_progress_categories, list) and category["id"] in sln.print_progress_categories ): print_category(category["name"], category["measures"]) if sln.progress_use_fancy: measures = report_data["measures"] total_code = measures.get("total_code", 0) total_data = measures.get("total_data", 0) if total_code == 0 or total_data == 0: return code_frac = measures.get("complete_code", 0) / total_code data_frac = measures.get("complete_data", 0) / total_data progress_print( "\nYou have {} out of {} {} and {} out of {} {}.".format( math.floor(code_frac * sln.progress_code_fancy_frac), sln.progress_code_fancy_frac, sln.progress_code_fancy_item, math.floor(data_frac * sln.progress_data_fancy_frac), sln.progress_data_fancy_frac, sln.progress_data_fancy_item, ) ) # Finalize GitHub Actions job summary if summary_file: summary_file.write("```\n") summary_file.close()