diff --git a/.gitignore b/.gitignore index 666e7db..cb8864e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ dist/ src-tauri/target/ src-tauri/gen/ public/mock/ +src-tauri/binaries/ +target-engine/ diff --git a/engine/build.py b/engine/build.py new file mode 100644 index 0000000..1d77c2e --- /dev/null +++ b/engine/build.py @@ -0,0 +1,52 @@ +"""Freeze the engine into src-tauri/binaries/prospack-engine-[.exe]. + +Tauri's sidecar convention needs the Rust target triple in the file name. +Run with the Python that has engine/requirements-build.txt installed. +""" + +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +ENGINE = ROOT / "engine" +OUT = ROOT / "src-tauri" / "binaries" + + +def host_triple() -> str: + out = subprocess.run(["rustc", "-vV"], check=True, capture_output=True, text=True).stdout + return next(l.split(": ", 1)[1] for l in out.splitlines() if l.startswith("host: ")) + + +def main() -> int: + work = ROOT / "target-engine" + subprocess.run( + [ + sys.executable, "-m", "PyInstaller", "--noconfirm", "--clean", "--onefile", + "--name", "prospack-engine", + "--collect-all", "mkpfs", + "--collect-all", "zlib_ng", + "--hidden-import", "deep_verify", + "--paths", str(ENGINE), + "--distpath", str(work / "dist"), + "--workpath", str(work / "build"), + "--specpath", str(work), + str(ENGINE / "prospack_engine.py"), + ], + check=True, + ) + exe = ".exe" if platform.system() == "Windows" else "" + OUT.mkdir(parents=True, exist_ok=True) + dest = OUT / f"prospack-engine-{host_triple()}{exe}" + shutil.copy2(work / "dist" / f"prospack-engine{exe}", dest) + if not exe: + os.chmod(dest, 0o755) + print(dest) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src-tauri/resources/deep_verify.py b/engine/deep_verify.py similarity index 100% rename from src-tauri/resources/deep_verify.py rename to engine/deep_verify.py diff --git a/engine/make_fake_dump.py b/engine/make_fake_dump.py new file mode 100644 index 0000000..ef7f04a --- /dev/null +++ b/engine/make_fake_dump.py @@ -0,0 +1,55 @@ +"""Write a small, reproducible fake PS5 dump for tests and CI (no real game data). + + python engine/make_fake_dump.py OUT_DIR/PPSA00001-app +""" + +import json +import os +import random +import struct +import sys +import zlib +from pathlib import Path + + +def png(path: Path, w: int, h: int, rgb: tuple[int, int, int]) -> None: + def chunk(tag: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + rows = b"".join(b"\0" + bytes(rgb) * w for _ in range(h)) + path.write_bytes( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(rows, 9)) + + chunk(b"IEND", b"") + ) + + +def main() -> int: + root = Path(sys.argv[1]) + title = root.name.split("-")[0] if root.name[:4].isalpha() else "PPSA00001" + rnd = random.Random(1234) + (root / "sce_sys").mkdir(parents=True, exist_ok=True) + (root / "fakelib").mkdir(exist_ok=True) + (root / "data/prein/bg").mkdir(parents=True, exist_ok=True) + (root / "sce_sys/param.json").write_text(json.dumps({ + "titleId": title, + "contentVersion": "01.000.000", + "localizedParameters": {"defaultLanguage": "en-US", "en-US": {"titleName": "TEST GAME"}}, + })) + png(root / "sce_sys/icon0.png", 512, 512, (40, 90, 220)) + png(root / "sce_sys/pic0.png", 1280, 720, (30, 30, 60)) + (root / "eboot.bin").write_bytes(rnd.randbytes(2 * 1024 * 1024)) + (root / "fakelib/libSceAmpr.sprx").write_bytes(rnd.randbytes(64 * 1024)) + for i in range(300): + (root / f"data/prein/bg/small_{i}.txt").write_bytes(b"a" * rnd.randrange(1, 90000)) + with open(root / "data/prein/bg/big_text.bin", "wb") as f: + line = b"level data compressible\n" + f.write(line * (40 * 1024 * 1024 // len(line))) + (root / "data/prein/bg/big_random.bin").write_bytes(rnd.randbytes(8 * 1024 * 1024)) + print(root) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/engine/prospack_engine.py b/engine/prospack_engine.py new file mode 100644 index 0000000..5b62499 --- /dev/null +++ b/engine/prospack_engine.py @@ -0,0 +1,39 @@ +"""Prospack engine: the compression and verification half of Prospack. + +Frozen into a single executable (see build.py) and shipped next to the app, so users +don't need Python. + + prospack-engine pack folder [mkpfs options] SOURCE IMAGE + prospack-engine verify IMAGE SOURCE [WORKERS] + prospack-engine --version +""" + +import multiprocessing +import sys + + +def main() -> int: + # Must run first: MkPFS and the verifier start worker processes, which re-enter this + # executable when frozen. + multiprocessing.freeze_support() + if len(sys.argv) < 2 or sys.argv[1] not in ("pack", "verify", "--version"): + print(__doc__.strip(), file=sys.stderr) + return 2 + cmd = sys.argv[1] + if cmd == "--version": + import importlib.metadata as md + + print(f"mkpfs {md.version('mkpfs')}") + return 0 + if cmd == "pack": + from mkpfs.cli import cli_mkpfs_main + + return cli_mkpfs_main(sys.argv[1:]) + import deep_verify + + sys.argv = [sys.argv[0], *sys.argv[2:]] + return deep_verify.main() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/engine/requirements-build.txt b/engine/requirements-build.txt new file mode 100644 index 0000000..dd72fe5 --- /dev/null +++ b/engine/requirements-build.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pyinstaller>=6.10 diff --git a/engine/requirements.txt b/engine/requirements.txt new file mode 100644 index 0000000..fd1bc49 --- /dev/null +++ b/engine/requirements.txt @@ -0,0 +1,2 @@ +mkpfs==1.1.0 +zlib-ng>=1.0 diff --git a/src-tauri/src/engine.rs b/src-tauri/src/engine.rs index fc4bb2d..49bd3ac 100644 --- a/src-tauri/src/engine.rs +++ b/src-tauri/src/engine.rs @@ -1,4 +1,8 @@ -//! The compression engine: a private Python venv with MkPFS, plus the deep verifier. +//! The compression engine: MkPFS plus the deep verifier, as one `prospack-engine` program. +//! +//! Release builds ship it frozen as a Tauri sidecar next to the app. Developers can point +//! `PROSPACK_ENGINE` at a frozen build, or let the app run `engine/prospack_engine.py` in a +//! private venv ("Install MkPFS" in the UI). use serde::Serialize; use std::io::Read; @@ -9,7 +13,9 @@ use std::sync::Arc; /// Pinned so a new MkPFS release can't silently change the image format. pub const MKPFS_VERSION: &str = "1.1.0"; -const VERIFIER: &str = include_str!("../resources/deep_verify.py"); +const EXE: &str = if cfg!(windows) { ".exe" } else { "" }; +/// The repo's engine sources, for the developer fallback. +const DEV_ENGINE_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../engine"); #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] @@ -17,9 +23,19 @@ pub struct EngineStatus { pub python: Option, pub mkpfs: Option, pub ready: bool, + /// True when the frozen engine shipped with the app is in use. + pub bundled: bool, + /// Whether "Install MkPFS" (developer venv) is possible on this machine. + pub can_install: bool, pub dir: String, } +#[derive(Debug, Clone, PartialEq)] +enum Runner { + Frozen(PathBuf), + Script { python: PathBuf, script: PathBuf }, +} + pub struct Engine { root: PathBuf, } @@ -32,65 +48,97 @@ impl Engine { fn venv(&self) -> PathBuf { self.root.join("venv") } - fn mkpfs_bin(&self) -> PathBuf { - self.venv().join("bin/mkpfs") - } fn python_bin(&self) -> PathBuf { - self.venv().join("bin/python") + if cfg!(windows) { + self.venv().join("Scripts/python.exe") + } else { + self.venv().join("bin/python") + } } - fn verifier(&self) -> PathBuf { - self.root.join("deep_verify.py") + fn dev_script() -> Option { + std::env::var_os("PROSPACK_ENGINE_SCRIPT") + .map(PathBuf::from) + .or_else(|| Some(PathBuf::from(DEV_ENGINE_DIR).join("prospack_engine.py"))) + .filter(|p| p.is_file()) + } + + /// Sidecar next to the app, then $PROSPACK_ENGINE, then the developer venv. + fn runner(&self) -> Option { + let sidecar = std::env::current_exe() + .ok() + .and_then(|e| e.parent().map(|d| d.join(format!("prospack-engine{EXE}")))) + .filter(|p| p.is_file()); + if let Some(p) = sidecar { + return Some(Runner::Frozen(p)); + } + if let Some(p) = std::env::var_os("PROSPACK_ENGINE").map(PathBuf::from).filter(|p| p.is_file()) { + return Some(Runner::Frozen(p)); + } + let python = self.python_bin(); + match (python.is_file(), Self::dev_script()) { + (true, Some(script)) => Some(Runner::Script { python, script }), + _ => None, + } + } + + fn command(&self) -> Result { + match self.runner() { + Some(Runner::Frozen(p)) => Ok(quiet(p)), + Some(Runner::Script { python, script }) => { + let mut c = quiet(python); + c.arg(script); + Ok(c) + } + None => Err("The compression engine is missing. Reinstall Prospack.".into()), + } } pub fn status(&self) -> EngineStatus { - let python = quiet("python3") - .arg("--version") - .output() + let runner = self.runner(); + let mkpfs = self + .command() .ok() + .and_then(|mut c| c.arg("--version").output().ok()) .filter(|o| o.status.success()) - .map(|o| String::from_utf8_lossy(&o.stdout).trim().replace("Python ", "")); - let mkpfs = if self.mkpfs_bin().is_file() { - quiet(self.python_bin()) - .args(["-c", "import importlib.metadata as m; print(m.version('mkpfs'))"]) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().trim_start_matches("mkpfs ").to_string()); + let can_install = Self::dev_script().is_some(); + let python = if matches!(runner, Some(Runner::Frozen(_))) || !can_install { + None + } else { + quiet(if cfg!(windows) { "python" } else { "python3" }) + .arg("--version") .output() .ok() .filter(|o| o.status.success()) - .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) - } else { - None + .map(|o| String::from_utf8_lossy(&o.stdout).trim().replace("Python ", "")) }; - let ready = mkpfs.is_some(); - EngineStatus { python, mkpfs, ready, dir: self.root.to_string_lossy().into_owned() } + EngineStatus { + python, + ready: mkpfs.is_some(), + mkpfs, + bundled: matches!(runner, Some(Runner::Frozen(_))), + can_install, + dir: self.root.to_string_lossy().into_owned(), + } } - /// Creates the venv and installs the pinned MkPFS. Streams pip's output to `log`. + /// Developer setup: a venv with the engine's requirements. Release builds never need this. pub fn install(&self, log: &mut dyn FnMut(String)) -> Result<(), String> { + let script = Self::dev_script() + .ok_or("The compression engine ships with Prospack. Reinstall Prospack to restore it.")?; + let reqs = script.with_file_name("requirements.txt"); std::fs::create_dir_all(&self.root).map_err(|e| e.to_string())?; if !self.python_bin().is_file() { log("Creating Python environment".into()); - run_logged(quiet("python3").arg("-m").arg("venv").arg(self.venv()), log)?; + run_logged(quiet(if cfg!(windows) { "python" } else { "python3" }).arg("-m").arg("venv").arg(self.venv()), log)?; } log(format!("Installing MkPFS {MKPFS_VERSION}")); run_logged( - quiet(self.python_bin()).args([ - "-m", - "pip", - "install", - "--disable-pip-version-check", - "--upgrade", - &format!("mkpfs=={MKPFS_VERSION}"), - ]), + quiet(self.python_bin()) + .args(["-m", "pip", "install", "--disable-pip-version-check", "-r"]) + .arg(reqs), log, )?; - self.write_verifier()?; - Ok(()) - } - - pub fn write_verifier(&self) -> Result<(), String> { - std::fs::create_dir_all(&self.root).map_err(|e| e.to_string())?; - if std::fs::read_to_string(self.verifier()).ok().as_deref() != Some(VERIFIER) { - std::fs::write(self.verifier(), VERIFIER).map_err(|e| e.to_string())?; - } Ok(()) } @@ -107,7 +155,7 @@ impl Engine { ) -> Result<(), String> { let tmp_dir = out.parent().unwrap().join("tmp"); std::fs::create_dir_all(&tmp_dir).map_err(|e| e.to_string())?; - let mut cmd = quiet(self.mkpfs_bin()); + let mut cmd = self.command()?; cmd.args(["pack", "folder", "--no-adjust-output-file-extension", "--ampr-skip-regen-if-exists"]) .arg("--compression-level") .arg(level.min(9).to_string()) @@ -141,14 +189,13 @@ impl Engine { cancel: &Arc, on: &mut dyn FnMut(VerifyEvent), ) -> Result { - self.write_verifier()?; let workers = if workers == 0 { (std::thread::available_parallelism().map(|n| n.get()).unwrap_or(2) / 2).clamp(1, 8) as u32 } else { workers }; - let mut cmd = quiet(self.python_bin()); - cmd.arg(self.verifier()).arg(image).arg(src).arg(workers.to_string()); + let mut cmd = self.command()?; + cmd.arg("verify").arg(image).arg(src).arg(workers.to_string()); let mut buf = String::new(); let mut verdict: Option = None; run_streaming(cmd, cancel, &mut |chunk| { @@ -464,6 +511,7 @@ mod cancel_test { /// PACKER_E2E_SCRATCH= cargo test cancel -- --ignored #[test] #[ignore] + #[cfg(unix)] fn cancel_kills_mkpfs_promptly() { let scratch = PathBuf::from(std::env::var("PACKER_E2E_SCRATCH").unwrap()); let tmp = tempfile::tempdir().unwrap(); @@ -483,6 +531,7 @@ mod cancel_test { let out_dir = tmp.path().join("out"); std::fs::create_dir_all(&out_dir).unwrap(); let engine = Engine { root }; + assert!(engine.runner().is_some(), "no engine: set PROSPACK_ENGINE or provide the venv"); let cancel = Arc::new(AtomicBool::new(false)); let c2 = cancel.clone(); let t = std::time::Instant::now(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c3cf6a6..865796e 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -120,7 +120,6 @@ pub fn run() { let _ = handle.emit(event, payload); }); let engine = Arc::new(engine::Engine::new(&settings::data_dir())); - let _ = engine.write_verifier(); let queue = pipeline::Queue::new(emitter, engine.clone()); // Folders passed on the command line (or via "Open with") are queued straight away. for arg in std::env::args().skip(1).filter(|a| !a.starts_with('-')) { diff --git a/src-tauri/src/pipeline.rs b/src-tauri/src/pipeline.rs index 2069a7a..9b8cee8 100644 --- a/src-tauri/src/pipeline.rs +++ b/src-tauri/src/pipeline.rs @@ -876,6 +876,18 @@ mod tests { mod e2e { use super::*; + fn copy_dir(from: &Path, to: &Path) { + std::fs::create_dir_all(to).unwrap(); + for e in std::fs::read_dir(from).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + copy_dir(&p, &to.join(e.file_name())); + } else { + std::fs::copy(&p, to.join(e.file_name())).unwrap(); + } + } + } + fn wait_until(q: &Queue, id: u64, secs: u64) -> Job { let t = Instant::now(); loop { @@ -895,17 +907,19 @@ mod e2e { let tmp = tempfile::tempdir().unwrap(); std::env::set_var("PROSPACK_CONFIG_DIR", tmp.path().join("config")); std::env::set_var("PROSPACK_DATA_DIR", tmp.path().join("data")); - let engine_dir = tmp.path().join("data/engine"); - std::fs::create_dir_all(&engine_dir).unwrap(); - std::os::unix::fs::symlink(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap(); + // Engine: $PROSPACK_ENGINE (frozen build) if set, else the scratch venv as the dev venv. + if std::env::var_os("PROSPACK_ENGINE").is_none() { + let engine_dir = tmp.path().join("data/engine"); + std::fs::create_dir_all(&engine_dir).unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap(); + #[cfg(windows)] + std::os::windows::fs::symlink_dir(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap(); + } // Fresh copy of the fake dump so MkPFS's ampr_emu.index write can't leak between runs. let dump = tmp.path().join("PPSA00001-app"); - assert!(std::process::Command::new("cp") - .args(["-r", scratch.join("fakedump/PPSA00001-app").to_str().unwrap(), dump.to_str().unwrap()]) - .status() - .unwrap() - .success()); + copy_dir(&scratch.join("fakedump/PPSA00001-app"), &dump); let _ = std::fs::remove_file(dump.join("ampr_emu.index")); let ps5_root = tmp.path().join("ps5"); @@ -917,7 +931,8 @@ mod e2e { let _ = self.0.wait(); } } - let _server = KillOnDrop(std::process::Command::new(scratch.join("mkvenv/bin/python")) + let py = if cfg!(windows) { scratch.join("mkvenv/Scripts/python.exe") } else { scratch.join("mkvenv/bin/python") }; + let _server = KillOnDrop(std::process::Command::new(py) .arg(scratch.join("fake_ps5.py")) .arg(&ps5_root) .arg(port.to_string()) diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 1a2ba51..8247be0 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -40,7 +40,9 @@ "icons/128x128@2x.png", "icons/icon.png" ], - "resources": [] + "externalBin": [ + "binaries/prospack-engine" + ] }, "mainBinaryName": "prospack" } diff --git a/src/lib/EmptyState.svelte b/src/lib/EmptyState.svelte index 55cb265..b4b82f5 100644 --- a/src/lib/EmptyState.svelte +++ b/src/lib/EmptyState.svelte @@ -11,21 +11,30 @@
{#if engine && !engine.ready}
-
One-time setup
-

Install the compression engine

-

- Prospack uses MkPFS 1.1.0, the same format Game Compressor builds on. It goes into a private Python - environment in {engine.dir} and doesn't touch your system Python. -

- {#if !engine.python} -

Python 3 wasn't found. Install it with your package manager, then restart Prospack.

+ {#if engine.canInstall} +
Developer setup
+

Install the compression engine

+

+ This is a source checkout without the bundled engine. Prospack can set up MkPFS {"1.1.0"} in a private + Python environment in {engine.dir}, without touching your system Python. +

+ {#if !engine.python} +

Python 3 wasn't found. Install it, then restart Prospack.

+ {:else} + + {/if} + {#if installLog.length} + 
{installLog.slice(-6).join("\n")}
+ {/if} {:else} - - {/if} - {#if installLog.length} - 
{installLog.slice(-6).join("\n")}
+
Something's missing
+

The compression engine isn't there

+

+ Prospack ships with its engine (prospack-engine), but it can't be found or started. + Antivirus software sometimes quarantines it. Reinstall Prospack, and allow it in your antivirus if needed. +

{/if}
{/if} diff --git a/src/lib/mock.js b/src/lib/mock.js index 6e6544b..6d94079 100644 --- a/src/lib/mock.js +++ b/src/lib/mock.js @@ -113,7 +113,7 @@ const delay = (ms) => new Promise((r) => setTimeout(r, ms)); export const mock = { getSettings: async () => structuredClone(settings), saveSettings: async (v) => { settings = v; }, - engineStatus: async () => ({ python: "3.14.7", mkpfs: "1.1.0", ready: true, dir: "~/.local/share/ps5-packer/engine" }), + engineStatus: async () => ({ python: null, mkpfs: "1.1.0", ready: true, bundled: true, canInstall: false, dir: "~/.local/share/prospack/engine" }), installEngine: async () => { for (const l of ["Creating Python environment", "Installing MkPFS 1.1.0"]) { emit("engine-log", l); await delay(500); } return mock.engineStatus(); }, inspectDump: async () => { throw new Error("preview"); }, addJob: async () => { throw "Adding dumps needs the desktop app (this is a browser preview)"; },