Bundle the engine

.gitignore

@@ -3,3 +3,5 @@ dist/
 src-tauri/target/
 src-tauri/gen/
 public/mock/
+src-tauri/binaries/
+target-engine/

engine/build.py

@@ -0,0 +1,52 @@
+"""Freeze the engine into src-tauri/binaries/prospack-engine-<target-triple>[.exe].
+
+Tauri's sidecar convention needs the Rust target triple in the file name.
+Run with the Python that has engine/requirements-build.txt installed.
+"""
+
+import os
+import platform
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+ENGINE = ROOT / "engine"
+OUT = ROOT / "src-tauri" / "binaries"
+
+
+def host_triple() -> str:
+    out = subprocess.run(["rustc", "-vV"], check=True, capture_output=True, text=True).stdout
+    return next(l.split(": ", 1)[1] for l in out.splitlines() if l.startswith("host: "))
+
+
+def main() -> int:
+    work = ROOT / "target-engine"
+    subprocess.run(
+        [
+            sys.executable, "-m", "PyInstaller", "--noconfirm", "--clean", "--onefile",
+            "--name", "prospack-engine",
+            "--collect-all", "mkpfs",
+            "--collect-all", "zlib_ng",
+            "--hidden-import", "deep_verify",
+            "--paths", str(ENGINE),
+            "--distpath", str(work / "dist"),
+            "--workpath", str(work / "build"),
+            "--specpath", str(work),
+            str(ENGINE / "prospack_engine.py"),
+        ],
+        check=True,
+    )
+    exe = ".exe" if platform.system() == "Windows" else ""
+    OUT.mkdir(parents=True, exist_ok=True)
+    dest = OUT / f"prospack-engine-{host_triple()}{exe}"
+    shutil.copy2(work / "dist" / f"prospack-engine{exe}", dest)
+    if not exe:
+        os.chmod(dest, 0o755)
+    print(dest)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

engine/make_fake_dump.py

@@ -0,0 +1,55 @@
+"""Write a small, reproducible fake PS5 dump for tests and CI (no real game data).
+
+    python engine/make_fake_dump.py OUT_DIR/PPSA00001-app
+"""
+
+import json
+import os
+import random
+import struct
+import sys
+import zlib
+from pathlib import Path
+
+
+def png(path: Path, w: int, h: int, rgb: tuple[int, int, int]) -> None:
+    def chunk(tag: bytes, data: bytes) -> bytes:
+        return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
+
+    rows = b"".join(b"\0" + bytes(rgb) * w for _ in range(h))
+    path.write_bytes(
+        b"\x89PNG\r\n\x1a\n"
+        + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0))
+        + chunk(b"IDAT", zlib.compress(rows, 9))
+        + chunk(b"IEND", b"")
+    )
+
+
+def main() -> int:
+    root = Path(sys.argv[1])
+    title = root.name.split("-")[0] if root.name[:4].isalpha() else "PPSA00001"
+    rnd = random.Random(1234)
+    (root / "sce_sys").mkdir(parents=True, exist_ok=True)
+    (root / "fakelib").mkdir(exist_ok=True)
+    (root / "data/prein/bg").mkdir(parents=True, exist_ok=True)
+    (root / "sce_sys/param.json").write_text(json.dumps({
+        "titleId": title,
+        "contentVersion": "01.000.000",
+        "localizedParameters": {"defaultLanguage": "en-US", "en-US": {"titleName": "TEST GAME"}},
+    }))
+    png(root / "sce_sys/icon0.png", 512, 512, (40, 90, 220))
+    png(root / "sce_sys/pic0.png", 1280, 720, (30, 30, 60))
+    (root / "eboot.bin").write_bytes(rnd.randbytes(2 * 1024 * 1024))
+    (root / "fakelib/libSceAmpr.sprx").write_bytes(rnd.randbytes(64 * 1024))
+    for i in range(300):
+        (root / f"data/prein/bg/small_{i}.txt").write_bytes(b"a" * rnd.randrange(1, 90000))
+    with open(root / "data/prein/bg/big_text.bin", "wb") as f:
+        line = b"level data compressible\n"
+        f.write(line * (40 * 1024 * 1024 // len(line)))
+    (root / "data/prein/bg/big_random.bin").write_bytes(rnd.randbytes(8 * 1024 * 1024))
+    print(root)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

engine/prospack_engine.py

@@ -0,0 +1,39 @@
+"""Prospack engine: the compression and verification half of Prospack.
+
+Frozen into a single executable (see build.py) and shipped next to the app, so users
+don't need Python.
+
+    prospack-engine pack folder [mkpfs options] SOURCE IMAGE
+    prospack-engine verify IMAGE SOURCE [WORKERS]
+    prospack-engine --version
+"""
+
+import multiprocessing
+import sys
+
+
+def main() -> int:
+    # Must run first: MkPFS and the verifier start worker processes, which re-enter this
+    # executable when frozen.
+    multiprocessing.freeze_support()
+    if len(sys.argv) < 2 or sys.argv[1] not in ("pack", "verify", "--version"):
+        print(__doc__.strip(), file=sys.stderr)
+        return 2
+    cmd = sys.argv[1]
+    if cmd == "--version":
+        import importlib.metadata as md
+
+        print(f"mkpfs {md.version('mkpfs')}")
+        return 0
+    if cmd == "pack":
+        from mkpfs.cli import cli_mkpfs_main
+
+        return cli_mkpfs_main(sys.argv[1:])
+    import deep_verify
+
+    sys.argv = [sys.argv[0], *sys.argv[2:]]
+    return deep_verify.main()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

engine/requirements-build.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pyinstaller>=6.10

engine/requirements.txt

@@ -0,0 +1,2 @@
+mkpfs==1.1.0
+zlib-ng>=1.0

src-tauri/src/engine.rs

@@ -1,4 +1,8 @@
-//! The compression engine: a private Python venv with MkPFS, plus the deep verifier.
+//! The compression engine: MkPFS plus the deep verifier, as one `prospack-engine` program.
+//!
+//! Release builds ship it frozen as a Tauri sidecar next to the app. Developers can point
+//! `PROSPACK_ENGINE` at a frozen build, or let the app run `engine/prospack_engine.py` in a
+//! private venv ("Install MkPFS" in the UI).
 
 use serde::Serialize;
 use std::io::Read;
@@ -9,7 +13,9 @@ use std::sync::Arc;
 
 /// Pinned so a new MkPFS release can't silently change the image format.
 pub const MKPFS_VERSION: &str = "1.1.0";
-const VERIFIER: &str = include_str!("../resources/deep_verify.py");
+const EXE: &str = if cfg!(windows) { ".exe" } else { "" };
+/// The repo's engine sources, for the developer fallback.
+const DEV_ENGINE_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../engine");
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -17,9 +23,19 @@ pub struct EngineStatus {
     pub python: Option<String>,
     pub mkpfs: Option<String>,
     pub ready: bool,
+    /// True when the frozen engine shipped with the app is in use.
+    pub bundled: bool,
+    /// Whether "Install MkPFS" (developer venv) is possible on this machine.
+    pub can_install: bool,
     pub dir: String,
 }
 
+#[derive(Debug, Clone, PartialEq)]
+enum Runner {
+    Frozen(PathBuf),
+    Script { python: PathBuf, script: PathBuf },
+}
+
 pub struct Engine {
     root: PathBuf,
 }
@@ -32,65 +48,97 @@ impl Engine {
     fn venv(&self) -> PathBuf {
         self.root.join("venv")
     }
-    fn mkpfs_bin(&self) -> PathBuf {
-        self.venv().join("bin/mkpfs")
-    }
     fn python_bin(&self) -> PathBuf {
-        self.venv().join("bin/python")
+        if cfg!(windows) {
+            self.venv().join("Scripts/python.exe")
+        } else {
+            self.venv().join("bin/python")
+        }
     }
-    fn verifier(&self) -> PathBuf {
-        self.root.join("deep_verify.py")
+    fn dev_script() -> Option<PathBuf> {
+        std::env::var_os("PROSPACK_ENGINE_SCRIPT")
+            .map(PathBuf::from)
+            .or_else(|| Some(PathBuf::from(DEV_ENGINE_DIR).join("prospack_engine.py")))
+            .filter(|p| p.is_file())
+    }
+
+    /// Sidecar next to the app, then $PROSPACK_ENGINE, then the developer venv.
+    fn runner(&self) -> Option<Runner> {
+        let sidecar = std::env::current_exe()
+            .ok()
+            .and_then(|e| e.parent().map(|d| d.join(format!("prospack-engine{EXE}"))))
+            .filter(|p| p.is_file());
+        if let Some(p) = sidecar {
+            return Some(Runner::Frozen(p));
+        }
+        if let Some(p) = std::env::var_os("PROSPACK_ENGINE").map(PathBuf::from).filter(|p| p.is_file()) {
+            return Some(Runner::Frozen(p));
+        }
+        let python = self.python_bin();
+        match (python.is_file(), Self::dev_script()) {
+            (true, Some(script)) => Some(Runner::Script { python, script }),
+            _ => None,
+        }
+    }
+
+    fn command(&self) -> Result<Command, String> {
+        match self.runner() {
+            Some(Runner::Frozen(p)) => Ok(quiet(p)),
+            Some(Runner::Script { python, script }) => {
+                let mut c = quiet(python);
+                c.arg(script);
+                Ok(c)
+            }
+            None => Err("The compression engine is missing. Reinstall Prospack.".into()),
+        }
     }
 
     pub fn status(&self) -> EngineStatus {
-        let python = quiet("python3")
-            .arg("--version")
-            .output()
+        let runner = self.runner();
+        let mkpfs = self
+            .command()
             .ok()
+            .and_then(|mut c| c.arg("--version").output().ok())
             .filter(|o| o.status.success())
-            .map(|o| String::from_utf8_lossy(&o.stdout).trim().replace("Python ", ""));
-        let mkpfs = if self.mkpfs_bin().is_file() {
-            quiet(self.python_bin())
-                .args(["-c", "import importlib.metadata as m; print(m.version('mkpfs'))"])
+            .map(|o| String::from_utf8_lossy(&o.stdout).trim().trim_start_matches("mkpfs ").to_string());
+        let can_install = Self::dev_script().is_some();
+        let python = if matches!(runner, Some(Runner::Frozen(_))) || !can_install {
+            None
+        } else {
+            quiet(if cfg!(windows) { "python" } else { "python3" })
+                .arg("--version")
                 .output()
                 .ok()
                 .filter(|o| o.status.success())
-                .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
-        } else {
-            None
+                .map(|o| String::from_utf8_lossy(&o.stdout).trim().replace("Python ", ""))
         };
-        let ready = mkpfs.is_some();
-        EngineStatus { python, mkpfs, ready, dir: self.root.to_string_lossy().into_owned() }
+        EngineStatus {
+            python,
+            ready: mkpfs.is_some(),
+            mkpfs,
+            bundled: matches!(runner, Some(Runner::Frozen(_))),
+            can_install,
+            dir: self.root.to_string_lossy().into_owned(),
+        }
     }
 
-    /// Creates the venv and installs the pinned MkPFS. Streams pip's output to `log`.
+    /// Developer setup: a venv with the engine's requirements. Release builds never need this.
     pub fn install(&self, log: &mut dyn FnMut(String)) -> Result<(), String> {
+        let script = Self::dev_script()
+            .ok_or("The compression engine ships with Prospack. Reinstall Prospack to restore it.")?;
+        let reqs = script.with_file_name("requirements.txt");
         std::fs::create_dir_all(&self.root).map_err(|e| e.to_string())?;
         if !self.python_bin().is_file() {
             log("Creating Python environment".into());
-            run_logged(quiet("python3").arg("-m").arg("venv").arg(self.venv()), log)?;
+            run_logged(quiet(if cfg!(windows) { "python" } else { "python3" }).arg("-m").arg("venv").arg(self.venv()), log)?;
         }
         log(format!("Installing MkPFS {MKPFS_VERSION}"));
         run_logged(
-            quiet(self.python_bin()).args([
-                "-m",
-                "pip",
-                "install",
-                "--disable-pip-version-check",
-                "--upgrade",
-                &format!("mkpfs=={MKPFS_VERSION}"),
-            ]),
+            quiet(self.python_bin())
+                .args(["-m", "pip", "install", "--disable-pip-version-check", "-r"])
+                .arg(reqs),
             log,
         )?;
-        self.write_verifier()?;
-        Ok(())
-    }
-
-    pub fn write_verifier(&self) -> Result<(), String> {
-        std::fs::create_dir_all(&self.root).map_err(|e| e.to_string())?;
-        if std::fs::read_to_string(self.verifier()).ok().as_deref() != Some(VERIFIER) {
-            std::fs::write(self.verifier(), VERIFIER).map_err(|e| e.to_string())?;
-        }
         Ok(())
     }
 
@@ -107,7 +155,7 @@ impl Engine {
     ) -> Result<(), String> {
         let tmp_dir = out.parent().unwrap().join("tmp");
         std::fs::create_dir_all(&tmp_dir).map_err(|e| e.to_string())?;
-        let mut cmd = quiet(self.mkpfs_bin());
+        let mut cmd = self.command()?;
         cmd.args(["pack", "folder", "--no-adjust-output-file-extension", "--ampr-skip-regen-if-exists"])
             .arg("--compression-level")
             .arg(level.min(9).to_string())
@@ -141,14 +189,13 @@ impl Engine {
         cancel: &Arc<AtomicBool>,
         on: &mut dyn FnMut(VerifyEvent),
     ) -> Result<bool, String> {
-        self.write_verifier()?;
         let workers = if workers == 0 {
             (std::thread::available_parallelism().map(|n| n.get()).unwrap_or(2) / 2).clamp(1, 8) as u32
         } else {
             workers
         };
-        let mut cmd = quiet(self.python_bin());
-        cmd.arg(self.verifier()).arg(image).arg(src).arg(workers.to_string());
+        let mut cmd = self.command()?;
+        cmd.arg("verify").arg(image).arg(src).arg(workers.to_string());
         let mut buf = String::new();
         let mut verdict: Option<bool> = None;
         run_streaming(cmd, cancel, &mut |chunk| {
@@ -464,6 +511,7 @@ mod cancel_test {
     /// PACKER_E2E_SCRATCH=<dir with mkvenv> cargo test cancel -- --ignored
     #[test]
     #[ignore]
+    #[cfg(unix)]
     fn cancel_kills_mkpfs_promptly() {
         let scratch = PathBuf::from(std::env::var("PACKER_E2E_SCRATCH").unwrap());
         let tmp = tempfile::tempdir().unwrap();
@@ -483,6 +531,7 @@ mod cancel_test {
         let out_dir = tmp.path().join("out");
         std::fs::create_dir_all(&out_dir).unwrap();
         let engine = Engine { root };
+        assert!(engine.runner().is_some(), "no engine: set PROSPACK_ENGINE or provide the venv");
         let cancel = Arc::new(AtomicBool::new(false));
         let c2 = cancel.clone();
         let t = std::time::Instant::now();

src-tauri/src/lib.rs

@@ -120,7 +120,6 @@ pub fn run() {
                 let _ = handle.emit(event, payload);
             });
             let engine = Arc::new(engine::Engine::new(&settings::data_dir()));
-            let _ = engine.write_verifier();
             let queue = pipeline::Queue::new(emitter, engine.clone());
             // Folders passed on the command line (or via "Open with") are queued straight away.
             for arg in std::env::args().skip(1).filter(|a| !a.starts_with('-')) {

src-tauri/src/pipeline.rs

@@ -876,6 +876,18 @@ mod tests {
 mod e2e {
     use super::*;
 
+    fn copy_dir(from: &Path, to: &Path) {
+        std::fs::create_dir_all(to).unwrap();
+        for e in std::fs::read_dir(from).unwrap().flatten() {
+            let p = e.path();
+            if p.is_dir() {
+                copy_dir(&p, &to.join(e.file_name()));
+            } else {
+                std::fs::copy(&p, to.join(e.file_name())).unwrap();
+            }
+        }
+    }
+
     fn wait_until(q: &Queue, id: u64, secs: u64) -> Job {
         let t = Instant::now();
         loop {
@@ -895,17 +907,19 @@ mod e2e {
         let tmp = tempfile::tempdir().unwrap();
         std::env::set_var("PROSPACK_CONFIG_DIR", tmp.path().join("config"));
         std::env::set_var("PROSPACK_DATA_DIR", tmp.path().join("data"));
-        let engine_dir = tmp.path().join("data/engine");
-        std::fs::create_dir_all(&engine_dir).unwrap();
-        std::os::unix::fs::symlink(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap();
+        // Engine: $PROSPACK_ENGINE (frozen build) if set, else the scratch venv as the dev venv.
+        if std::env::var_os("PROSPACK_ENGINE").is_none() {
+            let engine_dir = tmp.path().join("data/engine");
+            std::fs::create_dir_all(&engine_dir).unwrap();
+            #[cfg(unix)]
+            std::os::unix::fs::symlink(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap();
+            #[cfg(windows)]
+            std::os::windows::fs::symlink_dir(scratch.join("mkvenv"), engine_dir.join("venv")).unwrap();
+        }
 
         // Fresh copy of the fake dump so MkPFS's ampr_emu.index write can't leak between runs.
         let dump = tmp.path().join("PPSA00001-app");
-        assert!(std::process::Command::new("cp")
-            .args(["-r", scratch.join("fakedump/PPSA00001-app").to_str().unwrap(), dump.to_str().unwrap()])
-            .status()
-            .unwrap()
-            .success());
+        copy_dir(&scratch.join("fakedump/PPSA00001-app"), &dump);
         let _ = std::fs::remove_file(dump.join("ampr_emu.index"));
 
         let ps5_root = tmp.path().join("ps5");
@@ -917,7 +931,8 @@ mod e2e {
                 let _ = self.0.wait();
             }
         }
-        let _server = KillOnDrop(std::process::Command::new(scratch.join("mkvenv/bin/python"))
+        let py = if cfg!(windows) { scratch.join("mkvenv/Scripts/python.exe") } else { scratch.join("mkvenv/bin/python") };
+        let _server = KillOnDrop(std::process::Command::new(py)
             .arg(scratch.join("fake_ps5.py"))
             .arg(&ps5_root)
             .arg(port.to_string())

src-tauri/tauri.conf.json

@@ -40,7 +40,9 @@
       "icons/128x128@2x.png",
       "icons/icon.png"
     ],
-    "resources": []
+    "externalBin": [
+      "binaries/prospack-engine"
+    ]
   },
   "mainBinaryName": "prospack"
 }

src/lib/EmptyState.svelte

@@ -11,21 +11,30 @@
 <div class="empty">
   {#if engine && !engine.ready}
     <section class="setup reveal">
-      <div class="kicker">One-time setup</div>
-      <h2>Install the compression engine</h2>
-      <p>
-        Prospack uses <b>MkPFS 1.1.0</b>, the same format Game Compressor builds on. It goes into a private Python
-        environment in <span class="mono">{engine.dir}</span> and doesn't touch your system Python.
-      </p>
-      {#if !engine.python}
-        <p class="err">Python 3 wasn't found. Install it with your package manager, then restart Prospack.</p>
+      {#if engine.canInstall}
+        <div class="kicker">Developer setup</div>
+        <h2>Install the compression engine</h2>
+        <p>
+          This is a source checkout without the bundled engine. Prospack can set up <b>MkPFS {"1.1.0"}</b> in a private
+          Python environment in <span class="mono">{engine.dir}</span>, without touching your system Python.
+        </p>
+        {#if !engine.python}
+          <p class="err">Python 3 wasn't found. Install it, then restart Prospack.</p>
+        {:else}
+          <button class="btn primary" disabled={installing} onclick={onInstall}>
+            {installing ? "Installing…" : "Install MkPFS"}
+          </button>
+        {/if}
+        {#if installLog.length}
+          <pre class="mono">{installLog.slice(-6).join("\n")}</pre>
+        {/if}
       {:else}
-        <button class="btn primary" disabled={installing} onclick={onInstall}>
-          {installing ? "Installing…" : "Install MkPFS"}
-        </button>
-      {/if}
-      {#if installLog.length}
-        <pre class="mono">{installLog.slice(-6).join("\n")}</pre>
+        <div class="kicker err">Something's missing</div>
+        <h2>The compression engine isn't there</h2>
+        <p>
+          Prospack ships with its engine (<span class="mono">prospack-engine</span>), but it can't be found or started.
+          Antivirus software sometimes quarantines it. Reinstall Prospack, and allow it in your antivirus if needed.
+        </p>
       {/if}
     </section>
   {/if}

src/lib/mock.js

@@ -113,7 +113,7 @@ const delay = (ms) => new Promise((r) => setTimeout(r, ms));
 export const mock = {
   getSettings: async () => structuredClone(settings),
   saveSettings: async (v) => { settings = v; },
-  engineStatus: async () => ({ python: "3.14.7", mkpfs: "1.1.0", ready: true, dir: "~/.local/share/ps5-packer/engine" }),
+  engineStatus: async () => ({ python: null, mkpfs: "1.1.0", ready: true, bundled: true, canInstall: false, dir: "~/.local/share/prospack/engine" }),
   installEngine: async () => { for (const l of ["Creating Python environment", "Installing MkPFS 1.1.0"]) { emit("engine-log", l); await delay(500); } return mock.engineStatus(); },
   inspectDump: async () => { throw new Error("preview"); },
   addJob: async () => { throw "Adding dumps needs the desktop app (this is a browser preview)"; },