diff --git a/.github/workflows/pc-host.yml b/.github/workflows/pc-host.yml new file mode 100644 index 0000000..422a23e --- /dev/null +++ b/.github/workflows/pc-host.yml @@ -0,0 +1,53 @@ +name: PC host libraries +# PC in the headset (Experimental). The Windows job builds the native adapter +# and runs the real-encoder protocol tests; the installer's own bundle is +# tested in release.yml. The Linux adapter from PR #38 isn't shipped or offered +# in the app, but shares capture.c and controller.c, so it's still built here. +# These builds add the GPL x264 encoder for headless tests only: they're never +# uploaded or shipped. +on: + pull_request: + paths: ["desktop/**", "ui/**", "tests/**", "scripts/macview-bench.py", ".github/workflows/pc-host.yml"] + workflow_dispatch: +jobs: + linux: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Native libraries + run: | + sudo apt-get update -qq + sudo apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libglib2.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pipewire libpipewire-0.3-modules libspa-0.2-modules + python3 desktop/build.py --with-x264 + - name: Tests (real x264, fake capture consent/input) + env: + FRAME_PC_REQUIRE_NATIVE: "1" + FRAME_PC_REQUIRE_X264: "1" + run: python3 -m unittest discover -s tests -p 'test_pc*.py' -v + windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + update: true + # A mirror's HTTP 429 once failed this install (2026-10-24): retry, so a + # flaky mirror can't break a release build. + - name: MSYS2 packages for the PC host + shell: msys2 {0} + run: | + for attempt in 1 2 3; do + pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-gstreamer mingw-w64-ucrt-x86_64-gst-plugins-base mingw-w64-ucrt-x86_64-gst-plugins-good mingw-w64-ucrt-x86_64-gst-plugins-bad mingw-w64-ucrt-x86_64-gst-plugins-ugly && exit 0 + sleep $((attempt * 30)) + done + exit 1 + - name: Native build and tests (x264; WGC and MF need a desktop/GPU) + shell: msys2 {0} + run: | + python desktop/build.py --with-x264 + FRAME_PC_REQUIRE_NATIVE=1 FRAME_PC_REQUIRE_X264=1 python -m unittest discover -s tests -p 'test_pc*.py' -v diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8ed1e88..30e278f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,14 +3,14 @@ name: release # Pushing a v* tag builds Frame Control for macOS, Windows and Linux and attaches # the installers to that tag's GitHub release (created as a draft if missing). # Pull requests that change how the installers are built (the Electron app, the -# macOS helper it compiles, or this workflow) build the same installers as +# Windows PC helper libraries, the macOS helper, or this workflow) build the same installers as # artifacts. ui/, scripts/, frame/ and apk-catalog/ are copied in as they are and # are tested by the checks workflow; run this one by hand to package them early. on: push: tags: ["v*"] pull_request: - paths: ["app/**", "mac/frame-mac-view/**", ".github/workflows/release.yml"] + paths: ["app/**", "desktop/**", "mac/frame-mac-view/**", ".github/workflows/release.yml"] workflow_dispatch: # A newer push to the same pull request supersedes a build still in progress. @@ -75,6 +75,28 @@ jobs: - uses: actions/setup-node@v4 with: node-version: "24" + # PC in the headset (Experimental, Windows only): our capture adapter and + # the GStreamer libraries it uses, bundled into resources/desktop/bundle. + - uses: msys2/setup-msys2@v2 + if: runner.os == 'Windows' + with: + msystem: UCRT64 + update: true + # A mirror's HTTP 429 once failed this install (2026-10-24): retry, so a + # flaky mirror can't break a release build. + - name: MSYS2 packages for the PC host + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + for attempt in 1 2 3; do + pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-gstreamer mingw-w64-ucrt-x86_64-gst-plugins-base mingw-w64-ucrt-x86_64-gst-plugins-good mingw-w64-ucrt-x86_64-gst-plugins-bad mingw-w64-ucrt-x86_64-gst-plugins-ugly && exit 0 + sleep $((attempt * 30)) + done + exit 1 + - name: Windows PC streaming libraries + if: runner.os == 'Windows' + shell: msys2 {0} + run: python desktop/build.py --out app/build/desktop/win-x64 - name: Build working-directory: app shell: bash @@ -95,6 +117,19 @@ jobs: for ($i = 0; $i -lt 60 -and -not (Test-Path (Join-Path $installed 'Uninstall Frame Control.exe')); $i++) { Start-Sleep 2 } node app/build/check-resources.js (Join-Path $installed 'resources') app/dist/win-unpacked/resources if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + # From the installed app, so this also proves the installer carries the + # libraries. After the check above, and without writing __pycache__, + # so it can't change what that check compares. + - name: Test the installed app's Python with the Windows PC libraries + if: runner.os == 'Windows' + shell: bash + env: + PYTHONDONTWRITEBYTECODE: "1" + run: | + resources="$(cygpath -u "$LOCALAPPDATA")/Programs/Frame Control/resources" + python="$resources/python/python.exe" + native="$(cygpath -w "$resources/desktop/bundle")" + FRAME_PC_NATIVE="$native" FRAME_PC_REQUIRE_NATIVE=1 FRAME_PC_INSTALLER_BUNDLE=1 "$python" -B -m unittest discover -s tests -p 'test_pc*.py' -v - name: Upload to the release if: startsWith(github.ref, 'refs/tags/') shell: bash diff --git a/.gitignore b/.gitignore index ba02656..7387aad 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,15 @@ compat-db/.lakebed/ tests/smoke/results/ mac/bin/ +# Shared desktop streaming controller build products +desktop/controller.dll +desktop/controller.dylib +desktop/controller.so +desktop/*.obj +desktop/*.lib +desktop/*.exp +desktop/bundle/ +app/build/desktop/ + # Downloaded at build time (frame/kdeconnect/fetch.py, app/build/fetch-deps.js) frame/kdeconnect/packages/ diff --git a/README.md b/README.md index 2b98bd6..167dc93 100644 --- a/README.md +++ b/README.md @@ -226,6 +226,7 @@ Frame's software fits together, all checked against a real headset and labelled | [Sideloading Linux and Windows games](docs/sideloading.md) | A .zip, folder or .exe as a Steam Devkit Game, runtime detection | | [Install links for websites](docs/web-install.md) | `frame-control://install` links and manifests, the rules, a button to paste | | [VR comfort and HUD](docs/vr-utilities.md) · [Steam games](docs/steam-games.md) · [VR video](docs/vr-video.md) · [WebXR in Chromium](docs/webxr-chromium.md) | Installing and buying, watching VR180/360, the Chromium build | +| [PC in the headset](docs/pc-in-headset.md) | Experimental, Windows only, off until you turn it on: your PC's windows and screens as panels in the Frame. Not yet tried on a real Windows desktop or headset | | [Mac in the headset](docs/mac-in-headset.md) | Mac windows and screens as panels in the Frame, with laser and keyboard input | | [VR mods and custom songs](docs/mods.md) | UEVR for Unreal games from Frame Control (install, play in VR, remove), the per-game table, Beat Saber blockers | | [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes | diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 8c2a8a5..104a650 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -10,6 +10,8 @@ ship other people's software, unchanged, each under its own licence: | adb (Android SDK Platform-Tools) | Desktop apps (not Linux arm64) | Apache-2.0 and others | `NOTICE.txt` in the app's `tools` folder | | Mozilla's CA certificate list, as published by curl | Desktop apps | MPL-2.0 | https://curl.se/docs/caextract.html | | Electron | Desktop apps | MIT (Chromium: BSD-3-Clause and others) | `LICENSE` and `LICENSES.chromium.html` in the app | +| GStreamer 1.x, a few of its plugins and the libraries they need, as [MSYS2](https://packages.msys2.org/) builds them | Windows app only, for the experimental [PC in the headset](docs/pc-in-headset.md) | LGPL-2.1-or-later and others (per package); the GPL x264 encoder is left out | `libraries.json` (exact package versions) and the `licenses` folder in the app's `desktop/bundle` folder | Frame Control starts KDE Connect and talks to it over its network protocol; -it doesn't link to it or include its code. +it doesn't link to it or include its code. On Windows, Frame Control's own +`pc-host.dll` links to GStreamer dynamically; the libraries are unmodified. diff --git a/app/package.json b/app/package.json index 0d2110b..a17541f 100644 --- a/app/package.json +++ b/app/package.json @@ -198,7 +198,16 @@ "zip" ], "icon": "build/icon.png", - "artifactName": "Frame-Control-win-${arch}.${ext}" + "artifactName": "Frame-Control-win-${arch}.${ext}", + "extraResources": [ + { + "from": "build/desktop/${os}-${arch}", + "to": "desktop/bundle", + "filter": [ + "**/*" + ] + } + ] }, "nsis": { "oneClick": false, diff --git a/app/preferences.js b/app/preferences.js index 0b85b58..624c092 100644 --- a/app/preferences.js +++ b/app/preferences.js @@ -1,7 +1,7 @@ // Client choices live outside the local server's changing URL/origin. const fs = require("node:fs"); const path = require("node:path"); -const KEYS = ["mvSeparate", "padUsed"]; +const KEYS = ["mvSeparate", "padUsed", "pcView"]; const valid = value => value === "0" || value === "1"; function preferenceStore(file) { diff --git a/app/test/preferences.test.js b/app/test/preferences.test.js index 6bfe9ac..4fab560 100644 --- a/app/test/preferences.test.js +++ b/app/test/preferences.test.js @@ -36,7 +36,7 @@ test("client choices survive a new native session and local server origin", t => assert.deepEqual(first.read(), { mvSeparate: "1", padUsed: "1" }); }); -test("only the two non-sensitive binary choices can enter the native store", t => { +test("only the non-sensitive binary choices can enter the native store", t => { const file = fixture(t), store = preferenceStore(file); for (const [key, value] of [["token", "secret"], ["__proto__", "1"], ["padUsed", true], ["mvSeparate", "yes"]]) { assert.throws(() => store.write(key, value), /Invalid preference/); @@ -58,3 +58,11 @@ test("a missing or damaged preference file retains usable defaults", t => { store.write("padUsed", "1"); assert.deepEqual(store.read(), { padUsed: "1" }); }); + +test("the PC in the headset opt-in is remembered and off unless saved", t => { + const file = fixture(t), store = preferenceStore(file); + assert.equal(store.read().pcView, undefined); + store.write("pcView", "1"); + assert.deepEqual(preferenceStore(file).read(), { pcView: "1" }); + assert.throws(() => store.write("pcView", "on"), /Invalid preference/); +}); diff --git a/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json b/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json new file mode 100644 index 0000000..abccae8 --- /dev/null +++ b/bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json @@ -0,0 +1,311 @@ +{ + "label": "pc-agent-frame-arm64-hairpin", + "date": "2026-09-28T22:36:19", + "commit": "c5fc552+dirty", + "config": { + "quality": "balanced", + "mode": "separate", + "net": "none", + "delay_ms": 0, + "buffer_ms": 250, + "host": "frame", + "usb": false, + "ssh_opts": [], + "encoder": "", + "duration_s": 10.0, + "browser_flags": [] + }, + "frame_build": "20260925.6191901", + "mac": "26.5.2", + "host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)", + "source": "", + "pc_host": true, + "headset": "0:03.056203 [Info] - 1 - leaving standby", + "scenarios": [ + { + "scenario": "test", + "frames_sent": 351, + "frames_drawn": 351, + "frames_shown": 324, + "duration_s": 10.1, + "stages_ms": { + "capture": { + "p50": 0.1, + "p95": 0.2, + "p99": 0.5, + "n": 351 + }, + "queue": { + "p50": 0.1, + "p95": 0.3, + "p99": 0.4, + "n": 351 + }, + "encode": { + "p50": 2.0, + "p95": 2.9, + "p99": 3.5, + "n": 351 + }, + "socket": { + "p50": 0.1, + "p95": 0.4, + "p99": 0.6, + "n": 351 + }, + "network": { + "p50": 11.6, + "p95": 75.7, + "p99": 130.1, + "n": 351 + }, + "decode": { + "p50": 0.7, + "p95": 2.1, + "p99": 4.1, + "n": 351 + }, + "draw": { + "p50": 0.4, + "p95": 0.8, + "p99": 1.1, + "n": 351 + }, + "present": { + "p50": 0.8, + "p95": 18.1, + "p99": 24.9, + "n": 324 + }, + "content": { + "p50": 15.1, + "p95": 80.3, + "p99": 134.2, + "n": 351 + }, + "content_shown": { + "p50": 21.0, + "p95": 71.8, + "p99": 146.3, + "n": 324 + } + }, + "fps": 34.8, + "fps_shown": 32.0, + "late_pct": 53.41, + "stall_max": 226.8, + "stalls_over_100ms": 7, + "mbps": 0.26, + "keyframes": 6, + "size": "1280x720", + "captured": 607, + "viewer_never_drawn": 0, + "input_ms": { + "p50": 43.3, + "p95": 98.4, + "p99": 110.1, + "n": 18 + }, + "input_shown_ms": { + "p50": 54.7, + "p95": 98.9, + "p99": 114.0, + "n": 18 + }, + "input_parts_ms": { + "uplink": { + "p50": 17.1, + "p95": 48.3, + "p99": 53.1, + "n": 18 + }, + "mac": { + "p50": 12.5, + "p95": 32.9, + "p99": 35.7, + "n": 18 + }, + "back": { + "p50": 16.8, + "p95": 50.0, + "p99": 55.9, + "n": 18 + } + }, + "inputs": { + "asked": 18, + "sent": 18, + "seen": 18 + }, + "timeline": [ + { + "t": 0, + "fps": 38, + "mbps": 0.3, + "content_p50": 14.8, + "content_p95": 25.3, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 1, + "fps": 33, + "mbps": 0.24, + "content_p50": 15.6, + "content_p95": 103.5, + "bitrate": 3845448, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 2, + "fps": 39, + "mbps": 0.28, + "content_p50": 15.6, + "content_p95": 29.9, + "bitrate": 5283790, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 3, + "fps": 25, + "mbps": 0.21, + "content_p50": 17.8, + "content_p95": 175.9, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 4, + "fps": 34, + "mbps": 0.22, + "content_p50": 15.4, + "content_p95": 69.8, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 5, + "fps": 38, + "mbps": 0.29, + "content_p50": 17.7, + "content_p95": 68.7, + "bitrate": 2764800, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 6, + "fps": 38, + "mbps": 0.26, + "content_p50": 15.0, + "content_p95": 18.2, + "bitrate": 3091280, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 7, + "fps": 30, + "mbps": 0.22, + "content_p50": 19.2, + "content_p95": 111.0, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 8, + "fps": 35, + "mbps": 0.26, + "content_p50": 13.4, + "content_p95": 102.6, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 9, + "fps": 38, + "mbps": 0.28, + "content_p50": 12.3, + "content_p95": 20.9, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 10, + "fps": 3, + "mbps": 0.02, + "content_p50": 20.9, + "content_p95": 24.7, + "bitrate": 1725204, + "tier": 0, + "w": 1280, + "net": null + } + ], + "adapt": [], + "content_p50": 15.1, + "content_p95": 80.3, + "input_p50": 43.3, + "input_p95": 98.4, + "grades": { + "input_replies": "local", + "content_p50": "local", + "content_p95": "acceptable", + "input_p50": "local", + "input_p95": "acceptable", + "fps": "bad", + "late_pct": "bad", + "stall_max": "acceptable" + }, + "controller": { + "target": 3442081, + "ceiling": 5529600, + "tier": 0, + "fps": 60, + "scale": 1.0, + "baseRtt": 13.334, + "inFlight": 1, + "slack": 71.738, + "adapt": true + }, + "events": [ + { + "t": 3.77, + "e": "target 2764800 bit/s; tier 0" + }, + { + "t": 7.42, + "e": "target 1725204 bit/s; tier 0" + } + ], + "source_fps": 60.1, + "cpu": { + "host": "not sampled" + }, + "viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 71 Hz", + "show_s": 1.88, + "panel": "valve.steam.desktopgame.2001932457", + "src": "test", + "route": "network" + } + ], + "proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input." +} \ No newline at end of file diff --git a/bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json b/bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json new file mode 100644 index 0000000..a034e83 --- /dev/null +++ b/bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json @@ -0,0 +1,302 @@ +{ + "label": "pc-agent-frame-arm64-final", + "date": "2026-09-28T22:55:27", + "commit": "cd20243", + "config": { + "quality": "balanced", + "mode": "native", + "net": "none", + "delay_ms": 0, + "buffer_ms": 250, + "host": "frame", + "usb": false, + "ssh_opts": [], + "encoder": "", + "duration_s": 10.0, + "browser_flags": [] + }, + "frame_build": "20260925.6191901", + "mac": "26.5.2", + "host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)", + "source": "", + "pc_host": true, + "headset": "0:03.056203 [Info] - 1 - leaving standby", + "scenarios": [ + { + "scenario": "test", + "frames_sent": 400, + "frames_drawn": 400, + "frames_shown": 339, + "duration_s": 10.2, + "stages_ms": { + "capture": { + "p50": 0.1, + "p95": 0.1, + "p99": 0.2, + "n": 400 + }, + "queue": { + "p50": 0.1, + "p95": 0.2, + "p99": 0.4, + "n": 400 + }, + "encode": { + "p50": 2.0, + "p95": 2.7, + "p99": 3.1, + "n": 400 + }, + "socket": { + "p50": 0.1, + "p95": 0.3, + "p99": 0.4, + "n": 400 + }, + "network": { + "p50": 11.8, + "p95": 25.2, + "p99": 37.8, + "n": 400 + }, + "decode": { + "p50": 0.7, + "p95": 1.5, + "p99": 2.0, + "n": 400 + }, + "draw": { + "p50": 0.3, + "p95": 0.7, + "p99": 1.0, + "n": 400 + }, + "present": { + "p50": 0.8, + "p95": 19.3, + "p99": 26.8, + "n": 339 + }, + "content": { + "p50": 15.1, + "p95": 28.6, + "p99": 41.3, + "n": 400 + }, + "content_shown": { + "p50": 17.6, + "p95": 39.0, + "p99": 47.2, + "n": 339 + } + }, + "fps": 39.3, + "fps_shown": 33.4, + "late_pct": 50.25, + "stall_max": 69.3, + "stalls_over_100ms": 0, + "mbps": 0.35, + "keyframes": 6, + "size": "1280x720", + "captured": 612, + "viewer_never_drawn": 0, + "input_ms": { + "p50": 76.6, + "p95": 98.2, + "p99": 127.8, + "n": 19 + }, + "input_shown_ms": { + "p50": 80.5, + "p95": 107.8, + "p99": 128.1, + "n": 16 + }, + "input_parts_ms": { + "uplink": { + "p50": 36.5, + "p95": 57.6, + "p99": 83.2, + "n": 19 + }, + "mac": { + "p50": 14.6, + "p95": 31.3, + "p99": 31.7, + "n": 19 + }, + "back": { + "p50": 15.1, + "p95": 32.8, + "p99": 36.2, + "n": 19 + } + }, + "inputs": { + "asked": 19, + "sent": 19, + "seen": 19 + }, + "timeline": [ + { + "t": 0, + "fps": 40, + "mbps": 0.34, + "content_p50": 11.7, + "content_p95": 27.4, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 1, + "fps": 39, + "mbps": 0.37, + "content_p50": 11.9, + "content_p95": 28.2, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 2, + "fps": 41, + "mbps": 0.35, + "content_p50": 11.4, + "content_p95": 16.6, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 3, + "fps": 39, + "mbps": 0.37, + "content_p50": 12.7, + "content_p95": 28.0, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 4, + "fps": 38, + "mbps": 0.33, + "content_p50": 15.3, + "content_p95": 30.0, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 5, + "fps": 39, + "mbps": 0.37, + "content_p50": 16.7, + "content_p95": 28.5, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 6, + "fps": 41, + "mbps": 0.38, + "content_p50": 15.9, + "content_p95": 29.4, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 7, + "fps": 41, + "mbps": 0.31, + "content_p50": 15.6, + "content_p95": 28.2, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 8, + "fps": 38, + "mbps": 0.38, + "content_p50": 15.5, + "content_p95": 25.9, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 9, + "fps": 38, + "mbps": 0.29, + "content_p50": 16.7, + "content_p95": 28.8, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + }, + { + "t": 10, + "fps": 6, + "mbps": 0.03, + "content_p50": 17.6, + "content_p95": 23.1, + "bitrate": 5529600, + "tier": 0, + "w": 1280, + "net": null + } + ], + "adapt": [], + "content_p50": 15.1, + "content_p95": 28.6, + "input_p50": 76.6, + "input_p95": 98.2, + "grades": { + "input_replies": "local", + "content_p50": "local", + "content_p95": "local", + "input_p50": "acceptable", + "input_p95": "acceptable", + "fps": "bad", + "late_pct": "bad", + "stall_max": "local" + }, + "controller": { + "target": 5529600, + "ceiling": 5529600, + "tier": 0, + "fps": 60, + "scale": 1.0, + "baseRtt": 12.332, + "inFlight": 0, + "slack": 74.12, + "adapt": true + }, + "events": [], + "source_fps": 60.0, + "cpu": { + "host": "not sampled" + }, + "viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 72 Hz", + "show_s": 1.75, + "panel": "valve.steam.desktopgame.2001932457", + "src": "test", + "route": "network" + } + ], + "proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input." +} \ No newline at end of file diff --git a/desktop/build-controller.py b/desktop/build-controller.py new file mode 100644 index 0000000..33c83af --- /dev/null +++ b/desktop/build-controller.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Build the common controller for the Python PC host (a C11 compiler needed).""" +import os +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parent +suffix = '.dll' if sys.platform == 'win32' else '.dylib' if sys.platform == 'darwin' else '.so' +out = root / ('controller' + suffix) +if sys.platform == 'win32' and os.environ.get('CC', 'cl') == 'cl': + cmd = ['cl', '/nologo', '/O2', '/LD', '/std:c11', str(root / 'controller.c'), '/link', '/OUT:' + str(out)] +else: + cmd = [os.environ.get('CC', 'cc'), '-O2', '-std=c11', '-shared', '-fPIC', str(root / 'controller.c'), '-o', str(out)] +subprocess.run(cmd, cwd=root, check=True) +print(out) diff --git a/desktop/build.py b/desktop/build.py new file mode 100644 index 0000000..d7f858d --- /dev/null +++ b/desktop/build.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Build our native PC adapter and bundle its ordinary shared libraries. + +Run on the target architecture after installing GStreamer development packages. +No GStreamer executable or desktop streaming application is shipped or invoked. +Linux libc, display/GPU drivers and the portal service remain platform pieces. + +x264 is GPL, so the bundle the Windows installer ships leaves it out: Windows +encodes with Media Foundation. --with-x264 adds it for CI's headless protocol +tests, which have no GPU or desktop to encode with. +""" +import argparse +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys + +ROOT = Path(__file__).resolve().parent + + +def output(*args): + return subprocess.check_output(args, text=True).strip() + + +def build(destination, with_x264=False): + win = sys.platform == 'win32' + destination.mkdir(parents=True, exist_ok=True) + modules = ['gstreamer-1.0', 'gstreamer-app-1.0', 'gstreamer-video-1.0'] + if not win: + modules += ['gio-unix-2.0'] + import shlex + flags = shlex.split(output('pkg-config', '--cflags', '--libs', *modules)) + lib = destination / ('pc-host.dll' if win else 'pc-host.so') + cmd = [os.environ.get('CC', 'gcc' if win else 'cc'), '-std=c11', '-O2', '-shared', '-Wall', '-Wextra'] + if not win: + cmd += ['-fPIC', '-Wl,-rpath,$ORIGIN/lib'] + subprocess.run(cmd + [str(ROOT / f) for f in ('controller.c', 'capture.c', 'portal.c')] + flags + ['-o', str(lib)], check=True) + prefix = Path(output('pkg-config', '--variable=prefix', 'gstreamer-1.0')) + plugin_dir = Path(output('pkg-config', '--variable=pluginsdir', 'gstreamer-1.0')) + plugins_out, libs_out = destination / 'lib' / 'gstreamer-1.0', destination / ('bin' if win else 'lib') + plugins_out.mkdir(parents=True, exist_ok=True) + libs_out.mkdir(parents=True, exist_ok=True) + names = ['coreelements', 'videotestsrc', 'videoconvertscale', 'app', 'jpeg', 'videoparsersbad'] + if with_x264: + names.append('x264') + names += ['d3d11', 'mediafoundation'] if win else ['pipewire', 'va'] + pending = [lib] + for name in names: + matches = list(plugin_dir.glob('*gst' + name + ('.dll' if win else '.so'))) + if not matches: + raise SystemExit('Missing GStreamer library: ' + name) + for source in matches: + target = plugins_out / source.name + shutil.copy2(source, target) + pending.append(source) + if not win: + # PipeWire dynamically loads support and protocol modules. ldd cannot + # discover those; bundle them with a private client-only config instead + # of accidentally loading a different distro's client modules/config. + libdir = plugin_dir.parent + for relative in ('pipewire-0.3/libpipewire-module-protocol-native.so', + 'pipewire-0.3/libpipewire-module-client-node.so', + 'pipewire-0.3/libpipewire-module-adapter.so', + 'spa-0.2/support/libspa-support.so', + 'spa-0.2/support/libspa-dbus.so', + 'spa-0.2/videoconvert/libspa-videoconvert.so'): + source = libdir / relative + if not source.is_file(): + raise SystemExit('Missing PipeWire runtime library: ' + str(source)) + target = destination / 'lib' / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + pending.append(source) + config = destination / 'share' / 'pipewire' + config.mkdir(parents=True, exist_ok=True) + (config / 'client.conf').write_text( + 'context.spa-libs = { support.* = support/libspa-support video.* = videoconvert/libspa-videoconvert }\n' + 'context.modules = [\n' + ' { name = libpipewire-module-protocol-native }\n' + ' { name = libpipewire-module-client-node }\n' + ' { name = libpipewire-module-adapter }\n' + ']\n') + copied = set() + while pending: + binary = pending.pop() + if win: + imported = re.findall(r'DLL Name:\s*(\S+)', output('objdump', '-p', str(binary))) + dependencies = [prefix / 'bin' / name for name in imported] + else: + dependencies = [Path(p) for p in re.findall(r'=>\s+(/\S+)', output('ldd', str(binary)))] + for dep in dependencies: + if not dep.is_file() or dep.name in copied: + continue + if not win and re.match(r'lib(c|m|dl|rt|pthread|resolv)\.so', dep.name): + continue + shutil.copy2(dep, libs_out / dep.name) + copied.add(dep.name) + pending.append(dep) + # License texts and package provenance accompany the dynamically linked + # libraries. Distribution builders keep the upstream package/source URLs. + licenses = destination / 'licenses' + licenses.mkdir(exist_ok=True) + if win: + source = prefix / 'share' / 'licenses' + if source.exists(): + shutil.copytree(source, licenses, dirs_exist_ok=True) + packages = output('pacman', '-Q') if shutil.which('pacman') else 'See MSYS2 build log' + else: + packages = output('dpkg-query', '-W', '-f=${Package} ${Version}\n') if shutil.which('dpkg-query') else '' + # Debian copyright files contain licenses and source homepage details. + for path in Path('/usr/share/doc').glob('*/copyright'): + shutil.copy2(path, licenses / (path.parent.name + '.copyright')) + (destination / 'libraries.json').write_text(json.dumps(dict(gstreamer=output('pkg-config', '--modversion', 'gstreamer-1.0'), + plugins=names, libraries=sorted(copied), packages=packages), indent=2)) + print(lib) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, default=ROOT / 'bundle') + parser.add_argument('--with-x264', action='store_true', help='add the GPL x264 encoder (CI tests only, never shipped)') + args = parser.parse_args() + build(args.out.resolve(), args.with_x264) diff --git a/desktop/capture.c b/desktop/capture.c new file mode 100644 index 0000000..adb28cb --- /dev/null +++ b/desktop/capture.c @@ -0,0 +1,125 @@ +/* Frame Control's capture/encode adapter. GStreamer is a bundled library; + * capture uses WGC on Windows and the consented PipeWire fd on Linux. */ +#include "controller.h" +#include +#include +#include +#include + +typedef int (*Gate)(int stage, int64_t pts, int64_t capture, int64_t arrived); +typedef struct { + GstElement *pipeline, *encoder, *sink, *raw_queue; + Gate gate; + GstSample *sample; + GstMapInfo map; + int mapped; + char error[512]; +} Capture; +typedef struct { const unsigned char *data; int size, key, width, height; int64_t pts; } Encoded; +FC_API int64_t fc_now(void) {return g_get_monotonic_time();} +FC_API void fc_gst_init(void) {gst_init(NULL,NULL);} +FC_API int fc_has_element(const char *name) { + GstElementFactory *f=gst_element_factory_find(name); + if(!f)return 0;gst_object_unref(f);return 1; +} +static GstPadProbeReturn probe(GstPad *pad,GstPadProbeInfo *info,gpointer data) { + Capture *c=data; GstBuffer *b=GST_PAD_PROBE_INFO_BUFFER(info); + if(!b)return GST_PAD_PROBE_OK; + int64_t now=fc_now(), cap=now; + GstClock *clock=gst_element_get_clock(c->pipeline); + if(clock && GST_BUFFER_PTS_IS_VALID(b)) { + GstClockTime current=gst_clock_get_time(clock),origin=gst_element_get_base_time(c->pipeline); + if(current>=origin && current-origin>=GST_BUFFER_PTS(b)) + cap-= (int64_t)((current-origin-GST_BUFFER_PTS(b))/1000); + } + if(clock)gst_object_unref(clock); + int stage=GPOINTER_TO_INT(g_object_get_data(G_OBJECT(pad),"stage")); + if(stage==1) return c->gate(stage,(int64_t)GST_BUFFER_PTS(b),cap,now)>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP; + int phase=0; + for(;;) { + /* While congested, prefer a newer raw picture queued upstream. If + * nothing changed, retain this last picture until the gate opens; + * an idle window must not stay stale after a dropped final update. */ + guint queued=0; + if(phase && c->raw_queue)g_object_get(c->raw_queue,"current-level-buffers",&queued,NULL); + if(queued)return GST_PAD_PROBE_DROP; + int decision=c->gate(phase,(int64_t)GST_BUFFER_PTS(b),cap,now); + if(decision)return decision>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP; + phase=2;g_usleep(2000); + } +} +FC_API Capture *fc_capture_open(const char *pipeline,Gate gate,char *error,int capacity) { + GError *e=NULL;Capture *c=g_new0(Capture,1);c->gate=gate; + c->pipeline=gst_parse_launch(pipeline,&e); + if(e || !c->pipeline) { + g_strlcpy(error,e?e->message:"No pipeline",capacity); + if(e)g_error_free(e);if(c->pipeline)gst_object_unref(c->pipeline);g_free(c);return NULL; + } + c->encoder=gst_bin_get_by_name(GST_BIN(c->pipeline),"enc"); + c->sink=gst_bin_get_by_name(GST_BIN(c->pipeline),"out"); + GstElement *raw=gst_bin_get_by_name(GST_BIN(c->pipeline),"gate"); + if(!c->encoder || !c->sink || !raw) { + g_strlcpy(error,"Pipeline is missing enc, out or gate",capacity); + if(raw)gst_object_unref(raw);if(c->encoder)gst_object_unref(c->encoder); + if(c->sink)gst_object_unref(c->sink);gst_object_unref(c->pipeline);g_free(c);return NULL; + } + c->raw_queue=gst_bin_get_by_name(GST_BIN(c->pipeline),"raw_queue"); + GstPad *p=gst_element_get_static_pad(raw,"src"); + g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(0)); + gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p);gst_object_unref(raw); + p=gst_element_get_static_pad(c->encoder,"sink"); + g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(1)); + gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p); + gst_element_set_state(c->pipeline,GST_STATE_PLAYING); + return c; +} +FC_API int fc_capture_pull(Capture *c,Encoded *out) { + if(c->mapped) {gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map);c->mapped=0;} + if(c->sample){gst_sample_unref(c->sample);c->sample=NULL;} + GstBus *bus=gst_element_get_bus(c->pipeline); + GstMessage *m=gst_bus_pop_filtered(bus,GST_MESSAGE_ERROR);gst_object_unref(bus); + if(m) { + if(GST_MESSAGE_TYPE(m)==GST_MESSAGE_ERROR) { + GError *e=NULL;char *debug=NULL;gst_message_parse_error(m,&e,&debug); + g_strlcpy(c->error,e->message,sizeof(c->error));g_error_free(e);g_free(debug); + } else g_strlcpy(c->error,"The capture source closed",sizeof(c->error)); + gst_message_unref(m);return -1; + } + c->sample=gst_app_sink_try_pull_sample(GST_APP_SINK(c->sink),100*GST_MSECOND); + if(!c->sample) { + if(gst_app_sink_is_eos(GST_APP_SINK(c->sink))) { + g_strlcpy(c->error,"The capture source closed",sizeof(c->error));return -1; + } + return 0; + } + GstBuffer *b=gst_sample_get_buffer(c->sample); + if(!gst_buffer_map(b,&c->map,GST_MAP_READ))return 0; + c->mapped=1;out->data=c->map.data;out->size=(int)c->map.size; + out->key=!GST_BUFFER_FLAG_IS_SET(b,GST_BUFFER_FLAG_DELTA_UNIT);out->pts=(int64_t)GST_BUFFER_PTS(b); + const GstStructure *s=gst_caps_get_structure(gst_sample_get_caps(c->sample),0); + gst_structure_get_int(s,"width",&out->width);gst_structure_get_int(s,"height",&out->height);return 1; +} +FC_API const char *fc_capture_error(Capture *c) {return c->error;} +FC_API void fc_capture_bitrate(Capture *c,int bps) { + g_object_set(c->encoder,"bitrate",(guint)MAX(1,bps/1000),NULL); +} +FC_API void fc_capture_test(Capture *c,uint32_t input) { + GstElement *source=gst_bin_get_by_name(GST_BIN(c->pipeline),"source"); + if(source) { + /* Each benchmark click visibly changes the ball, before injection is + * timestamped. This is a response, not merely a protocol input echo. */ + guint color=0xff000000u | ((input*2654435761u)&0x00ffffffu); + g_object_set(source,"foreground-color",color,NULL);gst_object_unref(source); + } +} +FC_API void fc_capture_key(Capture *c) { + GstPad *p=gst_element_get_static_pad(c->encoder,"src"); + gst_pad_send_event(p,gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE,TRUE,0));gst_object_unref(p); +} +FC_API void fc_capture_close(Capture *c) { + gst_element_set_state(c->pipeline,GST_STATE_NULL); + if(c->mapped)gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map); + if(c->sample)gst_sample_unref(c->sample); + if(c->raw_queue)gst_object_unref(c->raw_queue); + gst_object_unref(c->sink);gst_object_unref(c->encoder);gst_object_unref(c->pipeline);g_free(c); +} diff --git a/desktop/controller.c b/desktop/controller.c new file mode 100644 index 0000000..38a179e --- /dev/null +++ b/desktop/controller.c @@ -0,0 +1,130 @@ +/* A C port of the Mac agent's RateController (mac/frame-mac-view/Sources/ + * Controller.swift) for the Windows host. Same gate, demand protection, bitrate + * recovery and tier hysteresis; bounded sample buffers. The Mac agent keeps its + * Swift original, so a change to either must be made to both: + * tests/test_pc_controller.py checks this one against the Swift trace. */ +#include "controller.h" +#include +#include +#define CAP 1024 +#define MIN(a,b) ((a)<(b)?(a):(b)) +#define MAX(a,b) ((a)>(b)?(a):(b)) +typedef struct { int64_t t, v; uint32_t seq; } Sample; +typedef struct { Sample a[CAP]; int n; } Samples; +struct FCController { + int max_fps, enabled, ceiling, target, tier, frame_bytes, held, signal, saw_ack; + int64_t decrease, increase, below, above, slack; + Samples flight, rtts, acked, sent, captures; +}; +static const int fps[] = {60,45,30,30,30}, scale[] = {100,100,100,75,50}; +static const double floors[] = {.45,.28,.16,.08,0}; +static void remove_first(Samples *s, int n) { + s->n -= n; memmove(s->a, s->a+n, (size_t)s->n*sizeof(Sample)); +} +static void add(Samples *s, Sample v, int cap) { + if (s->n >= cap) remove_first(s, 1); + s->a[s->n++] = v; +} +static void expire(Samples *s, int64_t oldest) { + int n=0; while(nn && s->a[n].trtts.n;i++) if(!i || c->rtts.a[i].vrtts.a[i].v; + return v; +} +static int compare(const void *a, const void *b) { + int64_t x=*(const int64_t*)a, y=*(const int64_t*)b; + return (x>y)-(xn;i++) if(s->a[i].t>since) a[n++]=s->a[i].v; + if(!n) return 0; + qsort(a,(size_t)n,sizeof(int64_t),compare); + return a[n*numerator/denominator]; +} +static int rate(Samples *s) { + int64_t total=0; for(int i=0;in;i++) total+=s->a[i].v; + return (int)MIN(total*16,2147483647); +} +FCController *fc_new(int max_fps,int enabled) { + FCController *c=calloc(1,sizeof(*c)); + if(c) {c->max_fps=MAX(1,max_fps);c->enabled=enabled;c->slack=40000;} + return c; +} +void fc_free(FCController *c) {free(c);} +void fc_ceiling(FCController *c,int bps) { + if(!c->target || c->target>bps) c->target=bps; + c->ceiling=bps; +} +int fc_gate(FCController *c,int64_t now,int counts) { + if(!c->enabled || !c->saw_ack) return 1; + expire(&c->flight,now-2000000); + if(!c->flight.n) return 1; + int64_t interval=1000000/MIN(c->max_fps,fps[c->tier]); + int window=MAX(3,(int)((base(c)+c->slack)/interval)+1); + if(c->flight.nflight.a[0].t<=base(c)+c->slack) return 1; + if(counts) c->held++; + return 0; +} +void fc_capture(FCController *c,int64_t now) {add(&c->captures,(Sample){now,0,0},256);} +void fc_sent(FCController *c,uint32_t seq,int bytes,int64_t now) { + Sample s={now,bytes,seq};add(&c->flight,s,512);add(&c->sent,s,CAP); +} +int fc_ack(FCController *c,uint32_t seq,int64_t now) { + c->saw_ack=1; + for(int i=0;iflight.n;i++) if(c->flight.a[i].seq==seq) { + Sample s=c->flight.a[i];remove_first(&c->flight,i+1); + add(&c->rtts,(Sample){now,now-s.t,0},CAP); + add(&c->acked,(Sample){now,s.v,0},CAP);return 1; + } + return 0; +} +int fc_update(FCController *c,int64_t now) { + expire(&c->rtts,now-10000000);expire(&c->acked,now-500000); + expire(&c->sent,now-500000);expire(&c->flight,now-2000000); + expire(&c->captures,now-1000000); + if(!c->enabled || c->ceiling<=0) return 0; + int64_t baseline=base(c), spread=quantile(&c->rtts,now-2000000,9,10); + int64_t jitter=spread ? spread-baseline : 0; + c->slack=1000000/MIN(c->max_fps,fps[c->tier])+MIN(MAX(jitter*3/2,25000),80000); + int64_t recent=quantile(&c->rtts,now-300000,1,2); + int64_t queue=recent ? recent-baseline : 0; + int64_t age=c->flight.n ? now-c->flight.a[0].t : 0; + int delivered=rate(&c->acked),sending=rate(&c->sent); + if(c->sent.n) c->frame_bytes=sending/16/c->sent.n; + int demand=(int)MIN((int64_t)MIN(c->captures.n,MIN(c->max_fps,fps[c->tier]))*c->frame_bytes*8,2147483647); + int signal=(sending>=c->target/2 && queue>40000)||c->held>=3||age>baseline+100000; + int congested=signal && c->signal;c->signal=signal;c->held=0; + if(congested && now-c->decrease>300000) { + int next=MAX(300000,MIN((int64_t)c->target*4/5,MAX((int64_t)delivered*9/10,c->target/2))); + if(demand>0 && (int64_t)demand*2<=(int64_t)c->target*5/4) next=MAX(next,MIN(c->target,(int64_t)demand*2)); + c->target=next;c->decrease=now; + } else if(!congested && now-c->decrease>1000000 && now-c->increase>250000 && c->targetceiling && + (sending>(int64_t)c->target*6/10 || now-c->decrease>3000000)) { + c->target=MIN(c->ceiling,(int64_t)(c->target*1.1)+50000);c->increase=now; + } + double share=(double)c->target/MAX(c->ceiling,1); + if(c->tier<4 && sharetier] && sending>=(int64_t)c->target*7/10) { + if(!c->below)c->below=now; + if(now-c->below>500000) { + for(c->tier=0;c->tier<4 && sharetier];c->tier++) {} + c->below=0; + } + } else c->below=0; + if(c->tier>0 && share>floors[c->tier-1]*1.25) { + if(!c->above)c->above=now; + if(now-c->above>2000000) {c->tier--;c->above=0;} + } else c->above=0; + return c->target; +} +int64_t fc_value(FCController *c,int field) { + switch(field) { + case 0:return c->target;case 1:return c->ceiling;case 2:return c->tier; + case 3:return MIN(c->max_fps,fps[c->tier]);case 4:return scale[c->tier]; + case 5:return base(c);case 6:return c->flight.n;case 7:return c->slack; + default:return 0; + } +} diff --git a/desktop/controller.h b/desktop/controller.h new file mode 100644 index 0000000..5df7212 --- /dev/null +++ b/desktop/controller.h @@ -0,0 +1,29 @@ +/* Shared latency controller. Times are monotonic host microseconds. + * Callers serialize access. Capture is gated BEFORE encoding: dropping an + * encoded inter frame would break the decoder's reference chain. */ +#ifndef FRAME_CONTROLLER_H +#define FRAME_CONTROLLER_H +#include +#ifdef _WIN32 +#define FC_API __declspec(dllexport) +#else +#define FC_API +#endif +#ifdef __cplusplus +extern "C" { +#endif +typedef struct FCController FCController; +FC_API FCController *fc_new(int fps, int enabled); +FC_API void fc_free(FCController *c); +FC_API void fc_ceiling(FCController *c, int bps); +FC_API int fc_gate(FCController *c, int64_t now, int counts); +FC_API void fc_capture(FCController *c, int64_t now); +FC_API void fc_sent(FCController *c, uint32_t seq, int bytes, int64_t now); +FC_API int fc_ack(FCController *c, uint32_t seq, int64_t now); +FC_API int fc_update(FCController *c, int64_t now); +/* target, ceiling, tier, fps, scale percent, baseline us, in flight, slack us */ +FC_API int64_t fc_value(FCController *c, int field); +#ifdef __cplusplus +} +#endif +#endif diff --git a/desktop/portal.c b/desktop/portal.c new file mode 100644 index 0000000..fbd11d2 --- /dev/null +++ b/desktop/portal.c @@ -0,0 +1,147 @@ +/* xdg-desktop-portal RemoteDesktop + ScreenCast, one consented session per + * panel. PipeWire receives the portal fd, never the unrestricted daemon fd. + * Input uses only the devices granted in Start's response. */ +#ifndef _WIN32 +#include "controller.h" +#include +#include +#include +#include +#define BUS "org.freedesktop.portal.Desktop" +#define PATH "/org/freedesktop/portal/desktop" +#define RD "org.freedesktop.portal.RemoteDesktop" +#define SC "org.freedesktop.portal.ScreenCast" +typedef struct { + GDBusConnection *bus; + GMainContext *context; + char *session; + int fd; + unsigned node, devices; + int width,height; +} Portal; +typedef struct {GVariant *result;int done;unsigned code;} Response; +static void response(GDBusConnection *c,const char *sender,const char *path,const char *iface, + const char *signal,GVariant *params,gpointer data) { + (void)c;(void)sender;(void)path;(void)iface;(void)signal; + Response *r=data;g_variant_get(params,"(u@a{sv})",&r->code,&r->result);r->done=1; +} +static void option(GVariantBuilder *b,const char *key,GVariant *v) {g_variant_builder_add(b,"{sv}",key,v);} +static GVariant *request(Portal *p,const char *iface,const char *method,GVariant *args, + const char *token,char *error,int capacity) { + char *sender=g_strdup(g_dbus_connection_get_unique_name(p->bus)+1); + for(char *c=sender;*c;c++)if(*c=='.')*c='_'; + char *path=g_strdup_printf(PATH "/request/%s/%s",sender,token);g_free(sender); + Response r={0};GError *e=NULL; + guint sub=g_dbus_connection_signal_subscribe(p->bus,BUS,"org.freedesktop.portal.Request","Response", + path,NULL,G_DBUS_SIGNAL_FLAGS_NONE,response,&r,NULL); + GVariant *reply=g_dbus_connection_call_sync(p->bus,BUS,PATH,iface,method,args,G_VARIANT_TYPE("(o)"), + G_DBUS_CALL_FLAGS_NONE,10000,NULL,&e); + if(reply)g_variant_unref(reply); + int64_t deadline=g_get_monotonic_time()+120000000; + while(!e && !r.done && g_get_monotonic_time()context,FALSE)) {} + g_usleep(10000); + } + if(!r.done) { + GVariant *closed=g_dbus_connection_call_sync(p->bus,BUS,path,"org.freedesktop.portal.Request","Close", + NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL); + if(closed)g_variant_unref(closed); + } + g_dbus_connection_signal_unsubscribe(p->bus,sub);g_free(path); + if(e || !r.done || r.code) { + g_strlcpy(error,e?e->message:!r.done?"Screen sharing request timed out":"Screen sharing was cancelled or refused",capacity); + if(e)g_error_free(e);if(r.result)g_variant_unref(r.result);return NULL; + } + return r.result; +} +FC_API void fc_portal_close(Portal *p) { + if(!p)return; + if(p->session && p->bus) { + GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,p->session,"org.freedesktop.portal.Session","Close", + NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL); + if(r)g_variant_unref(r); + } + if(p->fd>=0){close(p->fd);p->fd=-1;} + g_free(p->session);if(p->bus)g_object_unref(p->bus); + if(p->context)g_main_context_unref(p->context);g_free(p); +} +/* Each pipeline gets a fresh restricted PipeWire connection. A dup of a + * previously consumed protocol socket is not a new connection. + * Ownership: the Portal owns p->fd for its whole life and is its only closer. + * pipewiresrc never takes the fd it is given: its core connects with + * pw_context_connect_fd(ctx, fcntl(fd, F_DUPFD_CLOEXEC, 3), ...) and that + * duplicate is what PipeWire closes on teardown (src/gst/gstpipewirecore.c, + * unchanged from 0.3.19 through 1.x). So closing p->fd here is not a double + * close; not closing it would leak one socket per pipeline reopen. The + * caller closes the previous pipeline before asking for a new fd. */ +FC_API int fc_portal_refresh(Portal *p,char *error,int capacity) { + GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + GUnixFDList *fds=NULL;GError *e=NULL; + GVariant *r=g_dbus_connection_call_with_unix_fd_list_sync(p->bus,BUS,PATH,SC,"OpenPipeWireRemote", + g_variant_new("(oa{sv})",p->session,&b),G_VARIANT_TYPE("(h)"),G_DBUS_CALL_FLAGS_NONE,10000,NULL,&fds,NULL,&e); + if(!r){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;} + int handle;g_variant_get(r,"(h)",&handle);g_variant_unref(r); + int fd=g_unix_fd_list_get(fds,handle,&e);g_object_unref(fds); + if(fd<0){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;} + if(p->fd>=0)close(p->fd); + p->fd=fd;return fd; +} +FC_API Portal *fc_portal_select(char *error,int capacity) { + Portal *p=g_new0(Portal,1);p->fd=-1;p->context=g_main_context_new(); + g_main_context_push_thread_default(p->context); + GError *e=NULL;GVariant *r=NULL; + p->bus=g_bus_get_sync(G_BUS_TYPE_SESSION,NULL,&e); + if(!p->bus) {g_strlcpy(error,e->message,capacity);g_error_free(e);goto fail;} + GVariantBuilder b;char token[64],session[64]; + g_snprintf(session,sizeof(session),"fc_%u",g_random_int()); + g_snprintf(token,sizeof(token),"fc_%u",g_random_int()); + g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + option(&b,"handle_token",g_variant_new_string(token));option(&b,"session_handle_token",g_variant_new_string(session)); + r=request(p,RD,"CreateSession",g_variant_new("(a{sv})",&b),token,error,capacity);if(!r)goto fail; + g_variant_lookup(r,"session_handle","s",&p->session);g_variant_unref(r);r=NULL; + if(!p->session){g_strlcpy(error,"Portal did not return a session",capacity);goto fail;} + g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3)); + r=request(p,RD,"SelectDevices",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r); + g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3)); + option(&b,"multiple",g_variant_new_boolean(FALSE));option(&b,"cursor_mode",g_variant_new_uint32(2)); + r=request(p,SC,"SelectSources",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r); + g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + option(&b,"handle_token",g_variant_new_string(token)); + r=request(p,RD,"Start",g_variant_new("(osa{sv})",p->session,"",&b),token,error,capacity);if(!r)goto fail; + g_variant_lookup(r,"devices","u",&p->devices); + GVariant *streams=g_variant_lookup_value(r,"streams",G_VARIANT_TYPE("a(ua{sv})")); + if(streams && g_variant_n_children(streams)>0) { + GVariant *entry=g_variant_get_child_value(streams,0),*props=NULL; + g_variant_get(entry,"(u@a{sv})",&p->node,&props); + g_variant_lookup(props,"size","(ii)",&p->width,&p->height); + g_variant_unref(props);g_variant_unref(entry); + } + if(streams)g_variant_unref(streams);g_variant_unref(r);r=NULL; + if(!p->node || p->width<=0 || p->height<=0) {g_strlcpy(error,"Portal returned no stream size",capacity);goto fail;} + if(fc_portal_refresh(p,error,capacity)<0)goto fail; + g_main_context_pop_thread_default(p->context);return p; +fail: + g_main_context_pop_thread_default(p->context);fc_portal_close(p);return NULL; +} +FC_API int fc_portal_value(Portal *p,int field) { + switch(field){case 0:return p->fd;case 1:return (int)p->node;case 2:return p->width; + case 3:return p->height;case 4:return (int)p->devices;default:return 0;} +} +FC_API int fc_portal_input(Portal *p,int type,double x,double y,int code,int down) { + const char *method=NULL;GVariant *args=NULL;GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT); + if(type==0 && (p->devices&2)) { + method="NotifyPointerMotionAbsolute";args=g_variant_new("(oa{sv}udd)",p->session,&b,p->node,x,y); + } else if(type==1 && (p->devices&2)) { + method="NotifyPointerButton";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down); + } else if(type==2 && (p->devices&2)) { + method="NotifyPointerAxis";args=g_variant_new("(oa{sv}dd)",p->session,&b,x,y); + } else if(type==3 && (p->devices&1)) { + method="NotifyKeyboardKeysym";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down); + } else return 0; + GError *e=NULL;GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,PATH,RD,method,args,NULL, + G_DBUS_CALL_FLAGS_NONE,2000,NULL,&e); + if(e)g_error_free(e);if(r)g_variant_unref(r);return r!=NULL; +} +#endif diff --git a/docs/mac-in-headset.md b/docs/mac-in-headset.md index ed1b90a..de9a196 100644 --- a/docs/mac-in-headset.md +++ b/docs/mac-in-headset.md @@ -4,7 +4,8 @@ Frame Control can show any Mac window, or a whole Mac screen, as its own panel in the Steam Frame. You place each panel anywhere in the room with the SteamVR dashboard. The laser clicks and drags, the thumbstick scrolls, and you type on the Mac's own keyboard. Find it under **Tools → Mac in the headset** (macOS -only). +only). On Windows there's an experimental port, +[PC in the headset](pc-in-headset.md). The confidence labels are the same as in [ssh.md](ssh.md). diff --git a/docs/pc-in-headset.md b/docs/pc-in-headset.md new file mode 100644 index 0000000..6d86337 --- /dev/null +++ b/docs/pc-in-headset.md @@ -0,0 +1,137 @@ +# PC in the headset (Experimental) + +**Tools → PC in the headset**, on Windows only. It's the Windows version of +[Mac in the headset](mac-in-headset.md): a window or a whole screen on your PC +becomes its own SteamVR panel in the Frame. Place it with the dashboard's Float +in World, Move and Size, point and click with the laser, scroll with the +thumbstick, and type on the PC's keyboard. + +**It's experimental and not yet tried on a real Windows desktop or with a +headset.** It builds and passes its protocol tests on Windows in CI, but desktop +capture, Media Foundation encoding and laser input have never run on a real +PC. Expect it not to work yet. Reports are welcome through the +[feedback form](https://frame-control.pages.dev/feedback/). + +## Turning it on + +It's off by default. Until you tick **Turn on PC in the headset** on the card, +Frame Control starts no helper, loads none of the streaming libraries and lists +no windows. The choice is remembered on this computer. Turning it off ends +every stream, closes the viewers in the headset and stops the helper. + +The card and its Tools link only appear in the Windows app. On macOS the card +is Mac in the headset, as before; on Linux and the iPhone app there's no card. + +## How it works + +The ported pieces are from [PR #38](https://github.com/saphid/frame-control/pull/38). +Frame Control owns the host helper, input, authentication, streaming protocol, +panel launcher and adaptation. It doesn't need Sunshine, OBS or any other +streaming app. GStreamer is bundled as ordinary libraries. + +| | Windows | +|---|---| +| Capture | Windows.Graphics.Capture, through GStreamer's `d3d11screencapturesrc capture-api=wgc`, by window (HWND) or screen (HMONITOR) | +| Encoding | Media Foundation's H.264 encoder (`mfh264enc`), low latency, no B-frames. Whether it uses the GPU depends on the PC; **not verified** | +| Input | `SendInput`, mapped through the source's bounds, with per-monitor DPI awareness | + +API choices are documented, not device-verified: +[Windows capture](https://learn.microsoft.com/en-us/windows/uwp/audio-video-camera/screen-capture), +[GStreamer WGC](https://gstreamer.freedesktop.org/documentation/d3d11/d3d11screencapturesrc.html), +[Media Foundation encoder](https://gstreamer.freedesktop.org/documentation/mediafoundation/mfh264enc.html). + +It needs Windows 10 or 11 with Windows.Graphics.Capture and a Media Foundation +H.264 encoder (Windows "N" editions need the Media Feature Pack). Clicks and +keys can't reach administrator apps or the sign-in screen, which an ordinary +process can't drive. If Windows won't bring the chosen window to the front, +input is refused rather than sent to the app covering it: bring the window +forward, then Stop and Show again. Minimised or closed windows stop sending +pictures. Protected video isn't supported. Mac in the headset's "own display +per window" is Mac-only. + +The pieces it shares with Mac in the headset: + +- `ui/frame_macview.py` owns the SSH tunnel, reconnects, quality presets and + panel launch. `ui/frame_pcview.py` picks the Windows helper + (`ui/frame_pc_agent.py`, `ui/frame_pc_capture.py`, `desktop/capture.c`) and + holds the on/off switch. `/api/macview` is the endpoint for both. +- `ui/mac-view.html` is the one viewer in the headset. Its frame header, H.264 + and JPEG payloads, reconnect handshake, clock sync, timing reports and input + messages are unchanged. +- `desktop/controller.c` is a C port of the Mac helper's Swift rate controller, + for the Windows helper. The Mac keeps its Swift original; + `tests/test_pc_controller.py` checks the port against a frozen 600-state + trace from it. Capture is gated before encoding, so encoded frames are never + dropped, and an idle window's last picture still goes out once the link + clears. +- Stream records use the Mac helper's JSON schema (`ui/frame_stream_stats.py`), + so the benchmark's analysis is shared. + +## Packaging + +The Windows installer carries `resources/desktop/bundle`: `pc-host.dll` (our +adapter and the controller), the GStreamer plugins it uses (`coreelements`, +`videotestsrc`, `videoconvertscale`, `app`, `jpeg`, `videoparsersbad`, +`d3d11`, `mediafoundation`) and the DLLs they depend on, all from MSYS2's +UCRT64 packages. `libraries.json` records the GStreamer version, the plugins, +each copied DLL and the exact package versions; licence texts are in +`licenses/`. + +The x264 encoder is **not** in the installer. It's GPL, and Windows encodes +with Media Foundation, so only CI's test builds include it +(`desktop/build.py --with-x264`). The macOS and Linux installers are unchanged. + +From a source checkout on Windows, build the bundle from an MSYS2 UCRT64 shell +with the packages listed in `.github/workflows/pc-host.yml`: + +```sh +python desktop/build.py # what the installer ships +python desktop/build.py --with-x264 # for the native tests +``` + +A missing bundle is reported on the card; nothing is downloaded on first use. + +The Linux adapter from #38 (desktop portal, PipeWire, VA-API) is in the tree, +because it shares `capture.c` and the controller, and CI still builds and tests +it. The app never offers it and the Linux installers don't include it. + +## Measuring + +The Mac benchmark accepts the PC host: + +```sh +python scripts/macview-bench.py run --pc --scenario test --label pc-test +python scripts/macview-bench.py run --pc --scenario capture --source window:12345 --label windows-window +``` + +`capture` needs a window or screen id from the helper's `/windows` or +`/displays`. CPU sampling isn't available on PC hosts. See +[Mac in the headset](mac-in-headset.md) for the targets and shaping options. + +## What's verified + +- **Verified in CI, this PR:** on `windows-latest`, the native adapter builds + and a real x264 synthetic stream passes the ticket, reconnect, timing and + input-echo tests through the helper. The Windows installer builds with the + bundle, and its bundled Python loads the bundle, starts the helper and + confirms x264 isn't in it. Unit tests check the Windows-only, off-by-default + gating, the card and the Tools link on macOS, Windows and Linux. +- **Verified, #38, 2026-09-28, real Frame on SteamOS BUILD_ID 20260925.6191901:** + the PC helper (ARM64, on the Frame itself, x264 test pattern) streamed + through a Mac SSH relay into the shared viewer: 400 frames at 39.3 fps, + content latency p50/p95 15.1/28.6 ms, synthetic input p50 76.6 ms, at + revision `cd20243`. The frame-rate and late-frame targets failed. This + proves the helper and the viewer work together; it isn't Windows capture or + a real PC link. + [Raw result](../bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json), + [earlier probe](../bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json). +- **Verified, #38, on a Mac:** the C controller matched all 600 states of the + Swift controller's congestion and recovery trace. +- **Not verified:** anything on a real Windows desktop. Windows.Graphics.Capture, + Media Foundation (hardware or software), encoder restarts when the bitrate + changes, mixed-DPI and multi-monitor input alignment, window resize and + minimise, non-US keyboard layouts, physical laser input, and how it performs + in a worn headset. No numbers are claimed for these. Not tested with a + headset from this branch. +- **Not reviewed independently.** #38 tried an independent review that never + completed. diff --git a/docs/testing.md b/docs/testing.md index d263739..3fd2660 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -187,6 +187,27 @@ refuses ids with a hyphen (`missing/invalid arguments`), which the fake had accepted. The fake now refuses them the same way, and Frame Control makes ids Steam accepts. +## PC panel host tests + +PC in the headset (Experimental, Windows only). `tests/test_pcview.py` checks +that only Windows gets the PC host and that it starts nothing until turned on, +the Tools link and card wording, source-bound tickets, reconnect/Stop +revocation, input release, source-coordinate mapping and use of the existing +panel launcher. `tests/test_pc_controller.py` checks the C rate controller +against a frozen trace of the Mac's Swift one. + +The `PC host libraries` workflow builds the native adapter on Windows (and +the unshipped Linux adapter on Ubuntu x64 and ARM64), then runs a real x264 +synthetic stream through the HTTP/WebSocket agent. x264 is GPL, so only these +CI builds include it (`desktop/build.py --with-x264`); the release workflow +tests the installer's own bundle with its bundled Python and checks x264 isn't +in it. `FRAME_PC_REQUIRE_NATIVE=1` and `FRAME_PC_REQUIRE_X264=1` make a missing +bundle or encoder fail CI instead of skipping. + +None of this captures a desktop, exercises Media Foundation or a GPU, or +checks laser alignment. The [Windows test VM](#windows-test-vm) is the next +step for that. See [what's verified](pc-in-headset.md#whats-verified). + ## Windows test VM The unit tests run on Windows in CI, but the app itself doesn't. Anything that diff --git a/scripts/macview-bench.py b/scripts/macview-bench.py index cffa8b5..ac4a332 100755 --- a/scripts/macview-bench.py +++ b/scripts/macview-bench.py @@ -55,6 +55,7 @@ from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponen ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(ROOT / "ui")) import frame_macview # noqa: E402 +import frame_pcview # noqa: E402 RESULTS = ROOT / "bench" / "results" PAGES = ROOT / "bench" / "pages" @@ -254,7 +255,10 @@ class LabView(frame_macview.MacView): def ssh_runner(base): def run(remote, stdin=None, timeout=30): - r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout) + r = subprocess.run([*base, remote], input=stdin.encode("utf-8") if stdin is not None else None, + capture_output=True, timeout=timeout) + r.stdout = r.stdout.decode("utf-8", errors="replace") + r.stderr = r.stderr.decode("utf-8", errors="replace") if r.returncode: e = RuntimeError((r.stderr or r.stdout).strip()) e.stdout = r.stdout @@ -381,7 +385,11 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): if args.browser_flag is not None: mv.browser_flags = [f for f in args.browser_flag if f] chrome = None - src = "test" + src = args.source or "test" + cpu = None + if args.pc: + mv.host = "windows" if sys.platform == "win32" else "linux" + mv.viewer_profile = "pc-view" try: if scenario in ("scroll", "type"): chrome = chrome_window(f"{scenario}.html") @@ -405,9 +413,10 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): since = max([f["s"] for f in first["frames"]] or [0]) captured_before = first["captured"] start_mac_us = mv.call("/stats", id=stream["id"])["now"] - agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"], - capture_output=True, text=True).stdout.split()[0]) - cpu = CpuSampler(frame_ssh, agent_pid) + if not args.pc: + agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"], + capture_output=True, text=True).stdout.split()[0]) + cpu = CpuSampler(frame_ssh, agent_pid) if relay: relay.begin() expected = 0 # input events the harness asked the viewer for @@ -443,7 +452,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): if not streams: raise SystemExit(f"{scenario}: the stream ended during the run (did the Frame go to sleep?)") data = streams[0] - cpu.stop() + if cpu: + cpu.stop() if args.raw: Path(f"{args.raw}-{scenario}.json").write_text(json.dumps(data)) result = summarize(scenario, data, start_mac_us, end_mac_us, args, relay, schedule, expected) @@ -452,7 +462,7 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): for e in data.get("events", []) if e["t"] >= start_mac_us] result["captured"] = (captured_end if captured_end is not None else data["captured"]) - captured_before result["source_fps"] = round(result["captured"] / max(result["duration_s"], 1), 1) - result["cpu"] = cpu.summary() + result["cpu"] = cpu.summary() if cpu else {"host": "not sampled"} result["viewer"] = data["summary"].get("decoder", "") result["show_s"] = show_s result["panel"] = shown.get("panel") @@ -460,6 +470,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh): result["route"] = mv.route return result finally: + if cpu: + cpu.stop() try: mv.stop(src) except Exception: # noqa: BLE001 - best effort @@ -670,6 +682,8 @@ def fmt(v): def add_run_args(r): r.add_argument("--scenario", default="test,scroll,type") + r.add_argument("--pc", action="store_true", help="run the bundled PC host; use --scenario test or capture") + r.add_argument("--source", default="", help="PC source: window:ID, display:ID, or choose (Linux portal)") r.add_argument("--duration", type=float, default=20) r.add_argument("--warmup", type=float, default=3) r.add_argument("--quality", default="balanced", choices=list(frame_macview.QUALITY)) @@ -717,7 +731,7 @@ def frame_ssh_for(args): def run_suite(args, frame_ssh, quiet=False): - agent_port, token = lab_agent() + agent_port, token = (args.pc_view.port, args.pc_view.token) if args.pc else lab_agent() results = [] for sc in args.scenario.split(","): if not quiet: @@ -744,11 +758,12 @@ def document(args, frame_ssh, results, **extra): commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "") return { "label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit, - "config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay, + "config": {"quality": args.quality, "mode": "native" if args.pc else args.mode, "net": args.net or "none", "delay_ms": args.delay, "buffer_ms": args.buffer, "host": args.host or args.frame, "usb": args.usb, "ssh_opts": args.ssh_opt, "encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration, "browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS}, - "frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], "headset": standby[-40:], + "frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], + "host_platform": platform.platform(), "source": args.source, "pc_host": args.pc, "headset": standby[-40:], "scenarios": results, **extra, } @@ -819,19 +834,46 @@ def main(): print("\nworse:\n " + "\n ".join(regs)) sys.exit(1 if regs else 0) - lab_agent() + if args.pc and (args.cmd == "ab" or args.scenario not in ("test", "capture")): + p.error("PC runs use --scenario test or --scenario capture; Mac automation is not portable") + if args.pc and args.scenario == "capture" and not args.source: + p.error("capture needs --source window:ID, display:ID, or choose") + if not args.pc: + lab_agent() frame_ssh = frame_ssh_for(args) # Keep the Mac's screen awake: virtual displays aren't removed while it sleeps. runs = 1 if args.cmd == "run" else args.repeat * len(args.arm) - awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))]) + awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))]) if sys.platform == "darwin" else None + args.pc_view = None try: + if args.pc: + args.pc_view = frame_pcview.PCView(frame_ssh[:-1], ssh_runner(frame_ssh), frame_ssh[-1]) + args.pc_view.ensure_agent() + if args.source == "choose": + args.pc_view.call("/permissions", method="POST") + print("Choose a window or screen in your desktop's sharing dialog.", flush=True) + deadline = time.monotonic()+125 + while time.monotonic() < deadline: + state = args.pc_view.call("/status") + if not state.get("selecting"): + sources = args.pc_view.call("/windows")["windows"] + if not sources: + raise SystemExit(state.get("selectionError") or "Nothing was shared") + args.source = sources[-1]["src"] + break + time.sleep(.5) + else: + raise SystemExit("Sharing dialog timed out") if args.cmd == "ab": arm_runs, table = ab(args, frame_ssh) write(document(args, frame_ssh, [], arms=args.arm, runs=arm_runs, table=table), args) return results = run_suite(args, frame_ssh) finally: - awake.terminate() + if args.pc_view: + args.pc_view.shutdown() + if awake: + awake.terminate() doc = document(args, frame_ssh, results) write(doc, args) bad = [f"{r['scenario']} {k}" for r in results for k, v in r["grades"].items() if v in ("bad", "missing")] diff --git a/tests/test_macview.py b/tests/test_macview.py index 16e0de5..bf35ac2 100644 --- a/tests/test_macview.py +++ b/tests/test_macview.py @@ -84,7 +84,10 @@ class Helpers(unittest.TestCase): @unittest.skipUnless(shutil.which("bash"), "needs bash") @unittest.skipIf(os.name == "nt", "Windows' bash.exe is WSL's launcher, and runners have no distribution") def test_launch_script_parses(self): - r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True) + bash = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/bin/bash.exe") if sys.platform == "win32" else "bash" + if sys.platform == "win32" and not Path(bash).exists(): + self.skipTest("Git Bash is not installed (WSL bash is not a local shell)") + r = subprocess.run([bash, "-n"], input=frame_macview.LAUNCH.encode("utf-8"), capture_output=True) self.assertEqual(r.returncode, 0, r.stderr) def test_show_checks_the_source_before_anything_else(self): diff --git a/tests/test_pc_controller.py b/tests/test_pc_controller.py new file mode 100644 index 0000000..07fbb34 --- /dev/null +++ b/tests/test_pc_controller.py @@ -0,0 +1,64 @@ +"""Frozen regression trace from the original mac-in-headset Swift controller. +The 600-state reference was produced at 1b90c64, before the C extraction: +60 seconds with a slow/congested middle, then recovery. No generated behavior +from the new implementation is used to calculate the expected digest. +""" +import ctypes as C +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'ui')) +from frame_pc_capture import Controller + + +class ControllerTrace(unittest.TestCase): + def test_original_mac_congestion_and_recovery(self): + cc = shutil.which('cc') or shutil.which('gcc') + if not cc: + self.skipTest('C compiler not installed; native build jobs cover this trace') + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / ('trace.dll' if sys.platform == 'win32' else 'trace.so') + subprocess.run([cc, '-O2', '-shared', '-fPIC', str(ROOT/'desktop/controller.c'), '-o', str(path)], check=True, capture_output=True) + lib = C.CDLL(str(path)) + sig = {'new': (C.c_void_p, [C.c_int, C.c_int]), 'free': (None, [C.c_void_p]), + 'ceiling': (None, [C.c_void_p, C.c_int]), 'value': (C.c_int64, [C.c_void_p, C.c_int]), + 'capture': (None, [C.c_void_p, C.c_int64]), 'gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]), + 'sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]), + 'ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]), 'update': (C.c_int, [C.c_void_p, C.c_int64])} + for name, (ret, args) in sig.items(): + fn = getattr(lib, 'fc_'+name) + fn.restype, fn.argtypes = ret, args + c = Controller(lib, 60, 8000000) + due, states, seq = [], [], 0 + try: + for tick in range(6000): + now = 10000000+tick*10000 + for t, s in due: + if t <= now: + c.call('ack', s, now) + due = [(t, s) for t, s in due if t > now] + if tick % 2 == 0: + c.call('capture', now) + if c.call('gate', now, 1): + seq += 1 + c.call('sent', seq, 8000 if tick < 1000 or tick > 4000 else 22000, now) + due.append((now+(240000 if 1500 < tick < 3500 else 20000), seq)) + if tick % 10 == 0: + c.update(now) + x = c.state() + states.append([x[k] for k in ('target', 'ceiling', 'tier', 'fps', 'inFlight')] + + [round(x['scale']*100), round(x['baseRtt']*1000), round(x['slack']*1000)]) + digest = hashlib.sha256(json.dumps(states, separators=(',', ':')).encode()).hexdigest() + self.assertEqual(digest, '1932f16aafce5c9157bcd2aba5d90719a366f9999422920aaeae78a6cdec08f6') + finally: + c.close() + if sys.platform == 'win32': + import _ctypes + _ctypes.FreeLibrary(lib._handle) diff --git a/tests/test_pcview.py b/tests/test_pcview.py new file mode 100644 index 0000000..df4ac60 --- /dev/null +++ b/tests/test_pcview.py @@ -0,0 +1,520 @@ +"""PC adapters: grants, input cleanup, launcher reuse and native codec contract. +Native tests require the bundled libraries; CI sets REQUIRE_NATIVE so a missing +bundle is a failure, never a silent skip. No desktop consent or GPU is faked as +successful real-host capture. + +The encoding tests need x264, which only CI's own builds bundle (it's GPL, so +the installer leaves it out). FRAME_PC_REQUIRE_X264=1 makes its absence a +failure; FRAME_PC_INSTALLER_BUNDLE=1 checks the installer's bundle instead. +""" +import ctypes +import http.client +import json +import os +from pathlib import Path +import shutil +import struct +import subprocess +import sys +import threading +import time +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / 'ui')) +import frame_pc_agent as agent +import frame_pc_capture as capture +import frame_pcview +import frame_macview +from frame_stream_stats import Stats +from test_macview import WS + + +class GrantsTests(unittest.TestCase): + def test_source_bound_retry_ack_and_stop(self): + g = agent.Grants('master') + ticket = g.ticket('window:1') + self.assertIsNone(g.redeem('window:2', {'t': ticket})) + key = g.redeem('window:1', {'t': ticket}) + self.assertEqual(key, g.redeem('window:1', {'t': ticket})) + g.ack(key) + self.assertIsNone(g.redeem('window:1', {'t': ticket})) + self.assertEqual(key, g.redeem('window:1', {'r': key})) + self.assertIsNone(g.redeem('window:2', {'r': key})) + g.revoke('window:1') + self.assertIsNone(g.redeem('window:1', {'r': key})) + + def test_expiry_and_stop_before_redeem(self): + g = agent.Grants('master') + with mock.patch.object(agent.time, 'monotonic', return_value=1): + ticket = g.ticket('test') + with mock.patch.object(agent.time, 'monotonic', return_value=62): + self.assertIsNone(g.redeem('test', {'t': ticket})) + ticket = g.ticket('test') + g.revoke() + self.assertIsNone(g.redeem('test', {'t': ticket})) + + +class GatingTests(unittest.TestCase): + """Experimental, Windows only, and off until the person turns it on.""" + + def view(self): + return frame_pcview.PCView(['ssh'], mock.Mock(), 'frame') + + def test_only_windows_gets_the_pc_host(self): + for platform, cls in (('win32', frame_pcview.PCView), ('linux', frame_macview.MacView), + ('darwin', frame_macview.MacView)): + with mock.patch.object(frame_pcview.sys, 'platform', platform): + view = frame_pcview.host_view(['ssh'], mock.Mock(), 'frame') + self.assertIs(type(view), cls, platform) + + def test_off_by_default_starts_nothing(self): + view = self.view() + with mock.patch.object(frame_pcview.sys, 'platform', 'win32'), \ + mock.patch.object(frame_macview.subprocess, 'Popen', side_effect=AssertionError('started while off')): + state = view.state() + self.assertEqual(state, {'available': False, 'reason': frame_pcview.OFF, 'host': view.host, + 'experimental': True, 'enabled': False}) + with self.assertRaises(frame_macview.MacViewError): + view.show('window:12') + view.run.assert_not_called() + + def test_never_offered_off_windows_even_when_turned_on(self): + view = self.view() + view.enabled = True + with mock.patch.object(frame_pcview.sys, 'platform', 'linux'): + self.assertIn('only available on Windows', view.unavailable()) + + def test_turning_on_checks_for_the_bundled_libraries(self): + view = self.view() + missing = mock.Mock(is_file=mock.Mock(return_value=False)) + with mock.patch.object(frame_pcview.sys, 'platform', 'win32'), \ + mock.patch.object(frame_pcview, 'LIBRARY', missing): + state = view.set_enabled(True) + self.assertTrue(state['enabled']) + self.assertFalse(state['available']) + self.assertIn('libraries are missing', state['reason']) + + def test_turning_off_stops_streams_and_the_helper(self): + view = self.view() + view.enabled = True + agent = mock.Mock() + agent.poll.return_value = None + view.agent, view.tunnel = agent, mock.Mock() + tunnel = view.tunnel + tunnel.poll.return_value = None + view.stop = mock.Mock() + with mock.patch.object(frame_pcview.sys, 'platform', 'win32'): + state = view.set_enabled(False) + view.stop.assert_called_once_with() + agent.stdin.close.assert_called_once() + agent.wait.assert_called() + tunnel.terminate.assert_called_once() + self.assertIsNone(view.agent) + self.assertEqual((state['enabled'], state['reason']), (False, frame_pcview.OFF)) + + def test_server_enable_action(self): + import sandbox # noqa: F401 (keeps the server off real data and services) + import server + with mock.patch.object(server, 'macview', frame_macview.MacView(['ssh'], mock.Mock(), 'frame')): + with self.assertRaises(server.Failure) as e: + server.macview_action({'action': 'enable', 'on': True}) + self.assertIn('only available on Windows', str(e.exception)) + view = self.view() + view.set_enabled = mock.Mock(return_value={'enabled': True}) + with mock.patch.object(server, 'macview', view): + self.assertEqual(server.macview_action({'action': 'enable', 'on': True}), {'enabled': True}) + server.macview_action({'action': 'enable', 'on': 'yes'}) # anything but true is off + self.assertEqual([c.args for c in view.set_enabled.call_args_list], [(True,), (False,)]) + + +class PageTests(unittest.TestCase): + page = (ROOT / 'ui' / 'index.html').read_text(encoding='utf-8') + + def test_tool_link_is_windows_only_and_marked_experimental(self): + self.assertIn('

Tools

Choose a task for your Frame.

- +

VR comfort and performance

Waiting for the Frame
@@ -1703,8 +1703,19 @@
@@ -5546,7 +5558,7 @@ $("sessionGame").onclick = e => { }; // ---- Mac in the headset: windows and displays as panels (ui/frame_macview.py) ---- -let mvSeq = 0, mvTimer = 0; +let mvSeq = 0, mvTimer = 0, mvPcAsked = false; // streaming: the src it's shown as (a window may be "separate:"), or "". // How a live stream is doing: frames shown, Mac-to-headset delay, bitrate, and // whether the network made it step down (fewer frames, then fewer pixels). @@ -5572,9 +5584,27 @@ async function loadMacView(restart) { try { s = await api("/api/macview" + (restart ? "?restart=1" : "")); } catch (e) { err = e; } if (seq !== mvSeq || location.hash !== "#macview") return; const box = $("macview"); - if (err) { box.hidden = false; return failed($("mvWindows"), err); } - if (!s.available) { box.hidden = true; return; } + if (err) { box.hidden = $("mvMain").hidden = false; return failed($("mvWindows"), err); } + // PC in the headset (Windows, Experimental) is off until turned on here. + const pc = !!s.host && s.host !== "mac"; + $("mvTitle").textContent = pc ? "PC in the headset" : "Mac in the headset"; + $("mvTag").hidden = $("mvPc").hidden = !pc; + $("mvPcOn").checked = !!s.enabled; + if (pc && !s.enabled && clientPreferences.get("pcView") === "1" && !restart && !mvPcAsked) { + mvPcAsked = true; // once per page: a failure to start shouldn't loop + return mvPcEnable(true, true); + } + if (!s.available) { + box.hidden = !pc; + $("mvMain").hidden = true; + $("mvCount").textContent = ""; + if (pc) { $("mvPcState").textContent = s.reason || ""; $("mvPcState").hidden = !s.reason; } + return; + } box.hidden = false; + $("mvMain").hidden = false; + $("mvPcState").hidden = true; + $("mvSeparate").closest("label").hidden = pc; // separate virtual displays are a Mac feature const live = new Set((s.streams || []).map(x => x.src)); const byStream = src => (s.streams || []).find(x => x.src === src); $("mvCount").textContent = live.size ? `${live.size} showing${s.route === "usb" ? " · over USB-C" : ""}` : ""; @@ -5584,6 +5614,7 @@ async function loadMacView(restart) { $("mvPerm").hidden = !perm.length; $("mvPerm").innerHTML = perm.join(" ") + `
Then press Refresh. macOS may ask you to reopen Frame Control.
`; + if (pc) $("mvPerm").hidden = true; // Windows asks for no permission; elevated apps just refuse input const liveAs = src => live.has(src) ? src : src.startsWith("window:") && live.has("separate:" + src.slice(7)) ? "separate:" + src.slice(7) : ""; $("mvDisplays").innerHTML = (s.displays || []).map(d => mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, liveAs(d.src), byStream(liveAs(d.src)))).join(""); @@ -5591,16 +5622,26 @@ async function loadMacView(restart) { $("mvWinCount").textContent = wins.length ? `${wins.length}` : ""; $("mvWindows").innerHTML = !s.screen ? `
Windows appear here once Screen Recording is allowed.
` : wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, liveAs(w.src), byStream(liveAs(w.src)))).join("") - : `
No windows open on this Mac's current desktop.
`; + : `
No windows open on this ${pc ? "PC" : "Mac's current desktop"}.
`; // Keep the numbers fresh while something is in the headset and the card is on screen. clearTimeout(mvTimer); const tick = () => { mvTimer = document.hidden || !box.offsetParent ? setTimeout(tick, 3000) : (loadMacView(), 0); }; if (live.size) mvTimer = setTimeout(tick, 3000); } $("mvRefresh").onclick = () => loadMacView(true); +// The choice is remembered on this computer; the server starts with it off. +async function mvPcEnable(on, quiet) { + if (!quiet) clientPreferences.set("pcView", on ? "1" : "0"); + const turn = () => api("/api/macview", { action: "enable", on }); + if (quiet) { try { await turn(); } catch {} } + else await act(on ? "Turn on PC in the headset" : "Turn off PC in the headset", async () => + ({ ...await turn(), message: on ? "PC in the headset is on." : "PC in the headset is off. Nothing is shared." }), $("mvPcOn")); + loadMacView(); +} +$("mvPcOn").onchange = e => mvPcEnable(e.target.checked); // ---- client preferences: native storage survives changing localhost origins ---- const clientPreferences = (() => { - const keys = ['mvSeparate', 'padUsed'], values = {}, changed = new Set(); + const keys = ['mvSeparate', 'padUsed', 'pcView'], values = {}, changed = new Set(); const valid = value => value === '0' || value === '1'; const remember = (key, value) => { values[key] = value; @@ -5637,7 +5678,7 @@ $("macview").onclick = async e => { const b = e.target.closest("[data-mv]"); if (!b) return; const src = b.dataset.src; if (b.dataset.mv === "show") { - const as = $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src; + const as = !$("mvSeparate").closest("label").hidden && $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src; await act(`Show ${b.dataset.title} in the headset`, async () => { const r = await api("/api/macview", { action: "show", src: as, quality: $("mvQuality").value, w: +b.dataset.w || undefined, h: +b.dataset.h || undefined }); @@ -7160,6 +7201,8 @@ api("/api/host").then(h => { Object.assign(HOST, h); window.frameControlCenter?.host(h); $("macToolLink").hidden = h.mobile || h.os !== "macOS"; + $("pcToolLink").hidden = h.mobile || h.os !== "Windows"; + if (!h.mobile && h.os === "Windows") TOOL_TITLES.macview = "PC in the headset"; $("backupToolLink").hidden = !!h.mobile; // backups are folders on a computer if (h.mobile) return; $("offSetup").hidden = false; $("offSetup").onclick = setUpActive; // the server opens it in a terminal diff --git a/ui/mac-view.html b/ui/mac-view.html index 4d639f4..520d790 100644 --- a/ui/mac-view.html +++ b/ui/mac-view.html @@ -22,7 +22,7 @@ -
Connecting to the Mac…
+
Connecting to your computer…
diff --git a/ui/server.py b/ui/server.py index b10e229..7caf6fa 100755 --- a/ui/server.py +++ b/ui/server.py @@ -59,6 +59,7 @@ import frame_contact # noqa: E402 import frame_host # noqa: E402 import frame_link # noqa: E402 import frame_macview # noqa: E402 +import frame_pcview # noqa: E402 import frame_media # noqa: E402 import frame_panels # noqa: E402 import frame_quest # noqa: E402 @@ -2832,12 +2833,13 @@ def push_media(path): "id": token + "/" + name} -# ---- Mac in the headset (frame_macview.py) ---------------------------------- +# ---- Mac in the headset (frame_macview.py), PC in the headset (frame_pcview.py) ---- # The tunnel gets its own connection: the shared master's options would win # over anything added after them. -macview = frame_macview.MacView(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"], - lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout), +macview = frame_pcview.host_view(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"], + lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin.encode("utf-8") if stdin is not None else None, + timeout=timeout, text=False).decode("utf-8", errors="replace"), FRAME, track=_live_tunnels.add) @@ -2854,7 +2856,8 @@ def macview_state(query=None): def macview_action(body): - """{action: show|stop|permissions, src, quality, w, h}.""" + """{action: show|stop|permissions, src, quality, w, h} | {action: enable, on}. + enable turns PC in the headset (Experimental, Windows only) on or off.""" if LOCAL: raise Failure("Runs on your computer, not the headset.", 400) action = body.get("action") @@ -2867,6 +2870,10 @@ def macview_action(body): return macview.stop(body.get("src") or None) if action == "permissions": return macview.request_permissions() + if action == "enable": + if not isinstance(macview, frame_pcview.PCView): + raise Failure("PC in the headset is only available on Windows.", 400) + return macview.set_enabled(body.get("on") is True) except frame_macview.MacViewError as e: raise Failure(str(e), 502) raise Failure("unknown action", 400)