mirror of
https://github.com/saphid/frame-control.git
synced 2026-10-28 15:02:10 +01:00
Merge pull request #121 from saphid/feat/windows-pc-in-headset
PC in the headset on Windows (Experimental)
This commit is contained in:
31 files changed
+3433
-55
No files matched your search
@@ -0,0 +1,53 @@
|
||||
name: PC host libraries
|
||||
# PC in the headset (Experimental). The Windows job builds the native adapter
|
||||
# and runs the real-encoder protocol tests; the installer's own bundle is
|
||||
# tested in release.yml. The Linux adapter from PR #38 isn't shipped or offered
|
||||
# in the app, but shares capture.c and controller.c, so it's still built here.
|
||||
# These builds add the GPL x264 encoder for headless tests only: they're never
|
||||
# uploaded or shipped.
|
||||
on:
|
||||
pull_request:
|
||||
paths: ["desktop/**", "ui/**", "tests/**", "scripts/macview-bench.py", ".github/workflows/pc-host.yml"]
|
||||
workflow_dispatch:
|
||||
jobs:
|
||||
linux:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, ubuntu-24.04-arm]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Native libraries
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libglib2.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-pipewire libpipewire-0.3-modules libspa-0.2-modules
|
||||
python3 desktop/build.py --with-x264
|
||||
- name: Tests (real x264, fake capture consent/input)
|
||||
env:
|
||||
FRAME_PC_REQUIRE_NATIVE: "1"
|
||||
FRAME_PC_REQUIRE_X264: "1"
|
||||
run: python3 -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
windows:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: msys2/setup-msys2@v2
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: true
|
||||
# A mirror's HTTP 429 once failed this install (2026-10-24): retry, so a
|
||||
# flaky mirror can't break a release build.
|
||||
- name: MSYS2 packages for the PC host
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
for attempt in 1 2 3; do
|
||||
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-gstreamer mingw-w64-ucrt-x86_64-gst-plugins-base mingw-w64-ucrt-x86_64-gst-plugins-good mingw-w64-ucrt-x86_64-gst-plugins-bad mingw-w64-ucrt-x86_64-gst-plugins-ugly && exit 0
|
||||
sleep $((attempt * 30))
|
||||
done
|
||||
exit 1
|
||||
- name: Native build and tests (x264; WGC and MF need a desktop/GPU)
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
python desktop/build.py --with-x264
|
||||
FRAME_PC_REQUIRE_NATIVE=1 FRAME_PC_REQUIRE_X264=1 python -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
@@ -3,14 +3,14 @@ name: release
|
||||
# Pushing a v* tag builds Frame Control for macOS, Windows and Linux and attaches
|
||||
# the installers to that tag's GitHub release (created as a draft if missing).
|
||||
# Pull requests that change how the installers are built (the Electron app, the
|
||||
# macOS helper it compiles, or this workflow) build the same installers as
|
||||
# Windows PC helper libraries, the macOS helper, or this workflow) build the same installers as
|
||||
# artifacts. ui/, scripts/, frame/ and apk-catalog/ are copied in as they are and
|
||||
# are tested by the checks workflow; run this one by hand to package them early.
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
paths: ["app/**", "mac/frame-mac-view/**", ".github/workflows/release.yml"]
|
||||
paths: ["app/**", "desktop/**", "mac/frame-mac-view/**", ".github/workflows/release.yml"]
|
||||
workflow_dispatch:
|
||||
|
||||
# A newer push to the same pull request supersedes a build still in progress.
|
||||
@@ -75,6 +75,28 @@ jobs:
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "24"
|
||||
# PC in the headset (Experimental, Windows only): our capture adapter and
|
||||
# the GStreamer libraries it uses, bundled into resources/desktop/bundle.
|
||||
- uses: msys2/setup-msys2@v2
|
||||
if: runner.os == 'Windows'
|
||||
with:
|
||||
msystem: UCRT64
|
||||
update: true
|
||||
# A mirror's HTTP 429 once failed this install (2026-10-24): retry, so a
|
||||
# flaky mirror can't break a release build.
|
||||
- name: MSYS2 packages for the PC host
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: |
|
||||
for attempt in 1 2 3; do
|
||||
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-gcc mingw-w64-ucrt-x86_64-pkgconf mingw-w64-ucrt-x86_64-python mingw-w64-ucrt-x86_64-gstreamer mingw-w64-ucrt-x86_64-gst-plugins-base mingw-w64-ucrt-x86_64-gst-plugins-good mingw-w64-ucrt-x86_64-gst-plugins-bad mingw-w64-ucrt-x86_64-gst-plugins-ugly && exit 0
|
||||
sleep $((attempt * 30))
|
||||
done
|
||||
exit 1
|
||||
- name: Windows PC streaming libraries
|
||||
if: runner.os == 'Windows'
|
||||
shell: msys2 {0}
|
||||
run: python desktop/build.py --out app/build/desktop/win-x64
|
||||
- name: Build
|
||||
working-directory: app
|
||||
shell: bash
|
||||
@@ -95,6 +117,19 @@ jobs:
|
||||
for ($i = 0; $i -lt 60 -and -not (Test-Path (Join-Path $installed 'Uninstall Frame Control.exe')); $i++) { Start-Sleep 2 }
|
||||
node app/build/check-resources.js (Join-Path $installed 'resources') app/dist/win-unpacked/resources
|
||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
||||
# From the installed app, so this also proves the installer carries the
|
||||
# libraries. After the check above, and without writing __pycache__,
|
||||
# so it can't change what that check compares.
|
||||
- name: Test the installed app's Python with the Windows PC libraries
|
||||
if: runner.os == 'Windows'
|
||||
shell: bash
|
||||
env:
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
run: |
|
||||
resources="$(cygpath -u "$LOCALAPPDATA")/Programs/Frame Control/resources"
|
||||
python="$resources/python/python.exe"
|
||||
native="$(cygpath -w "$resources/desktop/bundle")"
|
||||
FRAME_PC_NATIVE="$native" FRAME_PC_REQUIRE_NATIVE=1 FRAME_PC_INSTALLER_BUNDLE=1 "$python" -B -m unittest discover -s tests -p 'test_pc*.py' -v
|
||||
- name: Upload to the release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
shell: bash
|
||||
|
||||
+10
@@ -7,5 +7,15 @@ compat-db/.lakebed/
|
||||
tests/smoke/results/
|
||||
mac/bin/
|
||||
|
||||
# Shared desktop streaming controller build products
|
||||
desktop/controller.dll
|
||||
desktop/controller.dylib
|
||||
desktop/controller.so
|
||||
desktop/*.obj
|
||||
desktop/*.lib
|
||||
desktop/*.exp
|
||||
desktop/bundle/
|
||||
app/build/desktop/
|
||||
|
||||
# Downloaded at build time (frame/kdeconnect/fetch.py, app/build/fetch-deps.js)
|
||||
frame/kdeconnect/packages/
|
||||
@@ -226,6 +226,7 @@ Frame's software fits together, all checked against a real headset and labelled
|
||||
| [Sideloading Linux and Windows games](docs/sideloading.md) | A .zip, folder or .exe as a Steam Devkit Game, runtime detection |
|
||||
| [Install links for websites](docs/web-install.md) | `frame-control://install` links and manifests, the rules, a button to paste |
|
||||
| [VR comfort and HUD](docs/vr-utilities.md) · [Steam games](docs/steam-games.md) · [VR video](docs/vr-video.md) · [WebXR in Chromium](docs/webxr-chromium.md) | Installing and buying, watching VR180/360, the Chromium build |
|
||||
| [PC in the headset](docs/pc-in-headset.md) | Experimental, Windows only, off until you turn it on: your PC's windows and screens as panels in the Frame. Not yet tried on a real Windows desktop or headset |
|
||||
| [Mac in the headset](docs/mac-in-headset.md) | Mac windows and screens as panels in the Frame, with laser and keyboard input |
|
||||
| [VR mods and custom songs](docs/mods.md) | UEVR for Unreal games from Frame Control (install, play in VR, remove), the per-game table, Beat Saber blockers |
|
||||
| [SSH](docs/ssh.md) · [Streaming](docs/streaming.md) · [Files](docs/file-transfer.md) · [Panels](docs/panels.md) · [Tailscale](docs/tailscale.md) | Topic notes |
|
||||
|
||||
@@ -10,6 +10,8 @@ ship other people's software, unchanged, each under its own licence:
|
||||
| adb (Android SDK Platform-Tools) | Desktop apps (not Linux arm64) | Apache-2.0 and others | `NOTICE.txt` in the app's `tools` folder |
|
||||
| Mozilla's CA certificate list, as published by curl | Desktop apps | MPL-2.0 | https://curl.se/docs/caextract.html |
|
||||
| Electron | Desktop apps | MIT (Chromium: BSD-3-Clause and others) | `LICENSE` and `LICENSES.chromium.html` in the app |
|
||||
| GStreamer 1.x, a few of its plugins and the libraries they need, as [MSYS2](https://packages.msys2.org/) builds them | Windows app only, for the experimental [PC in the headset](docs/pc-in-headset.md) | LGPL-2.1-or-later and others (per package); the GPL x264 encoder is left out | `libraries.json` (exact package versions) and the `licenses` folder in the app's `desktop/bundle` folder |
|
||||
|
||||
Frame Control starts KDE Connect and talks to it over its network protocol;
|
||||
it doesn't link to it or include its code.
|
||||
it doesn't link to it or include its code. On Windows, Frame Control's own
|
||||
`pc-host.dll` links to GStreamer dynamically; the libraries are unmodified.
|
||||
+10
-1
@@ -198,7 +198,16 @@
|
||||
"zip"
|
||||
],
|
||||
"icon": "build/icon.png",
|
||||
"artifactName": "Frame-Control-win-${arch}.${ext}"
|
||||
"artifactName": "Frame-Control-win-${arch}.${ext}",
|
||||
"extraResources": [
|
||||
{
|
||||
"from": "build/desktop/${os}-${arch}",
|
||||
"to": "desktop/bundle",
|
||||
"filter": [
|
||||
"**/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"nsis": {
|
||||
"oneClick": false,
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
// Client choices live outside the local server's changing URL/origin.
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const KEYS = ["mvSeparate", "padUsed"];
|
||||
const KEYS = ["mvSeparate", "padUsed", "pcView"];
|
||||
const valid = value => value === "0" || value === "1";
|
||||
|
||||
function preferenceStore(file) {
|
||||
|
||||
@@ -36,7 +36,7 @@ test("client choices survive a new native session and local server origin", t =>
|
||||
assert.deepEqual(first.read(), { mvSeparate: "1", padUsed: "1" });
|
||||
});
|
||||
|
||||
test("only the two non-sensitive binary choices can enter the native store", t => {
|
||||
test("only the non-sensitive binary choices can enter the native store", t => {
|
||||
const file = fixture(t), store = preferenceStore(file);
|
||||
for (const [key, value] of [["token", "secret"], ["__proto__", "1"], ["padUsed", true], ["mvSeparate", "yes"]]) {
|
||||
assert.throws(() => store.write(key, value), /Invalid preference/);
|
||||
@@ -58,3 +58,11 @@ test("a missing or damaged preference file retains usable defaults", t => {
|
||||
store.write("padUsed", "1");
|
||||
assert.deepEqual(store.read(), { padUsed: "1" });
|
||||
});
|
||||
|
||||
test("the PC in the headset opt-in is remembered and off unless saved", t => {
|
||||
const file = fixture(t), store = preferenceStore(file);
|
||||
assert.equal(store.read().pcView, undefined);
|
||||
store.write("pcView", "1");
|
||||
assert.deepEqual(preferenceStore(file).read(), { pcView: "1" });
|
||||
assert.throws(() => store.write("pcView", "on"), /Invalid preference/);
|
||||
});
|
||||
@@ -0,0 +1,311 @@
|
||||
{
|
||||
"label": "pc-agent-frame-arm64-hairpin",
|
||||
"date": "2026-09-28T22:36:19",
|
||||
"commit": "c5fc552+dirty",
|
||||
"config": {
|
||||
"quality": "balanced",
|
||||
"mode": "separate",
|
||||
"net": "none",
|
||||
"delay_ms": 0,
|
||||
"buffer_ms": 250,
|
||||
"host": "frame",
|
||||
"usb": false,
|
||||
"ssh_opts": [],
|
||||
"encoder": "",
|
||||
"duration_s": 10.0,
|
||||
"browser_flags": []
|
||||
},
|
||||
"frame_build": "20260925.6191901",
|
||||
"mac": "26.5.2",
|
||||
"host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)",
|
||||
"source": "",
|
||||
"pc_host": true,
|
||||
"headset": "0:03.056203 [Info] - 1 - leaving standby",
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario": "test",
|
||||
"frames_sent": 351,
|
||||
"frames_drawn": 351,
|
||||
"frames_shown": 324,
|
||||
"duration_s": 10.1,
|
||||
"stages_ms": {
|
||||
"capture": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.2,
|
||||
"p99": 0.5,
|
||||
"n": 351
|
||||
},
|
||||
"queue": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.3,
|
||||
"p99": 0.4,
|
||||
"n": 351
|
||||
},
|
||||
"encode": {
|
||||
"p50": 2.0,
|
||||
"p95": 2.9,
|
||||
"p99": 3.5,
|
||||
"n": 351
|
||||
},
|
||||
"socket": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.4,
|
||||
"p99": 0.6,
|
||||
"n": 351
|
||||
},
|
||||
"network": {
|
||||
"p50": 11.6,
|
||||
"p95": 75.7,
|
||||
"p99": 130.1,
|
||||
"n": 351
|
||||
},
|
||||
"decode": {
|
||||
"p50": 0.7,
|
||||
"p95": 2.1,
|
||||
"p99": 4.1,
|
||||
"n": 351
|
||||
},
|
||||
"draw": {
|
||||
"p50": 0.4,
|
||||
"p95": 0.8,
|
||||
"p99": 1.1,
|
||||
"n": 351
|
||||
},
|
||||
"present": {
|
||||
"p50": 0.8,
|
||||
"p95": 18.1,
|
||||
"p99": 24.9,
|
||||
"n": 324
|
||||
},
|
||||
"content": {
|
||||
"p50": 15.1,
|
||||
"p95": 80.3,
|
||||
"p99": 134.2,
|
||||
"n": 351
|
||||
},
|
||||
"content_shown": {
|
||||
"p50": 21.0,
|
||||
"p95": 71.8,
|
||||
"p99": 146.3,
|
||||
"n": 324
|
||||
}
|
||||
},
|
||||
"fps": 34.8,
|
||||
"fps_shown": 32.0,
|
||||
"late_pct": 53.41,
|
||||
"stall_max": 226.8,
|
||||
"stalls_over_100ms": 7,
|
||||
"mbps": 0.26,
|
||||
"keyframes": 6,
|
||||
"size": "1280x720",
|
||||
"captured": 607,
|
||||
"viewer_never_drawn": 0,
|
||||
"input_ms": {
|
||||
"p50": 43.3,
|
||||
"p95": 98.4,
|
||||
"p99": 110.1,
|
||||
"n": 18
|
||||
},
|
||||
"input_shown_ms": {
|
||||
"p50": 54.7,
|
||||
"p95": 98.9,
|
||||
"p99": 114.0,
|
||||
"n": 18
|
||||
},
|
||||
"input_parts_ms": {
|
||||
"uplink": {
|
||||
"p50": 17.1,
|
||||
"p95": 48.3,
|
||||
"p99": 53.1,
|
||||
"n": 18
|
||||
},
|
||||
"mac": {
|
||||
"p50": 12.5,
|
||||
"p95": 32.9,
|
||||
"p99": 35.7,
|
||||
"n": 18
|
||||
},
|
||||
"back": {
|
||||
"p50": 16.8,
|
||||
"p95": 50.0,
|
||||
"p99": 55.9,
|
||||
"n": 18
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"asked": 18,
|
||||
"sent": 18,
|
||||
"seen": 18
|
||||
},
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"fps": 38,
|
||||
"mbps": 0.3,
|
||||
"content_p50": 14.8,
|
||||
"content_p95": 25.3,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"fps": 33,
|
||||
"mbps": 0.24,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 103.5,
|
||||
"bitrate": 3845448,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"fps": 39,
|
||||
"mbps": 0.28,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 29.9,
|
||||
"bitrate": 5283790,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"fps": 25,
|
||||
"mbps": 0.21,
|
||||
"content_p50": 17.8,
|
||||
"content_p95": 175.9,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"fps": 34,
|
||||
"mbps": 0.22,
|
||||
"content_p50": 15.4,
|
||||
"content_p95": 69.8,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"fps": 38,
|
||||
"mbps": 0.29,
|
||||
"content_p50": 17.7,
|
||||
"content_p95": 68.7,
|
||||
"bitrate": 2764800,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"fps": 38,
|
||||
"mbps": 0.26,
|
||||
"content_p50": 15.0,
|
||||
"content_p95": 18.2,
|
||||
"bitrate": 3091280,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"fps": 30,
|
||||
"mbps": 0.22,
|
||||
"content_p50": 19.2,
|
||||
"content_p95": 111.0,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"fps": 35,
|
||||
"mbps": 0.26,
|
||||
"content_p50": 13.4,
|
||||
"content_p95": 102.6,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"fps": 38,
|
||||
"mbps": 0.28,
|
||||
"content_p50": 12.3,
|
||||
"content_p95": 20.9,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"fps": 3,
|
||||
"mbps": 0.02,
|
||||
"content_p50": 20.9,
|
||||
"content_p95": 24.7,
|
||||
"bitrate": 1725204,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
}
|
||||
],
|
||||
"adapt": [],
|
||||
"content_p50": 15.1,
|
||||
"content_p95": 80.3,
|
||||
"input_p50": 43.3,
|
||||
"input_p95": 98.4,
|
||||
"grades": {
|
||||
"input_replies": "local",
|
||||
"content_p50": "local",
|
||||
"content_p95": "acceptable",
|
||||
"input_p50": "local",
|
||||
"input_p95": "acceptable",
|
||||
"fps": "bad",
|
||||
"late_pct": "bad",
|
||||
"stall_max": "acceptable"
|
||||
},
|
||||
"controller": {
|
||||
"target": 3442081,
|
||||
"ceiling": 5529600,
|
||||
"tier": 0,
|
||||
"fps": 60,
|
||||
"scale": 1.0,
|
||||
"baseRtt": 13.334,
|
||||
"inFlight": 1,
|
||||
"slack": 71.738,
|
||||
"adapt": true
|
||||
},
|
||||
"events": [
|
||||
{
|
||||
"t": 3.77,
|
||||
"e": "target 2764800 bit/s; tier 0"
|
||||
},
|
||||
{
|
||||
"t": 7.42,
|
||||
"e": "target 1725204 bit/s; tier 0"
|
||||
}
|
||||
],
|
||||
"source_fps": 60.1,
|
||||
"cpu": {
|
||||
"host": "not sampled"
|
||||
},
|
||||
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 71 Hz",
|
||||
"show_s": 1.88,
|
||||
"panel": "valve.steam.desktopgame.2001932457",
|
||||
"src": "test",
|
||||
"route": "network"
|
||||
}
|
||||
],
|
||||
"proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input."
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
{
|
||||
"label": "pc-agent-frame-arm64-final",
|
||||
"date": "2026-09-28T22:55:27",
|
||||
"commit": "cd20243",
|
||||
"config": {
|
||||
"quality": "balanced",
|
||||
"mode": "native",
|
||||
"net": "none",
|
||||
"delay_ms": 0,
|
||||
"buffer_ms": 250,
|
||||
"host": "frame",
|
||||
"usb": false,
|
||||
"ssh_opts": [],
|
||||
"encoder": "",
|
||||
"duration_s": 10.0,
|
||||
"browser_flags": []
|
||||
},
|
||||
"frame_build": "20260925.6191901",
|
||||
"mac": "26.5.2",
|
||||
"host_platform": "Linux aarch64, SteamOS Frame, BUILD_ID 20260925.6191901 (synthetic host only)",
|
||||
"source": "",
|
||||
"pc_host": true,
|
||||
"headset": "0:03.056203 [Info] - 1 - leaving standby",
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario": "test",
|
||||
"frames_sent": 400,
|
||||
"frames_drawn": 400,
|
||||
"frames_shown": 339,
|
||||
"duration_s": 10.2,
|
||||
"stages_ms": {
|
||||
"capture": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.1,
|
||||
"p99": 0.2,
|
||||
"n": 400
|
||||
},
|
||||
"queue": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.2,
|
||||
"p99": 0.4,
|
||||
"n": 400
|
||||
},
|
||||
"encode": {
|
||||
"p50": 2.0,
|
||||
"p95": 2.7,
|
||||
"p99": 3.1,
|
||||
"n": 400
|
||||
},
|
||||
"socket": {
|
||||
"p50": 0.1,
|
||||
"p95": 0.3,
|
||||
"p99": 0.4,
|
||||
"n": 400
|
||||
},
|
||||
"network": {
|
||||
"p50": 11.8,
|
||||
"p95": 25.2,
|
||||
"p99": 37.8,
|
||||
"n": 400
|
||||
},
|
||||
"decode": {
|
||||
"p50": 0.7,
|
||||
"p95": 1.5,
|
||||
"p99": 2.0,
|
||||
"n": 400
|
||||
},
|
||||
"draw": {
|
||||
"p50": 0.3,
|
||||
"p95": 0.7,
|
||||
"p99": 1.0,
|
||||
"n": 400
|
||||
},
|
||||
"present": {
|
||||
"p50": 0.8,
|
||||
"p95": 19.3,
|
||||
"p99": 26.8,
|
||||
"n": 339
|
||||
},
|
||||
"content": {
|
||||
"p50": 15.1,
|
||||
"p95": 28.6,
|
||||
"p99": 41.3,
|
||||
"n": 400
|
||||
},
|
||||
"content_shown": {
|
||||
"p50": 17.6,
|
||||
"p95": 39.0,
|
||||
"p99": 47.2,
|
||||
"n": 339
|
||||
}
|
||||
},
|
||||
"fps": 39.3,
|
||||
"fps_shown": 33.4,
|
||||
"late_pct": 50.25,
|
||||
"stall_max": 69.3,
|
||||
"stalls_over_100ms": 0,
|
||||
"mbps": 0.35,
|
||||
"keyframes": 6,
|
||||
"size": "1280x720",
|
||||
"captured": 612,
|
||||
"viewer_never_drawn": 0,
|
||||
"input_ms": {
|
||||
"p50": 76.6,
|
||||
"p95": 98.2,
|
||||
"p99": 127.8,
|
||||
"n": 19
|
||||
},
|
||||
"input_shown_ms": {
|
||||
"p50": 80.5,
|
||||
"p95": 107.8,
|
||||
"p99": 128.1,
|
||||
"n": 16
|
||||
},
|
||||
"input_parts_ms": {
|
||||
"uplink": {
|
||||
"p50": 36.5,
|
||||
"p95": 57.6,
|
||||
"p99": 83.2,
|
||||
"n": 19
|
||||
},
|
||||
"mac": {
|
||||
"p50": 14.6,
|
||||
"p95": 31.3,
|
||||
"p99": 31.7,
|
||||
"n": 19
|
||||
},
|
||||
"back": {
|
||||
"p50": 15.1,
|
||||
"p95": 32.8,
|
||||
"p99": 36.2,
|
||||
"n": 19
|
||||
}
|
||||
},
|
||||
"inputs": {
|
||||
"asked": 19,
|
||||
"sent": 19,
|
||||
"seen": 19
|
||||
},
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"fps": 40,
|
||||
"mbps": 0.34,
|
||||
"content_p50": 11.7,
|
||||
"content_p95": 27.4,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 11.9,
|
||||
"content_p95": 28.2,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"fps": 41,
|
||||
"mbps": 0.35,
|
||||
"content_p50": 11.4,
|
||||
"content_p95": 16.6,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 12.7,
|
||||
"content_p95": 28.0,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"fps": 38,
|
||||
"mbps": 0.33,
|
||||
"content_p50": 15.3,
|
||||
"content_p95": 30.0,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"fps": 39,
|
||||
"mbps": 0.37,
|
||||
"content_p50": 16.7,
|
||||
"content_p95": 28.5,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"fps": 41,
|
||||
"mbps": 0.38,
|
||||
"content_p50": 15.9,
|
||||
"content_p95": 29.4,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"fps": 41,
|
||||
"mbps": 0.31,
|
||||
"content_p50": 15.6,
|
||||
"content_p95": 28.2,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"fps": 38,
|
||||
"mbps": 0.38,
|
||||
"content_p50": 15.5,
|
||||
"content_p95": 25.9,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"fps": 38,
|
||||
"mbps": 0.29,
|
||||
"content_p50": 16.7,
|
||||
"content_p95": 28.8,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"fps": 6,
|
||||
"mbps": 0.03,
|
||||
"content_p50": 17.6,
|
||||
"content_p95": 23.1,
|
||||
"bitrate": 5529600,
|
||||
"tier": 0,
|
||||
"w": 1280,
|
||||
"net": null
|
||||
}
|
||||
],
|
||||
"adapt": [],
|
||||
"content_p50": 15.1,
|
||||
"content_p95": 28.6,
|
||||
"input_p50": 76.6,
|
||||
"input_p95": 98.2,
|
||||
"grades": {
|
||||
"input_replies": "local",
|
||||
"content_p50": "local",
|
||||
"content_p95": "local",
|
||||
"input_p50": "acceptable",
|
||||
"input_p95": "acceptable",
|
||||
"fps": "bad",
|
||||
"late_pct": "bad",
|
||||
"stall_max": "local"
|
||||
},
|
||||
"controller": {
|
||||
"target": 5529600,
|
||||
"ceiling": 5529600,
|
||||
"tier": 0,
|
||||
"fps": 60,
|
||||
"scale": 1.0,
|
||||
"baseRtt": 12.332,
|
||||
"inFlight": 0,
|
||||
"slack": 74.12,
|
||||
"adapt": true
|
||||
},
|
||||
"events": [],
|
||||
"source_fps": 60.0,
|
||||
"cpu": {
|
||||
"host": "not sampled"
|
||||
},
|
||||
"viewer": "h264 software; ANGLE (Mesa, zink Vulkan 1.4(Turnip Adreno (TM) 750 (MESA_TURNIP)), OpenGL 4.6); raf 72 Hz",
|
||||
"show_s": 1.75,
|
||||
"panel": "valve.steam.desktopgame.2001932457",
|
||||
"src": "test",
|
||||
"route": "network"
|
||||
}
|
||||
],
|
||||
"proof": "Synthetic x264 PC agent runs on the Frame itself, forwarded via Mac and back. Not Windows/Linux desktop capture or laser input."
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build the common controller for the Python PC host (a C11 compiler needed)."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
root = Path(__file__).resolve().parent
|
||||
suffix = '.dll' if sys.platform == 'win32' else '.dylib' if sys.platform == 'darwin' else '.so'
|
||||
out = root / ('controller' + suffix)
|
||||
if sys.platform == 'win32' and os.environ.get('CC', 'cl') == 'cl':
|
||||
cmd = ['cl', '/nologo', '/O2', '/LD', '/std:c11', str(root / 'controller.c'), '/link', '/OUT:' + str(out)]
|
||||
else:
|
||||
cmd = [os.environ.get('CC', 'cc'), '-O2', '-std=c11', '-shared', '-fPIC', str(root / 'controller.c'), '-o', str(out)]
|
||||
subprocess.run(cmd, cwd=root, check=True)
|
||||
print(out)
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build our native PC adapter and bundle its ordinary shared libraries.
|
||||
|
||||
Run on the target architecture after installing GStreamer development packages.
|
||||
No GStreamer executable or desktop streaming application is shipped or invoked.
|
||||
Linux libc, display/GPU drivers and the portal service remain platform pieces.
|
||||
|
||||
x264 is GPL, so the bundle the Windows installer ships leaves it out: Windows
|
||||
encodes with Media Foundation. --with-x264 adds it for CI's headless protocol
|
||||
tests, which have no GPU or desktop to encode with.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def output(*args):
|
||||
return subprocess.check_output(args, text=True).strip()
|
||||
|
||||
|
||||
def build(destination, with_x264=False):
|
||||
win = sys.platform == 'win32'
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
modules = ['gstreamer-1.0', 'gstreamer-app-1.0', 'gstreamer-video-1.0']
|
||||
if not win:
|
||||
modules += ['gio-unix-2.0']
|
||||
import shlex
|
||||
flags = shlex.split(output('pkg-config', '--cflags', '--libs', *modules))
|
||||
lib = destination / ('pc-host.dll' if win else 'pc-host.so')
|
||||
cmd = [os.environ.get('CC', 'gcc' if win else 'cc'), '-std=c11', '-O2', '-shared', '-Wall', '-Wextra']
|
||||
if not win:
|
||||
cmd += ['-fPIC', '-Wl,-rpath,$ORIGIN/lib']
|
||||
subprocess.run(cmd + [str(ROOT / f) for f in ('controller.c', 'capture.c', 'portal.c')] + flags + ['-o', str(lib)], check=True)
|
||||
prefix = Path(output('pkg-config', '--variable=prefix', 'gstreamer-1.0'))
|
||||
plugin_dir = Path(output('pkg-config', '--variable=pluginsdir', 'gstreamer-1.0'))
|
||||
plugins_out, libs_out = destination / 'lib' / 'gstreamer-1.0', destination / ('bin' if win else 'lib')
|
||||
plugins_out.mkdir(parents=True, exist_ok=True)
|
||||
libs_out.mkdir(parents=True, exist_ok=True)
|
||||
names = ['coreelements', 'videotestsrc', 'videoconvertscale', 'app', 'jpeg', 'videoparsersbad']
|
||||
if with_x264:
|
||||
names.append('x264')
|
||||
names += ['d3d11', 'mediafoundation'] if win else ['pipewire', 'va']
|
||||
pending = [lib]
|
||||
for name in names:
|
||||
matches = list(plugin_dir.glob('*gst' + name + ('.dll' if win else '.so')))
|
||||
if not matches:
|
||||
raise SystemExit('Missing GStreamer library: ' + name)
|
||||
for source in matches:
|
||||
target = plugins_out / source.name
|
||||
shutil.copy2(source, target)
|
||||
pending.append(source)
|
||||
if not win:
|
||||
# PipeWire dynamically loads support and protocol modules. ldd cannot
|
||||
# discover those; bundle them with a private client-only config instead
|
||||
# of accidentally loading a different distro's client modules/config.
|
||||
libdir = plugin_dir.parent
|
||||
for relative in ('pipewire-0.3/libpipewire-module-protocol-native.so',
|
||||
'pipewire-0.3/libpipewire-module-client-node.so',
|
||||
'pipewire-0.3/libpipewire-module-adapter.so',
|
||||
'spa-0.2/support/libspa-support.so',
|
||||
'spa-0.2/support/libspa-dbus.so',
|
||||
'spa-0.2/videoconvert/libspa-videoconvert.so'):
|
||||
source = libdir / relative
|
||||
if not source.is_file():
|
||||
raise SystemExit('Missing PipeWire runtime library: ' + str(source))
|
||||
target = destination / 'lib' / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(source, target)
|
||||
pending.append(source)
|
||||
config = destination / 'share' / 'pipewire'
|
||||
config.mkdir(parents=True, exist_ok=True)
|
||||
(config / 'client.conf').write_text(
|
||||
'context.spa-libs = { support.* = support/libspa-support video.* = videoconvert/libspa-videoconvert }\n'
|
||||
'context.modules = [\n'
|
||||
' { name = libpipewire-module-protocol-native }\n'
|
||||
' { name = libpipewire-module-client-node }\n'
|
||||
' { name = libpipewire-module-adapter }\n'
|
||||
']\n')
|
||||
copied = set()
|
||||
while pending:
|
||||
binary = pending.pop()
|
||||
if win:
|
||||
imported = re.findall(r'DLL Name:\s*(\S+)', output('objdump', '-p', str(binary)))
|
||||
dependencies = [prefix / 'bin' / name for name in imported]
|
||||
else:
|
||||
dependencies = [Path(p) for p in re.findall(r'=>\s+(/\S+)', output('ldd', str(binary)))]
|
||||
for dep in dependencies:
|
||||
if not dep.is_file() or dep.name in copied:
|
||||
continue
|
||||
if not win and re.match(r'lib(c|m|dl|rt|pthread|resolv)\.so', dep.name):
|
||||
continue
|
||||
shutil.copy2(dep, libs_out / dep.name)
|
||||
copied.add(dep.name)
|
||||
pending.append(dep)
|
||||
# License texts and package provenance accompany the dynamically linked
|
||||
# libraries. Distribution builders keep the upstream package/source URLs.
|
||||
licenses = destination / 'licenses'
|
||||
licenses.mkdir(exist_ok=True)
|
||||
if win:
|
||||
source = prefix / 'share' / 'licenses'
|
||||
if source.exists():
|
||||
shutil.copytree(source, licenses, dirs_exist_ok=True)
|
||||
packages = output('pacman', '-Q') if shutil.which('pacman') else 'See MSYS2 build log'
|
||||
else:
|
||||
packages = output('dpkg-query', '-W', '-f=${Package} ${Version}\n') if shutil.which('dpkg-query') else ''
|
||||
# Debian copyright files contain licenses and source homepage details.
|
||||
for path in Path('/usr/share/doc').glob('*/copyright'):
|
||||
shutil.copy2(path, licenses / (path.parent.name + '.copyright'))
|
||||
(destination / 'libraries.json').write_text(json.dumps(dict(gstreamer=output('pkg-config', '--modversion', 'gstreamer-1.0'),
|
||||
plugins=names, libraries=sorted(copied), packages=packages), indent=2))
|
||||
print(lib)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--out', type=Path, default=ROOT / 'bundle')
|
||||
parser.add_argument('--with-x264', action='store_true', help='add the GPL x264 encoder (CI tests only, never shipped)')
|
||||
args = parser.parse_args()
|
||||
build(args.out.resolve(), args.with_x264)
|
||||
@@ -0,0 +1,125 @@
|
||||
/* Frame Control's capture/encode adapter. GStreamer is a bundled library;
|
||||
* capture uses WGC on Windows and the consented PipeWire fd on Linux. */
|
||||
#include "controller.h"
|
||||
#include <gst/gst.h>
|
||||
#include <gst/app/gstappsink.h>
|
||||
#include <gst/video/video-event.h>
|
||||
#include <string.h>
|
||||
|
||||
typedef int (*Gate)(int stage, int64_t pts, int64_t capture, int64_t arrived);
|
||||
typedef struct {
|
||||
GstElement *pipeline, *encoder, *sink, *raw_queue;
|
||||
Gate gate;
|
||||
GstSample *sample;
|
||||
GstMapInfo map;
|
||||
int mapped;
|
||||
char error[512];
|
||||
} Capture;
|
||||
typedef struct { const unsigned char *data; int size, key, width, height; int64_t pts; } Encoded;
|
||||
FC_API int64_t fc_now(void) {return g_get_monotonic_time();}
|
||||
FC_API void fc_gst_init(void) {gst_init(NULL,NULL);}
|
||||
FC_API int fc_has_element(const char *name) {
|
||||
GstElementFactory *f=gst_element_factory_find(name);
|
||||
if(!f)return 0;gst_object_unref(f);return 1;
|
||||
}
|
||||
static GstPadProbeReturn probe(GstPad *pad,GstPadProbeInfo *info,gpointer data) {
|
||||
Capture *c=data; GstBuffer *b=GST_PAD_PROBE_INFO_BUFFER(info);
|
||||
if(!b)return GST_PAD_PROBE_OK;
|
||||
int64_t now=fc_now(), cap=now;
|
||||
GstClock *clock=gst_element_get_clock(c->pipeline);
|
||||
if(clock && GST_BUFFER_PTS_IS_VALID(b)) {
|
||||
GstClockTime current=gst_clock_get_time(clock),origin=gst_element_get_base_time(c->pipeline);
|
||||
if(current>=origin && current-origin>=GST_BUFFER_PTS(b))
|
||||
cap-= (int64_t)((current-origin-GST_BUFFER_PTS(b))/1000);
|
||||
}
|
||||
if(clock)gst_object_unref(clock);
|
||||
int stage=GPOINTER_TO_INT(g_object_get_data(G_OBJECT(pad),"stage"));
|
||||
if(stage==1) return c->gate(stage,(int64_t)GST_BUFFER_PTS(b),cap,now)>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP;
|
||||
int phase=0;
|
||||
for(;;) {
|
||||
/* While congested, prefer a newer raw picture queued upstream. If
|
||||
* nothing changed, retain this last picture until the gate opens;
|
||||
* an idle window must not stay stale after a dropped final update. */
|
||||
guint queued=0;
|
||||
if(phase && c->raw_queue)g_object_get(c->raw_queue,"current-level-buffers",&queued,NULL);
|
||||
if(queued)return GST_PAD_PROBE_DROP;
|
||||
int decision=c->gate(phase,(int64_t)GST_BUFFER_PTS(b),cap,now);
|
||||
if(decision)return decision>0 ? GST_PAD_PROBE_OK : GST_PAD_PROBE_DROP;
|
||||
phase=2;g_usleep(2000);
|
||||
}
|
||||
}
|
||||
FC_API Capture *fc_capture_open(const char *pipeline,Gate gate,char *error,int capacity) {
|
||||
GError *e=NULL;Capture *c=g_new0(Capture,1);c->gate=gate;
|
||||
c->pipeline=gst_parse_launch(pipeline,&e);
|
||||
if(e || !c->pipeline) {
|
||||
g_strlcpy(error,e?e->message:"No pipeline",capacity);
|
||||
if(e)g_error_free(e);if(c->pipeline)gst_object_unref(c->pipeline);g_free(c);return NULL;
|
||||
}
|
||||
c->encoder=gst_bin_get_by_name(GST_BIN(c->pipeline),"enc");
|
||||
c->sink=gst_bin_get_by_name(GST_BIN(c->pipeline),"out");
|
||||
GstElement *raw=gst_bin_get_by_name(GST_BIN(c->pipeline),"gate");
|
||||
if(!c->encoder || !c->sink || !raw) {
|
||||
g_strlcpy(error,"Pipeline is missing enc, out or gate",capacity);
|
||||
if(raw)gst_object_unref(raw);if(c->encoder)gst_object_unref(c->encoder);
|
||||
if(c->sink)gst_object_unref(c->sink);gst_object_unref(c->pipeline);g_free(c);return NULL;
|
||||
}
|
||||
c->raw_queue=gst_bin_get_by_name(GST_BIN(c->pipeline),"raw_queue");
|
||||
GstPad *p=gst_element_get_static_pad(raw,"src");
|
||||
g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(0));
|
||||
gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p);gst_object_unref(raw);
|
||||
p=gst_element_get_static_pad(c->encoder,"sink");
|
||||
g_object_set_data(G_OBJECT(p),"stage",GINT_TO_POINTER(1));
|
||||
gst_pad_add_probe(p,GST_PAD_PROBE_TYPE_BUFFER,probe,c,NULL);gst_object_unref(p);
|
||||
gst_element_set_state(c->pipeline,GST_STATE_PLAYING);
|
||||
return c;
|
||||
}
|
||||
FC_API int fc_capture_pull(Capture *c,Encoded *out) {
|
||||
if(c->mapped) {gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map);c->mapped=0;}
|
||||
if(c->sample){gst_sample_unref(c->sample);c->sample=NULL;}
|
||||
GstBus *bus=gst_element_get_bus(c->pipeline);
|
||||
GstMessage *m=gst_bus_pop_filtered(bus,GST_MESSAGE_ERROR);gst_object_unref(bus);
|
||||
if(m) {
|
||||
if(GST_MESSAGE_TYPE(m)==GST_MESSAGE_ERROR) {
|
||||
GError *e=NULL;char *debug=NULL;gst_message_parse_error(m,&e,&debug);
|
||||
g_strlcpy(c->error,e->message,sizeof(c->error));g_error_free(e);g_free(debug);
|
||||
} else g_strlcpy(c->error,"The capture source closed",sizeof(c->error));
|
||||
gst_message_unref(m);return -1;
|
||||
}
|
||||
c->sample=gst_app_sink_try_pull_sample(GST_APP_SINK(c->sink),100*GST_MSECOND);
|
||||
if(!c->sample) {
|
||||
if(gst_app_sink_is_eos(GST_APP_SINK(c->sink))) {
|
||||
g_strlcpy(c->error,"The capture source closed",sizeof(c->error));return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
GstBuffer *b=gst_sample_get_buffer(c->sample);
|
||||
if(!gst_buffer_map(b,&c->map,GST_MAP_READ))return 0;
|
||||
c->mapped=1;out->data=c->map.data;out->size=(int)c->map.size;
|
||||
out->key=!GST_BUFFER_FLAG_IS_SET(b,GST_BUFFER_FLAG_DELTA_UNIT);out->pts=(int64_t)GST_BUFFER_PTS(b);
|
||||
const GstStructure *s=gst_caps_get_structure(gst_sample_get_caps(c->sample),0);
|
||||
gst_structure_get_int(s,"width",&out->width);gst_structure_get_int(s,"height",&out->height);return 1;
|
||||
}
|
||||
FC_API const char *fc_capture_error(Capture *c) {return c->error;}
|
||||
FC_API void fc_capture_bitrate(Capture *c,int bps) {
|
||||
g_object_set(c->encoder,"bitrate",(guint)MAX(1,bps/1000),NULL);
|
||||
}
|
||||
FC_API void fc_capture_test(Capture *c,uint32_t input) {
|
||||
GstElement *source=gst_bin_get_by_name(GST_BIN(c->pipeline),"source");
|
||||
if(source) {
|
||||
/* Each benchmark click visibly changes the ball, before injection is
|
||||
* timestamped. This is a response, not merely a protocol input echo. */
|
||||
guint color=0xff000000u | ((input*2654435761u)&0x00ffffffu);
|
||||
g_object_set(source,"foreground-color",color,NULL);gst_object_unref(source);
|
||||
}
|
||||
}
|
||||
FC_API void fc_capture_key(Capture *c) {
|
||||
GstPad *p=gst_element_get_static_pad(c->encoder,"src");
|
||||
gst_pad_send_event(p,gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE,TRUE,0));gst_object_unref(p);
|
||||
}
|
||||
FC_API void fc_capture_close(Capture *c) {
|
||||
gst_element_set_state(c->pipeline,GST_STATE_NULL);
|
||||
if(c->mapped)gst_buffer_unmap(gst_sample_get_buffer(c->sample),&c->map);
|
||||
if(c->sample)gst_sample_unref(c->sample);
|
||||
if(c->raw_queue)gst_object_unref(c->raw_queue);
|
||||
gst_object_unref(c->sink);gst_object_unref(c->encoder);gst_object_unref(c->pipeline);g_free(c);
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
/* A C port of the Mac agent's RateController (mac/frame-mac-view/Sources/
|
||||
* Controller.swift) for the Windows host. Same gate, demand protection, bitrate
|
||||
* recovery and tier hysteresis; bounded sample buffers. The Mac agent keeps its
|
||||
* Swift original, so a change to either must be made to both:
|
||||
* tests/test_pc_controller.py checks this one against the Swift trace. */
|
||||
#include "controller.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#define CAP 1024
|
||||
#define MIN(a,b) ((a)<(b)?(a):(b))
|
||||
#define MAX(a,b) ((a)>(b)?(a):(b))
|
||||
typedef struct { int64_t t, v; uint32_t seq; } Sample;
|
||||
typedef struct { Sample a[CAP]; int n; } Samples;
|
||||
struct FCController {
|
||||
int max_fps, enabled, ceiling, target, tier, frame_bytes, held, signal, saw_ack;
|
||||
int64_t decrease, increase, below, above, slack;
|
||||
Samples flight, rtts, acked, sent, captures;
|
||||
};
|
||||
static const int fps[] = {60,45,30,30,30}, scale[] = {100,100,100,75,50};
|
||||
static const double floors[] = {.45,.28,.16,.08,0};
|
||||
static void remove_first(Samples *s, int n) {
|
||||
s->n -= n; memmove(s->a, s->a+n, (size_t)s->n*sizeof(Sample));
|
||||
}
|
||||
static void add(Samples *s, Sample v, int cap) {
|
||||
if (s->n >= cap) remove_first(s, 1);
|
||||
s->a[s->n++] = v;
|
||||
}
|
||||
static void expire(Samples *s, int64_t oldest) {
|
||||
int n=0; while(n<s->n && s->a[n].t<oldest) n++;
|
||||
remove_first(s,n);
|
||||
}
|
||||
static int64_t base(FCController *c) {
|
||||
int64_t v=0;
|
||||
for(int i=0;i<c->rtts.n;i++) if(!i || c->rtts.a[i].v<v) v=c->rtts.a[i].v;
|
||||
return v;
|
||||
}
|
||||
static int compare(const void *a, const void *b) {
|
||||
int64_t x=*(const int64_t*)a, y=*(const int64_t*)b;
|
||||
return (x>y)-(x<y);
|
||||
}
|
||||
static int64_t quantile(Samples *s,int64_t since,int numerator,int denominator) {
|
||||
int64_t a[CAP]; int n=0;
|
||||
for(int i=0;i<s->n;i++) if(s->a[i].t>since) a[n++]=s->a[i].v;
|
||||
if(!n) return 0;
|
||||
qsort(a,(size_t)n,sizeof(int64_t),compare);
|
||||
return a[n*numerator/denominator];
|
||||
}
|
||||
static int rate(Samples *s) {
|
||||
int64_t total=0; for(int i=0;i<s->n;i++) total+=s->a[i].v;
|
||||
return (int)MIN(total*16,2147483647);
|
||||
}
|
||||
FCController *fc_new(int max_fps,int enabled) {
|
||||
FCController *c=calloc(1,sizeof(*c));
|
||||
if(c) {c->max_fps=MAX(1,max_fps);c->enabled=enabled;c->slack=40000;}
|
||||
return c;
|
||||
}
|
||||
void fc_free(FCController *c) {free(c);}
|
||||
void fc_ceiling(FCController *c,int bps) {
|
||||
if(!c->target || c->target>bps) c->target=bps;
|
||||
c->ceiling=bps;
|
||||
}
|
||||
int fc_gate(FCController *c,int64_t now,int counts) {
|
||||
if(!c->enabled || !c->saw_ack) return 1;
|
||||
expire(&c->flight,now-2000000);
|
||||
if(!c->flight.n) return 1;
|
||||
int64_t interval=1000000/MIN(c->max_fps,fps[c->tier]);
|
||||
int window=MAX(3,(int)((base(c)+c->slack)/interval)+1);
|
||||
if(c->flight.n<window && now-c->flight.a[0].t<=base(c)+c->slack) return 1;
|
||||
if(counts) c->held++;
|
||||
return 0;
|
||||
}
|
||||
void fc_capture(FCController *c,int64_t now) {add(&c->captures,(Sample){now,0,0},256);}
|
||||
void fc_sent(FCController *c,uint32_t seq,int bytes,int64_t now) {
|
||||
Sample s={now,bytes,seq};add(&c->flight,s,512);add(&c->sent,s,CAP);
|
||||
}
|
||||
int fc_ack(FCController *c,uint32_t seq,int64_t now) {
|
||||
c->saw_ack=1;
|
||||
for(int i=0;i<c->flight.n;i++) if(c->flight.a[i].seq==seq) {
|
||||
Sample s=c->flight.a[i];remove_first(&c->flight,i+1);
|
||||
add(&c->rtts,(Sample){now,now-s.t,0},CAP);
|
||||
add(&c->acked,(Sample){now,s.v,0},CAP);return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
int fc_update(FCController *c,int64_t now) {
|
||||
expire(&c->rtts,now-10000000);expire(&c->acked,now-500000);
|
||||
expire(&c->sent,now-500000);expire(&c->flight,now-2000000);
|
||||
expire(&c->captures,now-1000000);
|
||||
if(!c->enabled || c->ceiling<=0) return 0;
|
||||
int64_t baseline=base(c), spread=quantile(&c->rtts,now-2000000,9,10);
|
||||
int64_t jitter=spread ? spread-baseline : 0;
|
||||
c->slack=1000000/MIN(c->max_fps,fps[c->tier])+MIN(MAX(jitter*3/2,25000),80000);
|
||||
int64_t recent=quantile(&c->rtts,now-300000,1,2);
|
||||
int64_t queue=recent ? recent-baseline : 0;
|
||||
int64_t age=c->flight.n ? now-c->flight.a[0].t : 0;
|
||||
int delivered=rate(&c->acked),sending=rate(&c->sent);
|
||||
if(c->sent.n) c->frame_bytes=sending/16/c->sent.n;
|
||||
int demand=(int)MIN((int64_t)MIN(c->captures.n,MIN(c->max_fps,fps[c->tier]))*c->frame_bytes*8,2147483647);
|
||||
int signal=(sending>=c->target/2 && queue>40000)||c->held>=3||age>baseline+100000;
|
||||
int congested=signal && c->signal;c->signal=signal;c->held=0;
|
||||
if(congested && now-c->decrease>300000) {
|
||||
int next=MAX(300000,MIN((int64_t)c->target*4/5,MAX((int64_t)delivered*9/10,c->target/2)));
|
||||
if(demand>0 && (int64_t)demand*2<=(int64_t)c->target*5/4) next=MAX(next,MIN(c->target,(int64_t)demand*2));
|
||||
c->target=next;c->decrease=now;
|
||||
} else if(!congested && now-c->decrease>1000000 && now-c->increase>250000 && c->target<c->ceiling &&
|
||||
(sending>(int64_t)c->target*6/10 || now-c->decrease>3000000)) {
|
||||
c->target=MIN(c->ceiling,(int64_t)(c->target*1.1)+50000);c->increase=now;
|
||||
}
|
||||
double share=(double)c->target/MAX(c->ceiling,1);
|
||||
if(c->tier<4 && share<floors[c->tier] && sending>=(int64_t)c->target*7/10) {
|
||||
if(!c->below)c->below=now;
|
||||
if(now-c->below>500000) {
|
||||
for(c->tier=0;c->tier<4 && share<floors[c->tier];c->tier++) {}
|
||||
c->below=0;
|
||||
}
|
||||
} else c->below=0;
|
||||
if(c->tier>0 && share>floors[c->tier-1]*1.25) {
|
||||
if(!c->above)c->above=now;
|
||||
if(now-c->above>2000000) {c->tier--;c->above=0;}
|
||||
} else c->above=0;
|
||||
return c->target;
|
||||
}
|
||||
int64_t fc_value(FCController *c,int field) {
|
||||
switch(field) {
|
||||
case 0:return c->target;case 1:return c->ceiling;case 2:return c->tier;
|
||||
case 3:return MIN(c->max_fps,fps[c->tier]);case 4:return scale[c->tier];
|
||||
case 5:return base(c);case 6:return c->flight.n;case 7:return c->slack;
|
||||
default:return 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
/* Shared latency controller. Times are monotonic host microseconds.
|
||||
* Callers serialize access. Capture is gated BEFORE encoding: dropping an
|
||||
* encoded inter frame would break the decoder's reference chain. */
|
||||
#ifndef FRAME_CONTROLLER_H
|
||||
#define FRAME_CONTROLLER_H
|
||||
#include <stdint.h>
|
||||
#ifdef _WIN32
|
||||
#define FC_API __declspec(dllexport)
|
||||
#else
|
||||
#define FC_API
|
||||
#endif
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
typedef struct FCController FCController;
|
||||
FC_API FCController *fc_new(int fps, int enabled);
|
||||
FC_API void fc_free(FCController *c);
|
||||
FC_API void fc_ceiling(FCController *c, int bps);
|
||||
FC_API int fc_gate(FCController *c, int64_t now, int counts);
|
||||
FC_API void fc_capture(FCController *c, int64_t now);
|
||||
FC_API void fc_sent(FCController *c, uint32_t seq, int bytes, int64_t now);
|
||||
FC_API int fc_ack(FCController *c, uint32_t seq, int64_t now);
|
||||
FC_API int fc_update(FCController *c, int64_t now);
|
||||
/* target, ceiling, tier, fps, scale percent, baseline us, in flight, slack us */
|
||||
FC_API int64_t fc_value(FCController *c, int field);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,147 @@
|
||||
/* xdg-desktop-portal RemoteDesktop + ScreenCast, one consented session per
|
||||
* panel. PipeWire receives the portal fd, never the unrestricted daemon fd.
|
||||
* Input uses only the devices granted in Start's response. */
|
||||
#ifndef _WIN32
|
||||
#include "controller.h"
|
||||
#include <gio/gio.h>
|
||||
#include <gio/gunixfdlist.h>
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
#define BUS "org.freedesktop.portal.Desktop"
|
||||
#define PATH "/org/freedesktop/portal/desktop"
|
||||
#define RD "org.freedesktop.portal.RemoteDesktop"
|
||||
#define SC "org.freedesktop.portal.ScreenCast"
|
||||
typedef struct {
|
||||
GDBusConnection *bus;
|
||||
GMainContext *context;
|
||||
char *session;
|
||||
int fd;
|
||||
unsigned node, devices;
|
||||
int width,height;
|
||||
} Portal;
|
||||
typedef struct {GVariant *result;int done;unsigned code;} Response;
|
||||
static void response(GDBusConnection *c,const char *sender,const char *path,const char *iface,
|
||||
const char *signal,GVariant *params,gpointer data) {
|
||||
(void)c;(void)sender;(void)path;(void)iface;(void)signal;
|
||||
Response *r=data;g_variant_get(params,"(u@a{sv})",&r->code,&r->result);r->done=1;
|
||||
}
|
||||
static void option(GVariantBuilder *b,const char *key,GVariant *v) {g_variant_builder_add(b,"{sv}",key,v);}
|
||||
static GVariant *request(Portal *p,const char *iface,const char *method,GVariant *args,
|
||||
const char *token,char *error,int capacity) {
|
||||
char *sender=g_strdup(g_dbus_connection_get_unique_name(p->bus)+1);
|
||||
for(char *c=sender;*c;c++)if(*c=='.')*c='_';
|
||||
char *path=g_strdup_printf(PATH "/request/%s/%s",sender,token);g_free(sender);
|
||||
Response r={0};GError *e=NULL;
|
||||
guint sub=g_dbus_connection_signal_subscribe(p->bus,BUS,"org.freedesktop.portal.Request","Response",
|
||||
path,NULL,G_DBUS_SIGNAL_FLAGS_NONE,response,&r,NULL);
|
||||
GVariant *reply=g_dbus_connection_call_sync(p->bus,BUS,PATH,iface,method,args,G_VARIANT_TYPE("(o)"),
|
||||
G_DBUS_CALL_FLAGS_NONE,10000,NULL,&e);
|
||||
if(reply)g_variant_unref(reply);
|
||||
int64_t deadline=g_get_monotonic_time()+120000000;
|
||||
while(!e && !r.done && g_get_monotonic_time()<deadline) {
|
||||
while(g_main_context_iteration(p->context,FALSE)) {}
|
||||
g_usleep(10000);
|
||||
}
|
||||
if(!r.done) {
|
||||
GVariant *closed=g_dbus_connection_call_sync(p->bus,BUS,path,"org.freedesktop.portal.Request","Close",
|
||||
NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL);
|
||||
if(closed)g_variant_unref(closed);
|
||||
}
|
||||
g_dbus_connection_signal_unsubscribe(p->bus,sub);g_free(path);
|
||||
if(e || !r.done || r.code) {
|
||||
g_strlcpy(error,e?e->message:!r.done?"Screen sharing request timed out":"Screen sharing was cancelled or refused",capacity);
|
||||
if(e)g_error_free(e);if(r.result)g_variant_unref(r.result);return NULL;
|
||||
}
|
||||
return r.result;
|
||||
}
|
||||
FC_API void fc_portal_close(Portal *p) {
|
||||
if(!p)return;
|
||||
if(p->session && p->bus) {
|
||||
GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,p->session,"org.freedesktop.portal.Session","Close",
|
||||
NULL,NULL,G_DBUS_CALL_FLAGS_NONE,2000,NULL,NULL);
|
||||
if(r)g_variant_unref(r);
|
||||
}
|
||||
if(p->fd>=0){close(p->fd);p->fd=-1;}
|
||||
g_free(p->session);if(p->bus)g_object_unref(p->bus);
|
||||
if(p->context)g_main_context_unref(p->context);g_free(p);
|
||||
}
|
||||
/* Each pipeline gets a fresh restricted PipeWire connection. A dup of a
|
||||
* previously consumed protocol socket is not a new connection.
|
||||
* Ownership: the Portal owns p->fd for its whole life and is its only closer.
|
||||
* pipewiresrc never takes the fd it is given: its core connects with
|
||||
* pw_context_connect_fd(ctx, fcntl(fd, F_DUPFD_CLOEXEC, 3), ...) and that
|
||||
* duplicate is what PipeWire closes on teardown (src/gst/gstpipewirecore.c,
|
||||
* unchanged from 0.3.19 through 1.x). So closing p->fd here is not a double
|
||||
* close; not closing it would leak one socket per pipeline reopen. The
|
||||
* caller closes the previous pipeline before asking for a new fd. */
|
||||
FC_API int fc_portal_refresh(Portal *p,char *error,int capacity) {
|
||||
GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
GUnixFDList *fds=NULL;GError *e=NULL;
|
||||
GVariant *r=g_dbus_connection_call_with_unix_fd_list_sync(p->bus,BUS,PATH,SC,"OpenPipeWireRemote",
|
||||
g_variant_new("(oa{sv})",p->session,&b),G_VARIANT_TYPE("(h)"),G_DBUS_CALL_FLAGS_NONE,10000,NULL,&fds,NULL,&e);
|
||||
if(!r){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;}
|
||||
int handle;g_variant_get(r,"(h)",&handle);g_variant_unref(r);
|
||||
int fd=g_unix_fd_list_get(fds,handle,&e);g_object_unref(fds);
|
||||
if(fd<0){g_strlcpy(error,e->message,capacity);g_error_free(e);return -1;}
|
||||
if(p->fd>=0)close(p->fd);
|
||||
p->fd=fd;return fd;
|
||||
}
|
||||
FC_API Portal *fc_portal_select(char *error,int capacity) {
|
||||
Portal *p=g_new0(Portal,1);p->fd=-1;p->context=g_main_context_new();
|
||||
g_main_context_push_thread_default(p->context);
|
||||
GError *e=NULL;GVariant *r=NULL;
|
||||
p->bus=g_bus_get_sync(G_BUS_TYPE_SESSION,NULL,&e);
|
||||
if(!p->bus) {g_strlcpy(error,e->message,capacity);g_error_free(e);goto fail;}
|
||||
GVariantBuilder b;char token[64],session[64];
|
||||
g_snprintf(session,sizeof(session),"fc_%u",g_random_int());
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());
|
||||
g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"session_handle_token",g_variant_new_string(session));
|
||||
r=request(p,RD,"CreateSession",g_variant_new("(a{sv})",&b),token,error,capacity);if(!r)goto fail;
|
||||
g_variant_lookup(r,"session_handle","s",&p->session);g_variant_unref(r);r=NULL;
|
||||
if(!p->session){g_strlcpy(error,"Portal did not return a session",capacity);goto fail;}
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3));
|
||||
r=request(p,RD,"SelectDevices",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r);
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));option(&b,"types",g_variant_new_uint32(3));
|
||||
option(&b,"multiple",g_variant_new_boolean(FALSE));option(&b,"cursor_mode",g_variant_new_uint32(2));
|
||||
r=request(p,SC,"SelectSources",g_variant_new("(oa{sv})",p->session,&b),token,error,capacity);if(!r)goto fail;g_variant_unref(r);
|
||||
g_snprintf(token,sizeof(token),"fc_%u",g_random_int());g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
option(&b,"handle_token",g_variant_new_string(token));
|
||||
r=request(p,RD,"Start",g_variant_new("(osa{sv})",p->session,"",&b),token,error,capacity);if(!r)goto fail;
|
||||
g_variant_lookup(r,"devices","u",&p->devices);
|
||||
GVariant *streams=g_variant_lookup_value(r,"streams",G_VARIANT_TYPE("a(ua{sv})"));
|
||||
if(streams && g_variant_n_children(streams)>0) {
|
||||
GVariant *entry=g_variant_get_child_value(streams,0),*props=NULL;
|
||||
g_variant_get(entry,"(u@a{sv})",&p->node,&props);
|
||||
g_variant_lookup(props,"size","(ii)",&p->width,&p->height);
|
||||
g_variant_unref(props);g_variant_unref(entry);
|
||||
}
|
||||
if(streams)g_variant_unref(streams);g_variant_unref(r);r=NULL;
|
||||
if(!p->node || p->width<=0 || p->height<=0) {g_strlcpy(error,"Portal returned no stream size",capacity);goto fail;}
|
||||
if(fc_portal_refresh(p,error,capacity)<0)goto fail;
|
||||
g_main_context_pop_thread_default(p->context);return p;
|
||||
fail:
|
||||
g_main_context_pop_thread_default(p->context);fc_portal_close(p);return NULL;
|
||||
}
|
||||
FC_API int fc_portal_value(Portal *p,int field) {
|
||||
switch(field){case 0:return p->fd;case 1:return (int)p->node;case 2:return p->width;
|
||||
case 3:return p->height;case 4:return (int)p->devices;default:return 0;}
|
||||
}
|
||||
FC_API int fc_portal_input(Portal *p,int type,double x,double y,int code,int down) {
|
||||
const char *method=NULL;GVariant *args=NULL;GVariantBuilder b;g_variant_builder_init(&b,G_VARIANT_TYPE_VARDICT);
|
||||
if(type==0 && (p->devices&2)) {
|
||||
method="NotifyPointerMotionAbsolute";args=g_variant_new("(oa{sv}udd)",p->session,&b,p->node,x,y);
|
||||
} else if(type==1 && (p->devices&2)) {
|
||||
method="NotifyPointerButton";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down);
|
||||
} else if(type==2 && (p->devices&2)) {
|
||||
method="NotifyPointerAxis";args=g_variant_new("(oa{sv}dd)",p->session,&b,x,y);
|
||||
} else if(type==3 && (p->devices&1)) {
|
||||
method="NotifyKeyboardKeysym";args=g_variant_new("(oa{sv}iu)",p->session,&b,code,(guint)down);
|
||||
} else return 0;
|
||||
GError *e=NULL;GVariant *r=g_dbus_connection_call_sync(p->bus,BUS,PATH,RD,method,args,NULL,
|
||||
G_DBUS_CALL_FLAGS_NONE,2000,NULL,&e);
|
||||
if(e)g_error_free(e);if(r)g_variant_unref(r);return r!=NULL;
|
||||
}
|
||||
#endif
|
||||
@@ -4,7 +4,8 @@ Frame Control can show any Mac window, or a whole Mac screen, as its own panel
|
||||
in the Steam Frame. You place each panel anywhere in the room with the SteamVR
|
||||
dashboard. The laser clicks and drags, the thumbstick scrolls, and you type on
|
||||
the Mac's own keyboard. Find it under **Tools → Mac in the headset** (macOS
|
||||
only).
|
||||
only). On Windows there's an experimental port,
|
||||
[PC in the headset](pc-in-headset.md).
|
||||
|
||||
The confidence labels are the same as in [ssh.md](ssh.md).
|
||||
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
# PC in the headset (Experimental)
|
||||
|
||||
**Tools → PC in the headset**, on Windows only. It's the Windows version of
|
||||
[Mac in the headset](mac-in-headset.md): a window or a whole screen on your PC
|
||||
becomes its own SteamVR panel in the Frame. Place it with the dashboard's Float
|
||||
in World, Move and Size, point and click with the laser, scroll with the
|
||||
thumbstick, and type on the PC's keyboard.
|
||||
|
||||
**It's experimental and not yet tried on a real Windows desktop or with a
|
||||
headset.** It builds and passes its protocol tests on Windows in CI, but desktop
|
||||
capture, Media Foundation encoding and laser input have never run on a real
|
||||
PC. Expect it not to work yet. Reports are welcome through the
|
||||
[feedback form](https://frame-control.pages.dev/feedback/).
|
||||
|
||||
## Turning it on
|
||||
|
||||
It's off by default. Until you tick **Turn on PC in the headset** on the card,
|
||||
Frame Control starts no helper, loads none of the streaming libraries and lists
|
||||
no windows. The choice is remembered on this computer. Turning it off ends
|
||||
every stream, closes the viewers in the headset and stops the helper.
|
||||
|
||||
The card and its Tools link only appear in the Windows app. On macOS the card
|
||||
is Mac in the headset, as before; on Linux and the iPhone app there's no card.
|
||||
|
||||
## How it works
|
||||
|
||||
The ported pieces are from [PR #38](https://github.com/saphid/frame-control/pull/38).
|
||||
Frame Control owns the host helper, input, authentication, streaming protocol,
|
||||
panel launcher and adaptation. It doesn't need Sunshine, OBS or any other
|
||||
streaming app. GStreamer is bundled as ordinary libraries.
|
||||
|
||||
| | Windows |
|
||||
|---|---|
|
||||
| Capture | Windows.Graphics.Capture, through GStreamer's `d3d11screencapturesrc capture-api=wgc`, by window (HWND) or screen (HMONITOR) |
|
||||
| Encoding | Media Foundation's H.264 encoder (`mfh264enc`), low latency, no B-frames. Whether it uses the GPU depends on the PC; **not verified** |
|
||||
| Input | `SendInput`, mapped through the source's bounds, with per-monitor DPI awareness |
|
||||
|
||||
API choices are documented, not device-verified:
|
||||
[Windows capture](https://learn.microsoft.com/en-us/windows/uwp/audio-video-camera/screen-capture),
|
||||
[GStreamer WGC](https://gstreamer.freedesktop.org/documentation/d3d11/d3d11screencapturesrc.html),
|
||||
[Media Foundation encoder](https://gstreamer.freedesktop.org/documentation/mediafoundation/mfh264enc.html).
|
||||
|
||||
It needs Windows 10 or 11 with Windows.Graphics.Capture and a Media Foundation
|
||||
H.264 encoder (Windows "N" editions need the Media Feature Pack). Clicks and
|
||||
keys can't reach administrator apps or the sign-in screen, which an ordinary
|
||||
process can't drive. If Windows won't bring the chosen window to the front,
|
||||
input is refused rather than sent to the app covering it: bring the window
|
||||
forward, then Stop and Show again. Minimised or closed windows stop sending
|
||||
pictures. Protected video isn't supported. Mac in the headset's "own display
|
||||
per window" is Mac-only.
|
||||
|
||||
The pieces it shares with Mac in the headset:
|
||||
|
||||
- `ui/frame_macview.py` owns the SSH tunnel, reconnects, quality presets and
|
||||
panel launch. `ui/frame_pcview.py` picks the Windows helper
|
||||
(`ui/frame_pc_agent.py`, `ui/frame_pc_capture.py`, `desktop/capture.c`) and
|
||||
holds the on/off switch. `/api/macview` is the endpoint for both.
|
||||
- `ui/mac-view.html` is the one viewer in the headset. Its frame header, H.264
|
||||
and JPEG payloads, reconnect handshake, clock sync, timing reports and input
|
||||
messages are unchanged.
|
||||
- `desktop/controller.c` is a C port of the Mac helper's Swift rate controller,
|
||||
for the Windows helper. The Mac keeps its Swift original;
|
||||
`tests/test_pc_controller.py` checks the port against a frozen 600-state
|
||||
trace from it. Capture is gated before encoding, so encoded frames are never
|
||||
dropped, and an idle window's last picture still goes out once the link
|
||||
clears.
|
||||
- Stream records use the Mac helper's JSON schema (`ui/frame_stream_stats.py`),
|
||||
so the benchmark's analysis is shared.
|
||||
|
||||
## Packaging
|
||||
|
||||
The Windows installer carries `resources/desktop/bundle`: `pc-host.dll` (our
|
||||
adapter and the controller), the GStreamer plugins it uses (`coreelements`,
|
||||
`videotestsrc`, `videoconvertscale`, `app`, `jpeg`, `videoparsersbad`,
|
||||
`d3d11`, `mediafoundation`) and the DLLs they depend on, all from MSYS2's
|
||||
UCRT64 packages. `libraries.json` records the GStreamer version, the plugins,
|
||||
each copied DLL and the exact package versions; licence texts are in
|
||||
`licenses/`.
|
||||
|
||||
The x264 encoder is **not** in the installer. It's GPL, and Windows encodes
|
||||
with Media Foundation, so only CI's test builds include it
|
||||
(`desktop/build.py --with-x264`). The macOS and Linux installers are unchanged.
|
||||
|
||||
From a source checkout on Windows, build the bundle from an MSYS2 UCRT64 shell
|
||||
with the packages listed in `.github/workflows/pc-host.yml`:
|
||||
|
||||
```sh
|
||||
python desktop/build.py # what the installer ships
|
||||
python desktop/build.py --with-x264 # for the native tests
|
||||
```
|
||||
|
||||
A missing bundle is reported on the card; nothing is downloaded on first use.
|
||||
|
||||
The Linux adapter from #38 (desktop portal, PipeWire, VA-API) is in the tree,
|
||||
because it shares `capture.c` and the controller, and CI still builds and tests
|
||||
it. The app never offers it and the Linux installers don't include it.
|
||||
|
||||
## Measuring
|
||||
|
||||
The Mac benchmark accepts the PC host:
|
||||
|
||||
```sh
|
||||
python scripts/macview-bench.py run --pc --scenario test --label pc-test
|
||||
python scripts/macview-bench.py run --pc --scenario capture --source window:12345 --label windows-window
|
||||
```
|
||||
|
||||
`capture` needs a window or screen id from the helper's `/windows` or
|
||||
`/displays`. CPU sampling isn't available on PC hosts. See
|
||||
[Mac in the headset](mac-in-headset.md) for the targets and shaping options.
|
||||
|
||||
## What's verified
|
||||
|
||||
- **Verified in CI, this PR:** on `windows-latest`, the native adapter builds
|
||||
and a real x264 synthetic stream passes the ticket, reconnect, timing and
|
||||
input-echo tests through the helper. The Windows installer builds with the
|
||||
bundle, and its bundled Python loads the bundle, starts the helper and
|
||||
confirms x264 isn't in it. Unit tests check the Windows-only, off-by-default
|
||||
gating, the card and the Tools link on macOS, Windows and Linux.
|
||||
- **Verified, #38, 2026-09-28, real Frame on SteamOS BUILD_ID 20260925.6191901:**
|
||||
the PC helper (ARM64, on the Frame itself, x264 test pattern) streamed
|
||||
through a Mac SSH relay into the shared viewer: 400 frames at 39.3 fps,
|
||||
content latency p50/p95 15.1/28.6 ms, synthetic input p50 76.6 ms, at
|
||||
revision `cd20243`. The frame-rate and late-frame targets failed. This
|
||||
proves the helper and the viewer work together; it isn't Windows capture or
|
||||
a real PC link.
|
||||
[Raw result](../bench/results/2026-09-28-cd20243-pc-agent-frame-arm64-final.json),
|
||||
[earlier probe](../bench/results/2026-09-28-c5fc552-dirty-pc-agent-frame-arm64-hairpin.json).
|
||||
- **Verified, #38, on a Mac:** the C controller matched all 600 states of the
|
||||
Swift controller's congestion and recovery trace.
|
||||
- **Not verified:** anything on a real Windows desktop. Windows.Graphics.Capture,
|
||||
Media Foundation (hardware or software), encoder restarts when the bitrate
|
||||
changes, mixed-DPI and multi-monitor input alignment, window resize and
|
||||
minimise, non-US keyboard layouts, physical laser input, and how it performs
|
||||
in a worn headset. No numbers are claimed for these. Not tested with a
|
||||
headset from this branch.
|
||||
- **Not reviewed independently.** #38 tried an independent review that never
|
||||
completed.
|
||||
@@ -187,6 +187,27 @@ refuses ids with a hyphen (`missing/invalid arguments`), which the fake had
|
||||
accepted. The fake now refuses them the same way, and Frame Control makes ids
|
||||
Steam accepts.
|
||||
|
||||
## PC panel host tests
|
||||
|
||||
PC in the headset (Experimental, Windows only). `tests/test_pcview.py` checks
|
||||
that only Windows gets the PC host and that it starts nothing until turned on,
|
||||
the Tools link and card wording, source-bound tickets, reconnect/Stop
|
||||
revocation, input release, source-coordinate mapping and use of the existing
|
||||
panel launcher. `tests/test_pc_controller.py` checks the C rate controller
|
||||
against a frozen trace of the Mac's Swift one.
|
||||
|
||||
The `PC host libraries` workflow builds the native adapter on Windows (and
|
||||
the unshipped Linux adapter on Ubuntu x64 and ARM64), then runs a real x264
|
||||
synthetic stream through the HTTP/WebSocket agent. x264 is GPL, so only these
|
||||
CI builds include it (`desktop/build.py --with-x264`); the release workflow
|
||||
tests the installer's own bundle with its bundled Python and checks x264 isn't
|
||||
in it. `FRAME_PC_REQUIRE_NATIVE=1` and `FRAME_PC_REQUIRE_X264=1` make a missing
|
||||
bundle or encoder fail CI instead of skipping.
|
||||
|
||||
None of this captures a desktop, exercises Media Foundation or a GPU, or
|
||||
checks laser alignment. The [Windows test VM](#windows-test-vm) is the next
|
||||
step for that. See [what's verified](pc-in-headset.md#whats-verified).
|
||||
|
||||
## Windows test VM
|
||||
|
||||
The unit tests run on Windows in CI, but the app itself doesn't. Anything that
|
||||
|
||||
+55
-13
@@ -55,6 +55,7 @@ from urllib.parse import quote, urlencode # %20, not +: the agent's URLComponen
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / "ui"))
|
||||
import frame_macview # noqa: E402
|
||||
import frame_pcview # noqa: E402
|
||||
|
||||
RESULTS = ROOT / "bench" / "results"
|
||||
PAGES = ROOT / "bench" / "pages"
|
||||
@@ -254,7 +255,10 @@ class LabView(frame_macview.MacView):
|
||||
|
||||
def ssh_runner(base):
|
||||
def run(remote, stdin=None, timeout=30):
|
||||
r = subprocess.run([*base, remote], input=stdin, capture_output=True, text=True, timeout=timeout)
|
||||
r = subprocess.run([*base, remote], input=stdin.encode("utf-8") if stdin is not None else None,
|
||||
capture_output=True, timeout=timeout)
|
||||
r.stdout = r.stdout.decode("utf-8", errors="replace")
|
||||
r.stderr = r.stderr.decode("utf-8", errors="replace")
|
||||
if r.returncode:
|
||||
e = RuntimeError((r.stderr or r.stdout).strip())
|
||||
e.stdout = r.stdout
|
||||
@@ -381,7 +385,11 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
if args.browser_flag is not None:
|
||||
mv.browser_flags = [f for f in args.browser_flag if f]
|
||||
chrome = None
|
||||
src = "test"
|
||||
src = args.source or "test"
|
||||
cpu = None
|
||||
if args.pc:
|
||||
mv.host = "windows" if sys.platform == "win32" else "linux"
|
||||
mv.viewer_profile = "pc-view"
|
||||
try:
|
||||
if scenario in ("scroll", "type"):
|
||||
chrome = chrome_window(f"{scenario}.html")
|
||||
@@ -405,9 +413,10 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
since = max([f["s"] for f in first["frames"]] or [0])
|
||||
captured_before = first["captured"]
|
||||
start_mac_us = mv.call("/stats", id=stream["id"])["now"]
|
||||
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
|
||||
capture_output=True, text=True).stdout.split()[0])
|
||||
cpu = CpuSampler(frame_ssh, agent_pid)
|
||||
if not args.pc:
|
||||
agent_pid = int(subprocess.run(["pgrep", "-f", "Frame Mac View Lab.app/Contents/MacOS/frame-mac-view"],
|
||||
capture_output=True, text=True).stdout.split()[0])
|
||||
cpu = CpuSampler(frame_ssh, agent_pid)
|
||||
if relay:
|
||||
relay.begin()
|
||||
expected = 0 # input events the harness asked the viewer for
|
||||
@@ -443,7 +452,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
if not streams:
|
||||
raise SystemExit(f"{scenario}: the stream ended during the run (did the Frame go to sleep?)")
|
||||
data = streams[0]
|
||||
cpu.stop()
|
||||
if cpu:
|
||||
cpu.stop()
|
||||
if args.raw:
|
||||
Path(f"{args.raw}-{scenario}.json").write_text(json.dumps(data))
|
||||
result = summarize(scenario, data, start_mac_us, end_mac_us, args, relay, schedule, expected)
|
||||
@@ -452,7 +462,7 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
for e in data.get("events", []) if e["t"] >= start_mac_us]
|
||||
result["captured"] = (captured_end if captured_end is not None else data["captured"]) - captured_before
|
||||
result["source_fps"] = round(result["captured"] / max(result["duration_s"], 1), 1)
|
||||
result["cpu"] = cpu.summary()
|
||||
result["cpu"] = cpu.summary() if cpu else {"host": "not sampled"}
|
||||
result["viewer"] = data["summary"].get("decoder", "")
|
||||
result["show_s"] = show_s
|
||||
result["panel"] = shown.get("panel")
|
||||
@@ -460,6 +470,8 @@ def run_scenario(args, scenario, agent_port, token, frame_ssh):
|
||||
result["route"] = mv.route
|
||||
return result
|
||||
finally:
|
||||
if cpu:
|
||||
cpu.stop()
|
||||
try:
|
||||
mv.stop(src)
|
||||
except Exception: # noqa: BLE001 - best effort
|
||||
@@ -670,6 +682,8 @@ def fmt(v):
|
||||
|
||||
def add_run_args(r):
|
||||
r.add_argument("--scenario", default="test,scroll,type")
|
||||
r.add_argument("--pc", action="store_true", help="run the bundled PC host; use --scenario test or capture")
|
||||
r.add_argument("--source", default="", help="PC source: window:ID, display:ID, or choose (Linux portal)")
|
||||
r.add_argument("--duration", type=float, default=20)
|
||||
r.add_argument("--warmup", type=float, default=3)
|
||||
r.add_argument("--quality", default="balanced", choices=list(frame_macview.QUALITY))
|
||||
@@ -717,7 +731,7 @@ def frame_ssh_for(args):
|
||||
|
||||
|
||||
def run_suite(args, frame_ssh, quiet=False):
|
||||
agent_port, token = lab_agent()
|
||||
agent_port, token = (args.pc_view.port, args.pc_view.token) if args.pc else lab_agent()
|
||||
results = []
|
||||
for sc in args.scenario.split(","):
|
||||
if not quiet:
|
||||
@@ -744,11 +758,12 @@ def document(args, frame_ssh, results, **extra):
|
||||
commit = git("rev-parse", "--short", "HEAD") + ("+dirty" if git("status", "--porcelain", "--", "mac", "ui") else "")
|
||||
return {
|
||||
"label": args.label or args.cmd, "date": datetime.datetime.now().isoformat(timespec="seconds"), "commit": commit,
|
||||
"config": {"quality": args.quality, "mode": args.mode, "net": args.net or "none", "delay_ms": args.delay,
|
||||
"config": {"quality": args.quality, "mode": "native" if args.pc else args.mode, "net": args.net or "none", "delay_ms": args.delay,
|
||||
"buffer_ms": args.buffer, "host": args.host or args.frame, "usb": args.usb, "ssh_opts": args.ssh_opt,
|
||||
"encoder": os.environ.get("FRAME_MAC_VIEW_ENCODER", ""), "duration_s": args.duration,
|
||||
"browser_flags": args.browser_flag if args.browser_flag is not None else frame_macview.BROWSER_FLAGS},
|
||||
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0], "headset": standby[-40:],
|
||||
"frame_build": build.strip().partition("=")[2], "mac": platform.mac_ver()[0],
|
||||
"host_platform": platform.platform(), "source": args.source, "pc_host": args.pc, "headset": standby[-40:],
|
||||
"scenarios": results, **extra,
|
||||
}
|
||||
|
||||
@@ -819,19 +834,46 @@ def main():
|
||||
print("\nworse:\n " + "\n ".join(regs))
|
||||
sys.exit(1 if regs else 0)
|
||||
|
||||
lab_agent()
|
||||
if args.pc and (args.cmd == "ab" or args.scenario not in ("test", "capture")):
|
||||
p.error("PC runs use --scenario test or --scenario capture; Mac automation is not portable")
|
||||
if args.pc and args.scenario == "capture" and not args.source:
|
||||
p.error("capture needs --source window:ID, display:ID, or choose")
|
||||
if not args.pc:
|
||||
lab_agent()
|
||||
frame_ssh = frame_ssh_for(args)
|
||||
# Keep the Mac's screen awake: virtual displays aren't removed while it sleeps.
|
||||
runs = 1 if args.cmd == "run" else args.repeat * len(args.arm)
|
||||
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))])
|
||||
awake = subprocess.Popen(["caffeinate", "-d", "-u", "-t", str(int(runs * (args.duration * 4 + 60) + 120))]) if sys.platform == "darwin" else None
|
||||
args.pc_view = None
|
||||
try:
|
||||
if args.pc:
|
||||
args.pc_view = frame_pcview.PCView(frame_ssh[:-1], ssh_runner(frame_ssh), frame_ssh[-1])
|
||||
args.pc_view.ensure_agent()
|
||||
if args.source == "choose":
|
||||
args.pc_view.call("/permissions", method="POST")
|
||||
print("Choose a window or screen in your desktop's sharing dialog.", flush=True)
|
||||
deadline = time.monotonic()+125
|
||||
while time.monotonic() < deadline:
|
||||
state = args.pc_view.call("/status")
|
||||
if not state.get("selecting"):
|
||||
sources = args.pc_view.call("/windows")["windows"]
|
||||
if not sources:
|
||||
raise SystemExit(state.get("selectionError") or "Nothing was shared")
|
||||
args.source = sources[-1]["src"]
|
||||
break
|
||||
time.sleep(.5)
|
||||
else:
|
||||
raise SystemExit("Sharing dialog timed out")
|
||||
if args.cmd == "ab":
|
||||
arm_runs, table = ab(args, frame_ssh)
|
||||
write(document(args, frame_ssh, [], arms=args.arm, runs=arm_runs, table=table), args)
|
||||
return
|
||||
results = run_suite(args, frame_ssh)
|
||||
finally:
|
||||
awake.terminate()
|
||||
if args.pc_view:
|
||||
args.pc_view.shutdown()
|
||||
if awake:
|
||||
awake.terminate()
|
||||
doc = document(args, frame_ssh, results)
|
||||
write(doc, args)
|
||||
bad = [f"{r['scenario']} {k}" for r in results for k, v in r["grades"].items() if v in ("bad", "missing")]
|
||||
|
||||
@@ -84,7 +84,10 @@ class Helpers(unittest.TestCase):
|
||||
@unittest.skipUnless(shutil.which("bash"), "needs bash")
|
||||
@unittest.skipIf(os.name == "nt", "Windows' bash.exe is WSL's launcher, and runners have no distribution")
|
||||
def test_launch_script_parses(self):
|
||||
r = subprocess.run(["bash", "-n"], input=frame_macview.LAUNCH, text=True, capture_output=True)
|
||||
bash = str(Path(os.environ.get("ProgramFiles", "C:/Program Files")) / "Git/bin/bash.exe") if sys.platform == "win32" else "bash"
|
||||
if sys.platform == "win32" and not Path(bash).exists():
|
||||
self.skipTest("Git Bash is not installed (WSL bash is not a local shell)")
|
||||
r = subprocess.run([bash, "-n"], input=frame_macview.LAUNCH.encode("utf-8"), capture_output=True)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
|
||||
def test_show_checks_the_source_before_anything_else(self):
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Frozen regression trace from the original mac-in-headset Swift controller.
|
||||
The 600-state reference was produced at 1b90c64, before the C extraction:
|
||||
60 seconds with a slow/congested middle, then recovery. No generated behavior
|
||||
from the new implementation is used to calculate the expected digest.
|
||||
"""
|
||||
import ctypes as C
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'ui'))
|
||||
from frame_pc_capture import Controller
|
||||
|
||||
|
||||
class ControllerTrace(unittest.TestCase):
|
||||
def test_original_mac_congestion_and_recovery(self):
|
||||
cc = shutil.which('cc') or shutil.which('gcc')
|
||||
if not cc:
|
||||
self.skipTest('C compiler not installed; native build jobs cover this trace')
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / ('trace.dll' if sys.platform == 'win32' else 'trace.so')
|
||||
subprocess.run([cc, '-O2', '-shared', '-fPIC', str(ROOT/'desktop/controller.c'), '-o', str(path)], check=True, capture_output=True)
|
||||
lib = C.CDLL(str(path))
|
||||
sig = {'new': (C.c_void_p, [C.c_int, C.c_int]), 'free': (None, [C.c_void_p]),
|
||||
'ceiling': (None, [C.c_void_p, C.c_int]), 'value': (C.c_int64, [C.c_void_p, C.c_int]),
|
||||
'capture': (None, [C.c_void_p, C.c_int64]), 'gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]),
|
||||
'sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]),
|
||||
'ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]), 'update': (C.c_int, [C.c_void_p, C.c_int64])}
|
||||
for name, (ret, args) in sig.items():
|
||||
fn = getattr(lib, 'fc_'+name)
|
||||
fn.restype, fn.argtypes = ret, args
|
||||
c = Controller(lib, 60, 8000000)
|
||||
due, states, seq = [], [], 0
|
||||
try:
|
||||
for tick in range(6000):
|
||||
now = 10000000+tick*10000
|
||||
for t, s in due:
|
||||
if t <= now:
|
||||
c.call('ack', s, now)
|
||||
due = [(t, s) for t, s in due if t > now]
|
||||
if tick % 2 == 0:
|
||||
c.call('capture', now)
|
||||
if c.call('gate', now, 1):
|
||||
seq += 1
|
||||
c.call('sent', seq, 8000 if tick < 1000 or tick > 4000 else 22000, now)
|
||||
due.append((now+(240000 if 1500 < tick < 3500 else 20000), seq))
|
||||
if tick % 10 == 0:
|
||||
c.update(now)
|
||||
x = c.state()
|
||||
states.append([x[k] for k in ('target', 'ceiling', 'tier', 'fps', 'inFlight')] +
|
||||
[round(x['scale']*100), round(x['baseRtt']*1000), round(x['slack']*1000)])
|
||||
digest = hashlib.sha256(json.dumps(states, separators=(',', ':')).encode()).hexdigest()
|
||||
self.assertEqual(digest, '1932f16aafce5c9157bcd2aba5d90719a366f9999422920aaeae78a6cdec08f6')
|
||||
finally:
|
||||
c.close()
|
||||
if sys.platform == 'win32':
|
||||
import _ctypes
|
||||
_ctypes.FreeLibrary(lib._handle)
|
||||
@@ -0,0 +1,520 @@
|
||||
"""PC adapters: grants, input cleanup, launcher reuse and native codec contract.
|
||||
Native tests require the bundled libraries; CI sets REQUIRE_NATIVE so a missing
|
||||
bundle is a failure, never a silent skip. No desktop consent or GPU is faked as
|
||||
successful real-host capture.
|
||||
|
||||
The encoding tests need x264, which only CI's own builds bundle (it's GPL, so
|
||||
the installer leaves it out). FRAME_PC_REQUIRE_X264=1 makes its absence a
|
||||
failure; FRAME_PC_INSTALLER_BUNDLE=1 checks the installer's bundle instead.
|
||||
"""
|
||||
import ctypes
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(ROOT / 'ui'))
|
||||
import frame_pc_agent as agent
|
||||
import frame_pc_capture as capture
|
||||
import frame_pcview
|
||||
import frame_macview
|
||||
from frame_stream_stats import Stats
|
||||
from test_macview import WS
|
||||
|
||||
|
||||
class GrantsTests(unittest.TestCase):
|
||||
def test_source_bound_retry_ack_and_stop(self):
|
||||
g = agent.Grants('master')
|
||||
ticket = g.ticket('window:1')
|
||||
self.assertIsNone(g.redeem('window:2', {'t': ticket}))
|
||||
key = g.redeem('window:1', {'t': ticket})
|
||||
self.assertEqual(key, g.redeem('window:1', {'t': ticket}))
|
||||
g.ack(key)
|
||||
self.assertIsNone(g.redeem('window:1', {'t': ticket}))
|
||||
self.assertEqual(key, g.redeem('window:1', {'r': key}))
|
||||
self.assertIsNone(g.redeem('window:2', {'r': key}))
|
||||
g.revoke('window:1')
|
||||
self.assertIsNone(g.redeem('window:1', {'r': key}))
|
||||
|
||||
def test_expiry_and_stop_before_redeem(self):
|
||||
g = agent.Grants('master')
|
||||
with mock.patch.object(agent.time, 'monotonic', return_value=1):
|
||||
ticket = g.ticket('test')
|
||||
with mock.patch.object(agent.time, 'monotonic', return_value=62):
|
||||
self.assertIsNone(g.redeem('test', {'t': ticket}))
|
||||
ticket = g.ticket('test')
|
||||
g.revoke()
|
||||
self.assertIsNone(g.redeem('test', {'t': ticket}))
|
||||
|
||||
|
||||
class GatingTests(unittest.TestCase):
|
||||
"""Experimental, Windows only, and off until the person turns it on."""
|
||||
|
||||
def view(self):
|
||||
return frame_pcview.PCView(['ssh'], mock.Mock(), 'frame')
|
||||
|
||||
def test_only_windows_gets_the_pc_host(self):
|
||||
for platform, cls in (('win32', frame_pcview.PCView), ('linux', frame_macview.MacView),
|
||||
('darwin', frame_macview.MacView)):
|
||||
with mock.patch.object(frame_pcview.sys, 'platform', platform):
|
||||
view = frame_pcview.host_view(['ssh'], mock.Mock(), 'frame')
|
||||
self.assertIs(type(view), cls, platform)
|
||||
|
||||
def test_off_by_default_starts_nothing(self):
|
||||
view = self.view()
|
||||
with mock.patch.object(frame_pcview.sys, 'platform', 'win32'), \
|
||||
mock.patch.object(frame_macview.subprocess, 'Popen', side_effect=AssertionError('started while off')):
|
||||
state = view.state()
|
||||
self.assertEqual(state, {'available': False, 'reason': frame_pcview.OFF, 'host': view.host,
|
||||
'experimental': True, 'enabled': False})
|
||||
with self.assertRaises(frame_macview.MacViewError):
|
||||
view.show('window:12')
|
||||
view.run.assert_not_called()
|
||||
|
||||
def test_never_offered_off_windows_even_when_turned_on(self):
|
||||
view = self.view()
|
||||
view.enabled = True
|
||||
with mock.patch.object(frame_pcview.sys, 'platform', 'linux'):
|
||||
self.assertIn('only available on Windows', view.unavailable())
|
||||
|
||||
def test_turning_on_checks_for_the_bundled_libraries(self):
|
||||
view = self.view()
|
||||
missing = mock.Mock(is_file=mock.Mock(return_value=False))
|
||||
with mock.patch.object(frame_pcview.sys, 'platform', 'win32'), \
|
||||
mock.patch.object(frame_pcview, 'LIBRARY', missing):
|
||||
state = view.set_enabled(True)
|
||||
self.assertTrue(state['enabled'])
|
||||
self.assertFalse(state['available'])
|
||||
self.assertIn('libraries are missing', state['reason'])
|
||||
|
||||
def test_turning_off_stops_streams_and_the_helper(self):
|
||||
view = self.view()
|
||||
view.enabled = True
|
||||
agent = mock.Mock()
|
||||
agent.poll.return_value = None
|
||||
view.agent, view.tunnel = agent, mock.Mock()
|
||||
tunnel = view.tunnel
|
||||
tunnel.poll.return_value = None
|
||||
view.stop = mock.Mock()
|
||||
with mock.patch.object(frame_pcview.sys, 'platform', 'win32'):
|
||||
state = view.set_enabled(False)
|
||||
view.stop.assert_called_once_with()
|
||||
agent.stdin.close.assert_called_once()
|
||||
agent.wait.assert_called()
|
||||
tunnel.terminate.assert_called_once()
|
||||
self.assertIsNone(view.agent)
|
||||
self.assertEqual((state['enabled'], state['reason']), (False, frame_pcview.OFF))
|
||||
|
||||
def test_server_enable_action(self):
|
||||
import sandbox # noqa: F401 (keeps the server off real data and services)
|
||||
import server
|
||||
with mock.patch.object(server, 'macview', frame_macview.MacView(['ssh'], mock.Mock(), 'frame')):
|
||||
with self.assertRaises(server.Failure) as e:
|
||||
server.macview_action({'action': 'enable', 'on': True})
|
||||
self.assertIn('only available on Windows', str(e.exception))
|
||||
view = self.view()
|
||||
view.set_enabled = mock.Mock(return_value={'enabled': True})
|
||||
with mock.patch.object(server, 'macview', view):
|
||||
self.assertEqual(server.macview_action({'action': 'enable', 'on': True}), {'enabled': True})
|
||||
server.macview_action({'action': 'enable', 'on': 'yes'}) # anything but true is off
|
||||
self.assertEqual([c.args for c in view.set_enabled.call_args_list], [(True,), (False,)])
|
||||
|
||||
|
||||
class PageTests(unittest.TestCase):
|
||||
page = (ROOT / 'ui' / 'index.html').read_text(encoding='utf-8')
|
||||
|
||||
def test_tool_link_is_windows_only_and_marked_experimental(self):
|
||||
self.assertIn('<a href="#macview" id="pcToolLink" hidden><span>PC in the headset</span>'
|
||||
'<small>Experimental:', self.page)
|
||||
self.assertIn('$("pcToolLink").hidden = h.mobile || h.os !== "Windows";', self.page)
|
||||
self.assertIn('$("macToolLink").hidden = h.mobile || h.os !== "macOS";', self.page)
|
||||
|
||||
def test_card_says_experimental_and_what_is_unverified(self):
|
||||
self.assertIn('<span class="tag" id="mvTag" hidden>Experimental</span>', self.page)
|
||||
self.assertIn('<input type="checkbox" id="mvPcOn">', self.page)
|
||||
self.assertIn("hasn't been tried on a real\n Windows desktop or with a headset yet", self.page)
|
||||
self.assertIn('docs/pc-in-headset.md', self.page)
|
||||
|
||||
def test_opt_in_is_remembered_with_the_client_preferences(self):
|
||||
self.assertIn("const keys = ['mvSeparate', 'padUsed', 'pcView']", self.page)
|
||||
prefs = (ROOT / 'app' / 'preferences.js').read_text(encoding='utf-8')
|
||||
self.assertIn('const KEYS = ["mvSeparate", "padUsed", "pcView"];', prefs)
|
||||
|
||||
|
||||
class AdapterTests(unittest.TestCase):
|
||||
def test_windows_uses_wgc_and_media_foundation(self):
|
||||
for kind in ('window', 'display'):
|
||||
pipeline = capture.pipeline(dict(src=kind+':42'), 'win32', 'mfh264enc', 640, 360, 30, 1000000)
|
||||
self.assertIn('capture-api=wgc', pipeline)
|
||||
self.assertIn('window-handle=42' if kind == 'window' else 'monitor-handle=42', pipeline)
|
||||
self.assertIn('mfh264enc name=enc low-latency=true bframes=0', pipeline)
|
||||
self.assertNotIn('drop=true', pipeline)
|
||||
with self.assertRaises(ValueError):
|
||||
capture.pipeline(dict(src='window:42 ! fakesink'), 'win32', 'mfh264enc', 640, 360, 30, 1000000)
|
||||
|
||||
def test_linux_uses_only_portal_fd_and_node(self):
|
||||
p = capture.pipeline(dict(src='window:consented', fd=19, node=47), 'linux', 'x264enc', 640, 360, 30, 1000000)
|
||||
self.assertIn('pipewiresrc fd=19 path=47', p)
|
||||
self.assertIn('tune=zerolatency', p)
|
||||
self.assertEqual(capture.dimensions(1920, 1080, 1280), (1280, 720))
|
||||
|
||||
def test_portal_input_clamps_and_releases_held_state(self):
|
||||
native = mock.Mock()
|
||||
native.lib.fc_portal_input.return_value = 1
|
||||
inp = capture.PortalInput(native, dict(portal=123, w=100, h=50))
|
||||
inp.handle(dict(t='m', e='down', b=0, x=2, y=-1))
|
||||
inp.handle(dict(t='k', e='down', key='Control'))
|
||||
inp.release()
|
||||
calls = native.lib.fc_portal_input.call_args_list
|
||||
self.assertEqual(calls[0].args, (123, 0, 99, 0, 0, 0))
|
||||
self.assertIn(mock.call(123, 1, 0, 0, 0x110, 0), calls)
|
||||
self.assertIn(mock.call(123, 3, 0, 0, 0xffe3, 0), calls)
|
||||
self.assertFalse(inp.buttons or inp.keys)
|
||||
|
||||
def test_windows_release_and_negative_monitor_origin(self):
|
||||
host = mock.Mock()
|
||||
host.user.GetSystemMetrics.side_effect = [-1920, 0, 3840, 1080]
|
||||
inp = capture.WindowsInput(host, dict(src='display:4', x=-1920, y=0, w=1920, h=1080))
|
||||
inp.handle(dict(t='m', e='down', b=2, x=0, y=0))
|
||||
self.assertEqual(host.send.call_args_list[0], mock.call(mouse=(0, 0, 0, 0xc001)))
|
||||
inp.handle(dict(t='k', e='down', code='ControlLeft', key='Control'))
|
||||
inp.release()
|
||||
self.assertIn(mock.call(mouse=(0, 0, 0, 0x10)), host.send.call_args_list)
|
||||
self.assertIn(mock.call(key=(162, 0, 2)), host.send.call_args_list)
|
||||
|
||||
def test_refused_windows_focus_never_clicks_the_covering_app(self):
|
||||
host = mock.Mock()
|
||||
host.rect.return_value = (0, 0, 800, 600)
|
||||
host.user.GetForegroundWindow.return_value = 99
|
||||
host.user.SetForegroundWindow.return_value = False
|
||||
inp = capture.WindowsInput(host, dict(src='window:12', w=800, h=600))
|
||||
with self.assertRaisesRegex(RuntimeError, 'could not focus'):
|
||||
inp.handle(dict(t='m', e='down', b=0, x=.5, y=.5))
|
||||
host.send.assert_not_called()
|
||||
|
||||
def test_pc_reuses_launcher_and_namespaces_panel_ids(self):
|
||||
view = frame_pcview.PCView(['ssh'], mock.Mock(return_value='panel created'), 'frame')
|
||||
view.call = mock.Mock(side_effect=lambda path, **kw: {'screen': True, 'ticket': 'one-use', 'streams': []})
|
||||
view.ensure_tunnel = mock.Mock()
|
||||
view.remote_port = 47900
|
||||
view.show('window:12')
|
||||
self.assertEqual(view.run.call_args.kwargs['stdin'], frame_macview.LAUNCH)
|
||||
self.assertNotEqual(frame_macview.panel_id('window:12'), frame_macview.panel_id('window:12', 'windows'))
|
||||
self.assertFalse(view.prefer_usb)
|
||||
with self.assertRaises(frame_macview.MacViewError):
|
||||
view.show('separate:12')
|
||||
|
||||
def test_session_end_releases_native_input_only_once(self):
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.input_lock = threading.RLock(), threading.RLock()
|
||||
session.stop_event = threading.Event()
|
||||
session.ws, session.input = mock.Mock(), mock.Mock()
|
||||
session.released = False
|
||||
session.end()
|
||||
session.end()
|
||||
session.input.release.assert_called_once()
|
||||
self.assertTrue(session.stop_event.is_set())
|
||||
|
||||
def test_idle_source_is_not_a_stall(self):
|
||||
# PipeWire/WGC deliver frames only on damage: a static desktop sends
|
||||
# its first frame and then nothing. That must not end the session.
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.pending = threading.RLock(), {}
|
||||
session.stats = Stats(lambda: 0)
|
||||
session.stats.captured = 1
|
||||
self.assertIsNone(session.stalled(60000000, 0, 0))
|
||||
# A frame that entered the encoder and never came out is a fault.
|
||||
session.pending[5] = dict(cap=0, arr=0, e0=1000000, tier=0, br=0)
|
||||
self.assertIn('encoder', session.stalled(12000000, 0, 0))
|
||||
self.assertIsNone(session.stalled(10000000, 0, 0))
|
||||
# No initial frame at all after the pipeline opened is a fault.
|
||||
session.pending.clear()
|
||||
self.assertIn('capture', session.stalled(12000000, 0, 1))
|
||||
self.assertIsNone(session.stalled(9000000, 0, 1))
|
||||
|
||||
def test_controller_is_inert_after_close(self):
|
||||
lib = mock.Mock()
|
||||
lib.fc_new.return_value = 1234
|
||||
lib.fc_value.return_value = 100
|
||||
controller = capture.Controller(lib, 60, 5000000)
|
||||
controller.close()
|
||||
lib.fc_free.assert_called_once_with(1234)
|
||||
lib.reset_mock()
|
||||
state = controller.state()
|
||||
self.assertEqual((state['fps'], state['target'], state['scale']), (60, 0, 1))
|
||||
self.assertEqual(controller.call('gate', 1, 1), 0)
|
||||
self.assertEqual(controller.update(1), 0)
|
||||
controller.close()
|
||||
self.assertEqual(lib.method_calls, []) # nothing touched the freed pointer
|
||||
|
||||
def test_library_path_prepend_adds_no_empty_entry(self):
|
||||
env = {}
|
||||
frame_pcview.prepend(env, 'LD_LIBRARY_PATH', '/opt/fc/lib')
|
||||
self.assertEqual(env['LD_LIBRARY_PATH'], '/opt/fc/lib')
|
||||
frame_pcview.prepend(env, 'LD_LIBRARY_PATH', '/x')
|
||||
self.assertEqual(env['LD_LIBRARY_PATH'], '/x' + os.pathsep + '/opt/fc/lib')
|
||||
with mock.patch.dict(os.environ, clear=True):
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
for name in ('PATH', 'LD_LIBRARY_PATH'):
|
||||
parts = env.get(name, 'x').split(os.pathsep)
|
||||
self.assertNotIn('', parts, name)
|
||||
|
||||
def test_malformed_viewer_input_is_dropped_not_fatal(self):
|
||||
session = object.__new__(agent.Session)
|
||||
session.lock, session.input_lock = threading.RLock(), threading.RLock()
|
||||
session.stop_event, session.key_event = threading.Event(), threading.Event()
|
||||
session.src, session.codec, session.key, session.w, session.h = 'display:1', 'h264', 'r', 100, 100
|
||||
session.source = dict(src='display:1', w=100, h=100, portal=1)
|
||||
session.input_enabled, session.released = True, False
|
||||
session.native = mock.Mock()
|
||||
session.native.now.return_value = 0
|
||||
session.controller, session.agent = mock.Mock(), mock.Mock()
|
||||
session.stats = Stats(lambda: 0)
|
||||
session.produce = lambda: None
|
||||
lib = mock.Mock()
|
||||
lib.fc_portal_input.return_value = 1
|
||||
session.input = capture.PortalInput(mock.Mock(lib=lib), session.source)
|
||||
session.ws = mock.Mock()
|
||||
session.ws.receive.side_effect = [
|
||||
dict(t='m', x='left', y=0), dict(t='wheel', dx=[1]), dict(t='k', e='down', key=None, i='x'),
|
||||
dict(t='fd', drop='many'), dict(t='rx', s=[1]), dict(t='m', x=.5, y=.5), ConnectionError('closed')]
|
||||
with self.assertRaises(ConnectionError):
|
||||
session.start()
|
||||
# Every malformed message was skipped; the good move after them still ran.
|
||||
moves = [c.args for c in lib.fc_portal_input.call_args_list if c.args[1] == 0]
|
||||
self.assertAlmostEqual(moves[-1][2], .5*99)
|
||||
self.assertEqual(session.ws.receive.call_count, 7)
|
||||
|
||||
def test_stats_keep_capture_time_and_bound_records(self):
|
||||
stats = Stats(lambda: 10000000)
|
||||
stats.input(dict(t='m', i=1, tv=9999990))
|
||||
f = stats.add(cap=10000001, arr=10000002, e0=10000003, e1=10000004)
|
||||
self.assertEqual(f['echo'], 1)
|
||||
stats.report(dict(t='rx', s=f['s'], r=10000010))
|
||||
stats.report(dict(t='fd', f=[[f['s'], 10000011, 10000012, 10000013]]))
|
||||
self.assertEqual(stats.snapshot(settle=0)['inputs'][0]['frame'], f['s'])
|
||||
for _ in range(4100):
|
||||
stats.add(cap=10000001)
|
||||
self.assertEqual(len(stats.frames), 4096)
|
||||
|
||||
|
||||
def x264_bundled():
|
||||
return any((capture.NATIVE / 'lib' / 'gstreamer-1.0').glob('*gstx264*'))
|
||||
|
||||
|
||||
def needs_x264(test):
|
||||
def run(self):
|
||||
if not x264_bundled():
|
||||
if os.environ.get('FRAME_PC_REQUIRE_X264') == '1':
|
||||
raise AssertionError('CI required x264 in the native PC bundle (desktop/build.py --with-x264)')
|
||||
raise unittest.SkipTest('x264 is only in CI test bundles, not the installer')
|
||||
return test(self)
|
||||
run.__name__, run.__doc__ = test.__name__, test.__doc__
|
||||
return run
|
||||
|
||||
|
||||
class NativeTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
if not capture.LIBRARY.exists():
|
||||
if os.environ.get('FRAME_PC_REQUIRE_NATIVE') == '1':
|
||||
raise AssertionError('CI required native PC libraries but they were not built')
|
||||
raise unittest.SkipTest('native PC libraries not built on this host')
|
||||
cls.token = 'test-' + os.urandom(8).hex()
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['FRAME_MAC_VIEW_TOKEN'] = cls.token
|
||||
cls.proc = subprocess.Popen([sys.executable, str(ROOT/'ui/frame_pc_agent.py'), 'serve', '--port', '0',
|
||||
'--page', str(ROOT/'ui/mac-view.html'), '--exit-on-eof'], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, text=True, env=env)
|
||||
line = cls.proc.stdout.readline()
|
||||
if 'listening' not in line:
|
||||
raise AssertionError('Native agent failed: ' + cls.proc.stderr.read())
|
||||
cls.port = int(line.rsplit(':', 1)[1])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.proc.stdin.close()
|
||||
try:
|
||||
rc = cls.proc.wait(10)
|
||||
except subprocess.TimeoutExpired:
|
||||
cls.proc.kill()
|
||||
raise AssertionError('Native agent did not stop after EOF')
|
||||
error = cls.proc.stderr.read()
|
||||
cls.proc.stdout.close()
|
||||
cls.proc.stderr.close()
|
||||
if rc:
|
||||
raise AssertionError('Native agent exited %s: %s' % (rc, error))
|
||||
|
||||
def request(self, path, method='GET'):
|
||||
conn = http.client.HTTPConnection('127.0.0.1', self.port, timeout=10)
|
||||
try:
|
||||
conn.request(method, path)
|
||||
response = conn.getresponse()
|
||||
return response.status, response.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def tearDown(self):
|
||||
self.request('/close?k='+self.token, 'POST')
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith('linux'), 'Linux PipeWire client libraries')
|
||||
def test_bundled_pipewire_context_needs_no_system_modules(self):
|
||||
# No portal request, remote connection or desktop capture: create and
|
||||
# destroy a private client context to test the dynamically loaded SPA
|
||||
# and protocol modules that ldd cannot discover.
|
||||
code = """
|
||||
import ctypes as C, os
|
||||
lib = C.CDLL(os.path.join(os.environ['FRAME_PC_NATIVE'], 'lib', 'libpipewire-0.3.so.0'))
|
||||
for name, ret, args in [
|
||||
('pw_init', None, [C.c_void_p, C.c_void_p]),
|
||||
('pw_main_loop_new', C.c_void_p, [C.c_void_p]),
|
||||
('pw_main_loop_get_loop', C.c_void_p, [C.c_void_p]),
|
||||
('pw_context_new', C.c_void_p, [C.c_void_p, C.c_void_p, C.c_size_t]),
|
||||
('pw_context_find_factory', C.c_void_p, [C.c_void_p, C.c_char_p]),
|
||||
('pw_context_find_spa_lib', C.c_char_p, [C.c_void_p, C.c_char_p]),
|
||||
('pw_context_destroy', None, [C.c_void_p]),
|
||||
('pw_main_loop_destroy', None, [C.c_void_p])]:
|
||||
fn = getattr(lib, name)
|
||||
fn.restype, fn.argtypes = ret, args
|
||||
lib.pw_init(None, None)
|
||||
loop = lib.pw_main_loop_new(None)
|
||||
assert loop, 'bundled SPA loop support did not load'
|
||||
context = lib.pw_context_new(lib.pw_main_loop_get_loop(loop), None, 0)
|
||||
assert context, 'bundled PipeWire client modules did not load'
|
||||
assert lib.pw_context_find_factory(context, b'adapter'), 'video adapter factory is missing'
|
||||
# The exported factory is video.adapt, which needs a live follower node to
|
||||
# instantiate. Check its configured library and exported factory without
|
||||
# pretending a headless context is a consented video stream.
|
||||
plugin_name = lib.pw_context_find_spa_lib(context, b'video.adapt')
|
||||
assert plugin_name, 'video.adapt has no bundled library mapping'
|
||||
plugin = C.CDLL(os.path.join(os.environ['SPA_PLUGIN_DIR'], plugin_name.decode() + '.so'))
|
||||
class Factory(C.Structure):
|
||||
_fields_ = [('version', C.c_uint32), ('name', C.c_char_p)]
|
||||
plugin.spa_handle_factory_enum.argtypes = [C.POINTER(C.POINTER(Factory)), C.POINTER(C.c_uint32)]
|
||||
plugin.spa_handle_factory_enum.restype = C.c_int
|
||||
factory, index, names = C.POINTER(Factory)(), C.c_uint32(), []
|
||||
while plugin.spa_handle_factory_enum(C.byref(factory), C.byref(index)) > 0:
|
||||
names.append(factory.contents.name)
|
||||
assert b'video.adapt' in names, 'bundled SPA video adapter factory is missing'
|
||||
lib.pw_context_destroy(context)
|
||||
lib.pw_main_loop_destroy(loop)
|
||||
"""
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['FRAME_PC_NATIVE'] = str(capture.NATIVE)
|
||||
result = subprocess.run([sys.executable, '-c', code], env=env, capture_output=True, text=True, timeout=15)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
def test_agent_reports_its_host_and_encoder(self):
|
||||
self.assertEqual(self.request('/status')[0], 403)
|
||||
status, body = self.request('/status?k='+self.token)
|
||||
self.assertEqual(status, 200)
|
||||
data = json.loads(body)
|
||||
if sys.platform == 'win32':
|
||||
self.assertEqual((data['host'], data['encoder']), ('windows', 'mfh264enc'))
|
||||
self.assertEqual(data['streams'], [])
|
||||
|
||||
@unittest.skipUnless(os.environ.get('FRAME_PC_INSTALLER_BUNDLE') == '1', 'checks the installer\'s bundle')
|
||||
def test_installer_bundle_leaves_out_gpl_x264(self):
|
||||
self.assertFalse(x264_bundled())
|
||||
self.assertFalse(list((capture.NATIVE / 'bin').glob('libx264*')))
|
||||
plugins = [p.name for p in (capture.NATIVE / 'lib' / 'gstreamer-1.0').iterdir()]
|
||||
for name in ('mediafoundation', 'd3d11', 'jpeg'):
|
||||
self.assertTrue(any(p.endswith('gst%s.dll' % name) for p in plugins), (name, plugins))
|
||||
|
||||
@needs_x264
|
||||
def test_last_raw_picture_survives_a_closed_gate(self):
|
||||
# A source sends ONE changed frame, then becomes idle. It must reach
|
||||
# the encoder after congestion clears, without needing another update.
|
||||
code = """
|
||||
import ctypes as C, sys, time
|
||||
from frame_pc_capture import Native, GATE, Encoded, pipeline
|
||||
native = Native()
|
||||
start = None
|
||||
closing = False
|
||||
phases = []
|
||||
def gate(stage, pts, capture, arrived):
|
||||
global start
|
||||
phases.append(stage)
|
||||
if closing: return -1
|
||||
if start is None: start = time.monotonic()
|
||||
return int(stage == 1 or time.monotonic()-start >= .15)
|
||||
callback = GATE(gate)
|
||||
p = pipeline(dict(src='test'), sys.platform, 'x264enc', 320, 180, 30, 300000)
|
||||
p = p.replace('is-live=true', 'is-live=true num-buffers=1')
|
||||
error = C.create_string_buffer(1024)
|
||||
handle = native.lib.fc_capture_open(p.encode(), callback, error, len(error))
|
||||
assert handle, error.value
|
||||
try:
|
||||
frame = Encoded()
|
||||
result = 0
|
||||
deadline = time.monotonic()+5
|
||||
while not result and time.monotonic() < deadline:
|
||||
result = native.lib.fc_capture_pull(handle, C.byref(frame))
|
||||
assert result == 1, native.lib.fc_capture_error(handle)
|
||||
assert frame.key and frame.size > 17
|
||||
assert phases.count(0) == 1 and 2 in phases, phases
|
||||
finally:
|
||||
closing = True
|
||||
native.lib.fc_capture_close(handle)
|
||||
"""
|
||||
env = frame_pcview.PCView([], lambda *a: None, 'frame').agent_environment()
|
||||
env['PYTHONPATH'] = str(ROOT / 'ui')
|
||||
result = subprocess.run([sys.executable, '-c', code], env=env, capture_output=True, text=True, timeout=10)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
|
||||
@needs_x264
|
||||
def test_auth_and_real_h264_stats(self):
|
||||
self.assertEqual(self.request('/status')[0], 403)
|
||||
status, body = self.request('/ticket?src=test&k='+self.token, 'POST')
|
||||
self.assertEqual(status, 200)
|
||||
ticket = json.loads(body)['ticket']
|
||||
ws = WS(self.port, '/stream?src=test&fps=30&max=640&t='+ticket)
|
||||
try:
|
||||
self.assertEqual(ws.status, 101)
|
||||
key, got, echo = None, 0, False
|
||||
ws.send_text(json.dumps(dict(t='m', e='down', i=7, tv=1, x=.2, y=.3)))
|
||||
deadline = time.monotonic()+10
|
||||
while time.monotonic() < deadline and (got < 5 or not echo):
|
||||
op, data = ws.recv()
|
||||
if op == 1:
|
||||
m = json.loads(data)
|
||||
self.assertNotEqual(m.get('t'), 'error', m)
|
||||
if m.get('t') == 'hello':
|
||||
key = m['r']
|
||||
ws.send_text(json.dumps(dict(t='ack')))
|
||||
elif op == 2:
|
||||
got += 1
|
||||
seq, echoed = struct.unpack('!II', data[9:17])
|
||||
now = struct.unpack('!Q', data[1:9])[0]
|
||||
echo |= echoed == 7
|
||||
self.assertIn(b'\x00\x00\x00\x01', data[17:])
|
||||
ws.send_text(json.dumps(dict(t='rx', s=seq, r=now+100)))
|
||||
ws.send_text(json.dumps(dict(t='fd', f=[[seq, now+200, now+300, now+400]])))
|
||||
self.assertGreaterEqual(got, 5)
|
||||
self.assertTrue(echo)
|
||||
time.sleep(.1)
|
||||
stats = json.loads(self.request('/stats?settle=0&k='+self.token)[1])['streams'][0]
|
||||
self.assertGreater(stats['frames'][0]['e1'], stats['frames'][0]['e0'])
|
||||
self.assertIn('controller', stats)
|
||||
used = WS(self.port, '/stream?src=test&t='+ticket)
|
||||
self.assertEqual(used.status, 403)
|
||||
used.close()
|
||||
self.request('/close?src=test&k='+self.token, 'POST')
|
||||
revoked = WS(self.port, '/stream?src=test&r='+key)
|
||||
self.assertEqual(revoked.status, 403)
|
||||
revoked.close()
|
||||
finally:
|
||||
ws.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
+27
-16
@@ -56,12 +56,12 @@ BROWSER_FLAGS = []
|
||||
PANEL_BOX = (1920, 1080)
|
||||
|
||||
# Opens one viewer on gamescope's X display and gives its window its own panel
|
||||
# id. Args: appid url width height tag [browser flags...]. The page puts "[tag]" in its title at
|
||||
# id. Args: appid url width height tag profile [browser flags...]. The page puts "[tag]" in its title at
|
||||
# once, which is how its X window is found (Chromium may hand the URL to an
|
||||
# instance that's already running, so there's no process to follow).
|
||||
LAUNCH = r"""set -u
|
||||
appid=$1 url=$2 w=$3 h=$4 tag=$5
|
||||
shift 5
|
||||
appid=$1 url=$2 w=$3 h=$4 tag=$5 profile=$6
|
||||
shift 6
|
||||
export DISPLAY=:0 LC_ALL=C.UTF-8
|
||||
unset WAYLAND_DISPLAY
|
||||
if ! xprop -root GAMESCOPE_FOCUSABLE_WINDOWS >/dev/null 2>&1; then
|
||||
@@ -73,15 +73,15 @@ common=(--ozone-platform=x11 --force-device-scale-factor=1 --no-first-run --no-d
|
||||
--disable-features=Translate,MediaRouter --autoplay-policy=no-user-gesture-required
|
||||
"--window-size=$w,$h" "$@" "--app=$url")
|
||||
if [ -x "$HOME/chromium-xr/chrome" ]; then
|
||||
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/mac-view" "${common[@]}")
|
||||
cmd=("$HOME/chromium-xr/chrome" "--user-data-dir=$HOME/.local/share/frame-control/$profile" "${common[@]}")
|
||||
elif flatpak info org.chromium.Chromium >/dev/null 2>&1; then
|
||||
cmd=(flatpak run org.chromium.Chromium
|
||||
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-mac-view" "${common[@]}")
|
||||
"--user-data-dir=$HOME/.var/app/org.chromium.Chromium/data/frame-$profile" "${common[@]}")
|
||||
else
|
||||
echo "NO_BROWSER"
|
||||
exit 3
|
||||
fi
|
||||
log=/tmp/frame-mac-view.log
|
||||
log=/tmp/frame-$profile.log
|
||||
setsid nohup "${cmd[@]}" >>"$log" 2>&1 </dev/null &
|
||||
for _ in $(seq 1 60); do
|
||||
sleep 0.5
|
||||
@@ -105,9 +105,9 @@ class MacViewError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def panel_id(src):
|
||||
def panel_id(src, host="mac"):
|
||||
"""A stable panel id per source, in the range panel-on-frame.sh uses."""
|
||||
return 2_001_000_000 + zlib.crc32(f"mac:{src}".encode()) % 1_000_000
|
||||
return 2_001_000_000 + zlib.crc32(f"{host}:{src}".encode()) % 1_000_000
|
||||
|
||||
|
||||
def fit(w, h, box=PANEL_BOX):
|
||||
@@ -116,6 +116,9 @@ def fit(w, h, box=PANEL_BOX):
|
||||
|
||||
|
||||
class MacView:
|
||||
host = "mac"
|
||||
viewer_profile = "mac-view"
|
||||
|
||||
def __init__(self, tunnel_ssh, run, frame, track=None):
|
||||
self.tunnel_ssh = list(tunnel_ssh)
|
||||
self.run = run
|
||||
@@ -174,6 +177,12 @@ class MacView:
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def agent_command(self):
|
||||
return [str(AGENT)]
|
||||
|
||||
def agent_environment(self):
|
||||
return {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
|
||||
|
||||
def ensure_agent(self):
|
||||
with self.lock:
|
||||
if self.agent and self.agent.poll() is None:
|
||||
@@ -182,11 +191,11 @@ class MacView:
|
||||
if reason:
|
||||
raise MacViewError(reason)
|
||||
self.build()
|
||||
env = {**os.environ, "FRAME_MAC_VIEW_TOKEN": self.token}
|
||||
env = self.agent_environment()
|
||||
# The same port as before when restarting, so a running tunnel still
|
||||
# fits; otherwise (or if it's gone) whatever the system gives.
|
||||
for port in dict.fromkeys([self.port or 0, 0]):
|
||||
self.agent = subprocess.Popen([str(AGENT), "serve", "--port", str(port), "--page", str(PAGE),
|
||||
self.agent = subprocess.Popen([*self.agent_command(), "serve", "--port", str(port), "--page", str(PAGE),
|
||||
"--exit-on-eof"], env=env, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
|
||||
line = self.agent.stdout.readline()
|
||||
@@ -195,7 +204,7 @@ class MacView:
|
||||
break
|
||||
self.agent.kill()
|
||||
else:
|
||||
raise MacViewError(f"The Mac streaming helper didn't start: {line.strip() or 'no output'}")
|
||||
raise MacViewError(f"The streaming helper didn't start: {line.strip() or 'no output'}")
|
||||
if self.port != int(m.group(1)):
|
||||
self._drop_tunnel()
|
||||
self.port = int(m.group(1))
|
||||
@@ -211,7 +220,7 @@ class MacView:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
return json.load(r)
|
||||
except (urllib.error.URLError, OSError, ValueError) as e:
|
||||
raise MacViewError(f"The Mac streaming helper didn't answer: {e}")
|
||||
raise MacViewError(f"The streaming helper didn't answer: {e}")
|
||||
|
||||
# ---- the tunnel from the Frame ----
|
||||
|
||||
@@ -396,7 +405,7 @@ class MacView:
|
||||
# the agent no longer counts them, so only move when none are out there.
|
||||
others = self.shown - {src}
|
||||
self.ensure_tunnel(allow_new_port=not others)
|
||||
appid = panel_id(src)
|
||||
appid = panel_id(src, self.host)
|
||||
# Unique per launch, so a new window is never confused with an old one.
|
||||
tag = "fc" + secrets.token_hex(4)
|
||||
# A single-use ticket for this source, not Frame Control's key: the URL
|
||||
@@ -406,7 +415,7 @@ class MacView:
|
||||
"bpp": q["bpp"]}
|
||||
url = f"http://127.0.0.1:{self.remote_port}/view?{urlencode(params)}"
|
||||
w, h = fit(width or 1280, height or 720)
|
||||
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, *self.browser_flags))
|
||||
args = " ".join(_quote(str(a)) for a in (appid, url, w, h, tag, self.viewer_profile, *self.browser_flags))
|
||||
try:
|
||||
out = self.run("bash -s -- " + args, stdin=LAUNCH, timeout=60)
|
||||
except Exception as e: # noqa: BLE001 - the server's Failure carries the Frame's words
|
||||
@@ -438,7 +447,8 @@ class MacView:
|
||||
if self.shown or self.shows != shows or self.launching:
|
||||
return
|
||||
try:
|
||||
self.run("pkill -f '[f]rame-control/mac-view|[d]ata/frame-mac-view' || true", timeout=10)
|
||||
pattern = "[f]rame-control/" + self.viewer_profile + "|[d]ata/frame-" + self.viewer_profile
|
||||
self.run("pkill -f " + _quote(pattern) + " || true", timeout=10)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -449,7 +459,8 @@ class MacView:
|
||||
status = self.call("/status")
|
||||
windows = self.call("/windows").get("windows", []) if status.get("screen") else []
|
||||
displays = self.call("/displays").get("displays", [])
|
||||
return {"available": True, "screen": status.get("screen", False),
|
||||
return {"available": True, "host": self.host, "selecting": status.get("selecting", False),
|
||||
"selectionError": status.get("selectionError", ""), "screen": status.get("screen", False),
|
||||
"accessibility": status.get("accessibility", False), "streams": status.get("streams", []),
|
||||
"windows": windows, "displays": displays, "tunnel": self.tunnel_up(),
|
||||
"route": self.route}
|
||||
|
||||
@@ -0,0 +1,624 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frame Control's Windows/Linux host. Same ticket, H.264/JPEG, timing and
|
||||
input protocol as frame-mac-view; serves the same ui/mac-view.html.
|
||||
|
||||
Only loopback is bound. Frame Control owns the master token, the Frame gets
|
||||
one-source tickets and reconnect keys. Native libraries are bundled.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import ctypes as C
|
||||
import hashlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from frame_pc_capture import Native, Controller, Windows, WindowsInput, PortalInput, Encoded, GATE, dimensions, pipeline
|
||||
from frame_stream_stats import Stats
|
||||
|
||||
|
||||
# Viewer fields are untrusted JSON: float('x'), int(None), m['t'] missing.
|
||||
MALFORMED = (ValueError, TypeError, KeyError, OverflowError)
|
||||
|
||||
|
||||
class Grants:
|
||||
"""Caller holds the agent lock, including replacement and Stop."""
|
||||
def __init__(self, token):
|
||||
self.token, self.tickets, self.keys = token, {}, {}
|
||||
|
||||
def master(self, key):
|
||||
return isinstance(key, str) and secrets.compare_digest(key, self.token)
|
||||
|
||||
def ticket(self, src):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if v[1] > time.monotonic()}
|
||||
if len(self.tickets) >= 128:
|
||||
raise ValueError('Too many pending viewers')
|
||||
ticket = secrets.token_urlsafe(24)
|
||||
self.tickets[ticket] = (src, time.monotonic()+60, None)
|
||||
return ticket
|
||||
|
||||
def redeem(self, src, q):
|
||||
if self.master(q.get('k')):
|
||||
key = secrets.token_urlsafe(24)
|
||||
self.keys[key] = src
|
||||
return key
|
||||
entry = self.tickets.get(q.get('t'))
|
||||
if entry and entry[0] == src and entry[1] > time.monotonic():
|
||||
key = entry[2] or secrets.token_urlsafe(24)
|
||||
self.tickets[q['t']] = (src, entry[1], key)
|
||||
self.keys[key] = src
|
||||
return key
|
||||
key = q.get('r')
|
||||
return key if key and self.keys.get(key) == src else None
|
||||
|
||||
def ack(self, key):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if v[2] != key}
|
||||
|
||||
def revoke(self, src=None):
|
||||
self.tickets = {k: v for k, v in self.tickets.items() if src is not None and v[0] != src}
|
||||
self.keys = {k: v for k, v in self.keys.items() if src is not None and v != src}
|
||||
|
||||
|
||||
class WebSocket:
|
||||
def __init__(self, handler):
|
||||
self.sock, self.reader = handler.connection, handler.rfile
|
||||
self.lock = threading.Lock()
|
||||
self.closed = False
|
||||
self.sock.settimeout(5)
|
||||
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
|
||||
def send(self, data, opcode=1):
|
||||
if not isinstance(data, bytes):
|
||||
data = json.dumps(data, separators=(',', ':')).encode()
|
||||
n = len(data)
|
||||
head = bytes([0x80 | opcode, n]) if n < 126 else bytes([0x80 | opcode, 126]) + struct.pack('!H', n) if n <= 65535 else bytes([0x80 | opcode, 127]) + struct.pack('!Q', n)
|
||||
with self.lock:
|
||||
if self.closed:
|
||||
raise ConnectionError('Viewer disconnected')
|
||||
self.sock.sendall(head + data)
|
||||
|
||||
def exact(self, n):
|
||||
data = self.reader.read(n)
|
||||
if len(data) != n:
|
||||
raise ConnectionError('Viewer disconnected')
|
||||
return data
|
||||
|
||||
def receive(self):
|
||||
a, b = self.exact(2)
|
||||
opcode, size = a & 15, b & 127
|
||||
if a & 0x70 or not a & 0x80 or not b & 0x80 or opcode not in (1, 8, 9, 10):
|
||||
raise ValueError('Unsupported WebSocket frame')
|
||||
if size == 126:
|
||||
size = struct.unpack('!H', self.exact(2))[0]
|
||||
elif size == 127:
|
||||
size = struct.unpack('!Q', self.exact(8))[0]
|
||||
if size > 65536 or (opcode >= 8 and size > 125):
|
||||
raise ValueError('WebSocket message too large')
|
||||
mask = self.exact(4)
|
||||
data = bytes(v ^ mask[i % 4] for i, v in enumerate(self.exact(size)))
|
||||
if opcode == 8:
|
||||
raise ConnectionError('Viewer closed')
|
||||
if opcode == 9:
|
||||
self.send(data, 10)
|
||||
if opcode != 1:
|
||||
return {}
|
||||
message = json.loads(data)
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError('Expected an input object')
|
||||
return message
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
try:
|
||||
self.sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class Session:
|
||||
def __init__(self, agent, ws, key, source, query):
|
||||
self.agent, self.native, self.ws, self.key, self.source = agent, agent.native, ws, key, source
|
||||
self.src = source['src']
|
||||
self.codec = query.get('codec', 'h264')
|
||||
if self.codec not in ('h264', 'jpeg'):
|
||||
raise ValueError('Unsupported codec')
|
||||
self.fps = min(120, max(5, int(query.get('fps', 60))))
|
||||
bpp = float(query.get('bpp', .1))
|
||||
if not math.isfinite(bpp):
|
||||
raise ValueError('Invalid bitrate')
|
||||
self.w, self.h = dimensions(source['w'], source['h'], min(3840, max(320, int(query.get('max', 1920)))))
|
||||
self.bitrate = max(300000, int(self.w*self.h*self.fps*min(.5, max(.02, bpp))))
|
||||
self.controller = Controller(self.native.lib, self.fps, self.bitrate)
|
||||
self.stats = Stats(self.native.now)
|
||||
self.stop_event, self.key_event = threading.Event(), threading.Event()
|
||||
self.lock, self.input_lock = threading.RLock(), threading.RLock()
|
||||
self.pending, self.last_submit = {}, 0
|
||||
self.reconfiguring = False
|
||||
self.test_inputs = queue.Queue(maxsize=128)
|
||||
self.input = None if self.src == 'test' else WindowsInput(agent.windows, source) if agent.windows else PortalInput(self.native, source)
|
||||
self.writer = None
|
||||
self.released = False
|
||||
self.input_enabled = self.src == 'test' or self.source.get('devices', 3) == 3
|
||||
|
||||
def gate(self, stage, pts, capture, arrived):
|
||||
# Exceptions cannot cross a ctypes callback boundary.
|
||||
try:
|
||||
with self.lock:
|
||||
if self.stop_event.is_set() or self.reconfiguring:
|
||||
return -1
|
||||
if stage == 1:
|
||||
if pts in self.pending:
|
||||
self.pending[pts]['e0'] = self.native.now()
|
||||
return 1
|
||||
if stage == 0:
|
||||
self.stats.captured += 1
|
||||
self.controller.call('capture', arrived)
|
||||
now = self.native.now()
|
||||
state = self.controller.state()
|
||||
if len(self.pending) >= 3 or now-self.last_submit < 1000000/state['fps'] or not self.controller.call('gate', now, int(stage == 0)):
|
||||
if stage == 0:
|
||||
self.stats.skipped += 1
|
||||
return 0
|
||||
self.pending[pts] = dict(cap=capture, arr=arrived, e0=now, tier=state['tier'], br=state['target'])
|
||||
self.last_submit = now
|
||||
return 1
|
||||
except Exception:
|
||||
self.stop_event.set()
|
||||
return -1
|
||||
|
||||
STALL = 10000000
|
||||
|
||||
def stalled(self, now, opened, captured_at_open):
|
||||
"""PipeWire and WGC deliver frames only on screen damage, so an idle
|
||||
desktop legitimately produces no output. Fail only on evidence of a
|
||||
fault: a frame entered the encoder and never came out, or the capture
|
||||
never produced its initial frame after the pipeline opened."""
|
||||
with self.lock:
|
||||
oldest = min((r['e0'] for r in self.pending.values()), default=None)
|
||||
captured = self.stats.captured
|
||||
if oldest is not None and now-oldest > self.STALL:
|
||||
return 'The encoder stopped producing frames; check the video encoder'
|
||||
if captured == captured_at_open and now-opened > self.STALL:
|
||||
return 'No frames captured for 10 seconds; check capture permissions'
|
||||
return None
|
||||
|
||||
def fresh_pipewire(self):
|
||||
if 'portal' not in self.source:
|
||||
return
|
||||
error = C.create_string_buffer(1024)
|
||||
fd = self.native.lib.fc_portal_refresh(self.source['portal'], error, len(error))
|
||||
if fd < 0:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
self.source['fd'] = fd
|
||||
|
||||
def produce(self):
|
||||
native, capture = self.native.lib, None
|
||||
try:
|
||||
encoder = "x264enc" if self.src == "test" and self.native.has("x264enc") else self.agent.encoder
|
||||
self.fresh_pipewire()
|
||||
description = pipeline(self.source, sys.platform, encoder, self.w, self.h, self.fps, self.bitrate, self.codec)
|
||||
self.callback = GATE(self.gate)
|
||||
error = C.create_string_buffer(1024)
|
||||
capture = native.fc_capture_open(description.encode(), self.callback, error, len(error))
|
||||
if not capture:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
last_update = last_stats = self.native.now()
|
||||
last_config = opened = last_update
|
||||
captured_at_open = self.stats.captured
|
||||
applied = (self.w, self.h, self.bitrate)
|
||||
wanted = applied
|
||||
while not self.stop_event.is_set():
|
||||
while not self.test_inputs.empty():
|
||||
event = self.test_inputs.get_nowait()
|
||||
try:
|
||||
native.fc_capture_test(capture, int(event.get('i', 0)) & 0xffffffff)
|
||||
self.stats.input(event)
|
||||
except MALFORMED:
|
||||
pass # a bad probe field drops the event, not the stream
|
||||
output = Encoded()
|
||||
result = native.fc_capture_pull(capture, C.byref(output))
|
||||
now = self.native.now()
|
||||
if result < 0:
|
||||
raise RuntimeError(native.fc_capture_error(capture).decode(errors='replace'))
|
||||
if result:
|
||||
with self.lock:
|
||||
# At most three raw frames are in flight; no B-frames.
|
||||
# x264 offsets PTS, so match the FIFO encode order while
|
||||
# retaining the actual pre-encode capture timestamp.
|
||||
raw_pts = next(iter(self.pending), None)
|
||||
record = self.pending.get(raw_pts)
|
||||
if record is None:
|
||||
raise RuntimeError('Encoder changed frame timestamps; timing cannot be matched')
|
||||
data = C.string_at(output.data, output.size)
|
||||
f = self.stats.add(**record, e1=now, snd=now, b=len(data), k=output.key,
|
||||
w=output.width or self.w, h=output.height or self.h)
|
||||
self.controller.call('sent', f['s'], len(data)+17, now)
|
||||
self.ws.send(struct.pack('!BQII', output.key, max(0, f['cap']), f['s'], f['echo'])+data, 2)
|
||||
with self.lock:
|
||||
f['wire'] = self.native.now()
|
||||
self.pending.pop(raw_pts, None)
|
||||
stall = self.stalled(now, opened, captured_at_open)
|
||||
if stall:
|
||||
raise RuntimeError(stall)
|
||||
if self.key_event.is_set():
|
||||
self.key_event.clear()
|
||||
if self.codec == 'h264':
|
||||
native.fc_capture_key(capture)
|
||||
if now-last_update >= 100000:
|
||||
target = self.controller.update(now)
|
||||
state = self.controller.state()
|
||||
w, h = dimensions(self.w, self.h, max(320, int(max(self.w, self.h)*state['scale'])))
|
||||
if target and self.codec == 'h264' and encoder == 'x264enc':
|
||||
native.fc_capture_bitrate(capture, target)
|
||||
self.bitrate = target
|
||||
# Hardware properties are not uniformly mutable in PLAYING.
|
||||
# Drain then reopen the pipeline at a keyframe when its
|
||||
# budget changes materially. The portal fd/session stays
|
||||
# alive, so this does not bypass or repeat user consent.
|
||||
desired_bitrate = target or applied[2]
|
||||
bitrate_change = self.codec == 'h264' and encoder != 'x264enc' and abs(desired_bitrate-applied[2]) > applied[2]*.2
|
||||
if not self.reconfiguring and now-last_config >= 1000000 and ((w, h) != applied[:2] or bitrate_change):
|
||||
wanted = (w, h, desired_bitrate)
|
||||
with self.lock:
|
||||
self.reconfiguring = True
|
||||
last_update = now
|
||||
if self.reconfiguring:
|
||||
with self.lock:
|
||||
drained = not self.pending
|
||||
if drained:
|
||||
native.fc_capture_close(capture)
|
||||
capture = None
|
||||
self.fresh_pipewire()
|
||||
description = pipeline(self.source, sys.platform, encoder, wanted[0], wanted[1], self.fps, wanted[2], self.codec)
|
||||
capture = native.fc_capture_open(description.encode(), self.callback, error, len(error))
|
||||
if not capture:
|
||||
raise RuntimeError(error.value.decode(errors='replace'))
|
||||
applied, self.bitrate, last_config = wanted, wanted[2], now
|
||||
opened, captured_at_open = now, self.stats.captured
|
||||
with self.lock:
|
||||
self.reconfiguring = False
|
||||
if now-last_stats >= 1000000:
|
||||
self.ws.send(dict(self.stats.summary(), t='stats', bitrate=self.bitrate,
|
||||
size='%dx%d' % (self.w, self.h), tier=self.controller.state()['tier']))
|
||||
last_stats = now
|
||||
except Exception as e:
|
||||
try:
|
||||
self.ws.send({'t': 'error', 'message': str(e)})
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self.stop_event.set()
|
||||
if capture:
|
||||
native.fc_capture_close(capture)
|
||||
self.ws.close()
|
||||
|
||||
def start(self):
|
||||
if self.stop_event.is_set():
|
||||
self.controller.close()
|
||||
return
|
||||
self.ws.send(dict(t='hello', r=self.key))
|
||||
self.ws.send(dict(t='info', src=self.src, title=self.source.get('title', self.source.get('name', 'Test pattern')),
|
||||
app='PC', codec=self.codec, input=self.input_enabled,
|
||||
inputMessage='Allow pointer and keyboard control in the host sharing dialog.',
|
||||
aspect=self.w/self.h, warm=0))
|
||||
with self.lock:
|
||||
if self.stop_event.is_set():
|
||||
self.controller.close()
|
||||
return
|
||||
self.writer = threading.Thread(target=self.produce, daemon=True)
|
||||
self.writer.start()
|
||||
try:
|
||||
while not self.stop_event.is_set():
|
||||
m = self.ws.receive()
|
||||
t = m.get('t')
|
||||
if t == 'ping':
|
||||
self.ws.send(dict(t='pong', c=m.get('c', 0), a=self.native.now()))
|
||||
elif t == 'ack':
|
||||
with self.agent.lock:
|
||||
self.agent.grants.ack(self.key)
|
||||
elif t == 'key-frame':
|
||||
self.key_event.set()
|
||||
elif t in ('rx', 'fd', 'clock'):
|
||||
try:
|
||||
self.stats.report(m)
|
||||
except MALFORMED:
|
||||
pass
|
||||
if t == 'rx' and isinstance(m.get('s'), int):
|
||||
self.controller.call('ack', m['s'] & 0xffffffff, self.native.now())
|
||||
elif t in ('m', 'wheel', 'k', 'text', 'release'):
|
||||
with self.input_lock:
|
||||
if self.stop_event.is_set():
|
||||
break
|
||||
if self.input:
|
||||
if not self.input_enabled and t != 'release':
|
||||
continue
|
||||
try:
|
||||
self.input.handle(m)
|
||||
self.stats.input(m)
|
||||
except MALFORMED:
|
||||
pass # drop a malformed viewer event; keep the session
|
||||
except RuntimeError as e:
|
||||
self.input_enabled = False
|
||||
try:
|
||||
self.input.release()
|
||||
except RuntimeError:
|
||||
pass
|
||||
self.ws.send(dict(t='error', message=str(e)))
|
||||
elif m.get('i'):
|
||||
try:
|
||||
self.test_inputs.put_nowait(m)
|
||||
except queue.Full:
|
||||
pass
|
||||
finally:
|
||||
self.end()
|
||||
self.writer.join(6)
|
||||
if not self.writer.is_alive():
|
||||
self.controller.close()
|
||||
|
||||
def end(self):
|
||||
with self.lock:
|
||||
self.stop_event.set()
|
||||
self.ws.close()
|
||||
with self.input_lock:
|
||||
if self.input and not self.released:
|
||||
self.released = True
|
||||
try:
|
||||
self.input.release()
|
||||
except (RuntimeError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, native, token, page):
|
||||
self.native, self.page = native, page
|
||||
self.lock = threading.RLock()
|
||||
self.grants = Grants(token)
|
||||
self.sessions, self.sources = {}, {}
|
||||
self.next_id, self.selecting, self.selection_error = 1, False, ''
|
||||
self.windows = Windows() if sys.platform == 'win32' else None
|
||||
self.encoder = 'mfh264enc' if self.windows else 'vah264enc' if native.has('vah264enc') else 'x264enc'
|
||||
self.shutting_down = False
|
||||
self.selection_generation = 0
|
||||
|
||||
def lists(self):
|
||||
if self.windows:
|
||||
windows, displays = self.windows.sources()
|
||||
self.sources = {s['src']: s for s in windows + displays}
|
||||
return windows, displays
|
||||
return [{k: v for k, v in s.items() if k not in ('portal', 'fd', 'node')} for s in self.sources.values()], []
|
||||
|
||||
def source(self, src):
|
||||
if src == 'test':
|
||||
return dict(src='test', title='Test pattern', w=1280, h=720)
|
||||
self.lists()
|
||||
if src not in self.sources:
|
||||
raise ValueError('Choose a window or screen on this computer first')
|
||||
return dict(self.sources[src])
|
||||
|
||||
def select(self):
|
||||
if self.windows or self.selecting or self.shutting_down:
|
||||
return
|
||||
if len(self.sources) >= 8:
|
||||
raise ValueError('Stop a panel before sharing another source')
|
||||
self.selecting, self.selection_error = True, ''
|
||||
generation = self.selection_generation
|
||||
def choose():
|
||||
error = C.create_string_buffer(1024)
|
||||
portal = self.native.lib.fc_portal_select(error, len(error))
|
||||
with self.lock:
|
||||
self.selecting = False
|
||||
if not portal:
|
||||
self.selection_error = error.value.decode(errors='replace')
|
||||
elif self.shutting_down or generation != self.selection_generation:
|
||||
self.native.lib.fc_portal_close(portal)
|
||||
else:
|
||||
fd, node, w, h, devices = [self.native.lib.fc_portal_value(portal, i) for i in range(5)]
|
||||
src = 'window:' + secrets.token_hex(8)
|
||||
self.sources[src] = dict(src=src, title='Shared window or screen', app='Linux portal',
|
||||
w=w, h=h, fd=fd, node=node, portal=portal, devices=devices)
|
||||
threading.Thread(target=choose, daemon=True).start()
|
||||
|
||||
def stop(self, src=None):
|
||||
with self.lock:
|
||||
if src is None:
|
||||
self.selection_generation += 1
|
||||
self.grants.revoke(src)
|
||||
sessions = [s for s in self.sessions.values() if src is None or s.src == src]
|
||||
for session in sessions:
|
||||
try:
|
||||
session.ws.send(dict(t='close'))
|
||||
except OSError:
|
||||
pass
|
||||
session.end()
|
||||
for session in sessions:
|
||||
if session.writer and session.writer is not threading.current_thread():
|
||||
session.writer.join(6)
|
||||
# The capture is stopped before releasing its PipeWire fd/session.
|
||||
with self.lock:
|
||||
if not self.windows:
|
||||
for key, source in list(self.sources.items()):
|
||||
if src is None or key == src:
|
||||
if any(s.src == key and s.writer and s.writer.is_alive() for s in sessions):
|
||||
continue
|
||||
self.native.lib.fc_portal_close(source['portal'])
|
||||
self.sources.pop(key, None)
|
||||
return {'closed': len(sessions)}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = 'HTTP/1.1'
|
||||
|
||||
def log_message(self, *args):
|
||||
pass # URLs contain credentials
|
||||
|
||||
def reply(self, data, status=200, content='application/json'):
|
||||
if not isinstance(data, bytes):
|
||||
data = json.dumps(data).encode()
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', content)
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.send_header('Cache-Control', 'no-store')
|
||||
self.send_header('Connection', 'close')
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
self.close_connection = True
|
||||
|
||||
def do_GET(self):
|
||||
self.dispatch('GET')
|
||||
|
||||
def do_POST(self):
|
||||
self.dispatch('POST')
|
||||
|
||||
def dispatch(self, method):
|
||||
agent = self.server.agent
|
||||
url = urlsplit(self.path)
|
||||
q = {k: v[-1] for k, v in parse_qs(url.query).items()}
|
||||
try:
|
||||
if method == 'GET' and url.path == '/ping':
|
||||
return self.reply(b'frame-mac-view', content='text/plain')
|
||||
if method == 'GET' and url.path == '/view':
|
||||
return self.reply(agent.page.read_bytes(), content='text/html; charset=utf-8')
|
||||
if method == 'GET' and url.path == '/stream':
|
||||
return self.stream(q)
|
||||
if not agent.grants.master(q.get('k', self.headers.get('X-Token'))):
|
||||
return self.reply({'error': 'forbidden'}, 403)
|
||||
if method == 'POST' and url.path == '/close':
|
||||
return self.reply(agent.stop(q.get('src')))
|
||||
with agent.lock:
|
||||
windows, displays = agent.lists()
|
||||
if method == 'GET' and url.path == '/status':
|
||||
data = dict(version=1, host='windows' if agent.windows else 'linux', screen=True,
|
||||
accessibility=True, selecting=agent.selecting, selectionError=agent.selection_error,
|
||||
encoder=agent.encoder, finished=[], streams=[dict(id=i, src=s.src,
|
||||
title=s.source.get('title', ''), stats=s.stats.summary(), controller=s.controller.state())
|
||||
for i, s in agent.sessions.items() if not s.stop_event.is_set()])
|
||||
elif method == 'GET' and url.path in ('/windows', '/displays'):
|
||||
data = dict(windows=windows, displays=displays, screen=True)
|
||||
elif method == 'POST' and url.path == '/ticket':
|
||||
agent.source(q.get('src'))
|
||||
data = {'ticket': agent.grants.ticket(q['src'])}
|
||||
elif method == 'POST' and url.path == '/permissions':
|
||||
agent.select()
|
||||
data = {'selecting': agent.selecting}
|
||||
elif method == 'GET' and url.path == '/stats':
|
||||
data = dict(now=agent.native.now(), streams=[dict(id=i, src=s.src, controller=s.controller.state(),
|
||||
events=list(s.controller.events), **s.stats.snapshot(max(0, int(q.get('since', 0))), max(0, int(q.get('settle', 1500000)))))
|
||||
for i, s in agent.sessions.items() if q.get('id', str(i)) == str(i) and not s.stop_event.is_set()])
|
||||
elif method == 'POST' and url.path == '/bench':
|
||||
data = {'sent': 0}
|
||||
for s in agent.sessions.values():
|
||||
if s.src == q.get('src'):
|
||||
m = {k: v for k, v in q.items() if k not in ('k', 'src')}
|
||||
for k in ('x', 'y', 'interval'):
|
||||
if k in m:
|
||||
m[k] = float(m[k])
|
||||
s.ws.send(dict(m, t='bench'))
|
||||
data['sent'] += 1
|
||||
else:
|
||||
return self.reply({'error': 'not found'}, 404)
|
||||
self.reply(data)
|
||||
except (ValueError, RuntimeError) as e:
|
||||
self.reply({'error': str(e)}, 400)
|
||||
except (OSError, ConnectionError):
|
||||
self.close_connection = True
|
||||
|
||||
def stream(self, query):
|
||||
agent = self.server.agent
|
||||
with agent.lock:
|
||||
key = agent.grants.redeem(query.get('src'), query)
|
||||
if not key:
|
||||
return self.reply({'error': 'forbidden'}, 403)
|
||||
source = agent.source(query.get('src'))
|
||||
if self.headers.get('Upgrade', '').lower() != 'websocket' or self.headers.get('Sec-WebSocket-Version') != '13':
|
||||
return self.reply({'error': 'expected WebSocket'}, 400)
|
||||
wskey = self.headers.get('Sec-WebSocket-Key', '')
|
||||
if len(base64.b64decode(wskey, validate=True)) != 16:
|
||||
raise ValueError('Bad WebSocket key')
|
||||
if len(agent.sessions) >= 8:
|
||||
raise ValueError('At most eight panels may be open')
|
||||
# Stop can revoke and close only a registered session. Register
|
||||
# under the same lock as redemption, before any capture starts.
|
||||
session = Session(agent, WebSocket(self), key, source, query)
|
||||
for old in list(agent.sessions.values()):
|
||||
if old.key == key or old.src == source['src']:
|
||||
if old.key != key:
|
||||
agent.grants.keys.pop(old.key, None)
|
||||
try:
|
||||
old.ws.send(dict(t='close'))
|
||||
except OSError:
|
||||
pass
|
||||
old.end()
|
||||
if old.writer:
|
||||
old.writer.join(6)
|
||||
if old.writer.is_alive():
|
||||
session.controller.close()
|
||||
raise ValueError('The previous capture is still stopping; retry shortly')
|
||||
ident = agent.next_id
|
||||
agent.next_id += 1
|
||||
agent.sessions[ident] = session
|
||||
accept = base64.b64encode(hashlib.sha1((wskey+'258EAFA5-E914-47DA-95CA-C5AB0DC85B11').encode()).digest()).decode()
|
||||
self.send_response(101)
|
||||
self.send_header('Upgrade', 'websocket')
|
||||
self.send_header('Connection', 'Upgrade')
|
||||
self.send_header('Sec-WebSocket-Accept', accept)
|
||||
self.end_headers()
|
||||
try:
|
||||
session.start()
|
||||
except (OSError, ValueError, RuntimeError):
|
||||
session.end()
|
||||
finally:
|
||||
session.end()
|
||||
if session.writer:
|
||||
session.writer.join(6)
|
||||
if not session.writer or not session.writer.is_alive():
|
||||
session.controller.close()
|
||||
with agent.lock:
|
||||
agent.sessions.pop(ident, None)
|
||||
self.close_connection = True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('command', choices=['serve'])
|
||||
parser.add_argument('--port', type=int, default=0)
|
||||
parser.add_argument('--page', type=Path, required=True)
|
||||
parser.add_argument('--exit-on-eof', action='store_true')
|
||||
args = parser.parse_args()
|
||||
token = os.environ.get('FRAME_MAC_VIEW_TOKEN')
|
||||
if not token:
|
||||
raise SystemExit('Frame Control must provide a private token')
|
||||
native = Native()
|
||||
server = ThreadingHTTPServer(('127.0.0.1', args.port), Handler)
|
||||
agent = server.agent = Agent(native, token, args.page)
|
||||
if args.exit_on_eof:
|
||||
def eof():
|
||||
sys.stdin.buffer.read()
|
||||
with agent.lock:
|
||||
agent.shutting_down = True
|
||||
agent.stop()
|
||||
server.shutdown()
|
||||
threading.Thread(target=eof, daemon=True).start()
|
||||
print('listening on 127.0.0.1:%d' % server.server_port, flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
finally:
|
||||
agent.shutting_down = True
|
||||
agent.stop()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (OSError, RuntimeError, AttributeError) as e:
|
||||
raise SystemExit('PC streaming helper: ' + str(e))
|
||||
@@ -0,0 +1,383 @@
|
||||
"""Native PC capture and input bindings. No desktop-streaming app required.
|
||||
|
||||
The packaged native library contains our adapter and the shared rate controller;
|
||||
GStreamer supplies WGC/Media Foundation and PipeWire/VA-API/x264 libraries.
|
||||
"""
|
||||
import ctypes as C
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
NATIVE = Path(os.environ.get('FRAME_PC_NATIVE', ROOT / 'desktop' / 'bundle'))
|
||||
LIBRARY = NATIVE / ('pc-host.dll' if sys.platform == 'win32' else 'pc-host.so')
|
||||
|
||||
|
||||
class Encoded(C.Structure):
|
||||
_fields_ = [('data', C.c_void_p), ('size', C.c_int), ('key', C.c_int),
|
||||
('width', C.c_int), ('height', C.c_int), ('pts', C.c_int64)]
|
||||
|
||||
|
||||
GATE = C.CFUNCTYPE(C.c_int, C.c_int, C.c_int64, C.c_int64, C.c_int64)
|
||||
|
||||
|
||||
class Native:
|
||||
def __init__(self, path=LIBRARY):
|
||||
self.dll_dirs = []
|
||||
if sys.platform == 'win32':
|
||||
# Set process coordinates before loading GStreamer: a plugin may
|
||||
# create a helper HWND during discovery, after which it is too late.
|
||||
user = C.WinDLL('user32')
|
||||
user.SetProcessDpiAwarenessContext.argtypes = [C.c_void_p]
|
||||
user.SetProcessDpiAwarenessContext(C.c_void_p(-4))
|
||||
for folder in (path.parent, path.parent / 'bin'):
|
||||
self.dll_dirs.append(os.add_dll_directory(str(folder)))
|
||||
self.lib = C.CDLL(str(path))
|
||||
signatures = {
|
||||
'fc_now': (C.c_int64, []), 'fc_gst_init': (None, []),
|
||||
'fc_has_element': (C.c_int, [C.c_char_p]),
|
||||
'fc_new': (C.c_void_p, [C.c_int, C.c_int]), 'fc_free': (None, [C.c_void_p]),
|
||||
'fc_ceiling': (None, [C.c_void_p, C.c_int]),
|
||||
'fc_gate': (C.c_int, [C.c_void_p, C.c_int64, C.c_int]),
|
||||
'fc_capture': (None, [C.c_void_p, C.c_int64]),
|
||||
'fc_sent': (None, [C.c_void_p, C.c_uint32, C.c_int, C.c_int64]),
|
||||
'fc_ack': (C.c_int, [C.c_void_p, C.c_uint32, C.c_int64]),
|
||||
'fc_update': (C.c_int, [C.c_void_p, C.c_int64]),
|
||||
'fc_value': (C.c_int64, [C.c_void_p, C.c_int]),
|
||||
'fc_capture_open': (C.c_void_p, [C.c_char_p, GATE, C.c_char_p, C.c_int]),
|
||||
'fc_capture_pull': (C.c_int, [C.c_void_p, C.POINTER(Encoded)]),
|
||||
'fc_capture_error': (C.c_char_p, [C.c_void_p]),
|
||||
'fc_capture_bitrate': (None, [C.c_void_p, C.c_int]),
|
||||
'fc_capture_key': (None, [C.c_void_p]),
|
||||
'fc_capture_test': (None, [C.c_void_p, C.c_uint32]), 'fc_capture_close': (None, [C.c_void_p]),
|
||||
}
|
||||
if sys.platform.startswith('linux'):
|
||||
signatures.update({
|
||||
'fc_portal_select': (C.c_void_p, [C.c_char_p, C.c_int]),
|
||||
'fc_portal_close': (None, [C.c_void_p]),
|
||||
'fc_portal_refresh': (C.c_int, [C.c_void_p, C.c_char_p, C.c_int]),
|
||||
'fc_portal_value': (C.c_int, [C.c_void_p, C.c_int]),
|
||||
'fc_portal_input': (C.c_int, [C.c_void_p, C.c_int, C.c_double, C.c_double, C.c_int, C.c_int]),
|
||||
})
|
||||
for name, (result, args) in signatures.items():
|
||||
fn = getattr(self.lib, name)
|
||||
fn.restype, fn.argtypes = result, args
|
||||
self.lib.fc_gst_init()
|
||||
|
||||
def has(self, name):
|
||||
return bool(self.lib.fc_has_element(name.encode()))
|
||||
|
||||
def now(self):
|
||||
return self.lib.fc_now()
|
||||
|
||||
|
||||
class Controller:
|
||||
def __init__(self, lib, fps, ceiling):
|
||||
# close() may run on the stream thread while /status reads state();
|
||||
# after close every entry point is a no-op, never a NULL dereference.
|
||||
self.lib, self.lock, self.fps = lib, threading.RLock(), fps
|
||||
self.enabled = os.environ.get('FRAME_MAC_VIEW_ADAPT') != '0'
|
||||
self.ptr = lib.fc_new(fps, self.enabled)
|
||||
if not self.ptr:
|
||||
raise MemoryError('Unable to allocate the streaming controller')
|
||||
lib.fc_ceiling(self.ptr, ceiling)
|
||||
self.events = []
|
||||
|
||||
def state(self):
|
||||
with self.lock:
|
||||
names = ['target', 'ceiling', 'tier', 'fps', 'scale', 'baseRtt', 'inFlight', 'slack']
|
||||
if not self.ptr:
|
||||
out = dict.fromkeys(names, 0)
|
||||
out.update(fps=self.fps, scale=100)
|
||||
else:
|
||||
out = {k: self.lib.fc_value(self.ptr, i) for i, k in enumerate(names)}
|
||||
out.update(scale=out['scale'] / 100, baseRtt=out['baseRtt'] / 1000,
|
||||
slack=out['slack'] / 1000, adapt=self.enabled)
|
||||
return out
|
||||
|
||||
def call(self, name, *args):
|
||||
with self.lock:
|
||||
return getattr(self.lib, 'fc_' + name)(self.ptr, *args) if self.ptr else 0
|
||||
|
||||
def update(self, now):
|
||||
with self.lock:
|
||||
if not self.ptr:
|
||||
return 0
|
||||
old = self.state()
|
||||
result = self.lib.fc_update(self.ptr, now)
|
||||
state = self.state()
|
||||
if state['target'] < old['target'] or state['tier'] != old['tier']:
|
||||
self.events.append({'t': now, 'e': 'target %s bit/s; tier %s' % (state['target'], state['tier'])})
|
||||
self.events = self.events[-200:]
|
||||
return result
|
||||
|
||||
def close(self):
|
||||
with self.lock:
|
||||
if self.ptr:
|
||||
self.lib.fc_free(self.ptr)
|
||||
self.ptr = None
|
||||
|
||||
|
||||
def dimensions(w, h, maximum):
|
||||
scale = min(1, maximum / max(w, h))
|
||||
return max(2, int(w * scale) // 2 * 2), max(2, int(h * scale) // 2 * 2)
|
||||
|
||||
|
||||
def pipeline(source, platform, encoder, w, h, fps, bitrate, codec='h264'):
|
||||
"""Only locally constructed numeric source IDs enter the pipeline parser."""
|
||||
if source['src'] == 'test':
|
||||
capture = 'videotestsrc name=source is-live=true pattern=ball'
|
||||
elif platform == 'win32':
|
||||
kind, ident = source['src'].split(':')
|
||||
if kind not in ('window', 'display') or not re.fullmatch(r'[0-9]+', ident):
|
||||
raise ValueError('Invalid Windows capture source')
|
||||
prop = 'window-handle' if kind == 'window' else 'monitor-handle'
|
||||
capture = 'd3d11screencapturesrc capture-api=wgc show-cursor=true show-border=true %s=%d' % (prop, int(ident))
|
||||
else:
|
||||
capture = 'pipewiresrc fd=%d path=%d do-timestamp=true' % (source['fd'], source['node'])
|
||||
# The source gate runs before conversion/encoding. There is no leaky queue
|
||||
# of H.264 frames; every encoded reference frame reaches the socket.
|
||||
raw = '%s ! video/x-raw,framerate=%d/1 ! queue name=raw_queue leaky=downstream max-size-buffers=1 max-size-bytes=0 max-size-time=0 ! identity name=gate ! videoconvert ! videoscale add-borders=false ! video/x-raw,width=%d,height=%d' % (capture, fps, w, h)
|
||||
if codec == 'jpeg':
|
||||
enc = 'jpegenc name=enc quality=80'
|
||||
parse = ''
|
||||
else:
|
||||
choices = {
|
||||
'mfh264enc': 'mfh264enc name=enc low-latency=true bframes=0 gop-size=60',
|
||||
'vah264enc': 'vah264enc name=enc b-frames=0 key-int-max=60',
|
||||
'x264enc': 'x264enc name=enc tune=zerolatency speed-preset=ultrafast bframes=0 key-int-max=60',
|
||||
}
|
||||
enc = choices[encoder] + ' bitrate=%d' % max(1, bitrate // 1000)
|
||||
raw += ',format=NV12' if encoder != 'x264enc' else ',format=I420'
|
||||
parse = ' ! h264parse config-interval=-1 ! video/x-h264,stream-format=byte-stream,alignment=au'
|
||||
return raw + ' ! ' + enc + parse + ' ! appsink name=out sync=false max-buffers=2 drop=false'
|
||||
|
||||
|
||||
# X11 keysyms work on Wayland through the RemoteDesktop portal too. The host's
|
||||
# keyboard layout handles physical keys; Unicode text is sent as Unicode keysyms.
|
||||
KEYSYMS = {'Enter': 0xff0d, 'Tab': 0xff09, 'Backspace': 0xff08, 'Escape': 0xff1b,
|
||||
'Delete': 0xffff, 'ArrowLeft': 0xff51, 'ArrowUp': 0xff52, 'ArrowRight': 0xff53,
|
||||
'ArrowDown': 0xff54, 'Home': 0xff50, 'End': 0xff57, 'PageUp': 0xff55,
|
||||
'PageDown': 0xff56, 'Shift': 0xffe1, 'Control': 0xffe3, 'Alt': 0xffe9, 'Meta': 0xffeb}
|
||||
|
||||
|
||||
class PortalInput:
|
||||
def __init__(self, native, source):
|
||||
self.lib, self.source = native.lib, source
|
||||
self.buttons, self.keys = set(), set()
|
||||
|
||||
def emit(self, kind, x=0, y=0, code=0, down=0):
|
||||
if not self.lib.fc_portal_input(self.source['portal'], kind, x, y, code, down):
|
||||
raise RuntimeError('The desktop portal refused input; check the sharing permission')
|
||||
|
||||
def release(self):
|
||||
error = None
|
||||
for values, kind in ((self.buttons, 1), (self.keys, 3)):
|
||||
for code in list(values):
|
||||
try:
|
||||
self.emit(kind, code=code, down=0)
|
||||
values.discard(code)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
if error:
|
||||
raise error
|
||||
|
||||
def handle(self, m):
|
||||
t = m['t']
|
||||
if t == 'release':
|
||||
return self.release()
|
||||
if t in ('m', 'wheel'):
|
||||
x, y = (max(0, min(1, float(m.get(k, 0)))) for k in ('x', 'y'))
|
||||
self.emit(0, x * (self.source['w'] - 1), y * (self.source['h'] - 1))
|
||||
if t == 'm' and m.get('e') in ('up', 'down'):
|
||||
b = {0: 0x110, 1: 0x112, 2: 0x111}.get(m.get('b', 0))
|
||||
if b is not None:
|
||||
down = m['e'] == 'down'
|
||||
self.emit(1, code=b, down=down)
|
||||
(self.buttons.add if down else self.buttons.discard)(b)
|
||||
elif t == 'wheel':
|
||||
self.emit(2, max(-4096, min(4096, float(m.get('dx', 0)))), max(-4096, min(4096, float(m.get('dy', 0)))))
|
||||
elif t == 'k':
|
||||
key = str(m.get('key', ''))
|
||||
k = KEYSYMS.get(key) or (ord(key) if len(key) == 1 and ord(key) < 256 else
|
||||
0x1000000 + ord(key) if len(key) == 1 else 0)
|
||||
if k:
|
||||
down = m.get('e') == 'down'
|
||||
self.emit(3, code=k, down=down)
|
||||
(self.keys.add if down else self.keys.discard)(k)
|
||||
elif t == 'text':
|
||||
for ch in str(m.get('s', ''))[:4096]:
|
||||
k = ord(ch) if ord(ch) < 256 else 0x1000000 + ord(ch)
|
||||
self.emit(3, code=k, down=1)
|
||||
self.emit(3, code=k, down=0)
|
||||
|
||||
|
||||
class Windows:
|
||||
"""WGC source handles and SendInput. No elevation or global input hook."""
|
||||
def __init__(self):
|
||||
from ctypes import wintypes as W
|
||||
self.W = W
|
||||
self.user = C.WinDLL('user32', use_last_error=True)
|
||||
self.dwm = C.WinDLL('dwmapi')
|
||||
self.user.IsWindow.argtypes = [W.HWND]
|
||||
self.user.IsWindowVisible.argtypes = [W.HWND]
|
||||
self.user.IsIconic.argtypes = [W.HWND]
|
||||
self.user.GetWindowTextLengthW.argtypes = [W.HWND]
|
||||
self.user.GetWindowTextW.argtypes = [W.HWND, W.LPWSTR, C.c_int]
|
||||
self.user.GetWindowRect.argtypes = [W.HWND, C.POINTER(W.RECT)]
|
||||
self.user.SetForegroundWindow.argtypes = [W.HWND]
|
||||
self.user.GetForegroundWindow.argtypes = []
|
||||
self.user.GetForegroundWindow.restype = W.HWND
|
||||
self.dwm.DwmGetWindowAttribute.argtypes = [W.HWND, W.DWORD, C.c_void_p, W.DWORD]
|
||||
self.callback = C.WINFUNCTYPE(W.BOOL, W.HWND, W.LPARAM)
|
||||
self.monitor_callback = C.WINFUNCTYPE(W.BOOL, W.HMONITOR, W.HDC, C.POINTER(W.RECT), W.LPARAM)
|
||||
self.user.EnumWindows.argtypes = [self.callback, W.LPARAM]
|
||||
self.user.EnumDisplayMonitors.argtypes = [W.HDC, C.c_void_p, self.monitor_callback, W.LPARAM]
|
||||
class Mouse(C.Structure):
|
||||
_fields_ = [('dx', W.LONG), ('dy', W.LONG), ('data', W.DWORD), ('flags', W.DWORD),
|
||||
('time', W.DWORD), ('extra', C.c_size_t)]
|
||||
class Key(C.Structure):
|
||||
_fields_ = [('vk', W.WORD), ('scan', W.WORD), ('flags', W.DWORD), ('time', W.DWORD), ('extra', C.c_size_t)]
|
||||
class Union(C.Union):
|
||||
_fields_ = [('mouse', Mouse), ('key', Key)]
|
||||
class Input(C.Structure):
|
||||
_fields_ = [('type', W.DWORD), ('data', Union)]
|
||||
self.Mouse, self.Key, self.Input = Mouse, Key, Input
|
||||
self.user.SendInput.argtypes = [W.UINT, C.POINTER(Input), C.c_int]
|
||||
self.user.SendInput.restype = W.UINT
|
||||
|
||||
def rect(self, hwnd):
|
||||
r = self.W.RECT()
|
||||
if not self.user.IsWindow(hwnd) or self.user.IsIconic(hwnd):
|
||||
raise RuntimeError('The captured window closed or was minimized')
|
||||
# WGC captures the visible extended frame, excluding invisible resize borders.
|
||||
if self.dwm.DwmGetWindowAttribute(hwnd, 9, C.byref(r), C.sizeof(r)):
|
||||
if not self.user.GetWindowRect(hwnd, C.byref(r)):
|
||||
raise RuntimeError('Cannot locate the captured window')
|
||||
return r.left, r.top, r.right - r.left, r.bottom - r.top
|
||||
|
||||
def sources(self):
|
||||
windows, displays = [], []
|
||||
def window(hwnd, _):
|
||||
if not self.user.IsWindowVisible(hwnd) or self.user.IsIconic(hwnd):
|
||||
return True
|
||||
n = self.user.GetWindowTextLengthW(hwnd)
|
||||
if n:
|
||||
title = C.create_unicode_buffer(n + 1)
|
||||
self.user.GetWindowTextW(hwnd, title, n + 1)
|
||||
try:
|
||||
x, y, w, h = self.rect(hwnd)
|
||||
if w > 0 and h > 0:
|
||||
windows.append(dict(src='window:%d' % hwnd, id=hwnd, title=title.value,
|
||||
app='Windows', w=w, h=h, x=x, y=y))
|
||||
except RuntimeError:
|
||||
pass
|
||||
return True
|
||||
def monitor(handle, dc, rect, data):
|
||||
r = rect.contents
|
||||
displays.append(dict(src='display:%d' % handle, name='Display %d' % (len(displays) + 1),
|
||||
x=r.left, y=r.top, w=r.right-r.left, h=r.bottom-r.top))
|
||||
return True
|
||||
self.user.EnumWindows(self.callback(window), 0)
|
||||
self.user.EnumDisplayMonitors(None, None, self.monitor_callback(monitor), 0)
|
||||
return windows, displays
|
||||
|
||||
def send(self, mouse=None, key=None):
|
||||
inp = self.Input()
|
||||
inp.type = 0 if mouse else 1
|
||||
if mouse:
|
||||
inp.data.mouse = self.Mouse(*mouse, 0, 0)
|
||||
else:
|
||||
inp.data.key = self.Key(*key, 0, 0)
|
||||
if self.user.SendInput(1, C.byref(inp), C.sizeof(inp)) != 1:
|
||||
raise RuntimeError('Windows refused input (elevated apps and the secure desktop cannot be controlled)')
|
||||
|
||||
|
||||
class WindowsInput:
|
||||
def __init__(self, host, source):
|
||||
self.host, self.source = host, source
|
||||
self.buttons, self.keys = set(), set()
|
||||
|
||||
def release(self):
|
||||
error = None
|
||||
for button in list(self.buttons):
|
||||
try:
|
||||
self.host.send(mouse=(0, 0, 0, {0: 4, 1: 0x40, 2: 0x10}[button]))
|
||||
self.buttons.discard(button)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
for vk in list(self.keys):
|
||||
try:
|
||||
extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0
|
||||
self.host.send(key=(vk, 0, 2 | extended))
|
||||
self.keys.discard(vk)
|
||||
except RuntimeError as e:
|
||||
error = e
|
||||
if error:
|
||||
raise error
|
||||
|
||||
def focus(self):
|
||||
if not self.source['src'].startswith('window:'):
|
||||
return
|
||||
hwnd = int(self.source['src'].split(':')[1])
|
||||
user = self.host.user
|
||||
if user.GetForegroundWindow() != hwnd and not user.SetForegroundWindow(hwnd):
|
||||
raise RuntimeError('Windows could not focus this window. Bring it to the front on your PC, then press Stop and Show again.')
|
||||
if user.GetForegroundWindow() != hwnd:
|
||||
raise RuntimeError('The selected window is not in front. Reopen the view after bringing it to the front on your PC.')
|
||||
|
||||
def handle(self, m):
|
||||
t, u = m['t'], self.host.user
|
||||
if t == 'release':
|
||||
return self.release()
|
||||
if t in ('m', 'wheel'):
|
||||
source = self.source
|
||||
if source['src'].startswith('window:'):
|
||||
hwnd = int(source['src'].split(':')[1])
|
||||
x, y, w, h = self.host.rect(hwnd)
|
||||
if m.get('e') == 'down' or t == 'wheel':
|
||||
self.focus()
|
||||
else:
|
||||
x, y, w, h = (source[k] for k in ('x', 'y', 'w', 'h'))
|
||||
x += max(0, min(1, float(m.get('x', 0)))) * (w - 1)
|
||||
y += max(0, min(1, float(m.get('y', 0)))) * (h - 1)
|
||||
left, top, width, height = (u.GetSystemMetrics(i) for i in (76, 77, 78, 79))
|
||||
self.host.send(mouse=(round((x-left)*65535/max(1,width-1)), round((y-top)*65535/max(1,height-1)), 0, 0xc001))
|
||||
if t == 'm' and m.get('e') in ('up', 'down'):
|
||||
b = m.get('b', 0)
|
||||
if b in (0, 1, 2):
|
||||
down = m['e'] == 'down'
|
||||
self.host.send(mouse=(0, 0, 0, {0: 2, 1: 0x20, 2: 8}[b] * (1 if down else 2)))
|
||||
(self.buttons.add if down else self.buttons.discard)(b)
|
||||
elif t == 'wheel':
|
||||
for name, flag, direction in (('dy', 0x800, -1), ('dx', 0x1000, 1)):
|
||||
delta = int(max(-4096, min(4096, float(m.get(name, 0))))) * direction
|
||||
if delta:
|
||||
self.host.send(mouse=(0, 0, delta & 0xffffffff, flag))
|
||||
elif t == 'k':
|
||||
code, down = str(m.get('code', '')), m.get('e') == 'down'
|
||||
if down:
|
||||
self.focus()
|
||||
special = {'Enter': 13, 'Escape': 27, 'Tab': 9, 'Backspace': 8, 'Space': 32,
|
||||
'ArrowLeft': 37, 'ArrowUp': 38, 'ArrowRight': 39, 'ArrowDown': 40,
|
||||
'Delete': 46, 'Home': 36, 'End': 35, 'PageUp': 33, 'PageDown': 34,
|
||||
'ShiftLeft': 160, 'ShiftRight': 161, 'ControlLeft': 162, 'ControlRight': 163,
|
||||
'AltLeft': 164, 'AltRight': 165, 'MetaLeft': 91, 'MetaRight': 92}
|
||||
vk = special.get(code, 0)
|
||||
if re.fullmatch(r'Key[A-Z]', code) or re.fullmatch(r'Digit[0-9]', code):
|
||||
vk = ord(code[-1])
|
||||
if vk:
|
||||
extended = 1 if vk in (33, 34, 35, 36, 37, 38, 39, 40, 45, 46, 91, 92, 163, 165) else 0
|
||||
self.host.send(key=(vk, 0, extended | (0 if down else 2)))
|
||||
(self.keys.add if down else self.keys.discard)(vk)
|
||||
elif down and len(str(m.get('key', ''))) == 1:
|
||||
self.handle({'t': 'text', 's': m['key']})
|
||||
elif t == 'text':
|
||||
self.focus()
|
||||
raw = str(m.get('s', ''))[:4096].encode('utf-16-le')
|
||||
for i in range(0, len(raw), 2):
|
||||
unit = int.from_bytes(raw[i:i+2], 'little')
|
||||
self.host.send(key=(0, unit, 4))
|
||||
self.host.send(key=(0, unit, 6))
|
||||
@@ -0,0 +1,130 @@
|
||||
"""PC in the headset (Experimental, Windows only), behind the existing MacView
|
||||
tunnel/panel controller.
|
||||
|
||||
The public /api/macview name and viewer URL are the Mac feature's. Only helper
|
||||
launch, platform availability and source validation are different. No second
|
||||
viewer, SSH supervisor or benchmark launcher.
|
||||
|
||||
It is off until the person turns it on in Tools: until then nothing starts,
|
||||
no native library is loaded and no window is listed. The Linux adapter from
|
||||
PR #38 is in the tree but never offered (host_view only picks this on Windows).
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
from frame_macview import MacView, MacViewError, ROOT
|
||||
from frame_pc_capture import LIBRARY, NATIVE
|
||||
|
||||
|
||||
def prepend(env, name, path):
|
||||
"""An empty entry means the current directory to the loader; never add one."""
|
||||
rest = env.get(name, '')
|
||||
env[name] = str(path) + (os.pathsep + rest if rest else '')
|
||||
|
||||
|
||||
OFF = 'Off. Turn it on to list your windows and screens.'
|
||||
|
||||
|
||||
class PCView(MacView):
|
||||
viewer_profile = 'pc-view'
|
||||
host = 'windows' if sys.platform == 'win32' else 'linux'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.prefer_usb = False # Mac networksetup probe is platform-specific
|
||||
self.enabled = False
|
||||
|
||||
def unavailable(self):
|
||||
if sys.platform != 'win32':
|
||||
return 'PC in the headset is only available on Windows for now.'
|
||||
if not self.enabled:
|
||||
return OFF
|
||||
if not LIBRARY.is_file():
|
||||
return 'The PC streaming libraries are missing from this build. See docs/pc-in-headset.md.'
|
||||
return None
|
||||
|
||||
def state(self):
|
||||
result = super().state()
|
||||
result.update(host=self.host, experimental=True, enabled=self.enabled)
|
||||
return result
|
||||
|
||||
def set_enabled(self, on):
|
||||
"""Turning it off ends every stream and the helper; nothing restarts
|
||||
it until it's turned on again (ensure_agent asks unavailable())."""
|
||||
if on:
|
||||
with self.lock:
|
||||
self.enabled = True
|
||||
return self.state()
|
||||
try:
|
||||
self.stop() # while still enabled, so the viewers on the Frame close too
|
||||
except MacViewError:
|
||||
pass
|
||||
with self.lock:
|
||||
self.enabled = False
|
||||
self._drop_tunnel()
|
||||
agent, self.agent = self.agent, None
|
||||
if agent and agent.poll() is None:
|
||||
# EOF gives the host a chance to release held input, including on
|
||||
# Windows where terminate is uncatchable.
|
||||
if agent.stdin:
|
||||
agent.stdin.close()
|
||||
try:
|
||||
agent.wait(10)
|
||||
except subprocess.TimeoutExpired:
|
||||
agent.kill()
|
||||
agent.wait(5)
|
||||
return self.state()
|
||||
|
||||
def build(self):
|
||||
pass # native libraries are built and bundled with the app
|
||||
|
||||
def agent_command(self):
|
||||
return [sys.executable, str(ROOT / 'ui' / 'frame_pc_agent.py')]
|
||||
|
||||
def agent_environment(self):
|
||||
env = super().agent_environment()
|
||||
env['GST_PLUGIN_PATH_1_0'] = str(NATIVE / 'lib' / 'gstreamer-1.0')
|
||||
env['GST_PLUGIN_SYSTEM_PATH_1_0'] = ''
|
||||
cache = os.path.join(os.path.expanduser('~'), '.cache', 'frame-control')
|
||||
os.makedirs(cache, exist_ok=True)
|
||||
env['GST_REGISTRY_1_0'] = os.path.join(cache, 'gstreamer-registry.bin')
|
||||
env['GST_REGISTRY_FORK'] = 'no'
|
||||
if sys.platform == 'win32':
|
||||
prepend(env, 'PATH', NATIVE / 'bin')
|
||||
else:
|
||||
prepend(env, 'LD_LIBRARY_PATH', NATIVE / 'lib')
|
||||
env['PIPEWIRE_MODULE_DIR'] = str(NATIVE / 'lib' / 'pipewire-0.3')
|
||||
env['SPA_PLUGIN_DIR'] = str(NATIVE / 'lib' / 'spa-0.2')
|
||||
env['PIPEWIRE_CONFIG_DIR'] = str(NATIVE / 'share' / 'pipewire')
|
||||
env['PIPEWIRE_CONFIG_NAME'] = 'client.conf'
|
||||
return env
|
||||
|
||||
def shutdown(self):
|
||||
self.closing = True
|
||||
try:
|
||||
self.stop()
|
||||
except MacViewError:
|
||||
pass
|
||||
if self.tunnel and self.tunnel.poll() is None:
|
||||
self.tunnel.terminate()
|
||||
if self.agent and self.agent.poll() is None:
|
||||
# EOF gives the host a chance to release held input and portal
|
||||
# sessions, including on Windows where terminate is uncatchable.
|
||||
if self.agent.stdin:
|
||||
self.agent.stdin.close()
|
||||
try:
|
||||
self.agent.wait(10)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.agent.kill()
|
||||
self.agent.wait(5)
|
||||
|
||||
def _show(self, src, quality, width, height):
|
||||
if src.startswith('separate:'):
|
||||
raise MacViewError('Separate virtual displays are a Mac-only feature. Choose a window or screen.')
|
||||
return super()._show(src, quality, width, height)
|
||||
|
||||
|
||||
def host_view(*args, **kwargs):
|
||||
"""Windows gets the PC host; everywhere else keeps the Mac view, which
|
||||
reports itself unavailable off macOS."""
|
||||
return (PCView if sys.platform == 'win32' else MacView)(*args, **kwargs)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""The desktop stream's timing records, in the Mac viewer/bench schema.
|
||||
|
||||
Times are host monotonic microseconds. Zero means unknown, never a fabricated
|
||||
capture/display timestamp. Storage is bounded to the same 4096/512 records as
|
||||
Stats.swift. The existing macview-bench.py grades these records unchanged.
|
||||
"""
|
||||
from collections import OrderedDict
|
||||
import threading
|
||||
|
||||
|
||||
def distribution(values):
|
||||
values = sorted(v for v in values if v is not None)
|
||||
if not values:
|
||||
return {}
|
||||
return {key: round(values[min(len(values)-1, int((len(values)-1)*p + .5))], 1)
|
||||
for key, p in (('p50', .5), ('p95', .95))}
|
||||
|
||||
|
||||
class Stats:
|
||||
def __init__(self, now):
|
||||
self.now, self.lock = now, threading.RLock()
|
||||
self.frames, self.inputs = OrderedDict(), OrderedDict()
|
||||
self.captured = self.skipped = self.dropped = 0
|
||||
self.rtt, self.decoder, self.synced = 0, '', False
|
||||
self.sequence = 0
|
||||
|
||||
def add(self, **values):
|
||||
with self.lock:
|
||||
self.sequence = (self.sequence + 1) & 0xffffffff
|
||||
record = dict.fromkeys(('s', 'k', 'b', 'w', 'h', 'cap', 'arr', 'e0', 'e1', 'snd',
|
||||
'wire', 'rx', 'dec', 'drw', 'vs', 'echo', 'tier', 'br'), 0)
|
||||
record.update(values, s=self.sequence)
|
||||
self.frames[self.sequence] = record
|
||||
while len(self.frames) > 4096:
|
||||
self.frames.popitem(last=False)
|
||||
for event in self.inputs.values():
|
||||
if not event['frame'] and event['inj'] <= record['cap']:
|
||||
event.update(frame=self.sequence, cap=record['cap'])
|
||||
record['echo'] = event['id']
|
||||
return record
|
||||
|
||||
def report(self, message):
|
||||
with self.lock:
|
||||
if message['t'] == 'rx':
|
||||
frame = self.frames.get(message.get('s'))
|
||||
if frame and isinstance(message.get('r'), (int, float)):
|
||||
frame['rx'] = int(message['r'])
|
||||
elif message['t'] == 'fd':
|
||||
for item in message.get('f', [])[:4096]:
|
||||
if isinstance(item, list) and len(item) >= 4:
|
||||
frame = self.frames.get(item[0])
|
||||
if frame:
|
||||
for name, val in zip(('dec', 'drw', 'vs'), item[1:4]):
|
||||
if isinstance(val, (int, float)):
|
||||
frame[name] = int(val)
|
||||
self.dropped += max(0, int(message.get('drop', 0)))
|
||||
elif message['t'] == 'clock':
|
||||
self.rtt = float(message.get('rtt', 0))
|
||||
self.decoder = str(message.get('dec', ''))[:1024]
|
||||
self.synced = True
|
||||
|
||||
def input(self, m):
|
||||
if not isinstance(m.get('i'), int) or not m['i']:
|
||||
return
|
||||
with self.lock:
|
||||
self.inputs[m['i']] = dict(id=m['i'], kind=m['t'], tv=m.get('tv') or 0,
|
||||
inj=self.now(), frame=0, cap=0)
|
||||
while len(self.inputs) > 512:
|
||||
self.inputs.popitem(last=False)
|
||||
|
||||
def summary(self):
|
||||
with self.lock:
|
||||
now = self.now()
|
||||
frames = [f for f in self.frames.values() if f['cap'] >= now-2000000]
|
||||
out = dict(captured=self.captured, skipped=self.skipped, dropped=self.dropped,
|
||||
rtt=self.rtt, decoder=self.decoder, synced=self.synced,
|
||||
fps=sum(f['vs'] > 0 for f in frames)/2, sentFps=len(frames)/2,
|
||||
mbps=round(sum(f['b'] for f in frames)*4/1e6, 2))
|
||||
for key, start, end in (('capture', 'cap', 'arr'), ('queue', 'arr', 'e0'),
|
||||
('encode', 'e0', 'e1'), ('network', 'e1', 'rx'),
|
||||
('decode', 'rx', 'dec'), ('draw', 'dec', 'drw'), ('total', 'cap', 'drw')):
|
||||
out[key] = distribution([(f[end]-f[start])/1000 for f in frames if f[start] and f[end]])
|
||||
return out
|
||||
|
||||
def snapshot(self, since=0, settle=1500000):
|
||||
with self.lock:
|
||||
return dict(frames=[dict(f) for f in self.frames.values() if f['s'] > since and f['cap'] <= self.now()-settle],
|
||||
inputs=[dict(i) for i in self.inputs.values()], captured=self.captured, summary=self.summary())
|
||||
+52
-9
@@ -1420,7 +1420,7 @@
|
||||
|
||||
<div class="page" data-page="tools">
|
||||
<div class="workspace-heading"><div><a href="#tools" id="toolBack" class="workspace-link" hidden>← All tools</a><h1 id="toolTitle">Tools</h1><p id="toolSubtitle">Choose a task for your Frame.</p></div></div>
|
||||
<div id="toolDirectory"><section class="tool-group"><h2>Content & files</h2><div class="quick-routes"><a href="#transfer"><span>Send to Frame</span><small>Add files, APKs or your own games</small></a><a href="#media"><span>Media player</span><small>Play your own videos and media</small></a><a href="#linuxApps"><span>Linux apps</span><small>Install and manage desktop apps</small></a></div></section><section class="tool-group"><h2>Play & control</h2><div class="quick-routes"><a href="#capture"><span>Capture & Live</span><small>See and control the headset view</small></a><a href="#pad" title="Type on the headset (T)"><span>Type on the headset</span><small>Keyboard, trackpad and clipboard</small></a><a href="#panels"><span>Panel switcher</span><small>Bring an open panel into view</small></a><a href="#vrTools"><span>VR tools</span><small>Performance HUD and VR utilities</small></a><a href="#comfort"><span>Family & comfort</span><small>Session reminders and breaks</small></a><a href="#macview" id="macToolLink" hidden><span>Mac in the headset</span><small>Share Mac windows and displays</small></a></div></section><section class="tool-group"><h2>Your Frame</h2><div class="quick-routes"><a href="#backup" id="backupToolLink"><span>Backup & recovery</span><small>Save games and settings, restore after a reset</small></a></div><a class="workspace-link" href="#settings">Connection, display & preferences →</a></section><section class="tool-group"><h2>Assistant</h2><a class="workspace-link" href="/assistant">Open assistant & AI agents →</a><p class="sub">Use your own model endpoint or review a proposed MCP action.</p></section></div>
|
||||
<div id="toolDirectory"><section class="tool-group"><h2>Content & files</h2><div class="quick-routes"><a href="#transfer"><span>Send to Frame</span><small>Add files, APKs or your own games</small></a><a href="#media"><span>Media player</span><small>Play your own videos and media</small></a><a href="#linuxApps"><span>Linux apps</span><small>Install and manage desktop apps</small></a></div></section><section class="tool-group"><h2>Play & control</h2><div class="quick-routes"><a href="#capture"><span>Capture & Live</span><small>See and control the headset view</small></a><a href="#pad" title="Type on the headset (T)"><span>Type on the headset</span><small>Keyboard, trackpad and clipboard</small></a><a href="#panels"><span>Panel switcher</span><small>Bring an open panel into view</small></a><a href="#vrTools"><span>VR tools</span><small>Performance HUD and VR utilities</small></a><a href="#comfort"><span>Family & comfort</span><small>Session reminders and breaks</small></a><a href="#macview" id="macToolLink" hidden><span>Mac in the headset</span><small>Share Mac windows and displays</small></a><a href="#macview" id="pcToolLink" hidden><span>PC in the headset</span><small>Experimental: share this PC's windows and screens</small></a></div></section><section class="tool-group"><h2>Your Frame</h2><div class="quick-routes"><a href="#backup" id="backupToolLink"><span>Backup & recovery</span><small>Save games and settings, restore after a reset</small></a></div><a class="workspace-link" href="#settings">Connection, display & preferences →</a></section><section class="tool-group"><h2>Assistant</h2><a class="workspace-link" href="/assistant">Open assistant & AI agents →</a><p class="sub">Use your own model endpoint or review a proposed MCP action.</p></section></div>
|
||||
<div class="grid-3" id="toolWorkspace">
|
||||
<section class="panel" id="vrTools">
|
||||
<div class="shelf-head"><h2>VR comfort and performance</h2><span class="spacer"></span><span class="sub" id="vrUpdated">Waiting for the Frame</span></div>
|
||||
@@ -1703,8 +1703,19 @@
|
||||
</section>
|
||||
|
||||
<section class="panel" id="macview" hidden>
|
||||
<div class="shelf-head"><h2>Mac in the headset</h2><span class="count" id="mvCount"></span><span class="spacer"></span>
|
||||
<div class="shelf-head"><h2 id="mvTitle">Mac in the headset</h2><span class="tag" id="mvTag" hidden>Experimental</span><span class="count" id="mvCount"></span><span class="spacer"></span>
|
||||
<button class="small" id="mvRefresh">Refresh</button></div>
|
||||
<div id="mvPc" hidden>
|
||||
<p class="hint" style="color:var(--warn)">Experimental. Shows this PC's windows and screens as panels in the
|
||||
headset, with laser and keyboard input. It builds and passes its tests on Windows in CI, but hasn't been tried on a real
|
||||
Windows desktop or with a headset yet: capture, hardware encoding and laser clicks may not work. Clicks and keys
|
||||
can't reach administrator apps or the sign-in screen.
|
||||
<a href="https://github.com/saphid/frame-control/blob/main/docs/pc-in-headset.md" target="_blank" rel="noopener noreferrer">What's verified</a></p>
|
||||
<label class="mv-opt"><input type="checkbox" id="mvPcOn"> Turn on PC in the headset
|
||||
<span class="sub">off by default; nothing runs and no windows are listed until it's on</span></label>
|
||||
<div class="sub" id="mvPcState" role="status" aria-live="polite" hidden></div>
|
||||
</div>
|
||||
<div id="mvMain">
|
||||
<label class="mv-opt"><input type="checkbox" id="mvSeparate" checked> Give each window its own display
|
||||
<span class="sub">its menus and sheets come along, nothing covers it, and it goes back when you stop</span></label>
|
||||
<div class="row" style="margin:0 0 12px"><label for="mvQuality">Picture quality</label>
|
||||
@@ -1721,7 +1732,8 @@
|
||||
<button class="small danger" data-mv="stopall">Stop all</button>
|
||||
</div>
|
||||
<div class="hint">Each window or screen becomes its own panel in the headset. Place it with the SteamVR dashboard
|
||||
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on the Mac's keyboard.</div>
|
||||
(Float in World, Move, Size). Point and click with the laser, scroll with the thumbstick, and type on your computer's keyboard.</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel" id="privacy">
|
||||
@@ -5546,7 +5558,7 @@ $("sessionGame").onclick = e => {
|
||||
};
|
||||
|
||||
// ---- Mac in the headset: windows and displays as panels (ui/frame_macview.py) ----
|
||||
let mvSeq = 0, mvTimer = 0;
|
||||
let mvSeq = 0, mvTimer = 0, mvPcAsked = false;
|
||||
// streaming: the src it's shown as (a window may be "separate:<id>"), or "".
|
||||
// How a live stream is doing: frames shown, Mac-to-headset delay, bitrate, and
|
||||
// whether the network made it step down (fewer frames, then fewer pixels).
|
||||
@@ -5572,9 +5584,27 @@ async function loadMacView(restart) {
|
||||
try { s = await api("/api/macview" + (restart ? "?restart=1" : "")); } catch (e) { err = e; }
|
||||
if (seq !== mvSeq || location.hash !== "#macview") return;
|
||||
const box = $("macview");
|
||||
if (err) { box.hidden = false; return failed($("mvWindows"), err); }
|
||||
if (!s.available) { box.hidden = true; return; }
|
||||
if (err) { box.hidden = $("mvMain").hidden = false; return failed($("mvWindows"), err); }
|
||||
// PC in the headset (Windows, Experimental) is off until turned on here.
|
||||
const pc = !!s.host && s.host !== "mac";
|
||||
$("mvTitle").textContent = pc ? "PC in the headset" : "Mac in the headset";
|
||||
$("mvTag").hidden = $("mvPc").hidden = !pc;
|
||||
$("mvPcOn").checked = !!s.enabled;
|
||||
if (pc && !s.enabled && clientPreferences.get("pcView") === "1" && !restart && !mvPcAsked) {
|
||||
mvPcAsked = true; // once per page: a failure to start shouldn't loop
|
||||
return mvPcEnable(true, true);
|
||||
}
|
||||
if (!s.available) {
|
||||
box.hidden = !pc;
|
||||
$("mvMain").hidden = true;
|
||||
$("mvCount").textContent = "";
|
||||
if (pc) { $("mvPcState").textContent = s.reason || ""; $("mvPcState").hidden = !s.reason; }
|
||||
return;
|
||||
}
|
||||
box.hidden = false;
|
||||
$("mvMain").hidden = false;
|
||||
$("mvPcState").hidden = true;
|
||||
$("mvSeparate").closest("label").hidden = pc; // separate virtual displays are a Mac feature
|
||||
const live = new Set((s.streams || []).map(x => x.src));
|
||||
const byStream = src => (s.streams || []).find(x => x.src === src);
|
||||
$("mvCount").textContent = live.size ? `${live.size} showing${s.route === "usb" ? " · over USB-C" : ""}` : "";
|
||||
@@ -5584,6 +5614,7 @@ async function loadMacView(restart) {
|
||||
$("mvPerm").hidden = !perm.length;
|
||||
$("mvPerm").innerHTML = perm.join(" ") + `<div class="row"><button class="small action" data-mv="perm">Allow…</button>
|
||||
<span class="sub">Then press Refresh. macOS may ask you to reopen Frame Control.</span></div>`;
|
||||
if (pc) $("mvPerm").hidden = true; // Windows asks for no permission; elevated apps just refuse input
|
||||
const liveAs = src => live.has(src) ? src : src.startsWith("window:") && live.has("separate:" + src.slice(7)) ? "separate:" + src.slice(7) : "";
|
||||
$("mvDisplays").innerHTML = (s.displays || []).map(d =>
|
||||
mvRow(d.src, `Whole screen: ${d.name}`, `${d.w}×${d.h}${d.main ? " · main display" : ""}`, d.w, d.h, liveAs(d.src), byStream(liveAs(d.src)))).join("");
|
||||
@@ -5591,16 +5622,26 @@ async function loadMacView(restart) {
|
||||
$("mvWinCount").textContent = wins.length ? `${wins.length}` : "";
|
||||
$("mvWindows").innerHTML = !s.screen ? `<div class="sub">Windows appear here once Screen Recording is allowed.</div>`
|
||||
: wins.length ? wins.map(w => mvRow(w.src, w.title || w.app, `${w.title ? w.app + " · " : ""}${w.w}×${w.h}`, w.w, w.h, liveAs(w.src), byStream(liveAs(w.src)))).join("")
|
||||
: `<div class="sub">No windows open on this Mac's current desktop.</div>`;
|
||||
: `<div class="sub">No windows open on this ${pc ? "PC" : "Mac's current desktop"}.</div>`;
|
||||
// Keep the numbers fresh while something is in the headset and the card is on screen.
|
||||
clearTimeout(mvTimer);
|
||||
const tick = () => { mvTimer = document.hidden || !box.offsetParent ? setTimeout(tick, 3000) : (loadMacView(), 0); };
|
||||
if (live.size) mvTimer = setTimeout(tick, 3000);
|
||||
}
|
||||
$("mvRefresh").onclick = () => loadMacView(true);
|
||||
// The choice is remembered on this computer; the server starts with it off.
|
||||
async function mvPcEnable(on, quiet) {
|
||||
if (!quiet) clientPreferences.set("pcView", on ? "1" : "0");
|
||||
const turn = () => api("/api/macview", { action: "enable", on });
|
||||
if (quiet) { try { await turn(); } catch {} }
|
||||
else await act(on ? "Turn on PC in the headset" : "Turn off PC in the headset", async () =>
|
||||
({ ...await turn(), message: on ? "PC in the headset is on." : "PC in the headset is off. Nothing is shared." }), $("mvPcOn"));
|
||||
loadMacView();
|
||||
}
|
||||
$("mvPcOn").onchange = e => mvPcEnable(e.target.checked);
|
||||
// ---- client preferences: native storage survives changing localhost origins ----
|
||||
const clientPreferences = (() => {
|
||||
const keys = ['mvSeparate', 'padUsed'], values = {}, changed = new Set();
|
||||
const keys = ['mvSeparate', 'padUsed', 'pcView'], values = {}, changed = new Set();
|
||||
const valid = value => value === '0' || value === '1';
|
||||
const remember = (key, value) => {
|
||||
values[key] = value;
|
||||
@@ -5637,7 +5678,7 @@ $("macview").onclick = async e => {
|
||||
const b = e.target.closest("[data-mv]"); if (!b) return;
|
||||
const src = b.dataset.src;
|
||||
if (b.dataset.mv === "show") {
|
||||
const as = $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
|
||||
const as = !$("mvSeparate").closest("label").hidden && $("mvSeparate").checked && src.startsWith("window:") ? "separate:" + src.slice(7) : src;
|
||||
await act(`Show ${b.dataset.title} in the headset`, async () => {
|
||||
const r = await api("/api/macview", { action: "show", src: as, quality: $("mvQuality").value,
|
||||
w: +b.dataset.w || undefined, h: +b.dataset.h || undefined });
|
||||
@@ -7160,6 +7201,8 @@ api("/api/host").then(h => {
|
||||
Object.assign(HOST, h);
|
||||
window.frameControlCenter?.host(h);
|
||||
$("macToolLink").hidden = h.mobile || h.os !== "macOS";
|
||||
$("pcToolLink").hidden = h.mobile || h.os !== "Windows";
|
||||
if (!h.mobile && h.os === "Windows") TOOL_TITLES.macview = "PC in the headset";
|
||||
$("backupToolLink").hidden = !!h.mobile; // backups are folders on a computer
|
||||
if (h.mobile) return;
|
||||
$("offSetup").hidden = false; $("offSetup").onclick = setUpActive; // the server opens it in a terminal
|
||||
|
||||
+5
-5
@@ -22,7 +22,7 @@
|
||||
</head>
|
||||
<body>
|
||||
<canvas id="c" width="16" height="9"></canvas>
|
||||
<div id="note">Connecting to the Mac…</div>
|
||||
<div id="note">Connecting to your computer…</div>
|
||||
<div id="hud" hidden></div>
|
||||
<script>
|
||||
"use strict";
|
||||
@@ -243,14 +243,14 @@ async function connect() {
|
||||
// Refused before ever getting a key: the ticket expired or was revoked,
|
||||
// and retrying can't help.
|
||||
if (!reconnectKey && ++failedStarts >= 3) {
|
||||
show("This view has expired. Press Show in Frame Control on the Mac to open it again.");
|
||||
show("This view has expired. Press Show in Frame Control on your computer to open it again.");
|
||||
return;
|
||||
}
|
||||
hideNoteOnFrame = true;
|
||||
// The Mac may be asleep or the tunnel restarting: keep trying.
|
||||
retry = Math.min(retry + 1, 6);
|
||||
// Keep the Mac's reason (a missing permission, a closed window) on screen.
|
||||
show(lastError ? `${lastError} Retrying…` : "Lost the Mac. Reconnecting…");
|
||||
show(lastError ? `${lastError} Retrying…` : "Lost the computer. Reconnecting…");
|
||||
setTimeout(connect, 500 * 2 ** retry);
|
||||
};
|
||||
}
|
||||
@@ -269,7 +269,7 @@ function control(m) {
|
||||
document.title = tag ? `${name} [${tag}]` : name;
|
||||
if (!m.input) {
|
||||
hideNoteOnFrame = false; // a warning, not a connection notice: let it stay its 8 s
|
||||
show("Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
|
||||
show(m.inputMessage || "Clicks and keys need Accessibility permission on the Mac (Frame Control asks for it).", 8000);
|
||||
}
|
||||
} else if (m.t === "error") {
|
||||
stats.errors.push(m.message);
|
||||
@@ -391,7 +391,7 @@ addEventListener("keydown", e => onKey(e, true));
|
||||
addEventListener("keyup", e => onKey(e, false));
|
||||
addEventListener("blur", () => send({ t: "release" }));
|
||||
|
||||
show("Connecting to the Mac…");
|
||||
show("Connecting to your computer…");
|
||||
connect();
|
||||
</script>
|
||||
</body>
|
||||
|
||||
+11
-4
@@ -59,6 +59,7 @@ import frame_contact # noqa: E402
|
||||
import frame_host # noqa: E402
|
||||
import frame_link # noqa: E402
|
||||
import frame_macview # noqa: E402
|
||||
import frame_pcview # noqa: E402
|
||||
import frame_media # noqa: E402
|
||||
import frame_panels # noqa: E402
|
||||
import frame_quest # noqa: E402
|
||||
@@ -2832,12 +2833,13 @@ def push_media(path):
|
||||
"id": token + "/" + name}
|
||||
|
||||
|
||||
# ---- Mac in the headset (frame_macview.py) ----------------------------------
|
||||
# ---- Mac in the headset (frame_macview.py), PC in the headset (frame_pcview.py) ----
|
||||
|
||||
# The tunnel gets its own connection: the shared master's options would win
|
||||
# over anything added after them.
|
||||
macview = frame_macview.MacView(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
|
||||
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin, timeout=timeout),
|
||||
macview = frame_pcview.host_view(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=8"],
|
||||
lambda remote, stdin=None, timeout=30: ssh(remote, stdin=stdin.encode("utf-8") if stdin is not None else None,
|
||||
timeout=timeout, text=False).decode("utf-8", errors="replace"),
|
||||
FRAME, track=_live_tunnels.add)
|
||||
|
||||
|
||||
@@ -2854,7 +2856,8 @@ def macview_state(query=None):
|
||||
|
||||
|
||||
def macview_action(body):
|
||||
"""{action: show|stop|permissions, src, quality, w, h}."""
|
||||
"""{action: show|stop|permissions, src, quality, w, h} | {action: enable, on}.
|
||||
enable turns PC in the headset (Experimental, Windows only) on or off."""
|
||||
if LOCAL:
|
||||
raise Failure("Runs on your computer, not the headset.", 400)
|
||||
action = body.get("action")
|
||||
@@ -2867,6 +2870,10 @@ def macview_action(body):
|
||||
return macview.stop(body.get("src") or None)
|
||||
if action == "permissions":
|
||||
return macview.request_permissions()
|
||||
if action == "enable":
|
||||
if not isinstance(macview, frame_pcview.PCView):
|
||||
raise Failure("PC in the headset is only available on Windows.", 400)
|
||||
return macview.set_enabled(body.get("on") is True)
|
||||
except frame_macview.MacViewError as e:
|
||||
raise Failure(str(e), 502)
|
||||
raise Failure("unknown action", 400)
|
||||
|
||||
Reference in new issue
Block a user