From 285871bc39a9739282bb9fc6662935ebded84d58 Mon Sep 17 00:00:00 2001 From: SAPHID Date: Sun, 25 Oct 2026 23:25:07 +0000 Subject: [PATCH] Add locked unattended Framey VR capture and native input checks --- docs/framey-loader.md | 13 +- docs/framey-vr-check.md | 109 ++++++++++++ scripts/framey-vr-check.py | 310 ++++++++++++++++++++++++++++++++++ scripts/framey-worn-check.py | 32 +++- scripts/framey_openvr.py | 108 ++++++++++++ scripts/framey_vr_input.py | 134 +++++++++++++++ tests/test_framey_vr_check.py | 74 ++++++++ tests/test_framey_worn.py | 8 +- 8 files changed, 778 insertions(+), 10 deletions(-) create mode 100644 docs/framey-vr-check.md create mode 100644 scripts/framey-vr-check.py create mode 100644 scripts/framey_openvr.py create mode 100644 scripts/framey_vr_input.py create mode 100644 tests/test_framey_vr_check.py diff --git a/docs/framey-loader.md b/docs/framey-loader.md index 303332f..9ce97de 100644 --- a/docs/framey-loader.md +++ b/docs/framey-loader.md @@ -159,7 +159,7 @@ The Decky and headset service manual removal commands are in ## Fast worn-headset check With an already installed Framey plugin and someone wearing the headset, run -`python3 scripts/framey-worn-check.py` from a paired computer. The script +`python3 scripts/framey-worn-check.py --worn-confirmed` from a paired computer. The script atomically acquires `/tmp/frame-test.lock` before reading battery or SteamVR activity. If another job owns the lock, it makes no sensor reads. It releases its own lock without starting the interaction when the headset is in standby. @@ -172,3 +172,14 @@ color count, then removes the private temporary image from both devices. Pixel variety alone cannot prove the UI text was legible; a person must still confirm the view in the headset. The script does not install or remove a plugin, alter Steam settings, or wake the headset. + + +## Unattended VR checks + +The [unattended check](framey-vr-check.md) can wake, inspect and capture the +real compositor, send OpenVR mouse input, and exercise a temporary sample +against live Decky with restoration and shared-lock protection. Its +[device evidence](evidence/framey-vr-2026-10-25/README.md) distinguishes these +checks from physical controller and wearer validation. A trusted pointer event +and active-headset telemetry are both reproducible without wearing the Frame; +they must not be labelled as physical controller proof. diff --git a/docs/framey-vr-check.md b/docs/framey-vr-check.md new file mode 100644 index 0000000..56b7914 --- /dev/null +++ b/docs/framey-vr-check.md @@ -0,0 +1,109 @@ +# Checking Framey without wearing the Frame + +Use the existing paired SSH alias, directly or through lxso2. Start with the +read-only check, then a temporary wake. A full input exercise is opt in: + +```sh +python3 scripts/framey-vr-check.py --via lxso2 --output /tmp/framey-inspect +python3 scripts/framey-vr-check.py --via lxso2 --wake --output /tmp/framey-wake +python3 scripts/framey-vr-check.py --via lxso2 --exercise --output /tmp/framey-exercise +``` + +Omit `--via` when the computer can already `ssh frame`; use `--host` for another +paired alias. No SSH trust or credentials are changed. The output folder is +private by default. Stereo screenshots and the OBS mirror can contain room +pixels. Review/redact them before committing or sharing. JSON reports have +screenshot hashes and no UI access keys. + +The runner acquires `/tmp/frame-test.lock` by atomic `mkdir`, writes a random +owner, then reads battery and SteamVR state. A foreign lock prevents sensor +reads, wake, installation and input. Unknown battery or below 20% aborts. +Battery is read again before release. The default check never wakes or +installs anything; it returns nonzero when the panel is absent/hidden. + +`--wake` saves the exact current `power.pauseCompositorOnStandby` and +`power.turnOffScreensTimeout`, sets them to `0` and `120`, calls the supported +`vrcmd --handlewakeup` and requests the Frame Control dashboard panel. It reads +OpenVR state, captures a stereo compositor screenshot through the production +`ui/frame_vrshot.py` API, and captures `/dev/video99`, the same SteamVR mirror +used by OBS. It restores the saved power values and reads them back. It hides +the test dashboard if the dashboard was originally closed. It does not change +the proximity sensor, tracking driver, display hardware, root filesystem, +room setup, or spatial panel placement. An unreachable suspended Frame still +needs a physical wake; an SSH command cannot reach it. + +`--exercise` requires the managed headset panel and Decky services to be absent; +it refuses to replace an existing installation. It creates a random test +plugin folder, checks that its settings folder does not exist, and installs +the repository's actual sample frontend/backend, with only the plugin name +and receipt changed to distinguish this test. It installs the verified official +ARM64 Decky release through Frame Control's existing installer, followed by the +normal headset app bundle. Only the test bundle's browser startup is modified: +loopback-only `--remote-debugging-port=0` and a disposable `/tmp` browser profile. +Production browser startup is unchanged. Decky's installer can reload Steam's +web helper; this is not a Steam/SteamVR restart, but its current UI can change. +The test should run while the headset is unattended. + +CDP opens the actual plugin through `openFrameyPanel`, observes the opaque +iframe through its own debugging target, and reads control geometry. OpenVR's +`IVROverlayView_003.PostOverlayEvent` supplies mouse move/down/up to the +actual gamescope overlay. The check verifies delivery to the intended element +and a trusted pointer event, toggles the sample, reads its real Decky settings +file, reopens a fresh iframe, and checks the saved value. It also clicks the +Settings/Panel tabs and quick action. Temporary controls exercise the shipped +`DFL.SliderField` and `DFL.TextField`; their values remain UI-only. Text uses +CDP `Input.insertText` after OpenVR pointer focus. This is synthetic input, +not proof of a physical controller or the SteamVR virtual keyboard. + +SteamVR can leave a recreated panel hidden with a stale focus request. The +exercise retries once by removing and reapplying `STEAM_GAME` on its own test +browser window, then requests focus again. It never retags another application. +The report records whether this recovery was needed. Registration, dashboard +visibility, actual panel visibility, and pixel capture are separate checks. +A visible docked panel can report `IsDashboardVisible=false` and +`IsActiveDashboardOverlay=false`. A zero-sized main texture or failed image +read must not be described as a texture-refresh measurement. The compositor +before/after images provide independent pixel evidence. + +Normal cleanup removes only the test-created managed services and random +plugin/profile paths, restores power, cancels its power watchdog, checks battery +and settings, then releases its own lock. A four-minute independent systemd +user timer restores the saved power values even if the controlling process is +killed or disconnected. It deliberately leaves the lock and test artifacts +for recovery. `recovery.json` records the host, owner, timer, remote restore +script and original values as soon as the timer starts. The final report is +`result.json`. Cleanup errors force a nonzero exit and retain the lock; inspect +that report before manually releasing it. Never remove another job's lock. +A power watchdog is not a promise to complete service cleanup after a hard +kill. The next owner must finish that recorded cleanup first. + +Exit 0 means the requested machine checks and cleanup passed. It never means +a wearer has approved readability, comfort, controller rays, haptics or text +entry on the VR keyboard. The [real-device evidence](evidence/framey-vr-2026-10-25/README.md) +records the exact tested version and limits. No public CI runner should contact +the shared headset: run this command on a paired, serialized self-hosted worker. +Ordinary CI runs `tests/test_framey_vr_check.py`, the sample backend tests, and +the same-plugin navigation regression without a headset. + +## Three-minute worn checklist + +1. **0:00–0:30:** Coordinate the shared lock. With an installed managed panel, + Decky backend and harmless sample, wear the headset and open SteamVR → + Frame Control. Check the panel is in front of you, both eyes agree, text is + legible, and the size is comfortable. Do not use a save-cleaning plugin. +2. **0:30–1:15:** Run + `python3 scripts/framey-worn-check.py --via lxso2 --worn-confirmed`. Within its 35-second + interaction window, use the physical controller to open the sample and + toggle **Example setting**. Confirm the laser aims where expected and the + script records a fresh input/panel event and visible stereo output. The flag + is the operator’s statement; sensor activity and trusted pointers can also + be produced synthetically and cannot identify controller hardware. +3. **1:15–2:15:** Use the controller to open Settings, return to Panel, invoke + **Read saved setting**, close/reopen the plugin, and confirm the value + remains. Try controller scroll, a slider and a harmless text field in an + installed compatible plugin; check SteamVR's actual keyboard and Back. +4. **2:15–3:00:** Restore the original sample setting. Switch away and back, + close/reopen the Frame Control panel, and confirm controller input still + works. Record comfort/legibility, controller model, any haptic behavior, + keyboard behavior and the command's JSON separately. Confirm battery and + release only your lock. Do not infer these human checks from synthetic input. diff --git a/scripts/framey-vr-check.py b/scripts/framey-vr-check.py new file mode 100644 index 0000000..b398f2d --- /dev/null +++ b/scripts/framey-vr-check.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Bounded real-Frame VR diagnostics. No device writes without the shared lock. + +Default: inspect only. --wake: reversible wake + stereo capture. --exercise: +temporary clean user-service install, live sample settings and OpenVR input. +Exit 0 means requested machine checks passed, never a worn/controller verdict. +""" +import argparse +import base64 +import datetime +import hashlib +import io +import json +from pathlib import Path +import secrets +import shlex +import subprocess +import sys +import tarfile +import time +import signal + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'ui')) +import frame_decky +import frame_headset + +LOCK = '/tmp/frame-test.lock' +VRCMD = '/opt/steamvr/bin/linuxarm64/vrcmd' +KEY = 'valve.steam.desktopgame.2001947813' + + +class Remote: + def __init__(self, host, via=None): + self.argv = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', + '-o', 'ServerAliveInterval=5', '-o', 'ServerAliveCountMax=2', host] + self.via = via + + def __call__(self, command, stdin=None, text=True, timeout=30): + argv = self.argv + [command] + if self.via: + argv = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', self.via, + shlex.join(argv)] + result = subprocess.run(argv, input=stdin, capture_output=True, text=text, timeout=timeout) + if result.returncode: + raise RuntimeError((result.stderr or result.stdout).strip()) + return result.stdout + + def python(self, source, timeout=30): + return json.loads(self('python3 -', stdin=source, timeout=timeout)) + + +PROBE = ''' +import json,pathlib,subprocess,ctypes as C +lib=C.CDLL('/opt/steamvr/bin/linuxarm64/libopenvr_api.so') +lib.VR_InitInternal2.argtypes=[C.POINTER(C.c_int),C.c_int,C.c_char_p] +lib.VR_GetGenericInterface.argtypes=[C.c_char_p,C.POINTER(C.c_int)] +lib.VR_GetGenericInterface.restype=C.c_void_p +error=C.c_int();lib.VR_InitInternal2(C.byref(error),3,None) +if error.value: raise RuntimeError('OpenVR unavailable') +try: + table=lib.VR_GetGenericInterface(b'FnTable:IVROverlay_027',C.byref(error)) + if error.value or not table: raise RuntimeError('Overlay ABI unavailable') + dashboard=bool(C.CFUNCTYPE(C.c_bool)(C.cast(table,C.POINTER(C.c_void_p))[66])()) +finally: lib.VR_ShutdownInternal() +battery=[] +for p in pathlib.Path('/sys/class/power_supply').glob('*/capacity'): + if (p.parent/'type').read_text().strip()=='Battery': battery.append(int(p.read_text())) +def cmd(*args): + p=subprocess.run(args,capture_output=True,text=True,timeout=8) + if p.returncode: raise RuntimeError(p.stderr or p.stdout) + return p.stdout.strip() +vr='/opt/steamvr/bin/linuxarm64/vrcmd' +print(json.dumps({'battery':battery,'dashboardVisible':dashboard,'stats':json.loads(cmd(vr,'--stats')), + 'overlays':cmd(vr,'--overlays'), + 'pause':cmd(vr,'--settings-bool','power.pauseCompositorOnStandby'), + 'timeout':cmd(vr,'--settings-float','power.turnOffScreensTimeout')})) +''' + + +def settings(state): + pause = state['pause'].split('=', 1)[1] + timeout = state['timeout'].split('=', 1)[1] + if pause not in ('true', 'false') or not 0 <= float(timeout) <= 86400: + raise RuntimeError('Cannot safely save original SteamVR power values') + return pause, timeout + + +def diagnostic_source(): + return (ROOT / 'scripts/framey_openvr.py').read_text() + ''' +import json +vr=Overlay() +try: print(json.dumps({'state':vr.snapshot(),'imageRead':vr.image_hash()})) +finally: vr.close() +''' + + +def debug_bundle(original, profile): + """Only the test-owned browser gets CDP and a disposable profile.""" + out = io.BytesIO() + with tarfile.open(fileobj=io.BytesIO(original), mode='r:gz') as src, tarfile.open(fileobj=out, mode='w:gz') as dst: + for item in src.getmembers(): + data = src.extractfile(item).read() + if item.name == 'ui/frame_headset_runtime.py': + code = data.decode() + needle = "'--noerrdialogs', '--window-size=1200,800'," + if needle not in code: + raise RuntimeError('Runtime changed; review test browser setup') + code = code.replace(needle, "'--noerrdialogs', '--remote-debugging-port=0', '--window-size=1200,800',") + code = code.replace("str(BASE / 'profile')", repr(profile)) + data = code.encode() + item.size = len(data) + dst.addfile(item, io.BytesIO(data)) + return out.getvalue() + + +def exercise_source(profile, plugin_id, folder): + # Runs entirely on the headset. The normal iframe, HTTP handler, CEF bridge + # and Decky Python backend are used. CDP observes DOM/coordinates; OpenVR + # delivers clicks. No JS .click() or backend stubs. + return (ROOT/'scripts/framey_openvr.py').read_text() + '\n' + (ROOT/'scripts/framey_vr_input.py').read_text() + '\n' + ( + 'print(json.dumps(exercise(%r,%r,%r)))\n' % (profile, plugin_id, folder)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', default='frame') + parser.add_argument('--via', help='Existing paired SSH host, e.g. lxso2') + parser.add_argument('--wake', action='store_true') + parser.add_argument('--exercise', action='store_true', help='Temporary user installs; requires no managed panel/Decky install') + parser.add_argument('--output', type=Path, required=True, help='Private evidence folder (screenshots may contain room pixels)') + args = parser.parse_args(argv) + args.output.mkdir(parents=True, exist_ok=True, mode=0o700) + report = {'schema': 1, 'time': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'controllerHardwareVerified': False, 'wornVisualComfortVerified': False} + git = subprocess.run(['git','rev-parse','HEAD'],cwd=ROOT,capture_output=True,text=True) + report['sourceCommit'] = git.stdout.strip() if git.returncode == 0 else None + remote = Remote(args.host, args.via) + token = 'framey-vr-' + secrets.token_hex(8) + folder = token + profile = '/tmp/' + token + '-profile' + locked = power_changed = plugin_created = decky_created = panel_created = watchdog = False + original_bundle = frame_headset.bundle + before = None + previous_term = signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(RuntimeError('Terminated; cleaning up'))) + try: + remote('mkdir ' + LOCK + ' && { printf %s ' + shlex.quote(token) + ' > ' + LOCK + '/owner || { rmdir ' + LOCK + '; exit 1; }; }') + locked = True + before = remote.python(PROBE) + report['before'] = before + pause, timeout = settings(before) + if not before['battery'] or min(before['battery']) < 20: + raise RuntimeError('Battery unknown or below 20%') + if args.wake or args.exercise: + # Independent systemd timer restores power even if the controller is + # killed or SSH disappears. It retains the test lock for recovery. + recovery = '/tmp/' + token + '-restore.py' + restore_source = ("import pathlib,subprocess\n" + "p=pathlib.Path('/tmp/frame-test.lock/owner')\n" + "if p.exists() and p.read_text()==" + repr(token) + ":\n" + " subprocess.run([" + repr(VRCMD) + ", '--set-settings-bool','power.pauseCompositorOnStandby'," + repr('1' if pause == 'true' else '0') + "],check=True)\n" + " subprocess.run([" + repr(VRCMD) + ", '--set-settings-float','power.turnOffScreensTimeout'," + repr(timeout) + "],check=True)\n") + remote('umask 077; cat > ' + shlex.quote(recovery), stdin=restore_source) + remote('systemd-run --user --quiet --unit=' + token + '-restore --on-active=240s /usr/bin/python3 ' + shlex.quote(recovery)) + watchdog = True + report['powerRecovery'] = {'timer':token+'-restore.timer','script':recovery, + 'originalPause':pause,'originalTimeout':timeout} + (args.output/'recovery.json').write_text(json.dumps({'host':args.host,'via':args.via,'lockOwner':token,**report['powerRecovery']},indent=2)+'\n') + if args.exercise: + d, h = frame_decky.status(remote), frame_headset.status(remote) + if d['managed'] or h['managed'] or d['upstreamInstall']: + raise RuntimeError('Exercise needs absent managed services; existing installs left intact') + # Refuse every collision, including retained plugin settings. + files = {name: base64.b64encode((ROOT/'examples/framey-settings'/name).read_bytes()).decode() + for name in ('plugin.json', 'main.py', 'dist/index.js')} + source = ''' +import base64,json,pathlib +base=pathlib.Path.home()/'.local/share/frame-control/decky' +folder=%r +p=base/'plugins'/folder +if p.exists() or (base/'settings'/folder).exists(): raise RuntimeError('Test path collision') +p.mkdir(parents=True); (p/'dist').mkdir() +for name,data in %r.items(): (p/name).write_bytes(base64.b64decode(data)) +manifest=json.loads((p/'plugin.json').read_text());manifest['name']='Framey VR Check '+folder +(p/'plugin.json').write_text(json.dumps(manifest)) +(p/'.frame-control-store.json').write_text(json.dumps({'storeId':2000000001,'version':'local-vr-check'})) +print(json.dumps({'created':True})) +''' % (folder, files) + remote.python(source) + plugin_created = True + decky_created = True + report['deckyInstall'] = frame_decky.change(remote, 'install') + # Decky reloads its Steam frontend; install the panel afterwards. + time.sleep(3) + frame_headset.bundle = lambda: debug_bundle(original_bundle(), profile) + panel_created = True + report['panelInstall'] = frame_headset.change(remote, 'install') + if args.wake or args.exercise: + power_changed = True # before first setter, so partial failure restores + remote(VRCMD + ' --set-settings-bool power.pauseCompositorOnStandby 0') + remote(VRCMD + ' --set-settings-float power.turnOffScreensTimeout 120') + remote(VRCMD + ' --handlewakeup') + remote(VRCMD + ' --showdashboard ' + KEY) + time.sleep(2) + report['overlay'] = remote.python(diagnostic_source()) + if args.exercise and not report['overlay']['state']['visible']: + report['retaggedTestWindow'] = True + remote.python(''' +import json,pathlib,subprocess,time,os +p=pathlib.Path.home()/'.local/share/frame-control/headset/state.json' +w=json.loads(p.read_text())['window'];env={**os.environ,'DISPLAY':':0'} +subprocess.run(['xprop','-id',w,'-remove','STEAM_GAME'],env=env,check=True) +time.sleep(1) +subprocess.run(['xprop','-id',w,'-f','STEAM_GAME','32c','-set','STEAM_GAME','2001947813'],env=env,check=True) +time.sleep(1) +subprocess.run(['/opt/steamvr/bin/linuxarm64/vrcmd','--showdashboard','valve.steam.desktopgame.2001947813'],capture_output=True,check=True) +time.sleep(2) +print('{}') +''') + report['overlay'] = remote.python(diagnostic_source()) + if args.wake or args.exercise: + shot = remote.python((ROOT/'ui/frame_vrshot.py').read_text(), timeout=20)['path'] + try: + image = remote('cat ' + shlex.quote(shot), text=False, timeout=20) + (args.output/'before-stereo-private.png').write_bytes(image) + report['beforeStereoSha256'] = hashlib.sha256(image).hexdigest() + finally: + remote('rm -f ' + shlex.quote(shot)) + if args.exercise: + report['input'] = remote.python(exercise_source(profile, 2000000001, folder), timeout=100) + if args.wake or args.exercise: + shot = remote.python((ROOT/'ui/frame_vrshot.py').read_text(), timeout=20)['path'] + try: + image = remote('cat ' + shlex.quote(shot), text=False, timeout=20) + (args.output/'stereo-private.png').write_bytes(image) + report['stereoSha256'] = hashlib.sha256(image).hexdigest() + report['compositorChanged'] = report['stereoSha256'] != report.get('beforeStereoSha256') + finally: + remote('rm -f ' + shlex.quote(shot)) + # Same SteamVR mirror used by OBS; FFmpeg is already on the Frame. + mirror = remote('ffmpeg -hide_banner -loglevel error -f v4l2 -i /dev/video99 -frames:v 1 -f image2pipe -vcodec png -', text=False, timeout=20) + (args.output/'mirror-private.png').write_bytes(mirror) + report['mirrorSha256'] = hashlib.sha256(mirror).hexdigest() + report['passed'] = bool(report['overlay']['state']['visible']) + if args.exercise: + report['passed'] = report['passed'] and report['input']['passed'] and report.get('compositorChanged',False) + except Exception as exc: + report['error'] = str(exc) + report['passed'] = False + finally: + frame_headset.bundle = original_bundle + cleanup = [] + def clean(name, action): + try: action() + except Exception as exc: cleanup.append({'step':name,'error':str(exc)}) + if locked: + if panel_created: + clean('remove test panel', lambda: frame_headset.change(remote, 'remove') if frame_headset.status(remote)['managed'] else None) + if decky_created: + clean('remove test Decky', lambda: frame_decky.change(remote, 'remove') if frame_decky.status(remote)['managed'] else None) + if plugin_created: + # Only random per-run paths created after collision checks. + clean('remove test files', lambda: remote.python(''' +import pathlib,shutil,json +base=pathlib.Path.home()/'.local/share/frame-control/decky' +for parent in ('plugins','settings','data','logs'): + p=base/parent/%r + if p.is_dir() and not p.is_symlink(): shutil.rmtree(p) +p=pathlib.Path(%r) +if p.is_dir() and not p.is_symlink(): shutil.rmtree(p) +print('{}') +''' % (folder, profile))) + if power_changed: + clean('restore compositor pause', lambda: remote(VRCMD + ' --set-settings-bool power.pauseCompositorOnStandby ' + ('1' if pause == 'true' else '0'))) + clean('restore screen timeout', lambda: remote(VRCMD + ' --set-settings-float power.turnOffScreensTimeout ' + shlex.quote(timeout))) + clean('restore dashboard visibility', lambda: remote(VRCMD + ' --hidedashboard') if not before['dashboardVisible'] else None) + try: + report['after'] = remote.python(PROBE) + if power_changed and settings(report['after']) != settings(before): + cleanup.append({'step':'read back power settings','error':'Original values not restored'}) + except Exception as exc: cleanup.append({'step':'after probe','error':str(exc)}) + if watchdog and not cleanup: + clean('stop power recovery timer', lambda: remote.python(''' +import subprocess,json +unit=%r +timer=subprocess.run(['systemctl','--user','show',unit+'.timer','--property=LoadState','--value'],capture_output=True,text=True) +if timer.stdout.strip()=='loaded': subprocess.run(['systemctl','--user','stop',unit+'.timer'],check=True,capture_output=True) +p=subprocess.run(['systemctl','--user','show',unit+'.service','--property=LoadState','--value'],capture_output=True,text=True) +if p.stdout.strip()=='loaded': subprocess.run(['systemctl','--user','stop',unit+'.service'],check=True,capture_output=True) +print('{}') +''' % (token+'-restore'))) + if not cleanup: + clean('remove power recovery script', lambda: remote('rm -f ' + shlex.quote(recovery))) + report['powerRecoveryRemoved'] = not cleanup + if not cleanup: (args.output/'recovery.json').unlink(missing_ok=True) + if not cleanup: + clean('release owned lock', lambda: remote('test "$(cat ' + LOCK + '/owner)" = ' + shlex.quote(token) + ' && rm ' + LOCK + '/owner && rmdir ' + LOCK)) + report['lockReleased'] = not cleanup + else: + report['lockRetainedForRecovery'] = token + report['cleanupErrors'] = cleanup + report['passed'] = bool(report.get('passed') and not cleanup) + (args.output/'result.json').write_text(json.dumps(report, indent=2)+'\n') + signal.signal(signal.SIGTERM, previous_term) + print(json.dumps(report, indent=2)) + return 0 if report['passed'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/framey-worn-check.py b/scripts/framey-worn-check.py index 71b25d0..57c7713 100644 --- a/scripts/framey-worn-check.py +++ b/scripts/framey-worn-check.py @@ -4,6 +4,7 @@ Requires an already installed Frame Control VR panel, Decky user service, and one Framey-managed plugin. It never installs, restarts, or removes a service. """ +import argparse import datetime import hashlib import json @@ -18,10 +19,16 @@ import time ROOT = Path(__file__).resolve().parents[1] SSH = ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=5', 'frame'] LOCK = '/tmp/frame-test.lock' +VIA = None + +def ssh_argv(command): + argv = [*SSH, command] + return ['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', VIA, shlex.join(argv)] if VIA else argv + def remote(command, stdin=None, timeout=15): - run = subprocess.run([*SSH, command], input=stdin, capture_output=True, + run = subprocess.run(ssh_argv(command), input=stdin, capture_output=True, text=True, timeout=timeout) if run.returncode: raise RuntimeError((run.stderr or run.stdout).strip() or f'SSH exited {run.returncode}') @@ -92,12 +99,14 @@ def color_count(text): return len(set(re.findall(r'#[0-9A-Fa-f]{6,8}', text))) -def main(): - result = {'time':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'proof':'worn-headset check'} +def main(worn_confirmed=False): + result = {'time':datetime.datetime.now(datetime.timezone.utc).isoformat(), 'proof':'operator-confirmed worn check', 'operatorWornConfirmed':worn_confirmed} token = 'framey-' + secrets.token_hex(12) locked = False challenge = False try: + if not worn_confirmed: + raise RuntimeError('A wearer must be present; pass --worn-confirmed only after checking in person. Wake/activity and trusted pointer events alone do not prove wearing or controller hardware.') # Do not read even standby status until this task owns the device lock. remote('mkdir ' + LOCK + ' && { printf %s ' + shlex.quote(token) + ' > ' + LOCK + '/owner || { rm -f ' + LOCK + '/owner; rmdir ' + LOCK + '; exit 1; }; }') @@ -126,11 +135,11 @@ def main(): time.sleep(1) result['interaction'] = state result['after'] = probe() - result['worn'] = isinstance(result['after']['activity'], int) and result['after']['activity'] < 3 + result['activeHeadset'] = isinstance(result['after']['activity'], int) and result['after']['activity'] < 3 result['overlay'] = json.loads(remote('python3 -', stdin=OVERLAY_PROBE, timeout=15)) result['panelVisible'] = result['overlay']['visible'] is True installed_ids = {p['id'] for p in before['plugins']} - result['controllerInput'] = bool(state.get('input', {}).get('at', 0) > state.get('started', 0)) + result['trustedPointerInput'] = bool(state.get('input', {}).get('at', 0) > state.get('started', 0)) result['pluginPanelReady'] = bool(state.get('panel', {}).get('at', 0) > state.get('started', 0) and state['panel'].get('pluginId') in installed_ids) if result['panelVisible'] and result['pluginPanelReady']: @@ -138,7 +147,7 @@ def main(): local = Path('/tmp')/('framey-worn-' + token + '.png') try: with local.open('wb') as out: - copy = subprocess.run([*SSH, 'cat ' + shlex.quote(shot)], stdout=out, stderr=subprocess.PIPE, timeout=20) + copy = subprocess.run(ssh_argv('cat ' + shlex.quote(shot)), stdout=out, stderr=subprocess.PIPE, timeout=20) if copy.returncode: raise RuntimeError('Could not copy the SteamVR screenshot') result['screenshotSha256'] = hashlib.sha256(local.read_bytes()).hexdigest() pixels = subprocess.run(['convert', str(local), '-resize', '64x64!', 'txt:-'], @@ -147,7 +156,7 @@ def main(): finally: local.unlink(missing_ok=True) remote('rm -f ' + shlex.quote(shot)) - result['passed'] = (result['worn'] and result['panelVisible'] and result['controllerInput'] and + result['passed'] = (result['activeHeadset'] and result['panelVisible'] and result['trustedPointerInput'] and result['pluginPanelReady'] and (result.get('screenshotColors') or 0) > 8) except Exception as exc: result['error'] = str(exc) @@ -167,4 +176,11 @@ def main(): if __name__ == '__main__': - sys.exit(main()) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', default='frame') + parser.add_argument('--via', help='Paired SSH hop, e.g. lxso2') + parser.add_argument('--worn-confirmed', action='store_true', help='Operator confirms someone is wearing the Frame') + args = parser.parse_args() + SSH[-1] = args.host + VIA = args.via + sys.exit(main(args.worn_confirmed)) diff --git a/scripts/framey_openvr.py b/scripts/framey_openvr.py new file mode 100644 index 0000000..a912024 --- /dev/null +++ b/scripts/framey_openvr.py @@ -0,0 +1,108 @@ +"""Small OpenVR diagnostic client; runs on the Frame, never polls another app's events. + +Indices/signatures are for IVROverlay_027 and IVROverlayView_003 from Valve's +shipped openvr_capi.h, not interchangeable with IVROverlay_028. +""" +import ctypes as C +import hashlib + +KEY = b'valve.steam.desktopgame.2001947813' + + +class Vector(C.Structure): + _fields_ = [('x', C.c_float), ('y', C.c_float)] + + +class Matrix(C.Structure): + _fields_ = [('m', (C.c_float * 4) * 3)] + + +class Mouse(C.Structure): + _fields_ = [('x', C.c_float), ('y', C.c_float), ('button', C.c_uint32)] + + +class Data(C.Union): + _fields_ = [('mouse', Mouse), ('reserved', C.c_uint64 * 6)] + + +class Event(C.Structure): + _pack_ = 4 # Linux OpenVR preserves its historical 4-byte event packing. + _fields_ = [('eventType', C.c_uint32), ('device', C.c_uint32), + ('age', C.c_float), ('data', Data)] + + +class Overlay: + def __init__(self): + self.vr = C.CDLL('/opt/steamvr/bin/linuxarm64/libopenvr_api.so') + self.vr.VR_InitInternal2.argtypes = [C.POINTER(C.c_int), C.c_int, C.c_char_p] + self.vr.VR_GetGenericInterface.argtypes = [C.c_char_p, C.POINTER(C.c_int)] + self.vr.VR_GetGenericInterface.restype = C.c_void_p + error = C.c_int() + self.vr.VR_InitInternal2(C.byref(error), 3, None) # background, no scene focus + if error.value: + raise RuntimeError('OpenVR init: ' + str(error.value)) + try: + self.table = self.interface(b'FnTable:IVROverlay_027') + self.handle = C.c_uint64() + rc = self.fn(0, C.c_int, C.c_char_p, C.POINTER(C.c_uint64))(KEY, C.byref(self.handle)) + if rc: + raise RuntimeError('Frame Control overlay not found: ' + str(rc)) + except Exception: + self.close() + raise + + def interface(self, version): + error = C.c_int() + ptr = self.vr.VR_GetGenericInterface(version, C.byref(error)) + if error.value or not ptr: + raise RuntimeError('OpenVR interface unavailable: ' + version.decode()) + return C.cast(ptr, C.POINTER(C.c_void_p)) + + def fn(self, index, result, *args): + return C.CFUNCTYPE(result, *args)(self.table[index]) + + def snapshot(self): + h = self.handle + width, height = C.c_uint32(), C.c_uint32() + texture_error = self.fn(64, C.c_int, C.c_uint64, C.POINTER(C.c_uint32), C.POINTER(C.c_uint32))(h, C.byref(width), C.byref(height)) + transform_type = C.c_int() + type_error = self.fn(31,C.c_int,C.c_uint64,C.POINTER(C.c_int))(h,C.byref(transform_type)) + origin, matrix = C.c_int(), Matrix() + transform_error = self.fn(33, C.c_int, C.c_uint64, C.POINTER(C.c_int), C.POINTER(Matrix))(h, C.byref(origin), C.byref(matrix)) + scale, method = Vector(), C.c_int() + scale_error = self.fn(49, C.c_int, C.c_uint64, C.POINTER(Vector))(h, C.byref(scale)) + method_error = self.fn(47, C.c_int, C.c_uint64, C.POINTER(C.c_int))(h, C.byref(method)) + return {'interface': 'IVROverlay_027', + 'visible': bool(self.fn(43, C.c_bool, C.c_uint64)(h)), + 'dashboardVisible': bool(self.fn(66, C.c_bool)()), + 'activeDashboard': bool(self.fn(67, C.c_bool, C.c_uint64)(h)), + 'texture': {'error': texture_error, 'width': width.value, 'height': height.value}, + 'transform': {'error': transform_error, 'typeError':type_error,'type':transform_type.value,'origin': origin.value, + 'matrix': [list(row) for row in matrix.m]}, + 'input': {'error': method_error, 'method': method.value, + 'scaleError': scale_error, 'scale': [scale.x, scale.y]}} + + def post_mouse(self, kind, x, y, button=0): + # PostOverlayEvent is the supported overlay-view injection API. It is + # not a tracked controller and its void return does not prove delivery. + view = self.interface(b'FnTable:IVROverlayView_003') + event = Event(kind, 0xFFFFFFFF, 0, Data(mouse=Mouse(x, y, button))) + C.CFUNCTYPE(None, C.c_uint64, C.POINTER(Event))(view[2])(self.handle, C.byref(event)) + + def image_hash(self): + width, height = C.c_uint32(), C.c_uint32() + call = self.fn(6, C.c_int, C.c_uint64, C.c_void_p, C.c_uint32, + C.POINTER(C.c_uint32), C.POINTER(C.c_uint32)) + rc = call(self.handle, None, 0, C.byref(width), C.byref(height)) + size = width.value * height.value * 4 + result = {'error': rc, 'width': width.value, 'height': height.value} + if 0 < size <= 32 * 1024 * 1024: + pixels = C.create_string_buffer(size) + rc = call(self.handle, pixels, size, C.byref(width), C.byref(height)) + result['error'] = rc + if not rc: + result['sha256'] = hashlib.sha256(pixels.raw).hexdigest() + return result + + def close(self): + self.vr.VR_ShutdownInternal() diff --git a/scripts/framey_vr_input.py b/scripts/framey_vr_input.py new file mode 100644 index 0000000..84a4813 --- /dev/null +++ b/scripts/framey_vr_input.py @@ -0,0 +1,134 @@ +"""Headset-side real input exercise, appended after framey_openvr.py by the runner.""" +import json +from pathlib import Path +import sys +import time +import urllib.request + + +def exercise(profile, plugin_id, folder): + sys.path.insert(0, str(Path.home()/'.local/share/frame-control/headset/bundle/ui')) + import frame_steam + port = int((Path(profile)/'DevToolsActivePort').read_text().splitlines()[0]) + frame_steam.CDP = f'http://127.0.0.1:{port}/json' + parent = frame_steam.Page('Frame Control VR [fc-headset]') + panel = None + vr = Overlay() + result = {'clickTransport':'IVROverlayView_003.PostOverlayEvent', + 'textTransport':'CDP Input.insertText after OpenVR focus', + 'controllerHardwareVerified':False} + + def wait(read, accept, seconds=15): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + value = read() + if accept(value): return value + time.sleep(.25) + raise RuntimeError('Timed out waiting for live UI: ' + str(value)) + + def connect_panel(): + def target(): + targets = json.load(urllib.request.urlopen(frame_steam.CDP, timeout=5)) + return next((p['title'] for p in targets if p.get('type') == 'iframe' + and '/framey/panel/' + str(plugin_id) in p.get('url','')), None) + page = frame_steam.Page(wait(target, bool)) + page.eval('window.__vrClickCount=0;addEventListener("pointerup",e=>{window.__vrClickCount++;window.__vrLastTarget=e.target;});true') + return page + + def command(page, method, params): + page.next_id += 1 + page._send(json.dumps({'id':page.next_id,'method':method,'params':params})) + while True: + msg = json.loads(page._recv()) + if msg.get('id') == page.next_id: + if 'error' in msg: raise RuntimeError(str(msg['error'])) + return msg.get('result', {}) + + def text(): return panel.eval('document.body.innerText') + + def click(selector): + # Account for CSS pixels, iframe position, border, HiDPI scaling, and + # OpenVR's bottom-left mouse origin; read geometry after scrolling. + panel.eval(f'document.querySelector({json.dumps(selector)}).scrollIntoView({{block:"center"}})') + time.sleep(.3) # let Chromium present the scrolled iframe before native input + pos = panel.eval(f'(()=>{{let r=document.querySelector({json.dumps(selector)}).getBoundingClientRect();return {{x:r.x+r.width/2,y:r.y+r.height/2}}}})()') + outer = parent.eval('(()=>{let f=byId("frameyFrame"),r=f.getBoundingClientRect();return {x:r.x+f.clientLeft,y:r.y+f.clientTop,w:innerWidth,h:innerHeight}})()') + sx, sy = vr.snapshot()['input']['scale'] + x = (outer['x'] + pos['x']) * sx / outer['w'] + y = sy - (outer['y'] + pos['y']) * sy / outer['h'] + if not 0 <= x <= sx or not 0 <= y <= sy: + raise RuntimeError('Control is outside the visible viewport') + count = panel.eval('__vrClickCount') + vr.post_mouse(300, x, y) + vr.post_mouse(301, x, y, 1) + vr.post_mouse(302, x, y, 1) + wait(lambda:panel.eval('__vrClickCount'), lambda n:n > count, seconds=3) + hit = panel.eval(f'!!window.__vrLastTarget?.closest({json.dumps(selector)})') + if not hit: + raise RuntimeError('Native click hit another element: ' + str(panel.eval('({tag:__vrLastTarget?.tagName,text:__vrLastTarget?.textContent?.slice(0,100)})'))) + + try: + parent.eval(f'(async()=>{{await refreshDecky();openFrameyPanel({plugin_id},"Framey VR Check");return true}})()') + wait(lambda:parent.eval('byId("frameyPanelStatus").innerText'), lambda s:s == 'Plugin panel ready') + panel = connect_panel() + initial = wait(text, lambda s:'Saved setting: off' in s or 'Could not read setting:' in s) + if 'Could not read setting:' in initial: raise RuntimeError(initial) + result['initialText'] = initial + panel.eval('window.__vrEvents=[];addEventListener("pointerup",e=>__vrEvents.push({trusted:e.isTrusted,x:e.clientX,y:e.clientY}));true') + click('input[type=checkbox]') + result['savedText'] = wait(text, lambda s:'Saved setting: on' in s) + result['pointerEvents'] = panel.eval('__vrEvents') + result['trustedOverlayPointer'] = any(e['trusted'] for e in result['pointerEvents']) + disk = Path.home()/'.local/share/frame-control/decky/settings'/folder/'settings.json' + result['diskSetting'] = json.loads(disk.read_text()) + # Fresh browser document, same normal parent API; no backend stub. + panel.sock.close(); panel = None + parent.eval(f'openFrameyPanel({plugin_id},"Framey VR Check")') + wait(lambda:parent.eval('byId("frameyPanelStatus").innerText'), lambda s:s == 'Plugin panel ready') + panel = connect_panel() + result['reopenedText'] = wait(text, lambda s:'Saved setting: on' in s) + result['reopenedChecked'] = panel.eval('document.querySelector("input[type=checkbox]").checked') + # Settings tab and quick action are real host controls too. + click('nav button:nth-child(2)') + wait(text, lambda s:'Saved setting: on' in s) + click('nav button:nth-child(1)') + wait(text, lambda s:'Saved setting: on' in s) + panel.eval('[...document.querySelectorAll("button")].find(b=>b.textContent==="Read saved setting").id="vr-quick"') + click('#vr-quick') + result['quickActionText'] = wait(text, lambda s:'Saved setting is on' in s) + # Isolated diagnostic controls use the actual shipped DFL shims. Their + # values are UI-only; they never change a user's settings or plugin. + panel.eval('''(()=>{ + const mount=document.createElement('div');mount.id='vr-controls';document.body.prepend(mount); + function Controls(){const R=SP_REACT,[v,S]=R.useState(10),[t,T]=R.useState('');return R.createElement('section',null, + R.createElement(DFL.SliderField,{label:'VR test slider',min:0,max:100,value:v,showValue:true,onChange:S}), + R.createElement(DFL.TextField,{label:'VR test text',value:t,onChange:e=>T(e.target.value)}), + R.createElement('p',{id:'vr-control-values'},v+':'+t));} + SP_REACTDOM.createRoot(mount).render(SP_REACT.createElement(Controls));return true; + })()''') + wait(lambda:panel.eval('!!document.querySelector("#vr-controls input[type=range]")'), bool) + click('#vr-controls input[type=range]') + result['sliderValue'] = wait(lambda:panel.eval('document.querySelector("#vr-controls input[type=range]").value'), lambda v:v != '10') + click('#vr-controls input[type=text]') + # CDP only types text; OpenVR supplies pointer focus into the field. + command(parent, 'Input.insertText', {'text':'framey-ci'}) + result['textValue'] = wait(lambda:panel.eval('document.querySelector("#vr-controls input[type=text]").value'), lambda v:v == 'framey-ci') + panel.eval('document.getElementById("vr-controls").remove()') + # Scroll saved status into the visible iframe for compositor evidence. + panel.eval('document.querySelector("[role=status]").scrollIntoView({block:"center"})') + result['finalOverlay'] = vr.snapshot() + result['passed'] = (result['trustedOverlayPointer'] and result['diskSetting'] == {'enabled':True} + and result['reopenedChecked'] and result['sliderValue'] != '10' + and result['textValue'] == 'framey-ci') + return result + except Exception as exc: + result['error'] = str(exc) + result['passed'] = False + if panel: + result['failureText'] = text() + result['focus'] = panel.eval('({tag:document.activeElement?.tagName,type:document.activeElement?.type})') + return result + finally: + vr.close() + parent.sock.close() + if panel: panel.sock.close() diff --git a/tests/test_framey_vr_check.py b/tests/test_framey_vr_check.py new file mode 100644 index 0000000..175e6be --- /dev/null +++ b/tests/test_framey_vr_check.py @@ -0,0 +1,74 @@ +"""Safety gates for unattended device work; no headset or OpenVR loaded.""" +import importlib.util +import contextlib +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +PATH = Path(__file__).resolve().parents[1]/'scripts/framey-vr-check.py' +spec = importlib.util.spec_from_file_location('framey_vr_check', PATH) +check = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check) + + +class Safety(unittest.TestCase): + def test_foreign_lock_prevents_sensor_reads_and_cleanup(self): + with tempfile.TemporaryDirectory() as folder, patch.object(check, 'Remote') as factory: + remote = factory.return_value + remote.side_effect = RuntimeError('File exists') + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(check.main(['--exercise','--output',folder]), 1) + remote.python.assert_not_called() + self.assertEqual(remote.call_count, 1) + self.assertIn('mkdir /tmp/frame-test.lock', remote.call_args.args[0]) + self.assertFalse(json.loads((Path(folder)/'result.json').read_text())['passed']) + + def test_low_battery_never_wakes_or_installs_and_releases_lock(self): + state = {'battery':[19], 'pause':'power.pauseCompositorOnStandby=true', + 'timeout':'power.turnOffScreensTimeout=17.000000','dashboardVisible':False} + with tempfile.TemporaryDirectory() as folder, patch.object(check, 'Remote') as factory, \ + patch.object(check.frame_decky, 'change') as install: + remote = factory.return_value + remote.python.return_value = state + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(check.main(['--exercise','--output',folder]), 1) + install.assert_not_called() + self.assertEqual(remote.call_count, 2) + self.assertIn('rmdir /tmp/frame-test.lock', remote.call_args.args[0]) + self.assertFalse(any('--handlewakeup' in c.args[0] for c in remote.call_args_list)) + + def test_power_restore_after_partial_setter_failure(self): + state = {'battery':[99], 'pause':'power.pauseCompositorOnStandby=true', + 'timeout':'power.turnOffScreensTimeout=17.000000','dashboardVisible':False} + with tempfile.TemporaryDirectory() as folder, patch.object(check, 'Remote') as factory: + remote = factory.return_value + remote.python.return_value = state + def run(cmd, **_): + if '--set-settings-float power.turnOffScreensTimeout 120' in cmd: + raise RuntimeError('setter failed') + return '' + remote.side_effect = run + with contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(check.main(['--wake','--output',folder]), 1) + commands = [c.args[0] for c in remote.call_args_list] + self.assertTrue(any('--set-settings-bool power.pauseCompositorOnStandby 1' in c for c in commands)) + self.assertTrue(any('--set-settings-float power.turnOffScreensTimeout 17.000000' in c for c in commands)) + self.assertIn('rmdir /tmp/frame-test.lock', commands[-1]) + + def test_exact_linux_event_abi_and_debug_profile(self): + import ctypes + overlay_spec = importlib.util.spec_from_file_location('framey_openvr', PATH.with_name('framey_openvr.py')) + overlay = importlib.util.module_from_spec(overlay_spec); overlay_spec.loader.exec_module(overlay) + self.assertEqual(ctypes.sizeof(overlay.Event), 60) + self.assertEqual(overlay.Event.data.offset, 12) + original = check.frame_headset.bundle() + patched = check.debug_bundle(original, '/tmp/owned-profile') + import tarfile + with tarfile.open(fileobj=io.BytesIO(patched),mode='r:gz') as archive: + source=archive.extractfile('ui/frame_headset_runtime.py').read().decode() + self.assertIn("'--remote-debugging-port=0'", source) + self.assertIn("'/tmp/owned-profile'", source) + self.assertNotIn("str(BASE / 'profile')", source) diff --git a/tests/test_framey_worn.py b/tests/test_framey_worn.py index 7d71e65..8bb154c 100644 --- a/tests/test_framey_worn.py +++ b/tests/test_framey_worn.py @@ -20,9 +20,15 @@ class WornGate(unittest.TestCase): with patch.object(worn, 'probe', return_value=state) as probe, \ patch.object(worn, 'remote', side_effect=remote_side_effect) as remote, \ contextlib.redirect_stdout(out): - result = worn.main() + result = worn.main(worn_confirmed=True) return result, out.getvalue(), remote, probe + def test_no_wearer_confirmation_means_no_sensor_or_input_claim(self): + with patch.object(worn,'remote') as remote, contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(worn.main(),1) + remote.assert_not_called() + self.assertIn('trusted pointer events alone do not prove',out.getvalue()) + def test_standby_is_read_only_under_owned_lock_and_releases_it(self): state = {'battery':[{'percent':99}], 'activity':3, 'panelService':True, 'plugins':[{'id':14,'name':'ProtonDB Badges'}], 'panel':None}