mirror of
https://github.com/DeeJanuz/frametop.git
synced 2026-10-17 16:00:36 +02:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d985573c0 | ||
|
|
537fdac419 | ||
|
|
87a63932c0 | ||
|
|
bd06207db9 |
No files matched your search
@@ -1,24 +0,0 @@
|
||||
# Keep the build context small: what .gitignore ignores, and regenerated files.
|
||||
# Unlike .gitignore, patterns here only match at the top unless they start
|
||||
# with **/, so the ones that can appear in any folder do.
|
||||
.git/
|
||||
.worktrees/
|
||||
.venv/
|
||||
**/build/
|
||||
**/target/
|
||||
**/captures/
|
||||
**/__pycache__/
|
||||
**/*.pyc
|
||||
# Eye-camera frame dumps are biometric (see .gitignore): never in an image.
|
||||
**/*.raw
|
||||
**/*.pgm
|
||||
**/.env
|
||||
**/.env.*
|
||||
frametop-report-*.txt
|
||||
.frame-job.d/
|
||||
# the fine-tuned hand models: the image fetches its own pinned copy (hands/models.sh)
|
||||
hands/models/student/
|
||||
hands/models/.student-*
|
||||
# model versions downloaded from Input Settings (models/catalog.py): each install fetches its own
|
||||
hands/models/versions/
|
||||
gaze/tracker/models/
|
||||
@@ -1,105 +0,0 @@
|
||||
# Build Frametop's release on Depot's arm64 runners (the Frame is aarch64): the image from
|
||||
# pack/Containerfile, the test gate inside it, then Frametop.zip (framedrop/build.sh), the
|
||||
# image with its installer: FrameDrop installs it from a PC, or you unpack it on the headset
|
||||
# and run Frametop/frametop-install.sh, or get.sh --release downloads it (pack/README.md,
|
||||
# Releases). It's about 1.1 GB, under GitHub's 2 GB a file.
|
||||
#
|
||||
# A v* tag makes a draft GitHub release with the zip, its FrameDrop manifest, its
|
||||
# frametop-release.json, and SHA256SUMS (a prerelease when the tag has a "-", like
|
||||
# v0.3.0-exp.1). Nothing is public until someone publishes the draft. A manual run keeps the
|
||||
# zip as the run's artifact for a week.
|
||||
#
|
||||
# podman, as on the Frame: the zip holds podman save's archive, which the headset loads with
|
||||
# podman. Not on pushes or pull requests: Depot's runners are paid, and a fork's pull request
|
||||
# would run on them. Actions are pinned to commit SHAs (the version in the comment).
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write # the draft release
|
||||
|
||||
concurrency:
|
||||
group: release-${{ github.ref }}
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: depot-ubuntu-24.04-arm-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FT_ENGINE: podman
|
||||
steps:
|
||||
# actions/checkout@v7.0.1
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
|
||||
- name: version
|
||||
run: |
|
||||
if [ "$GITHUB_REF_TYPE" = tag ]; then v=${GITHUB_REF_NAME#v}; else v=0.0.0-ci.${GITHUB_SHA::7}; fi
|
||||
echo "VERSION=$v" >> "$GITHUB_ENV"
|
||||
echo "FT_IMAGE=localhost/frametop:$v" >> "$GITHUB_ENV"
|
||||
|
||||
- name: build the image
|
||||
run: ./ft dev build
|
||||
|
||||
# The unit gate: Python (strict), C, and shell, inside the image, so what passes here is
|
||||
# what the zip installs.
|
||||
- name: test (python, c, bash)
|
||||
run: ./ft dev test
|
||||
|
||||
- name: smoke (what a release installs from the image)
|
||||
run: |
|
||||
podman run --rm --entrypoint sh "$FT_IMAGE" -c '
|
||||
set -e
|
||||
for p in ft-screens ft-spotpanel ft-pointer ft-powerd ft-gaze ft-gazepanel ft-stream ft-eyegrab whisper-server; do
|
||||
test -x /opt/frametop/bin/$p
|
||||
done
|
||||
test -f /src/frametop/pointer/driver/build/driver_ft_pointer.so
|
||||
test -x /src/frametop/hands/build/ft-camd
|
||||
test -x /src/frametop/hands/build/ft-hands
|
||||
test -x /src/frametop/hands/rec/build/ft-handpanel
|
||||
/src/frametop/hands/models.sh check
|
||||
test -x /src/frametop/pack/build/distrobox/install
|
||||
/src/frametop/gaze/tracker/build/venv/bin/python -c "import numpy, cv2"
|
||||
/src/frametop/voice/build/venv/bin/python -c "import numpy, sentencepiece, sherpa_onnx"
|
||||
python3 -c "import PySide6, huggingface_hub"
|
||||
command -v zstd >/dev/null'
|
||||
|
||||
# The FrameDrop manifest names the zip's URL in this repo's release (a fork's, in a fork).
|
||||
- name: Frametop.zip
|
||||
run: |
|
||||
url=()
|
||||
[ "$GITHUB_REF_TYPE" = tag ] &&
|
||||
url=("https://github.com/$GITHUB_REPOSITORY/releases/download/$GITHUB_REF_NAME/Frametop.zip")
|
||||
framedrop/build.sh --image "$FT_IMAGE" --version "$VERSION" --commit "$GITHUB_SHA" "${url[@]}"
|
||||
|
||||
- name: keep the zip (manual runs)
|
||||
if: github.ref_type != 'tag'
|
||||
# actions/upload-artifact@v7.0.2
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
|
||||
with:
|
||||
name: Frametop-${{ env.VERSION }}
|
||||
path: framedrop/build/
|
||||
retention-days: 7
|
||||
|
||||
- name: draft release (tags)
|
||||
if: github.ref_type == 'tag'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
pre=() notes=pack/release-notes.md
|
||||
if [[ $VERSION == *-* ]]; then
|
||||
pre=(--prerelease)
|
||||
# get.sh --release asks for a channel and offers stable first: an experimental
|
||||
# release's install line names its channel.
|
||||
notes=$RUNNER_TEMP/release-notes.md
|
||||
sed 's/bash -s -- --release`/bash -s -- --release --experimental`/' pack/release-notes.md >"$notes"
|
||||
grep -q -- '--release --experimental`' "$notes" ||
|
||||
echo "::warning::pack/release-notes.md has no get.sh --release line to mark experimental"
|
||||
fi
|
||||
gh release create "$GITHUB_REF_NAME" --draft --verify-tag "${pre[@]}" \
|
||||
--title "Frametop $VERSION" --notes-file "$notes" --generate-notes \
|
||||
framedrop/build/Frametop.zip framedrop/build/frametop.framedrop.json \
|
||||
framedrop/build/frametop-release.json framedrop/build/SHA256SUMS
|
||||
@@ -11,5 +11,3 @@ frametop-report-*.txt
|
||||
*.pgm
|
||||
.venv/
|
||||
.frame-job.d/
|
||||
# eye net versions picked in Input Settings (models/catalog.py downloads them; CC BY-NC 4.0)
|
||||
gaze/tracker/models/
|
||||
@@ -1 +0,0 @@
|
||||
3.14
|
||||
@@ -25,22 +25,10 @@ scripts/frame.sh --host '<cmd>' # runs on the SteamOS host
|
||||
A Steam Frame is someone's personal headset, and they may be wearing it while you work.
|
||||
|
||||
- Don't kill or restart `gamescope`, `steam`, `vrserver`, `vrcompositor`, the gamescope session, or the Frametop desktop without asking. Each one ends or disrupts whatever is happening in VR.
|
||||
- Don't run host `sudo`, `steamos-readonly disable`, `steamos-devmode` changes, pacman installs, or reboots without explicit approval. Three installers need host `sudo`, and they ask for it: the Bluetooth fixes (`setup/bluetooth/install.sh`), hand tracking (`hands/run.sh install` and `caps`, which set ft-camd's file capabilities with `setcap`, and `uninstall` and `uncaps`, which take them back), and our own eye tracker's frame grabber (`gaze/tracker/install.sh`).
|
||||
- Write only inside the repo, `/tmp`, and the container unless told otherwise. The installers are the exception: they write the user services, launchers, and the SteamVR driver into the home folder. The Bluetooth fixes and the eye tracker's frame grabber also install root-owned files and system services under `/etc` (`/etc/steamframe`, `/etc/frametop`, `/etc/systemd/system`). When an installer starts writing something new outside the repo, or sets file capabilities, add it to `uninstall.sh` too: users uninstall with that script, not with each installer's `uninstall`.
|
||||
- Don't run host `sudo`, `steamos-readonly disable`, `steamos-devmode` changes, pacman installs, or reboots without explicit approval. Only the Bluetooth fixes need host `sudo`, and they ask.
|
||||
- Write only inside the repo, `/tmp`, and the container unless told otherwise. The installers are the exception: they write the user services, launchers, and the SteamVR driver into the home folder.
|
||||
- Never copy `.netrc`, SSH keys, or Steam config off the Frame or into this repo.
|
||||
|
||||
## SteamOS updates
|
||||
|
||||
A SteamOS update replaces SteamVR, KWin, and gamescope with the rest of the OS image. When a change starts depending on something from the image (a host file, an OpenVR interface outside the bundled header, an undocumented layout or output format, a SteamVR or KWin quirk), add a check for it to `scripts/update-check.py`, or a retest hint for its package there. [docs/design.md](docs/design.md) has the background.
|
||||
|
||||
## The image (`pack/`)
|
||||
|
||||
`pack/Containerfile` builds Frametop as an OCI image, the groundwork for installing without building on the headset; no installer uses it yet. [pack/design.md](pack/design.md) explains why and what is open, and [pack/README.md](pack/README.md) documents the specifics. Rules:
|
||||
|
||||
- **Pin every input.** Base images by digest, Python via `uv.lock` (commit the lock, never a bare `uv pip install`), downloads by tag or commit and sha256, CI actions by commit SHA.
|
||||
- **One build recipe.** The image runs the components' own `build.sh` scripts (`FRAME_IN_BOX=1`), and the OpenVR SDK they build against is pinned in `scripts/openvr.sh`. Change a build there, not in the Containerfile, and keep the Containerfile buildable on arm64, the only architecture the Frame has.
|
||||
- **Keep the tests green in the image.** `./ft dev test` runs `just test` inside it, and CI runs the same recipes. A new offline test that needs no headset goes in the `justfile` too.
|
||||
|
||||
## Names
|
||||
|
||||
User-facing names are "Frametop", "Frametop Display Settings", and "Frametop Input Settings". Programs and files use the `ft-` / `ft_` prefix (`ft-screens`, `ft-pointer`, `ft-layout`, the `ft_pointer` driver); config, units, and overlay keys use `frametop`. Program names must stay within 15 characters: Linux truncates process names there, and the scripts find programs with `pgrep -x` / `pkill -x`.
|
||||
@@ -1,45 +1,34 @@
|
||||
# Frametop
|
||||
|
||||
Frametop is a desktop for the Steam Frame that runs on the headset itself, with no PC. Put several monitors around you, pull app windows out to float on their own, and point with a mouse, or with your eyes.
|
||||
Frametop puts a multi-monitor KDE Plasma desktop into SteamVR on the Valve Steam Frame, and lets a Bluetooth mouse drive all of SteamVR. It installs and runs on the headset itself.
|
||||
|
||||
- **Screens that are real monitors.** Each KDE Plasma screen has its own resolution and shape: an ultrawide in front, portrait screens beside it. Move, resize, curve, and roll them, or pin one to your wrist or your head. They come back to your layout when the desktop starts.
|
||||
- **Windows that float on their own.** Take any app window off the screens into a panel of its own with Meta+Shift+F, its title bar button, or Launch as Standalone. It stays part of the desktop, so drag and drop and the clipboard still work between floating windows and the screens.
|
||||
- **Your other computers' monitors.** Stream a Windows PC's or a Mac's monitors, or a virtual monitor at any size, as screens of their own, with the same controls. See [Remote displays](#remote-displays).
|
||||
- **Profiles.** Save where your screens are, which ones show, and which apps are open where. Switch to a profile from Display Settings, a key, or its launcher entry, or start the desktop in one.
|
||||
- **One mouse for all of SteamVR.** A Bluetooth mouse drives a small dot anchored in the room. It works the screens, the dashboard, Steam, and overlays, and hands the laser back when you pick up a controller.
|
||||
- **Your hands (experimental).** Hand tracking from the headset's own cameras: poke a screen to click, pinch and grab screens to move, resize, and throw them, and see your hands over the screens.
|
||||
- **Look and click (experimental).** In gaze mode the pointer goes where you look. Meta+J and Meta+K, or the mouse buttons, click and fine-tune, and each correction teaches the tracker. Calibration runs in the headset.
|
||||
- **Made for long sessions.** The displays turn off when the headset isn't used, even on a stand that makes it seem worn. It stays awake on the charger, and you can reach the desktop remotely over VNC.
|
||||
Each screen is its own monitor with its own resolution, so you can have an ultrawide in the middle and two portrait screens beside it, at whatever size and distance you like. The screens come back to your saved layout when the desktop starts. You can move, resize, curve, and roll them, pin one to your wrist, and put them all back with a shortcut.
|
||||
|
||||
Two settings apps come with it: Frametop Display Settings for the screens, profiles, and power, and Frametop Input Settings for mice, keyboards, gaze, and button mappings. Optional fixes let Bluetooth LE mice and keyboards like the Swiftpoint Z3 reconnect after they sleep. Install it with one command: see [Install on the headset](#install-on-the-headset).
|
||||
The mouse shows up as a small dot anchored in the room. It works on the SteamVR dashboard, Steam, overlays, and the desktop, and it hands the laser back to your controllers when you pick one up.
|
||||
|
||||
It comes with two settings apps, Frametop Display Settings for the screens and Frametop Input Settings for mice, keyboards, and button mappings, plus fixes that let Bluetooth LE mice and keyboards like the Swiftpoint Z3 reconnect after they sleep.
|
||||
|
||||
When you're not wearing the headset, Frametop can turn its displays off and keep it awake on the charger, so you can still reach it remotely. This works even on a stand or mount that makes the headset seem worn.
|
||||
|
||||
Frametop is an independent project, not made by or affiliated with Valve.
|
||||
|
||||
Frametop lives at [Frametop/frametop](https://github.com/Frametop/frametop): code, releases, issues, and pull requests. It moved there from DeeJanuz/frametop on 2026-10-09; the old links and clones still work.
|
||||
|
||||
Join the [Frametop Discord](https://discord.gg/W3X9f7z3Bc) for questions, ideas, and help with your setup.
|
||||
|
||||
## Install on the headset
|
||||
|
||||
> **SteamOS 0.4:** SteamOS 0.4 moved the eye tracker's data that gaze mode reads. This version of Frametop reads both SteamOS 0.3's and 0.4's, and it's tested on 0.4.5. Run `scripts/doctor.sh` after the update: it says whether the eye tracker's layout is one Frametop knows. It also says whether the update deleted the Bluetooth fixes or our eye tracker's frame grabber, which happens when they were installed by Frametop 0.3.0-exp.3 or older. Reinstall what it names (`setup/bluetooth/install.sh install`, `gaze/tracker/install.sh`). From then on they're kept through updates.
|
||||
|
||||
You need a Steam Frame with an internet connection, a keyboard (Bluetooth, or the on-screen one), and about 3 GB of free space.
|
||||
|
||||
1. In the launcher, choose Launch a program → Desktop.
|
||||
2. In the application menu, open System → Konsole.
|
||||
3. Run:
|
||||
3. Clone the repo and run the installer:
|
||||
|
||||
```
|
||||
curl -fsSL https://frametop.github.io/frametop/get.sh | bash
|
||||
git clone https://github.com/DeeJanuz/frametop.git ~/frametop
|
||||
cd ~/frametop
|
||||
./install.sh
|
||||
```
|
||||
|
||||
It asks which version you want: stable (the `main` branch, tested releases) or experimental (the `experimental` branch, the newest features, less tested). Then it clones the repo into `~/frametop` and runs `install.sh`. To choose without the question, add `-s -- --stable` or `-s -- --experimental` after `bash`. By hand, the same is `git clone https://github.com/Frametop/frametop.git ~/frametop`, then `cd ~/frametop` and `./install.sh` (add `--branch experimental` to the clone for experimental).
|
||||
The installer sets up distrobox in your home folder (the system files aren't touched), a Fedora build container, and everything else. The first run downloads 1–2 GB. It asks you two things along the way. The Bluetooth fixes need your `sudo` password; if you've never set one, run `passwd` first, or skip them for now. SteamVR has to restart once at the end, which closes everything open in VR, including the terminal. Rebooting the headset works too.
|
||||
|
||||
The third and fourth choices, stable release and experimental release, download Frametop already built (`Frametop.zip`, about 1.4 GB, from the [releases](https://github.com/Frametop/frametop/releases)) and install it without compiling anything. `-s -- --release` picks the stable release without the question, and `-s -- --release --experimental` the experimental one.
|
||||
|
||||
The installer sets up distrobox in your home folder (the system files aren't touched), a Fedora build container, and everything else. The first run downloads 1–2 GB. It asks along the way whether to install each optional part: voice dictation (yes by default; it downloads about 320 MB of speech models), hand tracking (experimental, yes by default), in a release the [Hand Recorder](https://frametop.github.io/frametop/hand-recorder) (yes by default; it records your hands for Frametop's open hand dataset), gaze mode (experimental, yes by default), our own eye tracker for it (yes by default), in a release the Eye Recorder (yes by default, with our eye tracker; it records your eyes for Frametop's eye tracking dataset), and the Bluetooth fixes, then whether to restart SteamVR. The stable version asks only about gaze mode, the eye tracker, and the Bluetooth fixes. Hand tracking, the Hand Recorder, the eye tracker, and the Bluetooth fixes need your `sudo` password; if you've never set one, run `passwd` first, or skip them for now. SteamVR has to restart once at the end, which closes everything open in VR, including the terminal. Rebooting the headset works too.
|
||||
|
||||
After the restart, Launch a program → Desktop opens the multi-screen desktop, with its screens arranged around where you're facing. Frametop Display Settings and Frametop Input Settings are in the desktop's application menu, under Settings. SteamOS's own single-screen desktop is still there, as Native Desktop in the same list.
|
||||
After the restart, Launch a program → Desktop opens the multi-screen desktop, with its screens arranged around where you're facing. Frametop Display Settings and Frametop Input Settings are in the desktop's application menu, under Settings.
|
||||
|
||||
If you work in the desktop for long stretches, or leave the headset on a stand, open Frametop Display Settings → Power. Turn on Stay awake while plugged in: by default Steam puts the Frame to sleep after an hour without input, even while it charges. And choose when the displays turn off while the headset isn't used. SteamVR turns them off a few seconds after you take the headset off, but a stand or mount that covers the proximity sensor inside it makes the headset seem worn, and its displays stay on all night.
|
||||
|
||||
@@ -51,114 +40,27 @@ If you work in the desktop for long stretches, or leave the headset on a stand,
|
||||
|
||||
## Use
|
||||
|
||||
### Screens
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Point near the bottom of a screen | Its controls fade in: the bar, the curve and roll buttons, and, near a corner, a soft glow round it: press there and drag to resize |
|
||||
| Move the mouse | The dot moves around you and snaps onto whatever panel it's over |
|
||||
| Click, right-click, scroll | Acts on the panel under the dot |
|
||||
| Pick up a controller | The controller gets its laser back; move the mouse to take over again |
|
||||
| Point near the bottom of a screen | Its controls fade in: the bar, the curve and roll buttons, and the resize tab on the corner |
|
||||
| Drag the bar under a screen | Moves the screen; scroll while dragging to push it away or pull it closer. With the mouse, hold right while dragging to tilt it |
|
||||
| Drag the tab on a screen's bottom right corner | Resizes the screen |
|
||||
| Click the curve button (next to the bar) | Curves the screen around you, or flattens it |
|
||||
| Drag the roll button sideways, or scroll on it | Rolls the screen; it snaps level near straight |
|
||||
| While carrying a screen, sweep its laser across your other controller's ring, then let go | Pins it to that wrist, at its size and distance, as you hold it when you let go; it shows while you see its front. Grab its bar to adjust it (it stays pinned); sweep across the ring again to take it off |
|
||||
| Set a screen to On your head (Frametop Display Settings, Visibility & pins) | Pins it to your head where it is, like a HUD. Grab its bar to move it; it stays on your head |
|
||||
| Save current arrangement… (Frametop Display Settings, Layout) | Saves where the screens are, with their sizes and pins, under a name. Pick a saved layout under Arrangement and press Arrange now to switch to it |
|
||||
| Meta+Shift+R in the desktop | Puts the screens back in their layout (also in the menu as Reset Screen Layout, and mappable to a mouse button) |
|
||||
| Meta+Alt+Tab, or Meta+Alt+Shift+Tab | Spins every screen and floating window around you together, like a lazy susan, so the next one on your right (or left) glides to straight ahead, with the pointer and typing going to it. Their arrangement stays the same: the room turns instead of you. Tap again to keep going; Meta+Shift+R puts the screens back. Pinned screens stay where they are |
|
||||
| Meta+Space, on any keyboard | Opens spotwatch, a search pill in front of you: type an app's or a remote display's name and press Enter. A running app's window spins to the front, an app that isn't running opens floating, and a remote display connects and spins to the front. Shift+Enter (or a right click) has more: a private window, a new window, docked on this screen, or one of its windows. Esc closes it. See [docs/spotwatch.md](docs/spotwatch.md) |
|
||||
| Meta+Shift+H in the desktop | Hides or shows all screens (also in the menu as Hide/Show Screens, and mappable). The Visibility & pins tab of Frametop Display Settings can instead show them only with the dashboard open, or while you look at your wrist |
|
||||
| Tap Meta, on any keyboard | Opens the Steam menu in the SteamVR dashboard, or closes the dashboard, wherever you are. The desktop's launcher is still on the taskbar and Alt+F1. Change it in Frametop Input Settings (Keyboard page), where any key combination or modifier tap can do a Frametop or Steam action, open a profile, or run a command of your own |
|
||||
| Switch a screen to Hidden (Frametop Display Settings, Visibility & pins → Screens shown) | Hides just that screen until you switch it back, whatever the other visibility settings say; new windows that would open on it float instead |
|
||||
| Play a VR game | Frametop pauses so the game gets the headset to itself (see [Pause for VR games](#pause-for-vr-games)): the screens hide and your controllers stay in the game. Click both thumbsticks together twice to bring Frametop back. With the automatic pause off, the screens still hide, and the SteamVR dashboard or Meta+Shift+H shows them; to keep them visible over games, change During VR games on the Visibility & pins tab |
|
||||
|
||||
Restarting the desktop (Restart desktop in Frametop Display Settings) closes its windows, but background work you started in it, such as servers, tmux sessions, or builds, keeps running.
|
||||
|
||||
### Mouse and controllers
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Move the mouse | The dot moves around you and snaps onto whatever panel it's over |
|
||||
| Click, right-click, scroll | Acts on the panel under the dot |
|
||||
| Pick up a controller | The controller gets its laser back; move the mouse to take over again |
|
||||
| Leave the headset on a stand | Its displays turn off once it has gone unused for the time set in Frametop Display Settings → Power, even if the stand covers its proximity sensor. Pick it up, or use any mouse, keyboard, or button, and they come back on |
|
||||
| Play a VR game | The screens hide and your controllers stay in the game. Open the SteamVR dashboard, or press Meta+Shift+H, to see and use them. To keep them visible over games, change During VR games on the Visibility & pins tab; the controllers still stay in the game, and you use the screens with the mouse or the dashboard |
|
||||
|
||||
You can map the mouse's extra buttons to actions such as Toggle SteamVR dashboard, Recenter pointer, or Head follow on/off on the Buttons page of Frametop Input Settings, and the Frame controllers' buttons on its Controllers page. Pointer speed, dot size, and the rest are on its Pointer page and take effect immediately. If a panel you only look at, such as a performance overlay that follows your view, keeps catching the dot, tick it (or its whole app) on the Ignored panels page, and the pointer passes through it. Head follow, which is experimental and off by default, makes the pointer come along when you turn your head: it stays put until your head turns past the leash angle, then glides back to its place in your view, and a leash of 0 keeps it fixed in your view. It's only lightly tested and not polished; tuning its settings, or improving how it feels, is open to anyone who wants to take it further.
|
||||
|
||||
### Floating windows
|
||||
|
||||
Any desktop window can float in VR as a panel of its own. It stays a window of the same desktop, so drag and drop and the clipboard work between floating windows and the screens, and it's still in the taskbar and Alt+Tab.
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Meta+Shift+F over a desktop window | Floats that window, or puts it back on its screen if it floats. It acts on the window under the pointer, or the active one if the pointer is over the wallpaper. Rebind it, or map it to a mouse or controller button, in Frametop Input Settings (Keyboard page, or Buttons and Controllers as Float window in VR) |
|
||||
| Click the float button, left of Close in a window's title bar | The same. Float in VR is also in every window's menu (Alt+F3). Apps that draw their own title bar, like Chromium and Electron apps, don't have the button: use Meta+Shift+F |
|
||||
| Right-click an app in the Application Launcher (or the taskbar) and pick Launch as Standalone | Starts the app with its window floating, where that app last floated, or in front of you the first time. From a terminal: `float/ft-float launch org.kde.dolphin`, or `float/ft-float run <command>` |
|
||||
| Meta+scroll over a floating window | Scales it up or down |
|
||||
|
||||
### Profiles
|
||||
|
||||
A profile is a named setup: where the screens are, with their sizes and pins, which ones are hidden, and which apps are open and where their windows are, on a screen or floating.
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Save as profile… (Frametop Display Settings, Layout & profiles) | Saves the current setup under a name, or updates the profile you're in |
|
||||
| Pick a profile under Arrangement and press Open profile | Switches to it: the screens move, open windows of its apps go to their places, and the apps that aren't open start. Nothing closes |
|
||||
| Pick a profile under Start in profile, or run its entry (Frametop: NAME) from SteamVR's Launch a program list | The desktop starts in that profile, or switches to it if it's running. A profile can also go on a key combination, mouse button, or controller button in Frametop Input Settings |
|
||||
|
||||
### Remote displays
|
||||
|
||||
Remote displays bring another computer's monitors into Frametop. Each one is a panel with a screen's controls: move it, resize it, curve it, pin it, and save it in a profile. The mouse, the keyboard and the lasers work on it, and its sound plays on the Frame. The computer streams with Sunshine's protocol, and the Frame decodes the streams in hardware.
|
||||
|
||||
Each computer needs a host kit once:
|
||||
|
||||
| Computer | What it streams | Set up with |
|
||||
| --- | --- | --- |
|
||||
| Windows PC | Its monitors, and virtual monitors at any size | Vibepollo with Frametop's build: [host/windows/README.md](host/windows/README.md) |
|
||||
| Mac with Apple silicon | Its monitors | Sunshine, one instance per monitor, and Frametop Streams in the menu bar to turn them on and off: [host/macos/README.md](host/macos/README.md) |
|
||||
| Linux PC (Arch, CachyOS) | Its monitors, and virtual monitors | Vibepollo with Frametop's build, not tested yet: [host/linux/README.md](host/linux/README.md) |
|
||||
|
||||
Then, on the Frame, open Frametop Remote Displays (in the app menu, or Remote displays in Display Settings), choose Add computer, sign in with the computer's Web UI login, and pick its displays. Each display has a Connected switch, a Shown switch, and its stream's resolution, frame rate and bitrate. Meta+Space finds a remote display by name and brings it to the front. A PC with a Steam Link dongle can stream over it, which skips your router.
|
||||
|
||||
A monitor you stream switches to the stream's resolution while it streams (Windows), so pick a size the monitor has, or one with the same shape. A computer that's asleep can't be reached. [docs/reference.md](docs/reference.md#frametop-remote-displays) has the details.
|
||||
|
||||
### Pause for VR games
|
||||
|
||||
Frametop pauses while a VR game runs, so the game gets the headset's CPU and GPU, and comes back a few seconds after the game ends. Paused, the screens hide and the desktop nearly stops drawing, but its windows stay open. Gaze mode's eye tracking stops, and so do remote desktop and hand tracking if they run. The mouse works as a plain mouse in SteamVR.
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Click both thumbsticks together, twice | Pauses Frametop, or brings it back, in a game or not. You hear a short sound. The game sees the clicks too |
|
||||
| Start a VR game | Frametop pauses, and comes back 5 seconds after the game ends. Bring it back during the game, and it stays on until that game ends |
|
||||
| Map Pause/resume Frametop to a mouse button, key combination, or controller button | The same, from that button (Frametop Input Settings) |
|
||||
|
||||
The Game optimization page of Frametop Input Settings turns the automatic pause off, changes the gesture, closes the desktop instead of hiding it (more for the game, but its windows close), and turns the sound off. From a terminal: `input/ft-pause on`, `off`, or `status`.
|
||||
|
||||
### Gaze mode (experimental)
|
||||
|
||||
In gaze mode the pointer goes where you look, and the mouse or the keyboard does the last bit. The installer offers it (or run `gaze/run.sh install` later), and then our own eye tracker for it, which is more accurate than SteamVR's (or run `gaze/tracker/install.sh` later; it needs `sudo`). Gaze mode uses ours once it's installed, and SteamVR's until then. Turn it on and calibrate it on the Gaze page of Frametop Input Settings.
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Tap Meta+J, or Meta+K | A left or right click where you look |
|
||||
| Hold Meta+J, turn your head onto what you meant, and let go | A click there. Held still for half a second, it becomes a press, and turning your head drags |
|
||||
| Hold the left mouse button, move the mouse onto what you meant, and let go | A click there. The right button does the same for a right click. Held still, the left button drags |
|
||||
| Double right click (or double Meta+K) while dragging a screen's bar | Pans and tilts the screen |
|
||||
| Put the headset on | A quick check: look at the dot for a moment, and the pointer lines up again |
|
||||
|
||||
By default the mouse only corrects: while the gaze has the pointer, moving the mouse does nothing until you hold a button. Each correction before a click teaches the tracker where it was off. A correction past the learning limit (55 degrees by default, about half of what you can see) starts a quick check instead. Calibrate, on the Gaze page, runs a full calibration in a panel in front of you: look at each dot and click. Check headset fit shows how well the eye tracker sees your eyes. [gaze/README.md](gaze/README.md) has the details.
|
||||
|
||||
### Hand tracking (experimental)
|
||||
|
||||
Hand tracking uses the headset's own cameras, with no controllers. The installer offers it (or run `hands/run.sh install` later; it needs `sudo` once, to let its camera broker read the cameras). Installed, it runs while the Frametop desktop does, and parks while Frametop is paused for a VR game. It uses a lot of the headset's CPU.
|
||||
|
||||
| Do this | To get this |
|
||||
| --- | --- |
|
||||
| Poke a screen with your index finger | A click there |
|
||||
| Pinch a screen's bar, edge, or corner, and move | Moves or resizes the screen |
|
||||
| Hold a screen like a card, or in a fist | Carries it; turn your wrist to turn it. Flick it away, or let go fast to throw it |
|
||||
| Hold both hands on a screen and pull them apart | Stretches it |
|
||||
| Hold an open hand out to a far screen, then make a fist and pull it back ("grab that") | Brings it to arm's reach |
|
||||
| Turn a hand palm up toward your face, look at it, and pinch the ring that shows | Opens the Steam menu |
|
||||
|
||||
The Hands page of Frametop Input Settings sets when it runs (with Frametop, always with SteamVR, or off) and switches each hand interaction on or off, live. `ft-handsctl status` shows how it's doing. [hands/README.md](hands/README.md) has the details.
|
||||
Restarting the desktop (Restart desktop in Frametop Display Settings) closes its windows, but background work you started in it, such as servers, tmux sessions, or builds, keeps running.
|
||||
|
||||
### Leave the headset on a stand and reach it remotely
|
||||
|
||||
@@ -171,20 +73,16 @@ With the displays off, the headset keeps tracking and rendering, so it uses abou
|
||||
|
||||
## Known limitations
|
||||
|
||||
This is an early release, tested on one Steam Frame (SteamOS 0.4.5 build 20261007, SteamVR 2.18.2; before that SteamOS 0.3.0 build 20260922, SteamVR 2.17.10).
|
||||
This is an early release, tested on one Steam Frame (SteamOS 0.3.0 build 20260922, SteamVR 2.17.10).
|
||||
|
||||
- A SteamOS or SteamVR update can break parts of it until Frametop catches up. After an update, run `cd ~/frametop && scripts/doctor.sh` in a terminal. It checks what Frametop needs from SteamOS, and says what changed since the versions you last marked as working and what to try. Once everything works, `scripts/doctor.sh --mark-good` records the versions. If something stops working, please report it.
|
||||
- A SteamOS or SteamVR update can break parts of it until Frametop catches up. If something stops working after an update, please report it.
|
||||
- The first install downloads 1–2 GB for the build container and compiles everything on the headset, which takes several minutes.
|
||||
- During a VR game, mapped controller buttons belong to the game, so they can't bring the screens up. The pause gesture still works: Frametop reads it without taking the thumbsticks from the game. With the automatic pause off, open the SteamVR dashboard, press Meta+Shift+H, or use a mapped mouse button instead.
|
||||
- Flatscreen games aren't detected as games, so they don't pause Frametop by themselves: click both thumbsticks twice to pause it. If your controllers end up working the screens instead of the game, set Controllers on the screens to "Only with the SteamVR dashboard open" (Frametop Display Settings, Visibility & pins tab).
|
||||
- During a VR game you can't show the screens with a controller button, because the game owns the buttons. Open the SteamVR dashboard, press Meta+Shift+H, or use a mapped mouse button instead.
|
||||
- Flatscreen games aren't detected as games. If your controllers end up working the screens instead of the game, set Controllers on the screens to "Only with the SteamVR dashboard open" (Frametop Display Settings, Visibility & pins tab).
|
||||
- Typing follows your last click. A controller click on a panel other than the screens (the dashboard, a Steam app) doesn't move typing there; click it with the mouse, or click a screen to bring typing back.
|
||||
- The screens don't draw a mouse cursor of their own. The 3D mouse's dot or SteamVR's laser shows where you're pointing.
|
||||
- Profiles reopen apps, not what the apps had open. Tabs, files, and folders are left to each app's own restore.
|
||||
- Dragging something from one panel to another (a screen and a floating window) works, but the dragged item's icon doesn't show while the pointer is between panels.
|
||||
- Gaze mode is only as good as its calibration, and that depends on how the headset sits on your face. If the pointer lands off after you adjust the headset, run Quick check or Calibrate on the Gaze page of Frametop Input Settings.
|
||||
- On SteamVR's Settings page, the 3D mouse shows a laser beam and a larger hit dot, like a controller. SteamVR doesn't tell other programs where that page is (unlike Steam's pages, such as Library), so the mouse used to miss most of it: clicks went through to a desktop screen behind, and the dot disappeared. As a workaround, on that page only, the laser starts near your eye and SteamVR finds the page itself. See docs/design.md.
|
||||
- Remote desktop over VNC (Frametop Remote Access in the app menu, or `./desktops.sh remote on`) needs Tailscale on the Frame. It shows the primary screen only. The app turns it on and off, shows the address, and shows, copies, or changes the VNC password. The password is made at random on the Frame and kept in `~/.config/frametop-remote` (only you can read it); VNC limits it to 8 characters, and the tailnet encrypts the connection. Turning it on in a desktop that started with it off takes a desktop restart. It costs almost nothing until a viewer connects; the picture then takes a few seconds to appear.
|
||||
- The desktop has no blur behind panels and menus, and no window animations, so it leaves the headset's GPU to SteamVR. Turn them back on in the Frametop desktop's System Settings (Desktop Effects, and Animation speed under General Behavior); Frametop won't turn them off again.
|
||||
- Remote desktop over VNC (Frametop Remote Access in the app menu, or `./desktops.sh remote on`) needs Tailscale on the Frame. It shows the primary screen only. The app turns it on and off, shows the address, and shows, copies, or changes the VNC password. The password is made at random on the Frame and kept in `~/.config/frametop-remote` (only you can read it); VNC limits it to 8 characters, and the tailnet encrypts the connection. Turning it on in a desktop that started with it off takes a desktop restart.
|
||||
- Turning the displays off on a stand only turns their backlight off. SteamVR has no way for other programs to put the headset in standby, so tracking and rendering keep running, and the headset draws nearly its full power.
|
||||
|
||||
## Reporting problems
|
||||
@@ -195,84 +93,45 @@ In a terminal on the headset, run:
|
||||
cd ~/frametop && scripts/report.sh
|
||||
```
|
||||
|
||||
From a release, start in `~/.local/share/frametop/releases/current` instead of `~/frametop`.
|
||||
|
||||
This writes `frametop-report-<date>.txt` with version numbers, settings, service states, CPU, memory and heat, and the state and recent logs of every part: the desktop (Frametop's keyboard, the Steam menu, the screens), gaze, hand tracking (its models, cameras, and how often it sees each hand), and voice (its models, the microphone, and whether whisper-server transcribes). It takes up to half a minute, because it starts the gaze service for a moment to see whether the eye tracker sends. Bluetooth addresses and the headset's serial number are masked, and what you said to Frametop Voice is left out.
|
||||
|
||||
To report on one part only, name it: `--system`, `--desktop`, `--gaze`, `--hands` or `--voice` (several are fine, for example `scripts/report.sh --hands --voice`).
|
||||
|
||||
If the problem is something you can make happen, like a window that won't drag, a pinch that doesn't register, or dictation that types nothing, add `--watch`. After the report it records for 60 seconds (`--watch 120` for longer) while you make it happen in the headset, and keeps the logs from that time:
|
||||
|
||||
- desktop: when the Steam menu opens and closes, which laser drags what, where typing goes, and when Frametop's keyboard opens or why it doesn't
|
||||
- gaze: which eye tracker the gaze service uses, and whether its samples come in
|
||||
- hands: each second, how many frames hand tracking made, how often it saw each hand, and how close the fingertips came
|
||||
- voice: what Frametop Voice did (the number of words, not the words) and the microphone's level each second (no sound is kept)
|
||||
- system: every 5 seconds, the CPU's load, clocks and heat, and the busiest programs
|
||||
|
||||
If gaze or its calibration doesn't work, run the report while you wear the headset. `scripts/gaze-report.py` prints only the gaze part, with what looks wrong first. Then [open an issue](https://github.com/Frametop/frametop/issues), describe what you did, what you expected, and what happened, and attach the file. Quick questions can go to [Discord](https://discord.gg/W3X9f7z3Bc) instead.
|
||||
This writes `frametop-report-<date>.txt` with version numbers, service states, settings, and recent logs. Bluetooth addresses and the headset's serial number are masked. Then [open an issue](https://github.com/DeeJanuz/frametop/issues), describe what you did, what you expected, and what happened, and attach the file.
|
||||
|
||||
## Update
|
||||
|
||||
Run the same command again. It updates `~/frametop` to the latest of the version you have (or switches, if you pick the other one) and installs it:
|
||||
|
||||
```
|
||||
curl -fsSL https://frametop.github.io/frametop/get.sh | bash
|
||||
cd ~/frametop && git pull && ./install.sh
|
||||
```
|
||||
|
||||
Or by hand: `cd ~/frametop && git pull && ./install.sh`.
|
||||
|
||||
## Uninstall
|
||||
|
||||
In a terminal on the headset, run:
|
||||
|
||||
```
|
||||
curl -fsSL https://frametop.github.io/frametop/uninstall.sh | bash
|
||||
./desktops.sh uninstall # the launcher's Desktop entry goes back to the stock desktop
|
||||
./desktops.sh relay uninstall
|
||||
pointer/helper/run.sh uninstall
|
||||
power/run.sh uninstall
|
||||
pointer/driver/install.sh uninstall # then restart SteamVR
|
||||
input-settings/install.sh uninstall
|
||||
display-settings/install.sh uninstall
|
||||
setup/bluetooth/install.sh uninstall # if you installed the Bluetooth fixes
|
||||
```
|
||||
|
||||
It works in two steps, so it never takes away the keyboard, mouse, or desktop you're using while it runs:
|
||||
|
||||
1. It stops Frametop from starting. Launch a program → Desktop opens the stock desktop again, and the Native Desktop entry, Frametop's services, its SteamVR driver, and its menu entries are removed, along with the system files of our eye tracker and the Bluetooth fixes and the file capabilities of hand tracking's camera broker (those need your `sudo` password). Everything running now keeps running until you restart the headset, and it offers to restart it for you.
|
||||
2. After the restart, run the same command again. It deletes the code in `~/frametop`, and asks whether to delete your settings, any eye or hand recordings, and the build container (1–2 GB) too.
|
||||
|
||||
To see what it would do without changing anything, add `-s -- --dry-run` after `bash`. If the code isn't in `~/frametop`, add `-s -- --dir <folder>`. From the repo, the same script is `./uninstall.sh`.
|
||||
|
||||
Don't delete `~/frametop` by hand before you uninstall and restart: the desktop and the input relay run from it, and without it Launch a program → Desktop no longer opens anything. If you've already deleted it, the command above still works, since it doesn't need the repo.
|
||||
|
||||
Unless you ask for them to go, your settings stay: `~/.config/frametop.conf`, `frametop-input.json` (button maps and key combinations), `frametop-layout.json` (the layout and profiles), `frametop-float.json`, and `frametop-remote/` in `~/.config`, the gaze calibration in `~/.local/state/frametop`, and the desktop's own Plasma setup in `~/.config/frametop`. A later install picks them up again.
|
||||
|
||||
## How it works
|
||||
|
||||
A Plasma session runs nested inside ft-screens (`screens/`), a small Wayland compositor. KWin opens one window per screen, ft-screens sets each window's size, and each frame goes to SteamVR as an overlay without being copied. An input relay (`input/`) keeps Bluetooth mice working in SteamVR and feeds the mouse to the 3D pointer, which drives a virtual SteamVR controller (`pointer/`). A power service (`power/`) turns the displays off while the headset isn't used. [docs/reference.md](docs/reference.md) covers each piece, and [docs/design.md](docs/design.md) explains the design and what we learned about SteamVR on the Frame. [docs/hazards.md](docs/hazards.md) lists known ways the input handling can go wrong.
|
||||
|
||||
| Folder | What it is |
|
||||
| --- | --- |
|
||||
| `get.sh` | The one-line installer: picks stable or experimental, clones or updates the repo, and runs `install.sh`; or installs a built release (`--release`). |
|
||||
| `install.sh` | The one-step installer. Safe to re-run. |
|
||||
| `uninstall.sh` | The uninstaller: run it, restart the headset, and run it again. It doesn't need the rest of the repo. |
|
||||
| `desktops.sh` | Start, stop, and configure the desktop, and install the input relay. |
|
||||
| `screens/` | ft-screens, the compositor (wlroots and OpenVR). |
|
||||
| `session/` | The desktop session script and its config example. |
|
||||
| `layout/` | ft-layout: where the screens float, and their sizes. |
|
||||
| `float/` | Floating windows: ft-floatd and the KWin script that float a desktop window in VR. |
|
||||
| `decoration/` | The desktop's window decoration: Breeze's look plus the float button. |
|
||||
| `input/` | The input relay (Bluetooth mice and keyboards, button maps). |
|
||||
| `pointer/` | The 3D mouse: SteamVR driver, helper service, and a probe tool. |
|
||||
| `power/` | ft-powerd: turns the displays off while the headset isn't used. |
|
||||
| `voice/` | Frametop Voice: dictation and wake words, recognised on the headset; the input relay types the text. See [voice/README.md](voice/README.md). |
|
||||
| `gaze/` | Gaze mode (experimental): the gaze service, its calibration panel, our own eye tracker, and the gaze probe. See [gaze/README.md](gaze/README.md). |
|
||||
| `hands/` | Hand tracking (experimental): the camera broker and the tracker. See [hands/README.md](hands/README.md). |
|
||||
| `display-settings/`, `input-settings/` | The two settings apps (Kirigami, Python). |
|
||||
| `remote/` | Frametop Remote Access, the app that turns remote desktop over VNC on and off. |
|
||||
| `remote-displays/` | Frametop Remote Displays, the app that adds other computers' monitors as screens. |
|
||||
| `stream/` | ft-stream, one per remote display: receives and decodes its stream for ft-screens. |
|
||||
| `host/` | The host kits for remote displays: Windows, macOS and Linux. |
|
||||
| `setup/` | The build container and the Bluetooth fixes. See [setup/README.md](setup/README.md). |
|
||||
| `scripts/` | Helpers the installers use. They run commands locally on the Frame, or over SSH from a PC. |
|
||||
|
||||
## Packaging
|
||||
|
||||
`pack/` builds Frametop as an OCI image: the toolchain, the native binaries, and the locked Python environment (via [uv](https://docs.astral.sh/uv/)), built and tested by GitHub Actions. It is the groundwork for installing Frametop without building anything on the headset. No installer uses it yet. For development, `./ft dev build` builds the image and `./ft dev test` runs the tests inside it. [pack/design.md](pack/design.md) explains why an image and what is still open, and [pack/README.md](pack/README.md) documents the image itself.
|
||||
|
||||
## Developing from a PC
|
||||
|
||||
The scripts also work from a Linux or WSL PC over SSH, which is easier for editing code. On the Frame they use the local checkout; on a PC they sync the repo to `~/dev/frametop` on the Frame and run there.
|
||||
@@ -287,17 +146,16 @@ The scripts also work from a Linux or WSL PC over SSH, which is easier for editi
|
||||
IdentityFile ~/.ssh/<your-key>
|
||||
```
|
||||
|
||||
3. The Bluetooth fixes need `sudo` on the Frame. The installer asks for the password in your terminal (over `ssh -t`). To skip the question, or to install with no terminal, put it in `.env` at the repo root instead. It's gitignored and never synced:
|
||||
3. The Bluetooth fixes need `sudo`, and there's no terminal on the Frame to type the password into, so put it in `.env` at the repo root. It's gitignored and never synced:
|
||||
|
||||
```
|
||||
steamos_root_pwd="<password>"
|
||||
```
|
||||
|
||||
Then run `./install.sh` from the PC. If SteamVR isn't running on the Frame, the services that need it start with it later. Daily use:
|
||||
Then run `./install.sh` from the PC. Daily use:
|
||||
|
||||
```
|
||||
scripts/doctor.sh # is the Frame reachable and ready?
|
||||
scripts/doctor.sh --mark-good # and record the versions Frametop works with
|
||||
scripts/sync.sh # copy the repo to ~/dev/frametop on the Frame
|
||||
scripts/frame.sh '<cmd>' # run in the dev container, in the Frame's copy
|
||||
scripts/frame.sh -C <dir> '<cmd>' # same, in a folder of the repo
|
||||
@@ -310,9 +168,4 @@ Programs built in the `dev` container link against its glibc, which is newer tha
|
||||
|
||||
## License
|
||||
|
||||
MIT. See [LICENSE](LICENSE). The rendered hands' 3D models in `screens/assets/hands` (see its [README](screens/assets/hands/README.md); each folder has its licence) are from:
|
||||
|
||||
- the W3C Immersive Web Working Group's WebXR Input Profiles, `generic-hand` (MIT, Copyright (c) 2019 Amazon), the default;
|
||||
- Babylon.js's hand meshes by Patrick Ryan, https://github.com/BabylonJS/Assets (CC BY 4.0, https://creativecommons.org/licenses/by/4.0/), unchanged;
|
||||
- the Mixed Reality Toolkit: MRTK3's OpenXR hand (BSD-3-Clause, Copyright (c) 2025, Mixed Reality Toolkit Contributors) and MRTK2's rigged hand (MIT, Copyright (c) Microsoft Corporation), converted to glTF;
|
||||
- Godot XR Tools' hands by DigitalN8m4r3 (Miodrag Sejic), CC0 1.0.
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Switch the running Frametop desktop's window decoration without restarting it: Frametop's
|
||||
# own (this folder, with the float button) or back to Breeze. The session script does the
|
||||
# same at every desktop start (session/frametop-session.sh), so this is for trying changes.
|
||||
# decoration/apply.sh install this folder's copy and use it
|
||||
# decoration/apply.sh --off back to Breeze (until the next desktop start)
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
deco=kwin4_decoration_qml_frametop
|
||||
cfg=$HOME/.config/frametop
|
||||
kwinrc=$cfg/kwinrc
|
||||
|
||||
# The desktop's D-Bus: from its Plasma shell, which uses Frametop's config folder (KWin's own
|
||||
# environment isn't readable: it runs with extra capabilities).
|
||||
bus=
|
||||
for pid in $(pgrep -x plasmashell); do
|
||||
env=$( (tr '\0' '\n' < "/proc/$pid/environ") 2>/dev/null) || continue
|
||||
grep -qx "XDG_CONFIG_HOME=$cfg" <<<"$env" || continue
|
||||
bus=$(sed -n 's/^DBUS_SESSION_BUS_ADDRESS=//p' <<<"$env")
|
||||
done
|
||||
[ -n "$bus" ] || { echo "the Frametop desktop isn't running" >&2; exit 1; }
|
||||
|
||||
if [ "${1:-}" = --off ]; then
|
||||
kwriteconfig6 --file "$kwinrc" --group org.kde.kdecoration2 --key library --delete
|
||||
kwriteconfig6 --file "$kwinrc" --group org.kde.kdecoration2 --key theme --delete
|
||||
else
|
||||
# Under a new name each time: KWin keeps a decoration's QML (even a broken one) by name
|
||||
# until it restarts. The next desktop start goes back to the plain name.
|
||||
decos=${XDG_DATA_HOME:-$HOME/.local/share}/kwin/decorations
|
||||
rm -rf "$decos/${deco}"_try*
|
||||
deco=${deco}_try$(date +%s)
|
||||
mkdir -p "$decos"
|
||||
cp -r "$here" "$decos/$deco"
|
||||
rm -f "$decos/$deco/apply.sh"
|
||||
sed -i "s/\"Id\": \"[^\"]*\"/\"Id\": \"$deco\"/" "$decos/$deco/metadata.json"
|
||||
kwriteconfig6 --file "$kwinrc" --group org.kde.kdecoration2 --key library org.kde.kwin.aurorae
|
||||
kwriteconfig6 --file "$kwinrc" --group org.kde.kdecoration2 --key theme "$deco"
|
||||
fi
|
||||
DBUS_SESSION_BUS_ADDRESS=$bus gdbus call --session -d org.kde.KWin -o /KWin -m org.kde.KWin.reconfigure >/dev/null
|
||||
echo "decoration: $(kreadconfig6 --file "$kwinrc" --group org.kde.kdecoration2 --key theme --default Breeze)"
|
||||
@@ -1,123 +0,0 @@
|
||||
/*
|
||||
A title bar button drawn the way Breeze draws its own: a glyph in the title's colour,
|
||||
a soft circle behind it on hover, and a red circle for Close.
|
||||
|
||||
The float button is KWin's Keep Below button with a glyph of its own: Frametop's KWin
|
||||
script (float/frametop-float.js) floats a window when keep-below is set on it and docks
|
||||
it when it's cleared, and keeps the flag set on every floating window, so "toggled" here
|
||||
means "floating" and the glyph turns into "back to the desktop".
|
||||
|
||||
SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
import QtQuick
|
||||
import QtQuick.Shapes
|
||||
import org.kde.kwin.decoration
|
||||
|
||||
DecorationButton {
|
||||
id: button
|
||||
property real size: 20
|
||||
property color fg: "white"
|
||||
property color bg: "black"
|
||||
readonly property bool isClose: buttonType === DecorationOptions.DecorationButtonClose
|
||||
readonly property bool isFloat: buttonType === DecorationOptions.DecorationButtonKeepBelow
|
||||
// The glyph, as polylines in an 18 x 18 box (Breeze's own sizes).
|
||||
readonly property var glyph: {
|
||||
switch (buttonType) {
|
||||
case DecorationOptions.DecorationButtonClose:
|
||||
return [[[5, 5], [13, 13]], [[13, 5], [5, 13]]];
|
||||
case DecorationOptions.DecorationButtonMaximizeRestore:
|
||||
return decoration.client.maximized ? [[[4.5, 9], [9, 4.5], [13.5, 9], [9, 13.5], [4.5, 9]]]
|
||||
: [[[4, 11.5], [9, 6.5], [14, 11.5]]];
|
||||
case DecorationOptions.DecorationButtonMinimize:
|
||||
return [[[4, 7], [9, 12], [14, 7]]];
|
||||
case DecorationOptions.DecorationButtonKeepAbove:
|
||||
return [[[4, 9], [9, 4], [14, 9]], [[4, 14], [9, 9], [14, 14]]];
|
||||
case DecorationOptions.DecorationButtonShade:
|
||||
return [[[4, 5], [14, 5]], [[4, 13], [9, 8], [14, 13]]];
|
||||
case DecorationOptions.DecorationButtonApplicationMenu:
|
||||
return [[[3.5, 5], [14.5, 5]], [[3.5, 9], [14.5, 9]], [[3.5, 13], [14.5, 13]]];
|
||||
case DecorationOptions.DecorationButtonKeepBelow:
|
||||
// A box with an arrow leaving it (float in VR), or coming back into it (put it back).
|
||||
return button.toggled
|
||||
? [[[8, 3.5], [3.5, 3.5], [3.5, 14.5], [14.5, 14.5], [14.5, 10]], [[14.5, 3.5], [8.5, 9.5]],
|
||||
[[8.5, 5], [8.5, 9.5], [13, 9.5]]]
|
||||
: [[[8, 3.5], [3.5, 3.5], [3.5, 14.5], [14.5, 14.5], [14.5, 10]], [[8.5, 9.5], [14.5, 3.5]],
|
||||
[[10, 3.5], [14.5, 3.5], [14.5, 8]]];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
readonly property string tip: isFloat ? (toggled ? "Back to Desktop" : "Float in VR") : ""
|
||||
|
||||
width: size
|
||||
height: size
|
||||
|
||||
Rectangle {
|
||||
anchors.fill: parent
|
||||
radius: width / 2
|
||||
visible: button.hovered || button.pressed || (button.toggled && !button.isFloat)
|
||||
color: button.isClose ? (button.pressed ? "#c0392b" : "#da4453")
|
||||
: Qt.rgba(button.fg.r, button.fg.g, button.fg.b,
|
||||
button.pressed ? 0.35 : button.hovered ? 0.2 : 0.12)
|
||||
}
|
||||
|
||||
// On all desktops: a dot, filled while the window is on all of them. Help: a question mark.
|
||||
Rectangle {
|
||||
visible: button.buttonType === DecorationOptions.DecorationButtonOnAllDesktops
|
||||
anchors.centerIn: parent
|
||||
width: button.size * 0.3
|
||||
height: width
|
||||
radius: width / 2
|
||||
color: button.toggled ? button.fg : "transparent"
|
||||
border.width: Math.max(1, button.size / 18)
|
||||
border.color: button.fg
|
||||
}
|
||||
Text {
|
||||
visible: button.buttonType === DecorationOptions.DecorationButtonQuickHelp
|
||||
anchors.centerIn: parent
|
||||
text: "?"
|
||||
color: button.fg
|
||||
font.pixelSize: button.size * 0.65
|
||||
font.bold: true
|
||||
}
|
||||
|
||||
// Up to three strokes per glyph.
|
||||
function stroke(i) {
|
||||
const k = button.size / 18;
|
||||
return i < glyph.length ? glyph[i].map(p => Qt.point(p[0] * k, p[1] * k)) : [];
|
||||
}
|
||||
readonly property color strokeColor: isClose && (hovered || pressed) ? "white" : fg
|
||||
readonly property real strokeWidth: Math.max(1, size / 18 * 1.1)
|
||||
// (An inline component can't see the ids around it, so everything comes in as properties.)
|
||||
component Stroke: ShapePath {
|
||||
property var points: []
|
||||
fillColor: "transparent"
|
||||
capStyle: ShapePath.RoundCap
|
||||
joinStyle: ShapePath.RoundJoin
|
||||
PathPolyline { path: points }
|
||||
}
|
||||
Shape {
|
||||
anchors.fill: parent
|
||||
preferredRendererType: Shape.CurveRenderer
|
||||
Stroke { points: button.stroke(0); strokeColor: button.strokeColor; strokeWidth: button.strokeWidth }
|
||||
Stroke { points: button.stroke(1); strokeColor: button.strokeColor; strokeWidth: button.strokeWidth }
|
||||
Stroke { points: button.stroke(2); strokeColor: button.strokeColor; strokeWidth: button.strokeWidth }
|
||||
}
|
||||
|
||||
onHoveredChanged: {
|
||||
if (!tip || typeof decoration.requestShowToolTip !== "function")
|
||||
return;
|
||||
if (hovered)
|
||||
decoration.requestShowToolTip(tip);
|
||||
else
|
||||
decoration.requestHideToolTip();
|
||||
}
|
||||
Component.onCompleted: {
|
||||
if (buttonType === DecorationOptions.DecorationButtonQuickHelp)
|
||||
visible = Qt.binding(() => decoration.client.providesContextHelp);
|
||||
if (buttonType === DecorationOptions.DecorationButtonApplicationMenu)
|
||||
visible = Qt.binding(() => decoration.client.hasApplicationMenu);
|
||||
// Like Breeze: no On All Desktops button with only one virtual desktop.
|
||||
if (buttonType === DecorationOptions.DecorationButtonOnAllDesktops)
|
||||
visible = Qt.binding(() => decorationSettings.onAllDesktopsAvailable);
|
||||
}
|
||||
}
|
||||
@@ -1,270 +0,0 @@
|
||||
/*
|
||||
Frametop's window decoration: Breeze's flat title bar, drawn in QML for KWin's Aurorae
|
||||
engine, plus a button left of Close that floats the window in VR (docs/floating-windows.md,
|
||||
decision 25). Aurorae loads QML without compiling anything, so this keeps working across
|
||||
SteamOS's KWin updates, which a C++ decoration wouldn't.
|
||||
|
||||
The float button is the Keep Below button (FtButton.qml): Frametop's KWin script floats a
|
||||
window when keep-below is set and docks it when it's cleared. A Keep Below button in the
|
||||
configured button order is left out, since the float button stands in for it.
|
||||
|
||||
SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
import QtQuick
|
||||
import org.kde.kwin.decoration
|
||||
|
||||
Decoration {
|
||||
id: root
|
||||
alpha: false
|
||||
|
||||
DecorationOptions {
|
||||
id: options
|
||||
deco: decoration
|
||||
}
|
||||
TextMetrics {
|
||||
id: metrics
|
||||
font: options.titleFont
|
||||
text: "Mj"
|
||||
}
|
||||
|
||||
readonly property bool maximized: decoration.client.maximized
|
||||
readonly property int buttonSize: Math.max(16, Math.round(metrics.height * 1.25))
|
||||
readonly property int titleHeight: buttonSize + 8
|
||||
readonly property int borderSize: decorationSettings.borderSize
|
||||
readonly property int side: {
|
||||
switch (borderSize) {
|
||||
case DecorationOptions.BorderNone:
|
||||
case DecorationOptions.BorderNoSides: return 0;
|
||||
case DecorationOptions.BorderTiny: return 2;
|
||||
case DecorationOptions.BorderLarge: return 6;
|
||||
case DecorationOptions.BorderVeryLarge: return 8;
|
||||
case DecorationOptions.BorderHuge: return 12;
|
||||
case DecorationOptions.BorderVeryHuge: return 18;
|
||||
case DecorationOptions.BorderOversized: return 27;
|
||||
default: return 4;
|
||||
}
|
||||
}
|
||||
readonly property int bottomBorder: borderSize === DecorationOptions.BorderNone ? 0
|
||||
: borderSize === DecorationOptions.BorderNoSides ? 4 : side
|
||||
readonly property color outline: Qt.tint(options.titleBarColor, Qt.rgba(0, 0, 0, 0.35))
|
||||
|
||||
function applyBorders() {
|
||||
borders.left = side;
|
||||
borders.right = side;
|
||||
borders.bottom = bottomBorder;
|
||||
borders.top = titleHeight;
|
||||
maximizedBorders.top = titleHeight;
|
||||
// Without visible side borders, keep a strip to grab for resizing.
|
||||
extendedBorders.left = side ? 0 : 4;
|
||||
extendedBorders.right = side ? 0 : 4;
|
||||
extendedBorders.bottom = bottomBorder ? 0 : 4;
|
||||
}
|
||||
onTitleHeightChanged: applyBorders()
|
||||
onSideChanged: applyBorders()
|
||||
onBottomBorderChanged: applyBorders()
|
||||
Component.onCompleted: applyBorders()
|
||||
|
||||
// The configured buttons, with the float button left of Close (or first on the right
|
||||
// when there's no Close), and no Keep Below of their own.
|
||||
function order(list, right) {
|
||||
const out = [];
|
||||
let placed = false;
|
||||
for (let i = 0; i < list.length; ++i) {
|
||||
const t = list[i];
|
||||
if (t === DecorationOptions.DecorationButtonKeepBelow)
|
||||
continue;
|
||||
if (t === DecorationOptions.DecorationButtonClose && !placed) {
|
||||
if (right) {
|
||||
out.push(DecorationOptions.DecorationButtonKeepBelow, t);
|
||||
} else {
|
||||
out.push(t, DecorationOptions.DecorationButtonKeepBelow);
|
||||
}
|
||||
placed = true;
|
||||
continue;
|
||||
}
|
||||
out.push(t);
|
||||
}
|
||||
return {buttons: out, placed: placed};
|
||||
}
|
||||
readonly property var leftOrder: order(options.titleButtonsLeft || [], false)
|
||||
readonly property var rightOrder: {
|
||||
const r = order(options.titleButtonsRight || [], true);
|
||||
if (!r.placed && !leftOrder.placed)
|
||||
r.buttons.unshift(DecorationOptions.DecorationButtonKeepBelow);
|
||||
return r;
|
||||
}
|
||||
|
||||
function componentFor(t) {
|
||||
switch (t) {
|
||||
case DecorationOptions.DecorationButtonMenu: return menuButton;
|
||||
case DecorationOptions.DecorationButtonExplicitSpacer: return spacer;
|
||||
case DecorationOptions.DecorationButtonClose: return closeButton;
|
||||
case DecorationOptions.DecorationButtonMaximizeRestore: return maximizeButton;
|
||||
case DecorationOptions.DecorationButtonMinimize: return minimizeButton;
|
||||
case DecorationOptions.DecorationButtonKeepBelow: return floatButton;
|
||||
case DecorationOptions.DecorationButtonKeepAbove: return keepAboveButton;
|
||||
case DecorationOptions.DecorationButtonShade: return shadeButton;
|
||||
case DecorationOptions.DecorationButtonOnAllDesktops: return allDesktopsButton;
|
||||
case DecorationOptions.DecorationButtonQuickHelp: return helpButton;
|
||||
case DecorationOptions.DecorationButtonApplicationMenu: return appMenuButton;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
Rectangle {
|
||||
anchors.fill: parent
|
||||
color: options.titleBarColor
|
||||
border.width: root.maximized ? 0 : 1
|
||||
border.color: root.outline
|
||||
}
|
||||
|
||||
Item {
|
||||
id: titleBar
|
||||
x: root.maximized ? 0 : Math.max(root.side, 1)
|
||||
y: root.maximized ? 0 : 1
|
||||
width: root.width - 2 * x
|
||||
height: root.titleHeight - y
|
||||
|
||||
Row {
|
||||
id: leftButtons
|
||||
anchors.left: parent.left
|
||||
anchors.leftMargin: 4
|
||||
anchors.verticalCenter: parent.verticalCenter
|
||||
spacing: 4
|
||||
Repeater {
|
||||
model: root.leftOrder.buttons
|
||||
delegate: Loader {
|
||||
required property var modelData
|
||||
sourceComponent: root.componentFor(modelData)
|
||||
}
|
||||
}
|
||||
}
|
||||
Row {
|
||||
id: rightButtons
|
||||
anchors.right: parent.right
|
||||
anchors.rightMargin: 4
|
||||
anchors.verticalCenter: parent.verticalCenter
|
||||
spacing: 4
|
||||
layoutDirection: Qt.LeftToRight
|
||||
Repeater {
|
||||
model: root.rightOrder.buttons
|
||||
delegate: Loader {
|
||||
required property var modelData
|
||||
sourceComponent: root.componentFor(modelData)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Centred over the whole bar, like Breeze, but never under the buttons.
|
||||
Text {
|
||||
id: caption
|
||||
readonly property real free: rightButtons.x - (leftButtons.x + leftButtons.width) - 16
|
||||
width: Math.min(implicitWidth, free)
|
||||
x: Math.max(leftButtons.x + leftButtons.width + 8,
|
||||
Math.min((parent.width - width) / 2, rightButtons.x - 8 - width))
|
||||
anchors.verticalCenter: parent.verticalCenter
|
||||
text: decoration.client.caption
|
||||
textFormat: Text.PlainText
|
||||
font: options.titleFont
|
||||
color: options.fontColor
|
||||
elide: Text.ElideRight
|
||||
renderType: Text.NativeRendering
|
||||
}
|
||||
Component.onCompleted: decoration.installTitleItem(titleBar)
|
||||
}
|
||||
|
||||
Component {
|
||||
id: menuButton
|
||||
MenuButton {
|
||||
width: root.buttonSize
|
||||
height: root.buttonSize
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: spacer
|
||||
Item {
|
||||
width: root.buttonSize
|
||||
height: root.buttonSize
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: closeButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonClose
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: maximizeButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonMaximizeRestore
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: minimizeButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonMinimize
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: floatButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonKeepBelow
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: keepAboveButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonKeepAbove
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: shadeButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonShade
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: allDesktopsButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonOnAllDesktops
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: helpButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonQuickHelp
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
Component {
|
||||
id: appMenuButton
|
||||
FtButton {
|
||||
buttonType: DecorationOptions.DecorationButtonApplicationMenu
|
||||
size: root.buttonSize
|
||||
fg: options.fontColor
|
||||
bg: options.titleBarColor
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"KPackageStructure": "KWin/Decoration",
|
||||
"KPlugin": {
|
||||
"Authors": [
|
||||
{
|
||||
"Name": "Frametop"
|
||||
}
|
||||
],
|
||||
"Description": "Breeze-style window decoration with a button that floats the window in VR (Frametop desktop)",
|
||||
"Id": "kwin4_decoration_qml_frametop",
|
||||
"License": "GPL",
|
||||
"Name": "Frametop"
|
||||
}
|
||||
}
|
||||
+2
-18
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start, stop, or inspect the multi-screen Plasma desktop in VR on the Frame.
|
||||
# Usage: desktops.sh start [screens] | stop | restart | status | log [lines]
|
||||
# desktops.sh install # make the VR launcher's "Desktop" entry start Frametop, and add
|
||||
# # "Native Desktop" for the stock SteamOS desktop
|
||||
# desktops.sh install # make the VR launcher's "Desktop" entry start Frametop
|
||||
# desktops.sh uninstall # give the launcher back the stock SteamOS desktop
|
||||
# desktops.sh screens N # set the default screen count in ~/.config/frametop.conf
|
||||
# desktops.sh remote on|off|info # VNC access over the tailnet (applies on next start)
|
||||
@@ -17,8 +16,6 @@ action=${1:-start}
|
||||
screens=${2:-${FT_SCREENS:-}}
|
||||
session=$FRAME_REPO/session
|
||||
override=.local/share/applications/deckard-nested-desktop.desktop
|
||||
native_copy=.local/share/applications/native-deckard-nested-desktop.desktop
|
||||
stock=/usr/share/applications/deckard-nested-desktop.desktop
|
||||
log=/tmp/frametop-session.log
|
||||
# Bracketed first letter so pgrep/pkill never match the ssh shell running them.
|
||||
match='[v]r-overlay-key frametop '
|
||||
@@ -38,25 +35,12 @@ systemd-run --user --collect --quiet --unit frametop-desktop \
|
||||
sleep 12; echo \"plasmashell processes: \$(pgrep -c plasmashell)\"
|
||||
$running && echo 'started' || { echo 'failed:'; tail -20 $log; exit 1; }" ;;
|
||||
install)
|
||||
# Also "Native Desktop", a copy of SteamOS's entry for its own desktop. Optional, so a SteamOS
|
||||
# without the stock entry still installs. No X-Steam-Special, so Steam can only single out ours.
|
||||
"$root/scripts/sync.sh" >/dev/null
|
||||
"$frame" --host "set -e; mkdir -p ~/.local/share/applications
|
||||
sed 's|@SESSION@|$session/frametop-session.sh|' $session/deckard-nested-desktop.desktop > ~/$override
|
||||
if [ -r $stock ]; then
|
||||
sed -e 's/^Name=.*/Name=Native Desktop/' -e '/^Name\\[/d' -e '/^X-Steam-Special=/d' -e '/pick this out/d' \\
|
||||
$stock > ~/$native_copy.new && mv ~/$native_copy.new ~/$native_copy
|
||||
else
|
||||
rm -f ~/$native_copy; echo 'no $stock: no Native Desktop entry' >&2
|
||||
fi
|
||||
[ -f ~/.config/frametop.conf ] || cp $session/frametop.conf.example ~/.config/frametop.conf
|
||||
echo \"installed ~/$override\"; grep ^Exec= ~/$override; echo; cat ~/.config/frametop.conf" ;;
|
||||
uninstall)
|
||||
# Also what the session puts in place at each start: Launch as Standalone's app copies and
|
||||
# the title bar decoration (float/ft_apps.py, decoration/).
|
||||
"$frame" --host "rm -f ~/$override ~/$native_copy
|
||||
rm -rf ~/.local/share/frametop/apps ~/.local/share/kwin/decorations/kwin4_decoration_qml_frametop
|
||||
rmdir ~/.local/share/frametop 2>/dev/null; echo 'removed; the launcher uses the stock desktop again'" ;;
|
||||
uninstall) "$frame" --host "rm -f ~/$override && echo 'removed; the launcher uses the stock desktop again'" ;;
|
||||
screens)
|
||||
[[ ${2:-} =~ ^[1-9]$ ]] || { echo "usage: $0 screens N (1-9)" >&2; exit 2; }
|
||||
"$frame" --host "set -e; f=~/.config/frametop.conf
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
# Launch Frametop Display Settings from a Plasma session on the Frame host.
|
||||
# The app runs in Frametop's container (PySide6 and Kirigami come from Fedora there).
|
||||
# The app runs in the dev container (PySide6 and Kirigami come from Fedora there).
|
||||
# podman needs the real XDG_RUNTIME_DIR and the real user bus (to reach systemd for
|
||||
# the container's cgroup; the Frametop session runs on a private bus from
|
||||
# dbus-run-session). The session's Wayland socket and bus go to the app itself.
|
||||
@@ -10,7 +10,8 @@ case $wl in /*) ;; *) wl="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/$wl" ;; esac
|
||||
session_bus=${DBUS_SESSION_BUS_ADDRESS:-}
|
||||
export XDG_RUNTIME_DIR=/run/user/$(id -u)
|
||||
export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus
|
||||
exec "$here/../scripts/in-box" env WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
"$here/../scripts/container-up.sh"
|
||||
exec "$HOME/.local/bin/distrobox" enter dev -- env WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
XAUTHORITY="${XAUTHORITY:-}" DBUS_SESSION_BUS_ADDRESS="$session_bus" \
|
||||
QT_QPA_PLATFORM="wayland;xcb" \
|
||||
python3 "$here/ft_display_settings.py" "$@"
|
||||
@@ -3,7 +3,7 @@ Type=Application
|
||||
Name=Reset Screen Layout
|
||||
GenericName=Put the VR desktop's screens back in their layout
|
||||
Comment=Float the screens and arrange them in the layout from Frametop Display Settings
|
||||
Exec=@REPO@/layout/ft-layout-reset
|
||||
Exec=@REPO@/layout/ft-layout apply
|
||||
Icon=view-restore
|
||||
Categories=Settings;
|
||||
Keywords=display;screen;layout;arrange;reset;steamvr;frametop;
|
||||
|
||||
@@ -3,7 +3,7 @@ Type=Application
|
||||
Name=Hide/Show Screens
|
||||
GenericName=Hide or show the VR desktop's screens
|
||||
Comment=Hide the screens (and SteamVR's laser) for a VR game; press again to bring them back
|
||||
Exec=@REPO@/layout/ft-hide-show
|
||||
Exec=@REPO@/layout/ft-layout toggle
|
||||
Icon=view-visible
|
||||
Categories=Settings;
|
||||
Keywords=display;screen;hide;show;steamvr;frametop;
|
||||
|
||||
@@ -10,15 +10,12 @@ the dev container:
|
||||
1920x1080 worth of pixels, rotation for portrait.)
|
||||
- Visibility (ft-screens): when the screens show (always, only with the SteamVR
|
||||
dashboard open, while you look at a controller, or only when toggled), the wrist
|
||||
angle within which a pinned screen shows, pinning each screen to a wrist or your
|
||||
head, and whether the desktop's notifications go to the Steam session
|
||||
(NOTIFY_FORWARD, session/ft-notifyfwd; its state from its log).
|
||||
angle within which a pinned screen shows, and pinning each screen to a wrist or
|
||||
your head.
|
||||
- Layout: a preset (curved or flat, rows, distance, gap, height) or a named layout
|
||||
saved from where the screens are, with a preview; arrange now; save the current
|
||||
arrangement under a name; rename and delete; arrange automatically when the
|
||||
desktop starts.
|
||||
- Remote displays (other computers' monitors as screens) have their own app, Frametop
|
||||
Remote Displays (remote-displays/); a button opens it.
|
||||
- Power: how long the headset can go unused before ft-powerd turns its displays off
|
||||
(DISPLAY_OFF_MIN; the service's state comes from its control socket, @ft_powerd),
|
||||
and whether the Frame stays awake while plugged in, which is Steam's own setting
|
||||
@@ -57,14 +54,11 @@ SCREEN_RESOLUTIONS = [(1920, 1080, ""), (2560, 1440, ""), (3840, 2160, "4K"), (2
|
||||
(2560, 1600, "16:10"), (1080, 1920, "portrait"), (1440, 2560, "portrait"),
|
||||
(2160, 3840, "portrait 4K")]
|
||||
FT_SCREENS = "\0ft_screens"
|
||||
REMOTE_DISPLAYS = os.path.join(HERE, "..", "remote-displays", "ft_remote_displays.py")
|
||||
FT_POWERD = "\0ft_powerd"
|
||||
# Steam's default for "When Plugged In and Idle -> Sleep after", to go back to when
|
||||
# nothing was saved.
|
||||
STEAM_SLEEP_AC_DEFAULT = 3600
|
||||
SCALES = [0.75, 1.0, 1.25, 4 / 3, 1.5, 1.75, 2.0]
|
||||
NOTIFY_FORWARD = ("auto", "on", "off") # session/ft-notifyfwd; frametop.conf's NOTIFY_FORWARD
|
||||
NOTIFYFWD_LOG = "/tmp/frametop-notifyfwd.log" # its last line is its state (/tmp is the host's)
|
||||
ROTATIONS = [("normal", "Landscape"), ("left", "Portrait"), ("right", "Portrait (flipped)")]
|
||||
|
||||
|
||||
@@ -124,7 +118,6 @@ class Backend(QObject):
|
||||
self._steam_busy = False
|
||||
self._steamDone.connect(self._steam_done, Qt.QueuedConnection)
|
||||
self._pins = [] # each running screen's pin: none | left | right | head
|
||||
self._notify_state = ""
|
||||
self.poll = QTimer(interval=3000, timeout=self._check_running)
|
||||
self.poll.start()
|
||||
self._check_running()
|
||||
@@ -137,10 +130,8 @@ class Backend(QObject):
|
||||
return cast(conf.get(key, default))
|
||||
except ValueError:
|
||||
return default
|
||||
notify = conf.get("NOTIFY_FORWARD", "auto")
|
||||
return {"screens": num("SCREENS", 2, int), "width": num("WIDTH", 1920, int),
|
||||
"height": num("HEIGHT", 1080, int), "physWidth": num("PHYS_WIDTH", 1.6),
|
||||
"notifyForward": notify if notify in NOTIFY_FORWARD else "off"}
|
||||
"height": num("HEIGHT", 1080, int), "physWidth": num("PHYS_WIDTH", 1.6)}
|
||||
|
||||
def _check_running(self):
|
||||
# The session's own Wayland socket (host processes' environments aren't readable
|
||||
@@ -148,15 +139,12 @@ class Backend(QObject):
|
||||
running = os.path.exists(f"/run/user/{os.getuid()}/frametop/wayland-0")
|
||||
count = self._screens_running() if running and ft_layout.backend() == "screens" else 0
|
||||
pins = self._read_pins(count)
|
||||
notify_state = self._read_notify_state() if running else ""
|
||||
if (running != self._running or count != self._running_count or pins != self._pins
|
||||
or notify_state != self._notify_state):
|
||||
if running != self._running or count != self._running_count or pins != self._pins:
|
||||
if running != self._running or count != self._running_count:
|
||||
self._started = self._conf() if running else {}
|
||||
self._running = running
|
||||
self._running_count = count
|
||||
self._pins = pins
|
||||
self._notify_state = notify_state
|
||||
self.changed.emit()
|
||||
self._check_powerd()
|
||||
|
||||
@@ -168,22 +156,6 @@ class Backend(QObject):
|
||||
pins.append(f[16] if len(f) > 16 else "none")
|
||||
return pins
|
||||
|
||||
@staticmethod
|
||||
def _read_notify_state():
|
||||
"""ft-notifyfwd's: forwarding | waiting (no usable server on the user bus) | "" (not running)."""
|
||||
try:
|
||||
with open(NOTIFYFWD_LOG) as f:
|
||||
lines = f.read().splitlines()
|
||||
except OSError:
|
||||
return ""
|
||||
# the last state line: errors it logs in between (a failed Notify) don't change it
|
||||
for line in reversed(lines):
|
||||
if line == "ft-notifyfwd: forwarding":
|
||||
return "forwarding"
|
||||
if line.startswith("ft-notifyfwd: not forwarding"):
|
||||
return "waiting"
|
||||
return ""
|
||||
|
||||
def _ask_screens(self, text):
|
||||
"""Request/reply to ft-screens; None if it isn't running."""
|
||||
try:
|
||||
@@ -408,26 +380,6 @@ class Backend(QObject):
|
||||
def toggleScreens(self):
|
||||
self._ask_screens("toggle")
|
||||
|
||||
# --- the desktop's notifications in the Steam session (session/ft-notifyfwd) ---
|
||||
@Property(str, notify=changed)
|
||||
def notifyForward(self):
|
||||
return self._conf()["notifyForward"]
|
||||
|
||||
@Property(str, notify=changed)
|
||||
def notifyForwardState(self):
|
||||
return self._notify_state
|
||||
|
||||
@Property(bool, notify=changed)
|
||||
def notifyRestartNeeded(self):
|
||||
"""The session writes ft-notifyfwd's autostart entry when it starts."""
|
||||
return self._running and bool(self._started) and self._started["notifyForward"] != self._conf()["notifyForward"]
|
||||
|
||||
@Slot(str)
|
||||
def setNotifyForward(self, mode):
|
||||
if mode in NOTIFY_FORWARD:
|
||||
write_conf_value("NOTIFY_FORWARD", mode)
|
||||
self.changed.emit()
|
||||
|
||||
# --- ft-screens: visibility and pinning ---
|
||||
@Property("QVariantMap", notify=changed)
|
||||
def visibility(self):
|
||||
@@ -457,31 +409,6 @@ class Backend(QObject):
|
||||
def pins(self):
|
||||
return self._pins
|
||||
|
||||
@Property("QVariantList", notify=changed)
|
||||
def screensShown(self):
|
||||
"""For each screen, whether it shows (False: hidden on its own, ft-layout hide N)."""
|
||||
layout = ft_layout.load_layout()
|
||||
return [not ft_layout.screen_entry(layout, i).get("hidden") for i in range(ft_layout.screen_count(layout))]
|
||||
|
||||
@Slot(int, bool)
|
||||
def setScreenShown(self, index, shown):
|
||||
"""Hide screen `index` (0-based) on its own, whatever the visibility mode, or show it."""
|
||||
layout = ft_layout.load_layout()
|
||||
screens = layout.setdefault("screens", [])
|
||||
while len(screens) <= index:
|
||||
screens.append({})
|
||||
if shown:
|
||||
screens[index].pop("hidden", None)
|
||||
else:
|
||||
screens[index]["hidden"] = True
|
||||
ft_layout.save_layout(layout)
|
||||
self.changed.emit()
|
||||
if self._running:
|
||||
reply = self._ask_screens(f"{'reveal' if shown else 'conceal'} {index + 1}")
|
||||
if not (reply and reply.startswith("ok")):
|
||||
self.message.emit("Saved; the desktop applies it when it next starts "
|
||||
"(its compositor is older than hiding screens one at a time)", False)
|
||||
|
||||
@Slot(str, str)
|
||||
def pin(self, which, where):
|
||||
"""Pin screen `which` (1-based, or "all") to "left", "right", or "head" as it is
|
||||
@@ -493,7 +420,7 @@ class Backend(QObject):
|
||||
f"{reply or 'the desktop is not running'}", True)
|
||||
elif which == "all" and where != "none":
|
||||
place = "on your head" if where == "head" else f"on your {where} wrist"
|
||||
self.message.emit(f"All screens ride {place} now. Save as profile… (Layout & profiles) keeps it.", False)
|
||||
self.message.emit(f"All screens ride {place} now. Save current arrangement (Layout) keeps it.", False)
|
||||
self._check_running()
|
||||
|
||||
# --- power: ft-powerd and Steam's sleep setting ---
|
||||
@@ -632,55 +559,16 @@ class Backend(QObject):
|
||||
def renameLayout(self, old, new):
|
||||
try:
|
||||
self._edit_layout(lambda l: ft_layout.rename_named(l, old, new))
|
||||
ft_layout.write_launchers(ft_layout.load_layout())
|
||||
except (RuntimeError, OSError) as e:
|
||||
except RuntimeError as e:
|
||||
self.message.emit(str(e), True)
|
||||
|
||||
@Slot(str)
|
||||
def deleteLayout(self, name):
|
||||
try:
|
||||
self._edit_layout(lambda l: ft_layout.delete_named(l, name))
|
||||
ft_layout.write_launchers(ft_layout.load_layout())
|
||||
except (RuntimeError, OSError) as e:
|
||||
except RuntimeError as e:
|
||||
self.message.emit(str(e), True)
|
||||
|
||||
# --- profiles (docs/profiles.md): a named layout's apps and hidden screens ---
|
||||
@Slot(str, result="QVariantList")
|
||||
def profileWindows(self, name):
|
||||
"""A profile's windows, as "app" and "where" for the list."""
|
||||
out = []
|
||||
for e in ft_layout.load_layout().get("profiles", {}).get(name, {}).get("windows", []):
|
||||
app = e.get("app") or os.path.basename((e.get("cmd") or ["?"])[0])
|
||||
app = app.rsplit(".", 1)[-1] if "." in app and not e.get("cmd") else app
|
||||
where = "floating" if "float" in e else f"screen {e.get('screen', 1)}" + (", maximized" if e.get("maximized") else "")
|
||||
out.append({"app": app, "where": where})
|
||||
return out
|
||||
|
||||
@Slot(str, result="QVariantList")
|
||||
def profileHidden(self, name):
|
||||
return ft_layout.load_layout().get("profiles", {}).get(name, {}).get("hidden", [])
|
||||
|
||||
@Slot(str, int)
|
||||
def removeProfileWindow(self, name, index):
|
||||
def edit(layout):
|
||||
windows = layout.get("profiles", {}).get(name, {}).get("windows", [])
|
||||
if 0 <= index < len(windows):
|
||||
windows.pop(index)
|
||||
self._edit_layout(edit)
|
||||
|
||||
@Property(str, notify=changed)
|
||||
def defaultProfile(self):
|
||||
return ft_layout.load_layout().get("default_profile", "")
|
||||
|
||||
@Slot(str)
|
||||
def setDefaultProfile(self, name):
|
||||
def edit(layout):
|
||||
if name:
|
||||
layout["default_profile"] = name
|
||||
else:
|
||||
layout.pop("default_profile", None)
|
||||
self._edit_layout(edit)
|
||||
|
||||
@Slot(str, "QVariant")
|
||||
def setPreset(self, key, value):
|
||||
def edit(layout):
|
||||
@@ -694,25 +582,12 @@ class Backend(QObject):
|
||||
|
||||
@Slot()
|
||||
def arrange(self):
|
||||
"""Arrange the screens; in a profile, also open its apps (ft-layout use)."""
|
||||
layout = ft_layout.load_layout()
|
||||
name = layout.get("active")
|
||||
if layout.get("mode") == "custom" and name in layout.get("layouts", {}):
|
||||
self._run(f"Opening {name}", "use", name)
|
||||
else:
|
||||
self._run("Arranging the screens", "apply")
|
||||
self._run("Arranging the screens", "apply")
|
||||
|
||||
@Slot()
|
||||
def capture(self):
|
||||
self._run("Saving the current arrangement", "capture")
|
||||
|
||||
# --- remote displays: their own app ---
|
||||
@Slot()
|
||||
def openRemoteDisplays(self):
|
||||
"""Frametop Remote Displays (we're in the dev container already, as it runs)."""
|
||||
if not QProcess.startDetached(sys.executable, [os.path.abspath(REMOTE_DISPLAYS)]):
|
||||
self.message.emit("Couldn't start Frametop Remote Displays", True)
|
||||
|
||||
# --- ft-layout on the host ---
|
||||
def _run(self, label, *args):
|
||||
if self._proc is not None:
|
||||
|
||||
@@ -17,12 +17,10 @@ case ${1:-install} in
|
||||
on_frame "chmod +x display-settings/ft-display-settings layout/ft-layout layout/ft_layout.py
|
||||
mkdir -p ~/.config/frametop
|
||||
kwriteconfig6 --file ~/$shortcuts --group services --group ft-layout-reset.desktop --key _launch 'Meta+Shift+R'
|
||||
kwriteconfig6 --file ~/$shortcuts --group services --group ft-screens-toggle.desktop --key _launch 'Meta+Shift+H'
|
||||
layout/ft-layout launchers # each profile's entry (Frametop: NAME), if there are profiles"
|
||||
kwriteconfig6 --file ~/$shortcuts --group services --group ft-screens-toggle.desktop --key _launch 'Meta+Shift+H'"
|
||||
echo "installed: Frametop Display Settings, Reset Screen Layout (Meta+Shift+R), Hide/Show Screens (Meta+Shift+H)" ;;
|
||||
uninstall)
|
||||
on_frame "rm -f ~/$apps/ft-display-settings.desktop ~/$apps/ft-layout-reset.desktop ~/$apps/ft-screens-toggle.desktop
|
||||
rm -f ~/$apps/frametop-profile-*.desktop # the profiles' entries (the profiles stay in ~/.config/frametop-layout.json)
|
||||
[ -f ~/$shortcuts ] && for f in ft-layout-reset ft-screens-toggle; do kwriteconfig6 --file ~/$shortcuts --group services --group \$f.desktop --key _launch --delete; done
|
||||
echo removed" ;;
|
||||
*) echo "usage: $0 [install|uninstall]" >&2; exit 2 ;;
|
||||
|
||||
+13
-152
@@ -180,13 +180,6 @@ Kirigami.ApplicationWindow {
|
||||
icon.name: "view-visible"
|
||||
onTriggered: backend.toggleScreens()
|
||||
},
|
||||
Kirigami.Action {
|
||||
visible: spage.md
|
||||
text: "Remote displays"
|
||||
icon.name: "network-workgroup"
|
||||
tooltip: "Other computers' monitors as screens: Frametop Remote Displays"
|
||||
onTriggered: backend.openRemoteDisplays()
|
||||
},
|
||||
Kirigami.Action {
|
||||
visible: backend.desktopRunning
|
||||
text: "Restart desktop"
|
||||
@@ -418,8 +411,8 @@ Kirigami.ApplicationWindow {
|
||||
text: spage.md
|
||||
? "Each screen is a real monitor of its own: any resolution, portrait by choosing a tall one. Resolution, "
|
||||
+ "width, and curve apply at once. In VR: move a screen by the bar underneath, curve it with the round "
|
||||
+ "button next to the bar, resize it by the tab on its bottom right corner; Save as profile… on the "
|
||||
+ "Layout & profiles page keeps all of it."
|
||||
+ "button next to the bar, resize it by the tab on its bottom right corner; Save current arrangement on the "
|
||||
+ "Layout page keeps all of it."
|
||||
: "gamescope draws every screen at the same resolution, at most 1920 × 1080 worth of pixels. Portrait turns "
|
||||
+ "a screen on its side. Rotation and scale apply at once; the rest when the desktop starts."
|
||||
}
|
||||
@@ -431,7 +424,7 @@ Kirigami.ApplicationWindow {
|
||||
id: layoutPage
|
||||
Kirigami.ScrollablePage {
|
||||
id: lpage
|
||||
title: "Layout & profiles"
|
||||
title: "Layout"
|
||||
property var layout: backend.layout
|
||||
property var preset: layout.preset || {}
|
||||
property bool hasCustom: (layout.screens || []).some(s => s.pos !== undefined)
|
||||
@@ -447,17 +440,16 @@ Kirigami.ApplicationWindow {
|
||||
|
||||
actions: [
|
||||
Kirigami.Action {
|
||||
text: lpage.named ? "Open profile" : "Arrange now"
|
||||
text: "Arrange now"
|
||||
icon.name: "view-restore"
|
||||
tooltip: lpage.named ? "Put the screens in this profile's places, around where you're facing, and open its apps (windows already open move; nothing closes)"
|
||||
: "Float the screens out of the dashboard and put them in this layout, around where you're facing"
|
||||
tooltip: "Float the screens out of the dashboard and put them in this layout, around where you're facing"
|
||||
enabled: backend.desktopRunning && backend.busy === ""
|
||||
onTriggered: backend.arrange()
|
||||
},
|
||||
Kirigami.Action {
|
||||
text: "Save as profile…"
|
||||
text: "Save current arrangement…"
|
||||
icon.name: "document-save"
|
||||
tooltip: "Save where the screens are now, which ones are hidden, and the open apps and where their windows are, under a name"
|
||||
tooltip: "Save where the screens are now (placed by hand) as a named layout, and use it"
|
||||
enabled: backend.desktopRunning && backend.busy === ""
|
||||
onTriggered: nameDialog.openFor("save", lpage.named ? lpage.layout.active
|
||||
: "Layout " + (lpage.names.length + 1))
|
||||
@@ -515,61 +507,16 @@ Kirigami.ApplicationWindow {
|
||||
Controls.Label {
|
||||
visible: lpage.layout.mode === "custom"
|
||||
Kirigami.FormData.label: ""
|
||||
text: lpage.named ? "Where the screens were when you saved it, and the apps that were open. Open "
|
||||
+ "profile puts the screens there and opens the apps. Save as profile updates "
|
||||
+ "it or saves a new one."
|
||||
: lpage.hasCustom ? "Where the screens were when you saved. Save as profile "
|
||||
text: lpage.named ? "Where the screens were when you saved it. Arrange now puts them there. "
|
||||
+ "Save current arrangement updates it or saves a new one."
|
||||
: lpage.hasCustom ? "Where the screens were when you saved. Save current arrangement "
|
||||
+ "names it. Pick a preset to edit."
|
||||
: "Nothing saved yet: place the screens by hand, open your apps, then Save as profile."
|
||||
: "Nothing saved yet: place the screens by hand, then Save current arrangement."
|
||||
opacity: 0.7
|
||||
wrapMode: Text.Wrap
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 20
|
||||
}
|
||||
|
||||
// The profile's apps (docs/profiles.md): each window and where it goes.
|
||||
ColumnLayout {
|
||||
id: profileApps
|
||||
visible: lpage.named
|
||||
Kirigami.FormData.label: "Apps:"
|
||||
property var windows: lpage.named ? backend.profileWindows(lpage.layout.active) : []
|
||||
property var hidden: lpage.named ? backend.profileHidden(lpage.layout.active) : []
|
||||
Connections {
|
||||
target: backend
|
||||
function onChanged() {
|
||||
profileApps.windows = lpage.named ? backend.profileWindows(lpage.layout.active) : []
|
||||
profileApps.hidden = lpage.named ? backend.profileHidden(lpage.layout.active) : []
|
||||
}
|
||||
}
|
||||
Controls.Label {
|
||||
visible: profileApps.windows.length === 0
|
||||
text: "None saved. Open the apps you want, place their windows, then Save as profile."
|
||||
opacity: 0.7
|
||||
wrapMode: Text.Wrap
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 20
|
||||
}
|
||||
Repeater {
|
||||
model: profileApps.windows
|
||||
delegate: RowLayout {
|
||||
required property var modelData
|
||||
required property int index
|
||||
Controls.Label { text: modelData.app + " (" + modelData.where + ")" }
|
||||
Controls.ToolButton {
|
||||
icon.name: "list-remove"
|
||||
text: "Leave out"
|
||||
display: Controls.AbstractButton.IconOnly
|
||||
Controls.ToolTip.text: "Leave this window out of the profile"
|
||||
Controls.ToolTip.visible: hovered
|
||||
onClicked: backend.removeProfileWindow(lpage.layout.active, index)
|
||||
}
|
||||
}
|
||||
}
|
||||
Controls.Label {
|
||||
visible: profileApps.hidden.length > 0
|
||||
text: "Hides screen" + (profileApps.hidden.length > 1 ? "s " : " ") + profileApps.hidden.join(", ")
|
||||
opacity: 0.7
|
||||
}
|
||||
}
|
||||
|
||||
Controls.SpinBox {
|
||||
Kirigami.FormData.label: "Rows:"
|
||||
visible: lpage.layout.mode !== "custom"
|
||||
@@ -607,19 +554,8 @@ Kirigami.ApplicationWindow {
|
||||
Kirigami.FormData.label: "When the desktop starts:"
|
||||
text: "Float the screens and arrange them"
|
||||
checked: lpage.layout.auto !== false
|
||||
enabled: backend.defaultProfile === ""
|
||||
onToggled: backend.setAuto(checked)
|
||||
}
|
||||
Controls.ComboBox {
|
||||
Kirigami.FormData.label: "Start in profile:"
|
||||
model: [{ text: "None", value: "" }].concat(lpage.names.map(n => ({ text: n, value: n })))
|
||||
textRole: "text"
|
||||
valueRole: "value"
|
||||
currentIndex: Math.max(0, indexOfValue(backend.defaultProfile))
|
||||
onActivated: backend.setDefaultProfile(currentValue)
|
||||
Controls.ToolTip.text: "The desktop starts in this profile: its screens, and its apps open. Each profile also has its own entry in SteamVR's Launch a program list"
|
||||
Controls.ToolTip.visible: hovered
|
||||
}
|
||||
}
|
||||
|
||||
// Preview: from above (you at the bottom) and from the front.
|
||||
@@ -809,7 +745,7 @@ Kirigami.ApplicationWindow {
|
||||
Repeater {
|
||||
model: [
|
||||
{ value: "hide", text: "Hide them unless the SteamVR dashboard is open", help: "The game has the view to itself; open the dashboard (or press Meta+Shift+H) to see the screens." },
|
||||
{ value: "visible", text: "Keep them visible over the game", help: "They float over the game as they are outside it. Turn off Pause while a VR game runs in Frametop Input Settings (Game optimization), or Frametop pauses and hides them anyway." }
|
||||
{ value: "visible", text: "Keep them visible over the game", help: "They float over the game as they are outside it." }
|
||||
]
|
||||
delegate: ColumnLayout {
|
||||
required property var modelData
|
||||
@@ -866,27 +802,6 @@ Kirigami.ApplicationWindow {
|
||||
}
|
||||
}
|
||||
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Screens shown" }
|
||||
|
||||
Repeater {
|
||||
model: backend.screensShown
|
||||
delegate: Controls.Switch {
|
||||
required property var modelData
|
||||
required property int index
|
||||
Kirigami.FormData.label: "Screen " + (index + 1) + ":"
|
||||
text: modelData ? "Shown" : "Hidden"
|
||||
checked: modelData
|
||||
onToggled: backend.setScreenShown(index, checked)
|
||||
}
|
||||
}
|
||||
Controls.Label {
|
||||
text: "A hidden screen stays hidden whatever the choices above say, and Meta+Shift+H doesn't bring it back. Windows on it stay there; new ones that would open on it float instead."
|
||||
opacity: 0.7
|
||||
font: Kirigami.Theme.smallFont
|
||||
wrapMode: Text.Wrap
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
}
|
||||
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Pinned screens" }
|
||||
|
||||
Repeater {
|
||||
@@ -956,61 +871,7 @@ Kirigami.ApplicationWindow {
|
||||
+ "it.\n\nPin a screen to your head: choose On your head above. It rides on the headset where it "
|
||||
+ "is now, like a HUD, and shows whenever the screens do. Grab its bar to move it; it stays on "
|
||||
+ "your head where you let go. Choosing a pin above keeps the screen where it is now, so place "
|
||||
+ "it first. Save as profile… (Layout) keeps pins."
|
||||
}
|
||||
|
||||
Kirigami.FormLayout {
|
||||
Layout.fillWidth: true
|
||||
|
||||
Kirigami.Separator { Kirigami.FormData.isSection: true; Kirigami.FormData.label: "Notifications" }
|
||||
|
||||
Repeater {
|
||||
model: [
|
||||
{ value: "auto", text: "Show them in the Steam session", help: "The desktop's notifications go to the Steam session's notification server, such as one that shows them in VR, with their buttons. Plasma's own popups stay off meanwhile; its history keeps them. Not to SteamOS's own server, which can't show them on the Frame." },
|
||||
{ value: "on", text: "Always send them to the Steam session", help: "Whatever serves notifications there, SteamOS's own server too, which can't show them on the Frame: they're lost then." },
|
||||
{ value: "off", text: "Keep them in the desktop", help: "Plasma shows them on the desktop's screens." }
|
||||
]
|
||||
delegate: ColumnLayout {
|
||||
required property var modelData
|
||||
spacing: 0
|
||||
// "on" can lose them (SteamOS's server): set in frametop.conf, shown only then
|
||||
visible: modelData.value !== "on" || backend.notifyForward === "on"
|
||||
Controls.RadioButton {
|
||||
text: modelData.text
|
||||
checked: backend.notifyForward === modelData.value
|
||||
onToggled: if (checked) backend.setNotifyForward(modelData.value)
|
||||
}
|
||||
Controls.Label {
|
||||
text: modelData.help
|
||||
opacity: 0.7
|
||||
font: Kirigami.Theme.smallFont
|
||||
leftPadding: Kirigami.Units.gridUnit * 1.6
|
||||
wrapMode: Text.Wrap
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
}
|
||||
}
|
||||
}
|
||||
Controls.Label {
|
||||
Kirigami.FormData.label: "Now:"
|
||||
visible: backend.desktopRunning
|
||||
text: backend.notifyForwardState === "forwarding" ? "In the Steam session"
|
||||
: backend.notifyForwardState === "waiting" ? "In the desktop: the Steam session has no notification server that can show them"
|
||||
: "In the desktop"
|
||||
wrapMode: Text.Wrap
|
||||
Layout.maximumWidth: Kirigami.Units.gridUnit * 26
|
||||
}
|
||||
RowLayout {
|
||||
visible: backend.notifyRestartNeeded
|
||||
Controls.Label {
|
||||
text: "Applies when the desktop starts again."
|
||||
wrapMode: Text.Wrap
|
||||
}
|
||||
Controls.Button {
|
||||
text: "Restart desktop"
|
||||
icon.name: "system-reboot"
|
||||
onClicked: restartDialog.open()
|
||||
}
|
||||
}
|
||||
+ "it first. Save current arrangement (Layout) keeps pins."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,26 +4,117 @@ Steam, not systemd, puts the Frame to sleep: after "When Plugged In and Idle ->
|
||||
(an hour by default) without input, even while it charges. That's a Steam client setting,
|
||||
`system_idle_suspend_ac_sec` (0 = never), with no file or command line to change it. Steam
|
||||
on the Frame runs with -cef-enable-debugging, so its UI's JavaScript context
|
||||
(SharedJSContext) is reachable over the Chrome DevTools Protocol on 127.0.0.1:8080
|
||||
(steam/steamui.py). There `settingsStore.clientSettings` has the current values, and
|
||||
`SteamClient.Settings.SetSetting` takes a change as a serialized CMsgClientSettings protobuf,
|
||||
which is what Steam's own Settings -> Power page sends.
|
||||
(SharedJSContext) is reachable over the Chrome DevTools Protocol on 127.0.0.1:8080. There
|
||||
`settingsStore.clientSettings` has the current values, and `SteamClient.Settings.SetSetting`
|
||||
takes a change as a serialized CMsgClientSettings protobuf, which is what Steam's own
|
||||
Settings -> Power page sends. Standard library only (a minimal WebSocket client).
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "steam"))
|
||||
from steamui import SteamUnreachable, evaluate # noqa: E402,F401 (callers catch SteamUnreachable here)
|
||||
import socket
|
||||
import struct
|
||||
import urllib.request
|
||||
|
||||
CDP_PORT = 8080
|
||||
# CMsgClientSettings field numbers (Steam's UI bundle maps the names to these).
|
||||
FIELDS = {"system_idle_suspend_ac_sec": 24004, "system_idle_suspend_battery_sec": 24003}
|
||||
|
||||
|
||||
class SteamUnreachable(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class _WebSocket:
|
||||
def __init__(self, url, timeout=5):
|
||||
host_port, path = url[len("ws://"):].split("/", 1)
|
||||
host, port = host_port.rsplit(":", 1)
|
||||
self.sock = socket.create_connection((host, int(port)), timeout=timeout)
|
||||
key = base64.b64encode(os.urandom(16)).decode()
|
||||
self.sock.sendall((f"GET /{path} HTTP/1.1\r\nHost: {host_port}\r\nUpgrade: websocket\r\n"
|
||||
f"Connection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n").encode())
|
||||
head = b""
|
||||
while b"\r\n\r\n" not in head:
|
||||
chunk = self.sock.recv(4096)
|
||||
if not chunk:
|
||||
raise SteamUnreachable("Steam closed the connection")
|
||||
head += chunk
|
||||
if b" 101 " not in head.split(b"\r\n", 1)[0]:
|
||||
raise SteamUnreachable(head.split(b"\r\n", 1)[0].decode(errors="replace"))
|
||||
self.buf = head.split(b"\r\n\r\n", 1)[1]
|
||||
|
||||
def send(self, text):
|
||||
data, mask = text.encode(), os.urandom(4)
|
||||
n = len(data)
|
||||
if n < 126:
|
||||
head = struct.pack(">BB", 0x81, 0x80 | n)
|
||||
elif n < 65536:
|
||||
head = struct.pack(">BBH", 0x81, 0x80 | 126, n)
|
||||
else:
|
||||
head = struct.pack(">BBQ", 0x81, 0x80 | 127, n)
|
||||
self.sock.sendall(head + mask + bytes(b ^ mask[i % 4] for i, b in enumerate(data)))
|
||||
|
||||
def _take(self, n):
|
||||
while len(self.buf) < n:
|
||||
chunk = self.sock.recv(65536)
|
||||
if not chunk:
|
||||
raise SteamUnreachable("Steam closed the connection")
|
||||
self.buf += chunk
|
||||
out, self.buf = self.buf[:n], self.buf[n:]
|
||||
return out
|
||||
|
||||
def recv(self):
|
||||
message = b""
|
||||
while True:
|
||||
b0, b1 = self._take(2)
|
||||
n = b1 & 0x7F
|
||||
if n == 126:
|
||||
n = struct.unpack(">H", self._take(2))[0]
|
||||
elif n == 127:
|
||||
n = struct.unpack(">Q", self._take(8))[0]
|
||||
message += self._take(n)
|
||||
if b0 & 0x80:
|
||||
return message.decode(errors="replace")
|
||||
|
||||
def close(self):
|
||||
self.sock.close()
|
||||
|
||||
|
||||
def _evaluate(expression):
|
||||
"""Runs JavaScript in Steam's SharedJSContext and returns its (awaited) value."""
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{CDP_PORT}/json", timeout=3) as r:
|
||||
targets = json.load(r)
|
||||
except OSError as e:
|
||||
raise SteamUnreachable(f"Steam isn't reachable on port {CDP_PORT} ({e})") from e
|
||||
url = next((t["webSocketDebuggerUrl"] for t in targets if t.get("title") == "SharedJSContext"), None)
|
||||
if not url:
|
||||
raise SteamUnreachable("Steam's UI isn't running")
|
||||
try:
|
||||
ws = _WebSocket(url)
|
||||
try:
|
||||
ws.send(json.dumps({"id": 1, "method": "Runtime.evaluate",
|
||||
"params": {"expression": expression, "awaitPromise": True, "returnByValue": True}}))
|
||||
while True:
|
||||
reply = json.loads(ws.recv())
|
||||
if reply.get("id") == 1:
|
||||
break
|
||||
finally:
|
||||
ws.close()
|
||||
except OSError as e:
|
||||
raise SteamUnreachable(str(e)) from e
|
||||
result = reply.get("result", {})
|
||||
if "exceptionDetails" in result:
|
||||
details = result["exceptionDetails"]
|
||||
raise SteamUnreachable(details.get("exception", {}).get("description") or details.get("text", "error"))
|
||||
return result.get("result", {}).get("value")
|
||||
|
||||
|
||||
def sleep_settings():
|
||||
"""{"ac": seconds, "battery": seconds}: when Steam puts the Frame to sleep without input,
|
||||
plugged in and on battery (0 = never)."""
|
||||
value = evaluate("(() => { const c = settingsStore.clientSettings; "
|
||||
"return {ac: c.system_idle_suspend_ac_sec, battery: c.system_idle_suspend_battery_sec}; })()")
|
||||
value = _evaluate("(() => { const c = settingsStore.clientSettings; "
|
||||
"return {ac: c.system_idle_suspend_ac_sec, battery: c.system_idle_suspend_battery_sec}; })()")
|
||||
if not isinstance(value, dict) or not all(isinstance(value.get(k), int) for k in ("ac", "battery")):
|
||||
raise SteamUnreachable("Steam's settings don't have the sleep timeouts")
|
||||
return value
|
||||
@@ -34,7 +125,7 @@ def set_sleep_setting(name, seconds):
|
||||
field, seconds = FIELDS[name], int(seconds)
|
||||
if seconds < 0:
|
||||
raise ValueError("seconds must be 0 (never) or more")
|
||||
ok = evaluate(f"""(async () => {{
|
||||
ok = _evaluate(f"""(async () => {{
|
||||
const bytes = [];
|
||||
const varint = n => {{ while (n > 127) {{ bytes.push((n & 127) | 128); n = Math.floor(n / 128); }} bytes.push(n); }};
|
||||
varint({field} * 8); varint({seconds});
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
# Controller desktop click stability
|
||||
|
||||
Trigger presses reach KDE immediately, but controller motion within 32 logical
|
||||
pixels of the press stays at that position until release. Releasing without a motion outside this
|
||||
zone delivers the click at the original position, even if the hand moved during
|
||||
release. Moving outside the zone begins a normal drag immediately; returning to
|
||||
the zone does not turn it back into a click. There is no hold-duration timer.
|
||||
|
||||
This filters overlay pointer content events on desktop monitors only, and only
|
||||
presses from hand controllers start it. The 3D mouse (whose laser comes from the
|
||||
`ft_pointer` virtual controller), SteamVR UI, separate screen grab bars and
|
||||
floating-app title-bar carrying are unaffected. Multi-button gestures keep their
|
||||
existing behavior. A motion onto another desktop monitor starts a drag;
|
||||
cross-monitor motion is not stabilized.
|
||||
|
||||
CLI (runtime preferences, reset to 32 on desktop restart):
|
||||
|
||||
```sh
|
||||
input/ft-clickctl status
|
||||
input/ft-clickctl threshold 32
|
||||
input/ft-clickctl threshold 0 # disable without a restart
|
||||
```
|
||||
|
||||
Thresholds are 0–64 logical pixels, normalized to each panel's KDE scale.
|
||||
Status reports held state, suppressed motions, stabilized clicks and drags.
|
||||
Changing the threshold while a controller button is held is refused.
|
||||
|
||||
This is a separate contribution from desktop mouse/controller ownership. Its
|
||||
hardware validation must check small controls, intentional text selection,
|
||||
long presses, cross-monitor dragging and simultaneous mouse use. The existing
|
||||
renderer laser remains tracked; this change stabilizes desktop input rather
|
||||
than smoothing the visual laser. The default was 8 at first. That's about 0.2° on a 3.4 m wide 3440-pixel screen 2 m away, and clicking took a very still hand, so it's 32 (about 0.9°) since 2026-10-03.
|
||||
|
||||
Run `scripts/test-controller-click.sh` for the isolated gesture-state tests.
|
||||
+14
-116
@@ -20,12 +20,10 @@ SteamOS already shows a single-screen Plasma desktop in VR by running KWin neste
|
||||
|
||||
It also leaves the panels to SteamVR's dashboard, which places them and doesn't expose their position to other programs.
|
||||
|
||||
ft-screens replaces it. It's a small wlroots compositor that hosts the nested KWin. KWin's nested Wayland backend opens one window per output and resizes an output when the host configures its window, so ft-screens can give each screen any size, live. KWin needs `wl_compositor` v4 or later, `wl_shm`, `wl_seat`, `xdg_wm_base`, and `zwp_linux_dmabuf_v1` v4 with feedback from its host. KWin 6.6, which SteamOS build 20261009.6183636 ships, also refuses to start unless the host offers `wp_single_pixel_buffer_manager_v1`, `wp_viewporter`, `zwp_pointer_constraints_v1`, and `wp_presentation`. It draws each output's pixels on a synchronized subsurface and attaches a 1×1 single-pixel buffer to the toplevel, stretched to the output with viewporter. ft-screens presents the largest DMA-BUF in that tree, the toplevel's own included (KWin 6.2 draws there), and sends presentation feedback for the toplevel: KWin stops rendering after two frames without it. `xdg_toplevel_icon_manager_v1` and `zwp_keyboard_shortcuts_inhibit_manager_v1` stay optional. `scripts/update-check.py` fails when a newer KWin requires a host protocol this ft-screens binary does not name.
|
||||
ft-screens replaces it. It's a small wlroots compositor that hosts the nested KWin. KWin's nested Wayland backend opens one window per output and resizes an output when the host configures its window, so ft-screens can give each screen any size, live. KWin needs `wl_compositor` v4 or later, `wl_shm`, `wl_seat`, `xdg_wm_base`, and `zwp_linux_dmabuf_v1` v4 with feedback from its host; the rest is optional.
|
||||
|
||||
Frames go to SteamVR the same way gamescope sends them: OpenVR's `IVRIPCResourceManagerClient` imports each DMA-BUF (`ImportDmabuf`), and the overlay shows it as a `TextureType_SharedTextureHandle` texture. There's no copy and no size limit. The Frame's SteamVR supports this interface, but the OpenVR header bundled with SteamVR's samples predates it, so the build fetches a pinned header from Valve's openvr repository.
|
||||
|
||||
The small textures ft-screens draws itself (the controls, the hand glows, "grab that"'s halo, the carousel) go the same way, through `screens/overlaytex.cpp`: one linear DMA-BUF per distinct image, imported once and shared. Not `SetOverlayRaw`: the Frame's SteamVR never gives back the memory block each call takes, so a process's 201st call fails ("Refusing to create memory block because 201 blocks are already outstanding" in the journal) and the overlay keeps its old texture at its new size. That was the glow bug of 2026-10-15, after about 40 minutes of hand use. Other programs call it only a few times, or as a fallback.
|
||||
|
||||
Before settling on this, we tried KWin's screencast virtual outputs. KWin 6.2.5 crashed (`std::out_of_range` in `textureForOutput`) streaming one while nested in gamescope, and its headless backend doesn't implement virtual outputs at all.
|
||||
|
||||
A few wlroots details worth knowing: `wlr_shm_create` wants DRM format codes, not `wl_shm` ones, and KWin asks for server-side decorations before its first commit, when setting the mode would assert, so ft-screens answers on the first commit.
|
||||
@@ -40,12 +38,10 @@ The curved layout chains screens edge to edge, like monitors on a desk: the midd
|
||||
|
||||
A resize handle has to be able to shrink a screen from any direction, so the dragged corner follows the laser along the screen's diagonal rather than taking the larger of its horizontal and vertical reach. Pushing and pulling a carried screen moves it along the line from your head, because the 3D mouse's virtual controller sits just in front of the bar, below the screen's centre, so the line from the device points mostly upward.
|
||||
|
||||
Wherever ft-screens needs to know where a laser points (showing the controls, the resize tab, the roll knob), it uses the laser's own pose, the render model's `tip` component, rather than the controller's pose. On the Frame's controllers the tip points 40° below the pose's forward axis, so rays from the pose missed what the laser was actually on. The 3D mouse's virtual controller has no tip, and its laser runs along its pose. ft-screens reads the tip with `GetComponentState`: `GetComponentStateForDevicePath` without an input source handle fails for every component while a VR game runs, so in games the rays came from the pose, 40° too high.
|
||||
Wherever ft-screens needs to know where a laser points (showing the controls, the resize tab, the roll knob), it uses the laser's own pose, the render model's `tip` component, rather than the controller's pose. On the Frame's controllers the tip points 40° below the pose's forward axis, so rays from the pose missed what the laser was actually on. The 3D mouse's virtual controller has no tip, and its laser runs along its pose.
|
||||
|
||||
`ComputeOverlayIntersection` ignores `SetOverlayIntersectionMask`, and a control can't be allowed to cover part of its screen, so the resize tab sits entirely outside the corner.
|
||||
|
||||
What SteamVR hits isn't the texture's shape but the mouse scale's: an overlay is as tall, for SteamVR's laser and `ComputeOverlayIntersection`, as its width times the mouse scale's height over its width, and the default scale is 1 × 1. With it, the grab bar (a 256 × 24 texture) took hits in a square as tall as the bar is wide, so it caught clicks meant for the bottom tenth or so of the screen above it. Measured with a 0.2 m wide 256 × 24 overlay: a hit area 199 mm tall at the default scale, 18 mm at 256 × 24 (the bar itself is 18.8 mm), and no change from an intersection mask. `MakeChrome` sets each control's mouse scale to its texture size.
|
||||
|
||||
### Pinning
|
||||
|
||||
Pinning started as "bring the screen to your wrist", which doesn't work for big screens, because their centre is far from the edge you bring close. It became aiming: while a screen is carried, the line from the carrying device to its bar is tested against the other hand controllers. Crossing a controller's 6 cm ring arms the pin (leaving past 9 cm, so it doesn't flicker), and crossing it again disarms it. The pin happens on release, with the screen's pose at that moment, so you can arm it and then turn the screen. An earlier version pinned the moment the laser touched the wrist, which left the screen at whatever angle the carrying hand had while pointing there.
|
||||
@@ -56,9 +52,7 @@ A head pin is the same pin on the headset (device index 0): the screen's transfo
|
||||
|
||||
### Named layouts
|
||||
|
||||
Named layouts are now profiles, which also hold which screens are hidden and which apps to open, with where their windows go. See [profiles.md](profiles.md).
|
||||
|
||||
A profile's screen part is the custom arrangement under a name: each screen's pose relative to your head, width, curve, and pin, but not its resolution or scale, which need a desktop restart or belong to KWin. Using one copies it into the custom arrangement, so everything that applies the layout (desktop start, Meta+Shift+R, Arrange now) works unchanged, and `active` remembers which name it came from. Saving without a name (`ft-layout capture`) clears `active`, because the screens have been placed by hand since. Layouts are kept per screen number, so one saved with a different screen count still applies: missing screens keep their last saved place or the preset's.
|
||||
A named layout is the custom arrangement under a name: each screen's pose relative to your head, width, curve, and pin, but not its resolution or scale, which need a desktop restart or belong to KWin. Using one copies it into the custom arrangement, so everything that applies the layout (desktop start, Meta+Shift+R, Arrange now) works unchanged, and `active` remembers which name it came from. Saving without a name (`ft-layout capture`) clears `active`, because the screens have been placed by hand since. Layouts are kept per screen number, so one saved with a different screen count still applies: missing screens keep their last saved place or the preset's.
|
||||
|
||||
### Visibility and VR games
|
||||
|
||||
@@ -66,47 +60,6 @@ A profile's screen part is the custom arrangement under a name: each screen's po
|
||||
|
||||
`IVRApplications::GetCurrentSceneProcessId()` is 0 when no game is running (the Frame's home environment isn't a scene app) and the game's process ID while one is. ft-screens checks it twice a second, turns the flag off while a game runs, and by default hides the screens unless the dashboard is open. Flatscreen games run inside Steam's gamescope overlay and aren't scene apps, which is why "only with the dashboard open" is offered as a controller setting.
|
||||
|
||||
In a game, Frametop's panels work like SteamVR's own floating windows: point a controller at one and its laser comes on, point away and the game has the controllers again. ft-screens turns the flag on for a panel while a hand controller's laser pose meets it, its controls, or a floating window's popups. It finds that from the poses it already reads to show the controls, so SteamVR's laser doesn't have to be on first. Leaving takes a margin two control-sizes wide and 0.3 s, a drag or a held button keeps the flag on, and the keyboard, a single overlay, uses SteamVR's `ComputeOverlayIntersection`. The 3D mouse doesn't need any of this: it has its own laser mode.
|
||||
|
||||
## Floating windows
|
||||
|
||||
[floating-windows.md](floating-windows.md) describes the feature and its parts. Drag and drop and the clipboard only work between windows of one compositor, so a floating window stays a KWin window and gets a KWin output of its own: one of the spare outputs KWin opens after the screens, shown by ft-screens as a panel cropped to the window. What follows is how KWin 6.2.5 behaves underneath that, from its source (`src/backends/wayland/`) and from trying it on the Frametop desktop.
|
||||
|
||||
### KWin's nested outputs
|
||||
|
||||
- Disabling a nested output keeps its host window. `Output::applyChanges` only flips `enabled`, KWin stops rendering it (no more commits), and Plasma drops its desktop view. So ft-screens keeps the same toplevel, and its screen numbers stay put. Enabling the output again resumes on the same toplevel.
|
||||
- Each output's host window is titled `KDE Wayland Compositor WL-<n>`, with `- Output disabled` appended while it's disabled (`WaylandOutput::updateWindowTitle`, on every `enabledChanged`). ft-screens reads the title to tell screens (`WL-0` to `WL-<SCREENS-1>`) from spares, and to see a spare turn on and off.
|
||||
- A spare resized while it's disabled comes up at the new size on its first frame, so floating a window needn't blink. Outputs with gaps between them are accepted, so ft-floatd places spares apart from the screens and from each other: a grid below them, 4000 logical pixels a cell. Xwayland's coordinates are the layout's times its largest output scale, and they wrap past 32767, so an X11 window on a spare past that took no clicks (Brave on the 4th spare at x 27000 and scale 1.25 sat at x -31506; KWin still focused it). The grid ends at 20000 for 16 spares, within the limit up to scale 1.6.
|
||||
- KWin keeps a Wayland popup inside its parent's output (`XdgPopupWindow::updateRelativePlacement` uses the output's placement area), and X11 apps place their menus within the monitor. That's why a floating window's output has a margin around the window: menus and dropdowns open past the window's edges, into the margin.
|
||||
- KWin makes a nested output the size it's configured to times its scale, rounded (at 1.5 it lays out 1067 × 667 on a 1600 × 1000 buffer), and gives the buffer a whole buffer scale (1.2 becomes 2). A buffer whose size isn't a multiple of that is a protocol error that disconnects KWin, so ft-floatd sizes spares in multiples of it. After a scale change, ft-floatd asks for the output's size again in the new scale's terms, or the next configure would make it the old size times the scale.
|
||||
- **Virtual outputs don't work.** `createVirtualOutput` makes an output window but never adds it to the backend's `m_outputs`, so `findOutput()` returns null when the pointer enters it, and the next line dereferences it (`Q_ASSERT` is compiled out). A click on such a panel would crash KWin. This rules out virtual outputs (`stream_virtual_output`) for floating windows without a patched KWin.
|
||||
|
||||
### The pointer
|
||||
|
||||
- Pointer positions reach KWin only through motion events: the output's position in the layout plus the position on its window. When ft-screens stops sending motion, KWin's pointer stays put.
|
||||
- KWin starts an interactive move on the press itself, before any motion. So ft-screens stops sending motion as soon as a press lands in a floating window's title bar (from the frame and client rectangles ft-floatd sends it), with no round trip, and carries the panel instead. KWin gets the release at the press point, and the window moves by nothing on its output.
|
||||
- KWin's nested backend ignores the position in `wl_pointer.enter`, and wlroots drops a motion to the position it entered at, so the first click after crossing onto another panel landed where KWin's pointer had been. ft-screens enters one unit off.
|
||||
|
||||
### The KWin script
|
||||
|
||||
The KWin side is a script (`float/frametop-float.js`), not a C++ effect, because a script keeps working across KWin updates and an effect would have to match the host's exact KWin build. KWin scripts can call D-Bus but can't serve it, so ft-floatd's commands come back through a long poll: the script calls `NextCommand`, which answers when a command is ready, or empty after 20 seconds, under KWin's 25-second D-Bus timeout. If no reply comes within 30 seconds, a watchdog calls again, and waits twice as long each time until a reply comes (up to 5 minutes). The 30 seconds have to stay above KWin's timeout, or a late reply would start a second poll. A few things about KWin's script engine:
|
||||
|
||||
- `windowAdded` reports popups as windows of their own (`popupWindow` true, `transient` true) with their geometry.
|
||||
- Setting `frameGeometry` applies asynchronously: the app has to answer the new size first.
|
||||
- A script can't read a window's maximize mode, so the script counts a window as maximized when it fills its output's maximize area.
|
||||
- `globalThis` isn't defined. `print` goes to the journal unless `QT_FORCE_STDERR_LOGGING=1`.
|
||||
- `callDBus` never calls back when a call fails (an error reply, the name gone from the bus, or the 25-second timeout). KWin only logs `Received D-Bus message is error`, so a script that waits for the callback waits forever.
|
||||
|
||||
The title bar's float button is Frametop's own window decoration (`decoration/`), written in QML for KWin's Aurorae engine, which loads it without compiling. A C++ fork of Breeze would have to match SteamOS's exact KDecoration build. A decoration can only make the window requests KWin offers it, so the button toggles keep-below, which has no visible effect on a window alone on its own output, and the script treats keep-below as the floating flag.
|
||||
|
||||
### KWin's placement memory
|
||||
|
||||
KWin keeps each window's geometry, full screen, and maximized state for each layout of the outputs (its `PlacementTracker`, keyed by every enabled output's name and geometry). When the outputs come back to a layout it has seen, it puts every window back as it was in it. That's for plugging monitors in and out, and it does harm here. A spare output changes size after its window does, so what KWin keeps for a spare's size is the window's next size. Resizing a floating window back to an earlier size made the window and its output flip between two sizes for good (Dolphin went between 1187 and 1424 logical pixels wide, its output between 1687 and 1925). Full screen flipped the same way, and floating or docking one window could move others, even onto a spare or off one.
|
||||
|
||||
So the KWin script keeps where each window belongs: where ft-floatd put it, or where it went outside an output change. While KWin changes the outputs, it reports nothing to ft-floatd. Once KWin is done (`screensChanged` comes after its restore), it puts floating windows back, and the screens' windows too when only spares changed. KWin's resize request hasn't reached the app by then, so the app never sees it. A size asked for is held for a second, since an app can still answer an older request, and then the script takes the size the window has. If `screensChanged` doesn't come within 2 seconds, the script stops waiting for it.
|
||||
|
||||
KWin also ends an interactive move or resize whenever the outputs change. So during a resize by a floating window's edge, ft-floatd only crops the panel to the window, and resizes the output when the drag ends. The margin is the room to grow until then.
|
||||
|
||||
## The 3D mouse
|
||||
|
||||
The mouse works like the pointer on the Apple Vision Pro: a small cursor floats in the room, lands on whatever panel it meets, and acts on it like a controller's laser.
|
||||
@@ -119,19 +72,13 @@ Driver poses are in SteamVR's raw tracking space, and client programs work in th
|
||||
|
||||
Frametop's SteamVR clients (ft-pointer, ft-screens, ft-gaze) connect as a background app first and switch to an overlay app only once that works. `VR_Init` as an overlay app starts vrserver itself when none is running, and one started that way from the dev container never finds the headset. At a boot where the gamescope session timed out, systemd dropped `steamvr.service`'s start job, the pointer service (ordered only `After=` it) started anyway, and its vrserver made every SteamVR launch fail with `HmdNotFound`. SteamOS's health check then kept resetting the Steam client and tried to fall back to the previous OS slot. The units also say `Requisite=steamvr.service`, so they don't start at all when SteamVR's start fails.
|
||||
|
||||
The driver starts disconnected, because holding the right-hand role while SteamVR starts leaves the Steam UI stuck on its loading icon. It connects when the mouse is used. With a hand role (`POINTER_ROLE=right`, the default before the treadmill role) it claims that hand: SteamVR keeps a hand role reserved for a disconnected device that still asks for it, so the driver switches its role hint between the hand (connected) and opt-out (not connected). The treadmill role (below), now the default, needs no switching.
|
||||
|
||||
#### Treadmill role
|
||||
|
||||
With a hand role, a Frame controller held in that hand takes the role back (its touch sensors count as use), and the virtual controller is left with no role and no clicks. So mixing a controller in one hand with the 3D mouse or a bare hand's ray in the other fails whenever that controller is merely held. The treadmill role (4) is no hand: the real controllers keep both hand roles, and the virtual controller never fights them. It held the dashboard laser in the gaze-first tests (2026-09-30, branch `gaze-first`), with two conditions. SteamVR gives a device the `/user/treadmill` path only when the device is added, so the driver must hint the role from `Activate`, not on connect: the `role` in its `default.vrsettings` (or `driver_ft_pointer/role` in SteamVR's settings), and switching it needs a SteamVR restart. And the compositor binding needs `/user/treadmill` entries next to the hand ones (`ft_pointer_vrcompositor.json`). A treadmill reports no controller role (`GetControllerRoleForTrackedDeviceIndex` is invalid), so the helper's "no hand role" release skips it. The laser claim (`/input/a`, bound to `switchlaserhand`) works the same; SteamVR took the laser back to our device 11-15 ms after a controller took it.
|
||||
|
||||
A treadmill can't move SteamVR's own floating windows by their grab bars, though: SteamVR's dashboard (a web page in vrwebhelper) starts a window's move in `UndockedOverlay.startFloatingWindowMove`, which reads the pressing device from the mouse event's CapsLock and NumLock bits (head, left hand, right hand) and, for anything else, logs "mouse event carried no input path" and does nothing. Clicks and resizes don't ask, so only moves broke, from 2026-10-09, when treadmill became the default (found 2026-10-13, SteamVR 2.18.2; with role 2 the same laser moved them). `pointer/dashpatch` puts that back. `treadmill-move.js` wraps the grab bar's handler: a press with no device runs it as the right hand, with that one pose query answered by `/user/treadmill`, and the move then follows `/user/treadmill`. `ft-dashpatch.py run` (`frametop-dashpatch.service`, started with SteamVR) puts it into the dashboard over the Chrome DevTools Protocol, again whenever the page is new, and checks every 30 s that it's still in. The dashboard's debugger is off unless `VRWebHelper/DebuggerEnabled` is on; the installer turns it on (`ft-dashpatch.py on`, from SteamVR's next start) on port 8089, bound to 127.0.0.1. Not the usual 8087: SteamOS's `steamvr-web-debug-portforward.service` forwards 0.0.0.0:8088 to 127.0.0.1:8087, which would open the dashboard to the network. Like Steam's own debugger on 8080, any local program can use it. The uninstaller turns it off again if Frametop turned it on. The patch finds the handler by its name and React by its module, so a SteamVR update that renames them leaves windows unmovable for the treadmill again, without breaking anything else; `ft-dashpatch.py status` shows whether it's in and how many moves it made.
|
||||
The driver starts disconnected, because holding the right-hand role while SteamVR starts leaves the Steam UI stuck on its loading icon. It connects when the mouse is used and claims the right hand. SteamVR keeps a hand role reserved for a disconnected device that still asks for it, so the driver switches its role hint between right hand (connected) and opt-out (not connected).
|
||||
|
||||
### The cursor
|
||||
|
||||
Mouse motion turns into yaw and pitch around an anchor, the head position at the last recenter. A ray from the anchor is tested against every visible overlay with `ComputeOverlayIntersection`. On a hit, the cursor sits on that surface; otherwise it floats at `POINTER_DISTANCE`. Since the anchor isn't your current eye position, a second test runs along your line of sight to the cursor point, and anything nearer wins, so the cursor always lands on what you see under it. Overlays in `POINTER_IGNORE` are left out of both tests. A display-only panel, like a performance overlay locked to your view, has no input method, so SteamVR's laser passes through it, but `ComputeOverlayIntersection` still hits it, and the cursor stuck to it. The laser starts just before the cursor point, so an ignored panel nearer to you doesn't catch it either. Both tests ask SteamVR about every visible overlay, so a frame where nothing moved (the mouse, the anchor, the eye by more than 5 mm, which overlays show) reuses the last result, for up to 100 ms, since overlays can also move on their own. The dots' overlay settings go to SteamVR only when they change, and the pose goes to the driver, which keeps the last one, only when the laser would land 0.1 mm or more elsewhere, and at least every 100 ms.
|
||||
Mouse motion turns into yaw and pitch around an anchor, the head position at the last recenter. A ray from the anchor is tested against every visible overlay with `ComputeOverlayIntersection`. On a hit, the cursor sits on that surface; otherwise it floats at `POINTER_DISTANCE`. Since the anchor isn't your current eye position, a second test runs along your line of sight to the cursor point, and anything nearer wins, so the cursor always lands on what you see under it. Overlays in `POINTER_IGNORE` are left out of both tests. A display-only panel, like a performance overlay locked to your view, has no input method, so SteamVR's laser passes through it, but `ComputeOverlayIntersection` still hits it, and the cursor stuck to it. The laser starts just before the cursor point, so an ignored panel nearer to you doesn't catch it either.
|
||||
|
||||
OpenVR has no call to list other programs' overlays, so the helper runs `vrcmd --overlays` in the background. It includes hidden overlays, because a floating window's controls only appear while something hovers the window, and the cursor has to find them immediately. Each run is a shell and a new SteamVR client, about 30 ms of CPU, and it ran every second while the pointer was awake, which in gaze mode is all the time. Now it runs every 20 seconds, and at once when the pointer wakes, when the dashboard opens or closes, when a game starts or ends, and when a left click hits nothing (a panel that came up since). An overlay already on the list showing or hiding needs no new list: the helper checks the visibility of the ones it knows every 50 ms.
|
||||
OpenVR has no call to list other programs' overlays, so the helper runs `vrcmd --overlays` in the background. It includes hidden overlays, because a floating window's controls only appear while something hovers the window, and the cursor has to find them immediately.
|
||||
|
||||
The laser starts partway along your line of sight to the cursor rather than at your eye. SteamVR sizes its hit dot by distance from the laser's origin, and a laser from the eye still shows a beam in each eye. Starting it close to the target makes the beam and the dot tiny, while `POINTER_ORIGIN_MARGIN` keeps the origin in front of the small window controls, which float a few centimetres in front of their panels. The helper's own white dot is the visible cursor. In empty space it's an interactive overlay that the laser lands on, so SteamVR never draws a laser into nothing.
|
||||
|
||||
@@ -144,7 +91,7 @@ A few overlays need special handling:
|
||||
|
||||
Head follow is experimental and off by default. It works, but it's only lightly tested, and the feel is mostly a matter of its settings; polishing it is left open. With it on (`POINTER_FOLLOW=1`, or a mouse button mapped to Head follow on/off), the cursor rides on a reference direction, where you were facing when your head last settled, and keeps its offset from it. The mouse can put the cursor anywhere up to `POINTER_FOLLOW_REACH` (70 degrees) from the reference, a corner of your view included. While your head stays within `POINTER_LEASH_DEG` of the reference, nothing moves on its own. Once your head has been past the leash for `POINTER_LEASH_DELAY` (0.2 s, so a glance out and back doesn't count), the reference eases to where you're facing (time constant `POINTER_LEASH_RETURN`, 0.2 s), never falling further behind than the leash, and the cursor ends up back where it was in your view. Then it waits for the leash again. Two earlier versions didn't work out. Moving the reference only while your head pulled at the end of the leash left it up to the leash off after you turned back, and getting it centred again meant overshooting with your head. Easing it toward your facing all the time moved the cursor on every small head movement. A leash of 0 makes the reference your facing direction, so the cursor is locked to your view, and mouse movement shifts it within the view. Head roll is ignored, so tilting your head doesn't swing the cursor around. While the left button is held the cursor stays put in the room, so your head can't nudge a click or a drag. When you let go, it carries on from where it is instead of jumping.
|
||||
|
||||
Gaze mode is experimental and off by default (`POINTER_GAZE=1`, the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, or a mouse button or key combination mapped to Gaze pointer on/off). It's MAGIC pointing (Zhai, Morimoto and Ihde, 1999): the pointer goes where you look, and the mouse does the last bit. The gaze service (`gaze/ft-gazed`) sends the helper the corrected gaze at 90 Hz (from one eye while the tracker has lost the other), and while the gaze has the pointer, the cursor ray is that gaze from the eye. The pointer is aimed at the gaze each frame, not steered toward it, so nothing can pile up. An earlier try in the gaze probe steered the pointer with relative moves, and lost it when the pointer went idle or a controller had the laser. By default (`POINTER_GAZE_MOUSE_MOVE=held`, the Gaze page's Mouse movement switch) moving the mouse does nothing while the gaze has the pointer: it moves the pointer only while a button is held, as a correction. A bumped or drifting mouse can't pull the pointer off what you're looking at, and every mouse move is a correction, so the lessons aren't polluted by mouse moves to somewhere else (they used to be kept out by an 8 degree limit, which also dropped real corrections when the tracker was further off). With the gaze stale for a second, in a game, or with the headset off, the mouse moves the pointer as usual; with `free`, moving the mouse takes the pointer from the gaze. A left press while the gaze has the pointer isn't sent at once: the pointer stops where the gaze put it, you drag it onto what you meant with the button still down (panels only see it hover), and the release clicks there. Clicking at once clicked wherever the gaze was, often the wrong thing, before you could correct it. The drag is the correction. Snapping the pointer onto buttons and links is deferred: the session now starts an AT-SPI registry, but apps still need to expose useful accessibility trees (and may need restarting), and it makes Chromium and Electron apps use more CPU. A press held still for `POINTER_GAZE_HOLD` (0.5 s) becomes a real press, so drags still work: hold, then move. The right button works the same way, with the right click on the release, and pressing it while the left press is held back starts a drag where the pointer is, like Meta+J then Meta+K. That drag lasts while either button (or key) is held, so a second right press, or a second Meta+K, is free to pan and tilt the panel being dragged; with the keyboard, the head turns it. Outside games the pointer then stays: the mouse going idle doesn't release it. A moving controller still releases it, as without gaze. Gaze mode is a mouse and keyboard feature: Steam reads the Frame controllers itself, outside SteamVR's bindings, so controller clicks at the gaze kept knocking SteamVR out of laser mode (see `docs/gaze-controllers.md`). Keyboard clicks (Meta+J, Meta+K) hold the dot still in your view while the keys are down, so the head, not the mouse, does the last bit; a quick tap clicks where the dot was at the press, since the head moves as you hit the keys. The relay hides Meta from the desktop as soon as such a combination fires, because KWin takes Meta with a mouse button as a window move or resize, which swallowed the clicks. The dot shows all the time by default. With `POINTER_GAZE_DOT=moving` it shows only while the mouse moves it (`POINTER_GAZE_SHOW`), while a press is held, and as a pulse for each click; otherwise it's transparent, so the laser still lands on it. Looking more than `POINTER_GAZE_RETAKE` (5 degrees) away from it, with the mouse still, gives it back, so small eye movements around the pointer don't pull it off what you're doing. A mouse nudge before a click whose correction is within `POINTER_GAZE_NUDGE_MAX` (55 degrees, half of what the headset shows across) is sent to the gaze service as a lesson: you were looking at where you clicked when the mouse took over, so the nudge is the eye tracker's error there. Using it is what calibrates it. A one-dot check in a panel fixed to the headset tops that up when the headset goes on, when our tracker thinks it moved, and when a correction is past that limit (the tracker is far off, so a click there isn't trusted as a lesson), and the full calibration and the headset fit check run in the same panel, so everything a user does to calibrate happens in one place in the headset; the gaze probe, a fullscreen GTK app, is the development tool. The limit was 8 degrees, which dropped every correction while our tracker was 12 off. Its dots sit at known directions from the headset, so the panel needs no screen geometry. The quick check's dot takes the gaze once it has held still, so what the tracker says doesn't have to be close for the capture to work. The full calibration's and the five-dot check's dots wait for a click while you look at the dot (a left click or Meta+J), because a steady gaze isn't always on the dot, and take the gaze held still up to the click; a rightLine truncated
|
||||
Gaze mode is experimental and off by default (`POINTER_GAZE=1`, the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, or a mouse or controller button mapped to Gaze pointer on/off). It's MAGIC pointing (Zhai, Morimoto and Ihde, 1999): the pointer goes where you look, and the mouse does the last bit. The gaze service (`gaze/ft-gazed`) sends the helper the corrected gaze at 90 Hz (from one eye while the tracker has lost the other), and while the gaze has the pointer, the cursor ray is that gaze from the eye. The pointer is aimed at the gaze each frame, not steered toward it, so nothing can pile up. An earlier try in the gaze probe steered the pointer with relative moves, and lost it when the pointer went idle or a controller had the laser. Moving the mouse takes the pointer from the gaze. A left press while the gaze has the pointer isn't sent at once: the pointer stops where the gaze put it, you drag it onto what you meant with the button still down (panels only see it hover), and the release clicks there. Clicking at once clicked wherever the gaze was, often the wrong thing, before you could correct it. The drag is the correction. Snapping the pointer onto buttons and links is deferred: it needs accessibility (AT-SPI) on in the Frametop session, where it's off (no registry runs), plus app restarts, and it makes Chromium and Electron apps use more CPU. A press held still for `POINTER_GAZE_HOLD` (0.5 s) becomes a real press, so drags still work: hold, then move. Outside games the pointer then stays: the mouse going idle doesn't release it. A moving controller still releases it, as without gaze; the mouse is gaze mode's only pointer device for now. The dot shows only while the mouse moves it (`POINTER_GAZE_SHOW`), while a press is held, and as a pulse for each click; otherwise it's transparent, so the laser still lands on it. The gaze moving the pointer doesn't show it: you know where you're looking. Looking more than `POINTER_GAZE_RETAKE` (5 degrees) away from it, with the mouse still, gives it back, so small eye movements around the pointer don't pull it off what you're doing. A mouse nudge of up to `POINTER_GAZE_NUDGE_MAX` (8 degrees) before a click is sent to the gaze service as a lesson: you were looking at where you clicked when the mouse took over, so the nudge is the eye tracker's error there. Using it is what calibrates it. See `gaze/README.md` for the service, the calibration, and what was measured.
|
||||
|
||||
Replacing a loaded driver's files, as re-running the installer used to do, leaves SteamVR honoring the virtual controller's hand role but not its laser claim: the dashboard pointer stays unassigned until SteamVR restarts. The driver installer now leaves an unchanged driver in place.
|
||||
|
||||
@@ -154,7 +101,7 @@ Replacing a loaded driver's files, as re-running the installer used to do, leave
|
||||
|
||||
The dashboard follows whichever device summoned it or last pressed its trigger. Frametop adds "last used wins": moving a real controller releases the pointer, and the next mouse movement takes the laser back. Moving means faster than 0.35 m/s or 2 rad/s (both times `POINTER_CONTROLLER_PICKUP`, 1 by default) for 100 ms in a row, while the controller is tracked normally. A single sample over the limit used to be enough, and controllers resting on a desk took the laser back on a knock or a tracking jump while the mouse was in use. Small movements don't count; waking needs `POINTER_WAKE_COUNTS` of mouse motion within a second, so desk jitter doesn't steal the laser. While the pointer is awake, a tiny transparent overlay with `MakeOverlaysInteractiveIfVisible` keeps SteamVR's laser mouse on, since otherwise the first click would only switch the laser on.
|
||||
|
||||
When the headset comes off, SteamVR reports its activity level as idle at once and turns the displays off 5 seconds later (`power.turnOffScreensTimeout`), unless something keeps it awake. An awake pointer did, and so did the helper's `vrcmd` runs: each is a new SteamVR client, and a new client every second kept SteamVR out of standby. The helper now releases the pointer as soon as the headset is idle, ignores the mouse until you're wearing it again, and pauses the overlay list whenever the pointer is off. With the pointer off it also stops running its loop every 8 ms, about 116 wakeups a second for nothing: it waits up to 250 ms for a command on its socket (20 ms while it reads mapped controller buttons, which SteamVR input only offers by polling), and leaves the overlay lookups until the pointer wakes.
|
||||
When the headset comes off, SteamVR reports its activity level as idle at once and turns the displays off 5 seconds later (`power.turnOffScreensTimeout`), unless something keeps it awake. An awake pointer did, and so did the helper's `vrcmd` runs: each is a new SteamVR client, and a new client every second kept SteamVR out of standby. The helper now releases the pointer as soon as the headset is idle, ignores the mouse until you're wearing it again, and pauses the overlay list whenever the pointer is off.
|
||||
|
||||
### Moving floating windows
|
||||
|
||||
@@ -166,65 +113,29 @@ SteamVR opens every input device only when it starts. When a Bluetooth mouse sle
|
||||
|
||||
Keyboards aren't grabbed by default, because a grabbed keyboard's keys went into a virtual keyboard nothing typed from; the relay forwards them to ft-screens instead.
|
||||
|
||||
The relay never waits on the pointer helper. Its socket to the helper used to block, so when the helper stalled (a layout placement or `grabprobe` holds it for seconds, and the gaze service fills its socket 90 times a second meanwhile), the whole relay stopped with it: keyboards, the volume keys, and pausing. Now what the helper doesn't take waits in order and goes out on the next loops. Mouse moves add up into one while they wait, and a scroll notch is dropped, since scrolling seconds late is no use; presses and releases are kept, so no button stays down. Mouse motion goes to the helper at most every 4 ms, rather than once per report, which from a 1000 Hz mouse was 1000 datagrams a second to a helper that runs every 8 ms; a button sends the motion before it first, so the click lands where the pointer was.
|
||||
|
||||
An ungrabbed keyboard reaches both sides at once. In VR, gamescope reads every input device itself (the SteamOS build's `InputStealer`, libinput with udev hotplug, so new devices too) and types into its focused app, and ft-screens types the same keys into the desktop. So Space in the desktop also paused Spotify on the dashboard. Typing now follows the last click. ft-screens sees clicks on its own screens, from the mouse or a controller. A click anywhere else is only visible for the mouse: overlay apps get SteamVR's `OverlayFocusChanged` (which panel the laser is on) but no controller button events, so the pointer helper reports the panel under the dot on each left press. ft-screens tells the relay where typing goes every second, from an unbound socket so the relay's replies can't loop back into its control socket, and the relay grabs pass-through keyboards while it's the desktop. A grab waits until the keyboard has no key down, so no key stays held on either side, and the relay lets go if ft-screens stops reporting. A program that reads every keyboard for a hotkey (a dictation tool, say) loses a grabbed keyboard. Repeating the keys on another input device doesn't work: gamescope reads that device too, whether it's the relay's virtual keyboard or one created later, and every Space, typed or dictated, paused Spotify again. Until 2026-10-10, `SHARE_KEYS=1` sent a grabbed keyboard's keys to `@frametop_keys` as datagrams for frame-voice. It was off by default, because the relay can't tell who is listening: abstract sockets have no permissions, and any local process that binds the name first got every key typed into the desktop, passwords included. Now frame-voice is part of Frametop (Frametop Voice), and the relay does its keyboard side itself: its push-to-talk key is a key combination, and its text comes to the relay as `type TEXT`, taken only from the relay's own user (`SO_PASSCRED`), and goes where typing goes, like a keyboard's keys: to ft-screens for the desktop, or the virtual keyboard for Steam, never both. Its own uinput keyboard went with it, so SteamVR no longer has to have opened a second keyboard for dictation to reach games.
|
||||
|
||||
A mouse button that pointer mode passes through as a key (a side button for Back) is a pointer button to ft-screens, not a key, so it goes to the screen the pointer is on rather than where typing goes. As a key, its release was lost whenever typing moved while it was held (the dashboard closing, a click on another panel, a pause), and wlroots counts presses per button: from then on that button, the lasers' left click included, did nothing in the desktop, and KWin kept it held. So ft-screens tracks the relay's buttons itself, lets a release through whenever it took the press, and releases them when the pointer leaves the screens, its screen hides, or Frametop pauses.
|
||||
An ungrabbed keyboard reaches both sides at once. In VR, gamescope reads every input device itself (the SteamOS build's `InputStealer`, libinput with udev hotplug, so new devices too) and types into its focused app, and ft-screens types the same keys into the desktop. So Space in the desktop also paused Spotify on the dashboard. Typing now follows the last click. ft-screens sees clicks on its own screens, from the mouse or a controller. A click anywhere else is only visible for the mouse: overlay apps get SteamVR's `OverlayFocusChanged` (which panel the laser is on) but no controller button events, so the pointer helper reports the panel under the dot on each left press. ft-screens tells the relay where typing goes every second, from an unbound socket so the relay's replies can't loop back into its control socket, and the relay grabs pass-through keyboards while it's the desktop. A grab waits until the keyboard has no key down, so no key stays held on either side, and the relay lets go if ft-screens stops reporting. A program that reads every keyboard for a hotkey (a dictation tool, say) loses a grabbed keyboard. Repeating the keys on another input device doesn't work: gamescope reads that device too, whether it's the relay's virtual keyboard or one created later, and every Space, typed or dictated, paused Spotify again. So with `SHARE_KEYS=1` the relay sends a grabbed keyboard's keys to `@frametop_keys` as datagrams (`key <code> <value> <device name>`). It's off by default, because the relay can't tell who is listening: abstract sockets have no permissions, and any local process that binds the name first gets every key typed into the desktop, passwords included. A listener should accept only its own user (`SO_PASSCRED`) and skip any keyboard of its own that the relay grabs too.
|
||||
|
||||
Volume keys must never reach gamescope. With the openvr backend, gamescope sends volume up and down to Steam by moving keyboard focus to Steam for the key and then back to the previously focused surface. When nothing had focus, the one it moves back to is null, and wlroots aborts on a null focus surface (`wlr_seat_keyboard_notify_enter: Assertion 'surface' failed`), which ends the whole VR session. Keyboard focus is often empty while you work in VR, so one press of the headset's volume button could take everything down. gamescope reads the headset's buttons and every keyboard itself (`InputStealer`), as do SteamVR's processes, so the relay has to stop volume keys at the device. Grabbing `gpio-keys` would also take the headset's click button, so the relay remaps the volume entries in each device's keymap (`EVIOCSKEYCODE`) and handles the stand-in codes itself. That fix covers every device at once, including keyboards that aren't grabbed.
|
||||
|
||||
Frametop's keyboard opens by itself for a text field on the desktop. The apps run inside the nested KWin, so only KWin knows when a text field has focus, and the way it tells anyone is its input method protocol (`zwp_input_method_v1`): KWin starts one input method program and activates it whenever the focused app turns on text input. `input/ft-textinput` is that program, speaking the Wayland wire protocol directly so it needs nothing but Python on the host. It only reports focus. The gamescope session puts `QT_IM_MODULE=xim` and `GTK_IM_MODULE=xim` in the systemd user environment; with those, Qt and GTK apps use X input methods and never turn on Wayland text input, so the session script drops them.
|
||||
|
||||
The keyboard itself is ft-screens' own panel (`screens/keyboard.cpp`). We tried SteamVR's first (`ShowKeyboardForOverlay`), and on the Frame it doesn't fit a desktop. It's Steam's own panel (`valve.steam.gamepadui.keyboard`), which SteamVR mounts in the dashboard's scene, so with the dashboard closed it opened but wasn't drawn. Placing it in the room ourselves (`SetKeyboardTransformAbsolute`) made it show, but SteamVR moves it to whichever overlay the laser goes to and mounts it again, and while it's open the controllers switch to SteamVR's own laser. Our panel is an overlay like the screens' controls: any laser or the 3D mouse clicks it, nothing moves it, and its keys go out as key presses on ft-screens' seat rather than as text handed back to the input method. So nothing typed leaves ft-screens (a socket to the input method could be claimed by any local process, as the old `@frametop_keys` could), apps without text input (X11, Electron) take the keys too, and they mean what the desktop's keyboard layout says. It's drawn on the CPU into one of three shared buffers when a key's look changes; the labels come from stb_truetype, so the container needs no text rendering stack.
|
||||
The keyboard itself is ft-screens' own panel (`screens/keyboard.cpp`). We tried SteamVR's first (`ShowKeyboardForOverlay`), and on the Frame it doesn't fit a desktop. It's Steam's own panel (`valve.steam.gamepadui.keyboard`), which SteamVR mounts in the dashboard's scene, so with the dashboard closed it opened but wasn't drawn. Placing it in the room ourselves (`SetKeyboardTransformAbsolute`) made it show, but SteamVR moves it to whichever overlay the laser goes to and mounts it again, and while it's open the controllers switch to SteamVR's own laser. Our panel is an overlay like the screens' controls: any laser or the 3D mouse clicks it, nothing moves it, and its keys go out as key presses on ft-screens' seat rather than as text handed back to the input method. So nothing typed leaves ft-screens (a socket to the input method could be claimed by any local process, like `@frametop_keys`), apps without text input (X11, Electron) take the keys too, and they mean what the desktop's keyboard layout says. It's drawn on the CPU and uploaded with `SetOverlayRaw` when a key's look changes; the labels come from stb_truetype, so the container needs no text rendering stack.
|
||||
|
||||
The Frame controllers can be mapped like mouse buttons, but they aren't input devices on the host: they reach SteamVR over the headset's own radio, and no evdev or hidraw node exists for them. So only a SteamVR client can read them. Overlay apps normally get controller input only while they have input focus, which a background helper never has. SteamVR's experimental global action set priority (`steamvr/globalActionSetPriority`, "Enable global input from overlays") lets an overlay's action set receive input anyway, and takes the inputs it binds from the scene app. Binding every button would take them all from games, so the helper's action manifest puts each button in an action set of its own, and it activates only the sets of mapped buttons. The mapping itself stays in the relay, which does the action, so mice and controllers share one list of actions.
|
||||
|
||||
## The desktop session
|
||||
|
||||
The session is modeled on SteamOS's `steamos-nested-desktop` and runs beside it. It has its own runtime directory, config (`~/.config/frametop`), and state, so it never disturbs the stock desktop's layout or panels. It runs on a private D-Bus from `dbus-run-session`, which has two consequences. KDE only launches apps in systemd scopes when systemd is on the session bus, so everything started in the desktop lands in its systemd unit, and stopping the unit would kill all of it; `session/keep-apps.sh` moves those programs out first. And tools that need the real user bus, like podman and `distrobox-host-exec`, have to be pointed at it explicitly. Its own config folder also hides SteamVR's path registry (`~/.config/openvr/openvrpaths.vrpath`) from everything started in it: OpenVR programs there fail with `VRInitError_Init_PathRegistryNotFound`, and `vrpathreg adddriver` writes a new registry under `~/.config/frametop/openvr` that has no SteamVR in it and that SteamVR never reads. So Frametop's scripts run SteamVR's tools with `XDG_CONFIG_HOME=~/.config`.
|
||||
The session is modeled on SteamOS's `steamos-nested-desktop` and runs beside it. It has its own runtime directory, config (`~/.config/frametop`), and state, so it never disturbs the stock desktop's layout or panels. It runs on a private D-Bus from `dbus-run-session`, which has two consequences. KDE only launches apps in systemd scopes when systemd is on the session bus, so everything started in the desktop lands in its systemd unit, and stopping the unit would kill all of it; `session/keep-apps.sh` moves those programs out first. And tools that need the real user bus, like podman and `distrobox-host-exec`, have to be pointed at it explicitly.
|
||||
|
||||
The VR launcher starts the session from the Steam client, and the client's environment came along: `LD_LIBRARY_PATH` pointing at Steam's own runtime, whose `libavcodec` has no H.264 decoder, so VLC in the desktop couldn't play most videos, plus the client's overlay and launch settings. The session script drops the client's variables before it starts anything. SteamOS's global Mesa settings (`/usr/share/deckard/mesavars.sh`) stay, and the gamescope session's Vulkan layer (`ENABLE_GAMESCOPE_WSI`) is only kept for the gamescope backend.
|
||||
|
||||
### Nested accessibility
|
||||
|
||||
The session drops an inherited `AT_SPI_BUS_ADDRESS`, so apps cannot accidentally use the host desktop's registry. It autostarts `session/ft-atspi` in Plasma phase 2, after KWin has set the nested display environment. The helper gets the live accessibility address from `org.a11y.Bus` on the private session bus, preserves any existing registry owner, updates the accessibility bus's activation environment, and tries `StartServiceByName` first.
|
||||
|
||||
On SteamOS 0.3.0 with at-spi2-core 2.52.0, the native launcher can choose dbus-broker because its process belongs to a systemd user unit. Registry activation then fails: this desktop's private session bus does not have a systemd activation manager. In that case the helper starts only `at-spi2-registryd` on the already-existing accessibility bus. The registry refuses duplicate ownership. Unlike native activation's `--use-gnome-session`, the fallback does not try to register with GNOME's session manager; that flag did not explain the observed native activation failure.
|
||||
|
||||
The fallback registry does not exit merely when its bus disconnects in the isolated SteamOS test. Its small watcher checks both private buses every 5 seconds, and terminates and reaps only the child it started when either bus disappears or the watcher is stopped. Each check runs `gdbus` twice; once a second, that cost about 1% of a core. `keep-apps.sh` keeps the watcher in the desktop unit when `desktops.sh start` runs the desktop as `frametop-desktop`. Started from the VR launcher, the desktop runs in steam.service, which doesn't stop with it, so there the watcher is the only thing that stops the registry. There is no second accessibility bus, global systemd environment update, process-name kill, or host registry replacement. Missing accessibility files or bus errors are nonfatal; the desktop still starts. Toolkit-specific accessibility opt-ins and pointer snapping are separate work.
|
||||
|
||||
Run the isolated checks on the host with `/usr/bin/python3 session/test/test_accessibility.py`. They use private D-Bus buses, Xvfb and a GTK3 app, never the production display or input. Native activation uses a small `org.a11y.Bus` test provider pointing to a real private dbus-daemon with the installed registry service; the SteamOS fallback uses the installed bus launcher and broker. The tests check real app-tree discovery, existing owners, concurrent starts, session stop/restart, and teardown. They require test-only PyGObject (Gio and GTK3), Xvfb, and at-spi2-core; the runtime helper uses Python's standard library and the existing host `gdbus`. Actual Plasma autostart and VR desktop restart still require an approved hardware test.
|
||||
|
||||
### Notifications outside the desktop
|
||||
|
||||
The private bus keeps the desktop's notifications in it: apps notify Plasma, and Plasma's popups appear on the desktop's screens, whose panels may be hidden, or behind a VR game. Putting the desktop on the user bus instead would collide with the Steam session: Steam owns `org.freedesktop.ScreenSaver` there, the Steam session's portals and accessibility bus are there, Plasma would start its parts as units of the user's systemd and push its environment (`DISPLAY`, `WAYLAND_DISPLAY`, its runtime directory) into the bus's and systemd's activation environment, which programs started later in the Steam session would inherit.
|
||||
|
||||
`session/ft-notifyfwd` forwards instead. Taking `org.freedesktop.Notifications` from Plasma isn't reliable: plasmashell, the bus's "D-Bus master", registers it with `ReplaceExistingService` and `DontAllowReplacement` (plasma-workspace's `libnotificationmanager/server_p.cpp`), so another owner only keeps it by starting first, and loses it to a Plasma restart in between. Plasma 6.2.5 has a watcher API for this: `org.kde.NotificationManager.RegisterWatcher` on Plasma's notification object, after which Plasma calls `org.kde.NotificationWatcher.Notify` (its own id first, then the spec's arguments with the body as sent and the app's hints, image data included) and `CloseNotification` on the watcher's `/NotificationWatcher`, and `InvokeAction(id, key)`, which makes Plasma emit `ActionInvoked` to the app. The app only accepts that signal from its notification server's owner, which is why a click has to go back through Plasma. The helper copies each notification to the user bus's server keeping the hints' D-Bus types, maps the two servers' ids (a replacement reuses Plasma's), and maps that server's `ActionInvoked` to `InvokeAction`. Its `NotificationClosed` with reason 2 (dismissed) closes the notification in Plasma, so the app sees it closed (as revoked, reason 3); an expired one stays in Plasma's history. While it forwards, it holds an `Inhibit` on Plasma, which turns its popups off as Do not disturb does, and ends with the helper's connection; watchers get every notification regardless. It follows both servers' owners: it registers and inhibits again when Plasma restarts, and in `auto` mode checks `GetServerInformation` whenever the user bus's server changes. SteamOS's `steam_notif_daemon` reports itself as `simple_notif_daemon`; it passes notifications to `~/.steam/root/ubuntu12_32/steam` with a `steam://open_xdg_notification` URL, an x86 program that can't run on the Frame, and the Frame's Steam client ignores that URL anyway, so forwarding there would lose them. The helper is event-driven (GLib's main loop, PyGObject), so it costs nothing between notifications, and exits when the desktop's bus closes.
|
||||
|
||||
Notifications sent through the notification portal (`org.freedesktop.portal.Notification.AddNotification`, as Flatpak apps do) reach the watchers too: xdg-desktop-portal-kde passes them to Plasma's `Notify`. Plasma sends no activation token with `InvokeAction`, so KWin's focus stealing prevention may keep a window that an action raises from coming to the front. KDE's inline reply needs the `inline-reply` capability from the other server; without it, it's an action like any other.
|
||||
|
||||
Run the isolated checks on the host with `/usr/bin/python3 session/test/test_notifyfwd.py`. Two private buses stand in for the desktop's and the user bus, with a fake Plasma (the watcher API, Inhibit) on one and a fake notification server on the other. They check forwarding with the hints' types, actions coming back, dismissed and expired notifications, replacements and closes mapping ids, `auto` skipping `simple_notif_daemon` until another server replaces it and giving Plasma its popups back when none is left, registering again after a Plasma restart, exiting with the desktop's bus, and doing nothing outside the Frametop desktop.
|
||||
|
||||
### Other session behavior
|
||||
|
||||
Steam, not systemd, suspends the Frame: after `system_idle_suspend_ac_sec` (an hour by default) without input on AC power, it logs `Switching to power state: k_ESystemPowerState_Sleep` and suspends, even while charging. It's a Steam setting (Settings → Power → When Plugged In and Idle → Sleep after), which the Stay awake while plugged in switch in Frametop Display Settings sets to Never. SteamVR's standby, which turns the displays off when the headset comes off, is separate; see below.
|
||||
|
||||
Flatpak apps need `XDG_DATA_DIRS` to include Flatpak's exports, or Plasma opens Discover instead of launching them, so the session sources `/etc/profile.d/flatpak.sh`.
|
||||
|
||||
The private runtime directory also moves the session's document portal to `$XDG_RUNTIME_DIR/frametop/doc`, and that broke saving and uploading in Flatpak apps. The file picker (xdg-desktop-portal 1.18.4 on SteamOS) gives a sandboxed app the host path of the file it picked, `/run/user/1000/frametop/doc/ID/NAME`. Inside the sandbox the portal is at `/run/flatpak/doc`, and `/run/user/1000` is a private per-app folder (`.flatpak/APP/xdg-run` in the runtime directory). So Brave created the missing folder there, "finished" the download into it, and the file vanished when the session cleaned up. The session script now links that path to `/run/flatpak/doc` in each installed app's folder before Plasma starts. Upstream xdg-desktop-portal fixed this after 1.22.1 (commit `69ba5e1`) by handing Flatpak apps `/run/flatpak/doc` paths, after which the links go unused.
|
||||
|
||||
A podman container's monitor process (conmon) stays in the cgroup of whatever started the container, and `distrobox enter` starts it on demand. When a Frametop service happened to start the `dev` container, stopping that service stopped the container and everything in it, including the desktop's compositor. `scripts/container-up.sh` starts the container in a systemd scope of its own before anything enters it. It then waits for distrobox-init to log `container_setup_done`, as `distrobox enter` does only for containers it starts itself. A new container's first start takes a minute or more (it installs distrobox's dependencies and sets up passwordless sudo), and an install that entered right away met a sudo password prompt with no terminal to answer it ([#9](https://github.com/DeeJanuz/frametop/issues/9)).
|
||||
|
||||
KWin renders with OpenGL through zink on Turnip, Vulkan on the same GPU vrcompositor needs to hit its frame time, and on the Frame that costs CPU too. The nested session started with KWin's defaults: blur and background contrast on (no `[Plugins]` group in its kwinrc) and animations at full length. Blur re-renders what's behind every translucent panel and menu each time it changes, and every animated frame is one more frame for KWin and ft-screens to draw and send. They're off by default in the Frametop desktop. The session script writes them before KWin starts, only where the desktop's own config has no value, once: System Settings deletes a setting put back to KDE's default rather than writing it, so without the marker in `frametoprc` a user who turned blur back on would lose it at the next start. The effect ids (`blur`, `contrast`) are the ones built into KWin 6.2.5 on SteamOS; KWin reads `<id>Enabled` from `[Plugins]`.
|
||||
|
||||
The nested session also runs the system's XDG autostart entries, being a KDE session. Discover's update notifier started `plasma-discover --mode update` in it (520 to 620 MB resident and about 9% of a core, plus `flatpak-system-helper` and AppStream downloads), and IBus started a daemon, the kimpanel panel and its GTK extension that nothing can use: KWin hands text input to the one input method it starts (`ft-textinput`), and the session drops `QT_IM_MODULE`, `GTK_IM_MODULE` and `XMODIFIERS`. Steam's entry (`steam -silent`, from steamdeck-kde-presets) reached the running Steam client as a command line it ran (`ExecCommandLine` in its console log), since the desktop starts from Steam; SteamOS 0.4 added `-vrdisable -deckard` to it, for Desktop Mode, where Plasma starts Steam itself. The session hides all three for this desktop only, with `Hidden=true` copies in its own autostart folder. The geoclue demo agent stays: it's what answers apps' location requests to Geoclue outside GNOME, and it costs nothing while idle. Orca's entry only starts in GNOME-family desktops.
|
||||
|
||||
Plasma 6.2.5 keeps each panel on a screen number (`lastScreen` in `plasma-org.kde.plasma.desktop-appletsrc`), and the numbers rank the enabled outputs by priority, so 0 is the primary screen. A panel whose number is past the screen count gets no view, and Plasma never moves it: the remap it runs at every start only moves a panel whose number has no desktop, and this desktop keeps a desktop for every output it has seen, spares included. So the taskbar was lost when the number of screens went down, and once it was found saved on a spare output, number 8 of a desktop with three screens ([#18](https://github.com/DeeJanuz/frametop/issues/18)). Before Plasma starts, the session runs `session/fix-panels.py`, which moves any panel numbered past the screen count, with its system tray's containment, to screen 0, keeping its widgets and settings. A panel stays put when screen 0 already has one on that edge, and comes back by itself if the screens do. Before each repair the file is backed up to `<file>.ft-bak.last`. `<file>.ft-bak` keeps it as it was before the first repair and is never overwritten. The repair writes over a moved panel's old screen number, so the backups are the only record of it, and `.ft-bak.last` also keeps everything changed since the first repair. Plasma's scripting can't do this while it runs (`panel.screen` is read-only in 6.2.5), so a lost taskbar comes back at the desktop's next start. `scripts/doctor.sh` and `scripts/report.sh` list the panels and their screens.
|
||||
|
||||
Remote desktop is a chain (krdp, then FreeRDP inside Xvnc) because nothing on SteamOS serves KWin over VNC directly. Kept connected all the time, it cost about a core with nobody watching: krdpserver 55 to 78% (it encodes H.264 in software with openh264: VA-API finds no driver for the Frame's GPU in the container), FreeRDP 16 to 27%, Xvnc 6 to 11%, and the bridge's layout check every 5 seconds another 4%. krdp creates its screencast session per RDP connection and drops it when the connection closes (`SessionController::onNewConnection` in krdp 6.7), so an idle krdpserver costs nothing and can stay up; only the RDP connection has to go. The bridge connects FreeRDP when a VNC client appears and disconnects 45 seconds after the last one leaves. Xvnc has no hook for its clients, so the bridge counts established connections to its port with `ss`, woken early by Xvnc's log output; looking with `ss` once a second cost about 0.9% of a core in bash, against about 0.1% this way. `Xvnc -inetd` from a systemd socket would start a server per connection and lose sharing between viewers. The layout check (`ft-layout remote-view`, which scans `/proc` for plasmashell and runs `kscreen-doctor -j`) now runs only while FreeRDP runs, and then only after `kwinoutputconfig.json` or `frametop-layout.json` changes, with one check a minute in case a change touched neither.
|
||||
|
||||
KWin 6.2.5 can crash while Plasma logs out (a SIGSEGV in its libei backend's teardown, with a Deskflow client connected), and `kwin_wayland_wrapper` then starts a fresh KWin with no Plasma left to end it: a black desktop that never goes away. The session script watches for this. It finds Plasma's session manager (`ksmserver`) once it has started, waits for it to exit with `waitpid`, and if the session's KWin wrapper is still running 9 seconds later, ends it, which ends the session. Looking for `ksmserver` with `pgrep` every 3 seconds instead cost about 1.3% of a core (38 ms a look). `scripts/update-check.py` has a retest hint for KWin updates: if logout with Deskflow no longer crashes, the watcher can go.
|
||||
A podman container's monitor process (conmon) stays in the cgroup of whatever started the container, and `distrobox enter` starts it on demand. When a Frametop service happened to start the `dev` container, stopping that service stopped the container and everything in it, including the desktop's compositor. `scripts/container-up.sh` starts the container in a systemd scope of its own before anything enters it.
|
||||
|
||||
Program names stay within 15 characters, because Linux truncates process names there and the scripts find programs with `pgrep -x` and `pkill -x`. That's why the prefix is `ft-`.
|
||||
|
||||
@@ -238,21 +149,6 @@ Movement is judged within 10-second windows. On the mount, the head pose jittere
|
||||
|
||||
Staying awake while charging uses Steam's own setting rather than a logind sleep inhibitor. Steam suspends with `dbus-send ... login1.Manager.Suspend boolean:true`, and a block inhibitor does stop that (`CanSuspend` answers "challenge" while one is held), but it stops the power button too. `system_idle_suspend_ac_sec` is field 24004 of Steam's CMsgClientSettings. In Steam's SharedJSContext, reachable over CDP on port 8080 because Steam runs with `-cef-enable-debugging`, `SteamClient.Settings.SetSetting` takes a change as a base64 protobuf, the way Steam's Power page sends it (0 is never), and `settingsStore.clientSettings` has the current values.
|
||||
|
||||
## Pausing for VR games
|
||||
|
||||
Hiding the screens during a game kept them out of view, but Frametop kept using the headset. Measured on 2026-10-02 with gaze mode off and no game running, in shares of one core: our eye tracker (ft-eyes) about 60%, ft-eyegrab, ft-gaze and ft-gazed about 3 to 4% each; remote desktop (krdpserver, FreeRDP, Xvnc) about 2 cores while it ran; KWin about 13%, ft-screens about 4%. The gaze service ran at full rate whether gaze mode was on or not; now it idles while the gaze isn't used (gaze/README.md), and pausing stops it outright. Reading SteamVR's eye tracking 90 times a second also made it restart every 10 to 13 seconds during Beat Saber, and each restart took input focus from the game, which paused it (PR #13; since then ft-gaze skips SteamVR's gaze action during games, but our own tracker kept running). So pausing stops what costs the most and leaves windows where they are.
|
||||
|
||||
- A hidden screen still cost as much as a visible one. ft-screens sent every committed screen its frame callback at 90 Hz whether its overlay showed or not (since then, a hidden screen always gets one a second; see the frame rates in reference.md), so KWin kept drawing, and its apps with it. Paused, ft-screens sends the callbacks once a second. A Wayland client draws again only after its last frame's callback, so KWin's output stalls, KWin's own clients stop getting theirs, and the whole desktop idles, without anything losing its connection. A second's pace, rather than none, keeps any client that waits on a callback from waiting forever. Stopping KWin or the apps with SIGSTOP would free the same, but a Wayland peer that stops reading overflows the other side's 4 KB socket buffer, which ends the connection: that's how the live desktop died once when its KWin stalled (`Data too big for buffer`). They also sit in different cgroups (KWin under steam.service when the VR launcher starts it, ft-screens in the dev container's), so no single freeze stops them together.
|
||||
- The relay does the pausing because it's the one part that always runs, and the pointer helper keeps running because stopping it leaves its virtual controller connected with its last pose (the driver has no staleness timeout), maybe holding a hand role, with the 3D mouse dead. Releasing it does the job. The helper already checks for a scene app twice a second, so it's what tells the relay a game started.
|
||||
- The gesture has to work during a game, but SteamVR input reaches only the app with input focus, and an overlay with global input (`steamvr/globalActionSetPriority`) takes the buttons it binds from the game. vrserver's web socket on 127.0.0.1:27062, which its controller binding page uses for the live view, reports every controller component whatever has focus, and reading it takes nothing. The game sees the clicks too, so the default is a gesture games hardly use: both thumbsticks, together, twice. "Together" means within 0.3 seconds of each other, so a stick held down to sprint while the other clicks doesn't count. The stream is about 160 messages a second, nearly all capacitive sensing, so the reader parses only the few that mention a gesture's button. A controller's root path changes while the 3D mouse holds its hand role (`/devices/cv/<serial>` instead of `/user/hand/right`), so the reader looks the controllers up again (an HTTP request to vrserver): when the relay's 3D mouse connects or lets go, when a message comes from a device it doesn't know, and every 30 seconds. It used to be every 3 seconds.
|
||||
- Resuming starts remote desktop through `systemd-run --scope`: started straight from the relay, it would join the relay's cgroup and end with the next relay restart.
|
||||
|
||||
## SteamOS updates
|
||||
|
||||
On the Frame, SteamVR is part of the OS image (`/opt/steamvr`, the `deckard-steamvr-rel` package), next to KWin, gamescope, and the kernel, so every SteamOS update can bring a new SteamVR too. Frametop survives updates: it lives in the home folder and the `dev` container, the Bluetooth fixes are in `/etc`, which SteamOS keeps across updates, and nothing goes into `/usr`. What an update can break is what Frametop uses from the image. The public OpenVR API is versioned and stays put. The rest is less certain: `IVRIPCResourceManagerClient`, which is newer than the header SteamVR ships; the text `vrcmd --overlays` prints; the eye tracker's shared memory layout; XRService's camera buffers; KWin's nested backend; and behavior Frametop works around, such as the SteamVR Settings page that `ComputeOverlayIntersection` can't find or the scale KWin's nested backend doesn't undo.
|
||||
|
||||
`scripts/update-check.py`, which `scripts/doctor.sh` runs, checks what it can directly: that SteamVR still serves every OpenVR interface version the installed programs were built against (read from the binaries), that `vrcmd`'s format still parses, that the eye tracker's shared memory still has a layout ft-gaze knows (SteamOS 0.3's, or 0.4's, with every field from the timestamp on 5 bytes later), by the same test ft-gaze uses to pick one, and the host files, services, sockets, and driver registration. Behavior can't be checked without someone in the headset, so it records the versions of the packages that matter once things work (`--mark-good`), and after an update names what changed and what to try by hand.
|
||||
|
||||
## Approaches we dropped
|
||||
|
||||
- WayVR, an existing Wayland desktop for VR. It built and connected to SteamVR on the Frame, but nothing showed in the headset. It has no bindings for the Frame's controllers, and its KDE screen capture needs `xdg-desktop-portal-kde`, which SteamOS doesn't ship.
|
||||
@@ -261,7 +157,9 @@ On the Frame, SteamVR is part of the OS image (`/opt/steamvr`, the `deckard-stea
|
||||
|
||||
## Open questions
|
||||
|
||||
- A head-locked screen, like a HUD.
|
||||
- A controller button that shows the screens during a game. Games own the controllers, so this needs SteamVR input actions for ft-screens.
|
||||
- Drawing KWin's cursor on the screens.
|
||||
- Plasma can lose its panels when the number of screens goes down, because they're saved against a screen that no longer exists. Removing `plasma-org.kde.plasma.desktop-appletsrc` and `plasmashellrc` from `~/.config/frametop` brings the default panels back.
|
||||
- Frame pacing and GPU cost with several busy screens haven't been measured.
|
||||
- Real standby on a stand, with rendering and tracking paused, not just the backlight off. SteamVR has no call for it, and its activity level follows the proximity sensor.
|
||||
+132
-93
@@ -1,79 +1,76 @@
|
||||
# Floating windows
|
||||
# Floating windows (plan)
|
||||
|
||||
Any desktop app can float in VR in a panel of its own, like SteamVR's floating windows, while it stays part of the Frametop desktop. Drag and drop, the clipboard, and focus keep working between floating windows and the screens.
|
||||
Status: design settled 2026-09-29 (see "Decisions"); being built on the `floating-windows` branch.
|
||||
|
||||
- **Float a window** with "Float in VR" in its window menu (Alt+F3), the float button left of Close in its title bar, or the float key (Meta+Shift+F by default). Start an app floating with "Launch as Standalone" in its right-click menu in the Application Launcher or the taskbar, with `ft-float launch` or `ft-float run`, or from a profile ([profiles.md](profiles.md)).
|
||||
- **Put it back** with the dock button under its panel, the title bar button, the float key, or "Back to Desktop" in the window menu. It returns to the screen, position, and size it came from. The `dock_all` action docks every floating window.
|
||||
- **Move it** by its title bar or the bar under its panel, **resize it** by its edges or the corner tab, and **change its scale** with Meta+scroll over it. Each app's last floating place, size, and scale are remembered.
|
||||
Built so far (2026-09-29; the KWin side tested on the headless test desktop, `screens/test/headless.sh`; nothing yet tried in the headset):
|
||||
|
||||
- The catcher (a release off every panel still reaches KWin), and the pointer helper's "up" backstop.
|
||||
- `float/frametop-float.js` (the KWin script), `float/ft-floatd`, and `float/ft-float`. "Float in VR" is in the window menu under Extensions, and Meta+Shift+F toggles the active window. Floating, docking (back where it came from), closing, full screen, per-window scale (Meta+scroll), popups reported with their rectangles, windows of a floating app floating too, and the notification when every spare is in use.
|
||||
- ft-screens: a panel per spare output (`frametop.float.N`) with the crop, density, popups and dialogs as small panels over it, title-bar carrying, the corner tab resizing the window, and dock and close buttons. `--spares`, and the commands `float`, `unfloat`, `pose`, `sub`, `minimized`, `carry`.
|
||||
- The session adds `FLOAT_SLOTS` spares and starts ft-floatd from the desktop's autostart; ft-layout leaves the spares alone.
|
||||
- The 3D mouse's drag lock crosses onto other Frametop panels (not while carrying one).
|
||||
|
||||
Not built yet: phase 2 (the ghost, tear-off by dragging, push-flush docking), phase 3 (launching floating, the Frametop Apps entry and picker, remembered placement), and phase 4. Frametop Apps (decision 10) needs a per-screen hide, which ft-screens doesn't have yet: its hide and show are for all screens.
|
||||
|
||||
The goal is to let any desktop app float in VR in a panel of its own, like SteamVR's floating windows, while it stays part of the Frametop desktop. That means drag and drop, the clipboard, and focus keep working between floating windows and the screens.
|
||||
|
||||
- There are two ways to get a floating window. Launch the app floating, or drag a desktop window by its title bar off a screen and let go in the air.
|
||||
- There are two ways to put one back. Push it flush against a screen and let go, or press its "back to desktop" button.
|
||||
- Files, text, and images drag between any two floating windows, and between floating windows and the screens.
|
||||
|
||||
How KWin behaves underneath all this, and what the KWin script does about it, is in [design.md](design.md#floating-windows).
|
||||
|
||||
## Not built
|
||||
|
||||
These were decided (see the table) but aren't built:
|
||||
|
||||
- Tearing a window off a screen by dragging its title bar into the air, with a ghost of it on the laser (decision 3; see "Tearing a window off a screen").
|
||||
- Docking by pushing a floating window flush against a screen, with the landing spot highlighted (decisions 4 and 17).
|
||||
- New windows of a floating app placed where that app's windows went last time, or to the parent's right (decision 15).
|
||||
- +/- scale buttons on the floating panel's bar (decision 18). Meta+scroll changes the scale.
|
||||
- The glow at the edge of your view toward a floating window activated out of sight, and the setting that brings it in front of you instead (decision 19).
|
||||
- A Floating windows section in Frametop Display Settings (decision 1). `FLOAT_SLOTS` and `FLOAT_MARGIN` are set in `~/.config/frametop.conf`.
|
||||
- A drag proxy on the catcher, so a drag's icon shows while the laser is between panels.
|
||||
- Keeping floating windows out of Show Desktop (Meta+D) (decision 9). Nothing handles it yet.
|
||||
|
||||
## Decisions
|
||||
|
||||
The numbers are cited in the code, so they stay as they are. Struck-out text was replaced by a later decision.
|
||||
Settled with the user on 2026-09-29. The sections below follow them.
|
||||
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| 1 | How many windows can float at once | 16 spare outputs by default (also the most), configurable with `FLOAT_SLOTS`; a change needs a desktop restart. A Display Settings control isn't built |
|
||||
| 1 | How many windows can float at once | 8 spare outputs by default, configurable (`FLOAT_SLOTS`, and Display Settings); a change needs a desktop restart |
|
||||
| 2 | Menus and dropdowns | Each floating output has a margin around the window. The panel shows only the window, and each open popup gets a small overlay of its own, cut from the same buffer |
|
||||
| 3 | Tearing off | Not built. Drag the title bar past a screen's edge and let go in the air, with a small dead zone past the edge |
|
||||
| 4 | Docking by dragging | Not built. Push the window flush against a screen (within about 10 cm), with the landing spot highlighted, and let go |
|
||||
| 3 | Tearing off | Drag the title bar past a screen's edge and let go in the air, with a small dead zone past the edge |
|
||||
| 4 | Docking by dragging | Push the window flush against a screen (within about 10 cm), with the landing spot highlighted, and let go |
|
||||
| 5 | Visibility | Floating windows follow the same rules as the screens: the hide hotkey, the visibility modes, and the games rule |
|
||||
| 6 | Windows a floating app opens | They float too |
|
||||
| 7 | Launching floating from the headset | ~~One "Frametop Apps" launcher entry with a picker~~ Replaced by 26 and profiles (27) |
|
||||
| 8 | Build order | Not kept here: it only set the order of the work |
|
||||
| 9 | Show Desktop (Meta+D) | Floating windows stay. Not built |
|
||||
| 10 | Frametop Apps and visibility | ~~The entry starts the desktop with each screen hidden on its own, so only floating windows show~~ Replaced: a profile can hide screens (27) |
|
||||
| 7 | Launching floating from the headset | One "Frametop Apps" launcher entry with a picker |
|
||||
| 8 | Build order | The catcher first, as a fix that stands on its own; `pointer-ignore` and `layouts-headpin` merged before phase 1; the hand cutouts stay out |
|
||||
| 9 | Show Desktop (Meta+D) | Floating windows stay |
|
||||
| 10 | Frametop Apps and visibility | The entry starts the desktop with each screen hidden on its own (the existing per-screen hide), so only floating windows show. No special mode |
|
||||
| 11 | Window frame | KWin's title bar and border stay. Frametop's bar, close, and "back to desktop" are extras |
|
||||
| 12 | Resizing | The window's own edges and Frametop's corner tab both change the size in pixels at the same density; the output follows |
|
||||
| 13 | Margin | 300 px on each side, configurable (`FLOAT_MARGIN`) |
|
||||
| 14 | All spares in use | The window stays on the screens, with a notification |
|
||||
| 15 | Where a floating app's new windows go | Not built: where that app's windows went last time, otherwise to the parent's right, curving around you. For now, a new window that opens on a floating window's output floats a little in front of it |
|
||||
| 16 | Where the code is written | Not kept here: it was about the work, not about Frametop |
|
||||
| 17 | Size when docked by dragging | Not built (see 4): the current floating size in pixels, shrunk to fit the screen |
|
||||
| 18 | Bigger text | A scale for each window (KWin's output scale): Meta+scroll over the window. Remembered for each app. +/- buttons on its bar aren't built |
|
||||
| 19 | Switching to a window you can't see | It's focused. Not built: a glow at the edge of your view that points to it, and a setting that moves it in front of you |
|
||||
| 13 | Margin | 300 px on each side, configurable |
|
||||
| 14 | All spares in use | The window opens on the screens, with a notification |
|
||||
| 15 | Where a floating app's new windows go | Where that app's windows went last time; otherwise to the parent's right, curving around you |
|
||||
| 16 | Where the code is written | A branch in the PC's clone of the repo |
|
||||
| 17 | Size when docked by dragging | The current floating size in pixels, shrunk to fit the screen |
|
||||
| 18 | Bigger text | A scale for each window (KWin's output scale): Meta+scroll over the window, or +/- on its bar. Remembered for each app |
|
||||
| 19 | Switching to a window you can't see | It's focused, and a glow at the edge of your view points to it. Moving it in front of you is a setting |
|
||||
| 20 | Full screen | The window fills its own panel. The margin drops to zero while it's full screen, and the panel keeps its size and place |
|
||||
| 21 | Named layouts | ~~They cover the screens only~~ Replaced by profiles (27). Outside a profile, floating windows use the placement remembered for each app |
|
||||
| 22 | The float key | One toggle: it floats a window, or docks it if it already floats. The input relay owns it (`float_toggle`), Meta+Shift+F by default, rebindable in Frametop Input Settings and mappable to mouse and controller buttons. KWin has no shortcut of its own for it, so one press can't fire twice |
|
||||
| 23 | Which window the key acts on | The window under the desktop's pointer; the active window if there's none there (the wallpaper, the taskbar) |
|
||||
| 24 | Docking everything | A `dock_all` action, with no default binding |
|
||||
| 25 | A button on every window | A float button left of Close in the title bar, from Frametop's own QML window decoration, made to look like Breeze. It shows a dock icon on floating windows. Apps that draw their own title bar (Chromium, Electron, GTK) use the key |
|
||||
| 26 | Launching one app floating | "Launch as Standalone" in the right-click menu of every app in the Application Launcher and the taskbar, from copies of the apps' desktop files that only the Frametop desktop reads. It replaces the Frametop Apps entry (7, 10) |
|
||||
| 27 | Profiles | Named layouts become profiles: the screens' places, which screens show, and the apps and their windows, floating or not. See `docs/profiles.md` |
|
||||
| 21 | Named layouts | They cover the screens only. Floating windows use the placement remembered for each app |
|
||||
|
||||
Also: floating windows get the wrist pin, the head pin, and pass-through (`pointer-ignore`) like screens. Every gesture works with the controllers as well as the 3D mouse. A window launched floating uses the primary screen's density. VNC shows only the primary screen.
|
||||
Also assumed: floating windows get the wrist pin, the head pin, and pass-through (`pointer-ignore`) like screens. Every gesture works with the controllers as well as the 3D mouse. A window launched floating uses the primary screen's density. VNC shows only the primary screen, as now. Anything that restarts the live desktop waits for the user's OK.
|
||||
|
||||
## The approach: each floating window gets a KWin output of its own
|
||||
|
||||
Drag and drop and the clipboard only work between windows of the same compositor. A Wayland window can't move from one compositor to another. So a floating window has to stay a KWin window.
|
||||
|
||||
ft-screens already shows each KWin output as a panel. It sets the output's size with an `xdg_toplevel` configure, and KWin resizes the output to match. So a floating window gets an output of its own, sized to fit it, and ft-screens shows that output as a panel with its own controls. To KWin this is an ordinary desktop with more monitors. Dragging between two floating windows is the same as dragging between two monitors, which KWin already handles. ft-screens moves the pointer between panels in the middle of a drag: `handle_vr_event` moves pointer focus to another KWin window even while a button is held.
|
||||
ft-screens already shows each KWin output as a panel. It sets the output's size with an `xdg_toplevel` configure, and KWin resizes the output to match. So a floating window can get an output of its own, sized to fit it, and ft-screens shows that output as a panel with its own controls. To KWin this is an ordinary desktop with more monitors. Dragging between two floating windows is the same as dragging between two monitors, which KWin already handles. ft-screens already moves the pointer between panels in the middle of a drag: `handle_vr_event` moves pointer focus to another KWin window even while a button is held.
|
||||
|
||||
Alternatives considered:
|
||||
Alternatives we considered:
|
||||
|
||||
- **Run floating apps directly on ft-screens.** It's a wlroots compositor, so apps could connect to it and get a panel per window. But they would get no drag and drop or clipboard with desktop apps without a bridge. Also, a window that's already on the desktop could never float, because a Wayland client can't change compositors. Rejected.
|
||||
- **One large hidden "canvas" output.** Every floating window would sit on one big output, and each panel would show a crop of it (`SetOverlayTextureBounds`). That needs only one extra output, with no copies. But an 8K canvas uses about 128 MB per buffer, with two or three buffers in KWin's swapchain. It would also have to repack windows whenever one resized, full screen would fill the whole canvas, and every window would share one scale. It was the fallback in case per-window outputs didn't work.
|
||||
- **Run floating apps directly on ft-screens.** It's a wlroots compositor, so apps could connect to it and get a panel per window. But they would get no drag and drop or clipboard with desktop apps unless we wrote a bridge. Also, a window that's already on the desktop could never be torn off, because a Wayland client can't change compositors. Rejected.
|
||||
- **One large hidden "canvas" output.** Every floating window would sit on one big output, and each panel would show a crop of it (`SetOverlayTextureBounds`). That needs only one extra output, with no copies. But an 8K canvas uses about 128 MB per buffer, with two or three buffers in KWin's swapchain. It would also have to repack windows whenever one resized, full screen would fill the whole canvas, and every window would share one scale. This is the fallback if per-window outputs don't work.
|
||||
- **Screencast single windows** (`zkde_screencast` `stream_window`, over PipeWire). This adds copies and latency, and the window still needs a real place in KWin's layout to receive input. Rejected.
|
||||
- **SteamOS's own floating windows** (Launch a program from the dashboard). Those apps run in gamescope, outside KWin, so they can't drag and drop with the desktop.
|
||||
|
||||
### Where the extra outputs come from: spare outputs
|
||||
|
||||
KWin's nested backend opens its outputs at start (`--output-count`). The session starts KWin with the screen count plus `FLOAT_SLOTS` outputs (default and most 16). ft-floatd turns off the spares nothing floats on with `kscreen-doctor` once it starts. Floating a window enables a spare, and docking the window disables it again. `FLOAT_SLOTS` limits how many windows can float at once, and changing it means restarting the desktop. How KWin's nested backend treats disabled outputs, and why its virtual outputs can't be used instead, is in [design.md](design.md#floating-windows).
|
||||
KWin's nested backend opens its outputs at start (`--output-count`). The session starts KWin with the screen count plus `FLOAT_SLOTS` outputs (default 8). Each spare is disabled until it's needed, with `kscreen-doctor` (or in the session's `kwinoutputconfig.json`, so it starts disabled). Floating a window enables a spare, and docking the window disables it again. `FLOAT_SLOTS` limits how many windows can float at once, and changing it means restarting the desktop.
|
||||
|
||||
Checked in KWin 6.2.5's source (`src/backends/wayland/`, 2026-09-29):
|
||||
|
||||
- Disabling a nested output keeps its host window. `Output::applyChanges` only flips `enabled`, KWin stops rendering it, and Plasma drops its desktop view. So ft-screens keeps the same toplevel, and its screen numbers stay put.
|
||||
- Each output's host window is titled `KDE Wayland Compositor WL-<n>`, with `- Output disabled` appended while it's disabled (`WaylandOutput::updateWindowTitle`, on every `enabledChanged`). ft-screens reads the title to tell screens (`WL-0` to `WL-<SCREENS-1>`) from spares, and to see a spare turn on and off.
|
||||
- Pointer positions reach KWin only through motion events: the output's position in the layout plus the position on its window. When ft-screens stops sending motion, KWin's pointer stays put.
|
||||
- **Virtual outputs don't work.** `createVirtualOutput` makes an output window but never adds it to the backend's `m_outputs`, so `findOutput()` returns null when the pointer enters it, and the next line dereferences it (`Q_ASSERT` is compiled out). A click on such a panel would crash KWin. This rules out the virtual-output fallback (`stream_virtual_output`) without a patched KWin.
|
||||
|
||||
ft-screens creates a `screen` for each toplevel in the order they appear, and indexes its settings by that order. Spares come after the screens, so they get indices `SCREENS` and up. Their panels are hidden while their output is disabled.
|
||||
|
||||
@@ -82,38 +79,35 @@ ft-screens creates a `screen` for each toplevel in the order they appear, and in
|
||||
```
|
||||
KWin script "frametop-float" ft-floatd (host, Python) ft-screens
|
||||
window events, moves, menus ── D-Bus ──▶ window ↔ output ↔ panel table ── @ft_screens ──▶ panels, controls,
|
||||
runs commands ◀─ long poll ─ spare outputs (kscreen-doctor) ◀─ @frametop_float ─ lasers, catcher
|
||||
runs commands ◀─ long poll ─ spare outputs (kscreen-doctor) ◀─ @frametop_float ─ lasers, ghost, catcher
|
||||
```
|
||||
|
||||
- **KWin script `frametop-float`** (`float/frametop-float.js`). ft-floatd loads it into the desktop's KWin over D-Bus (`org.kde.kwin.Scripting`). A script keeps working across KWin updates. A C++ effect would have to match the host's exact KWin build, and the build container is Fedora, not SteamOS. The script watches windows (`windowAdded`/`windowRemoved`, `frameGeometryChanged`, `outputChanged`, `interactiveMoveResizeStarted`/`Finished`, `fullScreenChanged`, `maximizedChanged`, `minimizedChanged`, `keepBelowChanged`, `windowActivated`) and the outputs (`screensChanged`). It runs commands: move a window to an output, set its geometry, put it on all virtual desktops, and restore it. It adds "Float in VR" ("Back to Desktop" on a floating window) to the window menu (`registerUserActionsMenu`). It registers no shortcut: the float key belongs to the input relay. KWin scripts can call D-Bus but can't serve it, so commands come back through a long poll. The script calls ft-floatd's `NextCommand`, which answers when a command is ready, and then the script calls it again, or after 30 seconds without an answer. It also keeps KWin's placement memory from moving windows (see design.md).
|
||||
- **ft-floatd** (`float/ft-floatd`, Python). The host has dbus-python and PyGObject. It runs inside the desktop's Plasma session, started from its autostart. It owns `org.frametop.Float` on the session's private bus, and it keeps the table of which window is on which output and panel. It enables and disables spare outputs and sets their scale and position with `kscreen-doctor`, and their size through ft-screens. It tells ft-screens where each floating window goes and tells the script which window goes where. It launches apps floating, opens profiles' apps, and remembers each app's placement and scale, keyed by desktop file name. Commands come in on `@frametop_float`, from `ft-float`, the input relay, and ft-screens.
|
||||
- **ft-screens.** A spare output's panel is a floating window's. Floating panels get the same bar, curve, roll, resize tab, and wrist and head pins as screens, plus dock and close buttons left of the bar. Other parts: the catcher, popup and dialog overlays, and carrying a panel during a KWin move. `MAX_SCREENS` (screens and spares together) is 24. Commands arrive on `@ft_screens`. Events go out to `@frametop_float` from an unbound socket, the same way ft-screens talks to the input relay.
|
||||
- **Session script.** Adds `FLOAT_SLOTS` to KWin's output count, starts ft-floatd from the desktop's autostart, installs Frametop's window decoration and chooses it in the session's `kwinrc`, and writes the Launch as Standalone copies of the apps' desktop files.
|
||||
- **ft-layout.** Arranges only the screens' outputs, and leaves the spares (`WL-<SCREENS>` and up) to ft-floatd, enabled or not.
|
||||
- **ft-pointer.** The drag lock crosses onto other Frametop panels (see "Drag and drop between panels"), and a left release also goes to ft-screens as a backstop for the catcher.
|
||||
- **Input relay.** Owns the float key: `float_toggle` and `dock_all` send `float pointer` and `dock all` to ft-floatd.
|
||||
- **KWin script `frametop-float`** (JavaScript). A script keeps working across KWin updates. A C++ effect would have to match the host's exact KWin build, and our build container is Fedora, not SteamOS. The script watches windows (`windowAdded`/`windowRemoved`, `interactiveMoveResizeStarted`/`Stepped`/`Finished`, `outputChanged`, `minimizedChanged`, `windowActivated`, `fullScreenChanged`). It runs commands: move a window to an output, set its geometry, put it on all virtual desktops, and restore it. It adds "Float in VR" to the window menu (`registerUserActionsMenu`) and registers a shortcut (`registerShortcut`, Meta+Shift+F). KWin scripts can call D-Bus but can't serve it, so commands come back through a long poll. The script calls ft-floatd's `NextCommand`, which answers when a command is ready, and then the script calls it again. The fallback is loading one-shot scripts through `org.kde.kwin.Scripting`, the way kdotool does. All of these API names are present in the host's KWin 6.2.5.
|
||||
- **ft-floatd** (Python). The host has dbus-python and PyGObject. It owns `org.frametop.Float` on the session's private bus, and it keeps the table of which window is on which output and panel. It enables and disables spare outputs and sets their size, scale, and position with `kscreen-doctor`, as ft-layout does. It tells ft-screens where each floating window goes and tells the script which window goes where. It also remembers each app's placement and scale, keyed by desktop file name.
|
||||
- **ft-screens.** `Screen` becomes a panel with a kind: screen or floating window. Floating panels get the same bar, curve, roll, resize tab, and wrist and head pins, plus close, "back to desktop", and scale buttons. New parts are the tear-off ghost, the catcher, popup overlays, carrying a panel during a KWin move, and the dock target highlight. `MAX_SCREENS` goes from 8 to 16. Commands arrive on `@ft_screens`. Events go out to `@frametop_float` from an unbound socket, the same way ft-screens talks to the input relay.
|
||||
- **Session script.** Adds `FLOAT_SLOTS` to the output count, starts ft-floatd, and enables the KWin script in the session's `kwinrc`.
|
||||
- **ft-layout.** Arranges only the screens' outputs. Today it arranges everything in `kscreen-doctor -j`, so it has to skip the spares (`WL-<SCREENS>` and up), enabled or not.
|
||||
- **ft-pointer.** Changes to the drag lock (see "Drag and drop between panels").
|
||||
- **Frametop Display Settings.** Gets a Floating windows section: slots, margin, and "bring a window in front of you when it's activated".
|
||||
|
||||
Program names stay within 15 characters (`ft-floatd`). Overlay keys are `frametop.float.N` and `frametop.float.N.bar`, and so on; a floating window's popups and dialogs are `frametop.float.N.sub.K`.
|
||||
Program names stay within 15 characters (`ft-floatd`). Overlay keys are `frametop.float.N` and `frametop.float.N.bar`, and so on.
|
||||
|
||||
## A floating window
|
||||
|
||||
- **Output and margin.** Its output is the window's frame plus a margin on each side (`FLOAT_MARGIN`, default 300 px). KWin keeps a Wayland popup inside its parent's output, so the margin gives menus and dropdowns room past the window's edges. X11 apps place their own menus within the monitor, so the same applies. Enabled spares sit apart from the screens and from each other in KWin's layout, so nothing spills from one to the next. Memory: a 1600 × 1000 window with a 300 px margin is about 14 MB per buffer, 42 MB for three.
|
||||
- **What the panel shows.** Only the window's frame: ft-screens crops the output's buffer with `SetOverlayTextureBounds` and maps mouse positions through the crop. Each open popup or dialog gets a small overlay of its own, cut from the same buffer and placed a few millimetres in front of the window, so the main panel never changes size. KWin tells scripts about popups as windows of their own (`windowAdded` with `popupWindow`), so the script reports their rectangles.
|
||||
- **Where it appears.** Floated from a screen, the panel starts where the window was on that screen, 30 cm in front of it. Launched floating, it goes where that app last floated, or in front of you, 0.8 to 2 m away.
|
||||
- **Size and scale.** The panel's width is the window's pixel width times the source screen's metres per pixel, so text stays the same size in VR. A window launched floating uses the primary screen's density. Each window also has a scale (KWin's output scale), changed with Meta+scroll over the window in steps of 10% and remembered for each app. A bigger scale makes the content bigger at the same panel size.
|
||||
- **Output and margin.** Its output is the window's frame plus a margin on each side (`FLOAT_MARGIN`, default 300 px). KWin keeps a Wayland popup inside its parent's output (`XdgPopupWindow::updateRelativePlacement` uses the output's placement area), so the margin gives menus and dropdowns room past the window's edges. X11 apps place their own menus within the monitor, so the same applies. Enabled spares sit apart from the screens and from each other in KWin's layout, so nothing spills from one to the next. Memory: a 1600 × 1000 window with a 300 px margin is about 14 MB per buffer, 42 MB for three.
|
||||
- **What the panel shows.** Only the window's frame: ft-screens crops the output's buffer with `SetOverlayTextureBounds` and maps mouse positions through the crop. Each open popup gets a small overlay of its own, cut from the same buffer and placed a few millimetres in front of the window, so the main panel never changes size. KWin tells scripts about popups as windows of their own (`windowAdded` with `popupWindow`), so the script reports their rectangles.
|
||||
- **Size and scale.** The panel's width is the window's pixel width times the source screen's metres per pixel, so text stays the same size in VR. A window launched floating uses the primary screen's density. Each window also has a scale (KWin's output scale), changed with Meta+scroll over the window or +/- on its bar and remembered for each app. A bigger scale makes the content bigger at the same panel size.
|
||||
- **Window state.** An ordinary window, not maximized, placed inside its output with the margin around it, and set to show on all virtual desktops. It keeps its title bar and border. Apps that draw their own title bar (GTK, Chromium) keep theirs.
|
||||
- **Moving.** Press the title bar. KWin starts an interactive move on the press itself, before any motion, so ft-screens stops forwarding pointer motion to KWin as soon as a press lands in a floating window's title bar (from the frame and client rectangles ft-floatd sends it). KWin's pointer stays at the press point and the window moves by nothing. For apps that draw their own title bar, the script reports the move and ft-screens stops then (`carry`); ft-floatd puts back any few pixels the window slipped before that. Meanwhile ft-screens carries the panel with the pressing device, the same way the bar does: it follows rigidly, scroll pushes and pulls, and the 3D mouse's right-drag tilts. When the button comes up, KWin gets the release at the press point. The bar under the panel works too.
|
||||
- **Resizing.** The window's own edges (inside the margin, so KWin's resize works as on the desktop) and Frametop's corner tab both change the window's size in pixels at the same density, so the app lays itself out again. ft-floatd resizes the output to keep the margin, and the panel grows or shrinks around the window's top-left corner. KWin ends a resize by the window's edge whenever an output changes, so during one the panel follows the window and the output follows only when the drag ends: the margin is the room to grow until then. A screen's tab only scales the panel. Resizing is throttled to about 20 updates a second, with a minimum of 320 × 200, like screens.
|
||||
- **KWin's placement memory.** KWin puts windows back where they were for each layout of the outputs it has seen, which fights spare outputs that follow their windows' sizes. The KWin script undoes it ([design.md](design.md#kwins-placement-memory)).
|
||||
- **Moving.** Press the title bar. KWin starts an interactive move and the script reports it. ft-screens then stops forwarding pointer motion to KWin, so KWin's pointer stays at the press point and the window moves by nothing. Meanwhile ft-screens carries the panel with the pressing device, the same way the bar does today: it follows rigidly, scroll pushes and pulls, and the 3D mouse's right-drag tilts. When the button comes up, KWin gets the release at the press point. The bar under the panel works too.
|
||||
- **Resizing.** The window's own edges (inside the margin, so KWin's resize works as on the desktop) and Frametop's corner tab both change the window's size in pixels at the same density, so the app lays itself out again. ft-floatd resizes the output to keep the margin, and the panel grows or shrinks around the window's top-left corner. A screen's tab only scales the panel. Resizing is throttled to about 20 updates a second, with a minimum of 320 × 200, like screens.
|
||||
- **Full screen.** The window fills its own panel: the margin drops to zero while it's full screen, and the output is the panel's size in pixels. The panel keeps its size and place. On leaving full screen, the margin comes back.
|
||||
- **Buttons.** The close button closes the window. The dock button docks it where it came from.
|
||||
- **Buttons.** The close button closes the window. "Back to desktop" docks it where it came from.
|
||||
- **Minimize.** Minimizing, from the title bar or the taskbar, hides the panel, and restoring it shows the panel again. Floating windows stay in the desktop's taskbar and in Alt+Tab.
|
||||
- **New windows.** Popups and dialogs of a floating window (`transientFor`) show as small overlays over it. Another window of a floating app that opens on its output floats too, a little in front of it. Any other window that opens on a floating window's output goes to the first screen that shows. When every spare is in use, the window stays on the screens and a notification says so.
|
||||
- **On a hidden screen.** A new window that opens on a screen hidden on its own floats instead, where that app last floated or in front of you.
|
||||
- **Activated out of view.** When a floating window is activated (taskbar, Alt+Tab, a notification) and it's more than about 60° from where you're looking, it's focused and a glow at the edge of your view points to it. A setting moves it in front of you instead.
|
||||
- **New windows.** A dialog of a floating window (`transientFor`) floats in front of its parent. Other new windows of a floating app float too: where that app's windows went last time, otherwise to the parent's right at the same distance, curving around you, and to its left if that's taken. When every spare is in use, the window opens on the screen used last and a notification says so.
|
||||
- **Show Desktop.** Meta+D leaves floating windows alone.
|
||||
|
||||
## Tearing a window off a screen (not built)
|
||||
|
||||
The design for decision 3:
|
||||
## Tearing a window off a screen
|
||||
|
||||
1. Press a desktop window's title bar and drag it. KWin starts a move, and the script tells ft-floatd, which tells ft-screens: `move-start <output> <window> <rect>`.
|
||||
2. While the button is held, the laser leaves every Frametop panel by more than a small dead zone (a few centimetres past the edge). Letting go inside the dead zone is an ordinary drop.
|
||||
@@ -123,48 +117,93 @@ The design for decision 3:
|
||||
|
||||
## Putting it back
|
||||
|
||||
- **Button.** The dock button returns the window to the screen, position, and size it had before it floated. If that screen is hidden now, it goes onto the first screen that shows.
|
||||
- **Dragging (not built).** Carry the floating window, by its title bar or its bar, until the spot you're pointing at is on a screen. Then push it flush with the screen, within about 10 cm of its surface: scroll away with the mouse, or move the controller forward. The screen shows where the window will land, and letting go docks it there at its current size in pixels, shrunk to fit if the screen is smaller. A carried panel keeps its distance, so moving a floating window in front of a screen never docks it by accident.
|
||||
- Docking disables the output and hides the panel.
|
||||
|
||||
## Getting at it: the float key and the title bar button
|
||||
|
||||
Decisions 22 to 25.
|
||||
|
||||
- **The float key.** The input relay owns it: the action `float_toggle`, bound to Meta+Shift+F unless the rules file says otherwise (a rules file with no `key_bindings` gets that default; one with its own list, even an empty one, doesn't). It can be rebound or removed in Frametop Input Settings, and mapped to a mouse button or a Frame controller button like any other action. The relay takes the combination before it reaches the desktop and sends `float pointer` to ft-floatd, which asks the script for the window under KWin's pointer (`workspace.cursorPos`, top of `workspace.stackingOrder`, popups and dialogs counting as their parent). With none there, the wallpaper or the taskbar, it's the active window. KWin's pointer is where the 3D mouse or a laser last was on a Frametop panel. A window that floats docks; any other floats. The KWin script has no shortcut of its own (ft-floatd removes one that an older script registered), so one press can't float a window and dock it again.
|
||||
- **Docking everything.** `dock_all` (no default binding) sends `dock all`, which docks every floating window where it came from.
|
||||
- **The title bar button.** Breeze can't take a button of its own, and a C++ fork of it would have to match SteamOS's exact KDecoration build (Plasma 6.3 replaces KDecoration2 with KDecoration3). So the Frametop desktop gets its own window decoration, written in QML for KWin's Aurorae engine, which loads it without compiling (`decoration/`, installed to `~/.local/share/kwin/decorations/kwin4_decoration_qml_frametop`, chosen in the session's `kwinrc` only, so Desktop Mode keeps Breeze; `decoration/apply.sh` switches the running desktop to it or back to Breeze). It's drawn to look like Breeze, with a float button left of Close. The button calls `requestToggleKeepBelow()`, the one window request a decoration can make that has no visible effect here, and the KWin script reads the change: keep-below set on a window on the screens floats it, cleared on a floating window docks it. The script keeps keep-below set on every floating window, however it was floated, so the button shows its dock icon there. A window alone on its own output loses nothing by being kept below (only the wallpaper is under it). If the window can't float (every spare is in use), the script clears the flag again. Keep Below Others in a window's menu does the same as the button.
|
||||
- **Apps that draw their own title bar** (Chromium and Electron apps, GTK apps) never show KWin's decoration, so they don't get the button. They use the key, or the window menu (Alt+F3).
|
||||
- **Button.** "Back to desktop" returns the window to the screen, position, and size it had before it was torn off (saved at tear-off).
|
||||
- **Dragging.** Carry the floating window, by its title bar or its bar, until the spot you're pointing at is on a screen. Then push it flush with the screen, within about 10 cm of its surface: scroll away with the mouse, or move the controller forward. The screen shows where the window will land, and letting go docks it there at its current size in pixels, shrunk to fit if the screen is smaller. A carried panel keeps its distance, so moving a floating window in front of a screen never docks it by accident.
|
||||
- Docking disables the output and removes the panel.
|
||||
|
||||
## Launching an app floating
|
||||
|
||||
- **In the desktop.** Use "Float in VR" in any window's menu, its title bar button, or the float key.
|
||||
- **From the menu.** Right-click an app in the Application Launcher, or in the taskbar (where it starts another window of that app), and pick "Launch as Standalone" (decision 26). The launcher has no way to add an entry to every app's menu, but its menu shows each app's own desktop actions. So the Frametop desktop reads copies of the apps' desktop files with one more action added (`float/ft_apps.py`). They're written to `~/.local/share/frametop/apps/applications` from every desktop file in `XDG_DATA_DIRS`: by the session script before Plasma starts, and by ft-floatd whenever an app is installed, changed, or removed. The session puts `~/.local/share/frametop/apps` first in `XDG_DATA_DIRS`. Plasma's app cache is keyed by those directories, so Desktop Mode never sees the copies. Desktop files in `~/.local/share/applications` come before every data dir, so an app you've customized there keeps your copy and has no Launch as Standalone. The action runs `ft-float launch <desktop file name>`.
|
||||
- **From a command.** `ft-float run <command>` and `ft-float launch <app.desktop>` start an app and float its first window. ft-floatd records the process it started, and new windows are matched by PID, including child processes. Some single-instance apps (Firefox, D-Bus-activated apps) open the window from a process that was already running. Those are matched by desktop file name. Either way, the window has to show up within 30 seconds.
|
||||
- **From the headset with the desktop off.** A profile's launcher entry starts the desktop in that profile, and a profile can hide every screen and hold only floating apps (`docs/profiles.md`). There's no separate Frametop Apps entry or picker.
|
||||
- **Remembered placement.** Each app's last floating pose (relative to the primary screen's panel, so it moves with the screens' layout), size in pixels, and scale, keyed by desktop file name, in `~/.config/frametop-float.json`. It's kept whenever one of the app's windows stops floating. With nothing remembered, the window opens in front of you, at the primary screen's density, 0.8 to 2 m away. A profile's own placement wins when the profile opens the app.
|
||||
- **In the desktop.** Use "Float in VR" in any window's menu, or press Meta+Shift+F for the active window.
|
||||
- **From a command.** `ft-float run <command>` and `ft-float launch <app.desktop>` start an app and float its first window. ft-floatd records the process it started, and the script matches new windows by PID, including child processes. Some single-instance apps (Firefox, D-Bus-activated apps) open the window from a process that was already running. Those are matched by desktop file name within a few seconds, or by `XDG_ACTIVATION_TOKEN` where the app honors it.
|
||||
- **From the headset without the desktop open.** One new launcher entry, "Frametop Apps". It starts the Frametop session with each screen hidden on its own (the per-screen hide that already exists), and opens an app picker as a floating window. Picking an app launches it floating. The hide hotkey and visibility modes still apply to everything, and a screen comes back with one click. Everything runs in one session, so dragging between a standalone app and a desktop app works. If the desktop is already running, the entry just opens the picker.
|
||||
- **The picker.** Either KRunner, floated, or a small Kirigami app like the settings apps, with a grid of apps and their icons.
|
||||
- **Remembered placement.** Each app's last floating pose, size, and scale, keyed by desktop file name. Named layouts don't include floating windows.
|
||||
|
||||
## Drag and drop between panels
|
||||
|
||||
KWin handles the protocols: Wayland, X11 through Xwayland, and the portal's file transfer. Frametop has to get the pointer right between panels.
|
||||
|
||||
- **Crossing panels.** When the laser moves onto another panel mid-drag, ft-screens gives that panel's KWin window pointer focus. KWin puts its cursor at that output's position, and the drop target gets enter and motion events. Floating windows add nothing new here, but they make gaps between panels the normal case.
|
||||
- **Gaps (the catcher).** While the laser is between panels, none of Frametop's overlays get its events, and ft-screens clears pointer focus on `FT_LEAVE` even with a button held. A release in empty space would never reach KWin, and the drag or move would stay stuck until the next click. So while a button is held on a Frametop panel and the laser leaves all of them, ft-screens puts an invisible catcher overlay on the laser. A release on the catcher releases in KWin wherever the pointer last was. Dropping in a gap cancels, just as dropping outside any window does. The pointer helper also tells ft-screens when the mouse's left button comes up ("up"), in case the catcher misses it. This also covers window moves and drags on the screens that end off a panel.
|
||||
- **The 3D mouse's drag lock.** While the button is held, the drag lock keeps the cursor at its distance and stops hit tests, so a drag onto a nearer panel would pass behind it. So while the button is held, the helper keeps testing the other Frametop panels (not the one pressed on, and not while carrying one) and moves onto a panel the ray meets. Off the edge of the pressed panel, it keeps that panel's plane, so moves and resizes past the edge still work.
|
||||
- **Drag icon.** KWin 6 draws the drag icon as part of its scene, on the output its pointer is on. In a gap it stays at the source panel's edge.
|
||||
- **Flatpak apps.** Dropping files into a sandboxed app goes through the document portal, the same path that the session script's file-picker fix covers ([design.md](design.md#the-desktop-session)).
|
||||
- **Gaps (the catcher).** While the laser is between panels, none of our overlays get its events, and ft-screens clears pointer focus on `FT_LEAVE` even with a button held. If you let go in empty space, the release never reaches KWin, and the drag or move stays stuck until the next click. The fix: while a button is held on a Frametop panel and the laser leaves all of them, ft-screens puts an invisible catcher overlay on the laser (the tear-off ghost is the same thing with a picture on it). A release on the catcher releases in KWin wherever the pointer last was. Dropping in a gap cancels, just as dropping outside any window does. The pointer helper also tells ft-screens when the mouse's left button comes up, in case the catcher misses it. This also fixes window moves and drags that end off a panel today.
|
||||
- **The 3D mouse's drag lock.** While the button is held, the drag lock keeps the cursor at its distance and stops hit tests. So a drag onto a nearer panel passes behind it, and a farther panel works only if SteamVR's laser happens to reach it. The change: while the button is held, keep testing the other panels (not the one pressed on) and move onto a panel the ray meets. Off the edge of the pressed panel, keep that panel's plane, as now, so moves and resizes past the edge still work.
|
||||
- **Drag icon.** KWin 6 draws the drag icon as part of its scene, so it should show on the panel under the laser (to check). In a gap it stops at the source panel's edge. A later version can show a small drag proxy on the catcher.
|
||||
- **Flatpak apps.** Dropping files into a sandboxed app goes through the document portal, the same path that the session script's file-picker fix covers. Test it explicitly, for example Dolphin to Brave.
|
||||
|
||||
## Things that must keep working
|
||||
|
||||
- **Typing follows the last click.** A click on a floating panel counts as a click on the desktop, since the window is a KWin window.
|
||||
- **Visibility.** Floating windows follow the screens' rules: the hide hotkey, the visibility modes, and hiding during a VR game unless the dashboard is open. Controllers' lasers are off in games.
|
||||
- **Headset standby.** Nothing new may poll SteamVR with new clients, so no new `vrcmd` loops.
|
||||
- **The pointer helper's overlay list.** The helper learns about overlays by running `vrcmd --overlays` in the background, so a new floating panel appears in its next listing.
|
||||
- **The pointer helper's overlay list.** The helper learns about overlays by running `vrcmd --overlays` in the background, so a new floating panel appears in its next listing. Check the delay after a tear-off. If it's too long, ft-screens can send the helper new overlay keys directly.
|
||||
- **Plasma.** An enabled floating output gets a desktop view (wallpaper) under its window, hidden by the crop. Plasma doesn't add panels to new outputs by default. A floating output must never become primary. With spare outputs, the output count stays the same, which avoids the lost-taskbar problem in design.md's open questions.
|
||||
- **Restarting the desktop** closes every window, floating ones included. Each app's placement is remembered, so an app launched floating again comes back where it was.
|
||||
|
||||
## Plan
|
||||
|
||||
### Step 1: the catcher and the branches
|
||||
|
||||
- The catcher (see "Gaps"), as a fix that stands on its own, so it can go to `main` by itself.
|
||||
- Merge `pointer-ignore` and `layouts-headpin`.
|
||||
|
||||
### Phase 0: find out
|
||||
|
||||
Each item has a pass condition. Items that need a desktop restart with extra outputs wait until the headset is free, or run in the headless test mode from 0.1.
|
||||
|
||||
- **0.1 Headless test mode.** `ft-screens --no-vr` runs the compositor without SteamVR. It logs toplevels, titles, and sizes, and answers commands on a separate control socket name. This lets KWin and output experiments run without the headset, and without touching the running desktop.
|
||||
- **0.2 Outputs.** Start KWin with spare outputs, then disable and re-enable one with `kscreen-doctor`. Pass: the toplevel stays and its title changes (as the source says), a disabled output costs no frames, resizing a spare through configure works, and outputs with gaps between them are accepted. Also: an output larger than its window with the window placed inside, and a popup placed in the margin.
|
||||
- **0.3 KWin script API on 6.2.5.** Move signals fire for moves from both KWin's title bars and apps' own. `sendClientToScreen` and `frameGeometry` work on another output, the `callDBus` long poll works, and `registerUserActionsMenu` works. Popups show up in `windowAdded` with their geometry. The observers can load into the running desktop through `org.kde.kwin.Scripting` and move nothing, so this is safe while the headset is in use.
|
||||
- **0.4 Frozen-pointer move.** Pass: a KWin move with no pointer movement doesn't shift the window.
|
||||
- **0.5 SteamVR's laser.** Find out which overlay gets MouseMove and ButtonUp when a held laser moves from overlay A to overlay B, and when it's released over nothing, for both a controller and the 3D mouse. Pass: an interactive overlay placed on the laser reliably catches the release.
|
||||
- **0.6 Texture bounds.** Pass: `SetOverlayTextureBounds` crops a DMA-BUF (`SharedTextureHandle`) overlay correctly, mouse positions map to the cropped area, and two overlays can show different crops of one buffer.
|
||||
|
||||
#### Results (2026-09-29, headless, KWin 6.2.5)
|
||||
|
||||
`screens/test/headless.sh` runs these: ft-screens `--no-vr` with a bare nested KWin next to the running desktop, with an `input` command that feeds pointer events as if from a panel, KWin scripts loaded over D-Bus, and screenshots through ScreenShot2.
|
||||
|
||||
- **0.1 passes.** `--no-vr`, `--control`, `toplevels`, and `input` are in ft-screens.
|
||||
- **0.2 passes.** Disabling a spare with `kscreen-doctor` keeps its toplevel, its title gets `- Output disabled`, and it stops committing; enabling it resumes on the same toplevel. A spare resized (`size`) while disabled comes up at the new size on its first frame, so a tear-off needn't blink. Live resizing works, output scale works (1.5: KWin lays out 1067 × 667 on a 1600 × 1000 buffer), and outputs with gaps between them (x = 5000, 8000, 10000) are accepted. A window placed inside a 1600 × 1200 output with a 300 px margin opens a context menu past its bottom and right edges, into the margin.
|
||||
- **0.3 mostly passes.** `workspace.screens`, `sendClientToScreen`, `windowList`, `frameGeometry` (set; it applies asynchronously), `callDBus`, `registerShortcut`, `registerUserActionsMenu`, and `readConfig` exist. `windowAdded` reports popups (`popupWindow` true, `transient` true) with their geometry. Move signals fire for KWin's title bars (`interactiveMoveResizeStarted` with `move` true, `Stepped` with the geometry, `Finished`). Not yet checked: apps' own title bars, the `callDBus` long poll, and the window menu entry. `globalThis` isn't defined in KWin's script engine. `print` goes to the journal unless `QT_FORCE_STDERR_LOGGING=1`.
|
||||
- **0.4: KWin starts the move on the press itself,** before any motion. So ft-screens freezes pointer motion as soon as a press lands in a floating window's title bar band (from the frame and client rectangles ft-floatd sends it), with no round trip. For apps that draw their own title bars, the script reports the move and ft-screens freezes then; the script puts back any few pixels the window slipped before that.
|
||||
- **Found and fixed:** KWin's nested backend ignores the position in `wl_pointer.enter`, and wlroots drops a motion to the position it entered at, so the first click after crossing onto another screen landed where KWin's pointer had been. ft-screens now enters one unit off.
|
||||
- **0.5 and 0.6** need SteamVR and the headset.
|
||||
|
||||
### Phase 1: float a window from its menu
|
||||
|
||||
Build the KWin script, ft-floatd, and floating panels in ft-screens: the margin and popup overlays, controls, moving by the title bar, resizing (edges and tab), scale, full screen, close, and back to desktop. Add drag and drop across panels, with the pointer helper change.
|
||||
|
||||
Done when:
|
||||
|
||||
- Dolphin and Kate float from "Float in VR".
|
||||
- A file drags from the floating Dolphin to the floating Kate, to a screen, and back.
|
||||
- The clipboard works between them.
|
||||
- A menu near a floating window's edge opens past the edge.
|
||||
- Closing and docking give the output back.
|
||||
- Frame pacing and GPU memory are measured with several floating windows.
|
||||
|
||||
### Phase 2: tear off and dock by dragging
|
||||
|
||||
The ghost and tear-off, and the dock highlight with push-flush docking.
|
||||
|
||||
### Phase 3: launch floating
|
||||
|
||||
`ft-float run` and `ft-float launch`, window matching, new windows of floating apps, the Frametop Apps launcher entry, the picker, remembered placement, and the notification when every spare is in use.
|
||||
|
||||
### Phase 4: polish
|
||||
|
||||
The drag proxy, the glow toward a window activated out of view (and the setting to bring it in front), the Display Settings section, and docs (design.md, reference.md, and the Use table in the README).
|
||||
|
||||
## Risks
|
||||
|
||||
- A SteamOS update can change KWin's script API or its nested backend. The script and the output handling are the parts to recheck after one, and KWin's placement memory with them (the script undoes it, see [design.md](design.md#kwins-placement-memory)).
|
||||
- A SteamOS update can change KWin's script API or its nested backend. The script and the output handling are the parts to recheck after one.
|
||||
- GPU memory: each floating output has its own swapchain of two or three buffers, including the margin. The Frame has 16 GB shared, with about 4 GB free in normal use (2026-09-29).
|
||||
- Frame pacing with many panels hasn't been measured (already an open question in design.md). Each output is a separate render pass in KWin.
|
||||
@@ -1,28 +0,0 @@
|
||||
# Gaze with the controllers: a dead end
|
||||
|
||||
Gaze mode is a mouse and keyboard feature. On 2026-09-30 we tried to make the Frame controllers its buttons: with gaze mode on and no game running, either controller's trigger would click where you look (a tap clicks, moving the hand steers the pointer, holding still drags), the controllers' lasers would be muted, and SteamVR's dashboard would follow the gaze too. It can't be done cleanly, for the reasons below. The work wasn't merged, and it's kept outside the published history.
|
||||
|
||||
## What worked
|
||||
|
||||
- Our `ft_pointer` device can hold SteamVR's laser without a hand role, in the treadmill role. It has to hint that role when SteamVR activates it; a hint changed later never gets the `/user/treadmill` path.
|
||||
- A trimmed copy of the Frame controller's compositor binding mutes the controllers' laser buttons. It's chosen with `POST /input/selectconfig.action` on vrserver's port 27062 (a JSON body; a form-encoded one gets "Parse failed"). The helper's global action sets can't do it, because SteamVR marks them inactive while its laser mouse has focus.
|
||||
- vrserver's web socket on 127.0.0.1:27062 reports every controller component without taking it from anyone.
|
||||
- Steam's UI can be kept from acting on the controllers by wrapping its gamepad input source (webpack module 17900) through Steam's CEF debugger.
|
||||
- Our device takes the laser back 10 to 15 ms after a controller takes it.
|
||||
|
||||
## What broke it
|
||||
|
||||
- Steam reads the Frame controllers itself. They aren't devices on the host: vrserver owns their radio and passes their raw reports to Steam through SteamVR's private Steam interface. Steam's client library turns them into a virtual device ("SteamFrameVirtual", at `/steamvr/virtual`), outside every SteamVR binding. Steam's own SteamVR action manifest asks only for haptics.
|
||||
- Every press and every release that Steam sees takes SteamVR's dashboard, and Frametop's panels, out of laser mode about 40 ms later. That happens whatever the bindings say and whatever Steam's UI does with the event. None of these stopped it: dropping the events in Steam's UI, removing the controller's `dualanalog` bindings, binding every button to a harmless compositor action, or setting `dashboard.modalGamepadAndLaser` to false.
|
||||
- Taking the laser back after each switch leaves a gap of 20 to 40 ms, and panels treat it as the pointer leaving, so clicks and drags break.
|
||||
- Steam can't be told to ignore the controllers. It won't save a controller layout for the virtual controller ("Saved Binding Selection Failed - No Identity"), and its menus don't go through the layout anyway: a live preview of the empty layout for Steam's UI (app 769) changed nothing.
|
||||
- SteamVR hands the controllers to a VR app instead of Steam only while that app has the input focus, as games do. A dashboard overlay with overlay flag `1 << 4` is given the focus only in gamepad mode, and only for gamepad input.
|
||||
|
||||
## Options not taken
|
||||
|
||||
- Frametop as a transparent VR app (a scene application using OpenXR's alpha blend mode) whenever gaze mode is on. That would cut Steam off while the dashboard is closed, but Steam's dashboard pages would still take the controllers, and it costs a scene layer all the time.
|
||||
- Patching Steam's running process, with an eBPF probe that writes its memory or an injected hook, to drop the controller reports while gaze mode is on. It would cover everything, but it changes Valve's software, needs Steam's client library reverse-engineered, and breaks with Steam updates.
|
||||
|
||||
## What was built
|
||||
|
||||
The attempt had a plan and a test log, probes for the laser, input focus, and SteamVR settings, a reader for vrserver's web socket, a filter for Steam's UI, and controller code in the relay and the helper. None of it is in this repo. Only the gaze dot setting (`POINTER_GAZE_DOT`) came over.
|
||||
@@ -0,0 +1,158 @@
|
||||
# Gaze first: plan
|
||||
|
||||
**Abandoned on 2026-09-30.** Steam reads the Frame controllers itself, outside SteamVR's bindings, and every press and release it sees takes SteamVR out of laser mode. Gaze mode stays a mouse feature; `docs/gaze-controllers.md` on `experimental` explains what was tried and why it doesn't work. This branch keeps the plan, the probes, and the code for reference; it isn't merged.
|
||||
|
||||
Gaze first makes the eyes the pointer for everything flat in VR, and the controllers its buttons. With gaze on and no game running, the gaze drives Frametop's 3D pointer (the `ft_pointer` driver) everywhere the mouse can go, SteamVR's dashboard included. Either controller's trigger clicks where you look, and the controllers stop showing lasers. The work happens on branch `gaze-first` (worktree `~/frametop/.worktrees/gaze-first`, from `experimental`) and is merged into `experimental` after it's been tested in the headset.
|
||||
|
||||
The decisions below were made with the user on 2026-09-30. Nothing is built yet: the first step is a headset session with four tests (see "Tests before building").
|
||||
|
||||
## Decisions
|
||||
|
||||
Input, with gaze on and outside games:
|
||||
- The gaze drives `ft_pointer`, so anything the mouse does today works with the gaze, on Frametop's screens, floating windows, the keyboard, and SteamVR's and Steam's panels. The mouse keeps working as it does today in gaze mode.
|
||||
- The gaze wakes the pointer by itself: the headset is worn and the tracker sees your eyes. No mouse movement is needed.
|
||||
- Trigger, on either controller:
|
||||
- A tap clicks where you look.
|
||||
- Moving the controller within `POINTER_GAZE_HOLD` (0.5 s) of the press is precision: the pointer stops where you looked, and your hand's movement steers it, relative, like tap and drag on the Apple Vision Pro (the controller's position seen from the eye, not where it points). The release clicks there, and the correction goes to the gaze service as a lesson.
|
||||
- Holding still for 0.5 s makes a real press; then the hand's movement drags 1:1.
|
||||
- Bumper, on either controller: the same, with the right button.
|
||||
- Right thumbstick: scrolls at the pointer (vertical, and horizontal when pushed sideways).
|
||||
|
||||
Controllers:
|
||||
- They keep their hand roles and their stock SteamVR bindings: the Steam button (tap for the dashboard, double tap for the room view, hold to recenter, the screenshot chord), gamepad mode (both grips), locomotion, room setup, and every per-hand feature.
|
||||
- Only the trigger and the bumper are muted from SteamVR while gaze first is on, so they never click or take the laser.
|
||||
- When something still moves the laser to a controller (a Steam button summoning the dashboard, a grip), the helper moves it straight back to our device.
|
||||
- Later: mute the grips too, and maybe have turning gaze off switch the controllers to gamepad mode.
|
||||
|
||||
Turning gaze on and off:
|
||||
- Gaze is opt-in and off by default. On or off is remembered across restarts (`POINTER_GAZE`), whatever turned it on or off.
|
||||
- Any game gets the controllers: a VR game (a scene application) or a flatscreen Steam game. Gaze first is off during one.
|
||||
- The toggle macro, both thumbstick clicks held 1 s by default, turns gaze on or off. In a game it turns gaze first on for that game only, until the game exits.
|
||||
- You can record your own toggle macro on the Gaze page of Frametop Input Settings: a chord of buttons on one or both controllers, held together for 0.5 to 2 s. The Steam button can't be part of it. It needs 2 or more buttons, or 1 button held at least 1.5 s. Firing it cancels a gaze click in progress. "Reset to default" brings back the thumbsticks. The one-button "Gaze pointer on/off" mapping and key combinations keep working.
|
||||
- Gaze can't turn on without a calibration for the tracker in use, or without a working tracker service. Turning it on then opens the calibrator, or points to "Repair eye tracker".
|
||||
|
||||
Eye tracker:
|
||||
- Our own tracker (`gaze/tracker/`) is the default. Choosing SteamVR's tracker on the Gaze page turns ours off. The same calibration rule applies to SteamVR's.
|
||||
- Its root service, `frametop-eyegrab`, is installed by Frametop's installer (a step that defaults to yes) and stays enabled. Turning our tracker off means no longer asking it for frames: it then holds none of SteamVR's buffers, and nothing needs root.
|
||||
- The tracker waits for a calibration before tracking, and idles whenever gaze input is off (tracking costs roughly 7 to 36 % of a core).
|
||||
- If the service is missing or broken, gaze can't turn on, and the Gaze page offers "Repair eye tracker". There's no silent switch to SteamVR's tracker.
|
||||
- Updates: the gaze service compares the installed `ft-eyegrab` with the build, and when they differ the Gaze page offers "Update eye tracker". Both repair and update ask for the password each time, through polkit (`pkexec`). There's no passwordless sudoers or polkit rule: the build output is writable by the user, so such a rule would let any program running as the user get root.
|
||||
|
||||
Calibration, in one head-locked panel:
|
||||
- Quick check: one centre dot. It's captured by a dwell (about 0.6 s of steady fixation; steadiness, not position, so it works however far off the tracker is), the trigger accepts early, and it closes itself after about 4 s if ignored. It opens when the headset is put on, when our tracker notices the headset slipping (at most once every 2 minutes), and from a mappable action. If the first 3 nudges after it are still more than 2 degrees off, it asks for 5 dots.
|
||||
- Full calibration: the same panel, about 60 degrees across, running the probe's calibration (21 dots in dark, medium, and bright rounds). Frametop's screens hide while it runs. It opens when turning gaze on finds no calibration. Quitting it leaves gaze off; turning gaze on again reopens it. Resetting the calibration while gaze is on turns gaze off and opens it.
|
||||
- The dots move with your head, so there's no "keep your head still", and the calibration doesn't depend on where the screens are.
|
||||
- The gaze probe stays as the lab tool.
|
||||
|
||||
## What exists
|
||||
|
||||
- Gaze mode (`POINTER_GAZE`, `pointer/helper/ft-pointer.cpp`): the gaze aims the pointer; the mouse's held-back press, precision, hold to drag, and nudge lessons are the model for the trigger.
|
||||
- Holds in the helper (`struct Hold`): pinches and grips steer by the hand's movement seen from the eye, in the room, from where the eye was when the gesture began (`PoseHistory`). The trigger's steering is the same with the controller's position instead of the hand's.
|
||||
- `gaze_precision` and `gaze_drag` (relay actions, "precision|gazedrag <source> 1|0" to the helper) steer by the controller's aim. Controller holds switch to position steering.
|
||||
- Controller buttons through the helper's global action sets (`pointer/helper/vrbuttons.h`), one action set per button, active only for mapped buttons and only outside games.
|
||||
- ft-screens' `controllers always|outside_games|dashboard` and its `hide`/`show` switch.
|
||||
- Our tracker's headset-moved detection (`gaze/tracker/eyes_model.py`: a glint slip change over 3 px held 1 s) and its reseat after the frames stop.
|
||||
- `gaze/tracker/install.sh` already installs `ft-eyegrab` as a system service with sudo.
|
||||
|
||||
## What SteamVR does (found 2026-09-30)
|
||||
|
||||
- The Frame controller's compositor bindings (`/opt/steamvr/drivers/frame_controller/resources/input/vrcompositor_bindings_frame_controller.json`) put the laser on a controller in only two ways: a button that clicks or switches the laser (trigger and bumper click; trigger, bumper, and grip move the laser to that hand with `switchlaserhand`), or summoning the dashboard with its Steam button. Everything else there doesn't touch the laser.
|
||||
- The gamepad and laser modes are the `/actions/dualanalog` action set (`ModeSwitch1` and `ModeSwitch2` on the grips), with `dashboard.modalGamepadAndLaser`. Those actions are application-scoped: the mode lives in Steam's UI, and we can't set it from outside.
|
||||
- The laser doesn't need a hand. The headset's own binding (`/opt/steamvr/drivers/frame_hmd/resources/input/vrcompositor_bindings_frame_hmd.json`) runs the laser from `/user/head/pose/raw`. Our device has only tried the right, left, and stylus roles; the stylus attempt got no role at all.
|
||||
- A Frame controller in the hand takes its hand's role back through its touch sensors, so a device that needs a hand role loses it whenever both controllers are held. That's why the laser has to live somewhere else.
|
||||
- vrserver's web socket (`/input/getstate.json` and `request_input_state_updates`, as frame-voice uses) lists each controller's `/input/trigger/click`, `/input/bumper/click`, `/input/grip/click`, `/input/thumbstick/click`, `/input/thumbstick/x` and `y`, `/input/system/click`, and each device's `side`. The headset has `/proximity`, but it flickers off for 0.3 to 0.5 s at a time while worn, so the quick check follows SteamVR's activity level (as the helper already does) rather than the raw sensor. Reading the socket takes nothing from anyone. A controller's path is `/devices/cv/<serial>` instead of `/user/hand/<side>` while our device holds that hand.
|
||||
|
||||
## Test results (2026-09-30)
|
||||
|
||||
Run with the headset on its stand and the controllers on (`pointer/probe/lasertest`, `input/vrws.py`):
|
||||
|
||||
1. **Laser on the treadmill role: works.** Our device held the dashboard laser with no hand role. SteamVR gives a device the `/user/treadmill` path only if it hints treadmill when it's added, so the driver hints a role that's no hand from `Activate`; a hint changed on connecting kept it at `/devices/ft_pointer/ft_pointer_0`. `GetControllerRoleForTrackedDeviceIndex` reports no role for it, so the helper's "no hand role" release skips treadmill.
|
||||
2. **Muting through the helper's action sets: doesn't work.** SteamVR reported those actions inactive (`vrstatus`: `"active": []`) while its laser mouse had input focus, as frame-voice found on 2026-09-26, and a trigger pull moved the laser to its controller until the release. **Muting through the compositor binding works:** `pointer/bindings/vrcompositor_frame_controller_gazefirst.json` is the stock binding without its trigger and bumper laser entries, selected with `POST /input/selectconfig.action` on vrserver's port 27062 (a JSON body `{"app_key": "openvr.component.vrcompositor", "controller_type": "frame_controller", "url": "file:///..."}`; a form-encoded one gets "Parse failed"). `GET /input/getactions.json?app_key=openvr.component.vrcompositor` shows the choice (`current_binding_url`). With it, the triggers showed no laser. Selecting the stock file puts it back.
|
||||
3. **Snap back: works** in 11 to 15 ms after a grip or a Steam button summon. A trigger held the laser until its release (0.4 to 1 s), which the muting removes. Our device also takes the laser from "none".
|
||||
4. **Web socket: works.** Every click, the thumbstick axes, and both thumbsticks clicked together arrive. The headset's `/proximity` flickers off for 0.3 to 0.5 s at a time while worn, so the quick check goes by SteamVR's activity level instead.
|
||||
|
||||
Also found: the bumpers, the thumbsticks' movement, and their clicks switch Steam's dashboard into its controller (gamepad) mode, and the laser owner goes to "none"; both grips go back to laser mode. Steam's own Frame controller binding (`steam_vrgamepad_bindings_frame_controller.json`, app `steam.client`) has only haptics, so that input reaches Steam's UI some other way (most likely Steam Input's virtual gamepad), and a SteamVR binding can't mute it.
|
||||
|
||||
Decided after the tests: gaze replaces the laser pointer's controls, never the controller mode's. Controller mode takes precedence while it's on, and leaving it gives the laser back to the gaze. Right click is one grip held with a trigger (the bumpers belong to controller mode).
|
||||
|
||||
## Tests before building
|
||||
|
||||
One headset session, about 30 minutes, with the user wearing the headset. Installing the test driver needs a SteamVR restart, which closes everything in VR, so it's done at the start of the session and only when the user says so.
|
||||
|
||||
1. **Laser on the treadmill role.** The driver takes a `role treadmill` command, and its compositor bindings repeat the right hand's under `/user/treadmill`. Pass: after its `switchlaserhand` (`/input/a`), `GetPrimaryDashboardDevice()` is our device, and the pointer clicks Frametop's screens and the dashboard while both controllers are held. Fail: the fallback (below).
|
||||
2. **Muting.** The helper activates its trigger and bumper action sets on both sides. Pass: a real trigger or bumper, pointed at a panel, neither clicks nor moves the laser, with the dashboard open and closed, and the helper sees the press. Fail: we need our own compositor binding for the Frame controller, switched when gaze first turns on and off.
|
||||
3. **Snap back.** A Steam button tap on either controller, and a grip. Pass: the helper sees the laser move to a controller and moves it back within about 100 ms, with no stray click.
|
||||
4. **Web socket.** Update rates for the thumbstick axes and clicks while held, and `/proximity` at don and doff.
|
||||
|
||||
Fallback if test 1 fails: gaze input goes straight into ft-screens for Frametop's own panels (the helper already knows which panel it hits and where), with both controllers held. SteamVR's and Steam's panels then take the gaze only while one hand is empty.
|
||||
|
||||
## Design by component
|
||||
|
||||
### Driver (`pointer/driver`)
|
||||
|
||||
- `role treadmill` joins `right`, `left`, and `stylus`. The hint stays OptOut while disconnected, as now.
|
||||
- `ft_pointer_vrcompositor.json` and `ft_pointer_steam.json` get the same bindings under `/user/treadmill`. They're additive, so nothing changes while the device is a hand.
|
||||
|
||||
### Pointer helper (`pointer/helper/ft-pointer.cpp`)
|
||||
|
||||
- Gaze first = gaze mode on, the gaze service ready, no game, or a game with the macro's override.
|
||||
- Waking: in gaze first, fresh gaze with the headset worn wakes the pointer (connect, treadmill role, `switchlaserhand`), with no mouse counts. The "no hand role" release doesn't apply to the treadmill role.
|
||||
- The laser: while awake in gaze first, if the dashboard's primary device becomes a controller, press `/input/a` on ours again, at most every 100 ms. Last used wins stays off in gaze mode, as now.
|
||||
- Muting: the trigger and bumper action sets on both sides are active whenever gaze first is on, whatever the relay's mappings say. Their presses go to the built-in state machine, not to the relay.
|
||||
- Trigger and bumper: a hold with a new source, a controller's position. It starts as a held-back press at the gaze; moving past `POINTER_TRIGGER_DEADZONE` (about 1 degree, seen from the eye; the pull jolts the controller) within `POINTER_GAZE_HOLD` is precision at `POINTER_TRIGGER_GAIN` (0.5); still until `POINTER_GAZE_HOLD` is a real press, and dragging at `POINTER_GAZE_DRAG_GAIN` (1). Release: click, lesson (as with the mouse, up to `POINTER_GAZE_NUDGE_MAX`), or release the press. The existing controller holds (`gaze_precision`, `gaze_drag`) move to position steering too.
|
||||
- Scroll: "scroll <dx> <dy>" from the relay goes to the driver, as the mouse's wheel does.
|
||||
- Calibration: while ft-gazed says its panel is up ("calpanel 1|0"), the pointer hides, and a trigger press goes to ft-gazed as "calaccept" instead of clicking.
|
||||
- The macro's "cancel": drops a held-back press without clicking.
|
||||
- Headset worn or not goes to ft-gazed ("headset 1|0"), for the quick check.
|
||||
|
||||
### Input relay (`input/input-relay.py`)
|
||||
|
||||
- A web socket reader for vrserver, in the standard library (the host's Python has no `websockets` module, and the relay needs nothing but Python). It follows the controllers by `side`, since their paths change with the roles.
|
||||
- The toggle macro: `"gaze_macro": {"buttons": ["left/thumbstick", "right/thumbstick"], "hold": 1.0}` in the rules file. Outside games it writes `POINTER_GAZE` and tells the helper; in a game it toggles the override for that game. It sends the helper "cancel" when it fires.
|
||||
- Recording: Input Settings asks for "macro record" and gets back the chord and how long it was held, after the checks in "Decisions".
|
||||
- Scroll: the right thumbstick's axes, while gaze first is on, as "scroll" lines to the helper.
|
||||
- Games: a Steam game running (a `reaper` process with `SteamLaunch AppId=`, checked every 2 s; to confirm with a flatscreen game) goes to the helper, which already knows scene applications.
|
||||
- `gaze_toggle` writes `POINTER_GAZE` instead of lasting until a restart.
|
||||
|
||||
### Gaze service (`gaze/ft-gazed`)
|
||||
|
||||
- `GAZE_TRACKER` defaults to `own`.
|
||||
- Readiness goes to the helper ("gazeready 1|0"): the tracker's service works (for ours: `frametop-eyegrab` active, and its binary the same as the build) and the tracker in use has a calibration. When gaze is on but not ready, it opens the calibrator, or asks for the repair.
|
||||
- Our tracker runs only while gaze is on and ready, while a calibration runs, or for the probe's lease.
|
||||
- The quick check opens on "headset 1" from the helper, on our tracker's jump (at most once every 2 minutes), and on "quickcal" (the mappable action). For our tracker, the dot is a click on `@ft_eyes`, like the probe's one-dot check; for SteamVR's, it's a lesson.
|
||||
- The full calibration's logic (the dots, rounds, sample rejection, and fits) moves out of the probe into `gazecal.py`, so the probe and the service share it. Quitting writes `POINTER_GAZE=0`.
|
||||
- Hiding Frametop's screens during a full calibration goes through ft-screens' `hide` and `show`, keeping the user's own switch as it was.
|
||||
|
||||
### Calibration panel (`gaze/panel/ft-gazepanel`, new)
|
||||
|
||||
A small C++ OpenVR overlay program in the dev container: a head-locked overlay (placed relative to the headset) of a fixed angular size, drawn on the CPU and uploaded with `SetOverlayRaw`, like ft-screens' keyboard, with labels from stb_truetype. ft-gazed drives it over `@ft_gazepanel` (show quick or full, dot at yaw and pitch with its state, background brightness, a line of text, hide). It takes no input: the trigger comes through the helper, and the dwell is ft-gazed's. It's a separate program because the helper is already 2,000 lines, and the panel has nothing to do with pointing.
|
||||
|
||||
### Frametop Input Settings, Gaze page
|
||||
|
||||
- The gaze switch, blocked with the reason while not ready ("Calibrate first", "Repair eye tracker").
|
||||
- Eye tracker: ours (default) or SteamVR's.
|
||||
- Status: the service, the calibration, and when it was made. Buttons for Calibrate, Quick check, and Repair or Update eye tracker (`pkexec gaze/tracker/install.sh`, which then runs without sudo inside).
|
||||
- The toggle macro: what it is, Record (a 3 s countdown, hold the chord, confirm), and Reset to default.
|
||||
|
||||
### Installer (`install.sh`)
|
||||
|
||||
- A step for our eye tracker ("It needs your password (sudo)"), defaulting to yes, and the gaze service, which needs no root. Gaze itself stays off.
|
||||
|
||||
## Order of work
|
||||
|
||||
1. The tests above, then this plan updated with the results.
|
||||
2. Driver and helper: the treadmill role, muting, snap back, waking by gaze, and the trigger and bumper holds. Gaze still turns on the old way.
|
||||
3. Relay: the web socket reader, scroll, the macro, games and the override, `POINTER_GAZE` remembered.
|
||||
4. Gaze service: readiness, idling, the default tracker, the service check, and the quick check's triggers.
|
||||
5. The calibration panel and calibrator: the quick check, then the full calibration and the move to 5 dots.
|
||||
6. Input Settings, the installer, and the docs (`README.md`, `docs/design.md`, `docs/reference.md`, `gaze/README.md`).
|
||||
|
||||
Each step is tested in the headset before it's merged into `experimental`.
|
||||
|
||||
## Risks
|
||||
|
||||
- SteamVR may refuse the laser on a treadmill device, or a SteamVR update may change the Frame's compositor bindings.
|
||||
- The muting relies on "Enable global input from overlays" (`steamvr/globalActionSetPriority`), which SteamVR calls experimental.
|
||||
- `pkexec` runs a script the user can write. That's acceptable only because every run asks for the password.
|
||||
- Our tracker's CPU cost while gaze is on, with SteamVR and a busy desktop; it idles otherwise.
|
||||
- Head-locked panels can be uncomfortable; the panel stays small and short-lived, except for the full calibration.
|
||||
@@ -0,0 +1,140 @@
|
||||
# Hands in Frametop: migration plan
|
||||
|
||||
Hand tracking from the headset's own cameras has been built as a separate project, frame-hands (`~/Desktop/Projects/frame-hands` on the Frame, a local git repo with no remote). The plan is to make it a native Frametop component, like `gaze/` and `power/`, instead of a separate module. The work happens on branch `hands-migration` (worktree `frametop-hands/` in the PC workspace) and is merged into `experimental` after it's been tested in the headset.
|
||||
|
||||
Builds from this worktree must sync to their own folder on the Frame, never `~/dev/frametop`: run every script with `FRAME_REPO=/home/steamos/dev/frametop-hands`.
|
||||
|
||||
## Status (2026-09-30)
|
||||
|
||||
Steps 1-5 are done:
|
||||
|
||||
- frame-hands' pending work was committed there (6c63c9e).
|
||||
- Its filtered history was merged under `hands/` (1a76d15), then laid out (`trackd/` to `track/`).
|
||||
- The renames, the Frametop paths, and ft-camd's file capabilities are done. So are `hands/Makefile`, `build.sh`, `run.sh`, the two units, the README, the settings, the installer step, and ft-screens on the shared header.
|
||||
- Built in the dev container on the Frame, and on the 7i.
|
||||
- Checked without the headset:
|
||||
- `ft-handreplay` against frame-hands' `fh-replay`, both x86 with `--cost`, on the whole dim recording and the first 60 s of the bright one: identical summaries and byte-identical depth dumps. The Makefile's own ncnn build is included in that.
|
||||
- `ft-ringplay` into `ft-hands` on the 7i tracked, pinched, and wrote `/run/user/UID/frametop-hands/{hands,gestures}`.
|
||||
- On the Frame, ft-hands in the container finds the calibration through `/run/host/persist`, and ft-camd without its capabilities refuses with a clear message.
|
||||
|
||||
Step 6 has started (2026-09-30 10:30):
|
||||
- `hands/run.sh install` is done, and both services run from this worktree.
|
||||
- The files moved to `/run/user/UID/frametop-hands/`, because `/run/user/UID/frametop` is the desktop session's own runtime folder, deleted at every desktop start.
|
||||
- Until the desktop restarts from a build with this branch's ft-screens, the link `/run/user/UID/frame-hands -> frametop-hands` feeds the running one. It's tmpfs, so it's gone at reboot.
|
||||
|
||||
Found in the headset:
|
||||
- The side cameras were swapped (`HANDS_SWAP_SIDES=1`).
|
||||
- The cutout copy shader lost resolution at `mediump` (now `highp`).
|
||||
- Colour capture isn't reliable (see the README).
|
||||
- Two pinch fixes: one hand no longer pinches both sides, and the palm-down limit stops typing pinches.
|
||||
|
||||
## What frame-hands is today
|
||||
|
||||
| Part | What it is | Size |
|
||||
| --- | --- | --- |
|
||||
| `camd/` | `fh-camd`, the camera broker (C). It borrows XRService's camera DMA-BUFs read-only with `pidfd_getfd`, times them with the `v4l2_dqbuf` tracepoint, and publishes the four IR cameras (and optionally the two colour cameras) to a shared-memory ring. It starts as root and drops to the user after setup. Adapted in part from FrameEyeCameraFeed (MIT, licence file kept). `fh-camprobe` is its discovery and recording probe. | camd 1.1k lines, tp 0.4k, xrcams 0.8k, camprobe 1.1k |
|
||||
| `trackd/` | `fh-tracker` (C++): the tracker, the models on ncnn, the calibration (jsoncpp), the pinch detector, the publisher, and the recorder. Also `fh-replay` (offline replay and scoring), `fh-ringplay` (plays a recording into a ring), and `nettest`. | 2.9k lines |
|
||||
| `include/` | The hands file (`fh_hands.h`, read by ft-screens) and the gestures file (`fh_gestures.h`, pinches). | |
|
||||
| `models/ncnn/` | MediaPipe's palm detector and hand landmark model, from the OpenCV Zoo ONNX ports (Apache-2.0), converted to ncnn in float and int8. | 5.9 MB |
|
||||
| `tools/` | Python analysis: side-camera check, colour calibration check, frame viewer, gesture watcher, depth report, model comparison, int8 calibration, model conversion. | ~1.1k lines |
|
||||
| `tracker/` | The Python prototype of the tracker. Some tools import its `calib.py` and `models.py`. | 1.3k lines |
|
||||
| `probes/`, `notes/`, `re/`, `shim/` | One-off experiments, reverse-engineering notes on SteamVR's passthrough internals, a disassembly (not in git), and a header from an abandoned XRService shim approach. | |
|
||||
| `vendor/`, `captures/` | ncnn and FrameEyeCameraFeed clones, and recordings of the user's hands and room (tens of GB). Neither is in git. | |
|
||||
|
||||
Today it runs by hand: `sudo camd/fh-camd`, then `trackd/fh-tracker`. There are no units and no installer. Files: `/run/frame-hands/ir-ring` (the ring, in a root-owned folder), and `$XDG_RUNTIME_DIR/frame-hands/hands` and `gestures`.
|
||||
|
||||
Frametop already has the consumer side on `experimental`: `screens/handcut.{h,cpp}` cuts the hands out of the screens, with its own copy of the hands file layout, and `screens/handtest.cpp` tries it on a test panel.
|
||||
|
||||
## Where it goes
|
||||
|
||||
A top-level `hands/` folder, laid out like `gaze/`:
|
||||
|
||||
```
|
||||
hands/
|
||||
README.md # from trackd/README.md and camd/README.md
|
||||
build.sh # ft-camd, ft-hands; --tools also builds the replay tools
|
||||
run.sh # install|uninstall|start|stop|restart|status|log
|
||||
frametop-camd.service # user units (templates, @REPO@)
|
||||
frametop-hands.service
|
||||
include/ # fhring.h, fh_hands.h, fh_gestures.h: shared with screens/ and pointer/
|
||||
camd/ # ft-camd: camd.c tp.c xrcams.c, LICENSE.FrameEyeCameraFeed
|
||||
track/ # ft-hands: tracker, nets, calib, pinch, publish, record; replay.cpp
|
||||
# (ft-handreplay) and ringplay.cpp (ft-ringplay) for recordings
|
||||
models/ # the ncnn models, with NOTICE (Apache-2.0, MediaPipe / OpenCV Zoo)
|
||||
tools/ # the Python checks, watch_gestures, depth_report, calib.py, ring.py
|
||||
```
|
||||
|
||||
Left behind in frame-hands, which stays as the lab: the recordings, the Python prototype (the tools that need `calib.py` or `models.py` get a trimmed copy in `hands/tools/`), `probes/`, `notes/`, `re/`, `shim/`, `camprobe`, and `vendor/`. The reverse-engineering notes don't belong in a public repo, and recordings are images of the user's hands and room, so they never go into git.
|
||||
|
||||
## Names
|
||||
|
||||
Programs within 15 characters, `ft-` prefix; files under `frametop`:
|
||||
|
||||
| Now | In Frametop |
|
||||
| --- | --- |
|
||||
| `fh-camd` | `ft-camd` |
|
||||
| `fh-tracker` | `ft-hands` |
|
||||
| `fh-replay`, `fh-ringplay` | `ft-handreplay`, `ft-ringplay` |
|
||||
| `/run/frame-hands/ir-ring` | `/run/user/UID/frametop-hands/cam-ring` |
|
||||
| `$XDG_RUNTIME_DIR/frame-hands/hands`, `gestures` | `/run/user/UID/frametop-hands/hands`, `gestures` |
|
||||
|
||||
The source keeps its `fh_` identifiers and header names (`fh_hands.h`, `fh_gestures.h`, `fhring.h`), and the file formats keep their magic strings, so recordings and tools from frame-hands keep working. Programs, units and runtime paths change.
|
||||
|
||||
## Build
|
||||
|
||||
- `hands/build.sh` builds in the dev container through `scripts/frame.sh --build`, into `hands/build/`, like the other components. `FRAME_BUILDER=pc` can take the ncnn build.
|
||||
- ncnn: fetched at a pinned tag (20260526, as now) into `hands/build/ncnn` and built once, the way `screens/build.sh` fetches the OpenVR header, with frame-hands' options so results match. `NCNN=` points the build at an existing install instead. Every net runs single-threaded (`num_threads = 1`), with the tracker spreading nets over its own pinned threads, so OpenMP could go later.
|
||||
- ft-hands runs in the dev container like ft-pointer and ft-powerd (`distrobox enter dev --`, after `scripts/container-up.sh`). Today's fh-tracker runs on the host and works only because the host happens to have the same `libjsoncpp.so.25` and libgomp as the container. Inside the container the calibration is at `/run/host/persist`, and calib.cpp (and `tools/calib.py`) fall back to it when `/persist` isn't there.
|
||||
- ft-camd has to run on the host (below), so it's linked statically (only libc and libm; `glibc-static` goes into `setup/dev-container.sh`). The host has an older glibc than the container.
|
||||
|
||||
## Running it
|
||||
|
||||
**ft-camd needs privileges**, only while it sets up: `pidfd_getfd` on XRService (the Frame has `ptrace_scope=1`), system-wide tracepoints (`perf_event_paranoid=2`), and the tracepoint files, which are root-only (`/sys/kernel/tracing/events/v4l2/v4l2_dqbuf/{id,format}` are mode 0440). A rootless container's root can't do any of that, so it runs on the host. Two ways:
|
||||
|
||||
- **A. File capabilities (chosen, 2026-09-30).** The installer runs `sudo setcap cap_sys_ptrace,cap_perfmon,cap_dac_read_search+ep hands/build/ft-camd` once. ft-camd then runs as the user, in a user unit `PartOf=steamvr.service`, so it starts and stops with SteamVR, and its ring lives in the user's runtime folder. It drops all capabilities after setup, as it drops root today. Nothing ever runs as root. Writing the file clears its capabilities, so a rebuilt ft-camd needs the setcap again. It changes rarely. `/home` on the Frame is ext4 without `nosuid`, so file capabilities work there.
|
||||
- **B. Root system service**, like the Bluetooth fixes: a root-owned copy in `/var/lib/frametop/`, a unit in `/etc/systemd/system/`. It would have to watch for XRService itself, because a system unit can't follow the user's `steamvr.service`.
|
||||
|
||||
Either way the password is needed once at install, through the same `sudo -S` path the Bluetooth fixes use, and only after asking.
|
||||
|
||||
**ft-hands** is a user unit, `frametop-hands.service`: after `frametop-camd.service`, `PartOf=steamvr.service`, nice 5, model threads on CPUs 5-7 (measured best on 2026-09-29).
|
||||
|
||||
**Settings** in `~/.config/frametop.conf`: `HANDS_SWAP_SIDES=1` and `HANDS_CPUS=5,6,7`, read by ft-hands. It's on while its services are installed (`hands/run.sh install`, `uninstall`), so there's no `HANDS` switch. There's no setting for colour yet. Later, a switch in Frametop Display Settings.
|
||||
|
||||
**Installer:** an optional last step in `install.sh`, off by default, which asks first because it needs sudo.
|
||||
|
||||
## Interfaces
|
||||
|
||||
- `screens/handcut.cpp` includes `hands/include/ft_hands.h` instead of its own copy of the layout, and reads the new path. ft-screens and ft-hands change together on this branch.
|
||||
- Pinches go to the pointer helper. It maps the gestures file and checks the begin and end counters each tick. A begin is a press, an end the release, and the pinch point's movement a drag. In gaze mode, the press lands where you look. The counters mean a quick tap between two ticks isn't missed. The tracker knows nothing about the pointer.
|
||||
|
||||
## Open items that aren't part of the move
|
||||
|
||||
These block shipping hands to other people, not the migration:
|
||||
|
||||
- **The side-camera swap.** After some XRService restarts, fh-camd publishes the two side cameras under each other's names. Today it's caught by hand (`tools/check_sides.py --ring`, then `--swap-sides`). It needs fixing at the source (tell the buffers apart by the `dqbuf` tracepoint's device, the way the colour pair is split), or at least an automatic check at start-up.
|
||||
- **The colour cameras' calibration mapping** (`tools/check_color.py` on a recording with texture).
|
||||
- **Depth when one camera loses the hand.** From the 2026-09-30 replay measurements: drifting 10% per update toward the one-camera guess (`kMonoDepthGain`) makes the depth worse than keeping the last distance. Try 0.02.
|
||||
|
||||
## Public repo
|
||||
|
||||
Frametop is public. **Not pushed to GitHub until the user says it's ready** (user decision, 2026-09-30). When it is, it publishes:
|
||||
|
||||
- The camera borrowing (`pidfd_getfd` on XRService's buffers) and the tracepoint timing. FrameEyeCameraFeed already does the same publicly. Its MIT licence and credit stay with the code.
|
||||
- The models, under Apache-2.0, with a NOTICE.
|
||||
|
||||
It doesn't publish the reverse-engineering notes, the probes, or any recording. They stay in frame-hands.
|
||||
|
||||
## History
|
||||
|
||||
frame-hands' work was committed there first (6c63c9e, its 4th commit). Its history was then filtered to drop what stays behind (`notes/`, `probes/`, `shim/`, `camd/camprobe.c`, the camprobe tools, the Python prototype except `calib.py` and `models.py`, and `.frame-job`) from every commit. It was merged into this branch under `hands/` (a subtree merge), so blame still leads to where each line came from. The renames come after, as their own commits.
|
||||
|
||||
## Steps
|
||||
|
||||
1. In frame-hands: commit the pending work, as its last state before the move (needs the user's OK).
|
||||
2. On this branch: import it under `hands/`, then rename the programs and paths. The behaviour stays identical.
|
||||
3. `hands/build.sh`, `run.sh`, the two units, the README, the settings, and the installer step.
|
||||
4. ft-screens' hand cutouts on the shared header and the new path.
|
||||
5. Check without the headset. `ft-handreplay` on the 2026-09-29 recordings with `--cost` is repeatable, so its summary must match `fh-replay`'s exactly. And `ft-ringplay` into ft-hands must publish the same hands as into fh-tracker.
|
||||
6. In the headset, with the user and after asking: stop fh-camd and fh-tracker, install the services from `~/dev/frametop-hands`, and restart the desktop from this branch so ft-screens reads the new path.
|
||||
7. Pinch into the pointer helper (it can also follow the merge). The gaze work is on the Frame's `~/frametop` main: on 2026-09-30 that branch had 4 commits `experimental` doesn't have, plus uncommitted work in the pointer helper's gaze mode. Build this step on wherever that work lands, not on this branch's older copy.
|
||||
8. Merge into `experimental`. It's checked out in a worktree on the Frame (`~/frametop/.worktrees/experimental`, where the live desktop runs), so the merge happens there, or `experimental` is switched away first.
|
||||
@@ -1,87 +0,0 @@
|
||||
# Hands while typing on a real keyboard
|
||||
|
||||
Branch `hands-typing` (2026-10-10). The problem, from the user's headset session that day (hand
|
||||
tracking with student-v10a): typing on the real keyboard made hands click (pinches, middle pinches,
|
||||
grips, pokes), and the hand ray took the pointer from the mouse or the gaze during or right after
|
||||
typing.
|
||||
|
||||
## What the logs showed (2026-10-10)
|
||||
|
||||
Sources: `journalctl --user -u frametop-pointer` (the helper), `-u frametop-hands`, `-u
|
||||
frametop-input-relay`, and the four flags the user said "flag that" for
|
||||
(`frame-hands/captures/flags/flag-20261010-145753`, `-145841`, `-145857`, `-150115`: 16 s of hands
|
||||
in the head frame plus head poses). Keys aren't logged anywhere, so how long after a key a gesture
|
||||
came is known only as "within 1 s" (the old guard's "ignored (no ray up; typing)") or not.
|
||||
|
||||
- The old guard (`POINTER_PINCH_TYPING`, 1 s) caught 23 pinches today (09:12-09:42 and
|
||||
14:59-15:02), always "no ray up" ones. It never looked at ray pinches, middle pinches, grips,
|
||||
ft-screens' pokes and grabs, or the ray showing.
|
||||
- 14:57:10-14:57:30, typing into a text field on the TK-KB006 keyboard (connected 14:56:27, field
|
||||
focused 14:56:34): 11 pinches became "no ray up: a press where the pointer is, on nothing" and 2
|
||||
more were "ignored (a press is already held)". All began at 1.7-2.0 cm (deliberate pinches read
|
||||
0.4-1.0 cm), so typing thumbs. None was within 1 s of a key: typing pauses are longer than that.
|
||||
- 14:57:37 the right hand's ray came up ("hand ray: none -> right (raised)") with the hand on the
|
||||
keyboard, and at 14:57:38 a typing pinch became a ray press on `frametop.remote.103.bar`, held
|
||||
1.89 s. Flag 145753 shows that moment: both palms 0.29-0.33 m under the eyes, 0.26-0.34 m ahead,
|
||||
palms down (0.90-1.00 of the normal pointing down), 0.17-0.22 m apart. The knuckles sat 0.29-0.31 m
|
||||
under the eyes, right at the ray's "raised" limit (0.30 m to show).
|
||||
- 15:00:37 and 15:02:24: two more presses at the pointer between "ignored (typing)" pinches.
|
||||
- 15:01:15 (flag 150115): both hands typing at 0.40-0.42 m under the eyes, 0.18-0.21 m ahead, palms
|
||||
down 0.84-0.97, 0.15-0.16 m apart; the right hand rose, gripped, dragged a panel, then came back
|
||||
to 0.28 m under the eyes, palm down 0.84, and a ray pinch clicked `frametop.screen.1`.
|
||||
- Flags 145841 and 145857 are deliberate ray use: the pointing hand 0.15-0.30 m under the eyes and
|
||||
0.33-0.46 m ahead, palm anywhere from sideways to down; the other hand resting low (0.41-0.52 m
|
||||
under the eyes, -0.1 to 0.2 m ahead), 0.35-0.6 m away.
|
||||
- "pinch right: left to ft-screens (the hand is near its panel)" bursts (14:58:26-14:58:42,
|
||||
09:14:51-09:15:02) are hands near a low panel; with typing nearby those are ft-screens' to judge.
|
||||
|
||||
The hand recorder's desk take (PR #3, `frame-hands/dumps/pr3-04-desk-work`, head poses known):
|
||||
|
||||
| phase | palms under the eyes | ahead | palm down | apart | height difference |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| typing | 0.36-0.39 m | 0.22-0.29 m | 0.85-0.99 | 0.17 m (p90 0.40) | 0.00-0.01 m |
|
||||
| mouse (one hand on the keys) | 0.38-0.41 m | 0.21-0.29 m | 0.82-0.99 | 0.41 m | 0.00-0.02 m |
|
||||
| switching | 0.37-0.40 m | 0.21-0.29 m | 0.82-1.00 | 0.17-0.41 m | 0.00-0.02 m |
|
||||
|
||||
Deliberate gestures in the same recording (`pr3-03-gestures`): the hand 0.0-0.10 m under the eyes,
|
||||
0.26-0.37 m ahead. The touch take (`pr3-06-touch`, low dots) had the touching hand 0.24-0.30 m
|
||||
under the eyes, palm down 0.7-0.9, but 0.10-0.17 m higher than the other hand.
|
||||
|
||||
So typing hands sit anywhere from 0.29 to 0.42 m under the eyes depending on the desk and chair,
|
||||
which is also where deliberate gaze pinches with the elbow resting sit (0.35-0.45 m): no fixed
|
||||
height tells them apart. What does: both hands down together (level within a few cm, 0.15-0.25 m
|
||||
apart, palms down), and a hand that types rising well above where it typed when it's meant.
|
||||
|
||||
## What the branch does
|
||||
|
||||
`pointer/helper/typing.h` (tests: `pointer/tests/typing-test.cpp`) decides, per side, whether typing
|
||||
holds a hand back; the pointer helper feeds it the relay's keys and each hands frame, and ft-screens
|
||||
hears the result (`typing <L> <R>` on @ft_screens).
|
||||
|
||||
- Keys: for `POINTER_PINCH_TYPING` (2 s, was 1) after a key, every hand is held back, except one
|
||||
that has left the keyboard or is busy out of the keyboard zone (its ray up, a gesture held:
|
||||
pointing with one hand and pressing Enter with the other).
|
||||
- On the keyboard: a hand in the keyboard zone (palm at least `POINTER_TYPING_BELOW`, 0.2 m, under
|
||||
the eyes, 0.05-0.45 m ahead, palm down) while typing is parked, and stays held back through any
|
||||
pause until it rises `POINTER_TYPING_RAISE` (0.08 m) above where it typed, or turns its palm up,
|
||||
for 0.1 s. Freed, it works at once, keys or not. A hand on the mouse beside the keyboard is parked
|
||||
too once a key comes (its pinches are mouse clicks). Unseen 3 s, it's forgotten.
|
||||
- The posture: both hands in the zone, level within 8 cm and at most 32 cm apart, for
|
||||
`POINTER_TYPING_POSTURE` (0.3 s), is typing before any key, within `POINTER_TYPING_RECENT` (600 s)
|
||||
of a key. Without that limit, both hands resting on a desk would hold gaze pinches back.
|
||||
- Held back: no pinch, middle pinch or grip (ray, gaze or plain) in the helper; no ray (an owner
|
||||
drops the pointer to the mouse or the gaze, "hand ray: right -> none (typing)"); no poke, near grab
|
||||
or fingertip cursor in ft-screens. A gesture already held carries on.
|
||||
|
||||
Replaying the four flags and the PR #3 takes through typing.h (`/tmp` driver, not in the repo; the
|
||||
flags have no keys, so a key a minute before the window was assumed):
|
||||
|
||||
| recording | result |
|
||||
| --- | --- |
|
||||
| flag 145753 (typing, the ray click at 14:57:38) | both hands parked by the posture 0.3 s into the window, held back 98% of it; the 14:57:38 pinch came 0.27 s into the window, before the replay's posture (live, the hands had been on the keys since 14:57:10) |
|
||||
| flag 150115 (typing, then the right hand up for a grip, a drag and a click) | posture at once; the right hand freed by its raise (0.18 m), its grip and two pinches allowed; the left held back 98% |
|
||||
| flags 145841, 145857 (deliberate ray use, the other hand low) | nothing held back, all 11 pinches allowed |
|
||||
| desk take, keys during the typing prompt | typing, mouse and switching: both hands held back 100% |
|
||||
| desk take, no keys | the posture alone: typing 84-100%, mouse phase: the mouse hand free (0.41 m from the other hand) |
|
||||
| gestures take (pinches, grabs, pointing) | nothing held back |
|
||||
| touch take (low dots) | the posture took both hands while they waited low; the touching hand freed by its first raise, then held back 9% |
|
||||
+3
-5
@@ -16,15 +16,13 @@ The input relay takes the volume keys from every device that has them, so gamesc
|
||||
|
||||
ft-screens drops keys while no screen has focus or the SteamVR dashboard is open, but always lets through the release of a key the desktop saw pressed, so a modifier held as the dashboard opens doesn't stay down.
|
||||
|
||||
- **A key whose release never arrives stays held in the desktop until the relay clears it, within about a second.** KWin repeats held keys itself, so a stuck letter repeats and a stuck modifier changes every later key (Ctrl+Alt held turns T into Konsole). The relay remembers which keys it told the desktop went down, and once a second it releases any that no keyboard holds (`reconcile_desktop_keys`, which asks the kernel with `EVIOCGKEY`). Pressing and releasing the key again also clears it.
|
||||
- **A keyboard that disconnects mid-press, or a relay restart with a key down, is how it happens.** The once-a-second check catches the first. A relay that starts doesn't know what an earlier one left down, so it releases the modifiers and mouse buttons on the desktop; another key left down that way stays until it's pressed and released again.
|
||||
- **A release that never arrives leaves the key held in the desktop.** KWin repeats held keys itself, so a stuck letter repeats and a stuck modifier changes every later key (Ctrl+Alt held turns T into Konsole). Pressing and releasing the key again clears it.
|
||||
- **A keyboard that disconnects mid-press is one way to get there.** The relay forgets the held key without telling ft-screens. The same goes for the relay restarting while a key is down.
|
||||
- **To see where a key went,** run `scripts/keys-report.py` and reproduce the problem while it records. It logs the modifiers, Tab, and Esc (no other keys) as the relay reads them and as its virtual keyboard sends them on, with the device roles and grabs, which programs have each keyboard open, and the relay's and desktop's logs.
|
||||
- **Switching where typing goes waits for keys to come up.** The relay changes a keyboard's grab only while none of its keys are down, so a press and its release go to the same side. A key held for a long time delays the switch until it's let go.
|
||||
- **Mouse buttons passed through as keys follow the pointer, not typing.** In pointer mode, a mouse button set to Pass through as key (a side button for Back) goes to the screen under the pointer, only while there is one and Frametop isn't paused. Its release always goes through, and ft-screens releases it itself when the pointer leaves the screens, its screen hides or closes, or Frametop pauses; the real release that comes later is dropped. So a drag held with it ends as the laser leaves a screen, unlike a laser's own click, which KWin keeps until it comes up. wlroots counts presses per button, so a release lost on the way would leave that button dead in the desktop, the lasers' clicks included, until ft-screens restarts. The button also goes to the virtual mouse, which gamescope reads, so the app Steam has focused may see it too.
|
||||
- **Media keys reach both sides.** A keyboard's media keys come from its Consumer Control node, which has volume keys too, so the relay remaps that node rather than grabbing it. Its other keys reach gamescope as well as the desktop, even while typing goes to the desktop and the keyboard itself is grabbed: Play/Pause on the desktop can pause something in Steam too. The headset's own buttons don't go to the desktop.
|
||||
|
||||
## Typing and grabbed keyboards
|
||||
|
||||
- **Programs that watch every keyboard lose grabbed ones.** While typing goes to the desktop, the relay grabs pass-through keyboards, so a hotkey tool reading them directly stops seeing their keys. Bind the tool's key as a Frametop key combination that runs a command (`command:CMD`) instead: the relay sees every keyboard. Frametop Voice's push-to-talk key works this way. (Until 2026-10-10, `SHARE_KEYS=1` sent every key typed into the desktop to the abstract socket `@frametop_keys`, which any local process could bind first.)
|
||||
- **Programs that watch every keyboard lose grabbed ones.** While typing goes to the desktop, the relay grabs pass-through keyboards, so a hotkey tool reading them directly stops seeing their keys. `SHARE_KEYS=1` in `~/.config/frametop.conf` sends their keys to the abstract socket `@frametop_keys` instead. It's off by default: abstract sockets have no permissions, and any local process that binds the name first receives every key typed into the desktop, passwords included.
|
||||
- **Typing starts out going to Steam.** After the desktop starts, keys go to Steam until you click a screen.
|
||||
- **Controller clicks don't move typing.** Overlay apps don't see controller clicks on other panels, so after one typing stays where it was. A mouse click, or a click on a screen, moves it.
|
||||
@@ -1,56 +0,0 @@
|
||||
# Profiles
|
||||
|
||||
Profiles are built: Display Settings, `ft-layout`, each profile's launcher entry, and the input relay's `profile:NAME` action open them, and the desktop can start in one.
|
||||
|
||||
A profile is a named layout that also opens apps. It holds:
|
||||
|
||||
- where each screen goes, with its size in metres, curve, roll, and pin (what a named layout held before profiles);
|
||||
- which screens show and which are hidden;
|
||||
- the apps, one entry per window: on a screen at a place and size, or floating at a pose, size, and scale;
|
||||
- the remote displays connected when it was saved, each with its place and whether it's hidden ([remote-displays.md](remote-displays.md)). Opening the profile connects them, if their computer answers, and puts them back. Like its apps, it leaves other displays connected.
|
||||
|
||||
So a "Work" profile can put three screens around you with a browser, two terminals, and an editor on them, and a "Couch" profile can hide every screen and float one video player in front of you.
|
||||
|
||||
## Decisions
|
||||
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| 1 | Profiles and named layouts | One list. Named layouts grow into profiles; a layout saved before profiles existed is a profile with no apps and every screen shown |
|
||||
| 2 | Making one | Capture what's open: the screens and every app's windows. Display Settings lists a profile's apps, so one can be removed. No editor beyond that |
|
||||
| 3 | Apps with several windows | One entry per window. The app is launched once; when its first window shows up, it's launched again for each window still missing. A browser that restores its own windows gets launched once, a terminal twice |
|
||||
| 4 | What's recorded of an app | Its desktop file name, or its command line if it has none. Not what it had open: tabs, files, and folders are left to the app's own restore |
|
||||
| 5 | Switching while apps are open | Additive: launch what's missing, move the windows that match into place, leave the rest alone. Nothing is ever closed |
|
||||
| 6 | Saving changes | Only on an explicit save. Moving things after switching doesn't change the profile |
|
||||
| 7 | Starting one | Four ways: a default profile when the desktop starts, Display Settings, a launcher entry for each profile, and a mappable action |
|
||||
| 8 | Plasma's session restore | Off in the Frametop session, so a profile is the only thing that reopens apps |
|
||||
| 9 | Screen count and resolution | Not part of a profile. They're global, because changing them restarts the desktop, which closes every window |
|
||||
|
||||
## Where profiles live
|
||||
|
||||
`~/.config/frametop-layout.json` keeps its `layouts` as they are (`{"Work": [screen places]}`), so older copies of ft-layout still read it. What a profile adds goes in a parallel `profiles` map under the same names:
|
||||
|
||||
```
|
||||
"layouts": {"Work": [{"pos": ..., "face": ..., "roll": ..., "metres": ..., "curve": ..., "pin": ...}, ...]},
|
||||
"profiles": {"Work": {"hidden": [3],
|
||||
"windows": [
|
||||
{"app": "org.kde.konsole", "screen": 2, "rect": [40, 60, 1200, 800], "maximized": false},
|
||||
{"app": "com.brave.Browser", "screen": 1, "maximized": true},
|
||||
{"cmd": ["/opt/tool/run"], "class": "tool", "screen": 1, "rect": [...]},
|
||||
{"app": "org.kde.dolphin", "float": {"rel": [12 numbers], "pixels": [1400, 900],
|
||||
"scale": 1.2, "mpp": 0.00097}}]}},
|
||||
"default_profile": "Work"
|
||||
```
|
||||
|
||||
- `screen` is 1-based, as everywhere in Frametop. `rect` is the window's frame in KWin's logical units, relative to its screen's output, so it survives the screens being arranged differently.
|
||||
- A floating window's place (`rel`) is its panel's centre and axes in the frame of the primary screen's panel, the same way ft-floatd remembers each app's place. The screens go relative to your head when the profile is applied, and the floating windows follow them. `mpp` is its density in metres per pixel, and `scale` its scale, put back with the rest.
|
||||
- Renaming or deleting a layout renames or deletes its profile entry with it.
|
||||
|
||||
## How it works
|
||||
|
||||
- **Capture** (`ft-layout save NAME`, and Save as profile… in Display Settings). ft-layout captures the screens as before, then asks ft-floatd for the windows (`windows` on @frametop_float). ft-floatd has the KWin script report every window as it is now (`report-all`), then answers with every normal window: its desktop file name, the screen it's on, its rectangle there, and whether it's maximized. For floating windows it gives their panel's place, their size in pixels, and their scale. A window whose app id has no desktop file (a Flatpak app's X11 window can give its own: RustDesk's says `com.carriez.flutter_hbb`, its desktop file is `com.rustdesk.RustDesk`) is kept by the desktop file whose `StartupWMClass` names its window class. Windows with no desktop file name are kept by their process's command line (`/proc/<pid>/cmdline`) and window class. Windows of Plasma itself, the Frametop settings apps, and dialogs aren't recorded. If ft-floatd doesn't answer, the profile keeps the apps it had.
|
||||
- **Apply** (`ft-layout use NAME`, Open profile in Display Settings). ft-layout makes the profile's hidden screens the screens' own setting, arranges the screens (which hides and shows them: ft-screens' `conceal` and `reveal`), then has ft-floatd open the apps (`profile NAME`; ft-floatd reads the windows from the layout file). If the screens can't be arranged, for example with the headset off and no head pose, the apps still open: the screens stay where they are, the profile's hidden screens still hide, and floating windows go relative to the screens wherever they are. ft-floatd goes through the entries app by app. It claims windows of that app already open (oldest first, each claimed once), and moves each to its entry's place: onto its screen at its rect (or maximized), or floating at its pose. For the entries left over, it launches the app once (`ft-float launch`, the same path as Launch as Standalone) and waits up to 30 seconds for its first window. Each window that shows up goes to the next entry's place. Once the first window has been up for 3 seconds (time for an app that restores its own windows to show them), ft-floatd launches the app again for each entry still waiting, and waits up to 30 seconds more. New windows are matched to the launch by process (or a child of it), or by desktop file name (found the same way as at capture): single-instance and D-Bus-activated apps open their windows from a process that was already running.
|
||||
- **Default at start.** The session script runs `ft-layout start --wait 90`. That opens the profile in `FT_PROFILE` or `default_profile` (screens, then the apps once ft-floatd is up), or runs `apply --wait` if there's none. Start in profile on the Layout & profiles page sets `default_profile` (`ft-layout default NAME|none`). Plasma's session restore is turned off in the session (`ksmserverrc`: `loginMode=emptySession`).
|
||||
- **Launcher entries.** Each profile gets `~/.local/share/applications/frametop-profile-<name>.desktop` ("Frametop: Work"), written when it's saved and removed when it's deleted. They show in SteamVR's Launch a program list, the Application Launcher, and KRunner. Running one (`ft-layout open NAME`) switches to that profile if the desktop runs. Otherwise it starts the desktop with `FT_PROFILE` set (`systemd-run`, as `desktops.sh start` does), which overrides `default_profile` for that start. That needs SteamVR to be running.
|
||||
- **The quick reset.** Meta+Shift+R, the reset button on a screen's bar, the Reset Screen Layout menu entry, and a button mapped to Reset desktop screen layout run `ft-layout reset`: with a profile in use (`active`, and the custom arrangement), that's `use` on it again, so everything goes back as the profile has it. Without one, it's `apply`.
|
||||
- **The action.** `profile:NAME` in the input relay (it runs `ft-layout use NAME`) for key combinations, mouse buttons, and controller buttons, with or without pointer mode. Input Settings lists one "Open profile NAME" action per profile.
|
||||
- **Display Settings.** On the Layout & profiles page, the arrangement list has the profiles, which can be renamed and deleted. Open profile and Save as profile… are the page's actions. A profile's apps are listed with where each goes and a button to leave one out, plus which screens it hides. Start in profile picks the one the desktop starts with. The Visibility tab's Screens shown switches hide screens one at a time.
|
||||
+32
-176
@@ -6,34 +6,18 @@ How each part of Frametop works, where its settings live, and the commands for r
|
||||
|
||||
From the headset, open Launch a program → Desktop. The installer replaces that launcher entry with Frametop's (`~/.local/share/applications/deckard-nested-desktop.desktop`), and `desktops.sh uninstall` gives the stock single-screen desktop back.
|
||||
|
||||
The installer also adds Native Desktop to the same list: a copy of SteamOS's entry for its own desktop (`~/.local/share/applications/native-deckard-nested-desktop.desktop`), skipped when SteamOS has no such entry. The copy is made at install time, so `scripts/update-check.py` warns when SteamOS's entry changes, and `desktops.sh install` refreshes it. In Native Desktop, typing goes to Steam's side, so the input relay doesn't grab keyboards there, and a Meta tap runs Frametop's Meta action (by default, the Steam menu) as well as opening Plasma's launcher.
|
||||
|
||||
From a terminal, on the Frame or from a PC over SSH:
|
||||
|
||||
```
|
||||
desktops.sh install # the launcher's Desktop entry starts Frametop; Native Desktop is the stock one
|
||||
desktops.sh uninstall # back to the stock SteamOS desktop, as Desktop
|
||||
desktops.sh install # the launcher's Desktop entry starts Frametop
|
||||
desktops.sh uninstall # back to the stock SteamOS desktop
|
||||
desktops.sh start | stop | restart | status | log [lines]
|
||||
```
|
||||
|
||||
`session/frametop-session.sh` runs the desktop. It starts ft-screens in the `dev` container (log: `/tmp/frametop-screens.log`), then KWin and Plasma on the host inside it. Only one Frametop desktop runs at a time. Its check doesn't look for Native Desktop, and running both at once is untested. `desktops.sh start` runs it in its own systemd unit, `frametop-desktop`. It keeps its Plasma config in `~/.config/frametop`, separate from the stock desktop's.
|
||||
`session/frametop-session.sh` runs the desktop. It starts ft-screens in the `dev` container (log: `/tmp/frametop-screens.log`), then KWin and Plasma on the host inside it. Only one desktop runs at a time. `desktops.sh start` runs it in its own systemd unit, `frametop-desktop`. It keeps its Plasma config in `~/.config/frametop`, separate from the stock desktop's.
|
||||
|
||||
When the VR launcher starts the desktop, it inherits the Steam client's environment. The session script drops the client's runtime from it (`LD_LIBRARY_PATH`, the `STEAM_*` settings, and the Steam overlay's Vulkan layer), so apps in the desktop use the system's libraries, including its video codecs, just as they would after a normal login.
|
||||
|
||||
The nested session also starts an AT-SPI accessibility registry through `session/ft-atspi` in Plasma's autostart. It discovers the bus from this session, ignores an inherited host accessibility address, and leaves an existing registry alone. Accessibility errors do not stop the desktop. This supplies the registry infrastructure for apps that expose AT-SPI trees; it does not enable gaze snapping or force Chromium/Electron accessibility. After an approved desktop restart, an AT-SPI-aware app should be visible on the nested bus. See [design.md](design.md#nested-accessibility) for native activation, fallback lifecycle, and the isolated test command.
|
||||
|
||||
KWin's blur and background contrast effects and its animations are off in this desktop, because KWin draws on the headset's GPU, which SteamVR needs. The session script turns them off once, the first time it starts (it leaves a setting you already have alone, and marks it done in `~/.config/frametop/frametoprc`), so turning them back on sticks. In the Frametop desktop, System Settings → Window Management → Desktop Effects has Blur and Background Contrast, and General Behavior has Animation speed. Or from a terminal, then restart the desktop:
|
||||
|
||||
```
|
||||
kwriteconfig6 --file ~/.config/frametop/kwinrc --group Plugins --key blurEnabled true
|
||||
kwriteconfig6 --file ~/.config/frametop/kwinrc --group Plugins --key contrastEnabled true
|
||||
kwriteconfig6 --file ~/.config/frametop/kdeglobals --group KDE --key AnimationDurationFactor 1
|
||||
```
|
||||
|
||||
The desktop's notifications can show outside it, in the Steam session. Plasma there runs on the desktop's own D-Bus, so its popups only appear on the desktop's screens. `session/ft-notifyfwd` (Plasma's autostart; log: `/tmp/frametop-notifyfwd.log`) hands each one to the notification server on the user bus, the one the Steam session's apps use, for example a VR notification server that draws them as panels. Its buttons work: a click there reaches the app as if made in Plasma, and dismissing it there closes it in Plasma. While it forwards, Plasma's own popups are off (do not disturb, held by "frametop"), and Plasma's notification history still has every one. `NOTIFY_FORWARD` in `~/.config/frametop.conf` (or Notifications on the Visibility & pins tab of Frametop Display Settings) sets it: `auto` (the default) forwards only when the user bus's server isn't SteamOS's own `steam_notif_daemon`, which hands notifications to a Steam client that can't show them on the Frame, so with stock SteamOS nothing changes; `on` forwards to any server, SteamOS's too, where they're lost (it's in the file only; the settings app shows it while it's set); `off` keeps them in the desktop. Restart the desktop after a change. Notifications Flatpak apps send through the notification portal go along too. KDE's inline reply shows as a plain Reply button there; see [design.md](design.md#notifications-outside-the-desktop).
|
||||
|
||||
Three of the system's autostart programs don't start in this desktop: Discover's update notifier (`org.kde.discover.notifier`), which starts Discover to check for updates, IBus (`ibus`), which can't reach the desktop's apps because KWin's input method is `input/ft-textinput`, and Steam (`steam`), which is already running. The session script puts copies with `Hidden=true` in `~/.config/frametop/autostart` once (marked in `frametoprc`; Steam's was added later and is hidden once on desktops that already had the other two), and skips a name you already have a file for. Delete a copy to start that program again.
|
||||
|
||||
Settings are in two files, and Frametop Display Settings edits both. The screens (resolution, width in metres, scale, curve, which one has the taskbar) and their layout are in `~/.config/frametop-layout.json`. The backend, remote desktop, and pointer settings are in `~/.config/frametop.conf`; `session/frametop.conf.example` lists every key.
|
||||
|
||||
Restarting the desktop closes its windows. Before the unit stops, `session/keep-apps.sh` moves every program started in the desktop into a systemd scope of its own, so background work such as servers, tmux, and builds keeps running. An app that shuts down its own helper processes when its window closes will still lose them; run that kind of work outside the desktop, for example as a systemd user service.
|
||||
@@ -44,13 +28,12 @@ Restarting the desktop closes its windows. Before the unit stops, `session/keep-
|
||||
|
||||
Each KWin window is one screen. ft-screens sets its size with an `xdg_toplevel` configure and KWin resizes the output to match, live. Frames arrive as DMA-BUFs and go to SteamVR through OpenVR's `IVRIPCResourceManagerClient::ImportDmabuf`, with no copy and no size limit.
|
||||
|
||||
Every screen is an overlay named `frametop.screen.N` with five controls:
|
||||
Every screen is an overlay named `frametop.screen.N` with four controls:
|
||||
|
||||
- `.bar` moves the screen. Drag it with any laser or with the 3D mouse, whose right-drag tilts. Scrolling while you drag pushes the screen away or pulls it closer, along the line from your head.
|
||||
- `.curve` bends the screen into a cylinder around you, using your current distance as the radius, or makes it flat again.
|
||||
- `.roll` rolls the screen when you drag it sideways, like a knob. It snaps level within 2.5°, and scrolling on it turns 5° per notch.
|
||||
- `.resize`, the handle on a corner, sets the size from there, the opposite corner staying put (a screen keeps its aspect; a floating window gets that many pixels). It sits on the bottom right, and goes to whichever corner a laser, the gaze or a hand's pinch comes near, brightening as it comes; lit blue while resizing. Screens go down to 15 cm wide.
|
||||
- `.reset`, left of the bar, is the quick reset, like Meta+Shift+R (`ft-layout reset`): with a profile in use it opens that profile again, as Open profile does, and otherwise it puts every screen back in its layout around where you are now.
|
||||
- `.resize`, the tab on the bottom right corner, sets the width. Screens go down to 15 cm wide.
|
||||
|
||||
The controls are sized from both the screen's width and its distance from you, follow the surface of a curved screen, and stay invisible until a laser or the 3D mouse's cursor lands on one or comes within about 1.5 times a button's size of it. While invisible they're still there, fully transparent, so SteamVR's laser can find them. They're translucent until a laser is on them, like SteamVR's own window controls.
|
||||
|
||||
@@ -65,12 +48,12 @@ The Visibility & pins tab of Frametop Display Settings decides when the screens
|
||||
- While you look at a chosen controller (the wrist gesture).
|
||||
- Only after you show them with the hotkey.
|
||||
|
||||
In the last three modes the hotkey shows the screens anyway. A screen can also be hidden on its own (Screens shown on the same tab, or `ft-layout hide N`): it stays hidden whatever the mode or the hotkey says, until it's shown again there. Windows on it stay put, and a new window that would open on it floats instead (ft-floatd). Profiles use this to show only some screens. Two more settings on the same tab cover VR games, which ft-screens detects as SteamVR scene apps:
|
||||
In the last three modes the hotkey shows the screens anyway. Two more settings on the same tab cover VR games, which ft-screens detects as SteamVR scene apps:
|
||||
|
||||
- During VR games, the Always mode hides the screens unless the dashboard is open (the default), or leaves them up.
|
||||
- Controllers on the screens. Visible screens can keep SteamVR's laser mouse on, so controllers work them with the dashboard closed, but that also takes the controllers away from a game. By default this is off while a VR game runs, and the 3D mouse or the dashboard works the screens. Pointing a controller at a screen, a floating window, or the keyboard still turns its laser on, like SteamVR's own floating windows, and pointing away gives the game the controllers back. The other choices are always on, or only with the dashboard open, which also suits flatscreen games since they aren't scene apps.
|
||||
- Controllers on the screens. Visible screens can keep SteamVR's laser mouse on, so controllers work them with the dashboard closed, but that also takes the controllers away from a game. By default this is off while a VR game runs, and the 3D mouse or the dashboard works the screens. The other choices are always on, or only with the dashboard open, which also suits flatscreen games since they aren't scene apps.
|
||||
|
||||
Input from the lasers reaches KWin through ft-screens' own seat. Keys come from the input relay: from pass-through keyboards, a keyboard's media keys (its Consumer Control node), and any key a pointer device passes through. A mouse button passed through as a key in pointer mode (a side button for Back) goes to the screen the pointer is on instead, wherever typing goes, and only while the pointer is on a screen that shows and Frametop isn't paused. Its release always goes through, and ft-screens releases it itself when the pointer leaves the screens, its screen hides, or Frametop pauses (`scripts/test-relay-buttons.sh` checks this offline). Typing follows your last click: after a click on a screen it goes to the desktop, even with the SteamVR dashboard open, and after a mouse click on any other panel (the dashboard, Steam, an app like Spotify) it goes there instead. While it goes to the desktop, the relay grabs pass-through keyboards so gamescope, which reads every keyboard itself, doesn't type them into the Steam app too. A program that watches every keyboard for a hotkey loses a grabbed one; bind its key as a key combination with a command instead (Frametop Input Settings, Keyboard). Frametop Voice types its text through the relay the same way, so it lands where typing goes. Hidden screens don't take typing.
|
||||
Input from the lasers reaches KWin through ft-screens' own seat. Keys come from the input relay, from pass-through keyboards and any key a pointer device passes through. Typing follows your last click: after a click on a screen it goes to the desktop, even with the SteamVR dashboard open, and after a mouse click on any other panel (the dashboard, Steam, an app like Spotify) it goes there instead. While it goes to the desktop, the relay grabs pass-through keyboards so gamescope, which reads every keyboard itself, doesn't type them into the Steam app too. A program that watches every keyboard for a hotkey loses a grabbed one; with `SHARE_KEYS=1` in `~/.config/frametop.conf`, their keys also go to `@frametop_keys` for it. That's off by default, since any local process that binds the name first would get everything typed into the desktop. Hidden screens don't take typing.
|
||||
|
||||
Frametop's keyboard opens by itself when a text field on the desktop gets focus, and stays open until its Close key, a layout reset, or a mapped button closes it (or, with Keep it open off in Frametop Input Settings, until the text field loses focus). While the Steam menu (the dashboard) or Steam's own keyboard is up, it steps aside, and it comes back where it was when they're gone; one asked for meanwhile appears then. In the "only with the dashboard" visibility mode, the dashboard doesn't count. It doesn't open without a head pose (the headset in standby). It's a panel of keys (a US laptop layout, with Esc where Caps Lock would be, arrows, and a Close key) that ft-screens shows 0.7 m in front of you and below your eyes, facing you. It stays where it opened, and its grab bar (the pill along the top) moves it like a screen's. Type on it with a controller's laser or the 3D mouse. Shift, Ctrl and Alt latch for the next key, and a held key repeats. KWin starts `input/ft-textinput` as the desktop's input method, and KWin activates it whenever the focused app turns on text input for a field. It tells the relay (`textfield 1` or `0`), the relay decides by the Keyboard setting in Frametop Input Settings, and ft-screens opens the keyboard for the screen that has keyboard focus (`vrkeyboard show`, `hide`, or `toggle` from a mapped button). Its keys reach the focused screen as key presses, so it works in every app, but only apps that use Wayland text input (Qt, GTK, Firefox) open it by themselves; Chromium, Electron and X11 apps need the button. The session drops the `QT_IM_MODULE=xim` and `GTK_IM_MODULE=xim` that the gamescope session sets, or Qt and GTK apps wouldn't use Wayland text input either.
|
||||
|
||||
@@ -81,23 +64,11 @@ ft-screens listens for datagrams on the abstract socket `@ft_screens` and replie
|
||||
```
|
||||
place N x y z yaw pitch roll width N metres curve N radius|on|off
|
||||
pin N|all left|right|head [matrix] unpin N|all size N w h
|
||||
get N screens head state key code value scale N s vrkeyboard show|hide|toggle|close
|
||||
get N screens head state key code value scale N s vrkeyboard show|hide|toggle
|
||||
visibility always|dashboard|gesture|toggle wrist degrees gesture left|right degrees
|
||||
hide | show | toggle controllers always|outside_games|dashboard ingames hide|visible pause on|off|state
|
||||
conceal N|all reveal N|all concealed cutouts on|off|state cutouts predict on|off cutouts lead ms
|
||||
cutouts matte circle|mesh|capsules cutouts margin deg [ms [deg]] cutouts shift off|spec cutouts roomview auto|on|off
|
||||
hands L R allowed handinput on|off|force|state palmmenu on|off|state grabthat on|off|state
|
||||
float N mpp x y w h title unfloat N pose N matrix sub N k x y w h | sub N k off minimized N 0|1 carry N
|
||||
rates focused in_view hidden rates? watch seconds phase ms
|
||||
spin next|prev|DEGREES|to N front N spotwatch on|off
|
||||
hide | show | toggle controllers always|outside_games|dashboard ingames hide|visible
|
||||
```
|
||||
|
||||
Each screen draws at a frame rate for how much of it you see. KWin draws a screen only after ft-screens gives it a frame callback, and its apps wait for theirs, so the rate of callbacks is the screen's frame rate, for KWin and the apps on it alike. A screen is focused while you look at it (within 12 degrees of where your head points), while a laser or the mouse is on it or was in the last 1.5 seconds, while it's carried, and while you type on it; it gets every display frame. The rest of what you can see (within 60 degrees) gets 15 frames a second, and a hidden screen, one behind you, and everything while paused get one a second. A level goes up at once and comes down after a moment (1.5 s from focused, 0.5 s from in view). A video, or anything moving over a large part of a screen (6% or more of it, redrawn on 8 commits in a row, 10 or more a second), keeps every frame while in view. A floating window is a screen of its own here. Nothing that stands still costs anything at any rate: KWin sends a frame only when something on the screen changed. `rates F V H` sets the three rates in Hz (0: every display frame; default `0 15 1`, also `ft-screens --rates 0,15,1`), and `rates?` shows them, the display's rate, and for each screen its level, its milliseconds between frames, and whether it counts as a video. `watch S` gives every screen full rate for S seconds: remote desktop renews it while a VNC client is connected, since a viewer sees what KWin draws. The ticks (SteamVR events and the callbacks) come once per display frame, 1 ms after the vsync (`phase ms` changes that, for tuning), in step with the display rather than on a timer that drifted through the frame.
|
||||
|
||||
`spin` is the lazy susan. By default it brings panels to a front slot at reading distance and leaves the rest where they are: `next` and `prev` bring the next one round on the right or left (the one in the slot goes home), and `to N` brings panel N. With `SCREEN_SPIN_BRING=0` in `~/.config/frametop.conf` it turns every panel in the room about your head instead: `next` and `prev` bring the next one on the right or left to straight ahead, and `to N` turns the short way to panel N. Either way, a number of degrees turns the room that far. Typing goes to the panel that ends up in front. `front N` gives panel N the typing where it is, as a spin to it would. `spotwatch on`, renewed every second, gives spotwatch's search the typing, and `spotwatch off` gives it back ([Spotwatch](#spotwatch)).
|
||||
|
||||
`conceal` and `reveal` hide and show one screen on its own (`ft-layout hide` and `show` send them), and `concealed` lists those screens. `pause on` (from the input relay, when Frametop pauses for a VR game) hides every screen and floating window whatever else says, and slows the desktop down; `pause off` undoes it. `cutouts` turns the hand cutouts on and off (`ft-handsctl cutouts`), and sets their matte, its margin and the per-distance correction (hands/README.md, "Hand cutouts and near input"); they cut only with Room View on, detected or forced (`cutouts roomview`, hands/README.md "Rendered hands"). `hands` is the pointer helper's word on hand input (each side `hand`, `controller` or `off`, and whether it's allowed now); `handinput` turns near input on (following that word), off, or on without it, and `handinput state` reports what each hand is doing, its counts (presses, one-frame presses, pinch clicks, grabs by card and fist, poke attempts and those refused, crossings off the content, presses at the edge, flicks and throws, holds through tracking loss, second hands joining a held panel, pinches that waited to become fists, buttons pinched) and what held presses back; each press, release, refused poke and grab also goes to ft-screens' log as a `hands:` line. `palmmenu` turns the palm menu (a bare hand palm up, pinch: the Steam menu) on and off, and `palmmenu state` reports it. `grabthat` turns "grab that" (an open hand held out to a panel farther than arm's reach, then a fist pulled back: it comes to arm's reach) on and off, and `grabthat state` reports each hand's state, its counts and the panels that came. The last line is ft-floatd's, for floating windows: N is a floating window's panel, numbered on from the screens, one per spare output. `float` gives the window's rectangle in its output, metres per pixel, and the title bar's height, and shows the panel; `unfloat` hides it. `pose` places it (a 3x4 matrix, standing universe), `sub` shows popup or dialog k over it, `minimized` hides it while its window is minimized, and `carry` moves it with the laser that pressed the window's own title bar.
|
||||
|
||||
## Input relay
|
||||
|
||||
SteamVR opens input devices only when it starts. A Bluetooth mouse that sleeps and reconnects gets new device nodes, SteamVR keeps reading the dead ones, and the mouse stops working until SteamVR restarts. `input/input-relay.py` avoids this. It creates two virtual devices, `frametop virtual mouse` and `frametop virtual keyboard`, through `/dev/uinput` before SteamVR starts. It then grabs USB and Bluetooth mice and keyboards as they come and go and forwards their events, so SteamVR only ever sees the virtual devices, which never go away.
|
||||
@@ -110,8 +81,6 @@ The relay also owns the volume keys, on every device that has them, the headset'
|
||||
desktops.sh relay install # enable it (starts with the next reboot or SteamVR start)
|
||||
desktops.sh relay status | log | uninstall
|
||||
input/input-relay.py --no-grab # try it without taking devices from SteamVR
|
||||
input/test/keys-test.py # key combinations, modifier taps, and what reaches the desktop, against fake devices (safe next to the live relay)
|
||||
steam/ft-steam menu # what Open Steam menu does; ft-steam check: Steam's UI still has the calls
|
||||
```
|
||||
|
||||
The first time, the relay has to start before SteamVR, so reboot or restart SteamVR after installing it. After that it's safe to restart on its own: systemd keeps the virtual devices open in its file descriptor store (`FileDescriptorStorePreserve=yes`), so SteamVR keeps the same devices.
|
||||
@@ -122,51 +91,47 @@ A mouse drives SteamVR the way a controller's laser does, but shows up as a smal
|
||||
|
||||
- The input relay, in pointer mode (`POINTER=1`), sends mouse motion, clicks, and scrolling to the helper. A deliberate movement or a click wakes the pointer, and 30 seconds without mouse input releases it.
|
||||
- The helper, `pointer/helper/ft-pointer`, runs in the `dev` container as `frametop-pointer.service` and starts with SteamVR. It keeps the cursor, tests it against every visible overlay, draws the dot, and sends the driver an exact pose.
|
||||
- The driver, `pointer/driver/` (`ft_pointer`), is loaded by SteamVR. It's an invisible virtual controller whose laser follows the cursor. It takes the treadmill role, no hand, so the real controllers keep theirs and one can be used beside it (`POINTER_ROLE` and the driver's `role` setting; see docs/design.md, "Treadmill role").
|
||||
- The dashboard patch, `pointer/dashpatch/` (`frametop-dashpatch.service`, starts with SteamVR), lets that treadmill move SteamVR's own windows by their grab bars, which SteamVR allows only for the head or a hand. It goes into SteamVR's dashboard page through the dashboard's debugger, which the installer turns on at 127.0.0.1:8089 (`VRWebHelper/DebuggerEnabled`; the uninstaller turns it off). `pointer/dashpatch/run.sh status` shows whether it's in.
|
||||
- The driver, `pointer/driver/` (`ft_pointer`), is loaded by SteamVR. It's an invisible virtual right-hand controller whose laser follows the cursor.
|
||||
|
||||
Whichever device you used last wins. Picking up a controller hands the laser back at once, and moving the mouse takes it again. With hand interaction (`POINTER_HANDS=1`), this goes per hand: a controller used recently wins its hand, and the other hand can stay bare. When the headset comes off, the pointer lets go, so the displays can sleep, and it stays off until you're wearing the headset again.
|
||||
Whichever device you used last wins. Picking up a controller hands the laser back at once, and moving the mouse takes it again. When the headset comes off, the pointer lets go, so the displays can sleep, and it stays off until you're wearing the headset again.
|
||||
|
||||
To move a floating panel, left-drag its grab bar. The scroll wheel pushes and pulls it while you drag. Hold the right button while dragging and move the mouse to tilt the panel around the grab point; the right press isn't sent as a click. The tilt stays for the rest of the drag, and releasing the left button drops the panel as it is. A mapped Toggle dashboard button wakes the pointer if needed and holds the virtual system button for 0.12 s, because SteamVR ignores a press and release in the same instant. Open Steam menu / close dashboard (`steam_menu`) needs no pointer: `steam/ft-steam menu` asks Steam's UI, over its debugging port (`steam/steamui.py`), to show its dashboard overlay and focus the Steam frame's menu, or to hide the dashboard if it's up.
|
||||
To move a floating panel, left-drag its grab bar. The scroll wheel pushes and pulls it while you drag. Hold the right button while dragging and move the mouse to tilt the panel around the grab point; the right press isn't sent as a click. The tilt stays for the rest of the drag, and releasing the left button drops the panel as it is. A mapped Toggle dashboard button (or a Meta tap, with `META_DASHBOARD=1`) wakes the pointer if needed and holds the virtual system button for 0.12 s, because SteamVR ignores a press and release in the same instant.
|
||||
|
||||
```
|
||||
pointer/driver/build.sh && pointer/driver/install.sh install # then restart SteamVR
|
||||
pointer/helper/build.sh && pointer/helper/run.sh install
|
||||
pointer/helper/run.sh status | log | restart
|
||||
pointer/driver/install.sh probe # devices, hand roles, activity, charging, who owns the dashboard pointer
|
||||
pointer/probe/build.sh # probes: vrprobe, lasertest (who has the laser), ctlnoise (controller noise)
|
||||
pointer/driver/install.sh probe # devices, hand roles, who owns the dashboard pointer
|
||||
```
|
||||
|
||||
The pointer settings are in `~/.config/frametop.conf`: `POINTER_SENSITIVITY`, `POINTER_IDLE`, `POINTER_WAKE_COUNTS`, `POINTER_CONTROLLER_PICKUP`, `POINTER_DISTANCE`, `POINTER_CURSOR_DEG`, `POINTER_ORIGIN_FRACTION`, `POINTER_ORIGIN_MARGIN`, `POINTER_SCENE_RADIUS`, `POINTER_EDGE_REACH`, `POINTER_LASER_WIDTH`, `POINTER_IGNORE`, the head follow settings `POINTER_FOLLOW`, `POINTER_LEASH_DEG`, `POINTER_LEASH_DELAY`, `POINTER_LEASH_RETURN`, and `POINTER_FOLLOW_REACH`, and the gaze mode settings `POINTER_GAZE`, `POINTER_GAZE_RETAKE`, `POINTER_GAZE_NUDGE_MAX`, `POINTER_GAZE_HOLD`, `POINTER_GAZE_CONTROL_HOLD`, `POINTER_GAZE_DOT`, `POINTER_GAZE_SHOW`, `POINTER_GAZE_MOUSE`, `POINTER_GAZE_MOUSE_MOVE`, and the keyboard clicks' `POINTER_HEAD_DEADZONE` and `POINTER_KEY_TAP`, and the gaze service's `GAZE_TRACKER` (`auto`, the default: our own eye tracker when it's installed, else SteamVR's; or `own` or `steam`), `GAZE_EYE` (the eye bias), and `GAZE_FIT_WATCH` (the put-on check and the slip note, off by default: [gaze/README.md](../gaze/README.md), "Headset fit"). The example config explains each. Frametop Input Settings changes them live; after editing the file by hand, restart the relay or the helper (the gaze service reads its three again when the file changes).
|
||||
The pointer settings are in `~/.config/frametop.conf`: `POINTER_SENSITIVITY`, `POINTER_IDLE`, `POINTER_WAKE_COUNTS`, `POINTER_CONTROLLER_PICKUP`, `POINTER_DISTANCE`, `POINTER_CURSOR_DEG`, `POINTER_ORIGIN_FRACTION`, `POINTER_ORIGIN_MARGIN`, `POINTER_SCENE_RADIUS`, `POINTER_EDGE_REACH`, `POINTER_LASER_WIDTH`, `POINTER_IGNORE`, the head follow settings `POINTER_FOLLOW`, `POINTER_LEASH_DEG`, `POINTER_LEASH_DELAY`, `POINTER_LEASH_RETURN`, and `POINTER_FOLLOW_REACH`, and the gaze mode settings `POINTER_GAZE`, `POINTER_GAZE_RETAKE`, `POINTER_GAZE_NUDGE_MAX`, `POINTER_GAZE_HOLD`, and `POINTER_GAZE_SHOW`, and the gaze service's `GAZE_TRACKER` (SteamVR's eye tracker or our own) and `GAZE_EYE` (the eye bias). The example config explains each. Frametop Input Settings changes them live; after editing the file by hand, restart the relay or the helper (the gaze service reads its two again when the file changes).
|
||||
|
||||
## Frametop Input Settings
|
||||
|
||||
A Kirigami app with a Python backend, in the Plasma menu under Settings. It runs in the `dev` container and talks to the relay over its control socket, `@frametop_relay`. It has ten pages:
|
||||
A Kirigami app with a Python backend, in the Plasma menu under Settings. It runs in the `dev` container and talks to the relay over its control socket, `@frametop_relay`. It has eight pages:
|
||||
|
||||
- Devices lists every USB and Bluetooth mouse and keyboard, with a light that flashes when the device is used. Each device gets a role: 3D pointer (grabbed, drives the pointer; the default for anything with a mouse), Pass through (grabbed only while typing goes to the desktop; the default for keyboards, whose key combinations work everywhere), or Ignore. A device is identified by its Bluetooth address, or its USB ids and name, so all of its input nodes share one role. Forget drops everything saved for a device.
|
||||
- Buttons maps a pointer device's buttons. Choose Capture a button, press the button or key, then pick an action: a click, back, scroll, toggle dashboard, recenter, pointer on or off, head follow on or off, gaze pointer on or off, gaze precision, gaze drag, gaze quick check, faster or slower, reset the screen layout, hide or show the screens, open or close the keyboard, float a window in VR or put it back, put all floating windows back, pause or resume Frametop ([Pausing for VR games](#pausing-for-vr-games)), Open profile NAME (one per profile, [profiles.md](profiles.md)), pass the key through, or nothing. Devices with saved mappings are listed even while they're asleep.
|
||||
- Controllers maps the Frame controllers' buttons (every button but the system button) to the same actions, except passing a key through and the gaze actions: gaze mode is a mouse and keyboard feature ([gaze-controllers.md](gaze-controllers.md)). Capture a button and press it on a controller, or pick it from the list. The controllers aren't input devices on the host; only SteamVR sees them. So the pointer helper reads them with SteamVR input (`pointer/helper/vrbuttons.h`, `pointer/helper/actions/`) and sends presses to the relay (`vrbtn right/a 1`), which does the mapped action. The helper only takes the buttons that are mapped (the relay tells it with `vrbind`), at an overlay-global priority, and only while no game (scene application) runs, so games keep every button; with In games on (`controller_in_games`), a mapped button is taken from games too. That needs SteamVR's "Enable global input from overlays (Experimental)" setting (`steamvr/globalActionSetPriority`), which the page's Global input switch turns on and off. Mappings are saved as `controller_buttons` in `~/.config/frametop-input.json`.
|
||||
- Game optimization has the pause for VR games: its state with Pause now or Resume, whether VR games pause Frametop by themselves, the controller gesture (one or two buttons, pressed once or twice; one button always takes two presses), what happens to the desktop, and the sound. They're saved as `pause_auto`, `pause_gesture`, `pause_desktop`, and `pause_sound` in `~/.config/frametop-input.json`. See [Pausing for VR games](#pausing-for-vr-games).
|
||||
- Keyboard sets when Frametop's keyboard opens: whenever a text field is selected; only while no pass-through keyboard is connected (the default; keyboards other programs make through uinput don't count); only with a mouse or controller button mapped to Open/close keyboard; or never, which turns the button off too. Keep it open (on by default, `vr_keyboard_persist`) leaves it open after the text field loses focus. The mode is saved as `vr_keyboard` in `~/.config/frametop-input.json`, and the page lists the keyboards that count as connected. Its Key combinations section maps modifiers plus a key, or one modifier tapped on its own, on any keyboard, to any action but passing a key through or nothing, or to Run a command…: a command line the input relay runs with `sh -c` when you press the keys (`command:CMD`). The command runs as the relay's user service, outside the desktop's session, with `layout/`, `float/` and `steam/` on its `PATH` (so `ft-layout use Work` or `ft-float launch org.kde.dolphin` work as they are), and its output goes to the relay's journal. The gaze clicks (Gaze left click and Gaze right click) only go on key combinations. The defaults are a Meta tap (open the Steam menu, or close the dashboard), Meta+J (gaze left click), Meta+K (gaze right click), Meta+Shift+F (float window in VR or put it back), and Meta+Alt+Tab and Meta+Alt+Shift+Tab (spin the panels: every screen and floating window turns about your head, so the next one on the right or left comes to the front; ft-screens' `spin next|prev|<degrees>`); remove them or add others there. With Frametop Voice installed, a Right Ctrl tap starts and stops dictation too. A tap is a press with no other modifier held, then a release with no other key, mouse button, or scroll in between; a bound one sends the desktop F24 before the release, so Plasma's launcher doesn't open on it. Tapping a left modifier binds either side's; tapping a right one (Right Ctrl) binds that key alone. The combination's last key isn't typed, and the modifiers still reach the app; while typing goes to Steam rather than the desktop, keyboards aren't grabbed, so Steam or the game sees the keys too. They're saved as `key_bindings` in the same file; a file with its own list, even an empty one, gets no defaults.
|
||||
- Voice has Frametop Voice's settings (see Voice dictation): on or off, the dictation keys (set on the Keyboard page), the controller buttons and their mode, typing as you talk, sounds, and the wake word and its phrases. They're saved in `~/.config/frametop-voice.json`, and the service applies them within 2 seconds.
|
||||
- Devices lists every USB and Bluetooth mouse and keyboard, with a light that flashes when the device is used. Each device gets a role: 3D pointer (grabbed, drives the pointer; the default for anything with a mouse), Pass through (grabbed only while typing goes to the desktop; the default for keyboards, where a Meta tap toggles the dashboard if `META_DASHBOARD=1` is in `~/.config/frametop.conf`), or Ignore. A device is identified by its Bluetooth address, or its USB ids and name, so all of its input nodes share one role. Forget drops everything saved for a device.
|
||||
- Buttons maps a pointer device's buttons. Choose Capture a button, press the button or key, then pick an action: a click, back, scroll, toggle dashboard, recenter, pointer on or off, open or close the keyboard, head follow on or off, gaze pointer on or off, faster or slower, pass the key through, or nothing. Devices with saved mappings are listed even while they're asleep.
|
||||
- Controllers maps the Frame controllers' buttons (every button but the system button) to the same actions, except passing a key through. Capture a button and press it on a controller, or pick it from the list. The controllers aren't input devices on the host; only SteamVR sees them. So the pointer helper reads them with SteamVR input (`pointer/helper/vrbuttons.h`, `pointer/helper/actions/`) and sends presses to the relay (`vrbtn right/a 1`), which does the mapped action. The helper only takes the buttons that are mapped (the relay tells it with `vrbind`), at an overlay-global priority, and only while no game (scene application) runs, so games keep every button; with In games on (`controller_in_games`), a mapped button is taken from games too. That needs SteamVR's "Enable global input from overlays (Experimental)" setting (`steamvr/globalActionSetPriority`), which the page's Global input switch turns on and off. Mappings are saved as `controller_buttons` in `~/.config/frametop-input.json`.
|
||||
- Keyboard sets when Frametop's keyboard opens: whenever a text field is selected; only while no pass-through keyboard is connected (the default; keyboards other programs make through uinput, like frame-voice's, don't count); only with a mouse or controller button mapped to Open/close keyboard; or never, which turns the button off too. Keep it open (on by default, `vr_keyboard_persist`) leaves it open after the text field loses focus. The mode is saved as `vr_keyboard` in `~/.config/frametop-input.json`, and the page lists the keyboards that count as connected.
|
||||
- Pointer has a Head follow switch and sliders for the pointer settings, which apply immediately, and a Recenter button.
|
||||
- Ignored panels lists the SteamVR overlays that are showing, grouped by app (the first two parts of the overlay key, such as `sasaken.frame-perf-overlay`), from the pointer helper (`overlays`). Tick a panel, or Ignore the whole app, and the pointer passes through it to what's behind. It's for panels you only look at, like a performance overlay that follows your view. The list is saved as `POINTER_IGNORE` in `~/.config/frametop.conf`: comma-separated overlay keys, where a shell pattern like `vendor.app*` covers a whole app, including panels it opens later. The helper reloads at once. Frametop's own screens aren't listed, and entries for apps that aren't open are listed below, to remove.
|
||||
- Gaze has the gaze pointer switch (on now and from now on; a mapped button toggles it until the helper restarts), what the mouse's left button and movement do, the gaze dot, the eye tracker and eye bias, the gaze mode sliders, the gaze service's state (headset, samples per second, how often the tracker is losing each eye, the calibration, the nudges learned), and Quick check, Calibrate, and Check headset fit (each in a panel in the headset), Reload calibration, and Forget nudges. The gaze probe, a development tool, is in the page's overflow menu.
|
||||
- Gaze has the gaze pointer switch (on now and from now on; a mapped button toggles it until the helper restarts), the gaze mode sliders, the gaze service's state (headset, samples per second, how often the tracker is losing each eye, the calibration, the nudges learned), and Calibrate (opens the gaze probe), Check headset fit (opens the probe's Headset fit mode), Reload calibration, and Forget nudges.
|
||||
- Bluetooth lists paired devices and has Apply Bluetooth fixes, which runs `/etc/steamframe/bt-fixups.sh` through `pkexec`. Pair new devices in Steam.
|
||||
|
||||
Device rules are saved in `~/.config/frametop-input.json`. `input-settings/install.sh` installs the menu entry. Its launcher hands podman the real `XDG_RUNTIME_DIR` and user bus and gives the app the session's Wayland socket, because the desktop session runs on a private D-Bus and podman fails on it.
|
||||
|
||||
## Frametop Display Settings and ft-layout
|
||||
|
||||
When the desktop starts, its screens arrange themselves around where you're facing. You can move them by hand at any time and put them back with Meta+Shift+R, the reset button left of any screen's bar, the Reset Screen Layout menu entry, or a button mapped to Reset desktop screen layout. With a profile in use, these open it again, the same as Open profile: its screens, hidden screens, remote displays and apps. Arrange now in the app arranges the screens (and the profile's remote displays) without reopening its apps or hiding its hidden screens again.
|
||||
When the desktop starts, its screens arrange themselves around where you're facing. You can move them by hand at any time and put them back with Meta+Shift+R, the Reset Screen Layout menu entry, Arrange now in the app, or a mouse button mapped to Reset desktop screen layout.
|
||||
|
||||
The desktop's own screen arrangement follows where the screens are around you, whatever their numbers: a screen you see to the left of another is to its left in Plasma too, so the pointer and dragged windows cross straight to it. Screens one above the other stack, and screens pinned to a wrist or your head come last. It's updated at startup, after arranging or saving the layout, and half a second after you let go of a screen you moved. With the headset off there's no head pose to go by, and the arrangement stays as it was.
|
||||
|
||||
Frametop Display Settings has four tabs (three with the gamescope backend, which has no Visibility & pins):
|
||||
|
||||
- Screens: add and remove screens, and set each one's resolution (presets from 1080p to 4K, ultrawide, super ultrawide, portrait, or custom), its width in VR (0.5 to 6 m), its scale, whether it's curved, and whether it has the taskbar. Resolution, width, and curve apply at once. Adding or removing a screen takes a desktop restart, which the app offers.
|
||||
- Layout (the Layout & profiles page): the Arrangement list starts with two presets: Curved around you, with the screens hinged edge to edge like monitors on a desk and each turned to face you, and Flat wall. Both take rows, distance, gap, and height, and Arrange now applies them. Save as profile… saves where the screens are now (positions, sizes, curves, and pins), which ones are hidden, and the open apps and where their windows are, under a name. Profiles are listed in Arrangement after the presets: pick one and Open profile switches to it, and the buttons next to the list rename or delete it. Below it, Apps lists the profile's windows and where they go, and Leave out drops one. A profile saved with fewer screens than you have now leaves the others where they were saved last, or where the preset would put them. Start in profile picks the profile the desktop starts in; with None, a switch turns auto-arrange at startup on or off. A preview shows the layout from above and from the front. See [profiles.md](profiles.md).
|
||||
- Visibility & pins: the visibility, game, and controller settings described above, the wrist angle, where each screen is pinned (in the room, a wrist, or your head), and buttons to pin all screens or unpin them. Its Notifications section sets `NOTIFY_FORWARD` (whether the desktop's notifications show in the Steam session, see [The desktop](#the-desktop)), shows where they go now, and offers a desktop restart after a change.
|
||||
- Layout: a curve around you, with the screens hinged edge to edge like monitors on a desk and each turned to face you, or a flat wall. Both take rows, distance, gap, and height. Save current arrangement saves the positions, sizes, curves, and pins you set by hand under a name instead. Named layouts are listed with the presets: pick one and Arrange now to switch to it, and rename or delete it with the buttons next to the list. A layout saved with fewer screens than you have now leaves the others where they were saved last, or where the preset would put them. A preview shows the layout from above and from the front, and a switch turns auto-arrange at startup on or off.
|
||||
- Visibility & pins: the visibility, game, and controller settings described above, the wrist angle, where each screen is pinned (in the room, a wrist, or your head), and buttons to pin all screens or unpin them.
|
||||
- Power: when the displays turn off while the headset isn't used, their state now, Turn displays off now (to try it), and Stay awake while plugged in. See [Displays off and sleep](#displays-off-and-sleep).
|
||||
|
||||
`layout/ft-layout` does the arranging. It's a Python script that uses only the standard library and runs on the host:
|
||||
@@ -174,66 +139,17 @@ Frametop Display Settings has four tabs (three with the gamescope backend, which
|
||||
```
|
||||
layout/ft-layout apply # arrange every screen
|
||||
layout/ft-layout capture # save the current arrangement and sizes as the layout
|
||||
layout/ft-layout save NAME # ...under a name too, with the open apps and hidden screens (a profile, docs/profiles.md), and use it
|
||||
layout/ft-layout use NAME # switch to a profile: arrange the screens in it and open its apps
|
||||
layout/ft-layout open NAME # a profile's launcher entry: use it, or start the desktop in it
|
||||
layout/ft-layout default NAME|none # the profile the desktop starts with (start --wait runs it at desktop start)
|
||||
layout/ft-layout save NAME # ...under a name too, and use it
|
||||
layout/ft-layout use NAME # switch to a named layout and arrange the screens in it
|
||||
layout/ft-layout layouts # list the named layouts (* = in use); rename OLD NEW, delete NAME
|
||||
layout/ft-layout pin N|all left|right|head # pin as they are now; unpin N|all
|
||||
layout/ft-layout plan # print the arrangement as JSON (no VR needed)
|
||||
layout/ft-layout scale # per-screen scale, positions (as the screens are around you), and taskbar screen, to KWin
|
||||
layout/ft-layout toggle # hide or show all screens
|
||||
layout/ft-layout hide N|all # hide a screen on its own, whatever the visibility mode; show N|all brings it back, hidden lists them
|
||||
display-settings/install.sh # menu entries and the Meta+Shift+R and Meta+Shift+H shortcuts
|
||||
```
|
||||
|
||||
The layout is stored relative to your head when it's applied. `/run/user/<uid>/frametop-layout.log`, in the host's runtime directory (not the nested desktop's `/run/user/<uid>/frametop`), has the run from the last desktop start and ft-screens' layout runs after it.
|
||||
|
||||
## Frametop Remote Displays
|
||||
|
||||
Other computers' monitors as Frametop screens (ft-screens only), streamed from Vibepollo or Sunshine with Moonlight's protocol. Frametop Remote Displays (`remote-displays/`, also opened by the Remote displays button on Display Settings' Screens page) finds Vibepollo and Sunshine computers on the network, signs in to one with its Web UI login (it keeps a narrow API token, not the password, and pins the host's certificate), shows whether it answers, and adds its displays: its monitors, or a virtual one at any size. A computer with a Steam Link dongle on the Frame's hotspot streams over it (Connection: auto, network or dongle only); the others use the network. Each display has a Connected switch (and the host one for all of its displays), Shown, its stream's resolution, frame rate and bitrate, and its width in VR. On Windows, a monitor switches to its stream's resolution while it streams (or its closest mode with the same shape), and back when the stream ends. That takes Frametop's build `frametop-2.0.0-2` or newer; `frametop-2.0.0-1` scaled the monitor's own resolution into the stream. A disconnected display keeps its settings, and within the same desktop run, its place. In VR each one is a panel with a screen's controls, and profiles keep where they are. See [remote-displays.md](remote-displays.md).
|
||||
|
||||
```
|
||||
layout/ft-layout remote list # the remote displays and their streams' state
|
||||
layout/ft-layout remote connect|disconnect ID...
|
||||
remote-displays/install.sh # its menu entry
|
||||
```
|
||||
|
||||
On the PC, `host/windows/Setup Frametop host.cmd` sets it up for Frametop: Vibepollo 2.0.0 (installed if missing), Frametop's build of its `sunshine.exe`, the settings Frametop needs, the Web UI login you sign in with from the Frame, and a firewall check (`-Check` to see what it would change, `-Undo` to put things back). [host/windows/README.md](../host/windows/README.md) has the steps, and the [`windows-host-1`](https://github.com/Frametop/frametop/releases/tag/windows-host-1) release has the kit as a zip. A Linux PC (Arch or CachyOS) installs Frametop's Vibepollo package with Vibepollo's installer, as [host/linux/README.md](../host/linux/README.md) says; it hasn't been tested yet.
|
||||
|
||||
On a Mac, `host/macos/frametop-host-setup.sh` (or `Setup Frametop host.command`) installs Sunshine and runs one instance per real display, each on its own port (47989 + 100·N), started at login. You allow Sunshine Screen Recording and Accessibility once in System Settings. Without Accessibility the Frame's pointer moves on the Mac but clicks, scrolling and typing don't arrive; the setup checks both, and offers to ask again if Accessibility is missing. Remote Displays finds the Mac as one computer; signing in with the Web UI login pairs the displays you pick, and the password isn't kept, so adding more later signs in again. Macs have no virtual displays (`--check`, `--undo` as on the PC). Frametop Streams in the menu bar turns each display on or off with a click, or all of them, and `ft-sunshine off|on|status` does the same in Terminal. Off lasts through logins until on. [host/macos/README.md](../host/macos/README.md) has the steps for setting up a Mac, and the [`macos-host-1`](https://github.com/Frametop/frametop/releases/tag/macos-host-1) release has the kit as a zip.
|
||||
|
||||
## Floating windows
|
||||
|
||||
A desktop window can float in VR as a panel of its own, away from the screens. Meta+Shift+F floats the window under the pointer (or the active one, over the wallpaper), or puts it back on its screen if it floats. So do Float in VR in every window's menu (Alt+F3; Back to Desktop on a floating one), the button left of Close in its title bar, and a mouse button, controller button, or key combination mapped to Float window in VR in Frametop Input Settings; Put all floating windows back is mappable too. Launch as Standalone, in an app's right-click menu in the Application Launcher or the taskbar, starts the app with its first window floating, where that app last floated or in front of you. [floating-windows.md](floating-windows.md) explains how it works.
|
||||
|
||||
`float/ft-floatd` does this. It runs inside the desktop's Plasma session (log: `/tmp/frametop-floatd.log`), loads the KWin script `float/frametop-float.js`, and moves each floating window to a spare KWin output of its own, which ft-screens shows as a panel cropped to the window. The settings are in `~/.config/frametop.conf`: `FLOAT_SLOTS` is how many windows can float at once (16, the most; 0 turns floating off; restart the desktop after a change), and `FLOAT_MARGIN` the pixels around each window on its output, so menus have room past its edges (300). Each app's last floating place, size, and scale are kept in `~/.config/frametop-float.json` by desktop file name, relative to the primary screen, so they move with the screens. `FT_FLOAT_DEBUG=1` in ft-floatd's environment logs every event from the KWin script.
|
||||
|
||||
With floating on, the session gives the desktop's windows Frametop's own decoration (`decoration/`): Breeze's look plus the float button. Apps that draw their own title bar, like Chromium and Electron apps, don't have the button. `decoration/apply.sh` puts a changed copy into the running desktop, and `decoration/apply.sh --off` goes back to Breeze until the next desktop start.
|
||||
|
||||
```
|
||||
float/ft-float float [ID|active] # float a window (default: the active one, as a toggle)
|
||||
float/ft-float float pointer # the float key: the window under the pointer, floated or put back
|
||||
float/ft-float dock [ID|active|all] # put a floating window back (default: the active one), or all of them
|
||||
float/ft-float launch org.kde.dolphin # start an app (its desktop file name) floating
|
||||
float/ft-float run COMMAND [ARG...] # the same for a command
|
||||
float/ft-float close ID # close a window
|
||||
float/ft-float list # the spare outputs and what floats on them
|
||||
float/ft-float launch firefox action=new-private-window # one of the app's own actions, floating
|
||||
float/ft-float launch org.kde.dolphin screen=2 # its first window docked on screen 2
|
||||
float/ft-float activate ID # make a window the active one, unminimized
|
||||
```
|
||||
|
||||
## Spotwatch
|
||||
|
||||
Meta+Space opens spotwatch, a search pill straight ahead of where your head points, for apps and remote displays (`{host} - {display}`). Tab, Shift+Tab, and the arrow keys move through what it finds, and Enter or a click opens it. A running app's most recent window spins to the front with the lazy susan. An app that isn't running starts floating. A remote display connects if needed and spins to the front. Shift+Enter or a right click opens an item's menu: an app's own actions (a private window), Open new window, Open docked on this screen, and its windows; a remote display's Go to and Disconnect. Esc, Meta+Space again, or a click elsewhere closes it. It's the `spotwatch` action in Frametop Input Settings, bindable to mouse and controller buttons too, and opened that way it brings Frametop's keyboard. [spotwatch.md](spotwatch.md) has the decisions and how it works.
|
||||
|
||||
`spotwatch/ft-spotwatch` runs inside the desktop's Plasma session (log: `/tmp/frametop-spotwatch.log`) and runs the pill, `ft-spotpanel` (built by `screens/build.sh`), in the container. While the search is open, ft-screens sends it every key. Use counts are in `~/.config/frametop-spotwatch.json` and icons in `~/.cache/frametop/spotwatch`.
|
||||
|
||||
```
|
||||
spotwatch/test/spotwatch-test.py # the ranking and the search against a fake ft-screens and ft-floatd (offline)
|
||||
spotwatch/tools/render.py OUT.png # the panel's picture, without a headset
|
||||
```
|
||||
The layout is stored relative to your head when it's applied. `/tmp/frametop-layout.log` has the run from the last desktop start.
|
||||
|
||||
## Displays off and sleep
|
||||
|
||||
@@ -256,72 +172,14 @@ power/run.sh off | on # the displays off now, or back on
|
||||
power/run.sh log
|
||||
```
|
||||
|
||||
## Pausing for VR games
|
||||
|
||||
Paused, Frametop leaves the headset's CPU and GPU to a VR game. The input relay does it (`input/game_pause.py`), since it's the one part that always runs:
|
||||
|
||||
- The gaze service stops (`frametop-gaze`: ft-gazed, ft-gaze, our own eye tracker, the gaze panel), so nothing reads SteamVR's eye tracking. Our frame grabber, the root service `ft-eyegrab`, goes idle by itself 3 seconds after our eye tracker stops asking it for frames.
|
||||
- Hand tracking stops if it runs (`frametop-camd`, `frametop-hands`).
|
||||
- The desktop, as the Game optimization page of Frametop Input Settings says (`pause_desktop`): hidden (the default) or closed. Hidden, ft-screens hides every screen and floating window whatever the visibility mode, the hotkey, or the dashboard says, and gives KWin a frame callback once a second instead of every display frame. KWin draws a screen only after its frame callback, and its apps wait for theirs, so the desktop hardly draws, but its windows stay open. Remote desktop stops if it runs (`session/remote-ctl.sh`). Closed, `desktops.sh stop` closes the desktop and its windows, and resuming starts it again (about 12 seconds), in its start profile if it has one.
|
||||
- The relay lets go of the 3D mouse, releasing any click still held on it (a mouse button, a mapped controller button, or a key combination), and feeds pointer devices to its virtual mouse and keyboard, as with `POINTER=0`. Typing goes to Steam. Mapped buttons and key combinations do nothing but pausing, the Steam menu, dictation, and commands; a key combination that does nothing is typed as usual.
|
||||
|
||||
Resuming starts again only what pausing stopped, and plays a second sound. The pointer helper and ft-powerd keep running: they cost little, the helper is what says a game started, and stopping it would leave its virtual controller connected with its last pose.
|
||||
|
||||
Ways to pause and resume:
|
||||
|
||||
- The controller gesture, by default both thumbsticks clicked together twice: both go down within 0.3 seconds of each other, and the second time within 0.7 seconds of the first. The relay reads it from vrserver's web socket (`input/vrws.py`), which works whatever has input focus and takes nothing from the game, so the game sees the clicks too. The Game optimization page changes it: one or two of the buttons the Controllers page lists, pressed once or twice (one button always takes two), or none.
|
||||
- The Pause/resume Frametop action, on a mouse button, a key combination, or a controller button (outside games, like every mapped controller button).
|
||||
- VR games, with Pause while a VR game runs on (`pause_auto`, the default). The pointer helper tells the relay when a scene app starts and ends (`vrgame 1|0`, repeated every 5 seconds). A game starting pauses Frametop. A pause that starts while a game runs ends 5 seconds after the game does, unless another game starts first. Resumed during a game, Frametop stays on until that game ends. A pause that starts outside a game lasts until you resume. Flatscreen games aren't scene apps, so they don't pause it.
|
||||
- From a terminal or a script:
|
||||
|
||||
```
|
||||
input/ft-pause on | off | toggle # pause or resume
|
||||
input/ft-pause status # the state as JSON (the relay's "pause ?")
|
||||
input/vrws.py 10 # the controllers' buttons from vrserver's web socket, for 10 s
|
||||
input/test/pause-test.py # the gesture and the automatic pause, offline
|
||||
input/test/pause-buttons-test.py # a click held into a pause comes up, offline
|
||||
```
|
||||
|
||||
The state outlives a relay restart, in `/run/user/UID/frametop-pause.json`. A SteamVR restart while paused starts the gaze service with it, and the relay stops it again when the pointer helper comes back.
|
||||
|
||||
## Voice dictation
|
||||
|
||||
Frametop Voice (`voice/`, [voice/README.md](../voice/README.md)) types what you say where you type, recognised on the headset; nothing leaves it. `voice/ft-voiced` runs in Frametop's container as `frametop-voice.service` (installed by `voice/run.sh install`, which `install.sh` offers, yes by default), starting at login. It records with `pw-record` and transcribes live with Moonshine on the CPU; `frametop-voice-whisper.service` (whisper.cpp on the GPU) runs only while the wake word is on, for its commands. Its models, about 320 MB, are downloaded on install into `~/.local/share/frametop/voice/models` (`voice/models.sh`: fixed URLs, checked by sha256).
|
||||
|
||||
The input relay does the keyboard side. Dictation's keys are key combinations with the actions `dictate` (a press starts or stops) and `dictate_hold` (records while held); with Frametop Voice installed, the default key combinations add a Right Ctrl tap (`"97": "dictate"`, Right Ctrl alone: a right-hand modifier's own code is that key's tap only). Like every tap, it doesn't count with another modifier held, or with a key, click or scroll in between. The relay sends `toggle|start|stop <source>` to `@frametop_voice`, and dictation works while Frametop is paused. ft-voiced sends its text back as `type TEXT` on `@frametop_relay`; the relay types it, one key every 4 ms on a US layout, to ft-screens while typing goes to the desktop, otherwise on its virtual keyboard for Steam and games, and holds a character back while a modifier is held. Both sockets take these only from their own user. Frame controller buttons (left d-pad down, held, by default) are read by ft-voiced itself from vrserver's web socket (`input/vrws.py`), so games still get them.
|
||||
|
||||
Frametop Input Settings' Voice page turns it on and off and sets the controller buttons and their mode (hold or press to toggle), typing as you talk, sounds, and the wake word (off by default) and its phrases, in `~/.config/frametop-voice.json`; the keys are on its Keyboard page. Wake word commands that run a program, and advanced settings, are edited in that file by hand (voice/README.md).
|
||||
|
||||
```
|
||||
voice/build.sh && voice/run.sh install
|
||||
voice/run.sh status # the services, and ft-voiced's state as JSON
|
||||
voice/ft-voice toggle # start or stop dictation; also start, stop, cancel, type TEXT, wake on|off, events
|
||||
voice/run.sh log
|
||||
voice/models.sh check # every model's sha256
|
||||
voice/test/test_daemon.py # typing through a stand-in relay, push-to-talk, offline
|
||||
```
|
||||
|
||||
## Gaze pointer (experimental)
|
||||
|
||||
In gaze mode the 3D mouse's pointer goes where you look, and the mouse or the keyboard does the last bit. It needs the gaze service, which `install.sh` offers (yes by default) and `gaze/run.sh install` installs on its own: it builds it and runs `gaze/ft-gazed` as `frametop-gaze.service`, which starts with SteamVR. The service idles while the gaze isn't used: its eye tracker reader and our own eye tracker run only while gaze mode is on and someone wears the headset, while a check or the calibration runs, or while the Gaze page of Frametop Input Settings is open, and stop 30 seconds after ([gaze/README.md](../gaze/README.md)). Turn gaze mode on with the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1`, or a button or key combination mapped to Gaze pointer on/off.
|
||||
|
||||
- Meta+J left-clicks and Meta+K right-clicks where you look. A quick tap clicks where the dot was at the press. Hold instead, and the dot stays put in your view: turn your head until it's on what you meant, and let go to click there. Held still for `POINTER_GAZE_HOLD` (0.5 s), the press becomes a real one, and your head drags. Meta+K with Meta+J held presses where the dot is now, to drag from there, and a second Meta+K during that drag (a double Meta+K) pans and tilts what you're dragging while it's held.
|
||||
- The mouse's buttons work the same way, with the mouse steering instead of your head (`POINTER_GAZE_MOUSE=precision`, the default): the right button with the left held starts a drag, and a double right click pans and tilts what you're dragging. With `POINTER_GAZE_MOUSE_MOVE=held`, the default, the mouse only corrects: while the gaze has the pointer, moving it does nothing unless a button is held. `free` lets the mouse take the pointer any time. With the gaze stale for a second, in a game, or with the headset off, the mouse works as usual.
|
||||
- A correction before a click teaches the gaze service the tracker's error there. A correction bigger than `POINTER_GAZE_NUDGE_MAX` (55 degrees) isn't learned; it opens a quick check instead.
|
||||
- Calibration and checks run in a panel fixed to the headset (`gaze/panel/ft-gazepanel`, which the gaze service runs), from the Gaze page: Quick check is one dot, and also opens when you put the headset on. Calibrate is three rounds of dots, dark to bright; look at each dot and left click or press Meta+J to take it. Check headset fit shows, live, how well the tracker sees each eye. A right click or Meta+K closes the panel. Gaze mode on without a calibration opens Calibrate by itself, as soon as your eyes are seen. If gaze mode is on but can't follow your eyes yet (no calibration, the calibration can't open, the gaze service not running), the Gaze page says why under the Gaze pointer switch, and `gaze/ft-gazectl on` notes it.
|
||||
|
||||
[gaze/README.md](../gaze/README.md) has the details, our own eye tracker, and the gaze probe, a development tool.
|
||||
|
||||
## Hand tracking (experimental)
|
||||
|
||||
It costs a lot of the headset's CPU. `install.sh` offers it (yes by default, `--no-hands` skips it, and it needs `sudo` for ft-camd's file capabilities), and a release brings `ft-camd` and `ft-hands` built in its image. Installed, it runs while the Frametop desktop does (`HANDS_MODE=frametop`).
|
||||
Your hands show over the screens: where a tracked hand is between an eye and a screen, ft-screens lets that eye see the room through the screen. The same tracker also detects pinches, for clicking where you look with the gaze pointer (not wired to the pointer yet). It's optional: `hands/run.sh install`, or the last step of `install.sh`.
|
||||
|
||||
Your hands show over the screens: where a tracked hand is between an eye and a screen, ft-screens lets that eye see the room through the screen (a round hole per hand, `HANDS_MATTE=circle`; `mesh` and `capsules` are the older shapes). ft-handview (`frametop-handview.service`) draws your hands as a 3D hand model posed by the tracking: over games and the void with Room View off, and over the hole with it on (`HANDS_VIEW_ROOMVIEW=1`, the default). `HANDS_VIEW_MODEL` picks the model: `asset` (the default), `babylon`, `mrtk3`, `godot`, `mrtk2`, `robot` (a robot hand built in code from the tracked skeleton: light shells, dark joints, cyan accents), or `capsules` for the older tubes; `HANDS_VIEW_STYLE=ghost|solid` the look. The models, in `screens/assets/hands` with their licences: the WebXR Input Profiles' generic hand (`asset`; MIT, Copyright (c) 2019 Amazon), Babylon.js's hand meshes by Patrick Ryan (`babylon`; CC BY 4.0, https://github.com/BabylonJS/Assets), MRTK3's OpenXR hand (`mrtk3`; BSD-3-Clause, Copyright (c) 2025, Mixed Reality Toolkit Contributors), Godot XR Tools' hands by DigitalN8m4r3 (`godot`; CC0 1.0) and MRTK2's rigged hand (`mrtk2`; MIT, Copyright (c) Microsoft Corporation). The same tracker detects pinches and grips, and with `POINTER_HANDS=1` in `~/.config/frametop.conf` they work the pointer. In gaze mode your eyes aim and your hands click, Vision Pro style: a pinch clicks where you were looking as it began, and the hand ray stays off (`POINTER_GAZE_HAND_RAY=1` brings it back beside the gaze). Pinch, hold and move the hand a few centimetres to nudge the pointer slowly onto the target, then let go to click there; the correction teaches the gaze tracker, as the mouse's do. Pinch and hold still for half a second to press and drag with the hand. A middle pinch right-clicks the same way. Hold a pinch with one hand and pinch with the other to hold both buttons, left and right, each until its pinch opens. Without gaze mode a pinch is a press like the mouse's button, so a held pinch drags. A grip (closing the hand) presses and drags. To install it: `hands/run.sh install`. With `POINTER_HANDS=1` and gaze mode off, a bare hand raised and pointing also shows a ray that drives the pointer like a controller's laser (it comes up as the thumb nears the index or middle tip; a pinch clicks, a middle pinch right-clicks or scrolls, a three-finger pinch scrolls, reaching out and in pushes and pulls a carried panel), a controller used recently takes over its own hand only, and during a paused VR game hand tracking is parked rather than stopped, waking when the dashboard opens. Details: [hands/README.md](../hands/README.md), "Pinches and grips in the pointer". When tracking goes wrong, `ft-handsctl flag [note]` (or a voice command through frame-voice, "flag that") saves the last 12 seconds of hand crops and what the tracker made of them for the offline tools: ft-hands keeps them in RAM (`HANDS_FLAG_BUFFER=1`, about 50 MB), the drawn hands pulse cyan when a flag is taken (`HANDS_FLAG_FLASH`, `HANDS_FLAG_TONE`), and flags go to `HANDS_FLAG_DIR`. See [hands/README.md](../hands/README.md), "Flags".
|
||||
|
||||
- `ft-camd` borrows XRService's camera buffers and publishes the four IR tracking cameras to `/run/user/UID/frametop-hands/cam-ring`. It runs on the host as `frametop-camd.service`, with file capabilities that `hands/run.sh install` sets through sudo, and it drops them once set up. A rebuild clears them: `hands/run.sh caps`. The Hand Recorder's installer (`hands/rec/install.sh`) sets them the same way. `hands/run.sh uncaps` takes them back while neither the services nor the Hand Recorder is installed. `hands/run.sh uninstall` and `hands/rec/install.sh uninstall` run it after removing their own part, and `uninstall.sh` always takes them back.
|
||||
- `ft-hands` runs in the `dev` container as `frametop-hands.service`. It finds and triangulates the hands, and publishes `hands` (read by ft-screens' cutouts) and `gestures` (pinches and grips, read by the pointer helper) next to the ring.
|
||||
- `HANDS_MODE` says when they run: `off` (the default until install.sh installs hand tracking, which sets `frametop`) only after `ft-handsctl on`, until `ft-handsctl off` or SteamVR stops; `frametop` while the Frametop desktop runs; `always` whenever SteamVR runs, in games too. Frametop Input Settings' Hands page sets it and applies it at once (`hands/handsmode.py`), next to the hand cutouts' margin (`HANDS_MATTE_MARGIN`, live). The install links `ft-handsctl` into `~/.local/bin`. `ft-handsctl status` and `ft-handsctl log` (or `hands/run.sh status` and `log`) show how they're doing, `ft-handsctl cutouts on|off` turns just the cutouts off, and `ft-handsctl gestures` shows pinches and grips live.
|
||||
- Settings in `~/.config/frametop.conf`: `HANDS_SWAP_SIDES` (`auto`, the default: ft-hands tells from the hands when some SteamVR restart has swapped the side cameras' names, and fixes them; `0` or `1` force them, and `hands/tools/check_sides.py --ring` tells which is right), `HANDS_CPUS`, the cameras it tracks with (`HANDS_CAMERAS`, `HANDS_BRIGHT`, `HANDS_BRIGHT_ON`, `HANDS_BRIGHT_OFF`, `HANDS_COLOR_LEFT`, `HANDS_COLOR_CROP`), and the pointer helper's `POINTER_HANDS`, `POINTER_PINCH_GAIN`, `POINTER_PINCH_DEADZONE`, `POINTER_GRIP_GAIN`, `POINTER_GRIP_BELOW`, typing's `POINTER_PINCH_TYPING`, `POINTER_TYPING_RAISE`, `POINTER_TYPING_BELOW`, `POINTER_TYPING_POSTURE`, `POINTER_TYPING_RECENT`, the gaze pinch's `POINTER_GAZE_PINCH`, `POINTER_GAZE_HAND_RAY`, `POINTER_GAZE_PINCH_DEADZONE`, and `POINTER_GAZE_PINCH_GAIN`, the hand ray's `POINTER_HAND_RAY`, `POINTER_RAY_DEADZONE`, `POINTER_RAY_PUSH`, `POINTER_RAY_SCROLL`, `POINTER_RAY_RELEASE`, `POINTER_RAY_GRACE`, its smoothing and taps (`POINTER_RAY_SMOOTH_MINCUTOFF`, `POINTER_RAY_SMOOTH_BETA`, `POINTER_RAY_SMOOTH_HOLD`, `POINTER_RAY_STEADY`, `POINTER_RAY_TAP`, `POINTER_RAY_TAP_DEADZONE`), its body and hand models (`POINTER_RAY_TORSO_WRISTS`, `POINTER_RAY_HAND_WEIGHT`, `POINTER_RAY_HAND_PITCH`, `POINTER_RAY_HAND_YAW`, `POINTER_RAY_ANCHOR`), and ft-screens' near reach (`HANDS_NEAR_REACH`, 0.075 m), pokes (`HANDS_POKE_ANGLE`, `HANDS_POKE_TRAVEL`, `HANDS_POKE_REST_BELOW`, `HANDS_POKE_REST_NEAR`) and near grabs (the border band `HANDS_GRAB_BAND`, a pinch on the content clicking `HANDS_PINCH_CLICK`, the card pinch `HANDS_CARD_PINCH`, the fist grab `HANDS_FIST_GRAB` and its reach `HANDS_FIST_REACH`, a second hand joining a held panel `HANDS_JOIN_REACH`, two hands turning a panel on every axis `HANDS_STRETCH_TURN`, the round buttons' hit area for hands `HANDS_BUTTON_REACH` and pinches pressing them `HANDS_PINCH_BUTTONS`, `HANDS_GRAB_SMOOTH_MINCUTOFF`, `HANDS_GRAB_SMOOTH_BETA`, `HANDS_GRAB_DEADZONE`, the wrist turning a carried panel: `HANDS_GRAB_ROTATE`, `HANDS_GRAB_ROTATE_DEADZONE`, holding through tracking loss `HANDS_HOLD_LOST_MS`, ignoring tracking no hand makes `HANDS_TRACK_GATE`), throws (the flick `HANDS_FLICK`, `HANDS_FLICK_SPEED`, `HANDS_FLICK_OPENING`, a carry let go fast `HANDS_THROW`, `HANDS_THROW_SPEED`, the push `HANDS_PUSH`, `HANDS_PUSH_SPEED`, where they go, `HANDS_THROW_DISTANCE`, and at what width, `HANDS_SEND_AWAY_WIDTH`), its "grab that" gesture (`HANDS_GRAB_THAT`, `HANDS_GRAB_THAT_DISTANCE`, `HANDS_GRAB_THAT_WIDTH`, which panels: `HANDS_GRAB_THAT_FROM`, `HANDS_GRAB_THAT_MARGIN`, `HANDS_GRAB_THAT_MARGIN_DEG`, `HANDS_GRAB_THAT_FACING`), the soft white halo round a gesture's target panel (`HANDS_TARGET_GLOW`), and the soft glow round the corner, edge or button a hand is at (`HANDS_EDGE_GLOW`). The example config explains each.
|
||||
- `ft-camd` borrows XRService's camera buffers and publishes the four IR tracking cameras to `/run/user/UID/frametop-hands/cam-ring`. It runs on the host as `frametop-camd.service`, with file capabilities that `hands/run.sh install` sets through sudo, and it drops them once set up. A rebuild clears them: `hands/run.sh caps`.
|
||||
- `ft-hands` runs in the `dev` container as `frametop-hands.service`. It finds and triangulates the hands, and publishes `hands` (read by ft-screens' cutouts) and `gestures` (pinches) next to the ring.
|
||||
- Both start and stop with SteamVR. `hands/run.sh status` and `hands/run.sh log` show how they're doing.
|
||||
- Settings in `~/.config/frametop.conf`: `HANDS_SWAP_SIDES` (after some SteamVR restarts the side cameras' names come out swapped, and hands land beside the holes; `hands/tools/check_sides.py --ring` tells) and `HANDS_CPUS`.
|
||||
|
||||
Details, options, and the recording and replay tools are in [hands/README.md](../hands/README.md).
|
||||
|
||||
@@ -333,8 +191,6 @@ It listens on port 5900 on the Frame's Tailscale address only, not the LAN, so i
|
||||
|
||||
No VNC server can capture KWin on SteamOS directly: `krfb` needs `xdg-desktop-portal-kde`, which SteamOS doesn't ship, and `wayvnc` only works with wlroots compositors. So `session/remote-desktop.sh` captures the desktop with KDE's `krdpserver --plasma` on `127.0.0.1:3390`, and `session/vnc-bridge.sh` runs TigerVNC's `Xvnc` on display `:20` with a FreeRDP client inside it and serves that. Both run in the `dev` container, and the extra hop adds a little latency. krdp streams every screen; the VNC screen is the primary's size, and the FreeRDP window is shifted so the primary fills it (`ft-layout remote-view` gives the offset). krdp's own `--monitor` would stream just one screen, but it maps the pointer as if that screen sat at 0,0, so clicks would miss. When the layout changes, the VNC screen resizes and FreeRDP reconnects within a few seconds.
|
||||
|
||||
FreeRDP runs only while a VNC viewer is connected, because while it's connected krdp captures and encodes every redraw. With no viewer, krdp has no RDP connection and so captures nothing, and Xvnc shows a black screen. When a viewer connects, the bridge starts FreeRDP, and the desktop appears about 3 seconds later; FreeRDP stops 45 seconds after the last viewer leaves (`VNC_IDLE_SEC` in the bridge's environment). The bridge looks for viewers with `ss` whenever Xvnc logs something, as it does for every connection, and every 5 seconds otherwise. While a viewer is connected, the bridge asks ft-screens to draw every screen at full rate (`watch 15` on `@ft_screens`, renewed every 5 seconds), so screens you aren't looking at in the headset, or a headset on a stand, don't stream at a low rate. It reads the primary screen's place again (`ft-layout remote-view`) only while FreeRDP runs, after `~/.config/frametop/kwinoutputconfig.json` or `~/.config/frametop-layout.json` changes, and once a minute.
|
||||
|
||||
With remote access on, the nested KWin runs with `KWIN_WAYLAND_NO_PERMISSION_CHECKS=1` and `KWIN_SCREENSHOT_NO_PERMISSION_CHECKS=1`, so any app in the Frametop desktop could capture its screens or inject input. The second one lets scripts take screenshots through KWin's `org.kde.KWin.ScreenShot2` D-Bus interface. This applies only to that desktop, not the stock one. Port 3389 is SteamOS's own `xrdp`, which starts a separate X11 session rather than showing the VR desktop.
|
||||
|
||||
## Limits
|
||||
|
||||
@@ -1,501 +0,0 @@
|
||||
# Remote displays: design notes
|
||||
|
||||
These are the notes from building remote displays, from the first exploration (2026-10-06) on: what the Frame's hardware gives us, the spikes, and the decisions. To set a computer up, see [Remote displays in the README](../README.md#remote-displays) and the host kits: [Windows](../host/windows/README.md), [macOS](../host/macos/README.md), [Linux](../host/linux/README.md). How the pieces work now is in [reference.md](reference.md#frametop-remote-displays).
|
||||
|
||||
The idea: show desktops streamed from other machines as Frametop panels that behave like the native screens. They get placement, curve, pinning, layouts, lasers, gaze, and attention-based frame rates. The target is up to 5 remote displays from 2 hosts, with no more headset overhead than 5 native screens.
|
||||
|
||||
## What the hardware gives us
|
||||
|
||||
These were checked on the Frame (SteamOS kernel 6.18, SM8650 / Snapdragon 8 Gen 3, Mesa 26.3 Turnip on Adreno 750).
|
||||
|
||||
- **Decoder.** `/dev/video22` (`/dev/video-dec0`) is `qcom-iris-decoder`, Qualcomm's downstream iris driver (`drivers/media/platform/qcom/vcodec/iris`). It takes H.264, HEVC, and VP9 up to 8192x8192. AV1 isn't supported. The capture queue can produce `Q08C` (NV12 in Qualcomm's UBWC compressed layout), linear NV12, and NV21. It also lists RGBA (`AB24`, `QC24`), but those write a 10-bit UBWC YUV picture (S1), so the decoder has no usable RGB output.
|
||||
- **10-bit works on Valve's kernel.** Steam Link VR's client (`vrlink.txt`, `SVLCodecV4L2`) decodes into `Q10C`, 10-bit UBWC, at 1152x4608. That is both eyes stacked in one frame, so it uses one decode session. Valve drives the decoder through the raw V4L2 stateful API, as `vr-recorder` does for the encoder.
|
||||
- **Decoder budget.** The driver carries `MAX_SESSION_COUNT`, `MAX_MBPF`, and `MAX_MBPS` capability tables. For SM8650 the upstream values are 16 sessions, 278,528 macroblocks per frame, and 7,776,000 macroblocks per second (one 8K stream at 60 fps). A session that would go over the budget is refused when it starts. The downstream numbers on Valve's kernel are still to be confirmed by spike S2.
|
||||
- **SteamVR imports YUV but doesn't convert it.** `IVRIPCResourceManagerClient::GetDmabufModifiers(VRApplication_Overlay, …)` returns LINEAR and `0x0500000000000001` (`DRM_FORMAT_MOD_QCOM_COMPRESSED`) for NV12, P010, and the RGB formats. The probe is in `~/.cache/remote-displays-spike/modprobe.cpp`. `DmabufAttributes_t` takes multiple planes. This is the call ft-screens already makes for KWin's buffers (`ft_vr_screen_present`, `vr.cpp:2080`). The import works, but vrcompositor samples the planes as they are: red shows Cr, green Y, blue Cb (S1). `DmabufAttributes_t` has no colour-space fields to change that.
|
||||
|
||||
So a decoded frame needs one GPU pass before SteamVR sees it: NV12 to RGBA, about 0.13 ms of GPU time for a 3440x1440 frame at full clock. It is the same kind of pass as the hand cutouts' (`screens/handcut.cpp`), and native screens pay for one of the same size: KWin's composite of each output.
|
||||
|
||||
## Is the decoder the bottleneck?
|
||||
|
||||
Mostly no. Macroblocks per second against the 7,776,000 budget, at 60 fps:
|
||||
|
||||
| Displays | Share of the decoder |
|
||||
|---|---|
|
||||
| 5x 1920x1080 | 31% |
|
||||
| 5x 2560x1440 | 56% |
|
||||
| The current layout (3440x1440 + 2x 1440x1920) | 32% |
|
||||
| 5x 2560x1440 at 90 fps | 83% |
|
||||
| 4x 3840x2160 | 100% |
|
||||
| 5x 3840x2160 | 125%, refused |
|
||||
| Steam Link VR's own stream (1152x4608 at 120 Hz) | about 32% |
|
||||
|
||||
Five 1440p displays fit at full rate with room left. Five 4K displays only fit at about 48 fps or less. That number is what gets declared at session start; actual load is lower, because hosts send frames only when the screen changes (see "Keeping the decoder under budget").
|
||||
|
||||
### The planned setup
|
||||
|
||||
Two hosts that share the same two desk monitors: a Mac (14-inch MacBook Pro) with those two plus its built-in display, and the test PC (Windows, RTX 5090) with the two. Windows and Linux hosts run Vibepollo, which streams real displays and creates virtual ones on demand. The Mac runs Sunshine, which began with one display; it now runs one instance per display (see "Host side"). At 60 fps:
|
||||
|
||||
| Display | Share of the decoder |
|
||||
|---|---|
|
||||
| Super ultrawide 5120x1440 (5160x1440 as given; same load within 0.2%) | 22.2% |
|
||||
| Built-in 3024x1964 (native pixels) | 17.9% |
|
||||
| Ultrawide 3440x1440 | 14.9% |
|
||||
| Mac, one display | 14.9% to 22.2% |
|
||||
| The test PC, ultrawide + super ultrawide | 37.1% |
|
||||
| Both hosts today (three displays) | at most 59.4% |
|
||||
| Five: the three above plus two 3440x1440 virtual displays on the test PC | at most 89.2% |
|
||||
| The original five (all three Mac displays + the test PC's two) | 92.2% |
|
||||
|
||||
Every combination fits under the driver's limit at 60 fps. The original five stay as the worst case for spike S2, so the Mac can grow past one display later without a new budget. These are declared numbers, with every display changing every frame at once. With damage-driven hosts the real load is far lower.
|
||||
|
||||
Notes:
|
||||
|
||||
- The 5120-wide display needs HEVC. H.264 hardware encoders stop at 4096 pixels wide, and the iris decoder takes HEVC up to 8192. HEVC for every stream keeps it simple.
|
||||
- Streams don't need more than 60 fps. The Frame's display runs at 90 Hz, and the MacBook's 120 Hz ProMotion would double the built-in display's load for nothing.
|
||||
- A stream can be smaller than its display, because the host scales before encoding. A panel in VR covers far fewer headset pixels than the display has; a 1 m wide panel at 1 m spans about 53°, which is roughly 1,000 headset pixels across (estimate; the Frame's pixels per degree haven't been measured here). The built-in display at 2268x1473 instead of 3024x1964 would cost 10% instead of 18%. That's the lever when Steam Link VR also needs the decoder, or when a virtual display is made larger.
|
||||
- The desk monitors are shared through input switching. A monitor switched to the other machine may disappear from the first one, depending on the monitor and the cable. Real-display streams only work for monitors that host currently sees. In the headset, virtual displays avoid the question.
|
||||
|
||||
A remote display should cost less than a native one everywhere else:
|
||||
|
||||
| Per display | Native screen | Remote display |
|
||||
|---|---|---|
|
||||
| Apps | Run on the Frame's CPU | Run on the host |
|
||||
| Composition | KWin draws each output on the Adreno GPU | One GPU pass per new frame turns the decoder's NV12 into RGBA |
|
||||
| Hand-off to SteamVR | XRGB dmabuf, zero copy | RGBA UBWC dmabuf, zero copy |
|
||||
| Sampled by vrcompositor | 4 bytes per pixel | The same |
|
||||
| New work | — | Network receive and reassembly, V4L2 queueing |
|
||||
|
||||
The new costs are CPU for receiving packets, plus the decoder's and Wi-Fi radio's power and heat. Heat matters because the SoC slows down when it gets hot. Those are what the spikes need to measure.
|
||||
|
||||
## Keeping the decoder under budget
|
||||
|
||||
1. **Damage-driven hosts.** Sunshine and its forks send a new frame only when the captured screen changes, plus duplicates down to `minimum_fps_target` (default half the stream rate, settable to 1). With `minimum_fps_target = 1`, a static display costs about one decoded frame a second, the same idea as a native screen with no damage. A display playing video costs full rate, as a native video screen does.
|
||||
2. **Out of sight means 1 frame per second, whatever the host sends.** A display outside your view updates once a second, even when the host is sending 60 fps from a game. See "Displays out of sight" below for how. Concealed panels, and every panel while the desktop is paused for a game, disconnect fully after a grace period. That frees the host's encoder and the network.
|
||||
3. **An admission budget.** Before connecting, Frametop adds up the declared load of every stream: width/16 × height/16 × fps. If a new stream would go past the driver's limit, minus whatever else is decoding, Frametop lowers settings before connecting: first fps, then resolution, starting with the displays that get the least attention. Today's three displays come to at most 59%, which fits next to Steam Link VR's stream (about 32%). Five displays (89-92%) fit alone but not next to it, so a streamed PC game with five remote panels up would push them down to about 45 fps (or some lower and others higher).
|
||||
4. **No B-frames, low-latency decode.** Sunshine doesn't send B-frames, so every decoded frame can be shown at once. The decoder gets a small capture pool (6-8 buffers). Valve's client found the standard `DISPLAY_DELAY` controls unsupported on this driver (`EINVAL` in `vrlink.txt`). They aren't needed: S1 showed that with `Q08C` each frame comes out as soon as it's decoded (linear NV12 holds two more).
|
||||
5. **Tiling only as a fallback.** Valve tiles both eyes into one frame because they always change together. Desktops don't: one busy display would make the whole canvas decode at full rate, and a hidden tile can't be skipped. Per-display streams keep each display's cost proportional to its own changes. Tiling makes sense only for a host with many small, mostly static displays, and Sunshine can't capture a spanning desktop on Windows anyway.
|
||||
|
||||
The Moonlight protocol can't change resolution, fps, or bitrate mid-stream; that takes a reconnect. With stock hosts, attention changes therefore act on the client: in what gets decoded (2), not in what the host sends.
|
||||
|
||||
## Displays out of sight
|
||||
|
||||
Remote panels follow the native screens' attention rules (`UpdateAttention`, `vr.cpp:855`):
|
||||
|
||||
| Attention | Native screen | Remote display |
|
||||
|---|---|---|
|
||||
| Focused (within 12° of where you look) | Full rate | Full rate |
|
||||
| In view (within 60°) | 15 Hz, or full rate while it plays video | Full rate. Every frame has to be decoded anyway (below), and the host sends only what changed |
|
||||
| Hidden (out of view) | 1 Hz | 1 Hz, even when the host sends 60 |
|
||||
|
||||
The "video" signal comes for free: a damage-driven host sends frames only when its screen changes, so frames arriving faster than 10 a second for 8 frames in a row mean video, matching `VIDEO_COMMITS` and `VIDEO_HZ` in `compositor.c`.
|
||||
|
||||
The stream's fps is ours to choose. A game running at 144 Hz on the host is captured and sent at the fps the stream asked for (60 at most), so 144 never reaches the decoder.
|
||||
|
||||
**Why 1 Hz can't just skip frames.** Each frame in a stream is coded as changes to the frame before. Decoding frame 60 needs frames 1 to 59. Throwing away 59 of every 60 frames breaks the chain, and the next frame decodes as garbage. Valve's client works the same way: on a stall it asks the host for a new full frame (an "IFrame" in `vrlink.txt`).
|
||||
|
||||
**With stock hosts (Moonlight protocol): keyframe sampling.** While a panel is hidden, ft-stream:
|
||||
|
||||
1. drops every incoming frame before the decoder (moonlight-common-c's decode callback gets a `DECODE_UNIT` with `frameType`; return `DR_OK` without queueing);
|
||||
2. calls `LiRequestIdrFrame()` once a second, and decodes and shows only the IDR frame that comes back, which needs no earlier frames;
|
||||
3. when the panel comes back into view, requests one more IDR and goes back to decoding everything. The last frame, at most a second old, stays up until it arrives, about one round trip plus one frame later.
|
||||
|
||||
What that saves and what it doesn't, for a hidden display whose host sends 60 fps:
|
||||
|
||||
| Cost | Saved? |
|
||||
|---|---|
|
||||
| Decoder work | About 59 of 60 frames. One IDR costs a bit more to decode than one change frame |
|
||||
| Panel updates in vrcompositor | 59 of 60 |
|
||||
| Network traffic | No. The host keeps sending 60 fps, plus one larger IDR a second. A 5120x1440 IDR is a few hundred KB (estimate), so a few Mbit/s extra |
|
||||
| Frame CPU to receive, reassemble, and repair packets | No. moonlight-common-c still handles every packet. Patching it (it's GPL, and so is ft-stream) to discard hidden video packets early would cut most of this |
|
||||
| Host encoder | No |
|
||||
|
||||
**With a host that takes an fps change mid-stream: real 1 Hz.** If the host can be told "this display is out of sight, send 1 fps", the hidden display costs almost nothing anywhere. That's network, Frame CPU, decoder, and host encoder alike, and no IDRs are needed. The 1 Hz frames are ordinary change frames against the one a second earlier. This needs our own streamer, or a Sunshine fork with a control message that changes the capture rate. Owning a host streamer was rejected for its maintenance cost (see "Alternatives considered"), so this stays a possible upstream contribution. Vibepollo is the likeliest place to propose it: it already extends the protocol, since its own Moonlight fork sends a VRR pacing request that changes how the host captures.
|
||||
|
||||
Spike S3 checks the stock-host version: whether Sunshine honours an IDR request every second, how big and how late those IDRs are, and what receiving a hidden 60 fps stream costs the Frame's CPU.
|
||||
|
||||
## Proposed architecture
|
||||
|
||||
Frametop maintains only the headset side. The hosts run existing, separately maintained streamers that speak the Moonlight protocol: Vibepollo on Windows and Linux, Sunshine on macOS (see "Host side").
|
||||
|
||||
```
|
||||
host machine: Vibepollo (Windows, Linux) / Sunshine (macOS), one stream per display
|
||||
│ RTSP + RTP over UDP, ENet control
|
||||
▼
|
||||
ft-stream (one process per remote display, on the Frame, GPLv3)
|
||||
moonlight-common-c (protocol) + pairing (moonlight-embedded's libgamestream)
|
||||
iris decoder session: OUTPUT = HEVC, CAPTURE = Q08C, VIDIOC_EXPBUF
|
||||
GLES pass: Q08C → ring of 3 RGBA buffers (EGL YUV hints = the stream's colour space)
|
||||
│ unix socket, SCM_RIGHTS
|
||||
▼
|
||||
ft-screens (existing, MIT)
|
||||
remote screen type: ImportDmabuf once per ring buffer,
|
||||
SetOverlayTexture per frame, buffer returned to ft-stream when replaced
|
||||
panel input → ft-stream → LiSendMousePositionEvent / LiSendKeyboardEvent / …
|
||||
```
|
||||
|
||||
**Why one process per display.** moonlight-common-c keeps global state, so it runs one stream per process. It and libgamestream are GPLv3 while Frametop is MIT, and a helper that talks over a socket keeps the licences apart (this isn't legal advice). ft-stream lives in its own directory with its own licence file. Separate processes also mean a stalled stream or a decoder reset can't freeze the native screens. Each process pairs as its own client, which Vibepollo needs anyway: it ties each Remote Monitor to the client that opened it, one role per client, so every display needs its own client certificate and pairing.
|
||||
|
||||
**Pairing with a host token (chosen 2026-10-07).** Since every display is its own client, PIN pairing would mean one PIN and one round of permission clicks per display. Vibepollo has neither a multi-use PIN (its one-time PINs pair one client each) nor a way to pair several certificates at once. It does have scoped API tokens for its Web UI API (`POST /api/token`, sent as `Authorization: Bearer`). So the user creates one token on the host, limited to submitting pairing PINs (`POST /api/pin`), listing clients and setting their permissions (`GET /api/clients/list`, `POST /api/clients/update`), listing the host's monitors (`GET /api/display-devices`), and placing Remote Monitors (`GET`/`PUT /api/clients/display-layout`). A script on the host makes it from the Web UI login (`make-frametop-token`), and the user enters it once in Frametop's "add host" dialog. From then on, Frametop makes a client for each new display, pairs it by sending its own PIN, and grants it launch, mouse, and keyboard (Vibepollo gives a new client only list and view). It sizes Remote Monitors like the host's real monitors, and unpairs a display's client with that client's own certificate when the display is removed. The token can't change the host's settings and can be revoked in the Web UI. It can change any client's permissions, though, so Frametop keeps it like a password (a file only the user can read). Vibepollo's client update replaces the whole client record, so Frametop always sends every field.
|
||||
|
||||
**Why ft-stream converts.** ft-screens then gets RGBA dmabufs, as it does from KWin. Each decoder buffer goes back to the decoder right after the pass, so the decoder's pool doesn't depend on what SteamVR still holds. A GPU fault in one stream stays in its own process. ft-stream asks the host for BT.709 limited range (Moonlight's colour space and range settings) and gives the converter the same as EGL hints. S1 showed all four matrix and range combinations come out right when the hints match the stream.
|
||||
|
||||
**Opening a display.** For a host's main session, ft-stream launches the host's desktop app as Moonlight does. For a Vibepollo virtual display it launches the synthetic "Remote Monitor" app (id `2147483505` in `remote_session.h`), and the requested stream size and fps become the virtual display's mode. When the stream drops, Vibepollo keeps that display and its windows by default (`remote_monitor_disconnect_on_stream_end = false`), and ft-stream reconnects with "Resume" (id `2147483501`). So a concealed panel can disconnect fully without the host rearranging its windows.
|
||||
|
||||
**The socket protocol**, roughly:
|
||||
|
||||
- ft-stream → ft-screens: `buffers` (count, width, height, DRM format, modifier, offset and pitch, with the dmabuf fds of the RGBA ring) after each (re)configuration; `frame i` when ring buffer i holds a new picture (sent once the GPU is done, so no fence is needed); `title`, `state` (connecting, live, waiting for a keyframe, lost).
|
||||
- ft-screens → ft-stream: `release i` when buffer i is no longer on screen; `attention focused|view|hidden|concealed`; input events.
|
||||
|
||||
**In ft-screens**, the explorer's map gives the seams:
|
||||
|
||||
- Remote screens get `g_screens` entries through a `MakePanel` variant, with indices out of KWin's first-free-slot range (`compositor.c:279`) and the overlay keys `frametop.remote.N`. That brings visibility, attention, lasers, controls, spin, pinning, and hand cutouts.
|
||||
- Frames enter through an eventfd on the wl event loop and go through the same `ft_vr_screen_present` path, keyed by ring buffer, so the `g_imports` cache hits: 3 imports per display.
|
||||
- The pointer helper needs the new prefix in `FramePanel` (`ft-pointer.cpp:485`). The `screens` reply and `get N` need to list remote screens so gaze hit-testing sees them (`ft-gaze.cpp:302`).
|
||||
- `frametop-layout.json` gets a separate `hosts` list, since the `screens` array also sets KWin's output count. The user adds a host once, and its displays come as a bundle: Frametop pairs a client per display, starts their streams when the desktop starts, and stops them with it. Each host entry has the address, the token's file, and its displays. Each display has which one it is (the main session or a Remote Monitor), its own stream settings (size, fps, bitrate, and later codec and HDR), and the usual place, width, curve, and pin. ft-layout, profiles, and Display Settings learn the new list; Display Settings shows a host with its displays under it, each with its own settings (user decision, 2026-10-07).
|
||||
|
||||
**Input.** `handle_vr_event` (`compositor.c:333`) branches on the screen type. A remote screen sends:
|
||||
|
||||
- pointer motion as `LiSendMousePositionEvent(x, y, w, h)` in stream pixels;
|
||||
- buttons as `LiSendMouseButtonEvent`;
|
||||
- scroll as `LiSendHighResScrollEvent`, which matches the existing notches × 120.
|
||||
|
||||
A drag can cross panels, for example a window dragged from one of the host's displays to another. SteamVR sends a held button's moves only to the panel where the press began, with coordinates off that panel once the laser leaves it. The panel under the laser gets nothing (S3 measured this). So while a button is held on a remote screen, ft-screens hit-tests the laser (`ComputeOverlayIntersection`) against the host's other remote screens and sends the position to the ft-stream of the screen it hits. Moves that land on no panel are dropped, never clamped: clamping pins the host's cursor to the first display's edge. The host's input is shared across its sessions, so Windows sees one drag. But the release goes through the stream the press went through: Vibepollo takes a mouse button's release only from the client that pressed it (`mouse_press_owner` in its `src/input.cpp`). Sent through the display under the laser, it was dropped, and the window stayed on the pointer (found in the headset, 2026-10-07; `g_pressed_on` in `remote.c`).
|
||||
|
||||
Keyboard focus follows the last panel clicked. When it's a remote one, `send_key` and `relay_button` send evdev codes to its ft-stream, which maps them to Windows virtual-key codes (moonlight-qt has the table). Sunshine on macOS maps the Windows key to Cmd and Alt to Option. The host draws its own cursor into the video, so it lags the laser by one round trip.
|
||||
|
||||
**Won't work across the boundary:** drag and drop, the clipboard (the Moonlight protocol has none), floating windows, and KWin window rules. A remote display is a picture of another machine's monitor.
|
||||
|
||||
## Host side
|
||||
|
||||
**Decision for v1 (2026-10-07): real monitors only.** Vibepollo 2.0.0's Remote Monitors broke the test PC's monitor layout every time one was removed (see "S3 results so far"), while streaming a real monitor changes nothing on the host. Frametop can make extra screens in the headset itself, and the feature is mostly for controlling other machines, so v1 streams only a host's real monitors; virtual displays wait until the Vibepollo bugs are fixed. One host program streams one real monitor (a second client launching the main app joins the display already streaming), so each extra real monitor needs its own host instance: its own config file and ports (`port` 100 apart) and `output_name` set to that monitor. On the test PC that's Vibepollo for the primary (it also serves the Steam Deck and the Mac) and a plain Sunshine instance for the LC34G55T. Plain Sunshine has no scoped API tokens, so a Sunshine instance is paired once by PIN; it has a single Frametop client anyway. Frametop's host bundle becomes a list of instances, one per real monitor.
|
||||
|
||||
The requirement: each host streams its real displays, and virtual displays it creates on demand. Frametop doesn't maintain a host streamer; it uses existing ones and ships setup notes or scripts for them. State checked 2026-10-06.
|
||||
|
||||
| Host OS | Streamer | Displays |
|
||||
|---|---|---|
|
||||
| Windows | Vibepollo | 1 main session (real or virtual) + up to 4 virtual Remote Monitors |
|
||||
| Linux (Arch, CachyOS; beta) | Vibepollo | The same, with at most 4 virtual displays at once |
|
||||
| macOS | Sunshine | Every real display, one Sunshine instance each (`host/macos`); no virtual displays |
|
||||
|
||||
### Windows and Linux: Vibepollo
|
||||
|
||||
Vibepollo 2.0.0 (2026-09-30) is a Sunshine fork. One install gives:
|
||||
|
||||
- **A main session.** As on any Sunshine host, this is a real display (`virtual_display_mode = disabled`, picked by `output_name`) or a virtual display created for the client (`per_client`, the default on Windows 11 and Linux).
|
||||
- **Up to four Remote Monitors** (`max_client_vdds = 4` in `remote_session.h`). Each is a virtual display at the size and rate its client asks for, streamed and captured on its own (`capture_plan` in `remote_session.cpp`). They don't need a main session: with nothing running, the app list still offers Remote Monitor. They are always virtual and can't stream a real display.
|
||||
|
||||
That covers virtual displays completely, up to five from one host. Real displays are the limit: only the main session shows one, so one Vibepollo install streams one real display. A second real display at the same time needs a second host instance with its own config file, port, and `output_name`. On Windows that could be a plain Sunshine instance next to Vibepollo (untested). On Linux, Vibepollo expects to be the only host install on the machine. For the test PC this means one desk monitor streams as the main session and further displays are virtual, unless S3 shows a second instance works. Virtual displays also avoid the input-switching question (see "The planned setup").
|
||||
|
||||
Settings Frametop's setup notes change from the defaults:
|
||||
|
||||
- `virtual_display_layout`: the default `exclusive` turns off the host's other monitors while a virtual main session streams. `extended` keeps them on; `extended_isolated` also stops the host's own mouse from wandering onto the virtual display. Remote Monitors don't use it: Vibepollo always adds them next to the monitors already on (`apply_remote_monitor_composition` in `nvhttp.cpp`).
|
||||
- `minimum_fps_target = 1`, so a static display costs about one frame a second.
|
||||
- `remote_monitor_mute_audio = true`, unless that display should carry audio.
|
||||
|
||||
Both platforms send frames only when the screen changes. On Linux the virtual displays use presentation-driven capture: sparse changes are captured at once, and faster ones are coalesced to the stream's fps. The virtual outputs have no cursor plane, so the cursor is in the video, as with every Moonlight host.
|
||||
|
||||
PyroWave, Vibepollo's wavelet codec, isn't usable here. It decodes on the client's GPU and needs hundreds of Mbit/s over wired LAN. Frametop uses HEVC.
|
||||
|
||||
**Windows (the test PC).** The chosen setup (2026-10-06): one desk monitor streams as the main session, a real display (`virtual_display_mode = disabled`, `output_name` set to it). The other desk monitor is replaced by a Remote Monitor, a virtual display at that monitor's size, which Vibepollo releases when Frametop ends the connection (`remote_monitor_disconnect_on_client_disconnect = true`). A dropped connection keeps it (`remote_monitor_disconnect_on_stream_end = false`), so a Wi-Fi drop or a panel that disconnects while out of sight doesn't move its windows; ft-stream releases it explicitly with "Disconnect Monitor" (id `2147483502`) when the remote display is removed or Frametop exits. Each Remote Monitor's place next to the real monitors is set per client in the Web UI. GeForce allows 8 concurrent NVENC sessions per system. Installing Vibepollo and its display driver needs an admin; the test PC's `maptrainer` account isn't one.
|
||||
|
||||
How it's set up on the test PC (2026-10-06). Vibepollo installed over Apollo in `C:\Program Files\Apollo` (service `ApolloService`) and kept Apollo's settings, pairings, and Web UI login. The existing clients (a Steam Deck and a Mac) launch "Desktop" or "Steam Big Picture", which stream a virtual display with the other monitors turned off (`dd_configuration_option = ensure_only_display`). Frametop leaves them alone and gets its own app instead:
|
||||
|
||||
- "Frametop primary display": `"display-output": ""` streams the real primary monitor, and `"dd-configuration-option": "disabled"` leaves the other monitors as they are. ft-stream launches this app for the main session. These are the fields the Web UI writes for "use my own display" (`process.cpp` reads them).
|
||||
- `minimum_fps_target = 1` globally, which Remote Monitors use. "Desktop" and "Steam Big Picture" keep 120 as per-app `config-overrides`, so the Deck and the Mac stream as before.
|
||||
- `remote_monitor_mute_audio`, `remote_monitor_disconnect_on_client_disconnect` on, `remote_monitor_disconnect_on_stream_end` off.
|
||||
|
||||
The config folder is admin-only, so the change is a script for the user to run (`D:\remote-displays\vibepollo-setup\run-setup.cmd`, with a backup and an optional Web UI password reset). It ran on 2026-10-07. Vibepollo rewrites apps.json each time it starts and adds its built-in "Remote Input" and "Remote Monitor" apps, so any later change must start from the live file.
|
||||
|
||||
**Linux (beta).** x86_64 only: Arch Linux or CachyOS, KDE Plasma 6 on Wayland started by SDDM or Plasma Login Manager, Linux 6.16 or newer with matching headers, and a GPU with hardware H.264 encode (tested on NVIDIA and modern AMD). Virtual displays come from Vibepollo's own DKMS module, `vibeshine_drm`, which has four connectors, so at most four virtual displays exist at once. Streaming before login needs NVIDIA. Other distributions aren't covered; plain Sunshine still streams real displays there, without virtual ones.
|
||||
|
||||
**Risk.** Vibepollo is new, moves fast, and has one maintainer, and its README says about 99% of its code is AI-generated. Frametop depends only on its Moonlight-protocol behaviour and the Remote Monitor app ids, so on Windows Apollo (virtual displays through the SudoVDA driver) or plain Sunshine (real displays) stay as fallbacks.
|
||||
|
||||
### macOS: Sunshine, one instance per display
|
||||
|
||||
Vibepollo has no macOS build. Sunshine (labelled experimental on macOS) is the only Moonlight-protocol host for the Mac. One Sunshine streams one display, so the Mac runs one instance per real display, each with its own config file, ports (`port` 47989 + 100·N) and `output_name` (the display's CGDirectDisplayID). `host/macos/frametop-host-setup.sh` sets them up (see "The Mac" below). Sunshine can't create virtual displays on macOS, and no PR is open for it.
|
||||
|
||||
- Absolute mouse input reached only the main display from v2026.906 (Sunshine #5733). libvirtualhid #145 and Sunshine #5739 fixed it on 2026-10-09, first in the pre-release v2026.1010.141104, which the host setup pins until a release has it.
|
||||
- Capture is at backing pixel size, so the built-in display streams at 3024x1964. Closing the lid removes the built-in display, so a stream of it ends. Sunshine now keeps a display awake while it captures it (#5509 was the stall when one slept).
|
||||
- Sound comes from a Core Audio tap (macOS 14+), so nothing like BlackHole is needed. Every instance carries it; ft-stream's per-address lock plays one.
|
||||
|
||||
Full-motion video can still reach the encoder's limit. Sunshine forces VideoToolbox's low-latency mode, which roughly halves throughput (#5814); an M4 Pro managed 0.36-0.46 Gpix/s in that mode. The super ultrawide at 60 fps is 0.44 Gpix/s; the test Mac is an M3 Max, with two encode engines. Open PR #5817 (ScreenCaptureKit + OBS's VideoToolbox encoder) may raise the limit. Its capture (AVFoundation) also sends 30-55 fps of a still desktop, not `minimum_fps_target`'s 1 fps, at a low bitrate (1-6 Mbit/s).
|
||||
|
||||
### What stock hosts cost us
|
||||
|
||||
| Limit | Effect | What can be done |
|
||||
|---|---|---|
|
||||
| A stream's fps is fixed at connect | Out-of-sight panels save decoder work through keyframe sampling, but the host keeps encoding and sending 60 fps | Patch ft-stream's copy of moonlight-common-c to drop hidden video packets early (saves Frame CPU). A "change rate" control message needs a host upstream; Vibepollo is the likeliest |
|
||||
| One stream per display | Five ft-stream processes, connections, and client pairings on the Frame | Measure the CPU (S3) |
|
||||
| One real display per Vibepollo install | A second real display needs a second host instance | Use virtual displays; S3 tries a second instance on Windows |
|
||||
| Cursor drawn into the video | Lags the laser by a round trip; each mouse move over a still page becomes a video frame | Accept |
|
||||
| Mac: an instance per display | Each display pairs on its own, with the Web UI login (no tokens); no virtual displays | Remote Displays pairs all the picked displays from one sign-in |
|
||||
| Mac encoder | Full-motion video on the super ultrawide at 60 fps is at the limit; a still desktop still streams 30-55 fps | Lower fps or size for that stream; ScreenCaptureKit (#5817) upstream |
|
||||
|
||||
What Frametop maintains: ft-stream (pairing, decoder, keyframe sampling, input mapping, Remote Monitor launch and resume; the protocol itself is moonlight-common-c's), the remote screen type in ft-screens, the layout and settings changes, and host setup notes or scripts.
|
||||
|
||||
## Alternatives considered
|
||||
|
||||
- **Our own host streamer (ft-host)** on macOS, Windows, and Linux: capture and encode each display at a rate the headset can change mid-stream, real 1 Hz for hidden panels, a separate cursor, one connection per host, virtual displays built in. It solves every limit in the table above, but it means maintaining capture, encoding, transport, input, pairing, and virtual displays on three operating systems. Rejected for a free project (2026-10-06).
|
||||
- **Moonlight-qt in a window on a Frametop screen.** Nothing to build, and a quick way to check that a host and pairing work. But each frame goes decoder → Moonlight's GL renderer → KWin → ft-screens: two GPU passes per display where ft-stream needs one. Hidden panels can't drop to 1 Hz, and FFmpeg's v4l2m2m path on this device hasn't been tried (its encoder segfaults). Useful as a baseline, not as the feature.
|
||||
- **RDP (FreeRDP client, RDPGFX).** The best multi-monitor design on paper: one session, up to 16 monitors as separate surfaces, damage rectangles. But Windows' RDP host takes over the login session (the PC's own monitors lock) and runs at 30 fps by default, macOS has no RDP host, and FreeRDP decodes H.264 on the CPU or through VA-API, which the Frame lacks.
|
||||
- **Parsec, Steam Remote Play, RustDesk, NoMachine, Selkies, Apple Screen Sharing.** No aarch64 Linux client with hardware decode, no per-monitor streams, a closed protocol, or a Mac-only client.
|
||||
|
||||
## Spikes before any Frametop change
|
||||
|
||||
Spikes on the Frame run through `frame-job --local` as a standalone test overlay, with the headset on or with frame-testbench holding its worn state and pose. None of them touches the live desktop. S3 and S4 also need the hosts set up.
|
||||
|
||||
| # | Question | Pass |
|
||||
|---|---|---|
|
||||
| S1 (done) | Does a decoded `Q08C` buffer show in a SteamVR overlay with correct colours? Test pattern clip, BT.709 limited range, then full range | Correct colours; under 3% of a core; no extra missed frames (`~/.cache/frametop-perf/drops`); the decoder's real output delay |
|
||||
| S2 | Concurrency: the original five, the worst case (2x 5120x1440, 2x 3440x1440, 3024x1964), fed from files in real time at 60 fps | Admitted (92% declared); decode time per frame; SoC temperature and clocks over 10 minutes |
|
||||
| S3 | Live streams from the test PC (Vibepollo): the main session on a real desk monitor plus two Remote Monitors, through moonlight-common-c + the S1 decoder; then keyframe sampling on a hidden 60 fps stream | CPU per stream at real bitrates; glass-to-glass latency; time to picture after `LiRequestIdrFrame()`; the host honours one IDR request a second; IDR size; Frame CPU for a hidden stream; each Remote Monitor's mouse lands on its own display; Resume after a dropped stream keeps the windows; whether a second host instance can stream the other real monitor |
|
||||
| S4 | The Mac: one Sunshine instance on the main display. Once the mouse fix ships: one instance per display, then a BetterDisplay virtual display | Mouse and keyboard work; encode rate with full-motion video on the super ultrawide; `minimum_fps_target = 1` keeps a static display near 1 fps; later, all three stream at once with the mouse on the right display |
|
||||
| S5 | The baseline: 5 native screens, one playing video, four static | The CPU, GPU time (DRM fdinfo), temperature, and missed-frame numbers that the remote version must match |
|
||||
|
||||
### S1 results (2026-10-06)
|
||||
|
||||
S1 passes, with one change to the plan: a GPU pass between the decoder and SteamVR.
|
||||
|
||||
`stream/spike/ft-dectest` (built by `stream/build.sh`) decodes a clip with the V4L2 stateful API and hands each picture to SteamVR with `ImportDmabuf`, as the remote screen type would. Clips: a colour test pattern (`stream/spike/pattern.py`) with a moving box, 10 s at 60 fps, encoded by NVENC HEVC on the test PC (`-preset p1 -tune ull -bf 0`, as Sunshine-style streaming), in BT.709 and BT.601, limited and full range. The in-headset runs used frame-testbench to hold the worn state and a still pose, with nobody wearing the headset.
|
||||
|
||||
**Decoding.**
|
||||
|
||||
| | 3440x1440, Q08C | 3024x1964, Q08C | 3440x1440, linear NV12 |
|
||||
|---|---|---|---|
|
||||
| Decoder's buffer (coded size) | 3456x1440, 7,557,120 bytes | 3072x1984, 9,191,424 bytes | 3456x1440, 7,467,008 bytes |
|
||||
| SteamVR import | Works (NV12 + `QCOM_COMPRESSED`) | Works | Works (NV12, linear) |
|
||||
| Decode time, steady | 2.1 ms avg, 2.7 ms worst | 2.6 ms avg, 3.3 ms worst | 2.2 ms avg, 2.8 ms worst |
|
||||
| First picture after | 1 input frame | 1 input frame | 3 input frames |
|
||||
| CPU at 60 fps, no conversion | 1.2% of a core | 1.5% of a core | 1.4% of a core |
|
||||
|
||||
- The buffer layout from `msm_media_info.h` (per plane: UBWC metadata, then pixels, each 4 KiB aligned; Y stride aligned to 128, heights to 32) matches the driver's size to the byte, so the import offsets are right. The visible size comes from `G_SELECTION`; the decoder pads the coded size.
|
||||
- Q08C has no extra delay: each frame comes out as soon as it's decoded. Linear NV12 holds two more frames, so the remote screen type uses Q08C.
|
||||
- The CPU figure is ft-dectest's own process (file reading, V4L2 queueing, the overlay call). Network receive is S3's.
|
||||
|
||||
**Colours.** `stream/spike/s1-colours.py` shows the clip head-locked, switches the panel between the decoded video and an RGB copy of the pattern every 2.5 s, grabs the headset view (`/dev/video99`) in both states, and compares each colour patch.
|
||||
|
||||
- Decoder buffers straight into SteamVR come out wrong. vrcompositor doesn't convert NV12: red carries Cr, green Y, blue Cb. 100% red (Y 63, Cb 102, Cr 240 in BT.709 limited range) showed as 239/62/102, and greys as a dull magenta.
|
||||
- The decoder's RGBA formats don't help. It accepts `AB24` and `QC24` but writes a 10-bit UBWC YUV picture into them: 10,027,008 bytes used, exactly TP10 UBWC at 3456x1440, starting with UBWC metadata. It also reports their `bytesperline` in pixels.
|
||||
- A GPU pass gets them right (`ft-dectest --convert 709|601 --range tv|pc`). GLES samples the decoder's buffer through an external texture with the matrix and range as EGL hints (`EGL_YUV_COLOR_SPACE_HINT_EXT`, `EGL_SAMPLE_RANGE_HINT_EXT`). It draws into a ring of 3 RGBA buffers (UBWC, `QCOM_COMPRESSED`) that SteamVR imported once, set up as in `screens/handcut.cpp`.
|
||||
|
||||
| Clip, converted with its own matrix and range | Mean error | Worst patch |
|
||||
|---|---|---|
|
||||
| 3440x1440, BT.709 limited | 1.4 | 4.0 |
|
||||
| 3440x1440, BT.709 full | 1.2 | 4.0 |
|
||||
| 3440x1440, BT.601 limited | 0.2 | 2.4 |
|
||||
| 3440x1440, BT.601 full | 0.3 | 2.9 |
|
||||
| 3024x1964, BT.709 limited | 1.3 | 4.0 |
|
||||
|
||||
Errors are on the 0-255 scale, over 39-40 patches (27 for 3024x1964, where less of the panel is in view). The near-black steps (0-30) and near-white steps (225-255) all stay apart, so nothing is crushed or clipped. BT.709 sits about 3 low in red throughout, probably a rounding difference between ffmpeg's and Mesa's coefficients. That isn't visible.
|
||||
|
||||
**Cost of the pass**, 3440x1440 at 60 fps:
|
||||
|
||||
| Measure | Result |
|
||||
|---|---|
|
||||
| GPU work | About 118,000 cycles a frame: 0.13 ms at the 903 MHz top clock. With the headset idle, the GPU ran at about 230 MHz and was busy 0.51 ms a frame (DRM fdinfo) |
|
||||
| Time until the GPU is done | 1.1-1.4 ms avg, 2-6 ms worst |
|
||||
| CPU, whole ft-dectest process | 2.2-3.5% of a core, up from 1.2% |
|
||||
| Missed compositor frames, 30 s at 90 Hz | 0 of 2,701 with the stream shown, 0 of 2,700 idle |
|
||||
|
||||
- The CPU is at the 3% pass line. The increase is the GL driver plus the `glFinish` wait; ft-stream should try waiting on a sync file in its poll loop instead.
|
||||
- The five-display worst case (89% of the decoder) is about 360 converted 3440x1440-sized frames a second. That's about 5% of the GPU at top clock, and S2 measures it.
|
||||
- frame-testbench held the pose still and nobody wore the headset, so tracking and eye-tracking load may differ from a real session. S5 rechecks missed frames with the headset worn.
|
||||
|
||||
The benchmark is S5 against the same scene on remote displays: total Frame CPU (Frametop + ft-stream), GPU time, missed frames, and SoC temperature all at or below native.
|
||||
|
||||
### S3 results so far (2026-10-07)
|
||||
|
||||
`stream/spike/ft-streamtest.cpp` is a Moonlight client for one display: moonlight-common-c and libgamestream from moonlight-embedded (pinned in `stream/build.sh`), S1's decoder and GPU pass (`spike/iris.h`), and a SteamVR overlay. It pairs through the host's API token, launches the primary app or a Remote Monitor, and reports every 2 s. Tests ran from the test PC over the LAN, with the headset held by frame-testbench.
|
||||
|
||||
**The primary display (5120x1440 at 60 fps, HEVC, 50 Mbit/s asked).** It works end to end. The picture in the headset is right, colours included. The test PC's primary is an HDR monitor, and Vibepollo sends it as SDR Rec. 709, as asked.
|
||||
|
||||
| | Shown | Hidden (keyframe sampling) |
|
||||
|---|---|---|
|
||||
| Frames received | 60 fps | 60 fps |
|
||||
| Frames decoded and shown | 60 fps | 1 fps |
|
||||
| Network | 12-14 Mbit/s | the same |
|
||||
| ft-streamtest CPU | 4.2-4.6% of a core | 1.5-1.7% of a core |
|
||||
|
||||
- Time from a frame's first packet to the panel: 5.1-5.4 ms on average (worst about 12 ms). Of that, decoding takes 3.7 ms, and receiving the whole frame 0.1 ms. The host reports 3.2 ms from capture to encoded, and half the round trip is 1.5-2.5 ms. So a frame reaches the panel about 10 ms after the host captures it, before vrcompositor shows it. Glass to glass isn't measured yet.
|
||||
- Connecting took 225 ms, and the first picture showed 0.5 s after the launch.
|
||||
- Keyframe sampling: the host answered all 14 IDR requests, one a second. Each IDR arrived 18 ms after the request (worst 25 ms), reached the panel at 25 ms (worst 32 ms), and was about 81 KB for this desktop. Going back to full rate took 17 ms.
|
||||
- No packets were lost.
|
||||
- The test PC's desktop has an animated wallpaper, so the stream ran at 60 fps throughout, though Vibepollo applied a 0.5 fps floor. A still wallpaper would let an idle display drop to about one frame a second.
|
||||
|
||||
**Vibepollo bug: a stale display slot blocks Remote Monitors.** The first Remote Monitor launch failed with 503, "The composed display topology did not apply", for 40 s of retries. Vibepollo still held a display slot for the Mac's earlier "Desktop" session (`/api/clients/display-layout`: the Mac as a client node, its runtime "retryable" with the lease held), though that session's virtual display was gone. Composing the layout includes every held slot, and the Mac's can't be resolved to a device, so every composition fails. The Mac's session had been cut by a Vibepollo restart and reconnected before it ended. The failed launch still created frametop-2's virtual display, and "Disconnect Monitor" then reported success without removing it, because the launch never finished. A Vibepollo restart clears both. ft-stream must detect this state, a 503 that persists, and report it rather than retry forever. It's worth reporting upstream with the steps that cause it.
|
||||
|
||||
**Remote Monitor (frametop-2, 3440x1440 at 60 fps), after a Vibepollo restart.** The launch took 2.3 s (Vibepollo makes the virtual display, then answers), and the first picture came 0.9 s after connecting. The picture was right. An empty, still monitor dropped to 1 fps at once: 0.1 Mbit/s, 0.6% of a core, about 4-5 ms from first packet to panel. Windows moved a window it remembered for that spot onto the new monitor, so a Remote Monitor can take windows off the real screens without being asked.
|
||||
|
||||
**Vibepollo bug: a dropped Remote Monitor reset the host's real monitor layout.** The test killed ft-streamtest to imitate a dropped connection. Vibepollo kept the Remote Monitor for Resume, as configured, and recomposed the display layout. Windows refused it (`SetDisplayConfig`: ERROR_INVALID_PARAMETER, "failed to move device ... to new origin" for the monitor above the primary). Vibepollo's recovery (`SDC_USE_DATABASE_CURRENT`, a topology jog, `CDS_RESET`) then left Windows with a different layout: the monitor above became the primary, beside the old primary. "Disconnect Monitor" again reported success and did nothing. A Vibepollo restart was needed again.
|
||||
|
||||
One more defect that may have contributed: `GET /api/clients/display-layout` rebuilds Vibepollo's record of the real monitors without their positions or modes (every monitor at 0,0, 1920x1080; `refresh_remote_display_physical_baseline` in `confighttp.cpp`). A layout composed from that record stacks the monitors on one another. Launching a Remote Monitor reads the real positions again (`refresh_remote_monitor_baseline` in `nvhttp.cpp`), and the last such call came before the launch here, so this isn't proven to be the cause. Frametop must not call that endpoint until it's fixed.
|
||||
|
||||
So far, Vibepollo 2.0.0's Remote Monitors aren't reliable on the test PC: a stale slot blocks them, a drop can rearrange the host's real monitors, and "Disconnect Monitor" can't release a monitor whose ownership was lost. These go upstream with logs before Frametop depends on them. Further Remote Monitor tests on the test PC need the user's go-ahead, since they can move the user's own screens.
|
||||
|
||||
**The cause, and a fix (frametop-vibepollo, 2026-10-07).** All three Remote Monitor failures come from one call. When a display role ends, Vibepollo's coordinator (`src/remote_display_topology.cpp`) first recomposes the topology without the departing display, and only then removes it. On the test PC, Windows refuses that composed `SetDisplayConfig`. Then:
|
||||
- libdisplaydevice's automatic recovery (`SDC_USE_DATABASE_CURRENT`, a topology jog, `CDS_RESET`) rearranges the real monitors;
|
||||
- the release rolls back, so the virtual display stays attached and "Disconnect Monitor" does nothing;
|
||||
- for a normal game (the Mac's session), the per-client identity stays held, and every later composition fails on it.
|
||||
|
||||
Removing the virtual display alone, as a Vibepollo restart does, left the layout intact. The fork [Frametop/frametop-vibepollo](https://github.com/Frametop/frametop-vibepollo) (GPL-3.0, like Vibepollo), on branch `fix/windows-remote-monitor-release`, changes two things on Windows:
|
||||
- The coordinator removes the departing display first and recomposes only if another owned display remains (`retire_before_recompose`, set by the Windows runtime). A normal game's identity is released even when its display is already gone. Linux keeps the old order, which exists so KWin never has zero outputs.
|
||||
- Composed layout applies run with display recovery off, so a refused layout can't reset the host's arrangement.
|
||||
|
||||
Four new unit tests cover it, and the coordinator's 40 tests pass. The plan is to offer the fix upstream once it's proven on the test PC. Frametop stays MIT: it only talks to the host over the network.
|
||||
|
||||
**The fix works on the test PC (2026-10-07).** The dev build, `2.0.0` plus the fix, was built on the test PC with MSYS2 (UCRT64, as the CI does, without WebRTC, drivers or packaging; `D:\vp-build`). It links only Windows system DLLs, so it drops into the install as a plain exe swap (`D:\vp-build\deploy.ps1`; the original is kept as `sunshine.exe.2.0.0-original`, and `-Restore` puts it back). Three Remote Monitor cycles left the real layout exactly as it was, with no layout re-apply and no `SetDisplayConfig` errors in Vibepollo's log: a clean end, a killed client, and a relaunch after the kill. Before the fix, the clean end reset the layout twice in a row. Restarting Vibepollo for the deploy also ended a leftover Mac session cleanly and brought the real monitors back.
|
||||
|
||||
A dropped connection removes the Remote Monitor at once (Vibepollo saw the disconnect within 3 s), even with `remote_monitor_disconnect_on_stream_end = disabled`. With `remote_monitor_disconnect_on_client_disconnect = enabled`, a lost connection counts as a client disconnect. Keeping a monitor and its windows across a Wi-Fi drop would need that setting off, and Frametop releasing monitors explicitly with "Disconnect Monitor". That's still to test.
|
||||
|
||||
The development loop is an incremental Ninja build on the test PC (only changed files recompile), the exe swap through the admin SSH login, and `ft-streamtest`: a few minutes per change. Pure logic gets unit tests on the Frame.
|
||||
|
||||
**Real displays, several at once (frametop-vibepollo, 2026-10-07).** A new hidden control, "Frametop display" (id 2147483521), streams one of the host's existing displays, named by the launch argument `frametopDisplay` (a device id from `/api/display-devices`). The session captures that output exactly, the way a Remote Monitor captures its virtual display, but nothing is created and the layout is never touched. Each client holds one such stream, and the main app stays free for the Deck and the Mac. On the test PC:
|
||||
- both real monitors streamed at once (OLED at 5120x1440, LC34G55T at 3440x1440), each to its own client, at about 0.6% of a core each while idle;
|
||||
- a real display and a Remote Monitor streamed side by side, and the layout was unchanged afterwards.
|
||||
|
||||
With that, Frametop no longer needs the "Frametop primary display" app; every real display is captured the same way.
|
||||
|
||||
**Vibepollo bug: one idle HTTPS client froze the API for everyone.** While any stream ran, every other client's HTTPS requests (serverinfo, pairing, launch) waited until it ended, so a second display couldn't start. The stock 2.0.0 build did the same. A stack dump (MSYS2 gdb attached to the service) showed the HTTPS server's only I/O thread blocked in a synchronous TLS shutdown: `SunshineHTTPS`'s destructor (`nvhttp.h`) sends close_notify and then waits for the client's. libgamestream leaves its connection idle in curl's cache (it forbids reuse only on FreeBSD), so the wait lasted as long as the streaming client process. Fixed on both sides:
|
||||
- the fork marks the client's close_notify as received before shutting down, so nothing waits (a deliberately idle client no longer delays another client's request: 0.13 s);
|
||||
- `ft-streamtest` drops libgamestream's curl handle after every request, which ft-stream must do too.
|
||||
|
||||
**The 3D mouse, and dragging windows between displays (2026-10-07).** The setup had three panels from the test PC: the OLED and the LC34G55T as Frametop displays, and a 2560x1440 Remote Monitor. frame-testbench held the headset, and the 3D mouse was driven through `@ft_pointer_helper`. The 3D mouse moved and clicked the test PC's cursor on every panel.
|
||||
|
||||
The first version clamped every move to its own panel. A drag from the Remote Monitor to the OLED left the window below the Remote Monitor's bottom edge, mostly off screen. Removing the Remote Monitor brought it back to the OLED. Drags between monitors that touch in Windows' layout seemed to work, but only because the cursor, pinned at the shared edge, left the window hanging across it. `--input-log` showed why. With the button held, SteamVR kept sending the moves to the panel where the press began (for example -213,2093 on the 2560x1440 panel), and sent the panel under the laser no events at all. The release also went to the first panel, off its edge.
|
||||
|
||||
The spike's fix is the routing described under "Input", with one difference: each panel is its own process. The panel that got the press publishes the laser's device in a small shared file (`/dev/shm/frametop-streamtest-drag`). Every other panel hit-tests that device's ray against itself and moves the host's cursor while the ray hits it. When hovering, the panel's own hit test matched SteamVR's coordinates exactly, so the ray origin and the UV orientation (bottom-left origin, like the mouse events) are right. With the fix, three drags worked:
|
||||
- Remote Monitor to OLED;
|
||||
- OLED to Remote Monitor;
|
||||
- Remote Monitor to LC34G55T.
|
||||
|
||||
In each, the window followed the laser across panels and stayed where it was dropped. The release still went through the first panel's connection and landed right every time.
|
||||
|
||||
Windows' "Remember window locations based on monitor connection" moves windows on its own. When a Remote Monitor appears, Windows puts back windows it remembers there, and when it goes, they move to a real monitor.
|
||||
|
||||
## In ft-screens (2026-10-07)
|
||||
|
||||
The user decided remote displays are Frametop displays, with the same controls as the desktop's screens and their places saved in profiles. The spike's standalone overlays are replaced.
|
||||
|
||||
**What's built (branch `remote-displays`, uncommitted):**
|
||||
- `stream/ft-stream.cpp` is the per-display helper. It pairs, launches and decodes, and converts into a ring of three RGBA buffers. It hands their dmabufs to ft-screens once, then says which buffer holds each new picture. The protocol is in its header comment: frames one way, then release, attention, pointer, keys and blur the other way. `stream/host.cpp` holds the pairing and launch code ft-streamtest had.
|
||||
- `screens/remote.c` starts one ft-stream per remote screen with a socket pair, restarts a stream that ends (2 s, then backing off to 30 s), shows its frames through `ft_vr_screen_present`, and sends it the panel's input and attention. Commands: `remote <N> start <client> <host> <app> <W>x<H> <fps> <kbit/s> <metres> [label]` (`ok restored` when its panel is back where it was, below), `remote <N> stop`, `remote <N> info` (the stream's whole state, such as `lost can't connect`) and `remotes`. Remote screens are numbered from 101, so every other command (`place`, `width`, `curve`, `pin`, `get`, `conceal`) works on them as it is. `screens` leaves them out, because ft-layout, Display Settings and ft-floatd count KWin's outputs with it.
|
||||
- `vr.cpp` gives a remote screen a screen's panel (`frametop.remote.N`) with its controls. A press on a remote screen dragged onto another one is routed there (`UpdateRemoteDrag`). SteamVR keeps sending the moves to the panel the press began on, as if its surface went on past its edges. So every tick the pressing laser is hit-tested against all the remote screens, and the nearest one it meets takes the moves. On the panel the press began on, SteamVR's own moves count only while that panel is the nearest. Before this (2026-10-07), that panel's carried-on surface passed in front of or behind the other panel, and its moves pulled the host's pointer back: drags across worked only some of the time. The release comes up where the host's pointer is, through the stream the press went down on (Vibepollo takes a release only from the client that pressed).
|
||||
- `compositor.c` sends a remote screen's pointer events to its stream. A click on a remote screen takes the typing there (input relay keys and our VR keyboard), and leaving releases the keys it held (`blur`).
|
||||
- `--beside` runs a second ft-screens next to the desktop for tests. It shows remote screens only and sends nothing to the input relay, ft-floatd or ft-layout.
|
||||
- ft-pointer counts `frametop.remote.` panels as screens, and ft-gaze hit-tests them too (`remotes`, then `get N`). Without that, ft-gazed dropped looks below the keyboard pitch (-20°) on a low remote panel, as if you were looking at the keyboard.
|
||||
|
||||
**Tested through frame-testbench, with a `--beside` instance:**
|
||||
- the OLED and a Remote Monitor as panels;
|
||||
- Win+R, `notepad` and Enter typed through ft-screens' key path started Notepad on the test PC;
|
||||
- Notepad dragged from the OLED panel to the Remote Monitor's stayed where it was dropped;
|
||||
- the grab bar moves a remote panel;
|
||||
- stopping the instance released the Remote Monitor.
|
||||
|
||||
**SteamVR's overlay limit.** SteamVR allows 128 overlays in the whole system (`k_unMaxOverlayCount`), its own included. Each Frametop panel takes six or seven with its controls. The desktop's floating-window slots made all of theirs at start: eight empty slots held 56. With three screens, a third remote screen got `VROverlayError_OverlayLimitExceeded`. A floating window's panel and controls are now made when a window floats on it, and destroyed when it docks (`EnsureFloatPanel`, `DropFloatPanel`). This needs a test on the live desktop. The controls of the other panels stay up, invisible until a laser comes near, because SteamVR's laser hover is what brings them in.
|
||||
|
||||
**Layouts, profiles and settings (2026-10-07, same branch):**
|
||||
- `frametop-layout.json` has a `hosts` list. Each host has its displays: id, paired client, what it streams, label, stream size, fps, bitrate, and a screen's place, width, curve, pin and hidden flag. Each display keeps its screen number (101 and up), so the numbers don't shift when one is removed.
|
||||
- ft-layout starts the remote displays' streams on every apply and at desktop start, even without a head pose, and places them when there is one. Displays without a place go in a row above the screens.
|
||||
- `capture` and `save` record their places.
|
||||
- A profile keeps the displays connected when it's saved, like the apps open then: their places and hidden state by id (`profiles[NAME]["remote"]`). Opening it (`use`, `open`, desktop start), or arranging (`apply`, Reset Screen Layout) while it's the profile in use, connects each of them whose host answers on Vibepollo's Web UI port, at its address or its dongle's as its route allows, and puts it back where it was saved. A host that doesn't answer is skipped, and its displays stay disconnected. Displays the profile doesn't have stay as they are: a profile connects, but never disconnects (user decision, 2026-10-08).
|
||||
- `hide`/`show N` take their numbers.
|
||||
- `ft-layout remote list|monitors|add|set|connect|disconnect|remove`: `add` pairs the display's client with the host's token, and `remove` unpairs it (the host stays). `disconnect` sets the display's `off` flag and stops its stream; apply, profiles and desktop start skip it until `connect`.
|
||||
- ft-screens starts a host's streams one after another. Three started at once made Vibepollo refuse some ("Another stream operation is still running"), and the captures that started while the Remote Monitor appeared got no picture.
|
||||
- Frametop Remote Displays (`remote-displays/`) is the app for them. It was a Display Settings tab at first; the user wanted an app of its own for the connections. Display Settings' Screens page has a button that opens it.
|
||||
- Add a host with its token (written to `~/.local/share/frametop-stream/hosts/ADDRESS.token`, mode 0600). Whether it answers on its Web UI port (47990) is checked every 15 s.
|
||||
- Add a display: one of the host's monitors from its API, or a virtual one.
|
||||
- Per display: Connected, Shown, stream size, fps and bitrate (the stream starts over), width in VR, remove. Each host has a Connected switch for all of its displays. A lost stream shows why (`remote N info`).
|
||||
- A disconnected display comes back where it was. ft-screens keeps a stopped remote panel's place, width, curve, pin and hidden state for the rest of its run (`Parked` in `vr.cpp`), and the next start of that screen restores them and replies `ok restored`. Otherwise ft-layout places it from the head, or with the headset off, from where the last arrangement was made, worked out from screen 1's place (`remote_anchor`).
|
||||
- A vrcompositor crash: ft-screens freed a remote panel's texture imports while the panel still showed one. That happened at quit, at `remote stop`, and when a stream started over. vrcompositor crashed drawing it as the headset left standby. `ft_vr_forget` now clears a panel's texture before its import goes, and remote panels are destroyed before their buffers.
|
||||
|
||||
**vrcompositor crashed again at ft-screens quit (15:42, SIGBUS), and at 14:34.** Both times remote screens were up and the headset was in standby. SteamVR logs "leaving standby" within 2 ms of ft-screens disconnecting, so the compositor wakes in the middle of our cleanup, and it drew a texture that was already freed. The crash took the gamescope session and Steam down with it. The restart at 15:39 under the same conditions didn't crash, so it's a race. Hardened, not yet verified: ft-screens now tears SteamVR down first, while every buffer the panels show still exists (before KWin and the streams go). It clears the panels' textures, destroys the overlays, waits 200 ms, and only then releases the imports; nothing calls SteamVR after `VR_Shutdown`. Until that's checked, restart the desktop only with the compositor awake.
|
||||
|
||||
**Frozen panels, out-of-sight rate, sound (2026-10-07, after the user's test):**
|
||||
- The OLED froze while a video played on it: frames came in at 59 fps and none were shown. ft-stream's message loop asked ft-screens' socket once more after the queue ran dry, to see whether it had closed, and lost whatever came in between. A lost `release` kept one of the three ring buffers from ft-stream for good, and after three nothing could be shown. Every pointer move is a message, so drags made it likely. (A lost button or key release would have stuck on the host the same way.) Fixed; ft-stream also takes the ring back if none of it comes back for 2 s.
|
||||
- Out of sight, a stream used to decode only keyframes, one asked for each second. The user found that too aggressive. Every frame is now decoded, and 10 a second shown (`kHiddenFps`); coming back into view is immediate, with no keyframe requests (those are big frames, and the link already loses some).
|
||||
- ft-stream ignored SIGTERM, ft-screens' death signal included: posix_spawn passed on ft-screens' blocked signals (its event loop takes SIGTERM, SIGINT and SIGCHLD through a signalfd). ft-screens now spawns it with none blocked, and ft-stream clears its mask too.
|
||||
- Sound: ft-stream plays the host's sound (Opus, then PipeWire's PulseAudio server through libpulse-simple, about 40 ms queued). One stream per host plays it, the one holding `$XDG_RUNTIME_DIR/frametop-audio-HOST.lock`; the others try every 3 s, so another takes over when it ends. The host keeps playing its sound too. The test PC sent none: `remote_monitor_mute_audio = enabled` (our setup script) mutes the monitor and display roles. Turning it off needs an admin change on the test PC and a Vibepollo restart.
|
||||
|
||||
**Where the lost frames went, and the dongle (2026-10-07, ~16:40):**
|
||||
- With the streams on the home network, about one frame every 2 s (all three together) was unrecoverable: bursts of 5 to 14 packets of one frame lost, beyond what FEC (Vibepollo's default 20%) repairs, and each loss then waited for a keyframe.
|
||||
- Not on the Frame: UDP `RcvbufErrors` and `InErrors` stayed 0, wlan0 rx drops 0, the driver's misc drops +6 in 90 s. Not on the test PC: I225-V outbound discards and errors 0. The Frame's link was fine (-28 dBm, 2.1 Gbit/s, power save off). So the router loses them, as `~/.cache/wifitest` found for Steam Link VR (2026-10-01).
|
||||
- Same burst test (vrsim, 50 Mbit/s at 60 fps, 20 s) from the test PC: through the router 0.33% of packets and 21 of 1200 frames lost (worst frame 21 ms); over the test PC's Steam Link dongle on the Frame's hotspot (the PC on the Frame's hotspot, the Frame at 10.35.78.1), none (worst 5.6 ms).
|
||||
- So a host can go over its dongle. Its entry in `frametop-layout.json` has `direct` (the dongle's address) and `route`: `auto` (the default: the dongle when its port 47989 answers within 300 ms, else the network), `network` or `dongle` (only: while it's down, the stream is "lost the dongle link is down" and ft-screens tries again). ft-stream reads them itself, so ft-screens passes the host's own address as before. Its state says which way it went (`live via dongle|network`, `remote N info`). Remote Displays has a Connection setting per host, the dongle's address, and Find, which looks for the host among the hotspot's clients (`/proc/net/arp`, device `wlanap`) with the same `<uniqueid>` in its serverinfo. `ft-layout remote host NAME route=... direct=...` saves it and starts the host's running streams over in place.
|
||||
- On the dongle, none of the three streams lost a frame in 2 to 3 minutes each, all at 60 fps.
|
||||
- Sound: `remote_monitor_mute_audio = disabled` on the test PC (backup `D:\remote-displays\vibepollo-setup\sunshine.conf.before-audio-20261007-163556`, ApolloService restarted). One stream plays it (`Remote display: ADDRESS` in PipeWire, through SteamOS's spatial filter chain to the headset's speakers). That stream uses about 5% of a core more than the others (240-sample `pa_simple_write`s; batching them would help).
|
||||
- Seen on the way: a stream that started while Vibepollo restarted chose the network (the dongle didn't answer yet) and then got no video, only sound; starting it over fixed it. Starting one display's stream once ended another's ("lost", back 5 s later). Vibepollo takes about 5 s to answer serverinfo, so every stream start waits that long.
|
||||
|
||||
**Adding a computer: sign in instead of carrying a token (2026-10-07, ~17:00):**
|
||||
- Before, a host's API token came from `make-frametop-token` run on the host (its Web UI login typed there), and the file had to reach the Frame by hand. Now Remote Displays does it. Add computer lists the Vibepollo and Sunshine computers that announce `_nvstream._tcp` over mDNS (`avahi-browse -rpt` on the host; a computer also seen on the hotspot, `wlanap`, has a dongle, and that address is kept as its `direct`), or takes an address. You sign in once with the host's Web UI user name and password. Frametop posts them (HTTP Basic) to `/api/token` for a token with only what it uses: `/api/pin` POST, `/api/clients/list` GET, `/api/clients/update` POST and `/api/display-devices` GET (not `/api/clients/display-layout`, which the old script allowed). It keeps the token, mode 0600, and forgets the password. The sign-in goes over the dongle when there is one. Then Add displays lists the host's monitors, all ticked but those already added, and a virtual display.
|
||||
- Hosts without a dongle use the network, and their card says so, with Find a dongle; the Connection choices only show once a dongle is known. A laptop without one was the test: it's on the network and not on the hotspot.
|
||||
- The Web UI's certificate is self-signed. Its public key is pinned at the first sign-in (`hosts/ADDRESS.pin`, `sha256//...` as curl takes it): ft-stream's API calls set `CURLOPT_PINNEDPUBLICKEY`, and a later sign-in refuses another key ("If Vibepollo was installed again, remove the host and add it again"). The test PC's key is the same on both paths. Checked: the right pin works, a wrong one fails. The test PC's existing token got its pin too.
|
||||
- Checked without the password: discovery found the test PC (its LAN address and its dongle on the hotspot, marked added) and a laptop; a wrong password says "Wrong user name or password". A real sign-in is the user's to try (the password never goes through chat).
|
||||
- Sign in again (on a host card) makes a new token; the old one stays in the host's Web UI under API Tokens until revoked there (Frametop's token can't revoke tokens).
|
||||
|
||||
**The PC side: Frametop host setup (2026-10-07, ~17:40):**
|
||||
- `host/windows/Setup Frametop host.cmd` (it runs `frametop-host-setup.ps1` and asks for admin) does what the test PC got by hand, nothing else of its settings:
|
||||
1. Vibepollo 2.0.0 with its own installer when it isn't there (downloaded from Nonary/Vibepollo's release, SHA-256 checked first; you click through it).
|
||||
2. Frametop's build of `sunshine.exe` over the original (kept as `sunshine.exe.2.0.0-original`), SHA-256 checked: from `-FrametopBuild PATH|URL`, the script's `$BuildUrl` (the fork's release), or `sunshine-frametop.exe` next to it. Only over Vibepollo 2.0.0's own exe; another version stops it with a message. Without the build, virtual displays still work, but not the PC's own monitors.
|
||||
3. `remote_monitor_mute_audio = disabled`, `remote_monitor_disconnect_on_client_disconnect = enabled`, `remote_monitor_disconnect_on_stream_end = disabled`. The rest of `sunshine.conf` stays as it is.
|
||||
4. The Web UI login: keep the one there is, or set one (`sunshine.exe --creds`, the password typed into a hidden prompt and passed to it quoted).
|
||||
5. Checks: an inbound firewall rule for `sunshine.exe` on every network type (added if missing; Windows puts a Steam Link dongle's network in Public), whether a Steam Link dongle (an adapter "For Valve") is connected, and that the Web UI answers. It ends with what to pick and sign in as on the Frame.
|
||||
`-Check` only says what it would change, `-SkipLogin` leaves the login, `-Undo` puts back the original exe and the oldest backup of the settings. Backups and a log go to `%ProgramData%\Frametop`.
|
||||
- Tested on the test PC (admin over SSH, `FRAMETOP_NO_PAUSE=1`): `-Check`; a run with nothing to change (no restart); `-Undo` (the original exe back) and a run with `-FrametopBuild D:\vp-build\src\build\sunshine.exe` (backed up, swapped, Vibepollo restarted, the streams came back on their own). Not tested: a PC without Vibepollo (the download and its installer), setting the login (needs the user at the PC), adding the firewall rule.
|
||||
- Where other PCs get Frametop's build: the fork's GitHub releases, with the source as the release's tag (GPL-3.0). See below (2026-10-09).
|
||||
- The Frame side: `install.sh` now builds ft-stream (`stream/build.sh`, step 7) with Remote Displays' menu entry, and `uninstall.sh` offers to delete `~/.local/share/frametop-stream` (the clients' keys, the hosts' tokens and pins) with the other settings.
|
||||
- Linux hosts (Vibepollo's Arch package): a host setup like this one, later.
|
||||
- After Vibepollo restarted, a stream checked the dongle in 300 ms, too soon, and went over the network; ft-stream now gives it a second.
|
||||
|
||||
**Frametop's build released (2026-10-09):**
|
||||
- Release [`frametop-2.0.0-2`](https://github.com/Frametop/frametop-vibepollo/releases/tag/frametop-2.0.0-2) of Frametop/frametop-vibepollo (2026-10-14; `frametop-2.0.0-1` was the first) has `sunshine.exe` and the Arch package (`pacman -U`), each with its `.sha256`. The fork's CI (`frametop-build.yml`, Depot's runners) builds them from the tag and publishes them; a `frametop-*` tag is what makes a release. The tag is the source.
|
||||
- The host setup's `$BuildUrl` points at that `sunshine.exe`, so a PC needs only `host/windows`. It replaces the original, the hand-built `2f032252`, and the first CI build (`a84b6cfc`).
|
||||
- New in it: a Frametop display stream that asks for SDR turns the display's HDR off while it captures it, and back on when the stream ends (with `dd_hdr_option` automatic, Vibepollo's default). The test PC's HDR monitor looked washed out in SDR: Vibepollo's conversion clips at 80 nits while Windows draws SDR content at its SDR white level (240 nits there), and scaling for that left the colours heavily oversaturated. Displays it turned off are listed in `config\frametop_display_hdr.json` until they're back on, so the next start turns them back on after a crash or a forced stop. On the test PC's dev build: HDR off 0.4 s after the stream started, 8-bit capture, back on at the end, off again on a reconnect.
|
||||
- Tested on the test PC: with the first CI build installed and only `host/windows` in a folder, `-Check` downloaded the release and matched its SHA-256, and the run backed up, swapped and restarted Vibepollo; the Web UI answered.
|
||||
|
||||
**The monitor follows the stream's resolution (frametop-vibepollo, 2026-10-10):**
|
||||
- Before, a Frametop display stream captured the monitor at its own resolution and Vibepollo fit that picture into the stream's size, so a stream smaller than the monitor, or of another shape, never showed its desktop pixel for pixel.
|
||||
- Now the stream switches the monitor to the stream's width and height, at the monitor's own refresh rate, before capture starts, and puts it back when the stream ends, the way `dd_resolution_option` automatic (Vibepollo's default) matches a main session's display to the client. With it set to manual or disabled, nothing changes. A monitor without that mode gets its closest mode with the same shape, and the stream scales that; one with no mode of that shape keeps its own and is scaled as before.
|
||||
- The change is temporary: Windows' saved display settings stay as they were, so a reboot undoes it too. Windows may move the other monitors to keep them touching one that changed size, so every monitor's place is noted before the first change and put back after the last, in one change that either fully applies or doesn't.
|
||||
- Two streams of the same monitor share the first one's mode. HDR (above) goes off after the mode changes and back on before it's put back.
|
||||
- Monitors it changed are listed in `config\frametop_display_mode.json` (each one's own mode and the one it set, and the places) until they're back, so the next start puts them back after a crash or a forced stop. One someone has changed since keeps theirs.
|
||||
- In Frametop nothing changes: ft-stream already asks for the size set in Remote Displays, and changing it starts the stream over, which switches the monitor again. The note under a monitor's stream settings says what happens.
|
||||
|
||||
**The Mac: one Sunshine instance per display (2026-10-10, branch `mac-host`):**
|
||||
- `host/macos/frametop-host-setup.sh` (or `Setup Frametop host.command`): installs the pinned Sunshine pre-release (SHA-256 checked, notarized by LizardByte), lists the displays (`displays.js`, JavaScript for Automation, no compiler), and writes an instance per display in `~/Library/Application Support/Frametop/sunshine/instance-N`, the main display's first, each started at login by the LaunchAgent `frametop.sunshine.N`. Its settings: `port`, `sunshine_name` "COMPUTER: DISPLAY (WxH)", its own state and apps files, `minimum_fps_target = 1`, and the menu bar icon on the first only. The rest of each `sunshine.conf` (Web UI changes) is kept on a rerun. `--check`, `--undo` (removes the LaunchAgents; settings, pairings and Sunshine stay), `--set-login`, `--skip-login`.
|
||||
- Turning them off and on (2026-10-12): Frametop Streams, a menu bar icon, lists the displays with a check by each one that's on. A click turns one on or off, Turn All On and Turn All Off do every one, and the icon dims while they're all off. It's `Frametop Streams.applescript`, a stay-open AppleScriptObjC app (`NSStatusItem`, so no compiler) that the setup compiles into `~/Applications`, marks `LSUIElement` (no Dock icon), signs again ad hoc after that change, and starts at login with the LaunchAgent `frametop.streams`. Quit from its menu, it stays quit until the next login. It runs `ft-sunshine off|on|status [--tsv] [N...]`, in the background for a change, with the output in `~/Library/Logs/Frametop/streams.log`. `ft-sunshine` is installed next to `ft-sunshine-instance` and linked into `~/.local/bin` when there is one. Off disables each instance's LaunchAgent (`launchctl disable`) and stops it, so it stays off through logins. On enables it and starts it, or starts one quit from Sunshine's own menu bar icon, which launchd leaves stopped after a clean exit. Rerunning the setup turns them all on. The Frame's streams of the Mac keep trying while it's off and reconnect after on. Tested with throwaway LaunchAgents serving fake instances, through the menu (UI scripting) and `ft-sunshine`: on, on again (the running one kept its pid), off, on after off, one display, a stopped one, unknown numbers, a display that isn't connected, and the app started by its LaunchAgent and quit from its menu. On the M3 Max: the setup from the kit zip, unpacked with a download's quarantine (the installed scripts keep it, and run anyway: launchd and the app run them through bash and sh), which turned the three on; then Turn All Off, Turn All On and Turn All Off again from the menu, each done within 3 s.
|
||||
- The kit as a release (2026-10-12): [`macos-host-1`](https://github.com/Frametop/frametop/releases/tag/macos-host-1) of Frametop/frametop has `frametop-macos-host-1.zip`, `host/macos` of 188f107 in a `frametop-macos-host` folder (`zip -qr -X`), and its `.sha256`. Its notes are `host/macos/README.md` without the title, plus the files. It isn't marked Latest and has no `Frametop.zip`, so get.sh skips it: stable takes `releases/latest`, experimental the newest release with a `Frametop.zip`. Its tag doesn't start with `v`, so `release.yml` doesn't build it. A new kit is a new release, `macos-host-N`, with the zip renamed to match.
|
||||
- An instance is tied to its monitor by vendor, model and serial, not to a display id: `ft-sunshine-instance` looks the id up at every start and passes `output_name=ID`. While the monitor is unplugged it doesn't start Sunshine, which would fall back to the main display, and launchd tries again.
|
||||
- One certificate for all the instances, so Remote Displays pins one key per Mac; one Web UI login file for all, kept from an earlier Sunshine install if there was one. An earlier Homebrew Sunshine's LaunchAgent (the same ports) is turned off, and back on with `--undo`.
|
||||
- Screen Recording and Accessibility are granted by hand, once for all the instances: macOS lists them as Sunshine (the LaunchAgent runs a bash script that execs it; `AssociatedBundleIdentifiers` makes Login Items say Sunshine too). Sunshine started from a terminal gets that terminal's grants instead.
|
||||
- Without Accessibility (Post Event access) the pointer still moves, since libvirtualhid also warps it (`CGWarpMouseCursorPosition`, no permission needed), but clicks, scrolling and keys go only through `CGEventPost`, which macOS drops without a word: every Frame input method moved the pointer and clicked nothing, while the Mac's own trackpad worked. Sunshine notes at each start whether it had each permission (`defaults read dev.lizardbyte.app.Sunshine inputPostEventPermissionState`, 2 = granted, 1 = asked and not granted) and asks only once, so a grant macOS lost track of, or one given to an older Homebrew Sunshine (a separate app to macOS), stays broken with nothing asking again. The setup reports both permissions at the end and with `--check`. After the first instance's restart, if Accessibility is still missing, it offers to clear Sunshine's records (`tccutil reset Accessibility` and `PostEvent` for its bundle id, and that note) and restart the instance so macOS asks again, which fixed the test Mac (2026-10-10).
|
||||
- Frametop side: a display has a `port` when its instance isn't on 47989, and ft-stream takes `HOST:PORT`, so ft-screens passes it through as it is. The token, pin and audio lock stay per address. Given HOST alone, ft-stream takes its display's port from the layout, by its client: an ft-layout from before ports (an older release's Reset Screen Layout and profile launchers) restarted the Mac's streams at HOST, the instance on 47989, where they froze unpaired and their pointer went to the main display. `host_answers` checks the displays' ports. Remote Displays tries `/api/token` at sign-in; a bare 400 "Bad Request" (upstream Sunshine's answer to a path it doesn't have, before any login check) or a 404 means plain Sunshine, and the login is checked on `/api/clients/list`. It's kept in memory only, passed on stdin to `ft-layout remote add --port P --login` and on to `ft-stream pair --login` for each display picked, then forgotten. Upstream Sunshine (v2026.1010) lists a waiting pairing request with an id (`GET /api/pin`); ft-stream approves its own (libgamestream's device name "roth", from its own address) with `POST /api/pin {pairing_id, pin, name}`. The login is checked first: a wrong password's request, left waiting, blocked the next try. There are no permissions to grant. Add computer merges the instances' mDNS entries ("Mac (2)") into one computer, and Add displays lists the instances from their serverinfo on ports 47989 + 100·N.
|
||||
- Tested on the M3 Max (macOS 26.6.2: built-in 3024x1964, MPG491CX OLED 5120x1440, LC34G55T 3440x1440): the setup with `--check`, a first run and a rerun; the three instances answer from the Frame. With throwaway test instances (their own login, removed after): pairing with a wrong password then the right one; `ft-layout remote add --port --login` into a scratch layout; streams through a stand-in for ft-screens, the three at once (no losses; 35, 52 and 36 fps of still desktops, 3-9% of a Frame core each, about 6% CPU for Sunshine on the Mac for the OLED); the host's sound; a pointer move to two corners of the LC34G55T, not the main display, landing within a pixel of where it should. Not tested: the Remote Displays app itself, the live desktop, drags across the Mac's displays (separate processes; the release goes through the stream the press went down on), full-motion video's encode rate, replugging a monitor.
|
||||
|
||||
**Tested in the live desktop (frame-testbench, the 3D mouse through `@ft_pointer_helper`, 2026-10-07):**
|
||||
- An Explorer window carried by its title bar from the Remote Monitor to the OLED stayed there, and carried back, stayed there too. The log showed one change of screen each way.
|
||||
- Disconnect, connect: the panel came back where it was, at its width. Connect in a new run with the headset off: placed from screen 1's anchor.
|
||||
- Windows rescales a window that moves between displays with different scaling, so the point you held moves on the window. A second drag from the same spot can land on the address bar instead of the title bar. That's Windows, not the routing.
|
||||
|
||||
**Still to do:**
|
||||
- A headset test of all of it in the live desktop:
|
||||
- the row above the screens;
|
||||
- moving a remote display and saving a profile, then `use` putting it back;
|
||||
- adding a display from Remote Displays;
|
||||
- drags across with a controller, and gaze on the remote panels.
|
||||
- The shutdown hardening above, verified: restart the desktop with remote screens up and the headset in standby (a crash takes the gamescope session down, so only with the user's OK).
|
||||
- The floating windows' panels made on demand, tested live.
|
||||
- Sound in the headset, by ear: one copy, in time with the picture.
|
||||
- Over the network, a lost frame still waits for a keyframe. Reference frame invalidation (moonlight-common-c's `CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC`) would recover without one, if the decoder copes.
|
||||
- Find the dongle again if its address changes (the hotspot's DHCP), and a host whose own address changes (the token and pin files are named by it).
|
||||
- "Approve on the PC" instead of the password (our Vibepollo fork), later if wanted.
|
||||
- A per-display mouse mode (relative, for games).
|
||||
- A picture for a stream that's connecting or lost.
|
||||
- The installer doesn't build ft-stream yet (`stream/build.sh`), and `uninstall.sh` leaves `~/.local/share/frametop-stream` (the client keys and host tokens).
|
||||
- For the test, the pointer and gaze services run this branch's builds through drop-ins (`~/.config/systemd/user/frametop-{pointer,gaze}.service.d/remote-displays-worktree.conf`; `gaze/tracker/build` here links to the installed tracker's). Remove them when this is merged and installed.
|
||||
|
||||
## Open questions
|
||||
|
||||
- Which of the test PC's two desk monitors is the real one (the main session)?
|
||||
- Mac virtual displays: BetterDisplay switched on by `global_prep_cmd`, or a small helper on the private `CGVirtualDisplay` API. Not needed for v1.
|
||||
- Audio: none, the focused display's host, or a fixed one?
|
||||
- Remote displays outside the desktop: should they also show over games, where the decoder is shared with Steam Link VR?
|
||||
@@ -1,76 +0,0 @@
|
||||
# Spotwatch
|
||||
|
||||
Spotwatch is Frametop's search for apps and remote displays, like Spotlight on a Mac. Press Meta+Space and a glass search pill opens straight ahead of where your head points. Type part of an app's name, or a remote display's `{host} - {display}`, and pick from what it finds:
|
||||
|
||||
- **An app that's already running:** its most recent window comes to you. The lazy susan spins the panel it's on, a screen or its own floating window, to straight ahead, and typing goes to it.
|
||||
- **An app that isn't running:** it starts as a floating window, where that app last floated or in front of you.
|
||||
- **A remote display:** spotwatch connects it if needed, then spins it to the front.
|
||||
|
||||
Use Tab, Shift+Tab, or the arrow keys to move through the results, and Enter or a click to open one. Shift+Enter or a right click opens the item's menu:
|
||||
|
||||
- **For an app:** its own launch options, such as a browser's New Private Window, plus Open new window (even when one is open), Open docked on this screen, and each of its open windows.
|
||||
- **For a remote display:** Go to (connecting it first if needed) and Disconnect.
|
||||
|
||||
Esc goes back from a menu and closes the search. Meta+Space again, or a click anywhere else, closes it too.
|
||||
|
||||
Spotwatch works while the Frametop desktop is running and Frametop isn't paused for a VR game. It's the `spotwatch` action in Frametop Input Settings, so a mouse button, a controller button, or another key combination can open it too. When it opens from a mouse or controller, Frametop's keyboard opens with it.
|
||||
|
||||
## Not built
|
||||
|
||||
- Searching open windows by title, Frametop's own actions ("Reset layout"), and profiles (decision 4).
|
||||
- Moving the caret within the query. Typing and Backspace work at the end only.
|
||||
- A setting for its distance and size (`--distance` and `--width` on ft-spotpanel; 1 m and 0.56 m).
|
||||
|
||||
## Decisions
|
||||
|
||||
These were settled with the user on 2026-10-10. The numbers may be cited in the code, so they stay as they are.
|
||||
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| 1 | The look | A rounded glass pill with a results card under it. The glass is faux, because SteamVR's overlays have alpha but no backdrop blur: a translucent dark tint, a soft sheen from the top, a bright rim fading down the edges, and a shadow outside. Real blur would mean reading the panels back every frame on the Frame's GPU |
|
||||
| 2 | Where it opens | Fixed in the room, centred where the head points (not the eye gaze), at 1 m, upright, tilted no more than 60° up or down. The card hangs below the pill, so the pill stays put as results change |
|
||||
| 3 | An app with several windows | One row per app. Enter goes to its most recently active window, and Shift+Enter or a right click lists the others with the menu items |
|
||||
| 4 | What it searches | Apps and remote displays only. Hidden remote displays are left out, and disconnected ones show with Connect |
|
||||
| 5 | Disconnected remote displays | Enter connects the display (`ft-layout remote connect`), waits for its panel, then spins to it |
|
||||
| 6 | When it works | Only while Frametop is running and active: the session starts it, and the input relay doesn't run the action while Frametop is paused. A pause while it's open closes it |
|
||||
| 7 | Without a keyboard | `spotwatch` is an action like the others, so it can be bound to a mouse button or a controller button. Opened that way, Frametop's keyboard opens too and closes with it |
|
||||
| 8 | The menu | The app's desktop actions, Open new window (the app's own new-window action when it has one, else a plain launch), Open docked on this screen (the screen nearest straight ahead), and its windows |
|
||||
| 9 | An app that isn't running | It floats, where ft-floatd puts any launched app (where it last floated, or in front of you) |
|
||||
| 10 | A window on a pinned panel | Pinned panels don't spin. Typing goes to it where it is (`front N`), and a short note says where it is ("on your left wrist") |
|
||||
| 11 | Shortcut | Meta+Space by default. Plasma's KRunner has Alt+Space, and nothing in Frametop used Meta+Space |
|
||||
| 12 | Branch | `spotwatch`, off `experimental`, merged into `experimental` |
|
||||
|
||||
## How it works
|
||||
|
||||
Spotwatch has three parts, and needs changes in ft-screens, ft-floatd and the input relay:
|
||||
|
||||
- **`spotwatch/ft-spotwatch`** (`ft_spotwatch.py`) runs inside the desktop's Plasma session, started from its autostart like ft-floatd, so it's up exactly while the desktop is. It keeps the app index (Gio's, so it sees what the desktop's menus see), the remote displays from the layout, the usage counts, and the search state. It decides what each key and click does. Its log is `/tmp/frametop-spotwatch.log`.
|
||||
- **`spotwatch/ft-spotpanel.cpp`** is the pill: a SteamVR overlay drawn on the CPU, like ft-gazepanel, built by `screens/build.sh` and run by ft-spotwatch in the container. It holds no state beyond the picture. ft-spotwatch sends it the whole state each time, and it sends back hovers, clicks, right clicks, scrolls, and clicks beside the card. Its clear area catches those clicks.
|
||||
- **`spotwatch/spotsearch.py`** ranks the results with plain Python, so its test runs anywhere. Name matches come first, from the strongest kind of match to the weakest: the name's start, a word's start, initials, anywhere in the name, then letters in order. Generic names, keywords, and program names count for less and never match fuzzily. Every word of the query must match. Use counts and use in the last day raise an item, and so does an app that's running.
|
||||
|
||||
The search has to take the typing from wherever it went, Steam included. While it's open, ft-screens sends every key to `@ft_spotwatch` instead of KWin or a remote display's stream, from keyboards and Frametop's keyboard alike, and tells the input relay that typing goes to the desktop, so the relay grabs the keyboards. ft-spotwatch holds this with `spotwatch on`, renewed every second. ft-screens lets it lapse after 3 seconds, so a crashed ft-spotwatch can't keep the keys. A click on any other panel, or a pause, ends it, and ft-screens says `dismiss` so the pill closes.
|
||||
|
||||
ft-spotwatch turns key codes into text on the desktop's keyboard layout (Plasma's `kxkbrc`) through libxkbcommon, and falls back to a US layout without it. The input relay sends no key repeats, so ft-spotwatch repeats a held Backspace or arrow key itself.
|
||||
|
||||
Bringing a window to the front takes three commands:
|
||||
|
||||
1. ft-floatd's `activate ID` makes the window KWin's active window, unminimized.
|
||||
2. ft-screens' `spin to N` brings panel N to the front slot at reading distance (or, with `SCREEN_SPIN_BRING=0`, turns the ring of panels the short way so it ends straight ahead). Screens are 1 to S, floating windows come after them, and remote displays start at 101.
|
||||
3. When the spin settles, ft-screens gives that panel the typing, as any spin does. When the panel can't spin (pinned, or a floating window that's minimized), `front N` gives it the typing where it is.
|
||||
|
||||
ft-floatd's `spotlist` lists every app window with its desktop file name, title, panel, and when it was last active. The KWin script keeps that timestamp.
|
||||
|
||||
Launching uses ft-floatd's `launch APP.desktop`, which now takes `action=ACTION` (one of the desktop file's actions) and `screen=N` (open docked on screen N; the KWin script's `to-output`). Without ft-floatd (`FLOAT_SLOTS=0`), spotwatch launches through Gio itself, and nothing counts as running.
|
||||
|
||||
Apps' icons come from the desktop's icon theme (`kdeglobals`, then breeze and hicolor). GdkPixbuf rasterizes them to 64x64 RGBA files in `~/.cache/frametop/spotwatch` for the panel, one at a time in idle moments after the desktop starts. Use counts are kept in `~/.config/frametop-spotwatch.json`.
|
||||
|
||||
```
|
||||
spotwatch/test/spotwatch-test.py # the ranking, then ft-spotwatch against a fake ft-screens and ft-floatd (offline)
|
||||
spotwatch/tools/render.py OUT.png # what the panel draws, without a headset (ft-spotpanel --render)
|
||||
```
|
||||
|
||||
## Risks
|
||||
|
||||
- **Stray key releases.** A key pressed before the search opened is released to wherever it went, so nothing stays held. A key pressed in the search and released after it closes (the Enter that opened something) reaches the window that now has the typing as a release without a press. Apps ignore those.
|
||||
- **Laser input.** While the pill shows, it makes SteamVR's lasers active (`MakeOverlaysInteractiveIfVisible`), as Frametop's keyboard does. The search is short-lived, but during that time a game loses its controllers.
|
||||
- **Matching windows to apps.** Matching uses KWin's desktop file name, then the window class. An app whose windows report neither, such as some X11 apps under Flatpak, doesn't show as running, and Enter starts it again.
|
||||
+15
-325
@@ -2,7 +2,7 @@
|
||||
// loads it into the desktop's KWin over D-Bus (org.kde.kwin.Scripting) and talks to it:
|
||||
// - events go to ft-floatd as JSON strings (org.frametop.Float.Event), for the windows it
|
||||
// cares about: floating windows (the ones on a spare output, WL-<screens> and up), their
|
||||
// popups and dialogs, new windows, and requests to float or dock one;
|
||||
// popups and dialogs, new windows, and "Float in VR" requests;
|
||||
// - commands come back through a long poll: the script calls NextCommand, ft-floatd
|
||||
// answers when it has one (or after a while with nothing), and the script calls again.
|
||||
// KWin scripts can call D-Bus but can't serve it, hence the poll. Window ids are KWin's
|
||||
@@ -12,12 +12,6 @@ const SERVICE = "org.frametop.Float", PATH = "/Float", IFACE = "org.frametop.Flo
|
||||
let screens = 0; // outputs WL-0 .. WL-<screens - 1> are screens; the rest are spares
|
||||
let polling = false;
|
||||
const watched = {}; // id -> true once its signals are connected
|
||||
let marking = false; // the script itself is setting keep-below (see mark)
|
||||
const settled = {}; // id -> {output, frame, fullScreen, maximized}: where a window belongs (see putBack)
|
||||
const held = {}; // id -> {w, h, until, asked}: a size asked for, for a second (see hold)
|
||||
const moved = {}; // id -> true, or "back" once put back: it moved while KWin changed the outputs
|
||||
let layout = "", layoutOutputs = {}, layoutSince = 0; // the outputs at the last screensChanged
|
||||
const activeAt = {}; // id -> when it was last KWin's active window (ms), for spotwatch's most recent
|
||||
|
||||
function send(ev) {
|
||||
callDBus(SERVICE, PATH, IFACE, "Event", JSON.stringify(ev));
|
||||
@@ -53,203 +47,29 @@ function info(w) {
|
||||
frame: rect(w.frameGeometry), client: rect(w.clientGeometry), popup: w.popupWindow,
|
||||
transient: w.transient, parent: w.transientFor ? String(w.transientFor.internalId) : "",
|
||||
normal: w.normalWindow, dialog: w.dialog, fullScreen: w.fullScreen, minimized: w.minimized,
|
||||
onAllDesktops: w.onAllDesktops, maximized: isMaximized(w), activeAt: activeAt[String(w.internalId)] || 0
|
||||
onAllDesktops: w.onAllDesktops
|
||||
};
|
||||
}
|
||||
|
||||
// KWin 6.2's scripts have no maximize mode to read: a window is maximized when it fills its
|
||||
// output's maximize area.
|
||||
function isMaximized(w) {
|
||||
if (!w.normalWindow || !w.output) return false;
|
||||
const a = workspace.clientArea(KWin.MaximizeArea, w), g = w.frameGeometry;
|
||||
return g.x === a.x && g.y === a.y && g.width === a.width && g.height === a.height;
|
||||
}
|
||||
|
||||
function report(type, w) {
|
||||
if (w.deleted) return; // a window on its way out still changes output and size
|
||||
const ev = info(w);
|
||||
ev.ev = type;
|
||||
send(ev);
|
||||
}
|
||||
|
||||
// KWin's placement memory (its PlacementTracker) keeps each window's geometry for each layout of
|
||||
// the outputs (every enabled output's name and geometry), and when the outputs come back to a
|
||||
// layout it has seen, it puts the windows back where they were in it. That's for plugging monitors
|
||||
// in and out, and it does harm here. A spare output changes size after its window does, so what
|
||||
// KWin keeps for a spare's size is the window's next size: resizing a floating window back to a
|
||||
// size it had set off an endless flip between two sizes. And floating or docking one window could
|
||||
// move others, even onto a spare or off one. So the script keeps where each window belongs
|
||||
// (settled), tells ft-floatd nothing while KWin changes the outputs, and once KWin is done
|
||||
// (screensChanged comes after its restore) puts the floating windows back, and the screens' windows
|
||||
// too when only spares changed.
|
||||
function outputsNow() {
|
||||
const all = workspace.screens, out = {};
|
||||
for (let i = 0; i < all.length; ++i) {
|
||||
const g = all[i].geometry;
|
||||
out[all[i].name] = g.x + "," + g.y + " " + g.width + "x" + g.height;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
function keyOf(outputs) {
|
||||
return Object.keys(outputs).sort().map(n => n + "=" + outputs[n]).join(" ");
|
||||
}
|
||||
function takeLayout() {
|
||||
layoutOutputs = outputsNow();
|
||||
layout = keyOf(layoutOutputs);
|
||||
layoutSince = 0;
|
||||
}
|
||||
// KWin is changing the outputs: they differ from the last screensChanged.
|
||||
function changingOutputs() {
|
||||
if (keyOf(outputsNow()) === layout) {
|
||||
layoutSince = 0;
|
||||
return false;
|
||||
}
|
||||
if (!layoutSince) {
|
||||
layoutSince = Date.now();
|
||||
} else if (Date.now() - layoutSince > 2000) {
|
||||
takeLayout(); // screensChanged should have come by now: don't stay quiet for good
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function settle(w) {
|
||||
if (w.output) {
|
||||
settled[String(w.internalId)] = {output: w.output.name, frame: rect(w.frameGeometry), fullScreen: w.fullScreen};
|
||||
}
|
||||
}
|
||||
// ft-floatd put the window here: it's where it belongs now.
|
||||
function expect(w, output, c, fullScreen) {
|
||||
settled[String(w.internalId)] = {output: output, frame: {x: c.x, y: c.y, w: c.w, h: c.h}, fullScreen: fullScreen};
|
||||
hold(w, c.w, c.h);
|
||||
}
|
||||
|
||||
// A size asked for (by ft-floatd, or by the script putting a window back) comes in when the app
|
||||
// answers, and until then the app can still answer older requests: one KWin's restore made, or,
|
||||
// just after it opened, its own. For a second, the script asks again instead of taking those;
|
||||
// then it takes the size the window has (an app can refuse a size, below its minimum).
|
||||
const holdTimer = new QTimer();
|
||||
holdTimer.singleShot = true;
|
||||
holdTimer.timeout.connect(() => {
|
||||
const now = Date.now();
|
||||
Object.keys(held).forEach(id => {
|
||||
const h = held[id];
|
||||
if (h.until > now) return;
|
||||
delete held[id];
|
||||
const w = byId(id);
|
||||
if (!h.asked || !w || w.deleted) return; // (nothing held back: nothing to tell)
|
||||
settle(w);
|
||||
if (isSpare(w.output)) report("geometry", w);
|
||||
});
|
||||
if (Object.keys(held).length) holdTimer.start();
|
||||
});
|
||||
function hold(w, width, height) {
|
||||
held[String(w.internalId)] = {w: width, h: height, until: Date.now() + 1000};
|
||||
holdTimer.interval = 1100;
|
||||
holdTimer.start();
|
||||
}
|
||||
// A size change while a size is held: true when it isn't that size (the script asked again).
|
||||
// (ft-floatd's sizes can be fractional, the window's are whole: within a pixel is the same.)
|
||||
function holding(w) {
|
||||
const id = String(w.internalId), h = held[id], s = settled[id];
|
||||
if (!h) return false;
|
||||
const g = w.frameGeometry;
|
||||
if (!s || h.until < Date.now() || w.move || w.resize || (Math.abs(g.width - h.w) < 1 && Math.abs(g.height - h.h) < 1)) {
|
||||
delete held[id];
|
||||
return false;
|
||||
}
|
||||
w.frameGeometry = {x: s.frame.x, y: s.frame.y, width: h.w, height: h.h};
|
||||
h.asked = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Runs while KWin's output change still counts as going on (nothing reported), so the steps on
|
||||
// the way don't reach ft-floatd: told only where the window ends up (see reportMoved).
|
||||
function putBack(w, screensChanged) {
|
||||
const id = String(w.internalId), s = settled[id];
|
||||
if (w.deleted || !s || screens === 0 || !marked(w)) return;
|
||||
const o = outputByName(s.output);
|
||||
// KWin's restore sets full screen (and maximized) as it was in that layout too: with a
|
||||
// floating window that flipped forever, its output changing size with it. Ask for the
|
||||
// state it had: KWin's request hasn't reached the app yet, so it never sees it.
|
||||
w.fullScreen = s.fullScreen;
|
||||
if (o && isSpare(o) && !s.fullScreen) w.setMaximize(false, false);
|
||||
// Not where KWin had to move it: its output went, a screen changed, or it's full screen or
|
||||
// maximized (KWin fits those to their output).
|
||||
const back = o && !s.fullScreen && !w.fullScreen && !w.move && !w.resize && !s.maximized
|
||||
&& (isSpare(o) || (!screensChanged && !isMaximized(w)));
|
||||
if (!back) return;
|
||||
if (!w.output || w.output.name !== s.output) workspace.sendClientToScreen(w, o);
|
||||
w.frameGeometry = {x: s.frame.x, y: s.frame.y, width: s.frame.w, height: s.frame.h};
|
||||
if (isSpare(o)) hold(w, s.frame.w, s.frame.h);
|
||||
// Moved during the change (by KWin, by the lines above, or the size ft-floatd asked for came
|
||||
// in): report where it is, and keep settled as it is.
|
||||
if (moved[id]) moved[id] = "back";
|
||||
}
|
||||
// After an output change: tell ft-floatd where the windows that moved during it are now.
|
||||
function reportMoved(w) {
|
||||
const id = String(w.internalId), s = settled[id], how = moved[id];
|
||||
if (!how) return;
|
||||
delete moved[id];
|
||||
if (w.deleted) return;
|
||||
if (how !== "back") settle(w);
|
||||
if (!w.output || !s || w.output.name !== s.output) {
|
||||
report("output", w);
|
||||
mark(w);
|
||||
} else if (isSpare(w.output)) {
|
||||
report("geometry", w);
|
||||
}
|
||||
}
|
||||
workspace.screensChanged.connect(() => {
|
||||
const before = layoutOutputs, now = outputsNow();
|
||||
if (keyOf(now) === layout) return;
|
||||
let screensChanged = screens === 0;
|
||||
Object.keys(Object.assign({}, before, now)).forEach(name => {
|
||||
const m = /^WL-(\d+)$/.exec(name);
|
||||
if (before[name] !== now[name] && !(m && parseInt(m[1]) >= screens)) screensChanged = true;
|
||||
});
|
||||
const all = workspace.windowList();
|
||||
all.forEach(w => putBack(w, screensChanged));
|
||||
takeLayout();
|
||||
all.forEach(reportMoved);
|
||||
});
|
||||
|
||||
// Floating windows, and popups and dialogs on a spare output: tell ft-floatd about changes.
|
||||
function watch(w) {
|
||||
const id = String(w.internalId);
|
||||
if (watched[id]) return;
|
||||
watched[id] = true;
|
||||
const onSpare = () => isSpare(w.output);
|
||||
w.frameGeometryChanged.connect(() => {
|
||||
if (changingOutputs()) {
|
||||
moved[id] = true;
|
||||
return;
|
||||
}
|
||||
if (holding(w)) return;
|
||||
settle(w);
|
||||
if (onSpare()) report("geometry", w);
|
||||
});
|
||||
w.outputChanged.connect(() => {
|
||||
if (changingOutputs()) {
|
||||
moved[id] = true;
|
||||
return;
|
||||
}
|
||||
if (!held[id]) settle(w); // (held: the place asked for is settled already)
|
||||
report("output", w);
|
||||
mark(w);
|
||||
});
|
||||
w.keepBelowChanged.connect(() => keepBelowChanged(w));
|
||||
w.frameGeometryChanged.connect(() => { if (onSpare()) report("geometry", w); });
|
||||
w.outputChanged.connect(() => report("output", w));
|
||||
w.interactiveMoveResizeStarted.connect(() => {
|
||||
if (onSpare()) send({ev: "move-start", id: id, move: w.move, resize: w.resize, frame: rect(w.frameGeometry)});
|
||||
});
|
||||
w.interactiveMoveResizeFinished.connect(() => { if (onSpare()) report("move-end", w); });
|
||||
w.fullScreenChanged.connect(() => {
|
||||
if (changingOutputs()) {
|
||||
moved[id] = true;
|
||||
return;
|
||||
}
|
||||
if (settled[id]) settled[id].fullScreen = w.fullScreen;
|
||||
if (onSpare()) report("fullscreen", w);
|
||||
});
|
||||
w.fullScreenChanged.connect(() => { if (onSpare()) report("fullscreen", w); });
|
||||
w.minimizedChanged.connect(() => { if (onSpare()) report("minimized", w); });
|
||||
w.maximizedChanged.connect(() => {
|
||||
// A floating window stays an ordinary window: its output is its size plus a margin.
|
||||
@@ -259,54 +79,16 @@ function watch(w) {
|
||||
|
||||
workspace.windowAdded.connect(w => {
|
||||
watch(w);
|
||||
settle(w);
|
||||
report("added", w);
|
||||
});
|
||||
workspace.windowRemoved.connect(w => {
|
||||
const id = String(w.internalId);
|
||||
send({ev: "removed", id: id});
|
||||
delete watched[id];
|
||||
delete settled[id];
|
||||
delete held[id];
|
||||
delete moved[id];
|
||||
delete activeAt[id];
|
||||
send({ev: "removed", id: String(w.internalId)});
|
||||
delete watched[String(w.internalId)];
|
||||
});
|
||||
workspace.windowActivated.connect(w => {
|
||||
if (w) activeAt[String(w.internalId)] = Date.now();
|
||||
if (w && isSpare(w.output)) send({ev: "activated", id: String(w.internalId)});
|
||||
});
|
||||
takeLayout();
|
||||
workspace.windowList().forEach(w => {
|
||||
watch(w);
|
||||
settle(w);
|
||||
});
|
||||
|
||||
// Keep-below means "floating" in the Frametop desktop. The title bar's float button (Frametop's
|
||||
// window decoration, decoration/) is the Keep Below button, so setting the flag on a window on
|
||||
// the screens floats it, and clearing it on a floating one docks it. The script keeps the flag
|
||||
// set on every floating window, its dialogs included, and cleared everywhere else, however the
|
||||
// window got there. Kept below, a window alone on its own output only has the wallpaper under it.
|
||||
function marked(w) {
|
||||
return w.managed && !w.deleted && !w.specialWindow && !w.popupWindow;
|
||||
}
|
||||
function topOf(w) {
|
||||
let top = w;
|
||||
for (let n = 0; top.transientFor && n < 10; ++n) top = top.transientFor;
|
||||
return top;
|
||||
}
|
||||
function mark(w) {
|
||||
if (screens === 0 || !marked(w)) return;
|
||||
const want = isSpare(w.output);
|
||||
if (w.keepBelow === want) return;
|
||||
marking = true;
|
||||
w.keepBelow = want;
|
||||
marking = false;
|
||||
}
|
||||
function keepBelowChanged(w) {
|
||||
if (marking || screens === 0 || !marked(w)) return;
|
||||
const top = topOf(w);
|
||||
if (w.keepBelow !== isSpare(top.output)) requestFloat(top);
|
||||
}
|
||||
workspace.windowList().forEach(watch);
|
||||
|
||||
function requestFloat(w) {
|
||||
if (!w || !w.normalWindow || w.popupWindow) return;
|
||||
@@ -322,35 +104,15 @@ registerUserActionsMenu(w => {
|
||||
triggered: () => requestFloat(w)
|
||||
};
|
||||
});
|
||||
// The float key is the input relay's (float_toggle, Meta+Shift+F by default): it reaches us as
|
||||
// "request-pointer". No shortcut of KWin's own, so one press can't float a window and dock it again.
|
||||
|
||||
// The window under KWin's pointer (where the 3D mouse or a laser last was on a panel): the top
|
||||
// one there, a popup or dialog standing for the window it belongs to. Null over the wallpaper
|
||||
// or the taskbar.
|
||||
function underPointer() {
|
||||
const p = workspace.cursorPos;
|
||||
const order = workspace.stackingOrder;
|
||||
for (let i = order.length - 1; i >= 0; --i) {
|
||||
const w = order[i];
|
||||
if (w.deleted || w.minimized || w.hidden || !w.managed) continue;
|
||||
const g = w.frameGeometry;
|
||||
if (p.x < g.x || p.y < g.y || p.x >= g.x + g.width || p.y >= g.y + g.height) continue;
|
||||
const top = topOf(w);
|
||||
return top.normalWindow && !top.popupWindow ? top : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
registerShortcut("Frametop Float Window", "Frametop: Float Window in VR (or put it back)", "Meta+Shift+F",
|
||||
() => requestFloat(workspace.activeWindow));
|
||||
|
||||
function run(c) {
|
||||
const w = c.id ? byId(c.id) : null;
|
||||
switch (c.cmd) {
|
||||
case "config":
|
||||
screens = c.screens;
|
||||
workspace.windowList().forEach(w => { report("window", w); mark(w); });
|
||||
break;
|
||||
case "mark": // after a float that didn't happen: keep-below back as it was
|
||||
if (w) mark(w);
|
||||
workspace.windowList().forEach(w => report("window", w));
|
||||
break;
|
||||
case "place": { // onto an output, at a frame rectangle (logical, global)
|
||||
if (!w) break;
|
||||
@@ -358,109 +120,37 @@ function run(c) {
|
||||
if (!o) break;
|
||||
if (w.fullScreen && !c.keepFullScreen) w.fullScreen = false;
|
||||
w.setMaximize(false, false);
|
||||
expect(w, o.name, c, w.fullScreen && !!c.keepFullScreen);
|
||||
workspace.sendClientToScreen(w, o);
|
||||
w.frameGeometry = {x: c.x, y: c.y, width: c.w, height: c.h};
|
||||
if (c.onAllDesktops !== undefined) w.onAllDesktops = c.onAllDesktops;
|
||||
if (c.maximized) {
|
||||
// Maximized: KWin picks the size, and the place above is only where it goes.
|
||||
delete held[c.id];
|
||||
settled[c.id].maximized = true;
|
||||
w.setMaximize(true, true);
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "geometry":
|
||||
if (!w) break;
|
||||
expect(w, settled[c.id] ? settled[c.id].output : (w.output ? w.output.name : ""), c, w.fullScreen);
|
||||
w.frameGeometry = {x: c.x, y: c.y, width: c.w, height: c.h};
|
||||
if (w) w.frameGeometry = {x: c.x, y: c.y, width: c.w, height: c.h};
|
||||
break;
|
||||
case "close":
|
||||
if (w) w.closeWindow();
|
||||
break;
|
||||
case "activate":
|
||||
if (w && w.minimized) w.minimized = false;
|
||||
if (w) workspace.activeWindow = w;
|
||||
break;
|
||||
case "to-output": { // a launched window that opens docked, onto that screen (spotwatch)
|
||||
const o = outputByName(c.output);
|
||||
if (w && o && (!w.output || w.output.name !== o.name)) workspace.sendClientToScreen(w, o);
|
||||
if (w) workspace.activeWindow = w;
|
||||
break;
|
||||
}
|
||||
case "activate-output": { // the top window on that output (a spin brought it to the front)
|
||||
const order = workspace.stackingOrder;
|
||||
for (let i = order.length - 1; i >= 0; --i) {
|
||||
const o = order[i];
|
||||
if (o.deleted || o.minimized || o.hidden || !o.managed || !o.output) continue;
|
||||
if (o.output.name !== c.output || !o.normalWindow || o.popupWindow) continue;
|
||||
workspace.activeWindow = o;
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
case "minimize":
|
||||
if (w) w.minimized = c.on;
|
||||
break;
|
||||
case "info":
|
||||
if (w) report("window", w);
|
||||
break;
|
||||
case "report-all": // a profile's capture: every window as it is now, then a marker
|
||||
workspace.windowList().forEach(w => report("window", w));
|
||||
send({ev: "reported", token: c.token});
|
||||
break;
|
||||
case "request-float": // ft-float float ID: float it, if it isn't floating
|
||||
if (w && !isSpare(w.output)) requestFloat(w);
|
||||
break;
|
||||
case "request-active": // ft-float float|dock active
|
||||
case "request-active": // ft-float float|dock active: like the shortcut
|
||||
requestFloat(workspace.activeWindow);
|
||||
break;
|
||||
case "request-pointer": // the float key: the window under the pointer, else the active one
|
||||
requestFloat(underPointer() || workspace.activeWindow);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// The long poll has one failure mode: a reply that never comes. KWin's callDBus never calls
|
||||
// back when a call fails (an error reply, ft-floatd gone, or its 25 s D-Bus timeout, which
|
||||
// ft-floatd blocked that long hits); it only logs "Received D-Bus message is error". Then
|
||||
// polling stays true and the script stops hearing commands until KWin reloads it. A watchdog
|
||||
// re-arms the poll. Its period must stay above that 25 s timeout, whatever ft-floatd's
|
||||
// POLL_SECONDS is: by then the call it gives up on has ended. The serial is for a reply that
|
||||
// still comes later (KWin stalled with the reply queued): it runs its commands (ft-floatd
|
||||
// sends each one once) but doesn't poll again. Two polls would answer each other for good,
|
||||
// since ft-floatd answers a waiting poll empty when the next one comes.
|
||||
// When the watchdog fires again and again, ft-floatd is gone, and only starting it again
|
||||
// helps (it reloads the script then). Each failed call is a line in KWin's log, so the
|
||||
// script says so once and waits twice as long each time, up to 5 minutes. A reply starts
|
||||
// over at 30 s, so a lost reply is still polled again after 30 s.
|
||||
const POLL_WAIT = 30000, POLL_WAIT_MAX = 300000;
|
||||
const pollWatchdog = new QTimer();
|
||||
pollWatchdog.singleShot = true;
|
||||
pollWatchdog.interval = POLL_WAIT;
|
||||
let pollLost = false; // the watchdog fired, and no reply since
|
||||
pollWatchdog.timeout.connect(() => {
|
||||
if (!pollLost) print("frametop-float: NextCommand didn't answer, polling again");
|
||||
pollLost = true;
|
||||
pollWatchdog.interval = Math.min(pollWatchdog.interval * 2, POLL_WAIT_MAX);
|
||||
polling = false;
|
||||
poll();
|
||||
});
|
||||
let pollSerial = 0;
|
||||
|
||||
function poll() {
|
||||
if (polling) return;
|
||||
polling = true;
|
||||
const serial = ++pollSerial;
|
||||
pollWatchdog.start();
|
||||
callDBus(SERVICE, PATH, IFACE, "NextCommand", reply => {
|
||||
const current = serial === pollSerial; // not a call the watchdog gave up on
|
||||
if (current) {
|
||||
pollWatchdog.stop();
|
||||
pollWatchdog.interval = POLL_WAIT;
|
||||
pollLost = false;
|
||||
polling = false;
|
||||
}
|
||||
polling = false;
|
||||
if (reply) {
|
||||
try {
|
||||
JSON.parse(reply).forEach(run);
|
||||
@@ -468,7 +158,7 @@ function poll() {
|
||||
print("frametop-float: bad command " + reply + ": " + e);
|
||||
}
|
||||
}
|
||||
if (current) poll();
|
||||
poll();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+1
-12
@@ -2,21 +2,11 @@
|
||||
"""ft-float: talk to ft-floatd (floating windows in the Frametop desktop).
|
||||
|
||||
ft-float float [ID|active] float a window (default: the active one; for it, a toggle)
|
||||
ft-float float pointer the float key: float the window under the pointer (else the
|
||||
active one), or put it back if it floats
|
||||
ft-float dock [ID|active] put a floating window back on the desktop
|
||||
ft-float dock all put every floating window back
|
||||
ft-float launch APP start an app (its desktop file name, e.g. org.kde.dolphin) and
|
||||
float its first window where that app last floated
|
||||
ft-float launch APP action=ACTION ...one of its desktop file's actions (a private window)
|
||||
ft-float launch APP screen=N ...its first window docked on screen N instead
|
||||
ft-float run COMMAND [ARG...] the same for a command
|
||||
ft-float activate ID make a window KWin's active one, unminimized
|
||||
ft-float close ID close a window
|
||||
ft-float list the spare outputs and what floats on them
|
||||
FT_FLOAT_SOCKET names ft-floatd's socket (default frametop_float).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
@@ -27,8 +17,7 @@ s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
s.bind("")
|
||||
s.settimeout(5)
|
||||
try:
|
||||
text = "run " + json.dumps(sys.argv[2:]) if sys.argv[1] == "run" else " ".join(sys.argv[1:])
|
||||
s.sendto(text.encode(), "\0" + os.environ.get("FT_FLOAT_SOCKET", "frametop_float"))
|
||||
s.sendto(" ".join(sys.argv[1:]).encode(), "\0" + os.environ.get("FT_FLOAT_SOCKET", "frametop_float"))
|
||||
reply = s.recv(8192).decode()
|
||||
except OSError as e:
|
||||
sys.exit(f"ft-floatd didn't answer ({e}); is the Frametop desktop running?")
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Launch as Standalone: write the apps' desktop file copies (see ft_apps.py).
|
||||
exec python3 "$(dirname "$(readlink -f "$0")")/ft_apps.py" "$@"
|
||||
@@ -1,123 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Launch as Standalone: an action on every app in the Frametop desktop's menus.
|
||||
|
||||
Plasma's Application Launcher has no way to add an entry to every app's right-click menu,
|
||||
but its menu (and the taskbar's) shows each app's own desktop actions. So the Frametop
|
||||
desktop reads copies of the apps' desktop files with one more action, "Launch as
|
||||
Standalone", which runs `ft-float launch <desktop file name>`: the app starts and its first
|
||||
window floats in VR (docs/floating-windows.md, decision 26).
|
||||
|
||||
The copies go in OUT (~/.local/share/frametop/apps/applications), and the session puts
|
||||
OUT_ROOT first in XDG_DATA_DIRS, so a copy wins over the app's own file. Plasma's app cache
|
||||
is keyed by those directories, so Desktop Mode never sees the copies. Files in
|
||||
XDG_DATA_HOME/applications come before every data dir, so an app customized there keeps
|
||||
its own file and has no Launch as Standalone. A copy drops DBusActivatable: a D-Bus
|
||||
activated app would be asked to run the action itself, and it doesn't know ours.
|
||||
|
||||
ft-float-apps write the copies (the session script runs it before Plasma starts;
|
||||
ft-floatd again whenever an app's desktop file changes)
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
from gi.repository import GLib
|
||||
|
||||
ACTION = "frametop-standalone"
|
||||
OUT_ROOT = os.path.expanduser("~/.local/share/frametop/apps")
|
||||
OUT = os.path.join(OUT_ROOT, "applications")
|
||||
FT_FLOAT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "ft-float")
|
||||
GROUP = "Desktop Entry"
|
||||
|
||||
|
||||
def data_dirs():
|
||||
"""(XDG_DATA_HOME, the XDG_DATA_DIRS other than ours)."""
|
||||
home = os.environ.get("XDG_DATA_HOME") or os.path.expanduser("~/.local/share")
|
||||
ours = os.path.realpath(OUT_ROOT)
|
||||
dirs = [d for d in os.environ.get("XDG_DATA_DIRS", "/usr/local/share:/usr/share").split(":")
|
||||
if d and os.path.realpath(d) != ours]
|
||||
return home, dirs
|
||||
|
||||
|
||||
def app_dirs():
|
||||
"""Every applications folder that holds the apps' own desktop files, home first."""
|
||||
home, dirs = data_dirs()
|
||||
return [os.path.join(d, "applications") for d in [home] + dirs]
|
||||
|
||||
|
||||
def scan():
|
||||
"""Desktop file name -> (its path, whether it's in XDG_DATA_HOME), the first one found
|
||||
winning, as the desktop spec has it."""
|
||||
found = {}
|
||||
home_apps = app_dirs()[0]
|
||||
for root in app_dirs():
|
||||
for dirpath, _dirs, files in os.walk(root, followlinks=True):
|
||||
for name in files:
|
||||
if name.endswith(".desktop"):
|
||||
path = os.path.join(dirpath, name)
|
||||
found.setdefault(os.path.relpath(path, root).replace("/", "-"), (path, root == home_apps))
|
||||
return found
|
||||
|
||||
|
||||
def standalone(path, desktop_id):
|
||||
"""The desktop file with Launch as Standalone added, or None for one that isn't a
|
||||
visible app."""
|
||||
kf = GLib.KeyFile()
|
||||
try:
|
||||
kf.load_from_file(path, GLib.KeyFileFlags.KEEP_TRANSLATIONS | GLib.KeyFileFlags.KEEP_COMMENTS)
|
||||
except GLib.Error:
|
||||
return None
|
||||
|
||||
def get(key):
|
||||
try:
|
||||
return kf.get_string(GROUP, key)
|
||||
except GLib.Error:
|
||||
return None
|
||||
if get("Type") != "Application" or not get("Exec"):
|
||||
return None
|
||||
if (get("NoDisplay") or "").lower() == "true" or (get("Hidden") or "").lower() == "true":
|
||||
return None
|
||||
actions = [a for a in (get("Actions") or "").split(";") if a and a != ACTION]
|
||||
kf.set_string(GROUP, "Actions", ";".join(actions + [ACTION]) + ";")
|
||||
try:
|
||||
kf.remove_key(GROUP, "DBusActivatable")
|
||||
except GLib.Error:
|
||||
pass
|
||||
group = "Desktop Action " + ACTION
|
||||
kf.set_string(group, "Name", "Launch as Standalone")
|
||||
kf.set_string(group, "Icon", "window-new")
|
||||
exe = FT_FLOAT if " " not in FT_FLOAT else '"' + FT_FLOAT + '"'
|
||||
kf.set_string(group, "Exec", f"{exe} launch {desktop_id}")
|
||||
return kf.to_data()[0]
|
||||
|
||||
|
||||
def write_all():
|
||||
"""Write the copies, and remove the ones whose app is gone. Returns how many there are."""
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
keep = set()
|
||||
for desktop_id, (path, in_home) in sorted(scan().items()):
|
||||
if in_home:
|
||||
continue # XDG_DATA_HOME's own file wins over a copy anyway
|
||||
data = standalone(path, desktop_id)
|
||||
if data is None:
|
||||
continue
|
||||
keep.add(desktop_id)
|
||||
out = os.path.join(OUT, desktop_id)
|
||||
try:
|
||||
with open(out) as f:
|
||||
if f.read() == data:
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
with open(out + ".tmp", "w") as f:
|
||||
f.write(data)
|
||||
os.replace(out + ".tmp", out)
|
||||
for name in os.listdir(OUT):
|
||||
if name.endswith(".desktop") and name not in keep:
|
||||
os.remove(os.path.join(OUT, name))
|
||||
return len(keep)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1:
|
||||
sys.exit(__doc__)
|
||||
print(f"{write_all()} apps with Launch as Standalone in {OUT}")
|
||||
+30
-719
@@ -11,47 +11,17 @@ which window floats on which spare output and panel, and connects three parts:
|
||||
- kscreen-doctor, to turn spare outputs on and off and place them in KWin's layout.
|
||||
Commands and ft-screens' events arrive as datagrams on @frametop_float (ft-float is the
|
||||
command-line side). Replies go to the sender:
|
||||
float [ID|active|pointer] dock [ID|active|all] close ID list quit (ft-float)
|
||||
launch APP.desktop [action=ACTION] [screen=N] run ["argv", ...]
|
||||
(start an app, or one of its desktop file's actions, and float its first window;
|
||||
with screen=N its first window opens docked on screen N instead)
|
||||
windows (-> JSON: the open apps' windows and where they are, for a profile; ft-layout save)
|
||||
spotlist (-> JSON: every app window, its title, panel, and when it was last active; spotwatch)
|
||||
activate ID (make that window KWin's active one, unminimized; spotwatch)
|
||||
profile NAME (open a profile's apps: move the matching windows, launch the missing ones)
|
||||
("float pointer" is the float key: the window under the pointer, else the active one,
|
||||
floated or docked; the input relay sends it for float_toggle, and "dock all" for dock_all)
|
||||
float [ID|active] dock [ID|active] close ID list quit (ft-float)
|
||||
dock N | close N | resize N W H | scale N STEPS (ft-screens, N = its screen)
|
||||
front N (ft-screens: a spin brought panel N to the front: make its window, or the top one
|
||||
on a screen, KWin's active window)
|
||||
|
||||
Spare outputs are WL-<screens> .. WL-<screens + slots - 1>. A floating window's output is its
|
||||
frame plus a margin on each side (FLOAT_MARGIN pixels), so menus have room; the panel shows
|
||||
only the window. Spares are placed apart from the screens and from each other in KWin's
|
||||
layout, within Xwayland's 32767-pixel limit at scales up to 1.6.
|
||||
layout, all within Xwayland's 32767-pixel limit.
|
||||
|
||||
Usage: ft-floatd [--screens N] [--slots N] [--margin PX] [--control NAME] [--socket NAME]
|
||||
Defaults: FT_SCREEN_COUNT (from the session) or the layout's count, FLOAT_SLOTS and
|
||||
FLOAT_MARGIN from ~/.config/frametop.conf (8 and 300), @ft_screens, @frametop_float.
|
||||
|
||||
Launching floating: "launch" starts a desktop file's app (Gio), "run" a command. The next
|
||||
normal window from that process (or a child), or of that desktop file (desktop_name: by its app
|
||||
id, else its class, so Flatpak and X11 apps match too), within LAUNCH_SECONDS floats: where that app last floated, or in front of you. Single-instance and
|
||||
D-Bus-activated apps open their window from a process that was already running, hence the
|
||||
name. Each app's last floating place (its pose relative to the primary screen, so it moves
|
||||
with the screens' layout), size in pixels, and scale are kept in PLACES_PATH, by desktop
|
||||
file name, whenever one of its windows stops floating.
|
||||
|
||||
Profiles (docs/profiles.md): "windows" lists every app window with where it is: on a screen
|
||||
(its rectangle on that output, maximized or not) or floating (its pose relative to the primary
|
||||
screen, size, scale). "profile NAME" takes a profile's windows from the layout file: the
|
||||
windows of each app already open (oldest first) move to its entries, and for the entries
|
||||
left the app is launched once, and again for each window still missing 3 seconds after its
|
||||
first one shows up (a browser restores its own windows; a terminal opens one each time).
|
||||
Nothing is ever closed.
|
||||
|
||||
Launch as Standalone (float/ft_apps.py): copies of the apps' desktop files with that action,
|
||||
which only the Frametop desktop reads. ft-floatd rewrites them when apps change.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
@@ -66,27 +36,19 @@ import time
|
||||
import dbus
|
||||
import dbus.mainloop.glib
|
||||
import dbus.service
|
||||
from gi.repository import Gio, GLib
|
||||
from gi.repository import GLib
|
||||
|
||||
HERE = os.path.dirname(os.path.realpath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, "..", "layout"))
|
||||
import ft_layout # noqa: E402 (config and layout)
|
||||
import ft_apps # noqa: E402 (Launch as Standalone)
|
||||
|
||||
SERVICE = IFACE = "org.frametop.Float"
|
||||
PATH = "/Float"
|
||||
SCRIPT = "frametop-float"
|
||||
POLL_SECONDS = 20 # NextCommand answers empty after this (KWin's D-Bus timeout is 25 s)
|
||||
# Spares in KWin's layout: a grid 4 across, below the screens. X11 windows get Xwayland
|
||||
# coordinates of the layout's times its largest scale, which wrap past 32767: a window there
|
||||
# takes no clicks. 16 spares end at 20000, fine up to scale 1.6 (12000 across had the 4th
|
||||
# column past it at 1.25).
|
||||
SPARE_Y, SPARE_CELL = 4000, 4000
|
||||
SPARE_X, SPARE_CELL = 12000, 5000 # spares in KWin's layout: a grid from here, 4 across
|
||||
PULL_OUT = 0.3 # a floated window starts this far in front of its screen (metres), clear of it
|
||||
DEFAULT_MPP = 1.6 / 1920 # metres per pixel when ft-screens can't say (no SteamVR)
|
||||
PLACES_PATH = os.path.expanduser("~/.config/frametop-float.json") # each app's last floating place
|
||||
LAUNCH_SECONDS = 30 # a launched app's window must show up within this
|
||||
IN_FRONT = 2.0 # at most this far in front of you, for an app with no remembered place
|
||||
DEBUG = os.environ.get("FT_FLOAT_DEBUG") == "1" # log every event from the script
|
||||
|
||||
|
||||
@@ -146,20 +108,6 @@ def kscreen(*args):
|
||||
return ""
|
||||
|
||||
|
||||
def output_rects():
|
||||
"""Each output's place and size in KWin's layout (logical)."""
|
||||
try:
|
||||
data = json.loads(kscreen("-j") or "{}")
|
||||
except ValueError:
|
||||
return {}
|
||||
out = {}
|
||||
for o in data.get("outputs", []):
|
||||
pos, size, scale = o.get("pos") or {}, o.get("size") or {}, float(o.get("scale", 1)) or 1.0
|
||||
if o.get("name") and size:
|
||||
out[o["name"]] = (pos.get("x", 0), pos.get("y", 0), size.get("width", 0) / scale, size.get("height", 0) / scale)
|
||||
return out
|
||||
|
||||
|
||||
def output_scales():
|
||||
try:
|
||||
data = json.loads(kscreen("-j") or "{}")
|
||||
@@ -168,151 +116,6 @@ def output_scales():
|
||||
return {o["name"]: float(o.get("scale", 1)) for o in data.get("outputs", []) if o.get("name")}
|
||||
|
||||
|
||||
def load_places():
|
||||
try:
|
||||
with open(PLACES_PATH) as f:
|
||||
places = json.load(f)
|
||||
return places if isinstance(places, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def save_places(places):
|
||||
tmp = PLACES_PATH + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(places, f, indent=1)
|
||||
os.replace(tmp, PLACES_PATH)
|
||||
|
||||
|
||||
def to_local(ref, g):
|
||||
"""A panel's pose (ft_layout.parse_get) in the frame of another panel (ref): its centre
|
||||
and axes, 12 numbers."""
|
||||
axes = (ref["x"], ref["y"], ref["z"])
|
||||
d = [g["center"][k] - ref["center"][k] for k in range(3)]
|
||||
out = [ft_layout.dot(d, a) for a in axes]
|
||||
for v in (g["x"], g["y"], g["z"]):
|
||||
out += [ft_layout.dot(v, a) for a in axes]
|
||||
return [round(v, 5) for v in out]
|
||||
|
||||
|
||||
def to_world(ref, local):
|
||||
"""to_local's inverse: (centre, x, y, z) in the world."""
|
||||
axes = (ref["x"], ref["y"], ref["z"])
|
||||
|
||||
def turn(v):
|
||||
return [sum(v[i] * axes[i][k] for i in range(3)) for k in range(3)]
|
||||
c = turn(local[0:3])
|
||||
c = [c[k] + ref["center"][k] for k in range(3)]
|
||||
return c, turn(local[3:6]), turn(local[6:9]), turn(local[9:12])
|
||||
|
||||
|
||||
def pid_chain(pid):
|
||||
"""A process and its ancestors, nearest first."""
|
||||
out = []
|
||||
while pid > 1 and len(out) < 32:
|
||||
out.append(pid)
|
||||
try:
|
||||
with open(f"/proc/{pid}/stat") as f:
|
||||
pid = int(f.read().rsplit(")", 1)[1].split()[1])
|
||||
except (OSError, ValueError, IndexError):
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
# Windows a profile doesn't keep: Plasma's own, the Frametop settings apps, the login splash.
|
||||
SKIP_APPS = ("org.kde.plasmashell", "org.kde.krunner", "org.kde.ksplashqml", "org.kde.polkit-kde-authentication-agent-1")
|
||||
|
||||
|
||||
def recordable(ev):
|
||||
"""An app's top-level window, the kind a profile keeps."""
|
||||
if not ev.get("normal") or ev.get("popup") or ev.get("transient"):
|
||||
return False
|
||||
app, cls = ev.get("app", ""), ev.get("cls", "")
|
||||
return app not in SKIP_APPS and not any(n.startswith(("ft-", "frametop", "ksplash")) for n in (app, cls))
|
||||
|
||||
|
||||
def listable(ev):
|
||||
"""An app's top-level window, the kind spotwatch finds: Frametop's own settings apps too,
|
||||
which profiles leave out (an open Input Settings launched a second copy, 2026-10-13)."""
|
||||
if not ev.get("normal") or ev.get("popup") or ev.get("transient"):
|
||||
return False
|
||||
return ev.get("app", "") not in SKIP_APPS and not any(n.startswith("ksplash") for n in (ev.get("app", ""), ev.get("cls", "")))
|
||||
|
||||
|
||||
def cmdline(pid):
|
||||
try:
|
||||
with open(f"/proc/{int(pid)}/cmdline", "rb") as f:
|
||||
return [a.decode(errors="replace") for a in f.read().split(b"\0") if a]
|
||||
except (OSError, ValueError, TypeError):
|
||||
return []
|
||||
|
||||
|
||||
_names = {} # (app, cls) -> (desktop file name, when): desktop_name's answers for a minute
|
||||
|
||||
|
||||
def desktop_name(app, cls):
|
||||
now = time.monotonic()
|
||||
hit = _names.get((app, cls))
|
||||
if hit and now - hit[1] < 60:
|
||||
return hit[0]
|
||||
if len(_names) > 512:
|
||||
_names.clear()
|
||||
name = _desktop_name(app, cls)
|
||||
_names[(app, cls)] = (name, now)
|
||||
return name
|
||||
|
||||
|
||||
def _desktop_name(app, cls):
|
||||
"""The desktop file name (without .desktop) for a window's app id and class, or "" when no
|
||||
desktop file is its. Most windows give their desktop file name as the app id. The others:
|
||||
- A Flatpak app's window can give an id with no desktop file: RustDesk's (an X11 window)
|
||||
says com.carriez.flutter_hbb, and its desktop file is com.rustdesk.RustDesk.
|
||||
- An X11 window can give no app id at all, only its class: Brave's Flatpak says
|
||||
"Brave-browser" (2026-10-13), and its desktop file is com.brave.Browser.
|
||||
Then, by the class (or the id), in this order: a desktop file's StartupWMClass; a desktop
|
||||
file whose name, or its last dotted part, is that name; the program a desktop file runs
|
||||
(Flatpak's --command=, else the Exec's own), by its name."""
|
||||
try:
|
||||
if app and Gio.DesktopAppInfo.new(app + ".desktop"):
|
||||
return app
|
||||
except TypeError: # PyGObject raises for the NULL a missing desktop file returns
|
||||
pass
|
||||
names = {n.lower() for n in (app, cls) if n}
|
||||
if not names:
|
||||
return ""
|
||||
infos = [i for i in Gio.AppInfo.get_all() if isinstance(i, Gio.DesktopAppInfo) and i.get_id()]
|
||||
for info in infos:
|
||||
wm = info.get_startup_wm_class()
|
||||
if wm and wm.lower() in names:
|
||||
return info.get_id().removesuffix(".desktop")
|
||||
for info in infos:
|
||||
did = info.get_id().removesuffix(".desktop").lower()
|
||||
if did in names or did.rsplit(".", 1)[-1] in names:
|
||||
return info.get_id().removesuffix(".desktop")
|
||||
for info in infos:
|
||||
line = info.get_commandline() or ""
|
||||
m = re.search(r"--command=(\S+)", line)
|
||||
prog = os.path.basename(m.group(1) if m else (info.get_executable() or ""))
|
||||
if prog and prog.lower() not in ("flatpak", "env", "sh", "bash", "python3", "snap") and \
|
||||
any(n == prog.lower() or n.startswith(prog.lower() + "-") for n in names):
|
||||
return info.get_id().removesuffix(".desktop")
|
||||
return ""
|
||||
|
||||
|
||||
class Launch:
|
||||
"""An app we started: its first window floats, or (for a profile) its windows go to the
|
||||
profile's entries for it, in order."""
|
||||
|
||||
def __init__(self, app, pid, entries=None, argv=None):
|
||||
self.app = app # its desktop file name, without .desktop ("" for a command)
|
||||
self.argv = argv # the command, when it isn't a desktop file's app
|
||||
self.pids = [pid] if pid else [] # the processes we started (none when D-Bus started it)
|
||||
self.entries = entries # a profile's entries still waiting for a window, or None
|
||||
self.relaunched = False
|
||||
self.dock = None # its first window opens docked on this output (launch screen=N)
|
||||
self.until = time.monotonic() + LAUNCH_SECONDS
|
||||
|
||||
|
||||
class Float:
|
||||
"""A floating window."""
|
||||
|
||||
@@ -328,9 +131,7 @@ class Float:
|
||||
self.normal = None # its size in pixels when last not full screen
|
||||
self.unfull_until = 0.0 # left full screen just now (see follow)
|
||||
self.move_from = None # frame when a title-bar move started (put back after)
|
||||
self.resizing = False # KWin's resize by the window's edge is going on (see follow)
|
||||
self.subs = {} # popup or dialog id -> number on the panel
|
||||
self.app = "" # its desktop file name (for its remembered place)
|
||||
|
||||
|
||||
class Slot:
|
||||
@@ -338,7 +139,7 @@ class Slot:
|
||||
self.k = k
|
||||
self.output = f"WL-{screens + k}"
|
||||
self.index = screens + k + 1 # ft-screens' number (1-based)
|
||||
self.pos = ((k % 4) * SPARE_CELL, SPARE_Y + (k // 4) * SPARE_CELL)
|
||||
self.pos = (SPARE_X + (k % 4) * SPARE_CELL, (k // 4) * SPARE_CELL)
|
||||
self.size = None # its output's size in pixels, as last set
|
||||
self.want = None # the size in pixels asked for (size is at least that)
|
||||
self.kscale = 1.0 # its output's scale in KWin, as last set
|
||||
@@ -352,17 +153,11 @@ class Daemon:
|
||||
self.slots = [Slot(k, args.screens) for k in range(args.slots)]
|
||||
self.floats = {} # window id -> Float
|
||||
self.windows = {} # window id -> last info from the script
|
||||
self.gone = set() # ids of windows that closed (KWin's ids aren't reused)
|
||||
self.pending = [] # commands for the script
|
||||
self.waiter = None # (reply callback, timeout source) while the script waits
|
||||
self.screens = Screens(args.control)
|
||||
self.sub_numbers = {} # popup/dialog id -> (window id, number)
|
||||
self.next_sub = 1
|
||||
self.launches = [] # Launch: apps we started, waiting for their window
|
||||
self.captures = {} # token -> (sender, what): "windows" and "spotlist" requests waiting for the script's report
|
||||
self.next_token = 1
|
||||
self.sock = None # our socket (set by main), for deferred replies
|
||||
self.places = load_places() # app -> {"rel": pose by the primary screen, "pixels", "scale"}
|
||||
|
||||
# ------------------------------------------------------------ the script
|
||||
|
||||
@@ -400,19 +195,8 @@ class Daemon:
|
||||
sid = int(kwin.loadScript(os.path.join(HERE, "frametop-float.js"), SCRIPT, signature="ss"))
|
||||
if sid < 0:
|
||||
raise RuntimeError("KWin didn't load the script")
|
||||
# Scripting.start runs every loaded script that isn't running. Not /Scripting/Script<id>'s
|
||||
# run: a reloaded script gets the old one's id while the old one is still being deleted,
|
||||
# so that path is still the old script's, and the new one would never run.
|
||||
kwin.start()
|
||||
bus.get_object("org.kde.KWin", f"/Scripting/Script{sid}").run(dbus_interface="org.kde.kwin.Script")
|
||||
log(f"script loaded ({sid})")
|
||||
# Older scripts registered the float key with KWin; the input relay owns it now. Drop
|
||||
# that shortcut, so System Settings doesn't list one that does nothing.
|
||||
try:
|
||||
accel = dbus.Interface(bus.get_object("org.kde.kglobalaccel", "/kglobalaccel"), "org.kde.KGlobalAccel")
|
||||
if accel.unregister("kwin", "Frametop Float Window"):
|
||||
log("dropped KWin's old float shortcut")
|
||||
except dbus.DBusException:
|
||||
pass
|
||||
|
||||
# ------------------------------------------------------------ events from the script
|
||||
|
||||
@@ -427,10 +211,7 @@ class Daemon:
|
||||
# The script reports every window after "config"; spares nothing floats on are off.
|
||||
GLib.timeout_add(1500, self.disable_unused)
|
||||
return
|
||||
if wid in self.gone:
|
||||
return # an event that came after the window closed (it would bring it back)
|
||||
if kind == "removed":
|
||||
self.gone.add(wid)
|
||||
self.windows.pop(wid, None)
|
||||
if wid in self.floats:
|
||||
log(f"{wid[:9]} closed")
|
||||
@@ -440,29 +221,6 @@ class Daemon:
|
||||
if wid:
|
||||
self.windows[wid] = ev
|
||||
f = self.floats.get(wid)
|
||||
if kind == "reported":
|
||||
self.captured(ev.get("token"))
|
||||
return
|
||||
if kind == "added" and self.launches and not f:
|
||||
launch = self.launched(ev)
|
||||
if launch and launch.dock:
|
||||
log(f"{wid[:9]} ({ev.get('cls')}) opens docked on {launch.dock}")
|
||||
self.command(cmd="to-output", id=wid, output=launch.dock)
|
||||
self.seen(ev)
|
||||
return
|
||||
if launch and launch.entries is None:
|
||||
self.float_launched(ev, launch)
|
||||
return
|
||||
if launch:
|
||||
self.place_entry(ev, launch.entries.pop(0))
|
||||
if launch.entries and not launch.relaunched:
|
||||
GLib.timeout_add_seconds(3, lambda: self.relaunch(launch) and False)
|
||||
return
|
||||
if kind == "added" and not f and self.on_hidden_screen(ev):
|
||||
# Nobody would see it there (a profile can hide every screen): it floats instead.
|
||||
log(f"{wid[:9]} ({ev.get('cls')}) opened on a hidden screen")
|
||||
self.float_launched(ev, None)
|
||||
return
|
||||
if kind == "float-request":
|
||||
self.float_window(ev)
|
||||
elif kind == "dock-request":
|
||||
@@ -478,11 +236,6 @@ class Daemon:
|
||||
elif kind == "move-start" and f and ev.get("move"):
|
||||
f.move_from = ev["frame"]
|
||||
self.screens.ask(f"carry {f.slot.index}", quiet=True)
|
||||
elif kind == "move-start" and f and ev.get("resize"):
|
||||
f.resizing = True
|
||||
elif kind == "move-end" and f and f.resizing:
|
||||
f.resizing = False
|
||||
self.follow(f, ev)
|
||||
elif kind == "move-end" and f and f.move_from:
|
||||
# The panel carried the window; KWin may have slipped it a few pixels first.
|
||||
m, f.move_from = f.move_from, None
|
||||
@@ -525,14 +278,9 @@ class Daemon:
|
||||
return
|
||||
# Floating when ft-floatd (re)started: take it over where it is.
|
||||
f = Float(wid, slot, None)
|
||||
f.app = desktop_name(ev.get("app", ""), ev.get("cls", "")) or ev.get("app", "")
|
||||
slot.window = f
|
||||
self.floats[wid] = f
|
||||
f.scale = slot.kscale = output_scales().get(slot.output, 1.0)
|
||||
# Its panel's density as it is (follow below sets the panel again).
|
||||
g, px = self.panel_get(slot.index), round(ev["frame"]["w"] * f.scale)
|
||||
if g and g["metres"] > 0 and px > 0:
|
||||
f.mpp = g["metres"] / px
|
||||
log(f"{wid[:9]} ({ev.get('cls')}) already floats on {slot.output}")
|
||||
self.follow(f, ev)
|
||||
return
|
||||
@@ -542,16 +290,8 @@ class Daemon:
|
||||
for o in self.floats.values()):
|
||||
self.float_window(ev)
|
||||
else:
|
||||
# Onto the first screen that shows (or floating, if none does).
|
||||
hidden = self.concealed()
|
||||
shown = [i for i in range(self.screens_n) if i + 1 not in hidden]
|
||||
if not shown:
|
||||
self.float_launched(ev, None)
|
||||
return
|
||||
name = f"WL-{shown[0]}"
|
||||
x0, y0, _, _ = output_rects().get(name, (0, 0, 0, 0))
|
||||
self.command(cmd="place", id=wid, output=name, x=x0 + ev["frame"]["x"] % 400 + 100,
|
||||
y=y0 + ev["frame"]["y"] % 300 + 100, w=ev["frame"]["w"], h=ev["frame"]["h"])
|
||||
self.command(cmd="place", id=wid, output="WL-0", x=ev["frame"]["x"] % 400 + 100,
|
||||
y=ev["frame"]["y"] % 300 + 100, w=ev["frame"]["w"], h=ev["frame"]["h"])
|
||||
|
||||
# ------------------------------------------------------------ floating and docking
|
||||
|
||||
@@ -599,27 +339,22 @@ class Daemon:
|
||||
return float(metres) / max(1, int(size.split("x")[0]))
|
||||
return DEFAULT_MPP
|
||||
|
||||
def float_window(self, ev, place=None, mpp=None, scale=None):
|
||||
"""Float a window: its panel in front of where it was on its screen, or at place (centre
|
||||
and axes in the world), at the density of its screen, or mpp, and at its screen's scale,
|
||||
or scale (then ev's frame is the window's size at that scale). Returns its Float."""
|
||||
def float_window(self, ev):
|
||||
wid = ev["id"]
|
||||
if wid in self.floats:
|
||||
return self.floats[wid]
|
||||
return
|
||||
slot = self.free_slot()
|
||||
if slot is None:
|
||||
self.command(cmd="mark", id=wid) # the title bar button set keep-below for nothing
|
||||
self.notify(f"All {len(self.slots)} floating windows are in use. Put one back on the desktop "
|
||||
"to float another.")
|
||||
return None
|
||||
return
|
||||
f = Float(wid, slot, {"output": ev["output"], "frame": ev["frame"], "onAllDesktops": ev.get("onAllDesktops")})
|
||||
f.app = desktop_name(ev.get("app", ""), ev.get("cls", "")) or ev.get("app", "")
|
||||
slot.window = f
|
||||
self.floats[wid] = f
|
||||
scales = output_scales()
|
||||
f.scale = scale or scales.get(ev["output"], 1.0)
|
||||
f.scale = scales.get(ev["output"], 1.0)
|
||||
slot.kscale = scales.get(slot.output, slot.kscale)
|
||||
f.mpp = mpp or self.screen_mpp(ev["output"])
|
||||
f.mpp = self.screen_mpp(ev["output"])
|
||||
fr, s, m = ev["frame"], f.scale, self.margin
|
||||
w, h = round(fr["w"] * s), round(fr["h"] * s)
|
||||
log(f"{wid[:9]} ({ev.get('cls')}) floats on {slot.output}: {w}x{h} px, scale {s:g}")
|
||||
@@ -630,20 +365,12 @@ class Daemon:
|
||||
self.set_size(slot, (w + 2 * m, h + 2 * m), whole(slot.kscale, s))
|
||||
self.screens.ask(f"scale {slot.index} {s:g}") # for pointer positions (KWin's units)
|
||||
self.set_panel(f, (m, m, w, h), title=round((ev["client"]["y"] - fr["y"]) * s))
|
||||
if place:
|
||||
self.pose(f, *place)
|
||||
else:
|
||||
self.place_panel(f, ev)
|
||||
self.place_panel(f, ev)
|
||||
kscreen(f"output.{slot.output}.enable", f"output.{slot.output}.scale.{s:g}",
|
||||
f"output.{slot.output}.position.{slot.pos[0]},{slot.pos[1]}")
|
||||
self.rescaled(slot, s)
|
||||
self.command(cmd="place", id=wid, output=slot.output, x=slot.pos[0] + m / s, y=slot.pos[1] + m / s,
|
||||
w=fr["w"], h=fr["h"], onAllDesktops=True)
|
||||
return f
|
||||
|
||||
def pose(self, f, c, xa, ya, za):
|
||||
rows = [f"{xa[k]:.5f} {ya[k]:.5f} {za[k]:.5f} {c[k]:.4f}" for k in range(3)]
|
||||
self.screens.ask(f"pose {f.slot.index} {' '.join(rows)}")
|
||||
|
||||
def set_panel(self, f, crop, title=0):
|
||||
x, y, w, h = crop
|
||||
@@ -662,7 +389,8 @@ class Daemon:
|
||||
dx = ((fr["x"] - out["x"]) + fr["w"] / 2 - out["w"] / 2) * s * f.mpp
|
||||
dy = ((fr["y"] - out["y"]) + fr["h"] / 2 - out["h"] / 2) * s * f.mpp
|
||||
p = [c[k] + xa[k] * dx - ya[k] * dy + za[k] * PULL_OUT for k in range(3)]
|
||||
self.pose(f, p, xa, ya, za)
|
||||
rows = [f"{xa[k]:.5f} {ya[k]:.5f} {za[k]:.5f} {p[k]:.4f}" for k in range(3)]
|
||||
self.screens.ask(f"pose {f.slot.index} {' '.join(rows)}")
|
||||
|
||||
def follow(self, f, ev):
|
||||
"""The window moved or resized on its output: crop the panel to it, and keep the
|
||||
@@ -678,17 +406,9 @@ class Daemon:
|
||||
full = bool(ev.get("fullScreen")) or (fills and time.monotonic() > f.unfull_until)
|
||||
f.full = full
|
||||
w, h = round(fr["w"] * s), round(fr["h"] * s)
|
||||
if not full and not (f.normal and abs(f.normal[0] - w) <= math.ceil(s) and abs(f.normal[1] - h) <= math.ceil(s)):
|
||||
# (Within a logical pixel it's the same size: a size asked for in pixels comes out
|
||||
# rounded to whole logical pixels. Keeping it stops scale changes from creeping.)
|
||||
if not full:
|
||||
f.normal = (w, h)
|
||||
m = 0 if full else self.margin
|
||||
if f.resizing:
|
||||
# KWin ends a resize by the window's edge whenever an output changes: the output
|
||||
# follows when it's done (the margin is room to grow until then).
|
||||
x, y = round((fr["x"] - out["x"]) * s), round((fr["y"] - out["y"]) * s)
|
||||
self.set_panel(f, (x, y, w, h), title=round((cl["y"] - fr["y"]) * s))
|
||||
return
|
||||
self.set_size(slot, f.normal if full and f.normal else (w + 2 * m, h + 2 * m))
|
||||
if not full:
|
||||
x0, y0 = slot.pos[0] + m / s, slot.pos[1] + m / s
|
||||
@@ -703,14 +423,10 @@ class Daemon:
|
||||
panel stays the same size (KWin's output scale, in steps of 10%)."""
|
||||
if not f.frame or f.full or steps == 0:
|
||||
return
|
||||
self.set_scale(f, round(f.scale * 1.1 ** steps * 20) / 20)
|
||||
|
||||
def set_scale(self, f, s):
|
||||
"""The window's scale (KWin's output scale), at the same size in pixels."""
|
||||
s = min(3.0, max(0.5, s))
|
||||
if not f.frame or f.full or s == f.scale or self.floats.get(f.id) is not f:
|
||||
s = min(3.0, max(0.5, round(f.scale * 1.1 ** steps * 20) / 20))
|
||||
if s == f.scale:
|
||||
return
|
||||
w, h = f.normal or (round(f.frame["w"] * f.scale), round(f.frame["h"] * f.scale))
|
||||
w, h = round(f.frame["w"] * f.scale), round(f.frame["h"] * f.scale)
|
||||
slot, m = f.slot, self.margin
|
||||
log(f"{f.id[:9]} scale {f.scale:g} -> {s:g}")
|
||||
f.scale = s
|
||||
@@ -734,45 +450,17 @@ class Daemon:
|
||||
want, slot.want = slot.size, None
|
||||
self.set_size(slot, want)
|
||||
|
||||
def dock(self, f, frame=None, output=None, maximized=False):
|
||||
"""Back where it came from (or onto screen 1 if we don't know), or onto output at frame."""
|
||||
def dock(self, f, frame=None):
|
||||
"""Back where it came from (or onto screen 1 if we don't know)."""
|
||||
saved = f.saved or {"output": "WL-0", "frame": dict(f.frame or {"x": 100, "y": 100, "w": 800, "h": 600}),
|
||||
"onAllDesktops": False}
|
||||
fr = frame or saved["frame"]
|
||||
if not output:
|
||||
output = saved["output"]
|
||||
m = re.match(r"WL-(\d+)$", output)
|
||||
hidden = self.concealed()
|
||||
if m and int(m.group(1)) + 1 in hidden:
|
||||
# It came from a screen that's hidden now: onto the first one that shows.
|
||||
shown = [i for i in range(self.screens_n) if i + 1 not in hidden]
|
||||
if shown:
|
||||
x0, y0, ow, oh = output_rects().get(f"WL-{shown[0]}", (0, 0, 0, 0))
|
||||
output = f"WL-{shown[0]}"
|
||||
w, h = min(fr["w"], ow or fr["w"]), min(fr["h"], oh or fr["h"])
|
||||
fr = {"x": x0 + max(0, (ow - w) / 2), "y": y0 + max(0, (oh - h) / 2), "w": w, "h": h}
|
||||
log(f"{f.id[:9]} back to {output}")
|
||||
self.command(cmd="place", id=f.id, output=output, x=fr["x"], y=fr["y"], w=fr["w"], h=fr["h"],
|
||||
onAllDesktops=bool(saved.get("onAllDesktops")), maximized=maximized)
|
||||
|
||||
def remember(self, f):
|
||||
"""Keep where an app's window floated (before its panel goes)."""
|
||||
if not f.app or not f.frame:
|
||||
return
|
||||
g, ref = self.panel_get(f.slot.index), self.reference()
|
||||
if not g or not ref:
|
||||
return
|
||||
pixels = list(f.normal or (round(f.frame["w"] * f.scale), round(f.frame["h"] * f.scale)))
|
||||
self.places[f.app] = {"rel": to_local(ref, g), "pixels": pixels, "scale": f.scale,
|
||||
"mpp": round(f.mpp, 8)}
|
||||
try:
|
||||
save_places(self.places)
|
||||
except OSError as e:
|
||||
log(f"couldn't save {PLACES_PATH}: {e}")
|
||||
log(f"{f.id[:9]} back to {saved['output']}")
|
||||
self.command(cmd="place", id=f.id, output=saved["output"], x=fr["x"], y=fr["y"], w=fr["w"], h=fr["h"],
|
||||
onAllDesktops=bool(saved.get("onAllDesktops")))
|
||||
|
||||
def release(self, f):
|
||||
"""Its window left: hide the panel and turn the spare off."""
|
||||
self.remember(f)
|
||||
slot = f.slot
|
||||
if slot.window is f:
|
||||
slot.window = None
|
||||
@@ -782,312 +470,6 @@ class Daemon:
|
||||
self.screens.ask(f"unfloat {slot.index}", quiet=True)
|
||||
kscreen(f"output.{slot.output}.disable")
|
||||
|
||||
# ------------------------------------------------------------ launching floating
|
||||
|
||||
def panel_get(self, index):
|
||||
reply = self.screens.ask(f"get {index}", quiet=True)
|
||||
return ft_layout.parse_get(reply) if reply.startswith("ok") else None
|
||||
|
||||
def primary(self):
|
||||
"""The primary screen (0-based): the one with the taskbar."""
|
||||
try:
|
||||
return min(ft_layout.primary_screen(ft_layout.load_layout()), self.screens_n - 1)
|
||||
except (OSError, ValueError, KeyError):
|
||||
return 0
|
||||
|
||||
def reference(self):
|
||||
"""The primary screen's panel: remembered places are relative to it, so they move with
|
||||
the screens' layout."""
|
||||
return self.panel_get(self.primary() + 1)
|
||||
|
||||
def in_front(self):
|
||||
"""A place in front of you, facing you, a little nearer than the primary screen (so text
|
||||
looks as big as on the screens, at their density)."""
|
||||
reply = self.screens.ask("head", quiet=True)
|
||||
ref = self.reference()
|
||||
if not reply.startswith("ok"):
|
||||
return None
|
||||
h = reply.split()
|
||||
eye, heading = [float(v) for v in h[1:4]], float(h[4])
|
||||
# (Within reach: you may have walked away from the screens since they were arranged.)
|
||||
d = min(IN_FRONT, max(0.8, math.dist(eye, ref["center"]) - PULL_OUT)) if ref else IN_FRONT
|
||||
fwd = ft_layout.turn_yaw((0.0, 0.0, -1.0), heading)
|
||||
c = [eye[k] + fwd[k] * d for k in range(3)]
|
||||
c[1] -= 0.1 * d # a little below eye level, like a screen
|
||||
return c, ft_layout.turn_yaw((1.0, 0.0, 0.0), heading), (0.0, 1.0, 0.0), ft_layout.turn_yaw((0.0, 0.0, 1.0), heading)
|
||||
|
||||
def launch(self, app=None, argv=None, action=None, dock=None):
|
||||
"""Start an app (a desktop file name) or a command; its first window will float, or
|
||||
with dock (an output name) open docked there. action: one of the desktop file's
|
||||
actions instead (no pid: its window is known by the app's name). Returns (reply, pid or
|
||||
None)."""
|
||||
if app:
|
||||
app = app.removesuffix(".desktop")
|
||||
try:
|
||||
info = Gio.DesktopAppInfo.new(app + ".desktop")
|
||||
except TypeError: # PyGObject raises for the NULL a missing desktop file returns
|
||||
info = None
|
||||
if info is None:
|
||||
return f"error no app {app}", None
|
||||
if action and action not in info.list_actions():
|
||||
return f"error {app} has no action {action}", None
|
||||
pids = []
|
||||
try:
|
||||
if action:
|
||||
info.launch_action(action, Gio.AppLaunchContext())
|
||||
else:
|
||||
info.launch_uris_as_manager([], Gio.AppLaunchContext(), GLib.SpawnFlags.SEARCH_PATH,
|
||||
None, None, lambda _info, pid, *_: pids.append(pid), None)
|
||||
except GLib.Error as e:
|
||||
return f"error {app}: {e.message}", None
|
||||
la = Launch(app, pids[0] if pids else None)
|
||||
la.dock = dock
|
||||
self.launches.append(la)
|
||||
log(f"launched {app}{' (' + action + ')' if action else ''}"
|
||||
f"{' docked on ' + dock if dock else ''} (pid {pids[0] if pids else 'by D-Bus'})")
|
||||
return "ok", (pids[0] if pids else None)
|
||||
else:
|
||||
try:
|
||||
proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, start_new_session=True)
|
||||
except OSError as e:
|
||||
return f"error {argv[0]}: {e}", None
|
||||
GLib.child_watch_add(GLib.PRIORITY_DEFAULT, proc.pid, lambda *_: None) # reap it
|
||||
self.launches.append(Launch("", proc.pid))
|
||||
log(f"started {argv[0]} (pid {proc.pid})")
|
||||
return "ok", proc.pid
|
||||
|
||||
def launched(self, ev):
|
||||
"""Is this new window the one an app we started was to open? Takes it off the list."""
|
||||
now = time.monotonic()
|
||||
self.launches = [la for la in self.launches if la.until > now]
|
||||
if not ev.get("normal") or ev.get("popup") or ev.get("transient"):
|
||||
return None
|
||||
chain = pid_chain(int(ev.get("pid") or 0))
|
||||
# A window in a sandbox (Flatpak, Snap) gives its pid in there, and a single-instance app
|
||||
# opens it from a process that was already running: then only its desktop file can match.
|
||||
name = desktop_name(ev.get("app", ""), ev.get("cls", "")) if any(la.app for la in self.launches) else ""
|
||||
for la in self.launches:
|
||||
if any(p in chain for p in la.pids) or (la.app and name == la.app):
|
||||
return self.took(la)
|
||||
# Last: a window no desktop file claims, while exactly one app launch waits and none of
|
||||
# its processes made it (an app whose window names itself after nothing we can find).
|
||||
waiting = [la for la in self.launches if la.app]
|
||||
if not name and len(waiting) == 1 and not any(p in chain for la in self.launches for p in la.pids):
|
||||
log(f"{ev.get('id', '')[:9]} ({ev.get('cls', '') or '?'}): no desktop file claims it; "
|
||||
f"taken as {waiting[0].app}'s, the one launch waiting")
|
||||
return self.took(waiting[0])
|
||||
return None
|
||||
|
||||
def took(self, la):
|
||||
if not la.entries or len(la.entries) <= 1:
|
||||
self.launches.remove(la)
|
||||
return la
|
||||
|
||||
def concealed(self):
|
||||
"""The screens (1-based) hidden on their own (ft-layout hide N)."""
|
||||
reply = self.screens.ask("concealed", quiet=True)
|
||||
return {int(w) for w in reply.split()[1:] if w.isdigit()} if reply.startswith("ok") else set()
|
||||
|
||||
def on_hidden_screen(self, ev):
|
||||
"""A new top-level window on a screen that's hidden on its own."""
|
||||
m = re.match(r"WL-(\d+)$", ev.get("output", ""))
|
||||
if not m or int(m.group(1)) >= self.screens_n:
|
||||
return False
|
||||
if not ev.get("normal") or ev.get("popup") or ev.get("transient") or ev.get("cls") == "ksplashqml":
|
||||
return False
|
||||
return int(m.group(1)) + 1 in self.concealed()
|
||||
|
||||
def float_launched(self, ev, launch):
|
||||
"""Float a launched app's window (or one that opened on a hidden screen) where that app
|
||||
last floated (its size too), or in front of you, at the primary screen's density."""
|
||||
app = (launch.app if launch and launch.app else "") or desktop_name(ev.get("app", ""), ev.get("cls", "")) or ev.get("app", "")
|
||||
known = self.places.get(app) if app else None
|
||||
ref = self.reference()
|
||||
place = to_world(ref, known["rel"]) if known and ref and len(known.get("rel", [])) == 12 else self.in_front()
|
||||
mpp = (known or {}).get("mpp") or self.screen_mpp(f"WL-{self.primary()}")
|
||||
scale = None
|
||||
if known and known.get("pixels"):
|
||||
scale = known.get("scale") or output_scales().get(ev["output"], 1.0)
|
||||
ev = dict(ev, frame=dict(ev["frame"], w=known["pixels"][0] / scale, h=known["pixels"][1] / scale))
|
||||
self.float_window(dict(ev, app=app), place=place, mpp=mpp, scale=scale)
|
||||
|
||||
# ------------------------------------------------------------ profiles (docs/profiles.md)
|
||||
|
||||
def capture(self, sender, what="windows"):
|
||||
""""windows" and "spotlist": have the script report every window as it is now, and
|
||||
answer when the report is in (or after 3 seconds with what we know)."""
|
||||
if not sender:
|
||||
return f"error {what} needs a reply address"
|
||||
token = self.next_token
|
||||
self.next_token += 1
|
||||
self.captures[token] = (sender, what)
|
||||
self.command(cmd="report-all", token=token)
|
||||
GLib.timeout_add(3000, lambda: self.captured(token) and False)
|
||||
return None # answered by captured
|
||||
|
||||
def captured(self, token):
|
||||
sender, what = self.captures.pop(token, (None, None))
|
||||
if sender and self.sock:
|
||||
entries = self.spot_entries() if what == "spotlist" else self.window_entries()
|
||||
try:
|
||||
self.sock.sendto(("ok " + json.dumps(entries)).encode(), sender)
|
||||
except OSError as e:
|
||||
log(f"{what}: {e}")
|
||||
|
||||
def spot_entries(self):
|
||||
"""Every app window for spotwatch's search: its desktop file name, title, the panel
|
||||
it's on (a screen or a floating window's; None elsewhere), and when it was last active."""
|
||||
out = []
|
||||
for wid, ev in self.windows.items():
|
||||
if not listable(ev):
|
||||
continue
|
||||
f = self.floats.get(wid)
|
||||
m = re.match(r"WL-(\d+)$", ev.get("output", ""))
|
||||
panel = f.slot.index if f else (int(m.group(1)) + 1 if m and int(m.group(1)) < self.screens_n else None)
|
||||
out.append({"id": wid, "app": desktop_name(ev.get("app", ""), ev.get("cls", "")),
|
||||
"class": ev.get("cls", ""), "pid": ev.get("pid") or 0, "title": ev.get("caption", ""),
|
||||
"panel": panel, "floating": bool(f), "minimized": bool(ev.get("minimized")),
|
||||
"activeAt": ev.get("activeAt") or 0})
|
||||
return out
|
||||
|
||||
def window_entries(self):
|
||||
"""Every app window and where it is, as a profile keeps it."""
|
||||
ref = self.reference()
|
||||
rects = None
|
||||
out = []
|
||||
for wid, ev in self.windows.items():
|
||||
if not recordable(ev):
|
||||
continue
|
||||
app = desktop_name(ev.get("app", ""), ev.get("cls", ""))
|
||||
entry = {"app": app} if app else {"cmd": cmdline(ev.get("pid")), "class": ev.get("cls", "")}
|
||||
if not entry.get("app") and not entry.get("cmd"):
|
||||
continue
|
||||
f = self.floats.get(wid)
|
||||
if f:
|
||||
g = self.panel_get(f.slot.index)
|
||||
if not g or not ref or not f.frame:
|
||||
continue
|
||||
pixels = list(f.normal or (round(f.frame["w"] * f.scale), round(f.frame["h"] * f.scale)))
|
||||
entry["float"] = {"rel": to_local(ref, g), "pixels": pixels, "scale": f.scale, "mpp": round(f.mpp, 8)}
|
||||
else:
|
||||
m = re.match(r"WL-(\d+)$", ev.get("output", ""))
|
||||
if not m or int(m.group(1)) >= self.screens_n:
|
||||
continue
|
||||
fr, o = ev["frame"], ev.get("outputRect")
|
||||
if not o:
|
||||
rects = rects if rects is not None else output_rects()
|
||||
x0, y0, _, _ = rects.get(ev["output"], (0, 0, 0, 0))
|
||||
o = {"x": x0, "y": y0}
|
||||
entry["screen"] = int(m.group(1)) + 1
|
||||
entry["rect"] = [round(fr["x"] - o["x"]), round(fr["y"] - o["y"]), round(fr["w"]), round(fr["h"])]
|
||||
if ev.get("maximized"):
|
||||
entry["maximized"] = True
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
def window_key(self, ev):
|
||||
return desktop_name(ev.get("app", ""), ev.get("cls", "")) or json.dumps(cmdline(ev.get("pid")))
|
||||
|
||||
def open_profile(self, name):
|
||||
"""Open a profile's apps (additive: nothing closes)."""
|
||||
try:
|
||||
profile = ft_layout.load_layout().get("profiles", {}).get(name)
|
||||
except (OSError, ValueError):
|
||||
profile = None
|
||||
if profile is None:
|
||||
return f"error no profile {name!r}"
|
||||
groups = {}
|
||||
for e in profile.get("windows", []):
|
||||
key = e.get("app") or json.dumps(e.get("cmd") or [])
|
||||
if key != "[]":
|
||||
groups.setdefault(key, []).append(e)
|
||||
claimed = set()
|
||||
keys = {wid: self.window_key(ev) for wid, ev in self.windows.items() if recordable(ev)}
|
||||
for key, entries in groups.items():
|
||||
have = [ev for wid, ev in self.windows.items() if wid not in claimed and keys.get(wid) == key]
|
||||
for e, ev in zip(entries, have):
|
||||
claimed.add(ev["id"])
|
||||
self.place_entry(ev, e)
|
||||
rest = entries[len(have):]
|
||||
if rest:
|
||||
self.launch_entries(rest)
|
||||
log(f"profile {name!r}: {sum(len(v) for v in groups.values())} windows, {len(claimed)} already open")
|
||||
return "ok"
|
||||
|
||||
def launch_entries(self, entries):
|
||||
"""Launch an app for a profile's entries that have no window yet."""
|
||||
e = entries[0]
|
||||
if e.get("app"):
|
||||
reply, pid = self.launch(app=e["app"])
|
||||
app = e["app"]
|
||||
else:
|
||||
reply, pid = self.launch(argv=e["cmd"])
|
||||
app = ""
|
||||
if not reply.startswith("ok"):
|
||||
log(f"profile: {reply}")
|
||||
return
|
||||
la = self.launches[-1] # the one launch() just added
|
||||
la.entries, la.argv = list(entries), (None if app else e["cmd"])
|
||||
|
||||
def relaunch(self, la):
|
||||
"""3 seconds after a profile's app showed its first window: start it again for each
|
||||
entry still waiting (an app that restores its own windows has shown them by now)."""
|
||||
if la.relaunched or not la.entries or la not in self.launches:
|
||||
return
|
||||
la.relaunched = True
|
||||
la.until = time.monotonic() + LAUNCH_SECONDS
|
||||
for _ in la.entries:
|
||||
if la.app:
|
||||
reply, pid = self.launch(app=la.app)
|
||||
else:
|
||||
reply, pid = self.launch(argv=la.argv)
|
||||
if reply.startswith("ok"):
|
||||
self.launches.pop() # launch() added one of its own; this one waits for them all
|
||||
if pid:
|
||||
la.pids.append(pid)
|
||||
|
||||
def place_entry(self, ev, e):
|
||||
"""Move a window to a profile entry's place: on a screen, or floating."""
|
||||
f = self.floats.get(ev["id"])
|
||||
if "float" in e:
|
||||
fl, ref = e["float"], self.reference()
|
||||
if not ref or len(fl.get("rel", [])) != 12:
|
||||
return
|
||||
place = to_world(ref, fl["rel"])
|
||||
pixels, scale = fl.get("pixels"), fl.get("scale")
|
||||
if f:
|
||||
self.pose(f, *place)
|
||||
if scale:
|
||||
self.set_scale(f, scale)
|
||||
if pixels and f.frame:
|
||||
m = self.margin
|
||||
self.command(cmd="geometry", id=f.id, x=f.slot.pos[0] + m / f.scale,
|
||||
y=f.slot.pos[1] + m / f.scale, w=pixels[0] / f.scale, h=pixels[1] / f.scale)
|
||||
return
|
||||
if pixels:
|
||||
scale = scale or output_scales().get(ev["output"], 1.0)
|
||||
ev = dict(ev, frame=dict(ev["frame"], w=pixels[0] / scale, h=pixels[1] / scale))
|
||||
else:
|
||||
scale = None
|
||||
self.float_window(ev, place=place, mpp=fl.get("mpp"), scale=scale)
|
||||
return
|
||||
n = int(e.get("screen", 1)) - 1
|
||||
if not 0 <= n < self.screens_n:
|
||||
n = 0
|
||||
name = f"WL-{n}"
|
||||
x0, y0, ow, oh = output_rects().get(name, (0, 0, 0, 0))
|
||||
rx, ry, rw, rh = (e.get("rect") or [100, 100, ev["frame"]["w"], ev["frame"]["h"]])[:4]
|
||||
if ow and oh: # keep it on the screen if the screen got smaller
|
||||
rw, rh = min(rw, ow), min(rh, oh)
|
||||
rx, ry = max(0, min(rx, ow - rw)), max(0, min(ry, oh - rh))
|
||||
frame = {"x": x0 + rx, "y": y0 + ry, "w": rw, "h": rh}
|
||||
if f:
|
||||
self.dock(f, frame=frame, output=name, maximized=bool(e.get("maximized")))
|
||||
else:
|
||||
self.command(cmd="place", id=ev["id"], output=name, x=frame["x"], y=frame["y"], w=rw, h=rh,
|
||||
maximized=bool(e.get("maximized")))
|
||||
|
||||
# ------------------------------------------------------------ popups and dialogs
|
||||
|
||||
def sub(self, ev):
|
||||
@@ -1125,37 +507,11 @@ class Daemon:
|
||||
return s.window
|
||||
return None
|
||||
|
||||
def request(self, text, sender=None):
|
||||
def request(self, text):
|
||||
words = text.split()
|
||||
if not words:
|
||||
return "error empty"
|
||||
cmd, rest = words[0], words[1:]
|
||||
if cmd == "launch" and rest:
|
||||
opts = dict(w.split("=", 1) for w in rest[1:] if "=" in w)
|
||||
if len(opts) != len(rest) - 1 or set(opts) - {"action", "screen"}:
|
||||
return "error launch APP.desktop [action=ACTION] [screen=N]"
|
||||
screen = opts.get("screen")
|
||||
if screen is not None and not (screen.isdigit() and 1 <= int(screen) <= self.screens_n):
|
||||
return f"error no screen {screen}"
|
||||
return self.launch(app=rest[0], action=opts.get("action"),
|
||||
dock=f"WL-{int(screen) - 1}" if screen else None)[0]
|
||||
if cmd == "run" and rest:
|
||||
try:
|
||||
argv = json.loads(text.split(None, 1)[1])
|
||||
except ValueError:
|
||||
return "error run takes a JSON list"
|
||||
if not isinstance(argv, list) or not argv or not all(isinstance(a, str) for a in argv):
|
||||
return "error run takes a JSON list"
|
||||
return self.launch(argv=argv)[0]
|
||||
if cmd in ("windows", "spotlist"):
|
||||
return self.capture(sender, cmd)
|
||||
if cmd == "activate" and len(rest) == 1:
|
||||
if rest[0] not in self.windows:
|
||||
return f"error no window {rest[0]}"
|
||||
self.command(cmd="activate", id=rest[0])
|
||||
return "ok"
|
||||
if cmd == "profile" and rest:
|
||||
return self.open_profile(text.split(None, 1)[1])
|
||||
if cmd == "list":
|
||||
return "ok " + " ".join(f"{s.output}:{s.window.id if s.window else '-'}" for s in self.slots)
|
||||
if cmd == "quit":
|
||||
@@ -1164,20 +520,6 @@ class Daemon:
|
||||
if cmd in ("float", "dock") and (not rest or rest[0] == "active"):
|
||||
self.command(cmd="request-active")
|
||||
return "ok"
|
||||
if cmd == "float" and rest == ["pointer"]:
|
||||
self.command(cmd="request-pointer")
|
||||
return "ok"
|
||||
if cmd == "dock" and rest == ["all"]:
|
||||
for f in list(self.floats.values()):
|
||||
self.dock(f)
|
||||
return "ok"
|
||||
if cmd == "front" and len(rest) == 1 and rest[0].isdigit():
|
||||
f = self.by_panel(int(rest[0]))
|
||||
if f:
|
||||
self.command(cmd="activate", id=f.id)
|
||||
else: # a screen: its output is WL-<N - 1>
|
||||
self.command(cmd="activate-output", output=f"WL-{int(rest[0]) - 1}")
|
||||
return "ok"
|
||||
if cmd in ("dock", "close", "resize", "scale") and rest and rest[0].isdigit():
|
||||
f = self.by_panel(int(rest[0]))
|
||||
if not f:
|
||||
@@ -1197,8 +539,7 @@ class Daemon:
|
||||
if not ev:
|
||||
return f"error no window {rest[0]}"
|
||||
if cmd == "float":
|
||||
# The script's word for where it is now: a window on a screen isn't followed here.
|
||||
self.command(cmd="request-float", id=rest[0])
|
||||
self.float_window(ev)
|
||||
elif cmd == "dock" and rest[0] in self.floats:
|
||||
self.dock(self.floats[rest[0]])
|
||||
elif cmd == "close":
|
||||
@@ -1234,39 +575,11 @@ class Service(dbus.service.Object):
|
||||
self.daemon.wait(reply)
|
||||
|
||||
|
||||
def watch_apps(daemon):
|
||||
"""Keep Launch as Standalone's desktop file copies up to date (float/ft_apps.py), in a
|
||||
session that reads them (the session script puts them in XDG_DATA_DIRS)."""
|
||||
ours = os.path.realpath(ft_apps.OUT_ROOT)
|
||||
if ours not in (os.path.realpath(d) for d in os.environ.get("XDG_DATA_DIRS", "").split(":") if d):
|
||||
return
|
||||
later = []
|
||||
|
||||
def write():
|
||||
later.clear()
|
||||
try:
|
||||
log(f"Launch as Standalone: {ft_apps.write_all()} apps")
|
||||
except OSError as e:
|
||||
log(f"Launch as Standalone: {e}")
|
||||
return False
|
||||
|
||||
def changed(*_):
|
||||
if not later: # wait for an install or update to settle
|
||||
later.append(GLib.timeout_add_seconds(3, write))
|
||||
write()
|
||||
daemon.app_monitors = []
|
||||
for d in ft_apps.app_dirs():
|
||||
if os.path.isdir(d):
|
||||
m = Gio.File.new_for_path(d).monitor_directory(Gio.FileMonitorFlags.NONE, None)
|
||||
m.connect("changed", changed)
|
||||
daemon.app_monitors.append(m)
|
||||
|
||||
|
||||
def main():
|
||||
conf = ft_layout.read_conf()
|
||||
p = argparse.ArgumentParser(description="Floating windows for the Frametop desktop")
|
||||
p.add_argument("--screens", type=int, default=int(os.environ.get("FT_SCREEN_COUNT") or 0))
|
||||
p.add_argument("--slots", type=int, default=int(conf.get("FLOAT_SLOTS") or 16))
|
||||
p.add_argument("--slots", type=int, default=int(conf.get("FLOAT_SLOTS") or 8))
|
||||
p.add_argument("--margin", type=int, default=int(conf.get("FLOAT_MARGIN") or 300))
|
||||
p.add_argument("--control", default="ft_screens")
|
||||
p.add_argument("--socket", default="frametop_float")
|
||||
@@ -1285,7 +598,6 @@ def main():
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
sock.bind("\0" + args.socket)
|
||||
sock.setblocking(False)
|
||||
daemon.sock = sock
|
||||
|
||||
def readable(*_):
|
||||
while True:
|
||||
@@ -1293,8 +605,8 @@ def main():
|
||||
data, sender = sock.recvfrom(4096)
|
||||
except BlockingIOError:
|
||||
return True
|
||||
reply = daemon.request(data.decode(errors="replace").strip(), sender)
|
||||
if sender and reply is not None:
|
||||
reply = daemon.request(data.decode(errors="replace").strip())
|
||||
if sender:
|
||||
try:
|
||||
sock.sendto(reply.encode(), sender)
|
||||
except OSError:
|
||||
@@ -1303,7 +615,6 @@ def main():
|
||||
|
||||
log(f"{args.screens} screens, {args.slots} floating slots (WL-{args.screens} and up), margin {args.margin} px")
|
||||
daemon.load_script(bus)
|
||||
watch_apps(daemon)
|
||||
daemon.loop.run()
|
||||
|
||||
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# Install with FrameDrop (proof of concept)
|
||||
|
||||
[FrameDrop](https://framedropvr.com) is a Windows app that sideloads onto a Steam Frame: it copies a build to the headset and adds it to the Steam library. Issue #25 asks for an "Install with FrameDrop" button. Frametop isn't an app FrameDrop can copy over as is: it installs user services, a SteamVR driver, and a container, and four optional parts need sudo. So FrameDrop installs a small installer instead. Playing "Frametop" from the library opens a window that asks what to install, and your password for the parts that need it, then installs and shows its progress. A release's Frametop.zip (about 1.1 GB, built by CI on a tag, see [pack/README.md](../pack/README.md), Releases) carries Frametop built, as an image, and installs it with `install-release.sh`: nothing compiles on the headset and nothing else downloads. The same zip works unpacked on the headset. A test zip (a few KB) clones Frametop with `get.sh` instead.
|
||||
|
||||
Nothing here is published yet: no release, no button.
|
||||
|
||||
## How FrameDrop installs a Linux zip
|
||||
|
||||
It uses Valve's SteamOS Devkit path: pair once with the headset's devkit service, then rsync the unpacked zip into `~/devkit-game/<name>` over SSH, and register it with Steam as a Devkit Game with a start command. `devkit.sh` here makes the same calls with Valve's devkit-utils, so all of this can be tested on the Frame without a PC.
|
||||
|
||||
## What the probe found (SteamOS 0.3.0, build 20260922.6101926)
|
||||
|
||||
`probe/probe.sh`, started as a Devkit Game, recorded:
|
||||
|
||||
- Devkit titles run on the host, not in a container, as user `steamos`, from a process tree that Steam's reaper owns. This held even with the compat tool set to `SteamLinuxRuntime_4-arm64`: Steam recorded the mapping and still ran it on the host.
|
||||
- On the host, everything the installer needs works: git, curl to GitHub, podman (sees the `dev` container), `systemctl --user`, `systemd-run --user`, and GTK 4 with libadwaita.
|
||||
- A GTK window opens in gamescope (an X11 window on `:1`, drawn through gamescope's Vulkan WSI).
|
||||
- Steam puts its overlay in `LD_PRELOAD` and Steam runtime paths in `LD_LIBRARY_PATH` and `PATH`. Every host tool prints a preload error unless they're cleared.
|
||||
- Inside the Steam Linux Runtime 4 container (started by hand with its `run` script), there's no git, podman, systemctl, or GTK, but `flatpak-spawn --host` runs commands on the host.
|
||||
- Steam refuses Devkit Game names with a `-` ("missing/invalid arguments").
|
||||
- Steam doesn't make the start command executable: without the exec bit, the title exits in a second and nothing runs.
|
||||
|
||||
## The installer
|
||||
|
||||
`installer/frametop-install.sh` is the start command.
|
||||
|
||||
1. In the container, it starts itself again on the host with `flatpak-spawn --host`.
|
||||
2. It clears Steam's preload and library paths, and opens `installer/progress.py` (GTK 4 and libadwaita).
|
||||
3. The window asks which optional parts to install: voice dictation (on by default), hand tracking (on by default), the Hand Recorder (on by default; a release's zip only, since a clone builds it with `hands/rec/install.sh`), our own eye tracker (on by default), and the Bluetooth fixes. All but voice dictation need sudo, so it asks for your SteamOS password and checks it with `sudo -v`. If your user has no password (SteamOS starts without one), it says how to set one and leaves those out.
|
||||
4. It runs the zip's `install-release.sh --yes` (a test zip: `get.sh --yes`) in a transient user service, `frametop-framedrop-install`. The service is used because Steam ends the title's whole process tree when it's quit, and starts it with an OOM score of 900. Opened from a VR desktop (the zip unpacked by hand), it still uses the user's real bus and runtime folder for the service.
|
||||
5. It follows the service's log, shows the steps as a progress bar, and reports the result. Closing it leaves the install running. Playing the title again reattaches.
|
||||
|
||||
`--yes` keeps the version that's installed, or installs stable, and skips the SteamVR restart. The window says to restart SteamVR.
|
||||
|
||||
### The password
|
||||
|
||||
FrameDrop has no way to pass a password along, and the zip is the same file for everyone, so the password is typed on the headset, in the window. In VR that means the window's own keypad (shown when Steam starts the window; its Keypad button shows or hides it), whose keys you click with the controller's laser. SteamVR's keyboard doesn't come up for the field by itself, and when it's opened (`steam://open/keyboard`) its keys don't reach the window (tested with frame-testbench, 2026-10-07: the field stayed empty, while laser clicks on the window's checkboxes worked). A Bluetooth keyboard types as usual. The password stays in the window's memory until the install ends:
|
||||
|
||||
- The service gets `SUDO_ASKPASS=installer/askpass`. When install.sh's sudo asks, askpass connects to a socket the window keeps in `/run/user/UID/frametop-install` (mode 0700), and the window answers only a process in the install's own service (checked by its peer credentials and cgroup). If it doesn't have the password yet (the window was opened again), it asks you for it, or you skip that part.
|
||||
- The password is never written to a file, a log, the service's environment, or a command line. The window wipes its copy when the install ends or the window closes. Strings Python and GTK made from it along the way can't be wiped; they go with the process.
|
||||
- With the window closed there's nobody to answer: sudo fails, install.sh says which parts it skipped, and playing Frametop again finishes them.
|
||||
|
||||
`--dry-run` unpacks into `~/.cache/frametop-framedrop/dry-run` and stops there without installing (`install-release.sh --unpack-only`, or `get.sh --clone-only`).
|
||||
|
||||
## Try it on the Frame
|
||||
|
||||
```
|
||||
framedrop/build.sh # a test zip; or --image localhost/frametop:local --version 0.3.0-dev.1
|
||||
unzip -q framedrop/build/Frametop.zip -d /tmp/fd
|
||||
framedrop/devkit.sh add FrametopTest /tmp/fd/Frametop "./frametop-install.sh --dry-run"
|
||||
framedrop/devkit.sh run FrametopTest # or Play it from the Steam library
|
||||
framedrop/devkit.sh remove FrametopTest
|
||||
framedrop/devkit.sh add FrametopProbe framedrop/probe "./probe.sh native" # the probe
|
||||
```
|
||||
|
||||
The probe writes `~/.cache/frametop-framedrop/probe-native.log`, and the installer writes `~/.cache/frametop-framedrop/install.log`.
|
||||
|
||||
## Build the download
|
||||
|
||||
```
|
||||
framedrop/build.sh --image REF --version V [--commit SHA] [--channel C] [ZIP_URL] # a release
|
||||
framedrop/build.sh [ZIP_URL] # a test zip
|
||||
```
|
||||
|
||||
This writes `framedrop/build/Frametop.zip` (reproducible), `frametop.framedrop.json` (FrameDrop's manifest with the zip's sha256), and `SHA256SUMS`. A release's zip has the image REF (`podman save`) with its `frametop-release.json` (`pack/release-info.py`) and `install-release.sh`; CI builds it on a tag (`.github/workflows/release.yml`). A test zip has `get.sh` instead. By default, `ZIP_URL` is the release's asset (`releases/download/vV/Frametop.zip`; for a test zip, a `framedrop-installer` release's). Each release carries its manifest, so the button's link can point at the newest stable one: `https://framedropvr.com/install?manifest=https://github.com/Frametop/frametop/releases/latest/download/frametop.framedrop.json` (the exact URL format is FrameDrop's to confirm).
|
||||
|
||||
## Open questions, for a test with FrameDrop on a Windows PC
|
||||
|
||||
- Does FrameDrop keep or set the exec bit on `frametop-install.sh`? A zip unpacked on Windows loses it, and without it nothing runs.
|
||||
- What start command does FrameDrop pick for this zip, and which runtime?
|
||||
- Does the manifest's `name` become the Devkit Game name? It has to stay free of `-`.
|
||||
- Typing the password with the window's keypad, launched with Play from the library (frame-testbench reached the window only when started with devkit.sh run, where the systemui overlay hides its lower half).
|
||||
@@ -1,83 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build Frametop's download: framedrop/build/Frametop.zip, a "Frametop" folder that FrameDrop
|
||||
# installs from a PC, or that you unpack on the headset and run (frametop-install.sh). Also
|
||||
# framedrop/build/frametop.framedrop.json, the manifest an "Install with FrameDrop" button
|
||||
# points at, and SHA256SUMS. Same files in, same zip out.
|
||||
#
|
||||
# Usage: framedrop/build.sh --image REF --version V [--commit SHA] [--channel C] [ZIP_URL]
|
||||
# framedrop/build.sh [ZIP_URL]
|
||||
# --image REF a release: the image REF (built from this checkout, pack/Containerfile) goes
|
||||
# in the zip as frametop-image.tar with frametop-release.json, and the
|
||||
# installer installs it, built (pack/install-release.sh). About 1.1 GB.
|
||||
# --version V the release's version (0.3.0, or 0.3.0-exp.1 for an experimental one)
|
||||
# --commit SHA the commit it was built from (default: this checkout's HEAD)
|
||||
# --channel C stable or experimental (default: experimental if V has a "-")
|
||||
# Without --image, a few KB: the installer clones Frametop from GitHub (get.sh), for
|
||||
# testing the installer itself.
|
||||
# ZIP_URL where the zip will be downloaded from (default: the release's asset, or for
|
||||
# a test zip, the framedrop-installer release)
|
||||
set -euo pipefail
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
repo=$(cd "$here/.." && pwd)
|
||||
image= version= commit= channel=
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--image) image=${2:?--image needs an image}; shift ;;
|
||||
--version) version=${2:?--version needs a version}; shift ;;
|
||||
--commit) commit=${2:?--commit needs a commit}; shift ;;
|
||||
--channel) channel=${2:?--channel needs stable or experimental}; shift ;;
|
||||
-*) echo "unknown option: $1" >&2; exit 2 ;;
|
||||
*) break ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
out=$here/build
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
|
||||
files=("$here/installer/frametop-install.sh" "$here/installer/progress.py" "$here/installer/askpass")
|
||||
if [ -n "$image" ]; then
|
||||
[ -n "$version" ] || { echo "--image needs --version" >&2; exit 2; }
|
||||
commit=${commit:-$(git -C "$repo" rev-parse HEAD)}
|
||||
url=${1:-https://github.com/Frametop/frametop/releases/download/v$version/Frametop.zip}
|
||||
echo "saving $image"
|
||||
podman save -q --format oci-archive -o "$out/frametop-image.tar" "$image"
|
||||
python3 "$repo/pack/release-info.py" --image-file "$out/frametop-image.tar" --version "$version" \
|
||||
--commit "$commit" ${channel:+--channel "$channel"} >"$out/frametop-release.json"
|
||||
files+=("$repo/pack/install-release.sh" "$out/frametop-release.json" "$out/frametop-image.tar")
|
||||
else
|
||||
url=${1:-https://github.com/Frametop/frametop/releases/download/framedrop-installer/Frametop.zip}
|
||||
files+=("$repo/get.sh")
|
||||
fi
|
||||
|
||||
python3 - "$out/Frametop.zip" "${files[@]}" <<'EOF'
|
||||
import os, shutil, sys, zipfile
|
||||
dest, *files = sys.argv[1:]
|
||||
with zipfile.ZipFile(dest, "w", allowZip64=True) as z:
|
||||
for path in files:
|
||||
name = os.path.basename(path)
|
||||
info = zipfile.ZipInfo(f"Frametop/{name}", date_time=(2026, 1, 1, 0, 0, 0))
|
||||
info.create_system = 3 # unix, so the permissions below count
|
||||
info.external_attr = (0o100755 if os.access(path, os.X_OK) else 0o100644) << 16
|
||||
# The image's layers are compressed already.
|
||||
info.compress_type = zipfile.ZIP_STORED if name.endswith(".tar") else zipfile.ZIP_DEFLATED
|
||||
info.file_size = os.path.getsize(path)
|
||||
with open(path, "rb") as src, z.open(info, "w", force_zip64=True) as dst:
|
||||
shutil.copyfileobj(src, dst, 1 << 20)
|
||||
EOF
|
||||
rm -f "$out/frametop-image.tar"
|
||||
|
||||
sha=$(sha256sum "$out/Frametop.zip" | cut -d' ' -f1)
|
||||
python3 - "$url" "$sha" >"$out/frametop.framedrop.json" <<'EOF'
|
||||
import json, sys
|
||||
url, sha = sys.argv[1:]
|
||||
print(json.dumps({
|
||||
"schema": "framedrop.install/v1",
|
||||
"name": "Frametop",
|
||||
"files": [{"url": url, "sha256": sha}],
|
||||
}, indent=2))
|
||||
EOF
|
||||
(cd "$out" && sha256sum Frametop.zip frametop.framedrop.json ${image:+frametop-release.json} >SHA256SUMS)
|
||||
echo "built $out/Frametop.zip ($(du -h "$out/Frametop.zip" | cut -f1), sha256 $sha)"
|
||||
echo " $out/frametop.framedrop.json, $out/SHA256SUMS"
|
||||
@@ -1,95 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Do on the Frame what FrameDrop does from a PC: copy a folder into ~/devkit-game/NAME and
|
||||
# register it with Steam as a "Devkit Game" (Valve's devkit-utils, the same calls FrameDrop and
|
||||
# the SteamOS Devkit Client make over SSH). For testing the FrameDrop installer and the probe
|
||||
# without a PC. Steam must be running, and Developer Mode on.
|
||||
#
|
||||
# Usage: devkit.sh add NAME DIR COMMAND [--compat TOOL]
|
||||
# devkit.sh run NAME # start it, as the library's Play button does
|
||||
# devkit.sh remove NAME # delete the folder and the Steam entry
|
||||
# devkit.sh list
|
||||
#
|
||||
# NAME can't contain "-": Steam answers "missing/invalid arguments". COMMAND is relative to
|
||||
# the folder, like FrameDrop's start command ("./probe.sh native"). --compat sets the runtime
|
||||
# (SteamLinuxRuntime_4-arm64, say); on SteamOS 0.3.0 Steam recorded it but still ran the
|
||||
# title on the host. There's no environment option: this Steam ignores the devkit env file.
|
||||
#
|
||||
# devkit-utils is Valve's (LGPL, gitlab.steamos.cloud/devkit/steamos-devkit). FrameDrop and
|
||||
# the devkit client copy it to ~/devkit-utils; if that isn't there, it's fetched at a pinned
|
||||
# commit into ~/.cache/frametop-framedrop.
|
||||
set -euo pipefail
|
||||
|
||||
devkit_rev=a00ceb7d91ea44a0c3e714a91a06417d6e5cdb33
|
||||
cache=$HOME/.cache/frametop-framedrop
|
||||
|
||||
usage() { sed -n '7,10p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
|
||||
|
||||
utils() {
|
||||
if [ -e "$HOME/devkit-utils/steam-client-create-shortcut" ]; then
|
||||
echo "$HOME/devkit-utils"; return
|
||||
fi
|
||||
if [ ! -e "$cache/devkit-utils/steam-client-create-shortcut" ]; then
|
||||
echo "fetching devkit-utils ($devkit_rev)" >&2
|
||||
local tmp
|
||||
tmp=$(mktemp -d)
|
||||
git -C "$tmp" init -q
|
||||
git -C "$tmp" fetch -q --depth 1 https://gitlab.steamos.cloud/devkit/steamos-devkit.git "$devkit_rev"
|
||||
git -C "$tmp" checkout -q FETCH_HEAD
|
||||
mkdir -p "$cache"
|
||||
rm -rf "$cache/devkit-utils"
|
||||
cp -r "$tmp/client/devkit-utils" "$cache/devkit-utils"
|
||||
rm -rf "$tmp"
|
||||
fi
|
||||
echo "$cache/devkit-utils"
|
||||
}
|
||||
|
||||
[ $# -ge 1 ] || usage
|
||||
cmd=$1; shift
|
||||
case $cmd in
|
||||
add)
|
||||
[ $# -ge 3 ] || usage
|
||||
name=$1 src=$2 start=$3; shift 3
|
||||
case $name in *-*) echo "NAME can't contain '-' (Steam refuses it)" >&2; exit 2 ;; esac
|
||||
compat=
|
||||
while [ $# -gt 0 ]; do
|
||||
case $1 in
|
||||
--compat) compat=${2:?}; shift ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
u=$(utils)
|
||||
dir=$(python3 "$u/steamos-prepare-upload" --gameid "$name" | python3 -c 'import json,sys; print(json.load(sys.stdin)["directory"])')
|
||||
# FrameDrop rsyncs the unpacked zip; --delete as a clean upload would.
|
||||
rsync -a --delete "$src/" "$dir/"
|
||||
parms=$(python3 - "$name" "$dir" "$start" "$compat" <<'EOF'
|
||||
import json, sys
|
||||
name, directory, start, compat = sys.argv[1:]
|
||||
print(json.dumps({
|
||||
"gameid": name, "directory": directory,
|
||||
"argv": [start], "env": {},
|
||||
"settings": {"steam_play": "0", "compat_tool": compat},
|
||||
"force_appid": "", "lepton_args": "",
|
||||
}))
|
||||
EOF
|
||||
)
|
||||
res=$(python3 "$u/steam-client-create-shortcut" --parms "$parms" | tail -1)
|
||||
echo "Steam: $res"
|
||||
case $res in *'"error"'*) exit 1 ;; esac
|
||||
echo "registered $name ($dir)"
|
||||
;;
|
||||
run)
|
||||
[ $# -eq 1 ] || usage
|
||||
python3 "$(utils)/steam-devkit-rpc" run-game "gameid=$1" | tail -1
|
||||
echo
|
||||
;;
|
||||
remove)
|
||||
[ $# -eq 1 ] || usage
|
||||
python3 "$(utils)/steamos-delete" --delete-title "$1"
|
||||
rm -f "$HOME/devkit-game/$1"-*.json
|
||||
;;
|
||||
list)
|
||||
python3 "$(utils)/steam-devkit-rpc" list-shortcuts
|
||||
;;
|
||||
*) usage ;;
|
||||
esac
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/usr/bin/python3
|
||||
"""SUDO_ASKPASS for the FrameDrop install: sudo runs this for the password, and it asks the
|
||||
install window (progress.py) for it, over the window's socket in XDG_RUNTIME_DIR. The window
|
||||
answers only programs in the install's own service, and asks you on the headset if it doesn't
|
||||
have the password yet. The password goes to sudo on stdout and nowhere else.
|
||||
|
||||
With the window closed there's nobody to ask: this fails, so sudo does, and install.sh says
|
||||
which parts it skipped. Playing Frametop again reopens the window and the install.
|
||||
"""
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
path = os.environ.get("FRAMETOP_ASKPASS_SOCKET", f"/run/user/{os.getuid()}/frametop-install/askpass")
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.settimeout(600) # time for you to type it, if the window has to ask
|
||||
try:
|
||||
s.connect(path) # connecting is the question: the window answers, then closes
|
||||
answer = bytearray()
|
||||
while chunk := s.recv(4096):
|
||||
answer += chunk
|
||||
except OSError as e:
|
||||
print(f"askpass: the install window isn't answering ({e.strerror or e}): play Frametop again",
|
||||
file=sys.stderr)
|
||||
sys.exit(1)
|
||||
if not answer:
|
||||
sys.exit(1) # you cancelled
|
||||
sys.stdout.buffer.write(bytes(answer) + b"\n")
|
||||
answer[:] = bytes(len(answer))
|
||||
@@ -1,30 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Frametop's installer, in a release's Frametop.zip: the start command of the "Frametop" title
|
||||
# FrameDrop puts in the Steam library, or what you run after unpacking the zip on the headset.
|
||||
# It opens the install window (progress.py), which asks what to install, and your password
|
||||
# for the parts that need sudo, then installs Frametop (or updates it) in a service of its own
|
||||
# and shows its progress. A release's zip installs its own image, built
|
||||
# (install-release.sh); a test zip clones Frametop from GitHub (get.sh).
|
||||
#
|
||||
# Usage: frametop-install.sh [--dry-run]
|
||||
# --dry-run unpack into ~/.cache/frametop-framedrop/dry-run and stop there, without
|
||||
# installing, for testing this flow
|
||||
set -u
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
self=$here/$(basename "$0")
|
||||
|
||||
# A Linux title can be started in the Steam Linux Runtime container, which has no git, podman,
|
||||
# systemctl, or GTK. Start this again on the host. (SteamOS 0.3.0 runs devkit titles on the
|
||||
# host anyway; this is for when FrameDrop or Steam picks the runtime.)
|
||||
if [ -e /run/pressure-vessel ]; then
|
||||
exec flatpak-spawn --host --watch-bus --env=DISPLAY="${DISPLAY-}" \
|
||||
--env=XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR-}" --env=SteamAppId="${SteamAppId-}" \
|
||||
--env=ENABLE_GAMESCOPE_WSI="${ENABLE_GAMESCOPE_WSI-}" /usr/bin/bash "$self" "$@"
|
||||
fi
|
||||
|
||||
# Steam adds its overlay and runtime libraries to every child; host tools don't want them.
|
||||
unset LD_PRELOAD LD_LIBRARY_PATH
|
||||
export PATH=/usr/local/bin:/usr/bin:/bin
|
||||
|
||||
exec /usr/bin/python3 "$here/progress.py" "$here" "$@"
|
||||
@@ -1,563 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The FrameDrop installer's window: what to install, your password for the parts that need it,
|
||||
then the install's progress.
|
||||
|
||||
Usage: progress.py DIR [--dry-run]
|
||||
DIR the installer's folder. A release's Frametop.zip has install-release.sh, the
|
||||
image (frametop-image.tar), and frametop-release.json: it installs that, built.
|
||||
A test zip has get.sh instead, which clones Frametop from GitHub.
|
||||
--dry-run unpack into ~/.cache/frametop-framedrop/dry-run and stop there, without
|
||||
installing, for testing this flow
|
||||
|
||||
The install runs in a user service of its own (UNIT): Steam ends the title's whole process
|
||||
tree when it's quit, and starts it with a high OOM score. Closing the window doesn't stop the
|
||||
install; playing the title again reattaches to it.
|
||||
|
||||
Hand tracking, the Hand Recorder, our eye tracker and the Bluetooth fixes need sudo. The window asks for your password on the
|
||||
headset, checks it with sudo, and keeps it in this process's memory until the install ends.
|
||||
sudo in the install gets it through askpass (SUDO_ASKPASS), from a socket in a folder only you
|
||||
can open, and the window answers only programs in the install's service. It's never written to
|
||||
a file, a log, the service's environment, or a command line.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
import gi
|
||||
|
||||
gi.require_version("Gtk", "4.0")
|
||||
gi.require_version("Adw", "1")
|
||||
from gi.repository import Adw, GLib, Gtk, Pango # noqa: E402
|
||||
|
||||
HERE = Path(sys.argv[1]).resolve() if len(sys.argv) > 1 else Path(__file__).resolve().parent
|
||||
DRY_RUN = "--dry-run" in sys.argv[2:]
|
||||
UNIT = "frametop-framedrop-install"
|
||||
HOME = Path.home()
|
||||
STATE = HOME / ".cache" / "frametop-framedrop"
|
||||
LOG = STATE / "install.log"
|
||||
RELEASE = (HERE / "install-release.sh").exists() and (HERE / "frametop-image.tar").exists()
|
||||
SOCK_DIR = Path(f"/run/user/{os.getuid()}/frametop-install")
|
||||
SOCK = SOCK_DIR / "askpass"
|
||||
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
|
||||
# get.sh and install.sh mark each step with "== N/M what it is"
|
||||
STEP = re.compile(r"^== (\d+)/(\d+) (.*)$")
|
||||
DONE_TEXT = ("Frametop is installed. Restart SteamVR once, or reboot the headset, so it loads "
|
||||
"Frametop's driver. Then open Launch a program, then Desktop.")
|
||||
|
||||
|
||||
def host_env():
|
||||
"""The user's real runtime folder and bus, for systemctl and systemd-run: a window opened
|
||||
from a VR desktop's Dolphin or Konsole has that session's own. GTK keeps the session's."""
|
||||
env = dict(os.environ)
|
||||
env["XDG_RUNTIME_DIR"] = f"/run/user/{os.getuid()}"
|
||||
env["DBUS_SESSION_BUS_ADDRESS"] = f"unix:path=/run/user/{os.getuid()}/bus"
|
||||
return env
|
||||
|
||||
|
||||
def unit_state():
|
||||
out = subprocess.run(["systemctl", "--user", "show", "-P", "ActiveState", UNIT],
|
||||
capture_output=True, text=True, env=host_env()).stdout.strip()
|
||||
return out or "inactive"
|
||||
|
||||
|
||||
def release_version():
|
||||
try:
|
||||
return json.loads((HERE / "frametop-release.json").read_text())["version"]
|
||||
except (OSError, ValueError, KeyError, TypeError):
|
||||
return ""
|
||||
|
||||
|
||||
def password_set():
|
||||
"""Does this user have a password sudo can take? SteamOS starts without one."""
|
||||
user = pwd.getpwuid(os.getuid()).pw_name
|
||||
out = subprocess.run(["passwd", "-S", user], capture_output=True, text=True).stdout.split()
|
||||
return len(out) < 2 or out[1] == "P" # P: usable; NP: none; L: locked
|
||||
|
||||
|
||||
def check_password(pw):
|
||||
"""Does sudo take it? Checked with cached credentials ignored, and forgotten after."""
|
||||
try:
|
||||
r = subprocess.run(["sudo", "-S", "-k", "-v", "-p", ""], input=bytes(pw) + b"\n",
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=30)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return False
|
||||
subprocess.run(["sudo", "-k"], stdin=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
return r.returncode == 0
|
||||
|
||||
|
||||
def in_unit(pid):
|
||||
"""Is this process in the install's service?"""
|
||||
try:
|
||||
with open(f"/proc/{pid}/cgroup") as f:
|
||||
return any(line.rstrip("\n").endswith(f"/{UNIT}.service") for line in f)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def install_command(eye_tracker, bluetooth, voice=True, hands=True, hand_recorder=True):
|
||||
if RELEASE:
|
||||
args = [str(HERE / "install-release.sh"), "--yes"]
|
||||
if DRY_RUN:
|
||||
args += ["--unpack-only", "--dir", str(STATE / "dry-run")]
|
||||
else:
|
||||
args = [str(HERE / "get.sh"), "--yes"]
|
||||
if DRY_RUN:
|
||||
args += ["--clone-only", "--dir", str(STATE / "dry-run")]
|
||||
if not voice:
|
||||
args.append("--no-voice")
|
||||
if not hands:
|
||||
args.append("--no-hands")
|
||||
if RELEASE and not hand_recorder: # a clone (get.sh) doesn't offer it: hands/rec/install.sh builds it
|
||||
args.append("--no-hand-recorder")
|
||||
if not eye_tracker:
|
||||
args.append("--no-eye-tracker")
|
||||
if bluetooth:
|
||||
args.append("--bluetooth")
|
||||
return args
|
||||
|
||||
|
||||
def start_unit(args, askpass):
|
||||
env = host_env()
|
||||
subprocess.run(["systemctl", "--user", "stop", UNIT], stderr=subprocess.DEVNULL, env=env)
|
||||
subprocess.run(["systemctl", "--user", "reset-failed", UNIT], stderr=subprocess.DEVNULL, env=env)
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
LOG.write_bytes(b"")
|
||||
cmd = ["systemd-run", "--user", f"--unit={UNIT}", "--description=Frametop install (FrameDrop)",
|
||||
"--property=Type=oneshot", "--property=RemainAfterExit=yes",
|
||||
f"--property=StandardOutput=truncate:{LOG}", "--property=StandardError=inherit",
|
||||
f"--setenv=HOME={HOME}", f"--setenv=PATH={os.environ.get('PATH', '/usr/bin:/bin')}",
|
||||
"--setenv=TERM=dumb", f"--working-directory={HOME}", "--quiet", "--no-block"]
|
||||
if askpass:
|
||||
cmd += [f"--setenv=SUDO_ASKPASS={HERE / 'askpass'}", f"--setenv=FRAMETOP_ASKPASS_SOCKET={SOCK}"]
|
||||
cmd += ["/usr/bin/bash"] + args
|
||||
return subprocess.run(cmd, env=env).returncode == 0
|
||||
|
||||
|
||||
class Askpass:
|
||||
"""The socket askpass asks: answers programs in the install's service with the password,
|
||||
or holds them while the window asks you for it."""
|
||||
|
||||
def __init__(self, on_ask):
|
||||
self.on_ask = on_ask
|
||||
self.password = None # bytearray, while the install runs
|
||||
self.waiting = []
|
||||
SOCK_DIR.mkdir(mode=0o700, exist_ok=True)
|
||||
st = SOCK_DIR.stat()
|
||||
if st.st_uid != os.getuid():
|
||||
raise OSError(f"{SOCK_DIR} isn't yours")
|
||||
os.chmod(SOCK_DIR, 0o700)
|
||||
SOCK.unlink(missing_ok=True)
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
self.sock.bind(str(SOCK))
|
||||
self.sock.listen(4)
|
||||
self.sock.setblocking(False)
|
||||
self.watch = GLib.io_add_watch(GLib.IOChannel.unix_new(self.sock.fileno()), GLib.PRIORITY_DEFAULT,
|
||||
GLib.IO_IN, self.accept)
|
||||
|
||||
def accept(self, *_):
|
||||
try:
|
||||
conn, _ = self.sock.accept()
|
||||
except OSError:
|
||||
return True
|
||||
pid, uid, _ = struct.unpack("3i", conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED,
|
||||
struct.calcsize("3i")))
|
||||
if uid != os.getuid() or not in_unit(pid):
|
||||
conn.close()
|
||||
return True
|
||||
if self.password:
|
||||
self.answer(conn)
|
||||
else:
|
||||
self.waiting.append(conn)
|
||||
self.on_ask()
|
||||
return True
|
||||
|
||||
def answer(self, conn):
|
||||
try:
|
||||
conn.setblocking(True)
|
||||
conn.sendall(bytes(self.password) if self.password else b"")
|
||||
except OSError:
|
||||
pass
|
||||
conn.close()
|
||||
|
||||
def give(self, pw):
|
||||
self.password = pw
|
||||
while self.waiting:
|
||||
self.answer(self.waiting.pop())
|
||||
|
||||
def refuse(self):
|
||||
while self.waiting:
|
||||
self.waiting.pop().close()
|
||||
|
||||
def close(self):
|
||||
if self.password:
|
||||
self.password[:] = bytes(len(self.password))
|
||||
self.password = None
|
||||
self.refuse()
|
||||
GLib.source_remove(self.watch)
|
||||
self.sock.close()
|
||||
SOCK.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class Keypad(Gtk.Box):
|
||||
"""Keys to click with the controller's laser, for a password field: in VR, SteamVR's own
|
||||
keyboard comes up for a Steam title's window but its keys don't reach it (tested
|
||||
2026-10-07), while clicks on the window's buttons do. A physical keyboard types as usual."""
|
||||
|
||||
LETTERS = ("1234567890", "qwertyuiop", "asdfghjkl", "zxcvbnm")
|
||||
SYMBOLS = ("!@#$%^&*()", "-_=+[]{}\\|", ";:'\",.<>/?", "`~")
|
||||
|
||||
def __init__(self, entry):
|
||||
super().__init__(orientation=Gtk.Orientation.VERTICAL, spacing=4)
|
||||
self.entry = entry
|
||||
self.shift = self.symbols = False
|
||||
self.rows = []
|
||||
for _ in range(4):
|
||||
row = Gtk.Box(spacing=4, halign=Gtk.Align.CENTER, homogeneous=True)
|
||||
self.rows.append(row)
|
||||
self.append(row)
|
||||
bottom = Gtk.Box(spacing=4, halign=Gtk.Align.CENTER)
|
||||
for label, action in (("Shift", self.toggle_shift), ("!#1", self.toggle_symbols),
|
||||
("Space", lambda *_: self.type(" ")), ("Delete", self.backspace)):
|
||||
b = Gtk.Button(label=label)
|
||||
b.set_size_request(110 if label != "Space" else 260, 48)
|
||||
b.connect("clicked", action)
|
||||
bottom.append(b)
|
||||
if label == "!#1":
|
||||
self.symbols_key = b
|
||||
self.append(bottom)
|
||||
self.fill()
|
||||
|
||||
def fill(self):
|
||||
for row, keys in zip(self.rows, self.SYMBOLS if self.symbols else self.LETTERS):
|
||||
while (child := row.get_first_child()) is not None:
|
||||
row.remove(child)
|
||||
for k in keys:
|
||||
k = k.upper() if self.shift and not self.symbols else k
|
||||
b = Gtk.Button(label=k)
|
||||
b.set_size_request(56, 48)
|
||||
b.connect("clicked", lambda _b, k=k: self.type(k))
|
||||
row.append(b)
|
||||
|
||||
def type(self, text):
|
||||
self.entry.set_text(self.entry.get_text() + text)
|
||||
self.entry.set_position(-1)
|
||||
|
||||
def backspace(self, *_):
|
||||
self.entry.set_text(self.entry.get_text()[:-1])
|
||||
self.entry.set_position(-1)
|
||||
|
||||
def toggle_shift(self, *_):
|
||||
self.shift = not self.shift
|
||||
self.fill()
|
||||
|
||||
def toggle_symbols(self, *_):
|
||||
self.symbols = not self.symbols
|
||||
self.symbols_key.set_label("abc" if self.symbols else "!#1")
|
||||
self.fill()
|
||||
|
||||
|
||||
def keypad_row(entry, *extra):
|
||||
"""The password field, a button that shows or hides the keypad, and the keypad: shown by
|
||||
itself when Steam started this (FrameDrop's title, so most likely in VR)."""
|
||||
keypad = Keypad(entry)
|
||||
keypad.set_visible("SteamAppId" in os.environ)
|
||||
toggle = Gtk.Button(label="Keypad")
|
||||
toggle.connect("clicked", lambda *_: keypad.set_visible(not keypad.get_visible()))
|
||||
row = Gtk.Box(spacing=8)
|
||||
entry.set_hexpand(True)
|
||||
for w in (entry, toggle, *extra):
|
||||
row.append(w)
|
||||
return row, keypad
|
||||
|
||||
|
||||
class Window(Adw.ApplicationWindow):
|
||||
def __init__(self, app):
|
||||
super().__init__(application=app, title="Frametop")
|
||||
self.set_default_size(900, 860 if "SteamAppId" in os.environ else 640)
|
||||
self.offset = 0
|
||||
self.partial = ""
|
||||
self.askpass = None
|
||||
self.checking = False
|
||||
|
||||
self.status = Gtk.Label(label="Install Frametop", xalign=0, wrap=True)
|
||||
self.status.add_css_class("title-2")
|
||||
self.detail = Gtk.Label(xalign=0, wrap=True)
|
||||
self.box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12,
|
||||
margin_top=18, margin_bottom=18, margin_start=18, margin_end=18)
|
||||
self.box.append(self.status)
|
||||
self.box.append(self.detail)
|
||||
view = Adw.ToolbarView(content=self.box)
|
||||
view.add_top_bar(Adw.HeaderBar())
|
||||
self.set_content(view)
|
||||
self.connect("close-request", self.closing)
|
||||
|
||||
if unit_state() == "activating":
|
||||
self.show_progress() # playing the title again: the install is still going
|
||||
else:
|
||||
self.show_choices()
|
||||
|
||||
# --- what to install, and the password
|
||||
|
||||
def show_choices(self):
|
||||
what = f"Frametop {release_version()}" if RELEASE else "Frametop from GitHub"
|
||||
self.detail.set_label(f"This installs {what}: the multi-screen desktop, the 3D mouse, gaze "
|
||||
"mode, and Frametop's settings apps. Optional: voice dictation, and "
|
||||
"parts that need your SteamOS password (sudo):")
|
||||
self.voice = Gtk.CheckButton(label="Voice dictation: a Right Ctrl tap or holding the left d-pad down, and "
|
||||
"what you say is typed (downloads about 320 MB)", active=True)
|
||||
self.hands = Gtk.CheckButton(label="Hand tracking (experimental): poke, pinch and grab the screens with "
|
||||
"your hands. It uses a lot of the headset's CPU", active=True)
|
||||
# A release brings it built; a clone builds it with hands/rec/install.sh, so isn't offered it
|
||||
self.recorder = Gtk.CheckButton(label="Hand Recorder: record your hands for Frametop's open hand "
|
||||
"dataset, which trains hand tracking (nothing is uploaded "
|
||||
"until you press Upload)", active=True, visible=RELEASE)
|
||||
self.eye = Gtk.CheckButton(label="Our own eye tracker for gaze mode (more accurate than SteamVR's)",
|
||||
active=True)
|
||||
self.bt = Gtk.CheckButton(label="Bluetooth fixes (LE mice and keyboards, like the Swiftpoint Z3, "
|
||||
"reconnect after they sleep)")
|
||||
self.pw = Gtk.PasswordEntry(show_peek_icon=True, placeholder_text="Your SteamOS password")
|
||||
pw_note = Gtk.Label(label="It's used only for this install, and isn't saved anywhere.", xalign=0,
|
||||
wrap=True)
|
||||
pw_note.add_css_class("dim-label")
|
||||
self.error = Gtk.Label(xalign=0, wrap=True, visible=False)
|
||||
self.error.add_css_class("error")
|
||||
self.go = Gtk.Button(label="Install", halign=Gtk.Align.END)
|
||||
self.go.add_css_class("suggested-action")
|
||||
self.go.connect("clicked", self.install)
|
||||
self.pw.connect("activate", self.install)
|
||||
pw_row, keypad = keypad_row(self.pw)
|
||||
self.choices = [self.voice, self.hands, self.recorder, self.eye, self.bt, pw_row, keypad, pw_note,
|
||||
self.error, self.go]
|
||||
if not password_set():
|
||||
for c in (self.hands, self.recorder, self.eye, self.bt):
|
||||
c.set_active(False)
|
||||
c.set_sensitive(False)
|
||||
pw_row.set_visible(False)
|
||||
keypad.set_visible(False)
|
||||
pw_note.set_label("Your user has no password, so sudo can't run and these can't be "
|
||||
"installed from here. To set one, run passwd in Konsole; then play "
|
||||
"Frametop again, or install them later from a terminal "
|
||||
"(hands/run.sh install, hands/rec/install.sh, gaze/tracker/install.sh, "
|
||||
"setup/bluetooth/install.sh).")
|
||||
for c in (self.hands, self.recorder, self.eye, self.bt):
|
||||
c.connect("toggled", lambda *_: pw_row.set_sensitive(self.needs_password()))
|
||||
for w in self.choices:
|
||||
self.box.append(w)
|
||||
|
||||
def install(self, *_):
|
||||
if self.checking:
|
||||
return
|
||||
if not self.needs_password():
|
||||
return self.begin(None)
|
||||
text = self.pw.get_text()
|
||||
if not text:
|
||||
return self.say("Type your password, or untick the parts that need it.")
|
||||
pw = bytearray(text.encode())
|
||||
self.pw.set_text("")
|
||||
self.checking = True
|
||||
self.go.set_sensitive(False)
|
||||
self.say("Checking the password...")
|
||||
|
||||
def check():
|
||||
ok = check_password(pw)
|
||||
GLib.idle_add(checked, ok)
|
||||
|
||||
def checked(ok):
|
||||
self.checking = False
|
||||
self.go.set_sensitive(True)
|
||||
if ok:
|
||||
self.begin(pw)
|
||||
else:
|
||||
pw[:] = bytes(len(pw))
|
||||
self.say("sudo didn't take that password. Try again, or untick the parts that need it.")
|
||||
return False
|
||||
|
||||
threading.Thread(target=check, daemon=True).start()
|
||||
|
||||
def needs_password(self):
|
||||
return (self.hands.get_active() or (RELEASE and self.recorder.get_active()) or self.eye.get_active()
|
||||
or self.bt.get_active())
|
||||
|
||||
def say(self, text):
|
||||
self.error.set_label(text)
|
||||
self.error.set_visible(True)
|
||||
|
||||
def begin(self, pw):
|
||||
args = install_command(self.eye.get_active(), self.bt.get_active(), self.voice.get_active(),
|
||||
self.hands.get_active(), self.recorder.get_active())
|
||||
if pw is not None:
|
||||
try:
|
||||
self.askpass = Askpass(self.ask_again)
|
||||
except OSError as e:
|
||||
pw[:] = bytes(len(pw))
|
||||
return self.say(f"Couldn't make the password's socket: {e}")
|
||||
self.askpass.give(pw)
|
||||
if not start_unit(args, pw is not None):
|
||||
if self.askpass:
|
||||
self.askpass.close()
|
||||
self.askpass = None
|
||||
return self.say("Couldn't start the install service (systemd-run).")
|
||||
for w in self.choices:
|
||||
self.box.remove(w)
|
||||
self.show_progress()
|
||||
|
||||
# --- progress
|
||||
|
||||
def show_progress(self):
|
||||
self.status.set_label("Installing Frametop")
|
||||
self.detail.set_label("Starting...")
|
||||
self.bar = Gtk.ProgressBar()
|
||||
self.bar.pulse()
|
||||
self.text = Gtk.TextView(editable=False, cursor_visible=False, monospace=True,
|
||||
wrap_mode=Pango.WrapMode.WORD_CHAR)
|
||||
self.text.set_left_margin(8)
|
||||
self.text.set_right_margin(8)
|
||||
scroll = Gtk.ScrolledWindow(vexpand=True, child=self.text)
|
||||
|
||||
# The install wants the password and the window doesn't have it (played again).
|
||||
self.ask_pw = Gtk.PasswordEntry(show_peek_icon=True,
|
||||
placeholder_text="The next step needs your SteamOS password")
|
||||
ok = Gtk.Button(label="OK")
|
||||
skip = Gtk.Button(label="Skip that part")
|
||||
ok.connect("clicked", self.answer_ask)
|
||||
self.ask_pw.connect("activate", self.answer_ask)
|
||||
skip.connect("clicked", self.skip_ask)
|
||||
ask_row, ask_keypad = keypad_row(self.ask_pw, ok, skip)
|
||||
self.ask_bar = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=8, visible=False)
|
||||
self.ask_bar.append(ask_row)
|
||||
self.ask_bar.append(ask_keypad)
|
||||
self.ask_error = Gtk.Label(xalign=0, wrap=True, visible=False)
|
||||
self.ask_error.add_css_class("error")
|
||||
|
||||
keep = " Keep it open until the install is done: it gives the steps that need it your password." \
|
||||
if self.askpass else ""
|
||||
self.note = Gtk.Label(label="You can close this window: the install keeps going. Play Frametop "
|
||||
"again to come back to it." + keep, xalign=0, wrap=True)
|
||||
self.note.add_css_class("dim-label")
|
||||
close = Gtk.Button(label="Close", halign=Gtk.Align.END)
|
||||
close.connect("clicked", lambda *_: self.close())
|
||||
for w in (self.bar, scroll, self.ask_bar, self.ask_error, self.note, close):
|
||||
self.box.append(w)
|
||||
|
||||
if self.askpass is None:
|
||||
try:
|
||||
self.askpass = Askpass(self.ask_again)
|
||||
except OSError:
|
||||
self.askpass = None # askpass then fails, and install.sh skips those parts
|
||||
GLib.timeout_add(500, self.tick)
|
||||
self.tick()
|
||||
|
||||
def ask_again(self):
|
||||
if hasattr(self, "ask_bar"):
|
||||
self.ask_bar.set_visible(True)
|
||||
self.ask_pw.grab_focus()
|
||||
|
||||
def answer_ask(self, *_):
|
||||
text = self.ask_pw.get_text()
|
||||
if not text or self.checking:
|
||||
return
|
||||
pw = bytearray(text.encode())
|
||||
self.ask_pw.set_text("")
|
||||
self.checking = True
|
||||
|
||||
def check():
|
||||
GLib.idle_add(checked, check_password(pw))
|
||||
|
||||
def checked(ok):
|
||||
self.checking = False
|
||||
if ok and self.askpass:
|
||||
self.askpass.give(pw)
|
||||
self.ask_bar.set_visible(False)
|
||||
self.ask_error.set_visible(False)
|
||||
else:
|
||||
pw[:] = bytes(len(pw))
|
||||
self.ask_error.set_label("sudo didn't take that password.")
|
||||
self.ask_error.set_visible(True)
|
||||
return False
|
||||
|
||||
threading.Thread(target=check, daemon=True).start()
|
||||
|
||||
def skip_ask(self, *_):
|
||||
if self.askpass:
|
||||
self.askpass.refuse()
|
||||
self.ask_bar.set_visible(False)
|
||||
self.ask_error.set_visible(False)
|
||||
|
||||
def add_lines(self, lines):
|
||||
buf = self.text.get_buffer()
|
||||
for line in lines:
|
||||
m = STEP.match(line)
|
||||
if m:
|
||||
n, total, what = int(m[1]), int(m[2]), m[3]
|
||||
self.bar.set_fraction(max(0, n - 1) / total)
|
||||
self.detail.set_label(f"Step {max(n, 1)} of {total}: {what}")
|
||||
buf.insert(buf.get_end_iter(), line + "\n")
|
||||
# Keep the view at the newest line.
|
||||
end = buf.create_mark(None, buf.get_end_iter(), False)
|
||||
self.text.scroll_mark_onscreen(end)
|
||||
buf.delete_mark(end)
|
||||
|
||||
def tick(self):
|
||||
try:
|
||||
with open(LOG, "rb") as f:
|
||||
f.seek(self.offset)
|
||||
data = f.read()
|
||||
self.offset += len(data)
|
||||
except FileNotFoundError:
|
||||
data = b""
|
||||
if data:
|
||||
text = self.partial + ANSI.sub("", data.decode("utf-8", "replace")).replace("\r", "\n")
|
||||
*lines, self.partial = text.split("\n")
|
||||
self.add_lines(lines)
|
||||
|
||||
state = unit_state()
|
||||
if state == "activating":
|
||||
if self.bar.get_fraction() == 0:
|
||||
self.bar.pulse()
|
||||
return True
|
||||
if self.partial:
|
||||
self.add_lines([self.partial])
|
||||
self.partial = ""
|
||||
self.forget()
|
||||
self.note.set_visible(False)
|
||||
self.ask_bar.set_visible(False)
|
||||
if state == "active":
|
||||
self.status.set_label("Done")
|
||||
self.detail.set_label(DONE_TEXT)
|
||||
self.bar.set_fraction(1)
|
||||
else:
|
||||
self.status.set_label("The install stopped")
|
||||
self.detail.set_label("The log above says why. Play Frametop again to try again.")
|
||||
return False
|
||||
|
||||
def forget(self):
|
||||
if self.askpass:
|
||||
self.askpass.close()
|
||||
self.askpass = None
|
||||
|
||||
def closing(self, *_):
|
||||
self.forget()
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
app = Adw.Application(application_id="io.github.deejanuz.FrametopInstall")
|
||||
|
||||
def activate(a):
|
||||
# Played again while the window is open: show that one.
|
||||
win = a.get_active_window() or Window(a)
|
||||
win.present()
|
||||
|
||||
app.connect("activate", activate)
|
||||
app.run([])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,114 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# FrameDrop proof of concept, step 1: run as a Steam "Devkit Game" (what FrameDrop installs a
|
||||
# Linux zip as) and record what an installer started from Steam could use: host or Steam Linux
|
||||
# Runtime container, podman, the user's systemd, git, the network, GTK, and a window on screen.
|
||||
# Register and start it: framedrop/devkit.sh add FrametopProbe framedrop/probe "./probe.sh native"
|
||||
# framedrop/devkit.sh run FrametopProbe
|
||||
# Usage: probe.sh LABEL
|
||||
# Writes ~/.cache/frametop-framedrop/probe-LABEL.log, replacing the previous one.
|
||||
label=${1:-unlabeled}
|
||||
out=$HOME/.cache/frametop-framedrop
|
||||
mkdir -p "$out"
|
||||
log=$out/probe-$label.log
|
||||
exec >"$log" 2>&1
|
||||
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
section() { printf '\n== %s\n' "$1"; }
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
check() { # check NAME CMD...: one line, ok or failed with the exit status
|
||||
local name=$1; shift
|
||||
if out_=$(timeout 20 "$@" 2>&1); then echo "ok $name: $(echo "$out_" | head -1)"
|
||||
else echo "FAILED $name (exit $?): $(echo "$out_" | head -2 | tr '\n' ' ')"; fi
|
||||
}
|
||||
|
||||
section "when, who, where"
|
||||
date -Is
|
||||
echo "label=$label uid=$(id -u) user=$(id -un) pwd=$PWD here=$here"
|
||||
echo "args: $*"
|
||||
grep -E '^(ID|VARIANT_ID|VERSION_ID|BUILD_ID|PRETTY_NAME)=' /etc/os-release
|
||||
|
||||
section "container?"
|
||||
for p in /run/pressure-vessel /.flatpak-info /run/host/os-release; do
|
||||
[ -e "$p" ] && echo "present: $p" || echo "absent: $p"
|
||||
done
|
||||
echo "container=${container-} PRESSURE_VESSEL_RUNTIME=${PRESSURE_VESSEL_RUNTIME-}"
|
||||
[ -e /run/host/os-release ] && grep -E '^(ID|VARIANT_ID)=' /run/host/os-release
|
||||
|
||||
section "environment"
|
||||
env | grep -E '^(PATH|HOME|XDG_[A-Z_]+|DISPLAY|WAYLAND_DISPLAY|DBUS_SESSION_BUS_ADDRESS|SteamAppId|SteamGameId|STEAM_COMPAT_[A-Z_]+|LD_LIBRARY_PATH|LD_PRELOAD|SDL_[A-Z_]+|GDK_BACKEND|ENABLE_[A-Z_]+|PRESSURE_VESSEL_[A-Z_]+)=' | sort
|
||||
echo "process tree:"
|
||||
pid=$$
|
||||
for _ in 1 2 3 4 5 6 7 8; do
|
||||
[ "$pid" -le 1 ] 2>/dev/null && break
|
||||
printf ' %s %s\n' "$pid" "$(tr '\0' ' ' </proc/"$pid"/cmdline 2>/dev/null | cut -c1-200)"
|
||||
pid=$(awk '/^PPid:/{print $2}' /proc/"$pid"/status 2>/dev/null)
|
||||
done
|
||||
|
||||
# Steam adds its overlay (LD_PRELOAD) and runtime libraries to every child. An installer has
|
||||
# to drop them before running host tools, as the checks below do.
|
||||
unset LD_PRELOAD
|
||||
LD_LIBRARY_PATH=$(printf %s "${LD_LIBRARY_PATH-}" | tr ':' '\n' | grep -v -e '/Steam/' -e '^$' -e 'x86_64' -e 'i386' | paste -sd:)
|
||||
[ -n "$LD_LIBRARY_PATH" ] && export LD_LIBRARY_PATH || unset LD_LIBRARY_PATH
|
||||
PATH=$(printf %s "$PATH" | tr ':' '\n' | grep -v '/Steam/' | paste -sd:)
|
||||
echo "cleaned: PATH=$PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH-}"
|
||||
|
||||
section "tools"
|
||||
for t in bash git curl python3 podman distrobox systemctl systemd-run busctl gdbus flatpak-spawn \
|
||||
steam-runtime-launch-client konsole zenity kdialog; do
|
||||
printf '%-28s %s\n' "$t" "$(command -v "$t" || echo -)"
|
||||
done
|
||||
|
||||
section "what an installer needs"
|
||||
check "home writable" sh -c 'f=$HOME/.cache/frametop-framedrop/.w && : >"$f" && rm "$f" && echo yes'
|
||||
check "~/frametop visible" sh -c 'ls -d "$HOME/frametop" && git -C "$HOME/frametop" log -1 --format=%h'
|
||||
have git && check "git ls-remote github" git ls-remote --heads https://github.com/Frametop/frametop.git experimental
|
||||
have curl && check "curl get.sh" sh -c 'curl -fsSL https://frametop.github.io/frametop/get.sh | head -1'
|
||||
have podman && check "podman ps" podman ps --format '{{.Names}}'
|
||||
have distrobox && check "distrobox list" distrobox list
|
||||
have systemctl && check "systemctl --user" systemctl --user is-system-running
|
||||
have systemctl && check "user units visible" systemctl --user is-enabled frametop-input-relay.service
|
||||
have python3 && check "python3 gi Gtk 4" python3 -c 'import gi; gi.require_version("Gtk","4.0"); gi.require_version("Adw","1"); from gi.repository import Gtk, Adw; print(Gtk.get_major_version(), Gtk.get_minor_version())'
|
||||
|
||||
section "escape to the host (needed if this is a container)"
|
||||
# A transient user unit runs in the host's user manager, outside any container.
|
||||
if have systemd-run; then
|
||||
rm -f "$out/escape-$label"
|
||||
check "systemd-run --user" systemd-run --user --wait --collect --quiet -- \
|
||||
sh -c "grep -E '^(ID|VARIANT_ID)=' /etc/os-release > '$out/escape-$label'; command -v podman git >> '$out/escape-$label'"
|
||||
[ -s "$out/escape-$label" ] && sed 's/^/ host says: /' "$out/escape-$label"
|
||||
fi
|
||||
if have busctl; then
|
||||
check "busctl --user (systemd1)" busctl --user get-property org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager Version
|
||||
fi
|
||||
|
||||
section "window"
|
||||
# A plain GTK 4 window for 20 s. Watch for it on the Frame; "window: mapped" means GTK showed it.
|
||||
if have python3; then
|
||||
timeout 40 python3 - <<'EOF'
|
||||
import sys
|
||||
try:
|
||||
import gi
|
||||
gi.require_version("Gtk", "4.0")
|
||||
from gi.repository import Gtk, GLib
|
||||
except Exception as e:
|
||||
print("window: no GTK 4:", e); sys.exit(0)
|
||||
app = Gtk.Application(application_id="io.github.deejanuz.FrametopProbe")
|
||||
def on_activate(app):
|
||||
w = Gtk.ApplicationWindow(application=app, title="Frametop installer probe")
|
||||
w.set_default_size(640, 240)
|
||||
w.set_child(Gtk.Label(label="Frametop installer probe\n\nIf you can read this, a FrameDrop install can show progress.\nThis window closes in 20 seconds."))
|
||||
w.connect("map", lambda *_: print("window: mapped", flush=True))
|
||||
w.present()
|
||||
GLib.timeout_add_seconds(20, app.quit)
|
||||
app.connect("activate", on_activate)
|
||||
print("window: display", Gtk.Widget.get_display(Gtk.Label()) if hasattr(Gtk.Widget, "get_display") else "?", flush=True)
|
||||
app.run([])
|
||||
print("window: closed")
|
||||
EOF
|
||||
echo "window exit: $?"
|
||||
else
|
||||
echo "window: no python3"
|
||||
fi
|
||||
|
||||
section "done"
|
||||
date -Is
|
||||
@@ -1,202 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# ft — build and run Frametop out of its OCI image (pack/Containerfile).
|
||||
#
|
||||
# The image is the product: one container that holds the frozen toolchain, the
|
||||
# native binaries, and the locked Python environment. Everything Frametop runs
|
||||
# goes through this wrapper, so the integration points (mounts, IPC, devices)
|
||||
# live in exactly one place.
|
||||
#
|
||||
# The wrapper works from two homes:
|
||||
# Repo mode the script sits in a checkout (pack/Containerfile beside it):
|
||||
# development commands build from the sources, and runs mount
|
||||
# the repo at /src/frametop
|
||||
# Installed the script sits in ~/.local/bin (put there by install.sh):
|
||||
# programs run from the image's own copy, no repo needed
|
||||
#
|
||||
# Image reference, in order: $FT_IMAGE, then ~/.config/frametop/image — which
|
||||
# install.sh writes and `ft update` keeps digest-pinned. A moving :latest tag
|
||||
# is refused for what gets deployed (the pinned image and the update source):
|
||||
# what runs on a headset must be reproducible and rollback-able. Local
|
||||
# development is free to use any tag, :latest included.
|
||||
#
|
||||
# Usage:
|
||||
# ft <prog> [args] [both] run a program from /opt/frametop
|
||||
# ft update [both] pull the published image, pin its digest
|
||||
# ft clean [both] remove frametop's leftover containers and
|
||||
# unused frametop images (see below)
|
||||
# Development commands live behind "ft dev" (they need a repo checkout):
|
||||
# ft dev build [repo] build the image from the repo
|
||||
# ft dev shell [repo] interactive shell in the image
|
||||
# ft dev test [repo] run the Python test suites inside the image
|
||||
#
|
||||
# About `ft clean` and the shared podman store: on SteamOS, frametop's
|
||||
# containers share one rootless podman store with Valve's (named
|
||||
# lepton-<context>). `ft clean` touches ONLY objects that are provably
|
||||
# frametop's: containers named frametop-*, and images whose reference
|
||||
# contains a /frametop or frametop: component. Everything else — every
|
||||
# lepton-* container, every dangling image, every cache layer — is left
|
||||
# alone. Never run `podman rm -a`, `podman rmi -a`, or `podman system prune`
|
||||
# on a Frame: those are store-wide and would delete Valve's containers.
|
||||
set -euo pipefail
|
||||
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
[ -f "$root/pack/Containerfile" ] && repo_mode=1 || repo_mode=0
|
||||
config_dir=${XDG_CONFIG_HOME:-$HOME/.config}/frametop
|
||||
ref_file=$config_dir/image # installed: the pinned image reference
|
||||
published_file=$config_dir/published # installed: where updates come from
|
||||
|
||||
reject_latest() { # reject_latest <what> <ref>
|
||||
# A reference with neither a tag nor a digest means :latest too. The tag is
|
||||
# after the last "/", so a registry port (host:5000/...) isn't one.
|
||||
local name=${2%%@*} latest=0
|
||||
case ${name##*/} in
|
||||
*:latest) latest=1 ;;
|
||||
*:*) ;;
|
||||
*) [[ $2 == *@sha256:* ]] || latest=1 ;;
|
||||
esac
|
||||
if [ "$latest" = 1 ]; then
|
||||
echo "ft: $2 uses the moving :latest tag — refused for $1." >&2
|
||||
echo " Pin a version tag or a digest (repo@sha256:...) instead," >&2
|
||||
echo " e.g. in $published_file or $ref_file." >&2
|
||||
exit 2
|
||||
fi
|
||||
}
|
||||
|
||||
# The reference updates pull from: $FT_IMAGE_PUBLISHED, then the config file.
|
||||
# It must be configured and versioned — there is no silent :latest default,
|
||||
# and there is no default that appears when nothing is configured.
|
||||
# (Deferred: refreshing the installed wrapper from the image, so wrapper and
|
||||
# image ship as one artifact. The wiring broke CI under Docker; revisit after
|
||||
# the smoke job tells us exactly where.)
|
||||
default_published() {
|
||||
local ref
|
||||
if [ -n "${FT_IMAGE_PUBLISHED:-}" ]; then ref=$FT_IMAGE_PUBLISHED
|
||||
elif [ -r "$published_file" ]; then ref=$(<"$published_file")
|
||||
else
|
||||
echo "ft update: no published reference configured." >&2
|
||||
echo " Write a versioned reference (never :latest) to:" >&2
|
||||
echo " $published_file" >&2
|
||||
echo " e.g. ghcr.io/0x1f6/frametop:v0.2.1 — install.sh does this." >&2
|
||||
exit 2
|
||||
fi
|
||||
reject_latest "the update source" "$ref"
|
||||
echo "$ref"
|
||||
}
|
||||
|
||||
# The reference programs run: $FT_IMAGE, then the pinned file. Installed mode
|
||||
# refuses to run without a pin — a headset must never end up on :latest by
|
||||
# accident, and must always know exactly what image it is running.
|
||||
default_image() {
|
||||
if [ "$repo_mode" = 1 ]; then echo "frametop:local"; return; fi
|
||||
local ref
|
||||
if [ -r "$ref_file" ] && [ -n "$(<"$ref_file")" ]; then ref=$(<"$ref_file")
|
||||
elif [ -n "${FT_IMAGE:-}" ]; then echo "$FT_IMAGE"; return
|
||||
else
|
||||
echo "ft: no image pinned. Nothing has been installed yet." >&2
|
||||
echo " install.sh writes the pinned reference to:" >&2
|
||||
echo " $ref_file" >&2
|
||||
exit 2
|
||||
fi
|
||||
reject_latest "the pinned image" "$ref"
|
||||
echo "$ref"
|
||||
}
|
||||
image=${FT_IMAGE:-}
|
||||
published=${FT_IMAGE_PUBLISHED:-}
|
||||
|
||||
# FT_ENGINE picks one when both are installed (CI builds with docker, and
|
||||
# its runner has podman too).
|
||||
engine=${FT_ENGINE:-$(command -v podman || command -v docker || true)}
|
||||
[ -n "$engine" ] || { echo "ft: need podman or docker on PATH" >&2; exit 1; }
|
||||
|
||||
# Repo mode mounts the checkout; installed mode uses the image's own copy.
|
||||
mounts=()
|
||||
[ "$repo_mode" = 1 ] && mounts+=(-v "$root":/src/frametop)
|
||||
# A container's own network namespace starts with the kernel's datagram queue
|
||||
# of 10; systemd sets 512 on the host. The input relay sends without blocking
|
||||
# and drops what a full queue refuses, so at 10 its burst of key and button
|
||||
# releases loses the last ones (keys-test.py caught this).
|
||||
mounts+=(--sysctl net.unix.max_dgram_qlen=512)
|
||||
|
||||
# How the programs run on the Frame (mounts, devices, groups, namespaces) is
|
||||
# not decided yet: see "The runtime on the Frame" in pack/design.md. Runs here
|
||||
# get no access to the host beyond the repo mount.
|
||||
|
||||
run_in_image() {
|
||||
[ -n "$image" ] || image=$(default_image)
|
||||
# frametop-<program>-<pid>: podman ps names the program, two runs of one
|
||||
# program (python3 a.py, python3 b.py) don't replace each other, and
|
||||
# ft clean finds the leftovers of crashed runs by the prefix.
|
||||
local name
|
||||
name=frametop-$(printf '%s' "${1##*/}" | tr -c 'a-zA-Z0-9._-' '-')-$$
|
||||
exec "$engine" run --rm --name "$name" ${mounts[@]+"${mounts[@]}"} \
|
||||
--workdir /src/frametop "$image" "$@"
|
||||
}
|
||||
|
||||
need_repo() {
|
||||
[ "$repo_mode" = 1 ] || { echo "ft $1: only in a repo checkout (this copy runs installed)" >&2; exit 2; }
|
||||
}
|
||||
|
||||
update() {
|
||||
[ -n "$published" ] || published=$(default_published)
|
||||
reject_latest "the update source" "$published"
|
||||
"$engine" pull "$published"
|
||||
# Pin what was pulled, by digest, so every later run and any bug report
|
||||
# names exactly this image — and rollback is editing one file.
|
||||
local digest
|
||||
if digest=$("$engine" inspect --format '{{index .RepoDigests 0}}' "$published" 2>/dev/null) && [ -n "$digest" ]; then
|
||||
mkdir -p "$config_dir"
|
||||
# Written whole or not at all: an empty pin would stop every run.
|
||||
printf '%s\n' "$digest" > "$ref_file.new" && mv "$ref_file.new" "$ref_file"
|
||||
echo "ft update: pinned $digest"
|
||||
else
|
||||
echo "ft update: could not read a digest for $published — keeping $image." >&2
|
||||
fi
|
||||
}
|
||||
|
||||
clean() {
|
||||
# Frametop's leftovers, and nothing else. See the header: the podman store
|
||||
# is shared with Valve's lepton-* containers, so this only deletes objects
|
||||
# whose name proves they are ours.
|
||||
local found=0 obj
|
||||
while IFS= read -r obj; do
|
||||
[ -n "$obj" ] || continue
|
||||
found=1
|
||||
echo "removing container $obj"
|
||||
"$engine" rm -f "$obj" >/dev/null
|
||||
done < <("$engine" ps -a --format '{{.Names}}' 2>/dev/null | grep '^frametop-' || true)
|
||||
# Images: only references that name frametop, and only ones no container
|
||||
# uses (podman rmi refuses those anyway — the guard is belt and braces).
|
||||
while IFS= read -r obj; do
|
||||
[ -n "$obj" ] || continue
|
||||
case $obj in *frametop*) ;; *) continue ;; esac
|
||||
found=1
|
||||
echo "removing image $obj"
|
||||
"$engine" rmi "$obj" >/dev/null 2>&1 \
|
||||
|| echo " (in use or already gone — left in place)" >&2
|
||||
done < <("$engine" images --format '{{.Repository}}:{{.Tag}} {{.ID}}' 2>/dev/null \
|
||||
| grep -E '(^|[ /])frametop(:| |$)' | awk '{print $1}' || true)
|
||||
[ "$found" = 1 ] || echo "nothing of frametop's to clean"
|
||||
[ "$found" = 1 ] || exit 0
|
||||
}
|
||||
|
||||
case "${1:-help}" in
|
||||
dev)
|
||||
shift
|
||||
image=${FT_IMAGE:-frametop:local} # dev always works against the local build
|
||||
case "${1:-help}" in
|
||||
build) need_repo build
|
||||
exec "$engine" build -f "$root/pack/Containerfile" -t "$image" "$root" ;;
|
||||
shell) need_repo shell
|
||||
exec "$engine" run --rm -it ${mounts[@]+"${mounts[@]}"} --workdir /src/frametop "$image" bash ;;
|
||||
test) need_repo test; run_in_image just test ;;
|
||||
help|-h|--help|"") sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' ;;
|
||||
*) echo "ft dev: unknown command: $1 (build, shell, test)" >&2; exit 2 ;;
|
||||
esac ;;
|
||||
update) update ;;
|
||||
clean) clean ;;
|
||||
help|-h|--help)
|
||||
sed -n '2,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
[ -z "${1:-}" ] || exit 0 ;;
|
||||
*) [ -z "${1:-}" ] && { echo "ft: no command given" >&2; exit 2; }
|
||||
run_in_image "$@" ;;
|
||||
esac
|
||||
+15
-33
@@ -2,24 +2,18 @@
|
||||
|
||||
The Steam Frame's eye tracking as pointer input: a gaze mode for the 3D mouse (the pointer goes where you look, and the mouse does the last bit), and the tools to calibrate and measure it.
|
||||
|
||||
- `ft-gaze` (C++, OpenVR, runs in the dev container) reads the eye tracker and prints one JSON line per sample (90 Hz). For each source, it gives the gaze direction relative to the head and the Frametop screen pixel it lands on. The gaze service asks it for only the sources it uses (`--sources`, and `sources LIST` on its stdin): our tracker and mmap set 1 with Own tracker, about 700 bytes a line instead of 1.3 KB, and every source while a check or the calibration runs. So SteamVR's gaze action, the only source that calls into vrserver (twice a sample), is read only then. The probe gets them all.
|
||||
- `ft-gaze` (C++, OpenVR, runs in the dev container) reads the eye tracker and prints one JSON line per sample (90 Hz). For each source, it gives the gaze direction relative to the head and the Frametop screen pixel it lands on.
|
||||
- `gazecal.py` has what the probe and the gaze service share: the correction models, filters, and the reader for SteamVR's eye tracking log.
|
||||
- `tracker/` is our own eye tracker, an alternative to SteamVR's: `ft-eyes` finds the pupils and glints in the eye-camera frames that `ft-eyegrab` (a small root service) copies out of SteamVR's tracker. See "Our own eye tracker" below.
|
||||
- `probe/ft-gazeprobe` (GTK 4, host Python) is a fullscreen playground, for developing the gaze tracking: day to day, the calibration and the checks run in the headset panel (Quick check, Calibrate, and Check headset fit on the Gaze page). It runs ft-gaze, draws where you're looking, measures accuracy, and tries out hold-to-adjust clicking with a calibration that learns from your adjustments.
|
||||
|
||||
Day to day, install the gaze service, then turn gaze mode on and calibrate on the Gaze page of Frametop Input Settings (Calibrate). The installer offers the gaze service (`gaze/run.sh install`) and then our own tracker (`gaze/tracker/install.sh`), which gaze mode uses once it's installed; the probe is installed by hand.
|
||||
|
||||
The gaze service's install also turns on SteamVR's `steamvr.gazeDepEyeToHeadEnable` (`gaze/eye-rate.py on`), which counts from SteamVR's next start. The Frame's SteamVR driver runs the eye cameras at 15 to 90 fps (`eyeTrackingRateMin` and `eyeTrackingRateMax` in its `frame_hmd_additional.vrsettings`) and picks the rate as SteamVR starts: with its defaults, 15 (`eyetracking.txt`: "Set framerate 15"). That's enough for SteamVR's own tracker, but our tracker then sees 8 or 9 frames in a calibration dot's 0.6 s, where it needs 15, so it can never calibrate (a 2026-10-10 report: "the right eye was seen in only 8 frames" on every dot, while SteamVR's fit screen showed both eyes tracked). With that setting on, SteamVR asks for 90 at every start (vrserver.txt: "cv: Request eye tracking framerate: 90"). Valve's driver leaves it off and SteamVR's settings have no switch for it; what else it changes isn't documented, and the headset Frametop is developed on runs with it on. Uninstalling turns it off again only if Frametop turned it on. `scripts/doctor.sh` and `scripts/gaze-report.py` report the cameras' rate.
|
||||
- `probe/ft-gazeprobe` (GTK 4, host Python) is a fullscreen playground. It runs ft-gaze, draws where you're looking, measures accuracy, and tries out hold-to-adjust clicking with a calibration that learns from your adjustments.
|
||||
|
||||
```
|
||||
gaze/run.sh install # the gaze service: builds ft-gaze and the panel, starts with SteamVR
|
||||
gaze/eye-rate.py status # SteamVR's eye cameras: the setting that keeps them at 90 fps, and their rate now
|
||||
gaze/build.sh # build ft-gaze
|
||||
gaze/probe/install.sh # build, and add Frametop Gaze Probe to the app menu
|
||||
gaze/probe/ft-gazeprobe --screen 1
|
||||
gaze/run.sh install # the gaze service, with SteamVR
|
||||
gaze/ft-gazectl on # the pointer follows your gaze (off: the mouse alone)
|
||||
gaze/tracker/install.sh # our own eye tracker's frame grabber (asks for sudo)
|
||||
gaze/build.sh # build ft-gaze and the panel by hand
|
||||
gaze/probe/install.sh # development: build, and add Frametop Gaze Probe to the app menu
|
||||
gaze/probe/ft-gazeprobe --screen 1
|
||||
scripts/gaze-report.py # why gaze or its calibration doesn't work, with what looks wrong first
|
||||
```
|
||||
|
||||
## Gaze pointer
|
||||
@@ -27,20 +21,15 @@ scripts/gaze-report.py # why gaze or its calibration doesn't work, with w
|
||||
Gaze as an input method for the whole desktop, without replacing anything of SteamVR's:
|
||||
|
||||
- `ft-gazed` (host Python, a user service: `gaze/run.sh install`) runs ft-gaze and corrects its gaze. Two settings on the Gaze page of Frametop Input Settings (`GAZE_TRACKER` and `GAZE_EYE` in `~/.config/frametop.conf`, read again when the file changes) pick whose eye tracking it uses and how it weights the eyes:
|
||||
- **Eye tracker:** our own (Own tracker: see "Our own eye tracker" below) or SteamVR's. The default, `GAZE_TRACKER=auto`, is ours when it's installed (its frame grabber, and ft-eyes' Python in the gaze service's checkout), else SteamVR's, and it switches when ours is installed or removed; picking one on the Gaze page sets it for good. The gaze service runs ours while it's the one in use. It keeps its own calibration: with Own tracker chosen, Calibrate on the Gaze page calibrates it. The gaze pointer's settings (hand back, nudges, hold to drag, the dot) are the pointer helper's, so they're the same with either.
|
||||
- **Eye tracker:** SteamVR's (the default), or our own (Own tracker: see "Our own eye tracker" below). The gaze service runs ours while this is on. It keeps its own calibration: calibrate it in the probe with the tracker toggle on Own tracker. The gaze pointer's settings (hand back, nudges, hold to drag, the dot) are the pointer helper's, so they're the same with either.
|
||||
- **Eye bias:** Auto, Left, or Right. The gaze combines both eyes, each calibrated on its own, because their errors partly cancel: on 306 clicks with our tracker, the eyes' sideways errors were correlated -0.37, and both together were 0.65 degrees off (median) against 0.96 for the left eye alone and 1.11 for the right. So Left or Right leans instead of choosing: that eye counts twice as much as the other (0.03 degrees worse there toward the better eye, 0.13 toward the worse). Auto weights each eye by the inverse square of how far off it was at your last 20 nudges, once each eye has 5, and evenly before that. Each eye's miss is measured before that nudge teaches anything, so each is a fresh test. The calibration's own fit isn't used for this: on SteamVR's test of 2026-09-29, the calibration dots said the left eye was the better one, and new spots said the right. Either eye carries the gaze alone while the other is closed or lost.
|
||||
- **One eye:** SteamOS 0.4's Track Dominant Eye Only (SteamVR's settings, General, with Show advanced settings on; `steamvr.eyeTrackingDominantEyeOnly` with `steamvr.dominantEye`) makes SteamVR's tracker ignore the other eye. With SteamVR's tracker, Frametop then goes by that eye alone: the calibration and the checks wait only for it, the gaze is that eye's own reading once it's calibrated (SteamVR's combined gaze before, which follows that eye then), and the fit check shows the other eye as not tracked. The service reads the setting again when SteamVR's settings file changes. Not yet tried in the headset: what SteamVR's shared memory says about the ignored eye wasn't measured.
|
||||
|
||||
With SteamVR, each eye is its own reading (set 2), corrected by its calibration from the probe (the Left eye and Right eye sources) plus what the pointer has taught that eye since. On that test, the two eyes each calibrated and averaged were 1.70 degrees off (median; mean 1.62) against 1.72 (mean 1.84) for SteamVR's combined gaze with its calibration. A calibration from before the probe had the eyes as sources, or `--source`, uses the older path. That path runs on SteamVR's combined gaze (mmap set 1), corrected as a whole. When the tracker loses one eye (its variance for that eye jumps from about 0.001 to 0.02), the gaze comes from the other eye instead: that eye's own reading (set 2) plus what it usually reads against the combined gaze, learned while both eyes are seen, in 10 degree cells of where it looks. Set 1 keeps going on one eye too, but it holds the lost eye's yaw where it was, so the gaze moves half as far sideways as your eyes do. On a recording, one eye alone came out a median 0.8 degrees from both eyes' gaze over a steady look, a little more jittery.
|
||||
|
||||
Looks down past the screens (under 20 degrees down, on no Frametop screen: a glance at the keyboard) aren't sent, so the pointer stays where it was instead of following you down, and eyes lost there aren't counted. It drops blinks (both eyes closing or lost), smooths with a fixation lock, and sends the result to the pointer helper 90 times a second. It follows SteamVR's eye tracking log, and when the headset goes back on (SteamVR starts its eye model over, and the error moves), older lessons count less, so the first few after relearn the offset.
|
||||
- The pointer helper's **gaze mode** (off by default: the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1` in `~/.config/frametop.conf`, or a mouse button or key combination mapped to "Gaze pointer on/off") works like MAGIC pointing (Zhai et al., 1999). The pointer goes where you look, and the mouse does the last bit. By default the mouse moves it only while a button is held (see "The mouse only corrects" below). With `POINTER_GAZE_MOUSE_MOVE=free`, moving the mouse takes the pointer, from where the gaze put it, and looking well away (5 degrees) gives it back to the gaze. A press isn't sent at once: the pointer stops where the gaze put it, and if that's wrong, drag it onto what you meant with the button still held; the click happens where you let go. To drag something, hold the press still for half a second first (`POINTER_GAZE_HOLD`), then move. Outside games the pointer stays on while gaze mode is on, until a controller is picked up. The dot shows all the time (`POINTER_GAZE_DOT=moving`: only while the mouse moves it, while a press is held, and as a pulse when you click). Gaze mode works with the mouse and the keyboard, not the controllers ([docs/gaze-controllers.md](../docs/gaze-controllers.md) explains why).
|
||||
- **The mouse only corrects** (the default; the Gaze page's Mouse movement switch, `POINTER_GAZE_MOUSE_MOVE=held`): while the gaze has the pointer, moving the mouse does nothing. The buttons work like Meta+J and Meta+K: press and hold one and the pointer stops where you look; move the mouse onto what you meant and let go to click there (a left or a right click). Held still for half a second, a press is a real one (to drag). Once you've moved, the left button alone only clicks: press the right one while still holding the left to start a drag there; it lasts while either button is held. Press the right one again (a double right click, the left still held) to pan and tilt what you're dragging, as a right press does during any drag. A bumped or drifting mouse can't pull the pointer away, and every mouse move is a correction, so the tracker only learns from real ones. With the gaze stale for a second (the tracker stopped, eyes lost), in a game, or with the headset off, the mouse moves the pointer as usual. `free` (the switch off) lets the mouse take the pointer any time.
|
||||
- **Keyboard clicks** (Meta+J left, Meta+K right; other key combinations on the Keyboard page of Input Settings): tap to click where you look. A quick tap (let go within 0.25 s, `POINTER_KEY_TAP`) clicks where the dot was when you pressed, whatever your head did, and tells the gaze service it was right there. Hold instead, and the dot stays put in your view: turn your head until it sits on what you meant, and let go to click there (the correction is a lesson, as with the mouse, under the same limit: past `POINTER_GAZE_NUDGE_MAX` it opens the quick check instead). Hold still for half a second to press for real, then turn your head to drag. With Meta+J held, Meta+K presses where the dot is now, so you can correct first and then drag; the drag lasts while either key is held. Meta+K during a Meta+J drag (again, after starting it with Meta+K: a double Meta+K) pans and tilts what you're dragging while it's held: turn your head to turn it.
|
||||
- **Learning from nudges:** if the mouse took the pointer from the gaze and moved it (0.2 degrees or more, and the correction within `POINTER_GAZE_NUDGE_MAX`: 55 degrees by default, half of the 109 the headset shows across, and 1 to 110; the same limit for mouse, keyboard, and pinch clicks) before you clicked, or you dragged a held press that far, you were nudging it onto what you looked at. The helper sends that as a lesson, from the raw gaze when the mouse took over to where you clicked, and ft-gazed learns it. So using it is what calibrates it. The raw gaze is one ft-gazed sent, so it also finds when that look was, and what each eye read then. With SteamVR, each eye learns its own error. With our tracker, the look goes to it as a click, like the probe's, and it relearns how the headset sits on your face. After the headset was off, your first nudge and click there resets that (the quick check's dot does the same). A correction past `POINTER_GAZE_NUDGE_MAX` isn't learned: the helper asks ft-gazed for the quick check instead ("recheck", after its 2-minute cooldown). Tested on our tracker's 409 clicks since its Sep 29 calibration: a one-dot check set from any one of them put the next 2 minutes' clicks within 15 degrees (99% within 4.2) and the next 10 minutes' within 25 (the far ones after the headset moved), so the check gets back well under it. The limit used to be 8 degrees, and live on 2026-10-01 our tracker was 12 off after the headset went on, so every correction was dropped. One lesson moves the whole correction by only a third of what it measured (more near where it was taken), since in the first live test one 6 degree lesson moved everything and put the next target 7 degrees off. `ft-gazectl status` shows the lessons, and `ft-gazectl forget` drops them.
|
||||
- **Checks and calibration in the headset** (`gaze/gazecheck.py`, shown by `gaze/panel/ft-gazepanel`, a panel fixed to the headset that ft-gazed runs): a one-dot quick check opens when you put the headset on (SteamVR's tracker sees your eyes for 3 s after none for 3 s; its "HMD on" log line can't say, since it repeats every minute or so and can stay on for hours with nobody in the headset), when our tracker asks for a click (its "reseat", when the headset may sit differently), at most once every 2 minutes, and from Quick check on the Gaze page. Look at the dot: it takes your gaze once it has held still for 0.6 s (the steadiness counts, not where the tracker puts it, so it works however far off it is), or at once with a left click or Meta+J; a right click or Meta+K closes it, and ignoring it changes nothing. It also runs when a click's correction was past `POINTER_GAZE_NUDGE_MAX`. The dot is still and the ring fills in quarters, so the panel is drawn again only a few times per dot. If the first 3 lessons after it are still over 2 degrees off, five dots follow: the middle, and 12 degrees left and right and 9 up and down, in a see-through panel 40 degrees wide (the quick check's 16 degree square left four of them off the panel, unseen). The full calibration (Calibrate on the Gaze page, or by itself whenever gaze mode is on without one and your eyes are seen) is the probe's: three rounds, dark, medium and bright, of the middle and a ring around it, in a panel 64 degrees wide, with Frametop's screens hidden. Its dots (and the five-dot check's) wait for a click: look at the dot and left click or press Meta+J, and the gaze held still up to then is taken. A button mapped to gaze precision or gaze drag counts as the left click there (before 2026-10-09 the panel ignored it). Our tracker's first calibration has no gaze to go on, since ft-eyes maps pupils to a gaze only once it has a calibration: it opens once SteamVR's tracker sees an eye and ft-eyes answers, and a click takes the 0.6 s up to it, as long as ft-eyes saw each pupil held still then (before 2026-10-05 it waited for a gaze, so a fresh install could never calibrate ours). The dot's ring shows full while ft-eyes checks it; the service doesn't wait for that answer (before 2026-10-09 it did, up to 3 s a dot, so the pointer could come back over the panel, and a slow answer closed the calibration as if the headset came off). A dot that isn't taken says why, on an orange line over the instructions: with SteamVR's tracker, what dropped most of that look's samples (an eye lost, a blink, the two eyes disagreeing); with ours, its reply (an eye seen in too few frames, or moving). A dot gets two tries, then it's skipped. A calibration left with under two thirds of its dots fails and names the most common reason, as the Gaze page does after it. A click that has taken nothing after 1.5 s says what it waits for: the gaze to hold still, or an eye tracker that isn't sending. Capturing whenever the gaze held still sometimes took a look that wasn't on the dot. The panel draws into three shared buffers SteamVR imported once, as Frametop's keyboard does: uploading each picture anew (SetOverlayRaw) flickered, and in one live test left the headset showing an old picture. Quitting it while there's still no calibration turns gaze mode off; turning it on again reopens it. One that closes otherwise unfinished (ignored for 2 minutes, too few dots) opens again after the headset comes off and on. Why gaze mode, on, can't work yet goes in the service's status as `checks.problem`, which the Gaze page shows under the Gaze pointer switch. For our tracker a check is a click and the calibration is its own (calib-point per dot); for SteamVR's, a check is a lesson for each eye and the calibration replaces calibration.json, and the lessons start over. Checks go to `checks.jsonl`.
|
||||
- The pointer helper's **gaze mode** (off by default: the Gaze page of Frametop Input Settings, `gaze/ft-gazectl on`, `POINTER_GAZE=1` in `~/.config/frametop.conf`, or a mouse or controller button mapped to "Gaze pointer on/off") works like MAGIC pointing (Zhai et al., 1999). The pointer goes where you look. Move the mouse and it's the mouse's, from where the gaze put it, for the last bit. Look well away (5 degrees) and the gaze takes it back. A press isn't sent at once: the pointer stops where the gaze put it, and if that's wrong, drag it onto what you meant with the button still held; the click happens where you let go. To drag something, hold the press still for half a second first (`POINTER_GAZE_HOLD`), then move. Outside games the pointer stays on while gaze mode is on. The dot only shows while the mouse moves it, while a press is held, and as a pulse when you click.
|
||||
- **Learning from nudges:** if the mouse took the pointer from the gaze and moved it a little (0.2 to 8 degrees) before you clicked, or you dragged a held press that far, you were nudging it onto what you looked at. The helper sends that as a lesson, from the raw gaze when the mouse took over to where you clicked, and ft-gazed learns it. So using it is what calibrates it. The raw gaze is one ft-gazed sent, so it also finds when that look was, and what each eye read then. With SteamVR, each eye learns its own error. With our tracker, the look goes to it as a click, like the probe's, and it relearns how the headset sits on your face. After the headset was off, your first nudge and click there resets that (the probe's one-dot check does the same). The helper only sends nudges up to `POINTER_GAZE_NUDGE_MAX` (8 degrees), and right after putting the headset back on our tracker can be further off than that. If so, raise it for a moment, or do the probe's check. One lesson moves the whole correction by only a third of what it measured (more near where it was taken), since in the first live test one 6 degree lesson moved everything and put the next target 7 degrees off. `ft-gazectl status` shows the lessons, and `ft-gazectl forget` drops them.
|
||||
- Nothing writes to SteamVR, its eye tracker, or its files: ft-gaze maps the eye tracker's shared memory read-only. With no fresh gaze (a blink, the service stopped, the headset off), the pointer stays where it is, and the mouse works as always.
|
||||
- **Idle while the gaze isn't used:** ft-gaze and our own tracker run only while gaze mode is on and someone wears the headset (the pointer helper says both: SteamVR drops the headset's activity level as soon as it comes off), while a check or the calibration is open or asked for, or while the Gaze page of Frametop Input Settings is open (it renews a `wake` lease). 30 seconds after the last use they stop, and our frame grabber goes idle with our tracker. With our tracker, that saves over half a core: on 2026-10-02, with gaze mode off, ft-eyes took about 60% of a core, and ft-eyegrab, ft-gaze and ft-gazed 3 to 4% each. A check asked for while it idles starts the tracker and opens once it sends. When the gaze is used again, it takes a few seconds to come back, and our tracker's first click re-seats it, as after the headset was off: so the quick check opens when gaze mode comes on after the service idled, as it does when you put the headset on. `ft-gazectl status` says `"awake"`, and `"idle"` says why it isn't. A stand that covers the proximity sensor makes the headset seem worn, so with gaze mode on it doesn't idle there.
|
||||
|
||||
Lessons are logged to `pointer-lessons.jsonl`: the raw gaze, the true direction, the correction at the time, and how far off it was.
|
||||
|
||||
@@ -51,7 +40,7 @@ Lessons are logged to `pointer-lessons.jsonl`: the raw gaze, the true direction,
|
||||
| SteamVR action | An `eyetracking` action bound to `/user/head/eyetracking` (`actions/`), read with `IVRInput::GetEyeTrackingDataRelativeToNow`. This is the supported way. |
|
||||
| mmap set 1, set 2 | `/dev/shm/eye-server.mmap`, which SteamVR's eyetracking process writes for the HMD driver (`driver_cv.so`). It has two sets of per-eye directions in head space: set 1 is filtered, and its two eyes always share one pitch; set 2 is each eye's own reading. After each set come the tracker's variances for each eye, and at the end each eye's raw measurement and its variance (the tracker's confidence in that frame), which ft-gaze passes on for the fit check. |
|
||||
| Left eye, right eye | Each eye alone, from set 2: calibrate and test them to see what one eye is worth against both. The layout is undocumented (offsets are in `ft-gaze.cpp`) and may change with a SteamVR update. ft-gaze maps it read-only; the file also carries calibration clicks to the tracker and must never be written. |
|
||||
| Own tracker | Our own tracker (`tracker/`, experimental; see "Our own eye tracker"). It keeps its own calibration, not SteamVR's: Calibrate on the Gaze page fits it while Eye tracker is Own tracker (eight dots to a ring instead of six, on a slight oval, since the fit goes wrong past its dots; the probe's calibration with its tracker toggle on Own tracker does the same), and clicks teach it how far the headset has moved on your face since. After the headset was off, one look at a centre dot (the quick check, or the probe's first dot) resets that. With Own tracker on, the probe hides SteamVR's gaze and draws a red dot where each eye alone puts it, and asks the gaze service to keep the tracker running. The gaze pointer can use it too (Eye tracker: Own tracker, on the Gaze page of Frametop Input Settings). ft-gaze reports it as `own` while it's running, and as `{"ok":0}` otherwise. |
|
||||
| Own tracker | Our own tracker (`tracker/`, experimental; see "Our own eye tracker"). It keeps its own calibration, not SteamVR's: the probe's calibration with the tracker toggle on Own tracker fits it (its dots go out to the Calibration ring angle each way, on an oval, since the fit goes wrong past its dots), and practice clicks teach it how far the headset has moved on your face since. After the headset was off, the probe first asks for one look at a centre dot, which resets that. With Own tracker on, the probe hides SteamVR's gaze and draws a red dot where each eye alone puts it, and asks the gaze service to keep the tracker running. The gaze pointer can use it too (Eye tracker: Own tracker, on the Gaze page of Frametop Input Settings). ft-gaze reports it as `own` while it's running, and as `{"ok":0}` otherwise. |
|
||||
|
||||
The tracker stops when the headset is off your head. SteamVR also calibrates gaze on its own from laser-mouse clicks, treating each click as a spot you were looking at. That includes mouse clicks through the Frametop pointer, so a click where the pointer's dot isn't what you're looking at teaches SteamVR a wrong sample (it only takes clicks within 5 degrees of your gaze). In the probe, use Enter or Space as the trigger: keys don't go through SteamVR's laser. See `Accept usercal` in `~/.local/share/Steam/logs/eyetracking.txt`. When the tracker loses an eye, the same log says `CEyePoseUKF L: Large dt` (or `R`) as it starts that eye over.
|
||||
|
||||
@@ -59,32 +48,25 @@ The tracker stops when the headset is off your head. SteamVR also calibrates gaz
|
||||
|
||||
`gaze/tracker/` is an eye tracker of our own, because SteamVR's is about 1.5 degrees off after the best correction the gaze service can learn, and what's left is mostly look-to-look noise that no correction on top of its output can remove. Ours processes the eye cameras itself: 0.59 degrees (median) in its best live session against 0.83 for SteamVR's with the probe's correction, and after the headset was taken off and put back without recalibrating, 0.58 once your first clicks had taught it where the headset sat (`tracker/findings.md` has the measurements).
|
||||
|
||||
- `ft-eyegrab` (C, root, the system service `frametop-eyegrab.service`) copies the eye-camera frames (512x400, 90 fps per eye, or 15 without `gaze/eye-rate.py on`) out of the DMA-BUFs SteamVR's `eyetracking` process holds into `/dev/shm/frametop-eyes-cams`, owned by you. It maps them read-only, and it only copies while someone touches `/dev/shm/frametop-eyes-want` (ft-eyes and the recorder do, every second). Otherwise it holds none of the tracker's buffers. Its unit keeps only the capabilities that needs (`CAP_SYS_PTRACE`, `CAP_DAC_READ_SEARCH`, `CAP_CHOWN`). `gaze/tracker/install.sh` builds it and installs it to `/etc/frametop` with sudo, which it asks for (`uninstall`, `status`, and `log` too). It also adds it to SteamOS's update keep list (`/etc/atomic-update.conf.d/frametop-eyegrab.conf`): an update deletes `/etc` files that list doesn't name, and without the grabber gaze mode falls back to SteamVR's tracker and its calibration. `scripts/doctor.sh` reports a grabber an update deleted.
|
||||
- `ft-eyes` (Python with numpy and OpenCV, in the dev container: `gaze/tracker/build.sh` puts the pinned `requirements.txt` in `gaze/tracker/build/venv`) finds each eye's pupil (dark threshold, closing, ellipse fit) and glint pair (`eyes_pupil.py`), and maps them to a gaze with a quadratic fit per eye (`eyes_model.py`). It follows the headset moving on your face with a per-eye shift, which your clicks teach, and uses the glints only to notice a sudden jump. It publishes the gaze in `/dev/shm/frametop-eyes-gaze` (ft-gaze's source `own`) and takes calibration dots and clicks on `@ft_eyes`. The gaze service runs it while Eye tracker is Own tracker, or while the probe uses it. State (the calibration, each eye's shift, the clicks) is in `~/.local/state/frametop/gaze/eyes/`. Instead of the pupil finder it can run an eye net (`eyes_net.py`): `EYES_MODEL` is `classic` (the default), `latest`, or a version from `gaze/tracker/models.json`, picked with its notes and benchmarks on Input Settings' Gaze page, which downloads it into `gaze/tracker/models/NAME` (`models/README.md`). Each net keeps its own calibration in `eyes/net-NAME/`.
|
||||
- `lab/` has the tools for improving it on recordings. `ft-eyes-record NAME` (or `ft-eyes-session`, with SteamVR's gaze alongside) records the cameras. `ft-eyes-score` fits and scores on recordings against the probe's practice clicks. `ft-eyes-e2e` runs the whole live path on two recordings (calibrate on one, click through the other). `ft-eyes-replay` plays a recording into a scratch share. Heavy ones are meant for a PC: if you have `frame-job` (a personal tool, not in this repo), `gaze/tracker/.frame-job` sends them there. `lab/py` runs them with that Python (in the dev container on the Frame; on a PC, the same venv from `requirements.txt`, which frame-job's setup makes).
|
||||
- `ft-eyegrab` (C, root, the system service `frametop-eyegrab.service`) copies the eye-camera frames (512x400, 90 fps per eye) out of the DMA-BUFs SteamVR's `eyetracking` process holds into `/dev/shm/frametop-eyes-cams`, owned by you. It maps them read-only, and it only copies while someone touches `/dev/shm/frametop-eyes-want` (ft-eyes and the recorder do, every second). Otherwise it holds none of the tracker's buffers. Its unit keeps only the capabilities that needs (`CAP_SYS_PTRACE`, `CAP_DAC_READ_SEARCH`, `CAP_CHOWN`). `gaze/tracker/install.sh` builds it and installs it to `/etc/frametop` with sudo, which it asks for (`uninstall`, `status`, and `log` too).
|
||||
- `ft-eyes` (Python with numpy and OpenCV, in the dev container: `gaze/tracker/build.sh` puts the pinned `requirements.txt` in `gaze/tracker/build/venv`) finds each eye's pupil (dark threshold, closing, ellipse fit) and glint pair (`eyes_pupil.py`), and maps them to a gaze with a quadratic fit per eye (`eyes_model.py`). It follows the headset moving on your face with a per-eye shift, which your clicks teach, and uses the glints only to notice a sudden jump. It publishes the gaze in `/dev/shm/frametop-eyes-gaze` (ft-gaze's source `own`) and takes calibration dots and clicks on `@ft_eyes`. The gaze service runs it while Eye tracker is Own tracker, or while the probe uses it. State (the calibration, each eye's shift, the clicks) is in `~/.local/state/frametop/gaze/eyes/`.
|
||||
- `lab/` has the tools for improving it on recordings. `ft-eyes-record NAME` (or `ft-eyes-session`, with SteamVR's gaze alongside) records the cameras. `ft-eyes-score` fits and scores on recordings against the probe's practice clicks. `ft-eyes-e2e` runs the whole live path on two recordings (calibrate on one, click through the other). `ft-eyes-replay` plays a recording into a scratch share. Heavy ones run on a PC through `frame-job` (`gaze/tracker/.frame-job`). `lab/py` runs them with that Python (in the dev container on the Frame; frame-job's setup makes the same venv on the PC).
|
||||
|
||||
Ground rules, for anyone changing it:
|
||||
|
||||
- **Clean room.** Nothing of Valve's goes in: we don't decompile, disassemble, or patch the `eyetracking` binary or its network weights, and we don't copy their code or weights. Its public output (eye-server.mmap, read-only) is fair game as a baseline and as labels, and so are published papers and openly licensed pupil detectors (check each one's license: PuRe, PuReST, ElSe, and ExCuSe are non-commercial only).
|
||||
- **Root only reads.** ft-eyegrab never writes to, stops, or signals the `eyetracking` process, vrserver, or vrcompositor, never opens `/dev/adsp`, `/dev/cdsp`, or `/dev/spidev0.1`, and never writes to `/dev/shm/eye-server.mmap` (it also carries calibration clicks into SteamVR's tracker), `/opt`, or `/persist`.
|
||||
- **Eye images are biometric data.** Recordings live outside the repo, in `~/.local/share/frametop/eyes/captures` (0700), and `.gitignore` catches stray frame dumps. They go nowhere but the machine that runs your offline jobs.
|
||||
- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. ft-eyes keeps OpenCV and numpy to one thread: their pools of one per core spun idle workers at about a quarter of a core, for frames this small. It also runs at nice 10 with SCHED_BATCH, and ft-gaze at nice 5 (not batch, since each sample goes on to the pointer): both run in the dev container's podman scope, out of reach of the gaze service's unit, on the cores vrcompositor and vrserver use at nice 0. Replays, scoring, and training go to a PC.
|
||||
- **Mind the headset's budget.** Finding a pupil takes about 0.4 ms a frame while ft-eyes follows it, and 1.4-2.1 ms when it searches the whole frame. Replays, scoring, and training go to a PC.
|
||||
|
||||
## Headset fit
|
||||
|
||||
Check headset fit on the Gaze page opens it in the headset panel: a card per eye (tracked or lost, the tracker's signal, how much of the last 10 s it was seen) and the hints, live while you adjust the headset. A left click or Meta+J runs the guided check (dots, then looks down, up, left and right), and a right click or Meta+K closes it. The probe's Headset fit mode (`ft-gazeprobe --mode fit`, for development) has the same check with maps: it shows, for each eye, whether the tracker has it, how open it is, and the tracker's confidence in it, and a map of where you looked coloured by how often it lost that eye there. Hints under the maps say which eye gets lost where, and what to try. Enter runs a guided check: dots around the screen, then looking down at the keyboard, up, left and right. R starts over. Adjust the headset while you watch it.
|
||||
|
||||
**The fit watch** (`GAZE_FIT_WATCH=1` in `~/.config/frametop.conf`; off by default until it's been tried in the headset). The fit check keeps a good fit as yours: when its coach (`gaze/fitcoach.py`, from ft-eyes' pupils and glints, with the shipped reference) last read it as good while you looked straight ahead, closing the check saves that reading to `~/.local/state/frametop/gaze/fit-baseline.json`, with the IPD setting. The eye recorder's fitment step saves it too. Then, with the fit watch on:
|
||||
- **Put-on check:** when the headset goes on, the quick check first holds its dot for up to 4 s while the coach reads the fit against your saved one. If it sits the same, the dot is taken as usual. If it moved (5 degrees of gaze or more), the panel widens and says which way to move the headset, and a changed IPD over that, live, until it's back ("Good: it's back where it was"); then the dot is taken. A click takes the dot as the headset sits, and after 30 s it's taken anyway. It needs our tracker installed (ft-eyes runs for it with either tracker in use), and without a saved fit it's the plain quick check.
|
||||
- **Slip note:** with our tracker in use and gaze mode on, ft-eyes' glint slip (its `jump_px`, how far each pupil moved in its image since the last click) is turned into degrees through the reference. Past 5 degrees for 2 s, a small strip low in your view says "Your headset slipped" and which way to move it, for 8 s or until the slip is gone, at most once every 2 minutes. It isn't a check: the pointer stays, and a click or the quick check still teaches the tracker the new seating.
|
||||
`gaze/test/fit-watch-test.py` tests both offline.
|
||||
The probe's Headset fit mode (Check headset fit on the Gaze page, or `ft-gazeprobe --mode fit`) shows, for each eye, whether the tracker has it, how open it is, and the tracker's confidence in it, and a map of where you looked coloured by how often it lost that eye there. Hints under the maps say which eye gets lost where, and what to try. Enter runs a guided check: dots around the screen, then looking down at the keyboard, up, left and right. R starts over. Adjust the headset while you watch it.
|
||||
|
||||
Losing an eye is usually about where you look, not the tracker. On this Frame the left eye was lost 57 to 64 % of the time looking 30 to 50 degrees down (at the keyboard) and the right eye never; at screen height both were seen over 98 % of the time. Looking down, the lids come down over the eyes. That's harmless, since the gaze service ignores looks down past the screens: they show on the maps, but not in the counts or as a problem.
|
||||
|
||||
## Probe
|
||||
|
||||
The probe is a development tool (in the Gaze page's overflow menu): calibration experiments, accuracy tests, and practice modes. Users calibrate and check in the headset panel instead.
|
||||
|
||||
The trigger is Enter, Space, or a mouse button. Right-click anywhere in the window (or press the Menu key or Shift+F10) for a menu with Run calibration, Start accuracy test, Calibrate from last test, Reset calibration, the modes, the panel, fullscreen, and Quit. The buttons at the top right show and hide the panel, leave fullscreen, and quit. The arrow in the panel's title bar collapses it to just that bar, so the dot and targets behind it stay visible; the collapsed bar stays through tests. The keys do the same (Tab, C, F11, Esc), but only after you click the window once, since Frametop sends typing to the panel you clicked last. If ft-gaze stops, the probe starts it again after 3 s and shows why it stopped. Windowed mode stays on the screen it was on, and a small KWin script tells the probe where the window is, so the dot and targets are still in the right place.
|
||||
|
||||
- **Run calibration (start here):** the initial calibration, modeled on Apple Vision Pro's eye setup. Face the centre and keep your head still. Look at one dot and press the trigger, then at each of six dots in a circle. That happens in three rounds, and the screen goes dark, then medium, then bright, because pupil size changes with brightness and the tracker's error with it. Each round turns the ring 20 degrees, and the middle round's ring is half the size, so the 21 dots cover the middle, halfway out, and the edge of your view. The ring's size is `Calibration ring` (degrees, 20 by default, less if the window is too small). Error grows toward the edge, and the calibration can only correct as far out as it has seen dots. The current dot is a bright pulsing dot with a point in the middle; finished dots fade to specks, so your eyes don't go back to them. Samples from blinks and from moments when the tracker lost an eye are dropped: openness under half of what it was during that look (not a fixed level, because your lids come down when you look down, and you squint in the bright round), or the angle between the eyes jumping more than 1.5 degrees from its median (that angle depends on how far away you're looking, so only a jump counts). Each dot is measured with medians, so one bad sample can't fail it. A look that lands where the gaze was for another dot of the round is refused as a look at the wrong dot. Mouse clicks don't count during a calibration run or a test: use Enter or Space. If a dot still fails, the message says why and the next try listens longer. After two failures, S (or the menu) skips the dot. Every attempt is logged to `calibration-attempts.jsonl`. At the end it fits every source's calibration from all the dots, replacing what it had learned (quadratic if the model was none). Esc cancels. The run is saved as `calibration-*.json`. With "Test after calibration" on (the default), the accuracy test starts right after, on new spots.
|
||||
|
||||
+6
-12
@@ -1,19 +1,13 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build ft-gaze and the calibration panel ft-gazepanel in the dev container on the Frame
|
||||
# (gaze/build/; they also run there). The panel draws its text with stb_truetype (public
|
||||
# domain, one header, pinned as in screens/build.sh).
|
||||
# Build ft-gaze in the dev container on the Frame (gaze/build/ft-gaze; it also runs there).
|
||||
# The eye tracking API (IVRInput::GetEyeTrackingDataRelativeToNow) is newer than the header
|
||||
# shipped with SteamVR's samples, so this uses the pinned public header (scripts/openvr.sh).
|
||||
# shipped with SteamVR's samples, so this uses the pinned public header ft-screens fetches.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
"$root/scripts/sync.sh" >/dev/null
|
||||
exec "$root/scripts/frame.sh" -C gaze 'set -e; mkdir -p build/include
|
||||
. ../scripts/openvr.sh
|
||||
openvr=v2.15.6
|
||||
[ -f build/include/openvr-$openvr ] || { curl -fsSL "https://raw.githubusercontent.com/ValveSoftware/openvr/$openvr/headers/openvr.h" -o build/include/openvr.h && touch build/include/openvr-$openvr; }
|
||||
g++ -std=c++17 -O2 -Wall -Wno-unused-parameter -Wno-missing-field-initializers -Ibuild/include -I../pointer/common \
|
||||
-o build/ft-gaze ft-gaze.cpp $OPENVR_LIBS -lpthread
|
||||
stb=2c980bb59875b0d32144a71867fbdebb2f77cd20
|
||||
[ -f build/include/stb-$stb ] || { curl -fsSL "https://raw.githubusercontent.com/nothings/stb/$stb/stb_truetype.h" -o build/include/stb_truetype.h && touch build/include/stb-$stb; }
|
||||
g++ -std=c++17 -O2 -Wall -Wno-unused-parameter -Wno-missing-field-initializers -Ibuild/include $(pkg-config --cflags gbm libdrm) \
|
||||
-o build/ft-gazepanel panel/ft-gazepanel.cpp $OPENVR_LIBS \
|
||||
$(pkg-config --libs gbm libdrm) -lpthread
|
||||
echo "built build/ft-gaze build/ft-gazepanel"'
|
||||
-o build/ft-gaze ft-gaze.cpp -L/opt/steamvr/bin/linuxarm64 -lopenvr_api -Wl,-rpath,/opt/steamvr/bin/linuxarm64 -lpthread
|
||||
echo "built build/ft-gaze"'
|
||||
@@ -1,141 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Keep SteamVR's eye cameras at 90 fps, for gaze mode.
|
||||
|
||||
The Frame's SteamVR driver runs the eye cameras at 15 to 90 fps (steamvr.eyeTrackingRateMin
|
||||
and eyeTrackingRateMax, in drivers/frame_hmd/resources/frame_hmd_additional.vrsettings),
|
||||
and asks for the rate once, as SteamVR starts (vrserver.txt: "cv: Request eye tracking
|
||||
framerate"). With its defaults it asks for 15 (eyetracking.txt: "Set framerate 15"). SteamVR's
|
||||
own tracker copes, but our tracker gets 8 or 9 frames in a calibration dot's 0.6 s, where it
|
||||
needs 15, so it can never calibrate, and the gaze pointer would move 15 times a second.
|
||||
With steamvr.gazeDepEyeToHeadEnable on, SteamVR asks for 90 (seen 2026-10-10: a headset with
|
||||
it on asks for 90 at every start; a user's with it off, at 15, calibrated once it was on).
|
||||
SteamVR's settings have no switch for it.
|
||||
|
||||
gaze/eye-rate.py on # turn it on (gaze/run.sh install); it counts from SteamVR's next start
|
||||
gaze/eye-rate.py off # turn it off again, if this turned it on (gaze/run.sh uninstall)
|
||||
gaze/eye-rate.py status # the setting, and the rate the eye cameras run at now
|
||||
|
||||
With SteamVR running it goes through vrcmd, since SteamVR keeps its settings in memory and
|
||||
writes over the file; without it, it edits SteamVR's settings file.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
HOME = Path.home()
|
||||
SECTION, KEY = "steamvr", "gazeDepEyeToHeadEnable"
|
||||
STEAMVR_BIN = "/opt/steamvr/bin/linuxarm64"
|
||||
# The first that exists is SteamVR's (as gaze/gazecal.py reads them).
|
||||
STEAMVR_SETTINGS = (HOME / ".config/openvr/config/steamvr.vrsettings",
|
||||
HOME / ".steam/steam/config/steamvr.vrsettings")
|
||||
# There when this turned the setting on (it was off), so `off` leaves a setting someone chose alone.
|
||||
MARK = HOME / ".local/state/frametop/gaze/steamvr-eye-rate"
|
||||
EYE_LOG = HOME / ".local/share/Steam/logs/eyetracking.txt"
|
||||
# A terminal in the desktop has the desktop's own config folder, where SteamVR's path registry
|
||||
# isn't (see scripts/update-check.py): vrcmd needs the real one.
|
||||
ENV = dict(os.environ, XDG_CONFIG_HOME=str(HOME / ".config"), LD_LIBRARY_PATH=STEAMVR_BIN)
|
||||
|
||||
|
||||
def steamvr_up():
|
||||
return subprocess.run(["pgrep", "-x", "vrserver"], capture_output=True).returncode == 0
|
||||
|
||||
|
||||
def vrcmd(*args):
|
||||
p = subprocess.run([f"{STEAMVR_BIN}/vrcmd", *args], capture_output=True, text=True, timeout=15, env=ENV)
|
||||
return p.stdout
|
||||
|
||||
|
||||
def settings_file():
|
||||
return next((p for p in STEAMVR_SETTINGS if p.exists()), STEAMVR_SETTINGS[0])
|
||||
|
||||
|
||||
def get():
|
||||
"""The setting: True, False, or None when SteamVR isn't running and its file doesn't say
|
||||
(then the driver's default, off)."""
|
||||
if steamvr_up():
|
||||
# vrcmd prints "steamvr.gazeDepEyeToHeadEnable=true"
|
||||
m = re.search(rf"^{SECTION}\.{KEY}=(\w+)", vrcmd("--settings-bool", f"{SECTION}.{KEY}"), re.M)
|
||||
return None if not m else m.group(1) in ("true", "1")
|
||||
try:
|
||||
value = json.loads(settings_file().read_text()).get(SECTION, {}).get(KEY)
|
||||
except (OSError, ValueError, AttributeError):
|
||||
return None
|
||||
return value if isinstance(value, bool) else None
|
||||
|
||||
|
||||
def put(on):
|
||||
if steamvr_up():
|
||||
# 1 or 0: vrcmd reads "true" as 0.
|
||||
vrcmd("--set-settings-bool", f"{SECTION}.{KEY}", "1" if on else "0")
|
||||
return get() is on
|
||||
path = settings_file()
|
||||
try:
|
||||
d = json.loads(path.read_text())
|
||||
except FileNotFoundError:
|
||||
d = {}
|
||||
except (OSError, ValueError) as e:
|
||||
print(f"can't read {path}: {e}", file=sys.stderr)
|
||||
return False
|
||||
d.setdefault(SECTION, {})[KEY] = on
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".frametop-tmp")
|
||||
tmp.write_text(json.dumps(d, indent=3) + "\n")
|
||||
tmp.replace(path)
|
||||
return True
|
||||
|
||||
|
||||
def camera_rate():
|
||||
"""The rate the eye cameras run at since the eye tracker last started: the last "Set
|
||||
framerate" after its last start, or 90 when it set none (it starts at 90). None without
|
||||
the log or a start in it."""
|
||||
try:
|
||||
lines = EYE_LOG.read_text(errors="replace").splitlines()
|
||||
except OSError:
|
||||
return None
|
||||
starts = [i for i, line in enumerate(lines) if " startup with PID" in line]
|
||||
if not starts:
|
||||
return None
|
||||
rates = re.findall(r"CStereoAdspCams: Set framerate ([\d.]+)", "\n".join(lines[starts[-1]:]))
|
||||
return float(rates[-1]) if rates else 90.0
|
||||
|
||||
|
||||
def main():
|
||||
what = sys.argv[1] if len(sys.argv) == 2 else ""
|
||||
if what == "on":
|
||||
if get():
|
||||
print(f"SteamVR's eye cameras: {SECTION}.{KEY} is on already, so they run at 90 fps")
|
||||
return 0
|
||||
if not put(True):
|
||||
print(f"couldn't turn on {SECTION}.{KEY}: the eye cameras may stay at 15 fps, and our eye "
|
||||
"tracker can't calibrate at that. Try again with: gaze/eye-rate.py on", file=sys.stderr)
|
||||
return 1
|
||||
MARK.parent.mkdir(parents=True, exist_ok=True)
|
||||
MARK.write_text(f"{SECTION}.{KEY} was off; Frametop turned it on\n")
|
||||
print(f"SteamVR's eye cameras: turned on {SECTION}.{KEY}, so they run at 90 fps, not 15, "
|
||||
"once SteamVR restarts")
|
||||
return 0
|
||||
if what == "off":
|
||||
if not MARK.exists():
|
||||
print(f"SteamVR's eye cameras: left as they are (Frametop didn't turn on {SECTION}.{KEY})")
|
||||
return 0
|
||||
if not put(False):
|
||||
print(f"couldn't turn off {SECTION}.{KEY}. Try again with: gaze/eye-rate.py off", file=sys.stderr)
|
||||
return 1
|
||||
MARK.unlink()
|
||||
print(f"SteamVR's eye cameras: turned {SECTION}.{KEY} off again (back to 15 fps once SteamVR restarts)")
|
||||
return 0
|
||||
if what == "status":
|
||||
value = get()
|
||||
rate = camera_rate()
|
||||
print(f"{SECTION}.{KEY}: {'on' if value else 'off'}" + (" (set by Frametop)" if MARK.exists() else ""))
|
||||
print("eye cameras: " + (f"{rate:g} fps since the eye tracker started" if rate else "unknown (no eyetracking.txt)"))
|
||||
return 0
|
||||
print("usage: gaze/eye-rate.py on|off|status", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,46 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"made": "2026-10-14",
|
||||
"source": "the maintainer's headset: 2 of 3 own-tracker calibrations (2026-10-08, 2026-10-09), the third fitted worse than 4 px",
|
||||
"faces": 1,
|
||||
"ipd": 0.0651,
|
||||
"cameras_follow_lenses": null,
|
||||
"eyes": {
|
||||
"0": {
|
||||
"centre": [
|
||||
341.04,
|
||||
221.4
|
||||
],
|
||||
"jacobian": [
|
||||
[
|
||||
1.2749,
|
||||
-0.4221
|
||||
],
|
||||
[
|
||||
-0.4783,
|
||||
-1.3388
|
||||
]
|
||||
],
|
||||
"fits": 2,
|
||||
"fit_err": 2.339
|
||||
},
|
||||
"1": {
|
||||
"centre": [
|
||||
334.74,
|
||||
173.53
|
||||
],
|
||||
"jacobian": [
|
||||
[
|
||||
-1.3415,
|
||||
-0.5307
|
||||
],
|
||||
[
|
||||
-0.4822,
|
||||
1.4794
|
||||
]
|
||||
],
|
||||
"fits": 2,
|
||||
"fit_err": 2.146
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
-19
@@ -1,5 +1,5 @@
|
||||
"""fitcheck: how well the eye tracker sees each eye, for fitting the headset (the headset
|
||||
panel's fit check, gazecheck.py, and ft-gazeprobe's Headset fit mode).
|
||||
"""fitcheck: how well the eye tracker sees each eye, for fitting the headset (ft-gazeprobe's
|
||||
Headset fit mode).
|
||||
|
||||
From each ft-gaze sample it takes, per eye, whether the tracker has that eye (its variance
|
||||
for the eye's direction, "unc", under EYE_LOST; see gazecal), how open the eye is, and the
|
||||
@@ -51,10 +51,7 @@ GUIDE = [
|
||||
|
||||
|
||||
class FitCheck:
|
||||
def __init__(self, ignore=None):
|
||||
# The eye SteamVR's tracker ignores (0 left, 1 right) with Track Dominant Eye Only on
|
||||
# (gazecal.tracked_eye), or None: its losses say nothing about the fit.
|
||||
self.ignore = ignore
|
||||
def __init__(self):
|
||||
self.reset()
|
||||
|
||||
def reset(self):
|
||||
@@ -102,8 +99,7 @@ class FitCheck:
|
||||
if q and (eye.get("new") or [1, 1])[k]:
|
||||
self.q[k].append(q[k])
|
||||
closed = [opens[k] < CLOSED for k in (0, 1)]
|
||||
judged = [k for k in (0, 1) if k != self.ignore]
|
||||
if all(closed[k] for k in judged) or all(self.lost[k] for k in judged):
|
||||
if all(closed) or all(self.lost):
|
||||
return # a blink: says nothing about the fit
|
||||
key = (math.floor(hy / CELL), math.floor(hp / CELL))
|
||||
for k in (0, 1):
|
||||
@@ -147,8 +143,6 @@ class FitCheck:
|
||||
# --- Summaries ---
|
||||
|
||||
def status(self, k):
|
||||
if k == self.ignore:
|
||||
return "not tracked", (0.6, 0.6, 0.6)
|
||||
if not self.samples:
|
||||
return "no data", (0.6, 0.6, 0.6)
|
||||
if self.lost[k]:
|
||||
@@ -180,13 +174,8 @@ class FitCheck:
|
||||
return ["Look around slowly (the screen's corners, then down at your keyboard, up, left and right) "
|
||||
"or press Enter for a guided check."]
|
||||
out = []
|
||||
if self.ignore is not None:
|
||||
out.append(f"SteamVR tracks only your {EYES[1 - self.ignore].lower()} (Track Dominant Eye Only in "
|
||||
f"SteamVR's settings), so your {EYES[self.ignore].lower()} doesn't count here.")
|
||||
bad = {}
|
||||
for k in (0, 1):
|
||||
if k == self.ignore:
|
||||
continue
|
||||
for key, words, _ in REGIONS:
|
||||
share = self.region_share(k, key)
|
||||
if share is not None and share >= 0.15:
|
||||
@@ -208,7 +197,7 @@ class FitCheck:
|
||||
"face, not one eye's fit.")
|
||||
continue
|
||||
k = next(iter(eyes))
|
||||
other = self.region_share(1 - k, key) if 1 - k != self.ignore else None
|
||||
other = self.region_share(1 - k, key)
|
||||
vs = f", the {EYES[1 - k].lower()} {other:.0%}" if other is not None else ""
|
||||
line = f"{EYES[k]}: lost {eyes[k]:.0%} of the time looking {words}{vs}."
|
||||
if key == "down":
|
||||
@@ -222,12 +211,10 @@ class FitCheck:
|
||||
"eyes.")
|
||||
out.append(line)
|
||||
s0, s1 = self.signal(0), self.signal(1)
|
||||
if self.ignore is None and s0 is not None and s1 is not None and abs(s0 - s1) > 0.25:
|
||||
if s0 is not None and s1 is not None and abs(s0 - s1) > 0.25:
|
||||
k = 0 if s0 < s1 else 1
|
||||
out.append(f"The tracker is less sure of your {EYES[k].lower()} even when it has it "
|
||||
f"(signal {min(s0, s1):.0%} against {max(s0, s1):.0%}).")
|
||||
if self.ignore is not None and len(out) == 1:
|
||||
out.append(f"Your {EYES[1 - self.ignore].lower()} is tracked everywhere you've looked so far.")
|
||||
if not out:
|
||||
out.append("Both eyes are tracked everywhere you've looked so far.")
|
||||
return out
|
||||
|
||||
@@ -1,521 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""fitcoach: how the headset sits on this face, from where our tracker sees each eye, and what
|
||||
to do about it (gaze/rec/DESIGN.md, "Fit coach"). The eye recorder runs it before each round;
|
||||
the gaze service's fit check, put-on check and slip notice use it too (gaze/gazecheck.py).
|
||||
|
||||
It takes per-frame pupil finds (gaze/tracker/eyes_pupil.find_pupil: centre, axes, glints) for
|
||||
each camera (0 the right eye, 1 the left), keeps the last second, and compares the medians
|
||||
with a reference: where a well-fitted eye sits in that camera's image while looking straight
|
||||
ahead, and how the pupil moves per degree of gaze (the reference's Jacobian, fitted on
|
||||
calibration dots). Offsets go through that Jacobian into degrees as if the eye had turned, so
|
||||
the cameras' orientations don't matter (camera 1's image is turned 180 degrees from camera 0's:
|
||||
looking left moves its pupil the other way) and both eyes' readings can be added up:
|
||||
|
||||
both eyes appear to look up the eyes sit high in the headset: it's too low -> push it up
|
||||
both appear to look left the headset sits off to the right -> slide it left
|
||||
the eyes appear to look apart the eyes are wider apart than the lenses -> IPD wider
|
||||
one appears to look higher the headset is tilted -> level it
|
||||
|
||||
The IPD reading only becomes a hint once we know the eye cameras move with the lenses
|
||||
(HEADSET-TESTS, 2026-10-14): if they're fixed to the housing, the offset also depends on the IPD
|
||||
setting, and that needs a reference per setting. Until then it's reported, not advised.
|
||||
|
||||
A baseline is the same reading saved from one person's good fit. Compared with a baseline the
|
||||
glint pair's spacing says whether the eye is nearer the camera than then (the eye relief: on
|
||||
the maintainer's headset it moved 18 % between two seatings, and 4 % within one), and the put-on
|
||||
check says how far the headset moved since. The reference starts as the maintainer's
|
||||
(gaze/fit-reference.json); its hints are marked provisional until it's built from more faces.
|
||||
|
||||
Measured on the maintainer's headset (2026-10-14, three calibrations of 27 dots): straight-ahead
|
||||
pupils moved 15-20 px between seatings, at about 1.3 px per degree; so OK within OK_DEG and
|
||||
adjust past ADJUST_DEG, both provisional.
|
||||
|
||||
Usage (on the Frame, in the dev container, with frametop-eyegrab running):
|
||||
python3 gaze/fitcoach.py live [--baseline] print the reading twice a second; --baseline saves
|
||||
the last one as your baseline on Ctrl-C
|
||||
python3 gaze/fitcoach.py reference [DOTS...] fit a reference from calibration-dots.jsonl files
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REFERENCE = os.path.join(HERE, "fit-reference.json")
|
||||
BASELINE = os.path.expanduser("~/.local/state/frametop/gaze/fit-baseline.json")
|
||||
DOTS = os.path.expanduser("~/.local/state/frametop/gaze/eyes/calibration-dots.jsonl")
|
||||
EYES = ("right", "left") # camera 0, camera 1
|
||||
WINDOW = 1.0 # s of finds per reading
|
||||
MIN_FINDS = 10 # finds in the window before an eye is judged
|
||||
MIN_FOUND = 0.5 # share of frames with a pupil: under this, the eye is lost
|
||||
OK_DEG = 5.0 # every term within this: the fit is good (provisional)
|
||||
ADJUST_DEG = 8.0 # a term past this: say what to do (between the two: keep the last state)
|
||||
TILT_DEG = 4.0 # half the eyes' vertical difference
|
||||
RELIEF_TOL = 0.08 # glint spacing this much off the baseline: nearer or further
|
||||
MANY_GLINTS = 5 # a find with this many glints has stray reflections. Without glasses
|
||||
# (three maintainer captures, 2026-10-14) 3+ glints came in up to 37 % of
|
||||
# finds and 5+ in up to 12 %: a guess until the Glasses bounty's data
|
||||
EXTRA_GLINTS = 0.5 # share of finds with MANY_GLINTS: reflections (glasses, a smudge)
|
||||
MOVED_DEG = 5.0 # put-on check: this far from the baseline, the headset moved; the slip
|
||||
# notice: ft-eyes' glint jump this far
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Readings
|
||||
|
||||
def glint_spacing(p):
|
||||
"""The LED glint pair's spacing in pixels, or None (eyes_pupil.glint_pair's rule)."""
|
||||
pair = _glint_pair(p)
|
||||
if not pair:
|
||||
return None
|
||||
(x0, y0), (x1, y1) = pair
|
||||
return math.hypot(x1 - x0, y1 - y0)
|
||||
|
||||
|
||||
def _glint_pair(p):
|
||||
"""eyes_pupil.glint_pair without numpy: the vertical-ish pair nearest the pupil centre."""
|
||||
g, best = p.get("glints") or [], None
|
||||
for i in range(len(g)):
|
||||
for j in range(i + 1, len(g)):
|
||||
dx, dy = g[j][0] - g[i][0], g[j][1] - g[i][1]
|
||||
d = math.hypot(dx, dy)
|
||||
if not 5 <= d <= 45 or abs(dy) < 2 * abs(dx):
|
||||
continue
|
||||
cost = math.hypot((g[i][0] + g[j][0]) / 2 - p["x"], (g[i][1] + g[j][1]) / 2 - p["y"])
|
||||
if best is None or cost < best[0]:
|
||||
best = (cost, (g[i], g[j]) if dy > 0 else (g[j], g[i]))
|
||||
return best[1] if best else None
|
||||
|
||||
|
||||
class Observations:
|
||||
"""The last WINDOW seconds of pupil finds per camera. add() from any thread."""
|
||||
|
||||
def __init__(self, window=WINDOW):
|
||||
self.window = window
|
||||
self._rows = {0: deque(), 1: deque()} # (t, x, y, a, spacing or None, extra glints) or (t, None)
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add(self, cam, t, p):
|
||||
if p is None:
|
||||
row = (t, None)
|
||||
else:
|
||||
row = (t, float(p["x"]), float(p["y"]), float(p["a"]), glint_spacing(p), len(p.get("glints") or []) >= MANY_GLINTS)
|
||||
with self._lock:
|
||||
rows = self._rows[cam]
|
||||
rows.append(row)
|
||||
while rows and rows[0][0] < t - self.window:
|
||||
rows.popleft()
|
||||
|
||||
def summary(self, now=None):
|
||||
"""{camera: {"n", "found", "x", "y", "a", "spacing", "extra"}}: medians of the finds,
|
||||
found the share of frames with a pupil, extra the share with MANY_GLINTS or more. A camera with
|
||||
no frames in the window: {"n": 0}."""
|
||||
out = {}
|
||||
with self._lock:
|
||||
for cam, rows in self._rows.items():
|
||||
rs = [r for r in rows if now is None or r[0] >= now - self.window]
|
||||
finds = [r for r in rs if r[1] is not None]
|
||||
if not rs:
|
||||
out[cam] = {"n": 0}
|
||||
continue
|
||||
sp = [r[4] for r in finds if r[4] is not None]
|
||||
out[cam] = {"n": len(rs), "finds": len(finds), "found": len(finds) / len(rs),
|
||||
"x": statistics.median(r[1] for r in finds) if finds else None,
|
||||
"y": statistics.median(r[2] for r in finds) if finds else None,
|
||||
"a": statistics.median(r[3] for r in finds) if finds else None,
|
||||
"spacing": statistics.median(sp) if len(sp) >= 3 else None,
|
||||
"extra": sum(r[5] for r in finds) / len(finds) if finds else 0.0}
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The reference and the baseline
|
||||
|
||||
def _solve3(a, b):
|
||||
"""x for a 3x3 system a x = b (Gaussian elimination with pivoting), or None if singular."""
|
||||
m = [row[:] + [v] for row, v in zip(a, b)]
|
||||
for i in range(3):
|
||||
k = max(range(i, 3), key=lambda r: abs(m[r][i]))
|
||||
if abs(m[k][i]) < 1e-12:
|
||||
return None
|
||||
m[i], m[k] = m[k], m[i]
|
||||
for r in range(3):
|
||||
if r != i:
|
||||
f = m[r][i] / m[i][i]
|
||||
m[r] = [x - f * y for x, y in zip(m[r], m[i])]
|
||||
return [m[i][3] / m[i][i] for i in range(3)]
|
||||
|
||||
|
||||
def fit_linear(truths, values):
|
||||
"""value = c0 + c1 yaw + c2 pitch by least squares: [c0, c1, c2], or None."""
|
||||
rows = [(1.0, float(t[0]), float(t[1])) for t in truths]
|
||||
ata = [[sum(r[i] * r[j] for r in rows) for j in range(3)] for i in range(3)]
|
||||
atb = [sum(r[i] * v for r, v in zip(rows, values)) for i in range(3)]
|
||||
return _solve3(ata, atb)
|
||||
|
||||
|
||||
class Reference:
|
||||
"""Per camera: the straight-ahead pupil centre (px) and the Jacobian [[dx/dyaw, dx/dpitch],
|
||||
[dy/dyaw, dy/dpitch]] (px per degree); optionally the glint spacing, the IPD it was made
|
||||
at (m), whether the eye cameras move with the lenses, where it came from and from how many faces."""
|
||||
|
||||
def __init__(self, eyes, ipd=None, source="", faces=1, cameras_follow_lenses=None, made=""):
|
||||
self.eyes, self.ipd, self.source, self.faces = eyes, ipd, source, faces
|
||||
self.cameras_follow_lenses, self.made = cameras_follow_lenses, made
|
||||
|
||||
@property
|
||||
def provisional(self):
|
||||
return self.faces < 3
|
||||
|
||||
@classmethod
|
||||
def from_dots(cls, rows, source="", ipd=None, max_err=4.0):
|
||||
"""From calibration-dots.jsonl rows ({"made", "truth": [yaw, pitch], "eye": {"0": {"pupil": [x, y]}}}).
|
||||
Each calibration ("made") is fitted on its own; those that fit worse than max_err px
|
||||
(median) are left out, and the rest averaged."""
|
||||
fits = {0: [], 1: []}
|
||||
for made in sorted({r.get("made", "") for r in rows}):
|
||||
for cam in (0, 1):
|
||||
rs = [r for r in rows if r.get("made", "") == made and (r.get("eye") or {}).get(str(cam))]
|
||||
if len(rs) < 6:
|
||||
continue
|
||||
truths = [r["truth"] for r in rs]
|
||||
px = [r["eye"][str(cam)]["pupil"] for r in rs]
|
||||
cx, cy = fit_linear(truths, [p[0] for p in px]), fit_linear(truths, [p[1] for p in px])
|
||||
if cx is None or cy is None:
|
||||
continue
|
||||
err = statistics.median(math.hypot(cx[0] + cx[1] * t[0] + cx[2] * t[1] - p[0],
|
||||
cy[0] + cy[1] * t[0] + cy[2] * t[1] - p[1]) for t, p in zip(truths, px))
|
||||
if err <= max_err:
|
||||
fits[cam].append((cx, cy, err))
|
||||
eyes = {}
|
||||
for cam, fs in fits.items():
|
||||
if not fs:
|
||||
continue
|
||||
avg = lambda k, i: sum(f[k][i] for f in fs) / len(fs) # noqa: E731
|
||||
eyes[cam] = {"centre": [avg(0, 0), avg(1, 0)], "jacobian": [[avg(0, 1), avg(0, 2)], [avg(1, 1), avg(1, 2)]],
|
||||
"fits": len(fs), "fit_err": statistics.median(f[2] for f in fs)}
|
||||
return cls(eyes, ipd=ipd, source=source, made=time.strftime("%Y-%m-%d"))
|
||||
|
||||
def degrees(self, cam, dx, dy):
|
||||
"""A pupil offset in px as the gaze change it would be: (yaw, pitch) degrees."""
|
||||
(a, b), (c, d) = self.eyes[cam]["jacobian"]
|
||||
det = a * d - b * c
|
||||
if abs(det) < 1e-9:
|
||||
return 0.0, 0.0
|
||||
return (d * dx - b * dy) / det, (-c * dx + a * dy) / det
|
||||
|
||||
def to_json(self):
|
||||
eyes = {str(k): {"centre": [round(v, 2) for v in e["centre"]],
|
||||
"jacobian": [[round(v, 4) for v in row] for row in e["jacobian"]],
|
||||
**{k2: (round(v2, 3) if isinstance(v2, float) else v2) for k2, v2 in e.items()
|
||||
if k2 not in ("centre", "jacobian")}}
|
||||
for k, e in sorted(self.eyes.items())}
|
||||
return {"schema": 1, "made": self.made, "source": self.source, "faces": self.faces, "ipd": self.ipd,
|
||||
"cameras_follow_lenses": self.cameras_follow_lenses,
|
||||
"eyes": eyes}
|
||||
|
||||
@classmethod
|
||||
def from_json(cls, d):
|
||||
return cls({int(k): v for k, v in d.get("eyes", {}).items()}, ipd=d.get("ipd"), source=d.get("source", ""),
|
||||
faces=d.get("faces", 1), cameras_follow_lenses=d.get("cameras_follow_lenses"), made=d.get("made", ""))
|
||||
|
||||
@classmethod
|
||||
def load(cls, path=REFERENCE):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return cls.from_json(json.load(f))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
def save(self, path=REFERENCE):
|
||||
_write_json(path, self.to_json())
|
||||
|
||||
|
||||
def make_baseline(summary, ipd=None):
|
||||
"""A baseline from a good reading: per camera the centre and glint spacing, and the IPD."""
|
||||
eyes = {str(c): {"centre": [s["x"], s["y"]], "spacing": s.get("spacing"), "a": s.get("a")}
|
||||
for c, s in summary.items() if s.get("x") is not None}
|
||||
return {"schema": 1, "made": time.strftime("%Y-%m-%d %H:%M:%S"), "ipd": ipd, "eyes": eyes}
|
||||
|
||||
|
||||
def load_baseline(path=BASELINE):
|
||||
try:
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def save_baseline(baseline, path=None):
|
||||
"""Keep a good fit as the person's (the gaze service's fit check, the eye recorder's fitment)."""
|
||||
_write_json(path or BASELINE, baseline)
|
||||
|
||||
|
||||
def _write_json(path, obj):
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(obj, f, indent=1)
|
||||
f.write("\n")
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# The assessment
|
||||
|
||||
HINTS = {
|
||||
"up": "Push the headset up a little (the top strap).",
|
||||
"down": "Let the headset down a little.",
|
||||
"left": "Slide the headset a little to your left.",
|
||||
"right": "Slide the headset a little to your right.",
|
||||
"ipd-wider": "Turn the IPD wheel a little wider.",
|
||||
"ipd-narrower": "Turn the IPD wheel a little narrower.",
|
||||
"tilt-left-up": "Level the headset: lift its left side a little.",
|
||||
"tilt-right-up": "Level the headset: lift its right side a little.",
|
||||
"nearer": "The headset sits closer to your eyes than your saved fit: loosen the back strap a little.",
|
||||
"further": "The headset sits further from your eyes than your saved fit: tighten the back strap a little.",
|
||||
"reflections": "Stray reflections in the {eye} eye's camera: tilt your glasses a little, or wipe the lens.",
|
||||
"lost": "The {eye} eye is hard to see: lashes touching the lens, or the headset too low on that side?",
|
||||
"ipd-changed": "The IPD setting changed since your saved fit ({was:.1f} mm, now {now:.1f} mm).",
|
||||
}
|
||||
|
||||
|
||||
def assess(summary, ref, ipd=None, baseline=None, previous=None):
|
||||
"""The reading: {"state": "ok" | "adjust" | "lost" | "waiting", "eyes": {camera: {...}},
|
||||
"terms": {"up", "left", "ipd", "tilt"} in degrees, "hints": [{"key", "text", "amount"}],
|
||||
"provisional", "against": "baseline" | "reference"}.
|
||||
|
||||
Against a baseline (the person's own good fit) when there is one, else the reference.
|
||||
previous (an earlier assess() result) keeps the state between OK_DEG and ADJUST_DEG."""
|
||||
eyes, hints, offsets = {}, [], {}
|
||||
centres = {}
|
||||
if baseline:
|
||||
centres = {int(k): v["centre"] for k, v in baseline.get("eyes", {}).items()}
|
||||
for cam, s in summary.items():
|
||||
e = {"name": EYES[cam], "found": round(s.get("found", 0.0), 3), "seen": bool(s.get("n"))}
|
||||
if not s.get("n") or s.get("finds", 0) < MIN_FINDS:
|
||||
e["state"] = "waiting" if not s.get("n") else "lost"
|
||||
elif s["found"] < MIN_FOUND:
|
||||
e["state"] = "lost"
|
||||
else:
|
||||
e["state"] = "found"
|
||||
centre = centres.get(cam) or (ref.eyes.get(cam) or {}).get("centre") if ref else centres.get(cam)
|
||||
if centre and ref and cam in ref.eyes:
|
||||
dx, dy = s["x"] - centre[0], s["y"] - centre[1]
|
||||
offsets[cam] = ref.degrees(cam, dx, dy)
|
||||
e["offset_px"] = [round(dx, 1), round(dy, 1)]
|
||||
e["offset_deg"] = [round(v, 2) for v in offsets[cam]]
|
||||
if s.get("extra", 0.0) > EXTRA_GLINTS:
|
||||
hints.append({"key": "reflections", "text": HINTS["reflections"].format(eye=EYES[cam]), "amount": s["extra"]})
|
||||
if baseline:
|
||||
b = baseline.get("eyes", {}).get(str(cam)) or {}
|
||||
if b.get("spacing") and s.get("spacing"):
|
||||
rel = s["spacing"] / b["spacing"] - 1.0
|
||||
e["relief"] = round(rel, 3)
|
||||
eyes[cam] = e
|
||||
lost = [c for c, e in eyes.items() if e["state"] == "lost"]
|
||||
for c in lost:
|
||||
hints.append({"key": "lost", "text": HINTS["lost"].format(eye=EYES[c]), "amount": 1.0 - eyes[c]["found"]})
|
||||
terms = {}
|
||||
if offsets:
|
||||
yaws = {c: v[0] for c, v in offsets.items()}
|
||||
pitches = {c: v[1] for c, v in offsets.items()}
|
||||
terms["up"] = sum(pitches.values()) / len(pitches) # + : the eyes sit high: the headset is low
|
||||
terms["left"] = sum(yaws.values()) / len(yaws) # + : the eyes sit left: the headset is right
|
||||
if len(offsets) == 2:
|
||||
terms["ipd"] = (yaws[1] - yaws[0]) / 2 # + : the eyes are wider apart than the lenses
|
||||
terms["tilt"] = (pitches[1] - pitches[0]) / 2 # + : the left eye sits higher: its side is low
|
||||
t = terms
|
||||
if abs(t.get("up", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("up" if t["up"] > 0 else "down", t["up"]))
|
||||
if abs(t.get("left", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("left" if t["left"] > 0 else "right", t["left"]))
|
||||
ipd_advice = ref is not None and ref.cameras_follow_lenses is True
|
||||
if ipd_advice and abs(t.get("ipd", 0)) > ADJUST_DEG:
|
||||
hints.append(_hint("ipd-wider" if t["ipd"] > 0 else "ipd-narrower", t["ipd"]))
|
||||
if abs(t.get("tilt", 0)) > TILT_DEG:
|
||||
hints.append(_hint("tilt-left-up" if t["tilt"] > 0 else "tilt-right-up", t["tilt"]))
|
||||
reliefs = [e["relief"] for e in eyes.values() if "relief" in e]
|
||||
if reliefs:
|
||||
rel = sum(reliefs) / len(reliefs)
|
||||
terms["relief"] = rel
|
||||
if abs(rel) > RELIEF_TOL:
|
||||
hints.append({"key": "nearer" if rel > 0 else "further", "text": HINTS["nearer" if rel > 0 else "further"],
|
||||
"amount": rel})
|
||||
if baseline and ipd and baseline.get("ipd") and abs(ipd - baseline["ipd"]) >= 0.001:
|
||||
hints.insert(0, {"key": "ipd-changed", "amount": ipd - baseline["ipd"],
|
||||
"text": HINTS["ipd-changed"].format(was=baseline["ipd"] * 1000, now=ipd * 1000)})
|
||||
# The state, with hysteresis on the position terms.
|
||||
big = max([abs(t.get(k, 0.0)) for k in ("up", "left")] + ([abs(t.get("ipd", 0.0))] if ipd_advice else []), default=0.0)
|
||||
if any(e["state"] == "waiting" for e in eyes.values()) or not eyes:
|
||||
state = "waiting"
|
||||
elif lost:
|
||||
state = "lost"
|
||||
elif not offsets:
|
||||
state = "waiting"
|
||||
elif big > ADJUST_DEG or abs(t.get("tilt", 0)) > TILT_DEG or abs(t.get("relief", 0)) > RELIEF_TOL:
|
||||
state = "adjust"
|
||||
elif big <= OK_DEG:
|
||||
state = "ok"
|
||||
else:
|
||||
state = (previous or {}).get("state") if (previous or {}).get("state") in ("ok", "adjust") else "adjust"
|
||||
if state == "adjust" and not hints:
|
||||
# Between OK_DEG and ADJUST_DEG, and not OK before: name the biggest term.
|
||||
names = {"up": ("up", "down"), "left": ("left", "right")}
|
||||
if ipd_advice:
|
||||
names["ipd"] = ("ipd-wider", "ipd-narrower")
|
||||
term = max((k for k in names if k in t), key=lambda k: abs(t[k]), default=None)
|
||||
if term:
|
||||
hints.append(_hint(names[term][0] if t[term] > 0 else names[term][1], t[term]))
|
||||
hints.sort(key=lambda h: (h["key"] not in ("ipd-changed", "lost"), -abs(h["amount"])))
|
||||
return {"state": state, "eyes": eyes, "terms": {k: round(v, 3) for k, v in terms.items()}, "hints": hints,
|
||||
"provisional": bool(ref is None or ref.provisional), "against": "baseline" if centres else "reference",
|
||||
"ipd": ipd, "ipd_advice": ipd_advice}
|
||||
|
||||
|
||||
def _hint(key, amount):
|
||||
return {"key": key, "text": HINTS[key], "amount": amount}
|
||||
|
||||
|
||||
def moved(reading):
|
||||
"""Put-on check: the headset sits more than MOVED_DEG from the baseline (or changed IPD)."""
|
||||
t = reading.get("terms", {})
|
||||
return reading.get("against") == "baseline" and (
|
||||
max(abs(t.get("up", 0.0)), abs(t.get("left", 0.0))) > MOVED_DEG or
|
||||
any(h["key"] == "ipd-changed" for h in reading.get("hints", [])))
|
||||
|
||||
|
||||
def position_hint(terms, limit=MOVED_DEG):
|
||||
"""The biggest position term (up, left) past limit as a hint ("up", "down", "left" or
|
||||
"right"), or None."""
|
||||
names = {"up": ("up", "down"), "left": ("left", "right")}
|
||||
term = max((k for k in names if abs(terms.get(k, 0.0)) > limit), key=lambda k: abs(terms[k]), default=None)
|
||||
if term is None:
|
||||
return None
|
||||
return _hint(names[term][0] if terms[term] > 0 else names[term][1], terms[term])
|
||||
|
||||
|
||||
def slip_terms(jumps, ref):
|
||||
"""ft-eyes' glint jumps ({camera: (dx, dy)}, pixels: how far each pupil moved in its image
|
||||
since the last click, the headset having slipped) as the coach's position terms, {"up",
|
||||
"left"} in degrees, through the reference's Jacobian as assess() does; {} without any."""
|
||||
offsets = [ref.degrees(cam, dx, dy) for cam, (dx, dy) in jumps.items() if ref and cam in ref.eyes]
|
||||
if not offsets:
|
||||
return {}
|
||||
return {"up": sum(o[1] for o in offsets) / len(offsets), "left": sum(o[0] for o in offsets) / len(offsets)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
# Live finds from the shared frames (numpy and OpenCV: the dev container, or a venv with them)
|
||||
|
||||
class LiveFinder:
|
||||
"""Runs the pupil finder on each camera's newest frame, at most rate times a second, and adds
|
||||
the finds to obs. About 0.4 ms a frame while it follows the pupil, 2 ms when it searches."""
|
||||
|
||||
def __init__(self, obs, rate=30.0, share_path=None, want_path=None, log=None):
|
||||
sys.path.insert(0, os.path.join(HERE, "tracker"))
|
||||
import eyes_pupil # noqa: E402 (numpy, OpenCV)
|
||||
import eyes_share # noqa: E402
|
||||
import numpy as np # noqa: E402
|
||||
self._pupil, self._share_mod, self._np = eyes_pupil, eyes_share, np
|
||||
self.obs, self.period = obs, 1.0 / rate
|
||||
self.share_path, self.want_path = share_path, want_path
|
||||
self.log = log or (lambda line: None)
|
||||
self.error = None
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, name="fitcoach-finder", daemon=True)
|
||||
|
||||
def start(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
self._thread.join(2)
|
||||
|
||||
def _run(self):
|
||||
share = self._share_mod.open_share(10, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self.error = "no eye camera frames: is frametop-eyegrab.service running?"
|
||||
self.log("fit coach: " + self.error)
|
||||
return
|
||||
last, near, touched = {0: -1, 1: -1}, {0: None, 1: None}, 0.0
|
||||
np = self._np
|
||||
while not self._stop.is_set():
|
||||
t0 = time.monotonic()
|
||||
for cam in (0, 1):
|
||||
n = share.count(cam)
|
||||
if n == last[cam] or n == 0:
|
||||
continue
|
||||
last[cam] = n
|
||||
f = share.frame(cam, n - 1)
|
||||
if f is None:
|
||||
continue
|
||||
img = np.frombuffer(f[2], np.uint8).reshape(self._share_mod.H, self._share_mod.W)
|
||||
p = self._pupil.find_pupil(img, near[cam])
|
||||
near[cam] = p
|
||||
self.obs.add(cam, f[0], p)
|
||||
if t0 - touched >= 1.0:
|
||||
touched = t0
|
||||
self._share_mod.touch_want(self.want_path)
|
||||
if share.replaced():
|
||||
share.close()
|
||||
share = self._share_mod.open_share(10, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self.error = "the eye camera frames stopped"
|
||||
return
|
||||
self._stop.wait(max(0.0, self.period - (time.monotonic() - t0)))
|
||||
share.close()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------
|
||||
|
||||
def _print_reading(r):
|
||||
eyes = "; ".join("%s %s found %.0f%%%s" % (e["name"], e["state"], 100 * e["found"],
|
||||
" off %+.1f,%+.1f deg" % tuple(e["offset_deg"]) if "offset_deg" in e else "")
|
||||
for _c, e in sorted(r["eyes"].items()))
|
||||
terms = " ".join("%s %+.1f" % kv for kv in sorted(r["terms"].items()))
|
||||
print("%-7s %s | %s%s" % (r["state"], eyes, terms, " (provisional)" if r["provisional"] else ""))
|
||||
for h in r["hints"]:
|
||||
print(" -> " + h["text"])
|
||||
|
||||
|
||||
def main(argv):
|
||||
if not argv or argv[0] in ("-h", "--help"):
|
||||
sys.exit(__doc__)
|
||||
if argv[0] == "reference":
|
||||
paths = argv[1:] or [DOTS]
|
||||
rows = []
|
||||
for p in paths:
|
||||
with open(p) as f:
|
||||
rows += [json.loads(line) for line in f if line.strip()]
|
||||
ref = Reference.from_dots(rows, source="calibration dots: " + ", ".join(os.path.basename(p) for p in paths))
|
||||
print(json.dumps(ref.to_json(), indent=1))
|
||||
return
|
||||
if argv[0] == "live":
|
||||
ref = Reference.load()
|
||||
baseline = load_baseline()
|
||||
obs = Observations()
|
||||
finder = LiveFinder(obs, log=print).start()
|
||||
last = None
|
||||
try:
|
||||
while finder.error is None:
|
||||
time.sleep(0.5)
|
||||
last = assess(obs.summary(), ref, baseline=baseline, previous=last)
|
||||
_print_reading(last)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finder.stop()
|
||||
if "--baseline" in argv and last and last["state"] in ("ok", "adjust"):
|
||||
_write_json(BASELINE, make_baseline(obs.summary()))
|
||||
print("saved the baseline: " + BASELINE)
|
||||
return
|
||||
sys.exit(__doc__)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -8,8 +8,8 @@ PartOf=steamvr.service
|
||||
Requisite=steamvr.service
|
||||
|
||||
[Service]
|
||||
# Host Python; it runs ft-gaze in the container it was built in (scripts/in-box), which
|
||||
# quits when the service's pipe to it closes.
|
||||
# Host Python; it runs ft-gaze in the dev container (distrobox enter), which quits when
|
||||
# the service's pipe to it closes.
|
||||
ExecStart=/usr/bin/python3 @REPO@/gaze/ft-gazed
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
|
||||
+64
-278
@@ -1,17 +1,10 @@
|
||||
// ft-gaze: the headset's eye tracking as rays and Frametop screen pixels (OpenVR overlay
|
||||
// client, runs in the dev container). The gaze service (ft-gazed) and the gaze probe run it.
|
||||
// client, runs in the dev container). An experiment for gaze input; ft-gazeprobe reads it.
|
||||
//
|
||||
// Every eye tracker sample (90 Hz) becomes one JSON line on stdout with each gaze source
|
||||
// hit-tested against the Frametop screens:
|
||||
//
|
||||
// Options: -v (log action errors), --watch-stdin (quit when stdin closes; until then, a line
|
||||
// "sources LIST" on stdin switches the sources as --sources does), --sources LIST (comma-
|
||||
// separated: action, mmap1, mmap2, left, right, own, and eye for EYE; or all, the default).
|
||||
// Sources left out are read not at all and print as {"ok":0} ("eye" as null), so every line
|
||||
// keeps the same keys. The gaze service asks for the ones it uses (own and mmap1 with our
|
||||
// tracker), and all of them while a check or the calibration runs. Only the action costs
|
||||
// SteamVR anything (two calls into vrserver per sample), and a line with every source is
|
||||
// about 1.5 KB, 130 KB a second through podman's relay.
|
||||
// Options: -v (log action errors), --watch-stdin (quit when stdin closes).
|
||||
//
|
||||
// {"t":<sample time, CLOCK_MONOTONIC_RAW s>,"age":<ms old when read>,"n":<sample counter>,
|
||||
// "head":{"yaw":..,"pitch":..,"hit":HIT}, head forward ray (for head nudging)
|
||||
@@ -56,8 +49,8 @@
|
||||
// The mmap samples are in head space, 17 ms or so old when they appear, so each is turned
|
||||
// into the room with the head pose at its own timestamp, from a short pose history.
|
||||
//
|
||||
// Screens come from ft-screens (@ft_screens: "screens", "remotes" for other machines'
|
||||
// displays, "get N"), refreshed 4 times a second in the background. A curved screen is a cylinder toward its front (see OnSurface
|
||||
// Screens come from ft-screens (@ft_screens: "screens", "get N"), refreshed 4 times a
|
||||
// second in the background. A curved screen is a cylinder toward its front (see OnSurface
|
||||
// in screens/vr.cpp).
|
||||
#include <openvr.h>
|
||||
|
||||
@@ -65,7 +58,6 @@
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cerrno>
|
||||
#include <chrono>
|
||||
#include <climits>
|
||||
#include <cmath>
|
||||
@@ -80,7 +72,6 @@
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/un.h>
|
||||
@@ -98,11 +89,6 @@ double NowRaw() {
|
||||
}
|
||||
|
||||
// --- eye-server.mmap (packed, unaligned: read with memcpy) ---
|
||||
// SteamOS 0.4 (SteamVR 2.18.2; first on the 0.4.3 beta, the same on 0.4.5, the release)
|
||||
// moved every field from the timestamp on by 5 bytes (measured 2026-10-04 with the ftdiag
|
||||
// scan: timestamp 0x157 -> 0x15c, the vectors moved with it; the counter at 0x38 kept its
|
||||
// place). Which layout is live is detected at runtime (EyeFile::Detect), so one binary
|
||||
// serves both generations.
|
||||
constexpr size_t kCounter = 0x38; // u32, one per sample
|
||||
constexpr size_t kTime = 0x157; // f64, CLOCK_MONOTONIC_RAW seconds
|
||||
constexpr size_t kLeft1 = 0x15f, kRight1 = 0x16b; // set 1: unit vectors, head space
|
||||
@@ -115,19 +101,11 @@ constexpr size_t kVar1 = 0x177, kVar2 = 0x1b3;
|
||||
// The measurements the filter is fed: left x, y, right x, y, then the variance of each (left
|
||||
// x, y, right x, y). An eye's pair stops changing while the tracker can't see it.
|
||||
constexpr size_t kMeas = 0x1d3;
|
||||
constexpr size_t kNeed = 0x1f3 + 5; // enough for either layout
|
||||
constexpr size_t kShifts[] = {0, 5}; // the layouts EyeFile::Detect knows: SteamOS 0.3, 0.4
|
||||
constexpr size_t kNeed = 0x1f3;
|
||||
|
||||
struct EyeFile {
|
||||
// Everything from the timestamp on is read at base + shift: 0 on SteamOS 0.3, 5 on 0.4
|
||||
// (see the constants above). known once Detect has seen that layout's
|
||||
// timestamp tick; before that, reading would yield garbage that still passes
|
||||
// ReadSample's check.
|
||||
size_t shift = 0;
|
||||
bool known = false;
|
||||
const uint8_t *p = nullptr;
|
||||
size_t size = 0;
|
||||
size_t At(size_t base) const { return base + shift; }
|
||||
bool Open() {
|
||||
const int fd = open("/dev/shm/eye-server.mmap", O_RDONLY | O_CLOEXEC);
|
||||
if (fd < 0) return false;
|
||||
@@ -143,9 +121,6 @@ struct EyeFile {
|
||||
size = st.st_size;
|
||||
return true;
|
||||
}
|
||||
// The sample counter, or 0 without the mapping: the one read the loop makes whether or
|
||||
// not the file was there (Get, V and ReadSample need it).
|
||||
uint32_t Counter() const { return p ? Get<uint32_t>(kCounter) : 0; }
|
||||
template <class T> T Get(size_t off) const {
|
||||
T v;
|
||||
std::memcpy(&v, p + off, sizeof v);
|
||||
@@ -156,53 +131,6 @@ struct EyeFile {
|
||||
std::memcpy(f, p + off, sizeof f);
|
||||
return {f[0], f[1], f[2]};
|
||||
}
|
||||
// A live timestamp is near the clock it comes from (its samples are 17 ms or so old).
|
||||
static bool TimePlausible(double t, double now) { return t > now - 2.0 && t <= now + 2.0; }
|
||||
// The eye directions are unit vectors, so their length is a second, independent check
|
||||
// next to the timestamp: a mere coincidence in one field does not confirm a layout.
|
||||
static bool UnitVec(Vec3 v) {
|
||||
const float n = v.x * v.x + v.y * v.y + v.z * v.z;
|
||||
return n > 0.81f && n < 1.21f; // |v| within 0.9 .. 1.1
|
||||
}
|
||||
bool Fits(size_t layout, double now) const {
|
||||
return TimePlausible(Get<double>(kTime + layout), now) && UnitVec(V(kLeft1 + layout)) &&
|
||||
UnitVec(V(kRight1 + layout));
|
||||
}
|
||||
|
||||
// Which layout is live, a step per pass of the loop, so it never stalls it. A layout
|
||||
// fits when its timestamp is near the clock and its two set-1 directions are unit
|
||||
// vectors; it's taken once its timestamp has moved on too, within kConfirm. That allows
|
||||
// for 2 samples a second: the eye server writes 72 or 90 (15 were seen on the beta).
|
||||
// kWaiting: call again on the next pass. kNone: no layout fits (a server that stopped,
|
||||
// SteamVR's tracker warming up, or a layout we don't know), so the mmap stays unused;
|
||||
// try again later. One detection per run is enough: the layout can't change under a
|
||||
// running ft-gaze, since SteamVR starts the eye server that writes the file, and the
|
||||
// gaze service stops and starts with SteamVR.
|
||||
enum Detection { kWaiting, kFound, kNone };
|
||||
static constexpr double kConfirm = 0.5;
|
||||
static constexpr size_t kLayouts = sizeof kShifts / sizeof kShifts[0];
|
||||
Detection Detect(double now) {
|
||||
if (!fit_) {
|
||||
for (size_t i = 0; i < kLayouts; ++i)
|
||||
if (Fits(kShifts[i], now)) fit_ |= 1u << i, t0_[i] = Get<double>(kTime + kShifts[i]);
|
||||
if (!fit_) return kNone;
|
||||
since_ = now;
|
||||
return kWaiting;
|
||||
}
|
||||
for (size_t i = 0; i < kLayouts; ++i)
|
||||
if ((fit_ >> i & 1) && Get<double>(kTime + kShifts[i]) > t0_[i] && Fits(kShifts[i], now)) {
|
||||
fit_ = 0, shift = kShifts[i], known = true;
|
||||
return kFound;
|
||||
}
|
||||
if (now - since_ <= kConfirm) return kWaiting;
|
||||
fit_ = 0;
|
||||
return kNone;
|
||||
}
|
||||
bool Detecting() const { return fit_ != 0; }
|
||||
|
||||
// Detect's state: the layouts that fit at since_ (a bit each), and their timestamps then.
|
||||
unsigned fit_ = 0;
|
||||
double t0_[kLayouts] = {}, since_ = 0;
|
||||
};
|
||||
|
||||
struct EyeSample {
|
||||
@@ -214,22 +142,20 @@ struct EyeSample {
|
||||
};
|
||||
|
||||
// A consistent copy: the writer has no seqlock we can use, so read until the counter and
|
||||
// timestamp are the same before and after. Only call this once the layout is known
|
||||
// (EyeFile::known): on an unknown layout these reads still pass the consistency check,
|
||||
// but yield garbage.
|
||||
// timestamp are the same before and after.
|
||||
bool ReadSample(const EyeFile &f, EyeSample &s) {
|
||||
for (int attempt = 0; attempt < 4; ++attempt) {
|
||||
const uint32_t n0 = f.Get<uint32_t>(kCounter);
|
||||
const double t0 = f.Get<double>(f.At(kTime));
|
||||
const double t0 = f.Get<double>(kTime);
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
s.left1 = f.V(f.At(kLeft1)), s.right1 = f.V(f.At(kRight1)), s.fix1 = f.V(f.At(kFix1));
|
||||
s.left2 = f.V(f.At(kLeft2)), s.right2 = f.V(f.At(kRight2));
|
||||
std::memcpy(s.open, f.p + f.At(kOpen), sizeof s.open);
|
||||
std::memcpy(s.var1, f.p + f.At(kVar1), sizeof s.var1);
|
||||
std::memcpy(s.var2, f.p + f.At(kVar2), sizeof s.var2);
|
||||
std::memcpy(s.meas, f.p + f.At(kMeas), sizeof s.meas);
|
||||
s.left1 = f.V(kLeft1), s.right1 = f.V(kRight1), s.fix1 = f.V(kFix1);
|
||||
s.left2 = f.V(kLeft2), s.right2 = f.V(kRight2);
|
||||
std::memcpy(s.open, f.p + kOpen, sizeof s.open);
|
||||
std::memcpy(s.var1, f.p + kVar1, sizeof s.var1);
|
||||
std::memcpy(s.var2, f.p + kVar2, sizeof s.var2);
|
||||
std::memcpy(s.meas, f.p + kMeas, sizeof s.meas);
|
||||
std::atomic_thread_fence(std::memory_order_acquire);
|
||||
if (f.Get<uint32_t>(kCounter) == n0 && f.Get<double>(f.At(kTime)) == t0) {
|
||||
if (f.Get<uint32_t>(kCounter) == n0 && f.Get<double>(kTime) == t0) {
|
||||
s.n = n0, s.t = t0;
|
||||
return true;
|
||||
}
|
||||
@@ -368,32 +294,18 @@ private:
|
||||
Screen s;
|
||||
if (std::sscanf(p, " %d:%dx%d:%lf%n", &s.index, &s.wpx, &s.hpx, &s.metres, &used) != 4) break;
|
||||
p += used;
|
||||
if (Place(fd, s)) out.push_back(s);
|
||||
// "ok x y z xx xy xz yx yy yz zx zy zz width height curve hand"
|
||||
const std::string g = Ask(fd, "get " + std::to_string(s.index));
|
||||
double v[15];
|
||||
if (std::sscanf(g.c_str(), "ok %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf", &v[0], &v[1],
|
||||
&v[2], &v[3], &v[4], &v[5], &v[6], &v[7], &v[8], &v[9], &v[10], &v[11], &v[12], &v[13],
|
||||
&v[14]) != 15)
|
||||
continue;
|
||||
s.c = {v[0], v[1], v[2]};
|
||||
s.b = {{v[3], v[4], v[5]}, {v[6], v[7], v[8]}, {v[9], v[10], v[11]}};
|
||||
s.metres = v[12], s.height = v[13], s.curve = v[14];
|
||||
out.push_back(s);
|
||||
}
|
||||
// Other machines' displays (remote.c): "ok <count> <index>:<client>:<state>:<w>x<h> ..."
|
||||
const std::string remotes = Ask(fd, "remotes");
|
||||
if (remotes.rfind("ok ", 0) != 0) return;
|
||||
p = remotes.c_str() + 3;
|
||||
if (std::sscanf(p, "%d%n", &count, &used) != 1) return;
|
||||
p += used;
|
||||
for (int k = 0; k < count; ++k) {
|
||||
Screen s;
|
||||
if (std::sscanf(p, " %d:%*[^:]:%*[^:]:%dx%d%n", &s.index, &s.wpx, &s.hpx, &used) != 3) break;
|
||||
p += used;
|
||||
if (s.wpx > 0 && s.hpx > 0 && Place(fd, s)) out.push_back(s);
|
||||
}
|
||||
}
|
||||
static bool Place(int fd, Screen &s) {
|
||||
// "ok x y z xx xy xz yx yy yz zx zy zz width height curve hand"
|
||||
const std::string g = Ask(fd, "get " + std::to_string(s.index));
|
||||
double v[15];
|
||||
if (std::sscanf(g.c_str(), "ok %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf %lf", &v[0], &v[1], &v[2],
|
||||
&v[3], &v[4], &v[5], &v[6], &v[7], &v[8], &v[9], &v[10], &v[11], &v[12], &v[13], &v[14]) != 15)
|
||||
return false;
|
||||
s.c = {v[0], v[1], v[2]};
|
||||
s.b = {{v[3], v[4], v[5]}, {v[6], v[7], v[8]}, {v[9], v[10], v[11]}};
|
||||
s.metres = v[12], s.height = v[13], s.curve = v[14];
|
||||
return true;
|
||||
}
|
||||
|
||||
std::thread thread_;
|
||||
@@ -477,30 +389,6 @@ std::string SrcJson(const std::vector<Screen> &screens, const vr::HmdMatrix34_t
|
||||
return buf + extra + "\"hit\":" + HitJson(screens, head, yaw, pitch) + "}";
|
||||
}
|
||||
|
||||
// Sources (--sources, "sources LIST" on stdin): a bit each.
|
||||
enum : unsigned { kAction = 1, kMmap1 = 2, kMmap2 = 4, kLeft = 8, kRight = 16, kOwn = 32, kEye = 64, kAll = 127 };
|
||||
|
||||
bool ParseSources(const std::string &list, unsigned &mask) {
|
||||
static const struct {
|
||||
const char *name;
|
||||
unsigned bit;
|
||||
} names[] = {{"action", kAction}, {"mmap1", kMmap1}, {"mmap2", kMmap2}, {"left", kLeft},
|
||||
{"right", kRight}, {"own", kOwn}, {"eye", kEye}, {"all", kAll}};
|
||||
unsigned m = 0;
|
||||
size_t at = 0;
|
||||
while (at <= list.size()) {
|
||||
const size_t comma = std::min(list.find(',', at), list.size());
|
||||
const std::string name = list.substr(at, comma - at);
|
||||
bool known = false;
|
||||
for (const auto &n : names)
|
||||
if (name == n.name) m |= n.bit, known = true;
|
||||
if (!known) return false;
|
||||
at = comma + 1;
|
||||
}
|
||||
mask = m;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Head poses of the last half second, so a sample can use the pose at its own time.
|
||||
class PoseHistory {
|
||||
public:
|
||||
@@ -538,42 +426,17 @@ std::string ExeDir() {
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
bool verbose = false, watchStdin = false;
|
||||
std::atomic<unsigned> sources{kAll};
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
if (std::strcmp(argv[i], "-v") == 0) verbose = true;
|
||||
if (std::strcmp(argv[i], "--watch-stdin") == 0) watchStdin = true;
|
||||
if (std::strcmp(argv[i], "--sources") == 0 && i + 1 < argc) {
|
||||
unsigned m;
|
||||
if (ParseSources(argv[++i], m))
|
||||
sources = m;
|
||||
else
|
||||
std::fprintf(stderr, "ft-gaze: --sources %s: unknown source (all used)\n", argv[i]);
|
||||
}
|
||||
}
|
||||
// Nice 5, before any thread starts (they inherit it): we run in the dev container's podman
|
||||
// scope, beside vrcompositor and vrserver at nice 0, and the gaze service's unit doesn't
|
||||
// reach us. Not SCHED_BATCH, as ft-eyes is: that would let each wakeup wait out another
|
||||
// task's turn, and each sample goes on to the pointer.
|
||||
errno = 0;
|
||||
const int nice0 = getpriority(PRIO_PROCESS, 0);
|
||||
if (errno == 0 && nice0 < 5 && setpriority(PRIO_PROCESS, 0, 5) != 0)
|
||||
std::fprintf(stderr, "ft-gaze: nice: %s\n", std::strerror(errno));
|
||||
// --watch-stdin: quit when stdin closes. The probe runs us through distrobox, which
|
||||
// passes neither its signals nor a closed stdout on to us, but does pass stdin's end.
|
||||
// Lines on stdin until then: "sources LIST".
|
||||
std::atomic<bool> stdinClosed{false};
|
||||
if (watchStdin)
|
||||
std::thread([&stdinClosed, &sources] {
|
||||
std::thread([&stdinClosed] {
|
||||
char c[256];
|
||||
std::string line;
|
||||
ssize_t n;
|
||||
while ((n = read(0, c, sizeof c)) > 0) {
|
||||
line.append(c, size_t(n));
|
||||
for (size_t nl; (nl = line.find('\n')) != std::string::npos; line.erase(0, nl + 1)) {
|
||||
unsigned m;
|
||||
if (line.compare(0, 8, "sources ") == 0 && ParseSources(line.substr(8, nl - 8), m)) sources = m;
|
||||
}
|
||||
if (line.size() > 4096) line.clear(); // no newline in sight: not ours
|
||||
while (read(0, c, sizeof c) > 0) {
|
||||
}
|
||||
stdinClosed = true;
|
||||
}).detach();
|
||||
@@ -598,13 +461,10 @@ int main(int argc, char **argv) {
|
||||
vr::VRActionSetHandle_t set = vr::k_ulInvalidActionSetHandle;
|
||||
input->GetActionHandle("/actions/gaze/in/gaze", &gaze);
|
||||
input->GetActionSetHandle("/actions/gaze", &set);
|
||||
if (me == vr::VRInputError_None)
|
||||
std::fprintf(stderr, "ft-gaze: action manifest %s: ok\n", manifest.c_str());
|
||||
else
|
||||
std::fprintf(stderr, "ft-gaze: action manifest %s: error %d\n", manifest.c_str(), int(me));
|
||||
std::fprintf(stderr, "ft-gaze: action manifest %s: error %d\n", manifest.c_str(), int(me));
|
||||
|
||||
EyeFile eyes;
|
||||
bool haveMmap = eyes.Open();
|
||||
const bool haveMmap = eyes.Open();
|
||||
std::fprintf(stderr, "ft-gaze: eye-server.mmap %s\n", haveMmap ? "open" : "not available");
|
||||
|
||||
OwnFile ownFile;
|
||||
@@ -616,115 +476,51 @@ int main(int argc, char **argv) {
|
||||
double lastEmit = 0;
|
||||
int actionErrors = 0;
|
||||
vr::EVRInputError lastActionError = vr::VRInputError_None;
|
||||
// During a VR game, SteamVR's gaze action is left alone. With ft-gaze reading the eyes, SteamVR
|
||||
// restarted its eye tracker every 10 s or so in a game, as if the headset came off, and each
|
||||
// restart took input focus from the game: Beat Saber paused (PR #13). Of what ft-gaze reads,
|
||||
// only the action reaches SteamVR (the mmap and our tracker are files), so gaze still works
|
||||
// over the dashboard. Games are told apart the way ft-screens does it, by the scene app.
|
||||
bool inGame = false;
|
||||
double nextGameCheck = 0;
|
||||
// The mmap's layout (EyeFile::Detect) is looked for while it isn't known and the eye
|
||||
// server writes: the counter (0x38 in both layouts) ticks once per sample, so a silent
|
||||
// server (headset off) costs nothing and logs nothing. Until a layout is known, SteamVR's
|
||||
// tracker counts as unavailable. "Not recognized" waits until no layout has fitted for
|
||||
// kUnknownAfter of writing, longer than SteamVR's tracker takes to warm up after the
|
||||
// headset goes on (about 20 s), then goes to the journal at most once a minute; ft-gazed
|
||||
// shows it on the Gaze page.
|
||||
constexpr double kUnknownAfter = 30;
|
||||
double nextLayoutCheck = 0, nextLayoutLog = 0, lastWrite = 0, missSince = 0;
|
||||
uint32_t lastCounterSeen = eyes.Counter();
|
||||
// SteamVR's eye tracker creates the mmap a second or two after SteamVR starts, so it can
|
||||
// be missing when we start: look for it again every 2 s until it's there.
|
||||
double nextOpen = NowRaw() + 2.0;
|
||||
|
||||
while (true) {
|
||||
const double now = NowRaw();
|
||||
if (now >= nextGameCheck) {
|
||||
nextGameCheck = now + 0.5;
|
||||
const bool game = vr::VRApplications()->GetCurrentSceneProcessId() != 0;
|
||||
if (game != inGame)
|
||||
std::fprintf(stderr, "ft-gaze: %s\n",
|
||||
game ? "a VR game is running: SteamVR's gaze action left alone" : "the VR game ended");
|
||||
inGame = game;
|
||||
}
|
||||
vr::TrackedDevicePose_t hp;
|
||||
sys->GetDeviceToAbsoluteTrackingPose(vr::TrackingUniverseStanding, 0, &hp, 1);
|
||||
if (hp.bPoseIsValid) history.Add(now, hp.mDeviceToAbsoluteTracking);
|
||||
|
||||
if (!haveMmap && now >= nextOpen) {
|
||||
nextOpen = now + 2.0;
|
||||
if ((haveMmap = eyes.Open())) {
|
||||
std::fprintf(stderr, "ft-gaze: eye-server.mmap open\n");
|
||||
lastCounterSeen = eyes.Counter();
|
||||
}
|
||||
}
|
||||
|
||||
// One line per new eye sample, or at 90 Hz without a usable mmap: none, or one whose
|
||||
// layout isn't known (yet). Our own tracker needs nothing from the mmap, so it keeps
|
||||
// going then; the mmap's sources print as {"ok":0} and "eye" as null.
|
||||
// One line per new eye sample, or at 90 Hz without the mmap.
|
||||
EyeSample s;
|
||||
bool fresh = false;
|
||||
const uint32_t counter = eyes.Counter();
|
||||
const bool writing = haveMmap && counter != lastCounterSeen;
|
||||
lastCounterSeen = counter;
|
||||
if (writing) {
|
||||
if (now - lastWrite > 2.0) missSince = 0; // it was silent: start counting again
|
||||
lastWrite = now;
|
||||
}
|
||||
if (haveMmap && !eyes.known && (eyes.Detecting() || (writing && now >= nextLayoutCheck))) {
|
||||
const EyeFile::Detection d = eyes.Detect(now);
|
||||
if (d == EyeFile::kFound) {
|
||||
std::fprintf(stderr, "ft-gaze: eye-server.mmap layout: %s\n", eyes.shift ? "SteamOS 0.4 (+5)" : "SteamOS 0.3");
|
||||
} else if (d == EyeFile::kNone) {
|
||||
nextLayoutCheck = now + 1.0;
|
||||
if (!missSince) missSince = now;
|
||||
if (now - missSince >= kUnknownAfter && now >= nextLayoutLog) {
|
||||
nextLayoutLog = now + 60.0;
|
||||
std::fprintf(stderr, "ft-gaze: eye-server.mmap has eye data, but its layout is not recognized: "
|
||||
"SteamVR's eye tracking is unavailable\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
const bool mmapOk = haveMmap && eyes.known;
|
||||
if (mmapOk && ReadSample(eyes, s) && s.n != lastN) fresh = true, lastN = s.n;
|
||||
if (!mmapOk && now - lastEmit >= 1.0 / 90) fresh = true, s.t = now;
|
||||
if (haveMmap && ReadSample(eyes, s) && s.n != lastN) fresh = true, lastN = s.n;
|
||||
if (!haveMmap && now - lastEmit >= 1.0 / 90) fresh = true, s.t = now;
|
||||
|
||||
if (fresh && hp.bPoseIsValid) {
|
||||
lastEmit = now;
|
||||
const unsigned want = sources;
|
||||
const auto list = screens.Get();
|
||||
const vr::HmdMatrix34_t &headNow = hp.mDeviceToAbsoluteTracking;
|
||||
vr::HmdMatrix34_t headThen = headNow;
|
||||
if (mmapOk) history.At(s.t, headThen);
|
||||
if (haveMmap) history.At(s.t, headThen);
|
||||
|
||||
// SteamVR's action: a room-space origin and fixation point, turned into the head
|
||||
// frame so every source reports the same kind of angles.
|
||||
std::string action = "{\"ok\":0}";
|
||||
if (!inGame && (want & kAction)) {
|
||||
vr::VRActiveActionSet_t active{};
|
||||
active.ulActionSet = set;
|
||||
active.nPriority = vr::k_nActionSetOverlayGlobalPriorityMin;
|
||||
input->UpdateActionState(&active, sizeof active, 1);
|
||||
vr::VREyeTrackingData_t e{};
|
||||
const vr::EVRInputError ae =
|
||||
input->GetEyeTrackingDataRelativeToNow(gaze, vr::TrackingUniverseStanding, 0, &e, sizeof e);
|
||||
if (ae == vr::VRInputError_None && e.bActive && e.bValid) {
|
||||
const Vec3 o{e.vGazeOrigin.v[0], e.vGazeOrigin.v[1], e.vGazeOrigin.v[2]};
|
||||
const Vec3 t{e.vGazeTarget.v[0], e.vGazeTarget.v[1], e.vGazeTarget.v[2]};
|
||||
const Vec3 dHead = RotateInverse(headNow, Normalize(t - o));
|
||||
char extra[96];
|
||||
std::snprintf(extra, sizeof extra, "\"tracked\":%d,\"dist\":%.3f,", int(e.bTracked), Length(t - o));
|
||||
action = SrcJson(list, headNow, dHead, extra);
|
||||
} else if (ae != lastActionError || (verbose && ++actionErrors % 90 == 1)) {
|
||||
std::fprintf(stderr, "ft-gaze: action: error %d active %d valid %d\n", int(ae), int(e.bActive),
|
||||
int(e.bValid));
|
||||
lastActionError = ae;
|
||||
}
|
||||
vr::VRActiveActionSet_t active{};
|
||||
active.ulActionSet = set;
|
||||
active.nPriority = vr::k_nActionSetOverlayGlobalPriorityMin;
|
||||
input->UpdateActionState(&active, sizeof active, 1);
|
||||
vr::VREyeTrackingData_t e{};
|
||||
const vr::EVRInputError ae =
|
||||
input->GetEyeTrackingDataRelativeToNow(gaze, vr::TrackingUniverseStanding, 0, &e, sizeof e);
|
||||
if (ae == vr::VRInputError_None && e.bActive && e.bValid) {
|
||||
const Vec3 o{e.vGazeOrigin.v[0], e.vGazeOrigin.v[1], e.vGazeOrigin.v[2]};
|
||||
const Vec3 t{e.vGazeTarget.v[0], e.vGazeTarget.v[1], e.vGazeTarget.v[2]};
|
||||
const Vec3 dHead = RotateInverse(headNow, Normalize(t - o));
|
||||
char extra[96];
|
||||
std::snprintf(extra, sizeof extra, "\"tracked\":%d,\"dist\":%.3f,", int(e.bTracked), Length(t - o));
|
||||
action = SrcJson(list, headNow, dHead, extra);
|
||||
} else if (ae != lastActionError || (verbose && ++actionErrors % 90 == 1)) {
|
||||
std::fprintf(stderr, "ft-gaze: action: error %d active %d valid %d\n", int(ae), int(e.bActive),
|
||||
int(e.bValid));
|
||||
lastActionError = ae;
|
||||
}
|
||||
|
||||
std::string m1 = "{\"ok\":0}", m2 = m1, left = m1, right = m1, eye = "null";
|
||||
// Only from a known layout: the unread sample's zeros would print an "unc" of 0 (eyes seen).
|
||||
if (mmapOk) {
|
||||
if (haveMmap) {
|
||||
// lr: the angle between the two eyes' directions. It's a fraction of a degree
|
||||
// normally; when the tracker loses one eye (or during a blink) it jumps.
|
||||
auto lr = [](Vec3 l, Vec3 r) {
|
||||
@@ -744,33 +540,26 @@ int main(int argc, char **argv) {
|
||||
return std::string(b);
|
||||
};
|
||||
char extra[256];
|
||||
if (want & kMmap1) {
|
||||
std::snprintf(extra, sizeof extra, "\"dist\":%.3f,\"open\":[%.3f,%.3f],\"lr\":%.3f,", Length(s.fix1),
|
||||
s.open[0], s.open[1], lr(s.left1, s.right1));
|
||||
m1 = SrcJson(list, headThen, s.left1 + s.right1, extra + eyes(s.left1, s.right1) + unc(s.var1));
|
||||
}
|
||||
if (want & kMmap2) {
|
||||
std::snprintf(extra, sizeof extra, "\"lr\":%.3f,", lr(s.left2, s.right2));
|
||||
m2 = SrcJson(list, headThen, s.left2 + s.right2, extra + eyes(s.left2, s.right2) + unc(s.var2));
|
||||
}
|
||||
if (want & kLeft) left = SrcJson(list, headThen, s.left2);
|
||||
if (want & kRight) right = SrcJson(list, headThen, s.right2);
|
||||
// "new" compares with the last sample, so the last measurement is kept either way.
|
||||
std::snprintf(extra, sizeof extra, "\"dist\":%.3f,\"open\":[%.3f,%.3f],\"lr\":%.3f,", Length(s.fix1),
|
||||
s.open[0], s.open[1], lr(s.left1, s.right1));
|
||||
m1 = SrcJson(list, headThen, s.left1 + s.right1, extra + eyes(s.left1, s.right1) + unc(s.var1));
|
||||
std::snprintf(extra, sizeof extra, "\"lr\":%.3f,", lr(s.left2, s.right2));
|
||||
m2 = SrcJson(list, headThen, s.left2 + s.right2, extra + eyes(s.left2, s.right2) + unc(s.var2));
|
||||
left = SrcJson(list, headThen, s.left2);
|
||||
right = SrcJson(list, headThen, s.right2);
|
||||
const float *m = s.meas;
|
||||
const bool newL = m[0] != lastMeas[0] || m[1] != lastMeas[1];
|
||||
const bool newR = m[2] != lastMeas[2] || m[3] != lastMeas[3];
|
||||
std::memcpy(lastMeas, m, sizeof lastMeas);
|
||||
if (want & kEye) {
|
||||
std::snprintf(extra, sizeof extra, "{\"q\":[%.3g,%.3g],\"m\":[[%.4f,%.4f],[%.4f,%.4f]],\"new\":[%d,%d]}",
|
||||
(m[4] + m[5]) / 2, (m[6] + m[7]) / 2, m[0], m[1], m[2], m[3], int(newL), int(newR));
|
||||
eye = extra;
|
||||
}
|
||||
std::snprintf(extra, sizeof extra, "{\"q\":[%.3g,%.3g],\"m\":[[%.4f,%.4f],[%.4f,%.4f]],\"new\":[%d,%d]}",
|
||||
(m[4] + m[5]) / 2, (m[6] + m[7]) / 2, m[0], m[1], m[2], m[3], int(newL), int(newR));
|
||||
eye = extra;
|
||||
}
|
||||
|
||||
// Our tracker: its own sample time picks the head pose, like the mmap's.
|
||||
std::string own = "{\"ok\":0}";
|
||||
OwnSample o;
|
||||
if ((want & kOwn) && ownFile.Read(o) && now - o.t < 0.1) {
|
||||
if (ownFile.Read(o) && now - o.t < 0.1) {
|
||||
vr::HmdMatrix34_t headOwn = headNow;
|
||||
history.At(o.t, headOwn);
|
||||
auto pair = [](bool ok, float a, float b) {
|
||||
@@ -815,10 +604,7 @@ int main(int argc, char **argv) {
|
||||
break;
|
||||
}
|
||||
if (stdinClosed) break;
|
||||
// 250 passes a second: a new sample is printed within 4 ms (2 on average) of appearing,
|
||||
// and the pose history has a pose within 2 ms of any sample's time (a 0.2 degree head
|
||||
// turn at 100 degrees a second). Every 2 ms read the pose and the events twice as often.
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(4));
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
||||
}
|
||||
screens.Stop();
|
||||
vr::VR_Shutdown();
|
||||
|
||||
+1
-18
@@ -6,9 +6,6 @@
|
||||
ft-gazectl status the gaze service (ft-gazed): samples, lessons, the correction
|
||||
ft-gazectl forget drop what the pointer's lessons taught (the calibration stays)
|
||||
ft-gazectl reload the service reads calibration.json again
|
||||
ft-gazectl quickcal|calibrate|fitcheck|fivecheck
|
||||
open that check in the headset panel: the one-dot check, the
|
||||
full calibration, the headset fit check, the five-dot check
|
||||
"""
|
||||
|
||||
import json
|
||||
@@ -39,17 +36,8 @@ def main():
|
||||
if r is None:
|
||||
sys.exit("the pointer helper isn't running")
|
||||
print(f"gaze mode {r.removeprefix('ok ')}" if r else "no answer (an older pointer helper without gaze mode?)")
|
||||
st = ask("ft_gazed", "status", 0.3)
|
||||
if st is None:
|
||||
if ask("ft_gazed", "status", 0.3) is None:
|
||||
print("note: the gaze service (ft-gazed) isn't running, so the pointer has no gaze to follow")
|
||||
elif r == "ok on":
|
||||
try:
|
||||
calibrated = (json.loads(st).get("checks") or {}).get("calibrated")
|
||||
except ValueError:
|
||||
calibrated = None
|
||||
if calibrated is False:
|
||||
print("note: no calibration for this eye tracker yet; it opens in the headset "
|
||||
"(if it can't, ft-gazectl status says why: checks.problem)")
|
||||
elif cmd in ("status", "forget", "reload"):
|
||||
r = ask("ft_gazed", cmd)
|
||||
if r is None:
|
||||
@@ -58,11 +46,6 @@ def main():
|
||||
print(json.dumps(json.loads(r), indent=1))
|
||||
except ValueError:
|
||||
print(r)
|
||||
elif cmd in ("quickcal", "calibrate", "fitcheck", "fivecheck"):
|
||||
r = ask("ft_gazed", cmd)
|
||||
if r is None:
|
||||
sys.exit("the gaze service (ft-gazed) isn't running")
|
||||
print(r)
|
||||
else:
|
||||
sys.exit(__doc__)
|
||||
|
||||
|
||||
+46
-296
@@ -3,12 +3,8 @@
|
||||
|
||||
Two settings in ~/.config/frametop.conf (the Gaze page of Frametop Input Settings), read
|
||||
again when the file changes:
|
||||
GAZE_TRACKER=auto|own|steam
|
||||
our own eye tracker (gaze/tracker/ft-eyes, ft-gaze's source "own";
|
||||
this service runs it, see below) or SteamVR's. auto (the default)
|
||||
is ours when it's installed (gaze/tracker/install.sh: the frame
|
||||
grabber, and ft-eyes' Python in this checkout), else SteamVR's; it
|
||||
switches when ours is installed or removed.
|
||||
GAZE_TRACKER=steam|own SteamVR's eye tracker (default), or our own (gaze/tracker/ft-eyes,
|
||||
ft-gaze's source "own"; this service runs it, see below)
|
||||
GAZE_EYE=auto|left|right the eye bias (gazecal.EyeWeights): auto weights each eye by how far
|
||||
off it was at your recent nudges; left or right counts that eye twice
|
||||
as much as the other. Either eye alone carries the gaze when the
|
||||
@@ -48,11 +44,9 @@ click, it sends "lesson <raw yaw> <raw pitch> <true yaw> <true pitch>": where th
|
||||
was when the mouse took over, and where the pointer was when you clicked (you were looking
|
||||
there). The gap is the tracker's error there. The raw gaze is the one sent, so it also says
|
||||
when that look was (the history of what was sent), and so what each eye read then:
|
||||
- SteamVR: each eye learns its own error, unless the gaze was more than
|
||||
POINTER_GAZE_NUDGE_MAX degrees (frametop.conf, 55 by default: the helper's limit too)
|
||||
- SteamVR: each eye learns its own error, unless the gaze was more than LESSON_MAX degrees
|
||||
past the correction (then it wasn't a nudge onto what you looked at);
|
||||
- our tracker: unless it was more than POINTER_GAZE_NUDGE_MAX off, the look goes to it as
|
||||
a click ("click T YAW PITCH" on @ft_eyes), as the
|
||||
- our tracker: the look goes to it as a click ("click T YAW PITCH" on @ft_eyes), as the
|
||||
probe's clicks do, and it learns how far the headset has moved on your face. That's
|
||||
what it gets wrong, and after the headset was off, the first click resets it;
|
||||
- either way, how far off each eye was (before the lesson taught it anything) goes to the
|
||||
@@ -63,54 +57,20 @@ and the error moves): lessons from before count less then, so the first few afte
|
||||
relearn the offset.
|
||||
|
||||
Our own tracker (gaze/tracker/ft-eyes) runs here too, in the dev container, while
|
||||
GAZE_TRACKER=own and the gaze is in use (below), or the gaze probe asks for it ("eyes SECONDS",
|
||||
a lease the probe renews). It reads the eye-camera frames the root service frametop-eyegrab
|
||||
copies (gaze/tracker/install.sh), which copies them only while ft-eyes runs. Which eye net it runs
|
||||
(EYES_MODEL: models/catalog.py) is read at its start, so ft-eyes restarts when that changes; starting
|
||||
it also runs `models/ft-models auto eyes` in the background (the catalog from GitHub, and the net
|
||||
EYES_MODEL needs if it isn't downloaded), and ft-eyes moves to a net that brings.
|
||||
|
||||
Idle: ft-gaze and our own tracker run only while the gaze is in use: gaze mode on with someone
|
||||
wearing the headset (the pointer helper says both: "gaze ? headset" -> "ok on|off worn|away"), a
|
||||
check or calibration open or asked for, or a "wake" lease (the Gaze page of Frametop Input
|
||||
Settings renews one while it's open). IDLE_AFTER after the last use they stop, and the frame
|
||||
grabber goes idle with our tracker; with our tracker, that was over half a core with gaze mode
|
||||
off. A check asked for while idle waits for the tracker to start (at most WAKE_SETTLE). Our
|
||||
tracker's next click after it starts again re-seats it, as after the headset was off, so turning
|
||||
gaze mode on after a while opens the quick check.
|
||||
|
||||
ft-gaze prints only the sources this uses ("sources LIST" on its stdin, see wanted_sources):
|
||||
own and mmap1 with our tracker, left, right and mmap1 with SteamVR's eyes, and every source
|
||||
while a check or the calibration runs or is asked for, since those record them all. So
|
||||
SteamVR's gaze action, the one source that calls into vrserver, is read only then (or with
|
||||
--source action).
|
||||
|
||||
Checks and calibration (gaze/gazecheck.py): a one-dot quick check when the headset goes on or
|
||||
our tracker asks for a click, and the full calibration when gaze mode comes on without one,
|
||||
both in a panel fixed to the headset (gaze/panel/ft-gazepanel, which this service runs too).
|
||||
GAZE_TRACKER=own or the gaze probe asks for it ("eyes SECONDS", a lease the probe renews). It
|
||||
reads the eye-camera frames the root service frametop-eyegrab copies (gaze/tracker/install.sh),
|
||||
which copies them only while ft-eyes runs.
|
||||
|
||||
Nothing here writes to SteamVR, its eye tracker, or its files: ft-gaze reads the eye
|
||||
tracker's shared memory read-only.
|
||||
|
||||
Control socket: abstract unix datagram "@ft_gazed":
|
||||
lesson <rhy> <rhp> <thy> <thp> from the pointer helper (see above)
|
||||
recheck <deg> from the pointer helper: a click's correction was past
|
||||
POINTER_GAZE_NUDGE_MAX, so the quick check (gazecheck.py)
|
||||
status reply: one JSON object
|
||||
forget drop what the lessons taught (the calibration stays)
|
||||
reload read calibration.json and the settings again
|
||||
eyes <seconds> keep our own tracker running that much longer (at most 120),
|
||||
whatever GAZE_TRACKER says: the probe's lease. Reply: "ok"
|
||||
wake <seconds> keep the gaze running (not idle) that much longer (at most 120),
|
||||
whatever gaze mode says: Input Settings' Gaze page. Reply: "ok"
|
||||
quickcal the one-dot check now (the calibration if there's none)
|
||||
calibrate the full calibration in the panel
|
||||
fitcheck the headset fit check in the panel
|
||||
fivecheck the five-dot check now (it otherwise follows a quick check
|
||||
that didn't fix the tracker)
|
||||
calaccept | calquit from the pointer helper while the panel is up: take this dot
|
||||
now (a left click, Meta+J, or a gaze precision or gaze drag
|
||||
press) | close it (a right click, Meta+K)
|
||||
|
||||
Options: --source action|mmap1|mmap2 (the older one-source path with that source, whatever
|
||||
the settings say; set 2 was a little quieter in the probe, but loses the pointer whenever
|
||||
@@ -135,22 +95,16 @@ from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from gazecal import (DEFAULT_MODEL, EYE_FOUND, EYE_LOST, MODELS, STATE, Correction, EyeFallback, # noqa: E402
|
||||
EyeWeights, Fixation, LiveCorrection, SteamEyeLog, TrackedEye)
|
||||
from gazecheck import Checks # noqa: E402
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "models"))
|
||||
import catalog # noqa: E402 (which eye net EYES_MODEL means: models/catalog.py)
|
||||
EyeWeights, Fixation, LiveCorrection, SteamEyeLog)
|
||||
|
||||
REPO = Path(__file__).resolve().parents[1]
|
||||
HELPER = REPO / "gaze" / "build" / "ft-gaze"
|
||||
IN_BOX = REPO / "scripts" / "in-box" # runs a program in the container it was built in
|
||||
ME = "\0ft_gazed"
|
||||
POINTER = "\0ft_pointer_helper"
|
||||
EYES_PROG = REPO / "gaze" / "tracker" / "ft-eyes" # our own tracker
|
||||
EYES_PYTHON = REPO / "gaze" / "tracker" / "build" / "venv" / "bin" / "python" # numpy, OpenCV (build.sh)
|
||||
EYES_SOCKET = "\0ft_eyes" # its control socket
|
||||
EYES_CAMS = Path("/dev/shm/frametop-eyes-cams") # the frames it reads (frametop-eyegrab.service)
|
||||
EYEGRAB = (Path("/etc/frametop/ft-eyegrab"), Path("/etc/systemd/system/frametop-eyegrab.service")) # tracker/install.sh
|
||||
CONF = Path.home() / ".config" / "frametop.conf"
|
||||
CALIBRATION = STATE / "calibration.json"
|
||||
LESSONS = STATE / "pointer-lessons.json"
|
||||
@@ -158,19 +112,15 @@ LESSON_LOG = STATE / "pointer-lessons.jsonl"
|
||||
SOURCES = ("action", "mmap1", "mmap2", "left", "right") # the ones with a calibration here
|
||||
SIDES = ("left", "right") # ft-gaze's order, and the sources for each eye alone
|
||||
TRACKERS = ("steam", "own")
|
||||
TRACKER_SETTINGS = ("auto",) + TRACKERS
|
||||
BIASES = ("auto", "left", "right")
|
||||
NUDGE_MAX = 55.0 # degrees: POINTER_GAZE_NUDGE_MAX's default, the largest lesson taken
|
||||
LESSON_MAX = 8.0 # degrees past the correction
|
||||
OWN_LESSON_MAX = 25.0 # our tracker: after the headset was off, its first clicks can be 10-17 off
|
||||
HISTORY = 12.0 # seconds of the gaze sent, to find a lesson's look (the helper sends it up to 10 s later)
|
||||
LOOK = 0.3 # seconds of samples before that moment make the look (the probe's fixation)
|
||||
RETRY = 3.0 # seconds before starting ft-gaze again
|
||||
EYES_RETRY = 10.0 # seconds before starting ft-eyes again after it stopped on its own
|
||||
EYES_LEASE_MAX = 120.0
|
||||
MODELS_CLI = REPO / "models" / "ft-models" # "auto eyes": the eye catalog, and the net EYES_MODEL needs
|
||||
SETTLE = 0.3 # seconds after an eye is found again before the fallback learns from it
|
||||
IDLE_AFTER = 30.0 # seconds after the gaze was last in use before ft-gaze and our tracker stop
|
||||
WAKE_SETTLE = 15.0 # seconds after waking that the tracker may take to start sending
|
||||
WAKE_MAX = 120.0
|
||||
KEYBOARD_PITCH = -20.0 # degrees: gaze under this, on no screen, is a look at the keyboard
|
||||
|
||||
|
||||
@@ -190,14 +140,8 @@ def log(msg):
|
||||
print(f"ft-gazed: {msg}", file=sys.stderr, flush=True)
|
||||
|
||||
|
||||
def own_installed():
|
||||
"""Is our own tracker installed: its frame grabber, and ft-eyes' Python here?"""
|
||||
return all(p.exists() for p in EYEGRAB) and EYES_PYTHON.exists()
|
||||
|
||||
|
||||
def read_settings():
|
||||
"""(tracker, GAZE_TRACKER, eye bias, nudge max) from frametop.conf, defaults for anything
|
||||
missing or unknown. The tracker is "steam" or "own": auto picks ours when it's installed."""
|
||||
"""(tracker, eye bias) from frametop.conf, defaults for anything missing or unknown."""
|
||||
conf = {}
|
||||
try:
|
||||
for line in CONF.read_text().splitlines():
|
||||
@@ -207,15 +151,9 @@ def read_settings():
|
||||
conf[k.strip()] = v.strip().lower()
|
||||
except OSError:
|
||||
pass
|
||||
setting = conf.get("GAZE_TRACKER", "auto")
|
||||
setting = setting if setting in TRACKER_SETTINGS else "auto"
|
||||
tracker = setting if setting != "auto" else "own" if own_installed() else "steam"
|
||||
tracker = conf.get("GAZE_TRACKER", "steam")
|
||||
bias = conf.get("GAZE_EYE", "auto")
|
||||
try:
|
||||
nudge = min(max(float(conf.get("POINTER_GAZE_NUDGE_MAX", NUDGE_MAX)), 1.0), 110.0) # the helper's range
|
||||
except ValueError:
|
||||
nudge = NUDGE_MAX
|
||||
return tracker, setting, bias if bias in BIASES else "auto", nudge
|
||||
return tracker if tracker in TRACKERS else "steam", bias if bias in BIASES else "auto"
|
||||
|
||||
|
||||
def mtime(path):
|
||||
@@ -230,7 +168,7 @@ class Service:
|
||||
self.override, self.verbose, self.to = source, verbose, to
|
||||
self.source = source or "mmap1" # the older path's source
|
||||
STATE.mkdir(parents=True, exist_ok=True)
|
||||
self.tracker, self.tracker_setting, self.bias, self.nudge_max = read_settings()
|
||||
self.tracker, self.bias = read_settings()
|
||||
self.conf_mtime = mtime(CONF)
|
||||
self.models = {name: Correction() for name in SOURCES}
|
||||
self.mode = DEFAULT_MODEL
|
||||
@@ -247,9 +185,6 @@ class Service:
|
||||
self.opens = (deque(maxlen=90), deque(maxlen=90)) # left, right
|
||||
self.vergence = deque(maxlen=90)
|
||||
self.fallback = EyeFallback()
|
||||
# SteamVR's tracker following one eye only (gazecal.tracked_eye): 0 left, 1 right, or None.
|
||||
self.tracked = TrackedEye()
|
||||
self.one_eye = self.tracked()
|
||||
self.lost = [False, False]
|
||||
self.bad_at = [0.0, 0.0] # sample time an eye was last lost or closed
|
||||
self.counts = {"samples": 0, "sent": 0, "blinks": 0, "one_eye": 0, "one_eye_used": 0, "lost_left": 0,
|
||||
@@ -264,12 +199,6 @@ class Service:
|
||||
self.eyes_proc = None # ft-eyes, while it runs
|
||||
self.eyes_until = 0.0 # the probe's lease (monotonic time)
|
||||
self.eyes_restart_at = 0.0
|
||||
self.eyes_model = self.eyes_model_now() # what EYES_MODEL means now: ft-eyes restarts when it changes
|
||||
self.models_proc = None # ft-models auto eyes, while it runs
|
||||
self.awake = False # ft-gaze (and our tracker) run: the gaze is in use (see the top)
|
||||
self.idle_at = 0.0 # idle from then, unless it's in use again first
|
||||
self.woke_at = 0.0
|
||||
self.wake_until = 0.0 # a "wake" lease
|
||||
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC | socket.SOCK_NONBLOCK)
|
||||
self.sock.bind(ME)
|
||||
@@ -280,10 +209,7 @@ class Service:
|
||||
self.sel = selectors.DefaultSelector()
|
||||
self.sel.register(self.sock, selectors.EVENT_READ, "control")
|
||||
self.sel.register(self.eyes_sock, selectors.EVENT_READ, "own")
|
||||
self.checks = Checks(self, self.sel)
|
||||
self.proc = None
|
||||
self.proc_sources = None # what ft-gaze was last told to print
|
||||
self.mmap_unknown = False # ft-gaze said SteamVR's eye-server.mmap has a layout it doesn't know
|
||||
self.buf = b""
|
||||
self.restart_at = 0.0
|
||||
self.running = True
|
||||
@@ -296,54 +222,15 @@ class Service:
|
||||
return "source"
|
||||
if self.tracker == "own":
|
||||
return "own"
|
||||
# With one eye tracked, that eye's own calibration is enough.
|
||||
sides = SIDES if self.one_eye is None else (SIDES[self.one_eye],)
|
||||
return "eyes" if all(self.models[e].samples for e in sides) else "source"
|
||||
return "eyes" if all(self.models[e].samples for e in SIDES) else "source"
|
||||
|
||||
# --- Settings, calibration and lessons ---
|
||||
|
||||
@staticmethod
|
||||
def eyes_model_now():
|
||||
"""(EYES_MODEL, the folder it resolves to): ft-eyes reads it at start."""
|
||||
try:
|
||||
r = catalog.resolve("eyes")
|
||||
return r["setting"], r["folder"]
|
||||
except Exception as e: # noqa: BLE001 (a broken catalog: ft-eyes falls back to the pupil finder)
|
||||
return f"error {e}", None
|
||||
|
||||
def check_eyes_model(self):
|
||||
now = self.eyes_model_now()
|
||||
if now != self.eyes_model:
|
||||
log(f"eye model {now[0]}" + (f" ({now[1]})" if now[1] else "") + (": restarting ft-eyes" if self.eyes_proc else ""))
|
||||
self.eyes_model = now
|
||||
if self.eyes_proc:
|
||||
self.stop_eyes()
|
||||
self.eyes_restart_at = 0.0
|
||||
|
||||
def models_auto(self):
|
||||
"""The eye catalog from GitHub (at most every 15 minutes) and the net EYES_MODEL needs, if it's
|
||||
missing, in the background: ft-eyes starts with what's here, and moves to the new one after."""
|
||||
if self.models_proc and self.models_proc.poll() is None:
|
||||
return
|
||||
try:
|
||||
self.models_proc = subprocess.Popen([sys.executable, str(MODELS_CLI), "auto", "eyes"], stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
start_new_session=True)
|
||||
except OSError as e:
|
||||
log(f"couldn't check for eye models: {e}")
|
||||
|
||||
def load_settings(self):
|
||||
self.conf_mtime = mtime(CONF)
|
||||
self.check_eyes_model()
|
||||
tracker, self.tracker_setting, bias, self.nudge_max = read_settings()
|
||||
tracker, bias = read_settings()
|
||||
if (tracker, bias) != (self.tracker, self.bias):
|
||||
auto = ""
|
||||
if self.tracker_setting == "auto":
|
||||
auto = " (auto: ours is installed)" if tracker == "own" else " (auto: ours isn't installed)"
|
||||
if tracker == "steam" and EYEGRAB[1].exists() and not EYEGRAB[0].exists():
|
||||
auto = (f" (auto: ours lost {EYEGRAB[0]}, which SteamOS updates delete unless it's kept;"
|
||||
" reinstall it with gaze/tracker/install.sh)")
|
||||
log(f"tracker {tracker}{auto}, eye bias {bias}" + (f" (--source {self.override} wins)" if self.override else ""))
|
||||
log(f"tracker {tracker}, eye bias {bias}" + (f" (--source {self.override} wins)" if self.override else ""))
|
||||
self.tracker, self.bias = tracker, bias
|
||||
for w in self.weights.values():
|
||||
w.bias = bias
|
||||
@@ -386,13 +273,6 @@ class Service:
|
||||
tmp.replace(LESSONS)
|
||||
self.dirty = False
|
||||
|
||||
def forget_lessons(self):
|
||||
"""Drop what the lessons taught (the calibration stays)."""
|
||||
self.lives = {name: PointerLessons() for name in SOURCES}
|
||||
self.weights = {t: EyeWeights(self.bias) for t in TRACKERS}
|
||||
self.refit()
|
||||
self.save_lessons()
|
||||
|
||||
def refit(self):
|
||||
for name, live in self.lives.items():
|
||||
live.wear_time = self.steam.worn()
|
||||
@@ -424,8 +304,8 @@ class Service:
|
||||
cy, cp = self.correction(self.source, rhy, rhp)
|
||||
left = math.hypot(dy - cy, dp - cp)
|
||||
rec.update(source=self.source, model=self.mode, correction=[cy, cp], lesson_deg=left)
|
||||
if left > self.nudge_max:
|
||||
rec["refused"] = f"more than {self.nudge_max:g} deg past the correction"
|
||||
if left > LESSON_MAX:
|
||||
rec["refused"] = f"more than {LESSON_MAX} deg past the correction"
|
||||
else:
|
||||
self.lives[self.source].add({"time": rec["time"], "hy": rhy, "hp": rhp, "dy": dy, "dp": dp,
|
||||
"wy": 1.0, "wp": 1.0, "how": "pointer"}, self.models[self.source], self.mode)
|
||||
@@ -436,10 +316,11 @@ class Service:
|
||||
weights = self.weights[self.tracker if kind == "own" else "steam"]
|
||||
rec.update(tracker=self.tracker if kind == "own" else "steam", bias=self.bias, lesson_deg=left, look_t=t,
|
||||
eyes=eyes, weights=[round(w, 3) for w in weights.weights()])
|
||||
limit = OWN_LESSON_MAX if kind == "own" else LESSON_MAX
|
||||
if t is None:
|
||||
rec["refused"] = "that gaze isn't in the last few seconds sent"
|
||||
elif left > self.nudge_max:
|
||||
rec["refused"] = f"more than {self.nudge_max:g} deg off"
|
||||
elif left > limit:
|
||||
rec["refused"] = f"more than {limit} deg off"
|
||||
if "refused" in rec:
|
||||
return self.taken(rec)
|
||||
if kind == "own":
|
||||
@@ -477,73 +358,8 @@ class Service:
|
||||
log(f"lesson log: {e}")
|
||||
return rec
|
||||
|
||||
# --- Idle (see the top) ---
|
||||
|
||||
def use_reason(self):
|
||||
"""Why the gaze is in use now, or None."""
|
||||
c = self.checks
|
||||
if c.gaze_on and c.headset is not False:
|
||||
return "gaze mode is on"
|
||||
if c.active or c.pending:
|
||||
return "a check"
|
||||
if time.monotonic() < self.wake_until:
|
||||
return "asked to stay awake"
|
||||
return None
|
||||
|
||||
def idle_reason(self):
|
||||
c = self.checks
|
||||
if c.gaze_on is None:
|
||||
return "the pointer helper isn't answering (SteamVR not running?)"
|
||||
if c.gaze_on and c.headset is False:
|
||||
return "nobody is wearing the headset"
|
||||
return "gaze mode is off"
|
||||
|
||||
def waking(self):
|
||||
"""Idle, or awake too briefly for the tracker to be sending yet."""
|
||||
return not self.awake or time.monotonic() - self.woke_at < WAKE_SETTLE
|
||||
|
||||
def update_awake(self):
|
||||
now = time.monotonic()
|
||||
why = self.use_reason()
|
||||
if why:
|
||||
self.idle_at = now + IDLE_AFTER
|
||||
if why and not self.awake:
|
||||
self.awake, self.woke_at, self.restart_at = True, now, 0.0
|
||||
log(f"awake: {why}")
|
||||
elif not why and self.awake and now >= self.idle_at:
|
||||
self.awake = False
|
||||
log(f"idle: {self.idle_reason()}; ft-gaze and our tracker stop until the gaze is used again")
|
||||
self.stop_helper()
|
||||
if self.eyes_proc and not self.eyes_wanted():
|
||||
self.stop_eyes()
|
||||
|
||||
# --- ft-gaze ---
|
||||
|
||||
def wanted_sources(self):
|
||||
"""The sources ft-gaze should print (its --sources): what on_sample and the checks read.
|
||||
mmap1 always (blinks, lost eyes, and the headset going on, from its openness and
|
||||
variances); everything while a check runs or waits, since they record every source."""
|
||||
c = self.checks
|
||||
if c.active or c.pending:
|
||||
return "all"
|
||||
kind = self.kind
|
||||
if kind == "own":
|
||||
return "own,mmap1"
|
||||
if kind == "eyes":
|
||||
return "left,right,mmap1"
|
||||
return ",".join(dict.fromkeys((self.source, "mmap1", "mmap2"))) # mmap2: each eye, for the fallback
|
||||
|
||||
def sync_sources(self):
|
||||
want = self.wanted_sources()
|
||||
if not self.proc or want == self.proc_sources:
|
||||
return
|
||||
try:
|
||||
self.proc.stdin.write(f"sources {want}\n".encode())
|
||||
self.proc.stdin.flush()
|
||||
except (OSError, ValueError):
|
||||
return # it's stopping: read_stdout notices
|
||||
self.proc_sources = want
|
||||
|
||||
def start_helper(self):
|
||||
if not HELPER.exists():
|
||||
log(f"ft-gaze isn't built: run {REPO}/gaze/build.sh")
|
||||
@@ -551,18 +367,17 @@ class Service:
|
||||
return
|
||||
env = dict(os.environ)
|
||||
env["XDG_RUNTIME_DIR"] = f"/run/user/{os.getuid()}" # podman needs the real one
|
||||
subprocess.run([str(REPO / "scripts" / "container-up.sh")], env=env, check=False)
|
||||
distrobox = Path.home() / ".local" / "bin" / "distrobox"
|
||||
# ft-gaze quits when its stdin closes: the one thing distrobox passes on.
|
||||
sources = self.wanted_sources()
|
||||
self.proc = subprocess.Popen([str(IN_BOX), str(HELPER), "--watch-stdin", "--sources", sources], env=env,
|
||||
self.proc = subprocess.Popen([str(distrobox), "enter", "dev", "--", str(HELPER), "--watch-stdin"], env=env,
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
start_new_session=True)
|
||||
self.proc_sources = sources
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
self.mmap_unknown = False
|
||||
log("ft-gaze started")
|
||||
|
||||
def stop_helper(self):
|
||||
@@ -583,28 +398,24 @@ class Service:
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
self.proc = None
|
||||
self.mmap_unknown = False
|
||||
|
||||
def own_installed(self):
|
||||
return own_installed() # (for the checks: the fit coach needs ft-eyes)
|
||||
|
||||
def eyes_wanted(self):
|
||||
return (self.tracker == "own" and not self.override and self.awake) or time.monotonic() < self.eyes_until
|
||||
return (self.tracker == "own" and not self.override) or time.monotonic() < self.eyes_until
|
||||
|
||||
def start_eyes(self):
|
||||
"""Our own tracker, in the container, with build/venv's numpy and OpenCV. Like
|
||||
"""Our own tracker, in the dev container, with build/venv's numpy and OpenCV. Like
|
||||
ft-gaze, it quits when its stdin closes."""
|
||||
if not EYES_PYTHON.exists():
|
||||
log(f"ft-eyes isn't built: run {REPO}/gaze/tracker/build.sh")
|
||||
self.eyes_restart_at = time.monotonic() + 30
|
||||
return
|
||||
self.models_auto()
|
||||
self.eyes_model = self.eyes_model_now()
|
||||
env = dict(os.environ)
|
||||
env["XDG_RUNTIME_DIR"] = f"/run/user/{os.getuid()}"
|
||||
self.eyes_proc = subprocess.Popen([str(IN_BOX), str(EYES_PYTHON), str(EYES_PROG), "-v", "--watch-stdin"],
|
||||
env=env, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE, start_new_session=True)
|
||||
subprocess.run([str(REPO / "scripts" / "container-up.sh")], env=env, check=False)
|
||||
distrobox = Path.home() / ".local" / "bin" / "distrobox"
|
||||
self.eyes_proc = subprocess.Popen([str(distrobox), "enter", "dev", "--", str(EYES_PYTHON), str(EYES_PROG), "-v",
|
||||
"--watch-stdin"], env=env, stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, start_new_session=True)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
log("ft-eyes started" + ("" if EYES_CAMS.exists() else
|
||||
@@ -669,11 +480,6 @@ class Service:
|
||||
for line in data.decode("utf-8", "replace").splitlines():
|
||||
if line.strip():
|
||||
log(line)
|
||||
# For the Gaze page (gazecheck's problem): ft-gaze can't read SteamVR's tracker.
|
||||
if "layout is not recognized" in line:
|
||||
self.mmap_unknown = True
|
||||
elif "eye-server.mmap layout:" in line:
|
||||
self.mmap_unknown = False
|
||||
|
||||
def judge_eyes(self, m1, down):
|
||||
"""Which eyes (left, right) are closed, from SteamVR's openness (set 1); updates
|
||||
@@ -694,18 +500,11 @@ class Service:
|
||||
for k in (0, 1):
|
||||
self.lost[k] = unc[k] > (EYE_FOUND if self.lost[k] else EYE_LOST)
|
||||
if not down:
|
||||
self.counts["lost_left"] += self.lost[0] and self.one_eye != 1
|
||||
self.counts["lost_right"] += self.lost[1] and self.one_eye != 0
|
||||
self.counts["lost_left"] += self.lost[0]
|
||||
self.counts["lost_right"] += self.lost[1]
|
||||
return low
|
||||
|
||||
def on_sample(self, s):
|
||||
one = self.tracked()
|
||||
if one != self.one_eye:
|
||||
log("SteamVR now tracks both eyes" if one is None else
|
||||
f"SteamVR tracks the {SIDES[one]} eye only (Track Dominant Eye Only): going by that eye")
|
||||
self.one_eye = one
|
||||
self.fix.reset()
|
||||
self.checks.on_sample(s)
|
||||
kind = self.kind
|
||||
if kind != self.last_kind:
|
||||
log({"own": "our own tracker", "eyes": "SteamVR's eyes, each calibrated",
|
||||
@@ -731,8 +530,7 @@ class Service:
|
||||
eyes = [(p["hy"], p["hp"]) if "hy" in p else None for p in per]
|
||||
if not any(eyes):
|
||||
return
|
||||
one = self.one_eye
|
||||
hp, hit = (eyes[one] if one is not None and eyes[one] else next(e for e in eyes if e))[1], m1.get("hit")
|
||||
hp, hit = next(e[1] for e in eyes if e), m1.get("hit")
|
||||
self.counts["samples"] += 1
|
||||
self.last_sample = time.monotonic()
|
||||
down = hp < KEYBOARD_PITCH and not hit
|
||||
@@ -740,14 +538,8 @@ class Service:
|
||||
if down:
|
||||
self.counts["looking_down"] += 1
|
||||
return
|
||||
if own and self.one_eye is not None:
|
||||
# SteamVR judges only that eye's openness, and a blink closes both: ours still
|
||||
# sees the other, so it stays in.
|
||||
low[1 - self.one_eye] = low[self.one_eye]
|
||||
# Our tracker finds the pupils itself; SteamVR's openness still marks the blinks.
|
||||
bad = [eyes[k] is None or low[k] or (not own and self.lost[k]) for k in (0, 1)]
|
||||
if not own and self.one_eye is not None:
|
||||
bad[1 - self.one_eye] = True # SteamVR ignores that eye
|
||||
if all(bad):
|
||||
self.counts["blinks"] += 1
|
||||
return
|
||||
@@ -782,27 +574,11 @@ class Service:
|
||||
self.bad_at[k] = s["t"] # the fallback doesn't learn from these either
|
||||
return
|
||||
bad = [low[k] or self.lost[k] for k in (0, 1)]
|
||||
hy, hp = src["hy"], src["hp"]
|
||||
eyes = (s["src"].get("mmap2") or {}).get("eyes")
|
||||
one = self.one_eye
|
||||
if one is not None:
|
||||
# SteamVR's combined gaze already follows that eye alone; set 2's averages the
|
||||
# ignored one in, so mmap2 takes the eye's own reading. No fallback to learn.
|
||||
if bad[one]:
|
||||
self.counts["blinks"] += 1
|
||||
return
|
||||
if self.source == "mmap2":
|
||||
if not eyes:
|
||||
self.counts["dropped"] += 1
|
||||
return
|
||||
hy, hp = eyes[one]
|
||||
fy, fp = self.fix(hy, hp, s["t"], 1.0)
|
||||
cy, cp = self.correction(self.source, fy, fp)
|
||||
self.send(s["t"], fy + cy, fp + cp, fy, fp, None)
|
||||
return
|
||||
if all(bad):
|
||||
self.counts["blinks"] += 1
|
||||
return
|
||||
hy, hp = src["hy"], src["hp"]
|
||||
eyes = (s["src"].get("mmap2") or {}).get("eyes")
|
||||
for k in (0, 1):
|
||||
if bad[k]:
|
||||
self.bad_at[k] = s["t"]
|
||||
@@ -860,7 +636,6 @@ class Service:
|
||||
if words[:1] == ["lesson"] and len(words) == 5:
|
||||
try:
|
||||
rec = self.lesson(*map(float, words[1:]))
|
||||
self.checks.after_lesson(rec)
|
||||
reply = "refused" if "refused" in rec else f"ok {rec['lesson_deg']:.2f}"
|
||||
log(f"lesson {rec['lesson_deg']:.2f} deg at {rec['raw'][0]:+.1f},{rec['raw'][1]:+.1f}"
|
||||
+ (f", eyes off {', '.join('-' if m is None else f'{m:.2f}' for m in rec['miss'])}"
|
||||
@@ -871,11 +646,11 @@ class Service:
|
||||
elif words[:1] == ["status"]:
|
||||
reply = json.dumps(self.status())
|
||||
elif words[:1] == ["forget"]:
|
||||
self.forget_lessons()
|
||||
self.lives = {name: PointerLessons() for name in SOURCES}
|
||||
self.weights = {t: EyeWeights(self.bias) for t in TRACKERS}
|
||||
self.refit()
|
||||
self.save_lessons()
|
||||
reply = "ok"
|
||||
elif words[:1] and words[0] in ("quickcal", "calibrate", "fitcheck", "fivecheck", "calaccept", "calquit",
|
||||
"recheck"):
|
||||
reply = self.checks.command(words)
|
||||
elif words[:1] == ["eyes"] and len(words) == 2:
|
||||
try:
|
||||
secs = min(max(float(words[1]), 0.0), EYES_LEASE_MAX)
|
||||
@@ -883,14 +658,6 @@ class Service:
|
||||
reply = "ok"
|
||||
except ValueError:
|
||||
reply = "error bad seconds"
|
||||
elif words[:1] == ["wake"] and len(words) == 2:
|
||||
try:
|
||||
secs = min(max(float(words[1]), 0.0), WAKE_MAX)
|
||||
self.wake_until = max(self.wake_until, time.monotonic() + secs)
|
||||
self.update_awake()
|
||||
reply = "ok"
|
||||
except ValueError:
|
||||
reply = "error bad seconds"
|
||||
elif words[:1] == ["reload"]:
|
||||
self.load_settings()
|
||||
self.load_calibration()
|
||||
@@ -941,14 +708,12 @@ class Service:
|
||||
st.update(calibration_samples=cal.get("dots", 0), calibration_made=cal.get("made"),
|
||||
lessons=max(len(m) for m in w.misses), own_running=bool(own),
|
||||
own_reseat=any(e.get("reseat") for e in own.get("eyes", {}).values()))
|
||||
st.update(awake=self.awake, idle=None if self.awake else self.idle_reason())
|
||||
st.update(eyes_process=self.eyes_proc is not None, eyegrab=EYES_CAMS.exists(),
|
||||
tracker_setting=self.tracker_setting, own_installed=own_installed())
|
||||
st.update(eyes_process=self.eyes_proc is not None, eyegrab=EYES_CAMS.exists())
|
||||
st.update({"ft_gaze": self.proc is not None, "sample_age_s": round(time.monotonic() - self.last_sample, 2)
|
||||
if self.last_sample else None, "headset_on": self.steam.wearing(),
|
||||
"headset_on_since": self.steam.worn(), "last": [round(v, 2) for v in self.last] if self.last else None,
|
||||
"eyes_lost": self.lost, "fallback_ready": [self.fallback.ready(0), self.fallback.ready(1)],
|
||||
"checks": self.checks.status(), **self.counts})
|
||||
**self.counts})
|
||||
return st
|
||||
|
||||
def periodic(self):
|
||||
@@ -959,13 +724,8 @@ class Service:
|
||||
if mtime(CALIBRATION) != self.cal_mtime:
|
||||
self.load_calibration()
|
||||
self.refit()
|
||||
if mtime(CONF) != self.conf_mtime or (self.tracker_setting == "auto"
|
||||
and own_installed() != (self.tracker == "own")):
|
||||
if mtime(CONF) != self.conf_mtime:
|
||||
self.load_settings()
|
||||
if self.models_proc and self.models_proc.poll() is not None:
|
||||
self.models_proc = None
|
||||
self.check_eyes_model() # a net it fetched: ft-eyes moves to it
|
||||
self.update_awake()
|
||||
want = self.eyes_wanted()
|
||||
if want and not self.eyes_proc and time.monotonic() >= self.eyes_restart_at:
|
||||
self.start_eyes()
|
||||
@@ -977,7 +737,6 @@ class Service:
|
||||
self.eyes_sock.sendto(b"status", EYES_SOCKET)
|
||||
except OSError:
|
||||
pass # not up yet: status() says so once the last answer is old
|
||||
self.checks.periodic()
|
||||
if self.dirty:
|
||||
self.save_lessons()
|
||||
|
||||
@@ -986,17 +745,11 @@ class Service:
|
||||
next_verbose = time.monotonic() + 5
|
||||
while self.running:
|
||||
now = time.monotonic()
|
||||
if self.awake and not self.proc and now >= self.restart_at:
|
||||
if not self.proc and now >= self.restart_at:
|
||||
self.start_helper()
|
||||
for key, _ in self.sel.select(timeout=0.05 if self.checks.active else 0.5):
|
||||
for key, _ in self.sel.select(timeout=0.5):
|
||||
if key.data == "control":
|
||||
self.on_control()
|
||||
elif key.data == "checks":
|
||||
self.checks.on_readable()
|
||||
elif key.data == "eyes_reply":
|
||||
self.checks.on_eyes_reply(key.fileobj)
|
||||
elif key.data == "panel" and self.checks.panel_proc:
|
||||
self.checks.read_panel()
|
||||
elif key.data == "own":
|
||||
self.on_own()
|
||||
elif key.data == "eyes" and self.eyes_proc:
|
||||
@@ -1005,15 +758,12 @@ class Service:
|
||||
self.read_stdout()
|
||||
elif key.data == "stderr" and self.proc:
|
||||
self.read_stderr()
|
||||
self.checks.tick()
|
||||
self.sync_sources()
|
||||
if now >= next_periodic:
|
||||
self.periodic()
|
||||
next_periodic = now + 1.0
|
||||
if self.verbose and now >= next_verbose:
|
||||
log(json.dumps(self.status()))
|
||||
next_verbose = now + 5
|
||||
self.checks.stop()
|
||||
self.stop_helper()
|
||||
self.stop_eyes()
|
||||
if self.dirty:
|
||||
|
||||
+13
-101
@@ -7,7 +7,6 @@ the tracker has lost the other), EyeWeights (how much each eye counts), and Stea
|
||||
reports them.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
@@ -398,63 +397,6 @@ class LiveCorrection:
|
||||
EYE_LOST = 0.004
|
||||
EYE_FOUND = 0.0025
|
||||
|
||||
# SteamOS 0.4's "Track Dominant Eye Only" (SteamVR's settings, General, advanced): SteamVR's
|
||||
# tracker ignores the other eye, for someone whose eyes don't look at the same spot. Its
|
||||
# settings: steamvr.eyeTrackingDominantEyeOnly, and steamvr.dominantEye (0 left, 1 right,
|
||||
# SteamVR's default). Frametop then goes by that eye alone: a calibration that waits for both
|
||||
# eyes would never take a dot, and the other eye's reading isn't where the person looks.
|
||||
STEAMVR_SETTINGS = (Path.home() / ".config" / "openvr" / "config" / "steamvr.vrsettings",
|
||||
Path.home() / ".steam" / "steam" / "config" / "steamvr.vrsettings")
|
||||
|
||||
|
||||
def tracked_eye(paths=STEAMVR_SETTINGS):
|
||||
"""The one eye SteamVR's tracker follows (0 left, 1 right), or None for both. The first
|
||||
of the settings files that exists counts (SteamVR keeps only settings changed from its
|
||||
defaults, so a missing key is the default)."""
|
||||
for path in paths:
|
||||
try:
|
||||
text = Path(path).read_text()
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
steamvr = json.loads(text).get("steamvr")
|
||||
except (ValueError, AttributeError):
|
||||
return None
|
||||
if not isinstance(steamvr, dict) or steamvr.get("eyeTrackingDominantEyeOnly") is not True:
|
||||
return None
|
||||
return 0 if steamvr.get("dominantEye", 1) == 0 else 1
|
||||
return None
|
||||
|
||||
|
||||
class TrackedEye:
|
||||
"""tracked_eye(), read again when SteamVR's settings file changes (looked at no more than
|
||||
every CHECK seconds), since the setting can change while a service runs."""
|
||||
|
||||
CHECK = 2.0
|
||||
|
||||
def __init__(self, paths=STEAMVR_SETTINGS):
|
||||
self.paths = paths
|
||||
self.eye = None
|
||||
self.stamp = None
|
||||
self.checked = None
|
||||
|
||||
def __call__(self, now=None):
|
||||
now = time.monotonic() if now is None else now
|
||||
if self.checked is not None and now - self.checked < self.CHECK:
|
||||
return self.eye
|
||||
self.checked = now
|
||||
stamp = []
|
||||
for path in self.paths:
|
||||
try:
|
||||
st = os.stat(path)
|
||||
stamp.append((st.st_mtime_ns, st.st_size))
|
||||
except OSError:
|
||||
stamp.append(None)
|
||||
if stamp != self.stamp:
|
||||
self.stamp = stamp
|
||||
self.eye = tracked_eye(self.paths)
|
||||
return self.eye
|
||||
|
||||
|
||||
class EyeFallback:
|
||||
"""The gaze from one eye, while the tracker has lost the other.
|
||||
@@ -602,9 +544,6 @@ class SteamEyeLog:
|
||||
each time the headset goes on ("HMD on"): the eye model starts over then too."""
|
||||
|
||||
PATH = Path.home() / ".local" / "share" / "Steam" / "logs" / "eyetracking.txt"
|
||||
# It writes "HMD on" again every minute or so while on, and flickers off for 0.01-0.3 s:
|
||||
# only an on after an off of BLIP or longer counts.
|
||||
BLIP = 1.5
|
||||
|
||||
def __init__(self):
|
||||
self.pos = 0
|
||||
@@ -656,14 +595,9 @@ class SteamEyeLog:
|
||||
self.starts.append(t)
|
||||
restarted = not first
|
||||
elif "HMD on" in line:
|
||||
off = bool(self.offs) and (not self.wears or self.offs[-1] > self.wears[-1])
|
||||
if off and self.wears and t - self.offs[-1] < self.BLIP:
|
||||
self.offs.pop() # the sensor flickering: it never came off
|
||||
elif off or not self.wears:
|
||||
self.wears.append(t)
|
||||
self.wears.append(t)
|
||||
elif "HMD off" in line:
|
||||
if not self.offs or (self.wears and self.wears[-1] > self.offs[-1]):
|
||||
self.offs.append(t)
|
||||
self.offs.append(t)
|
||||
elif "Accept usercal" in line:
|
||||
self.accepts.append(t)
|
||||
elif "Reject usercal" in line:
|
||||
@@ -698,53 +632,31 @@ def cross_validate(points, mode):
|
||||
return errs
|
||||
|
||||
|
||||
def steady_samples(samples, vergence_jump=1.5, why=None, eye=None):
|
||||
def steady_samples(samples, vergence_jump=1.5):
|
||||
"""The samples of one look at one spot where the tracker had both eyes: none in a blink
|
||||
(openness under half its median over the samples), none where it had lost an eye (its
|
||||
variance over EYE_LOST), and none where the angle between the eyes' directions (`lr`, the
|
||||
vergence) is more than `vergence_jump` degrees from its median over the samples. The
|
||||
vergence itself depends on distance (about 2.8 degrees for a screen 1.3 m away, a
|
||||
fraction of one far off), so only a jump away from what it was during this look means
|
||||
the tracker lost an eye. Without the mmap there's nothing to judge by: all are kept.
|
||||
`eye` (0 left, 1 right; see tracked_eye) judges that eye alone: the other one's loss,
|
||||
openness and the vergence don't count.
|
||||
`why`, a dict, gets how many were dropped for each reason: "lost_left", "lost_right",
|
||||
"lost_both", "blink" and "vergence" (each sample once, for the first that applies)."""
|
||||
if why is None:
|
||||
why = {}
|
||||
|
||||
def openness(o):
|
||||
return o[eye] if eye is not None else min(o)
|
||||
the tracker lost an eye. Without the mmap there's nothing to judge by: all are kept."""
|
||||
# Openness: a blink is a sharp drop from what it was during this look. Not a fixed
|
||||
# level: looking down, the upper lids come down with the eyes, and in bright light you
|
||||
# squint, so the reading can stay under 0.5 for the whole look while the tracker follows
|
||||
# the eyes fine (a calibration dot at the bottom of the bright round failed that way).
|
||||
opens = [openness(o) for o in ((smp["src"].get("mmap1") or {}).get("open") for smp in samples) if o]
|
||||
opens = [min(o) for o in ((smp["src"].get("mmap1") or {}).get("open") for smp in samples) if o]
|
||||
floor = max(0.12, 0.5 * statistics.median(opens)) if len(opens) >= 5 else 0.12
|
||||
seen = []
|
||||
opened = []
|
||||
for smp in samples:
|
||||
m1 = smp["src"].get("mmap1") or {}
|
||||
o = m1.get("open")
|
||||
lost = [u > EYE_LOST for u in m1.get("unc") or [0, 0]]
|
||||
if eye is not None:
|
||||
lost[1 - eye] = False
|
||||
# A lost eye's openness reads 0 too, so a lost eye is named before a blink.
|
||||
key = ("lost_both" if all(lost) else "lost_left" if lost[0] else "lost_right") if any(lost) else \
|
||||
"blink" if o and openness(o) < floor else None
|
||||
if key:
|
||||
why[key] = why.get(key, 0) + 1
|
||||
else:
|
||||
seen.append(smp)
|
||||
if eye is not None:
|
||||
return seen
|
||||
o = (smp["src"].get("mmap1") or {}).get("open")
|
||||
if not o or min(o) >= floor:
|
||||
opened.append(smp)
|
||||
|
||||
def vergence(smp):
|
||||
return (smp["src"].get("mmap1") or {}).get("lr", (smp["src"].get("mmap2") or {}).get("lr"))
|
||||
have = [v for v in map(vergence, seen) if v is not None]
|
||||
opened = [smp for smp in opened if max((smp["src"].get("mmap1") or {}).get("unc") or [0]) <= EYE_LOST]
|
||||
have = [v for v in map(vergence, opened) if v is not None]
|
||||
if len(have) < 5:
|
||||
return seen
|
||||
return opened
|
||||
med = statistics.median(have)
|
||||
kept = [smp for smp in seen if vergence(smp) is None or abs(vergence(smp) - med) <= vergence_jump]
|
||||
if len(kept) < len(seen):
|
||||
why["vergence"] = why.get("vergence", 0) + len(seen) - len(kept)
|
||||
return kept
|
||||
return [smp for smp in opened if vergence(smp) is None or abs(vergence(smp) - med) <= vergence_jump]
|
||||
-1348
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -161,10 +161,11 @@ class GazeReader:
|
||||
env = dict(os.environ)
|
||||
# The Frametop desktop has its own runtime dir; podman needs the real one.
|
||||
env["XDG_RUNTIME_DIR"] = f"/run/user/{os.getuid()}"
|
||||
subprocess.run([str(REPO / "scripts" / "container-up.sh")], env=env, check=False)
|
||||
distrobox = Path.home() / ".local" / "bin" / "distrobox"
|
||||
# ft-gaze quits when its stdin closes, which is the one thing distrobox passes on
|
||||
# when we go away (even if we're killed). in-box starts the container first, and picks
|
||||
# the release's own on a release install.
|
||||
self.proc = subprocess.Popen([str(REPO / "scripts" / "in-box"), str(HELPER), "--watch-stdin"], env=env,
|
||||
# when we go away (even if we're killed).
|
||||
self.proc = subprocess.Popen([str(distrobox), "enter", "dev", "--", str(HELPER), "--watch-stdin"], env=env,
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
start_new_session=True)
|
||||
out = Gio.UnixInputStream.new(self.proc.stdout.fileno(), False)
|
||||
@@ -202,7 +203,7 @@ class GazeReader:
|
||||
return
|
||||
print(line, file=sys.stderr)
|
||||
text = line.removeprefix("ft-gaze: ")
|
||||
if not text.startswith(("action manifest", "eye-server.mmap open", "eye-server.mmap layout:")):
|
||||
if not text.startswith(("action manifest", "eye-server.mmap open")):
|
||||
self.last_err = text # worth repeating if ft-gaze stops (not the startup lines)
|
||||
self.on_status(text)
|
||||
stream.read_line_async(GLib.PRIORITY_DEFAULT, self.cancel, got)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
[Desktop Entry]
|
||||
Type=Application
|
||||
Name=Frametop Gaze Probe (development)
|
||||
GenericName=Eye tracking development tool
|
||||
Comment=For developing Frametop's gaze tracking. Day to day, the calibration and checks are on the Gaze page of Frametop Input Settings
|
||||
Name=Frametop Gaze Probe
|
||||
GenericName=Eye tracking playground
|
||||
Comment=How accurate the headset's eye tracking is on your screens, and gaze clicking with calibration
|
||||
Exec=@REPO@/gaze/probe/ft-gazeprobe
|
||||
Icon=view-visible
|
||||
Categories=Development;
|
||||
Categories=Utility;Development;
|
||||
Keywords=eye;gaze;tracking;calibration;pointer;steamvr;frametop;
|
||||
@@ -1,96 +0,0 @@
|
||||
# Recording your eyes for the Frametop eye dataset
|
||||
|
||||
Version: 2026-10-16
|
||||
|
||||
Frametop's eye tracking needs a better model than it has: one that keeps working when you look far to the side, blink, squint, wear glasses or makeup, or when the headset moves on your face. To train it, we're collecting recordings of many people's eyes into a dataset that only the maintainers can read. This page explains what the eye recorder records, where it goes, who can see it, and what you agree to if you take part. Please read all of it.
|
||||
|
||||
## Who runs this
|
||||
|
||||
Frametop is an independent open-source project, maintained by DeeJanuz (<https://github.com/DeeJanuz>). It isn't affiliated with or endorsed by Valve or Hugging Face. Steam and Steam Frame are trademarks of Valve Corporation. This text was written without a lawyer, in good faith; if something in it seems wrong or unclear, please say so before you take part.
|
||||
|
||||
You can reach the maintainer through the Frametop repository's issues (<https://github.com/Frametop/frametop/issues>) or the dataset's discussion page on Hugging Face. Both are public.
|
||||
|
||||
## Who can take part
|
||||
|
||||
- You must be **18 or older**.
|
||||
- For now, you can't take part if you live in **Illinois, Texas or Washington** (USA). Those states have laws about biometric data, which includes images of the iris, that need more than this project can provide yet.
|
||||
- Take part only if you're comfortable with the maintainers seeing close-up images of your eyes.
|
||||
|
||||
## Your eyes are biometric data
|
||||
|
||||
The headset's eye cameras film each eye from a few centimetres away. The iris in those images can identify a person, the way a fingerprint can. That's why this dataset works differently from Frametop's hand dataset:
|
||||
|
||||
- **The eye images are never public.** Your headset encrypts them before they're uploaded, so that only the maintainers' keys can read them. Hugging Face, other contributors and people who download the dataset get only the encrypted files.
|
||||
- **What's trained from them can be public.** Models trained on the dataset can be published and shared. They don't contain your images, though no one can rule out that a model remembers something about the eyes it was trained on.
|
||||
|
||||
## What is recorded
|
||||
|
||||
While a round runs, the eye recorder saves:
|
||||
|
||||
- **Eye images.** Infrared images from the headset's 2 eye cameras, about 90 a second each. They show your eyes, eyelids, eyelashes and the skin around them, and your glasses or contact lenses if you wear them. Encrypted in the export.
|
||||
- **Where you looked.** Where each dot was when you looked at it, what Frametop's eye tracker and SteamVR's found in each image (pupil and glint positions, gaze directions), and how the fit check rated the headset's position. These stay readable in the export, so a pull request can be checked before anyone decrypts it.
|
||||
- **Motion and settings.** The headset's position and rotation, many times a second, and its IPD setting (the distance between the lenses).
|
||||
- **Prompts.** What you were asked to do and when.
|
||||
- **Your answers.** Glasses, contacts or inserts, eye makeup, the face pad and strap, and your measured IPD if you gave it. Readable in the export.
|
||||
- **Health answers, only if you tick their box.** Eye surgery and conditions such as a turned eye, a droopy eyelid or nystagmus. They're health data, so they're kept in a separate file and encrypted like the images. You can leave them out, and still take part.
|
||||
- **Times.** When each round was recorded, with your time zone.
|
||||
- **A contributor id.** A random number made on your headset the first time you agree to this page. It isn't made from your name, your Steam account or your headset. It keeps your rounds together and lets you withdraw them.
|
||||
|
||||
The recorder doesn't record sound, the headset's outward cameras, your name, your email address or your Steam account.
|
||||
|
||||
**Your Hugging Face account.** You upload with your own Hugging Face account, and your pull request shows its username next to your contributor id, publicly. The images in it are encrypted, but anyone can see that this account contributed eye recordings, and read the readable parts listed above. Use an account you're happy to have linked to them.
|
||||
|
||||
## What it's used for
|
||||
|
||||
- Training and testing eye-tracking models: finding the pupil, the glints and the eyelids, and working out where you're looking, including when the headset sits badly. Mainly for Frametop, and possibly elsewhere (below, "The license").
|
||||
- Checking and improving the recorder's fit coach, which helps people put the headset on well.
|
||||
- It isn't used to recognise or identify people, and the maintainers won't try.
|
||||
|
||||
## Nothing leaves your headset unless you send it
|
||||
|
||||
- Recordings stay on your headset, in `~/.local/share/frametop/eyes/contrib`. Nothing is uploaded automatically.
|
||||
- Before you share anything, you can look through every recording in the Review page. You can delete any stretch of a take, a whole take or a whole round. Export leaves out what you deleted.
|
||||
- You upload the export yourself, from the Upload page. That opens a pull request on the dataset. The maintainer checks it before it becomes part of the dataset, and may decline it. Until it's merged, you can close the pull request yourself.
|
||||
|
||||
## Where it goes, and who can read it
|
||||
|
||||
- The dataset is hosted on Hugging Face (<https://huggingface.co/datasets/DeeJanuz/frametop-eyes>), whose servers may be outside your country, for example in the USA. It's gated: people must accept its terms to download it, and what they get has the eye images and health answers encrypted.
|
||||
- Only the maintainers hold the keys that decrypt them. Today that's DeeJanuz, with a key on each of two of their own computers (one of them trains the models) and a backup copy stored offline. The keys are never uploaded. If a key ever leaked, uploads made to it could be read: the maintainers would say so on the dataset's page and stop using it.
|
||||
- Decrypted images are kept only on the maintainers' own computers, for training and checking models, and aren't shared with anyone else or uploaded to other services.
|
||||
- Recordings stay in the dataset until they're withdrawn or the dataset is taken down.
|
||||
|
||||
## Safety
|
||||
|
||||
The rounds ask you to look at dots, including far to the sides, to blink and squint, and sometimes to turn your head or move the headset on your face. Sit where you normally use the headset and stop whenever anything is uncomfortable, or if your eyes get tired or sore. If you have a condition that bright or moving lights affect, such as photosensitive epilepsy, don't take part. You take part at your own risk.
|
||||
|
||||
## The license
|
||||
|
||||
- **You give the maintainer of Frametop a non-exclusive, worldwide, royalty-free license to use your contribution for any purpose, including commercially.** That includes copying it, changing it, and training, publishing and selling models made from it, in Frametop and elsewhere. It doesn't include publishing your eye images or health answers, or giving them to anyone outside the maintainers. The maintainer today is DeeJanuz. If someone else, or an organization, takes over maintaining Frametop, this license passes to them, with the same limits. It's non-exclusive: you keep any rights you have in your recordings and can do what you like with your own copies. It ends for a recording when you withdraw it, except for models already trained with it.
|
||||
- You confirm that you have the right to give this license: the recordings are of your own eyes.
|
||||
- There is no payment. The dataset and the eye recorder come with no warranty, and as far as the law allows, the maintainer isn't liable for any loss or damage from taking part.
|
||||
|
||||
## Withdrawing
|
||||
|
||||
You can withdraw a contribution at any time, without giving a reason:
|
||||
|
||||
- **Before it's merged:** close your pull request on Hugging Face. The maintainer deletes its files.
|
||||
- **After it's merged:** post on the dataset's discussion page, or in the Frametop repository's issues, with your contributor id (shown in the eye recorder) and which rounds, or "all". Post from the Hugging Face account that uploaded them if you can, so the maintainer can tell it's you; otherwise, say how to check. These requests are public, so don't add anything else that identifies you.
|
||||
|
||||
Then, usually within 30 days:
|
||||
|
||||
- your recordings are deleted from the dataset and purged from its repository's history, and the maintainers delete their decrypted copies;
|
||||
- they're left out of anything trained after that.
|
||||
|
||||
What can't be undone: models already trained with them.
|
||||
|
||||
## Your rights
|
||||
|
||||
Depending on where you live (for example in the EU or the UK, under the GDPR), the law may give you more rights over this data: to get a copy of it, to have it corrected or deleted, to object to its use, and to complain to your data protection authority. Eye images that can identify you and health answers are special kinds of data there, and they're used only because you explicitly agreed to it, here and with the health answers' own tick. You can withdraw that agreement at any time, as above. Withdrawing doesn't make earlier use unlawful. To use any of these rights, contact the maintainer as above.
|
||||
|
||||
## Changes to this text
|
||||
|
||||
If this text changes, the eye recorder shows the new version and asks you again before your next round. Each upload records the version you agreed to, and recordings you already uploaded stay under that version, unless you agree to a newer one or withdraw them.
|
||||
|
||||
## Agreeing
|
||||
|
||||
By ticking the three boxes and "Agree and continue", you confirm that you're 18 or older, that you don't live in Illinois, Texas or Washington, and that you agree to the above, including the use of images of your eyes as described. You can still decide not to upload anything.
|
||||
@@ -1,178 +0,0 @@
|
||||
# Eye recorder: design (2026-10-14)
|
||||
|
||||
The eye recorder guides a person through recording their eyes with the headset's two eye cameras, at known gaze targets and through the cases where eye tracking breaks. It also fits the headset: it shows whether each eye sits where the cameras see it best, shows the IPD setting, and catches the headset shifting on the face, with instructions to put it back. Recordings stay on the headset until the person uploads them. Only the maintainers can read what's uploaded (below, "Privacy").
|
||||
|
||||
It's the hand recorder's sibling (`hands/rec/DESIGN.md`): the same session runner pattern, headset button, review, export and Hugging Face upload, with eye cameras instead of hand cameras. Like the hand recorder it ships in Frametop and, later, as a standalone app for people without Frametop (Frametop/frametop-eye-recorder, Frametop as a submodule).
|
||||
|
||||
The data is for training a better eye tracker than `gaze/tracker/` (ft-eyes, a classic pupil and glint finder) and SteamVR's, one that holds up at the edges of view, through blinks and squints, with glasses and makeup, and when the headset moves.
|
||||
|
||||
## Decisions (2026-10-14)
|
||||
|
||||
| | Decided |
|
||||
|---|---|
|
||||
| Who reads the recordings | Maintainers only. The eye cameras see the iris, which identifies a person. Trained models can be public; the images never are. |
|
||||
| How | Encrypted frames in a gated dataset, `DeeJanuz/frametop-eyes` (below, "Privacy"). |
|
||||
| Keys | Two maintainer keys, both recipients of every upload: key 1 on the maintainer's training machine (the 7i), key 2 kept offline in the maintainer's password manager. |
|
||||
| Packaging | Backend here in `gaze/rec/`, then a standalone repo with Frametop as a submodule, as the hand recorder does. |
|
||||
| Sessions | Short rounds picked from a bounty board (below, "Bounty board"), not one long session. Each round starts with a fit check and 9 dots. |
|
||||
| Bounty list | `bounties.json` in the dataset repo, fetched by the app (a copy ships for offline use). The board shows your own done marks and a maintainer-set priority tag per bounty, not community counts. |
|
||||
| Fit and slip source | Our tracker, ft-eyes: pupil and glint positions in the camera image. It needs ft-eyegrab, the root service `gaze/tracker/install.sh` installs. |
|
||||
| Where the fit coach runs | A fitment step in the recorder before each round, for better recordings, and the put-on and slip check in Frametop's gaze service. |
|
||||
| Shared code | The hand recorder's non-hand parts move into a shared module first, and the bounty board is built there too, so the hand recorder can use it next. |
|
||||
| License | As hands: contributors give the maintainer a license for any use, including commercial models; the eye images and health answers are never published or shared outside the maintainers (user, 2026-10-14). |
|
||||
| Standalone repo | Created after the window and a full round pass in the headset; step 9 comes first (user, 2026-10-14). |
|
||||
| Checklist | Glasses/inserts/contacts, eye makeup, measured IPD, and eye surgery or conditions. That last one is health data: a separate opt-in, and its answers are encrypted like the images. |
|
||||
|
||||
## Privacy
|
||||
|
||||
A private Hugging Face repo can't take contributions: people outside the org can't see it, so they can't open a pull request, and org membership would let every member read every contribution. So the dataset is gated like `DeeJanuz/frametop-hands`, and the export encrypts what identifies a person before upload:
|
||||
- Every frame chunk is encrypted with [age](https://age-encryption.org) (pyrage, which has aarch64 wheels: `pyrage-1.4.0-cp310-abi3-manylinux_2_17_aarch64`) to the recipients in `recipients.txt`: the maintainers' public keys: key 1 on the 7i, key 2 offline. The private keys never go in a repo.
|
||||
- The health answers go in `health.json.age`, encrypted the same way.
|
||||
- Labels, timings, fit readings, the IPD and the rest of session.json stay readable, so `validate.py` and a review can check a pull request before anyone decrypts it.
|
||||
- Contributors open pull requests from their own accounts, so `hub.py` and the review tools carry over from hands.
|
||||
|
||||
The risk: if a private key leaks, every upload made to it can be read. A new key is one more recipient for new uploads; old uploads stay readable to the old key until they're re-encrypted. Withdrawal means deleting the files and squashing history, as with hands.
|
||||
|
||||
## What's measured already
|
||||
|
||||
- **The eye cameras** (`gaze/tracker/ft-eyegrab.c`): 512x400 8-bit IR, about 90 fps each when SteamVR runs them at 90 (`gaze/eye-rate.py`; at its default 15 fps a recording is worth little, so the recorder checks the rate first). Camera 0 is the right eye (steep angle), camera 1 the left (head-on). ft-eyegrab copies the frames read-only into `/dev/shm/frametop-eyes-cams` while someone touches `/dev/shm/frametop-eyes-want`.
|
||||
- **Size.** Raw, both cameras are 36.9 MB/s, 2.2 GB a minute. zstd level 1 on 200 MB of the `fit1` capture (2026-10-14): 56 % of the size at about 250 MB/s on one core, so compressing while recording costs about 15 % of a core. Level 3: 53 %, 2.3x slower. Level 9: 49 %, 9.5x slower. The recorder uses level 1 while recording: about 1.2 GB per recorded minute, so a session of about 10 recorded minutes (below) is about 12 GB, close to a hand session.
|
||||
- **IPD.** SteamVR reports it (2026-10-14, a probe against the live vrserver): `Prop_UserIpdMeters_Float` 0.0651, `Prop_IpdUIRangeMinMeters` 0.057, `Max` 0.071, `Prop_MinimumIpdStepMeters` 0.0015, and the eye-to-head transforms are ±IPD/2. `driver_cv.so` reads the range from the headset's EEPROM. So the recorder can read the IPD setting live and record it with every take. Not yet known: whether the eye cameras move with the lenses when the IPD changes. That decides how the IPD guidance works (below), and it's the first headset test.
|
||||
- **ft-eyes** already finds each pupil (centre, ellipse) and glint pair per frame, keeps a per-eye slip estimate from the glints (`eyes_model.SlipTracker`), and resets its shift when frames stop for 3 s (the headset was off). `gaze/fitcheck.py` already keeps per-eye tracked and lost by gaze direction, with hints.
|
||||
|
||||
## Parts
|
||||
|
||||
| Part | What it does |
|
||||
|---|---|
|
||||
| `gaze/rec/session.py` (Python, no Qt) | The session runner, like `hands/rec/session.py`: reads a round's script, drives the panel, starts and stops the frame writer per step, reads the live fit for feedback, writes each take. `--dry-run`, `--next-after`, stdin steering and `--button-device` as in hands. |
|
||||
| `gaze/rec/rounds/<id>.json` | Each kind of round's steps, targets and timings (below, "Bounty board"). |
|
||||
| `gaze/rec/eyerec.py` | The frame writer: reads the shared frames (as `lab/ft-eyes-record` does) and writes zstd-1 chunks, one per second per camera, with an index (below, "Files"). A thread of the session, not a process: ft-eyes-record shows the copy needs no numpy. |
|
||||
| `gaze/fitcoach.py` (shared) | The fit coach: from ft-eyes' pupil and glint positions, how the headset sits on this face, and what to do about it (below, "Fit coach"). The recorder, the Gaze page's fit check and the put-on check all use it. |
|
||||
| ft-gazepanel `rec` mode | The headset panel. The gaze panel already draws dots at exact head-relative directions, which the ground truth needs, so it gains a `rec` mode with the hand panel's text, step, countdown and paused lines rather than a second panel (below, "Panel"). |
|
||||
| `gaze/rec/takes.py`, `validate.py`, `hub.py` | Review, export, checks and upload, from `hands/rec/`'s. The shared parts move into a common module rather than being copied (below, "Shared with the hand recorder"). |
|
||||
| `gaze/rec/ft_eyerec.py` + `main.qml` | The desktop window: consent, the bounty board, the checklist and eye camera check, the round with its fit card, review, export, upload. Launched by `ft-eyerec` (the dev container), installed to the app menu by `install.sh`. The standalone app gets a Qt Quick Controls window, as the hand recorder's does. |
|
||||
| `gaze/rec/CONSENT.md`, `UPLOAD.md` | The texts. Eye images need a stronger consent than hands (biometric data, who can read it, how to withdraw). They need a review like `frame-hands/notes/legal-review-2026-10-03.md` before uploads open. |
|
||||
|
||||
## Fit coach
|
||||
|
||||
It reads, per eye, from ft-eyes: the pupil centre, its ellipse, the glint pair (both glints and their spacing), and how often the eye was found over the last second. It compares them with a reference: where a well-fitted eye sits in that camera's image. At first the reference comes from the maintainer's headset; once contributions come in, it comes from their rounds with good nine-dot fits (every round's fitment step records exactly that), per headset model. The coach says which reference it used, and until it's built from more than one face its hints are marked provisional.
|
||||
|
||||
What it can tell, and the instruction it gives:
|
||||
|
||||
| Reading | Means | Instruction |
|
||||
|---|---|---|
|
||||
| Both pupils high or low in their images | The headset sits low or high on the face | "Push the headset up" / "Let it down a little", with the top strap |
|
||||
| Both pupils left or right | The headset sits off to one side | "Slide the headset right" |
|
||||
| Pupils apart or together, compared with the reference at this IPD | IPD set too narrow or wide | "Turn the IPD wheel wider", until both are centred |
|
||||
| One pupil higher than the other | The headset is tilted | "Level the headset" |
|
||||
| Glint pair spacing larger or smaller than the reference | The eye is nearer or further than usual (eye relief: face pad, straps, glasses) | "Tighten the back strap" / "The headset is pressing close: loosen it" |
|
||||
| A glint missing, or extra bright spots | Glasses reflections, or the lens fogged or smudged | "Tilt your glasses" / "Wipe the lenses" |
|
||||
| An eye found in under half the frames, looking straight ahead | Lashes on the lens, a droopy lid, or the eye outside the camera's view | The per-eye card says which |
|
||||
|
||||
The IPD row depends on the first headset test. If the cameras move with the lenses, a pupil's sideways offset in its image directly says how far that eye is from its lens axis, and the coach turns the IPD wheel's direction into "wider" or "narrower" until both offsets are about zero, at any IPD. If the cameras are fixed, the offset depends on the IPD setting too, and the coach needs the reference at each IPD step, built from rounds recorded at different settings (there's no IPD sweep: the setting can't be changed with the headset on).
|
||||
|
||||
**Put-on and shift check.** After a good fit, the coach keeps that fit as the person's baseline (`~/.local/state/frametop/gaze/fit-baseline.json`: per eye, the pupil looking straight ahead and the glint pair, with the IPD setting). Then:
|
||||
- When the headset goes back on (frames resume after ft-eyes' 3 s gap), it shows the fit card for a few seconds while the person looks ahead. If the pupils sit more than a threshold (start with 1.5 mm equivalent, tuned from data) from the baseline, it says which way to move the headset, live, until it's back. If the IPD setting changed, it says so first.
|
||||
- While in use, ft-eyes' glint slip estimate already tracks small movement. A shift past the threshold that stays for 2 s gets a quiet notice ("Your headset slipped down: push it up, or look at the dot to recalibrate"), not a modal.
|
||||
In the recorder the fit coach runs as each round's fitment step, and the shift warning runs during the round. In Frametop the put-on and slip check becomes part of the gaze service, off by default until it's been tried.
|
||||
|
||||
## Bounty board
|
||||
|
||||
People pick what to record from a board of bounties. Each bounty is one round of about 5 minutes, in step mode: the headset's right button starts each step (press = next or pause, two presses = redo, hold = stop), as in the hand recorder. Every round starts the same way:
|
||||
1. **Fitment** (not recorded as a take, but its readings go into fit.json): the fit coach until the headset sits well (below, "Fit coach"). Skippable after a warning, and the round records that it was skipped. (The coach is off for now, 2026-10-16: then it only asks for a comfortable fit.) It shows the IPD setting, read from SteamVR when the round starts. The setting can't be changed with the headset on (2026-10-16), so there's no IPD step: a wrong setting means stopping, setting it, and starting again. Each take's ipd.jsonl still logs it.
|
||||
2. **Nine dots** (recorded): a 3x3 grid to 20 degrees on the mid background. It's the round's own reference, so each round can be labelled alone, even after a re-seat.
|
||||
3. **The bounty's steps.**
|
||||
|
||||
How to build a new one, and the rules a round script must follow (beeps for anything done with the eyes shut, cue words as labels, timing, reach, comfort): `rounds/README.md`.
|
||||
|
||||
Every round uploads on its own. The board marks the bounties you've done, by your contributor id in `profile.json`, with how many times. You can do one again.
|
||||
|
||||
The launch bounties:
|
||||
|
||||
| Bounty | What | Edge case it covers |
|
||||
|---|---|---|
|
||||
| View edges | Dots at the panel's edges and corners: 30 degrees to the sides, 23 up and down, about 35 in the corners (the rec panel is 64 degrees across; every round's dots are checked against it) | Eyes turned far: the lid, the glints lost, the pupil foreshortened |
|
||||
| Blinks and lids | Blink on a beep, close both eyes on a beep and open them on the next (sounds.py: the eyes are shut, so words can't cue them), wink each eye, squint, eyes wide, half-closed, looking down with lids lowered | Closed and partly closed eyes; one eye missing |
|
||||
| Headset slips | Nudge the headset up, down, left, right, tilt it; then take it off and put it back on; dots after each | Headset shifting on the face: the put-on check's data |
|
||||
| Glasses | A 5x5 grid and the edges with glasses or inserts on (offered when the checklist says so) | Lens reflections, the glasses' frame in view |
|
||||
| Jumps and pursuit | A dot jumping between random spots (0.5 to 1.5 s each), then moving on circles, lines and a figure eight at two speeds | Saccades, overshoot, smooth pursuit, the tracker's lag |
|
||||
| Depth | Dots at 0.5 m, 1.5 m and 5 m in turn (the panel's distance changes) | Vergence; the eyes' angle with distance |
|
||||
| Light changes | The background steps dark to bright and back while looking at a dot, then a 5x5 grid at each of three backgrounds | Pupil dilation and constriction; error against pupil size |
|
||||
| Face moves | Smile, raise eyebrows, frown, yawn, talk (read a line aloud: the microphone isn't recorded), chew | The face moving the headset and the lids |
|
||||
| Head turning | A dot fixed in the room while turning the head (head poses logged) | The eyes counter-rotating (VOR), headset jiggle |
|
||||
| Reading | Lines of text to read, then a page of icons to look over | Natural use |
|
||||
|
||||
Later candidates, not at launch: eye makeup (the grid and blinks with mascara or eyeliner; the checklist still records makeup on every round), long wear (the grid after 30+ minutes in the headset).
|
||||
|
||||
### bounties.json
|
||||
|
||||
In the dataset repo, fetched when the board opens (a copy in the app for offline use, used when the fetch fails):
|
||||
|
||||
```json
|
||||
{"schema": 1, "updated": "2026-10-14",
|
||||
"bounties": [{"id": "edges", "title": "View edges", "blurb": "...", "minutes": 5,
|
||||
"priority": "high|normal|low|retired", "script": "edges", "needs": {"glasses": true}}]}
|
||||
```
|
||||
|
||||
`script` names a round in the app's `rounds/` folder: a new kind of round needs a release, but the list, wording and priorities don't. `needs` hides a bounty unless the checklist matches. A bounty id the app doesn't know is shown as "needs an update".
|
||||
|
||||
The board is in the shared recorder module, so the hand recorder's v2 sections can become bounties later.
|
||||
|
||||
### Checklist
|
||||
|
||||
Asked once and kept in profile.json, editable before each round, all optional, none identifying: glasses (none, glasses, prescription inserts, contacts), eye makeup today (none, mascara, eyeliner, lash extensions), the face pad and strap in use, and a measured IPD if they know it (from an optician). Separately, behind its own consent tick: eye surgery or conditions (LASIK or other surgery, a lazy or turned eye, a droopy lid, nystagmus, other). Those answers go into `health.json`, which the export encrypts. Eye colour isn't asked: the IR images show what matters about it.
|
||||
|
||||
## Panel
|
||||
|
||||
ft-gazepanel gains `show rec`: a large head-locked panel (FULL_DEG, 64 degrees, with a solid background whose brightness the script sets) that keeps every existing command (`dot`, `bg`, `title`, `text`, `note`, `eye`, `hints`) and adds what the hand panel has: `step`, `big` (countdown), `action`, `keys`, `rec on|off`, `paused on|off`. New for the ground truth:
|
||||
- `path <kind> <params> <seconds>`: the panel moves a dot itself (circle, line, figure eight), so it's smooth at the headset's frame rate, not at the socket's.
|
||||
- `distance <metres>`: moves the panel for the Depth bounty.
|
||||
- `world <x> <y> <z>`: a dot fixed in the room (the Head turning bounty), as the hand panel's touch target.
|
||||
- Every picture the panel shows with a dot logs that dot's direction and the frame's time to `targets.jsonl` (CLOCK_MONOTONIC_RAW, the eye frames' clock). The ground truth is what was on screen when, not what was asked.
|
||||
|
||||
## Files
|
||||
|
||||
```
|
||||
~/.local/share/frametop/eyes/contrib/
|
||||
profile.json consent, contributor id, checklist, bounties done
|
||||
rounds/<YYYYMMDD-HHMMSS>-<bounty>/
|
||||
session.json the bounty, checklist, versions, takes, eye camera rate, IPD at start
|
||||
fit.json the fit coach's readings: the fitment step, and any slip warnings
|
||||
health.json the opt-in health answers (encrypted in the export)
|
||||
takes/<NN>-<step group>/ numbered in order: NN-dots (the nine dots), then the bounty's
|
||||
cam0/000000.zst ... right eye: one zstd chunk per second, frames back to back (512x400 u8)
|
||||
cam1/000000.zst ... left eye
|
||||
index.tsv per frame: camera, chunk, offset in it, slot, CLOCK_MONOTONIC_RAW time
|
||||
targets.jsonl what the panel showed, when (direction, distance, state, background)
|
||||
prompts.jsonl what the person was asked to do, when (as hands)
|
||||
poses.jsonl head poses, 250 Hz (as hands; needed for Head turning)
|
||||
ipd.jsonl the IPD setting at the start and when it changes
|
||||
steamvr.jsonl SteamVR's gaze from ft-gaze, per sample (a baseline, not a label)
|
||||
own.jsonl ft-eyes' pupils, glints and gaze (a baseline, not a label)
|
||||
take.json status, deleted ranges, parts, clock samples, as hands
|
||||
exports/<session>/
|
||||
```
|
||||
|
||||
The export recompresses nothing (the chunks are zstd already), drops deleted ranges by rewriting the chunks they touch, encrypts each chunk and `health.json` to `recipients.txt` (`.age`), and writes the manifest and `SHA256SUMS` over the encrypted files. It never includes `/persist` files: unlike hands, nothing here needs the unit's calibration, and the eye cameras' isn't in `/persist` anyway (2026-10-14: `device_config.json` has the displays' transforms only).
|
||||
|
||||
## Shared with the hand recorder
|
||||
|
||||
Much of `hands/rec/` isn't about hands: the headset button reader, profile and consent, take bookkeeping (deleted ranges, redo, parts, clock samples), the Hugging Face login and upload, validate's manifest and checksum checks, and most of the window. These live in `recorder/` at the top of the tree, with the bounty board, and both recorders use them: `common.py`, `units.py`, `button.py`, `bounties.py`, `files.py` (ranges, checksums, sizes), `checks.py` (validate's report, SHA256SUMS, typed manifest lookups), `hub.py` (login, errors, upload records, the pull request flow), `age.py` (the export's encryption) and `window.py` (the window backend's shared half: consent, disk, session controls, export, login and upload; each recorder's `Backend` subclasses it). The hand recorder's modules keep their names on top of them, and its tests pass.
|
||||
|
||||
Uploads: `gaze/rec/hub.py` sends nothing to the real dataset until `CONSENT.md` and `UPLOAD.md` exist and aren't drafts. `FT_EYEREC_ALLOW_UPLOAD=1` opens rehearsals, and only to another dataset (`FT_EYEREC_DATASET`). Rehearsed 2026-10-14 against the private `DeeJanuz/frametop-eyes-test` with a round of random bytes: uploaded, downloaded, checked, and decrypted with both keys by age 1.1.1 on the 7i.
|
||||
|
||||
## Build order
|
||||
|
||||
1. **Headset test 1:** do the eye cameras move with the lenses when the IPD changes? The setting can't be changed with the headset on, so compare nine-dot rounds recorded at two settings (ft-eyes' pupil x at the centre dot). Decides the IPD guidance.
|
||||
2. Pull the shared recorder code out of `hands/rec/` (hand recorder tests pass).
|
||||
3. `eyerec.py` (frame writer) and its replay test against a capture.
|
||||
4. `fitcoach.py` with readings from a capture, and the fit card on the existing fit panel.
|
||||
5. ft-gazepanel `rec` mode, `path`, `distance`, target logging.
|
||||
6. `session.py`, the nine-dot opener and the first rounds, run with `--dry-run`, then on a replay. The bounty board and `bounties.json`.
|
||||
7. Review, export with encryption (two recipients), `validate.py`, `hub.py` against a test dataset.
|
||||
8. The window, then the standalone repo.
|
||||
9. The put-on check in the gaze service. Built 2026-10-14 behind `GAZE_FIT_WATCH=1`: the quick check holds its dot while the coach reads the fit against the saved one (the fit check and the recorder's fitment save a good fit), and the slip note is a strip the gaze panel draws (`show note`) from ft-eyes' `jump_px`. Thresholds are the coach's 5 degrees, not 1.5 mm yet; tuned from the headset tests.
|
||||
|
||||
Before uploads open: the consent texts and a review like `frame-hands/notes/legal-review-2026-10-03.md`, with the health opt-in in it.
|
||||
|
||||
Not in scope here: the training. The dataset format is designed so that frame-eyes or a later trainer can label pupils from the targets and `own.jsonl`, but that's its own project.
|
||||
@@ -1,10 +0,0 @@
|
||||
### About uploading
|
||||
|
||||
- **What's sent:** the export in `@EXPORT_PATH@` (@EXPORT_SIZE@), into `contributions/@CONTRIBUTOR@/@ROUND@` in [@DATASET@](https://huggingface.co/datasets/@DATASET@). Upload first checks that every file is complete and matches its checksum, and that the eye images and health answers are encrypted.
|
||||
- **Who can read it:** the eye images and health answers are encrypted on this headset to the maintainers' keys. Hugging Face and anyone who downloads the dataset get only the encrypted files. The rest (where the dots were, what the trackers found, the fit readings and your checklist answers) stays readable.
|
||||
- **Your account:** the pull request comes from your Hugging Face account, and anyone can see its username next to your contributor id, `@CONTRIBUTOR@`. The login stays saved on this headset, so you only log in once.
|
||||
- **The dataset's terms:** the first time, open the dataset's page and accept its terms. Until you have, Upload stops with "accept the dataset's terms first".
|
||||
- **Once your pull request's link shows, plug in the headset and leave it plugged in until this page says Uploaded.** A round is several gigabytes, so this can take a while. You can take the headset off: the Eye Recorder keeps it awake until the upload is done. Keep the Eye Recorder open, since closing it stops the upload.
|
||||
- **If it stops partway** (Cancel, or the network drops), press Upload again. It carries on in the same pull request, and files already sent aren't sent twice.
|
||||
- **The maintainer's review:** they check that the files are complete and decrypt correctly before merging. You can follow it and answer questions on the pull request's page. Once it's merged, you can delete the round and its export here to free the space.
|
||||
- **Withdrawing:** see "Withdrawing" in the consent text. You'll need your contributor id, `@CONTRIBUTOR@`.
|
||||
@@ -1,13 +0,0 @@
|
||||
{"schema": 1, "updated": "2026-10-14",
|
||||
"bounties": [
|
||||
{"id": "edges", "title": "View edges", "blurb": "Dots far out to the sides and corners: eyes turned as far as they go.", "minutes": 5, "priority": "high", "script": "edges"},
|
||||
{"id": "blinks", "title": "Blinks and lids", "blurb": "Blinks, winks, squints and half-closed eyes.", "minutes": 5, "priority": "high", "script": "blinks"},
|
||||
{"id": "slips", "title": "Headset slips", "blurb": "Nudge the headset on your face, take it off and on: dots after each.", "minutes": 6, "priority": "high", "script": "slips"},
|
||||
{"id": "glasses", "title": "Glasses", "blurb": "The grid and the edges with glasses or inserts on.", "minutes": 5, "priority": "high", "script": "glasses", "needs": {"glasses": true}},
|
||||
{"id": "jumps", "title": "Jumps and pursuit", "blurb": "A jumping dot, then one gliding on circles, lines and a figure eight.", "minutes": 5, "priority": "normal", "script": "jumps"},
|
||||
{"id": "depth", "title": "Depth", "blurb": "Dots near, mid and far.", "minutes": 4, "priority": "normal", "script": "depth"},
|
||||
{"id": "light", "title": "Light changes", "blurb": "Dark to bright and back: pupils changing size.", "minutes": 5, "priority": "normal", "script": "light"},
|
||||
{"id": "face", "title": "Face moves", "blurb": "Smile, frown, yawn, talk and chew while looking at a dot.", "minutes": 4, "priority": "normal", "script": "face"},
|
||||
{"id": "head", "title": "Head turning", "blurb": "A dot fixed in the room while you turn your head.", "minutes": 4, "priority": "normal", "script": "head"},
|
||||
{"id": "reading", "title": "Reading", "blurb": "Read a few paragraphs and look around freely.", "minutes": 4, "priority": "low", "script": "reading"}
|
||||
]}
|
||||
@@ -1,326 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's frame writer (DESIGN.md: "Files"): every frame of both eye cameras, from
|
||||
the frames ft-eyegrab shares (gaze/tracker/eyes_share.py), into a take's folder as zstd chunks.
|
||||
|
||||
cam0/000000.zst, 000001.zst, ... the right eye (camera 0): up to CHUNK_FRAMES frames each
|
||||
cam1/... the left eye (camera 1)
|
||||
index.tsv a header line, then one line per frame:
|
||||
camera, chunk, frame in the chunk, the tracker's slot, time
|
||||
(CLOCK_MONOTONIC_RAW seconds, the share's)
|
||||
|
||||
A chunk is the frames back to back, 512x400 bytes each, compressed with zstd level 1 (56 % of the
|
||||
size at about 250 MB/s on one core, measured 2026-10-14; both cameras at 90 fps are 36.9 MB/s
|
||||
raw). A chunk ends when it's full or the recording pauses, so a pause or a deleted range never
|
||||
splits one. The index lines of a chunk are written once the chunk is on disk, so the index never
|
||||
names a chunk that isn't there; they're in chunk order, not time order across the cameras.
|
||||
|
||||
One thread copies frames out of the share as they come (it holds only 8 per camera, 89 ms);
|
||||
another compresses and writes, so a slow chunk never makes the copy fall behind. If the writer
|
||||
falls MAX_QUEUED chunks behind, the next chunk is dropped and counted rather than held in memory.
|
||||
|
||||
Plain Python, standard library: zstd is Python 3.14's compression.zstd (the dev container's), or
|
||||
zstandard, or the zstd command (SteamOS's host Python 3.12 has neither module).
|
||||
|
||||
From the command line, for testing (FT_EYES_CAMS points it at a replay, lab/ft-eyes-replay):
|
||||
python3 gaze/rec/eyerec.py DIR [SECONDS]
|
||||
python3 gaze/rec/eyerec.py --read DIR the frames' count and times, decompressed
|
||||
"""
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import deque
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "tracker"))
|
||||
import eyes_share # noqa: E402 (gaze/tracker/eyes_share.py)
|
||||
|
||||
W, H = eyes_share.W, eyes_share.H
|
||||
FRAME = W * H
|
||||
CAMERAS = (0, 1) # 0 the right eye, 1 the left (gaze/tracker/README: camera 0 is the steep one)
|
||||
CHUNK_FRAMES = 90 # a second at 90 fps
|
||||
LEVEL = 1
|
||||
MAX_QUEUED = 6 # chunks waiting for the writer (about 18 MB each) before one is dropped
|
||||
MIN_FREE = 3e9 # bytes: stop before the disk fills
|
||||
RATE_WINDOW = 2.0 # s of frame times for each camera's rate
|
||||
SHARE_WAIT = 10.0 # s to wait for the share at the start, or after it was replaced
|
||||
POLL = 0.002
|
||||
INDEX_HEADER = "# camera\tchunk\tframe\tslot\tt_raw\n"
|
||||
|
||||
|
||||
def _compressor():
|
||||
"""(compress(bytes) -> bytes, decompress(bytes) -> bytes, name)."""
|
||||
try:
|
||||
from compression import zstd # Python 3.14
|
||||
return (lambda b: zstd.compress(b, level=LEVEL)), zstd.decompress, "compression.zstd"
|
||||
except ImportError:
|
||||
pass
|
||||
try:
|
||||
import zstandard
|
||||
c, d = zstandard.ZstdCompressor(level=LEVEL), zstandard.ZstdDecompressor()
|
||||
return c.compress, (lambda b: d.decompress(b, max_output_size=CHUNK_FRAMES * FRAME)), "zstandard"
|
||||
except ImportError:
|
||||
pass
|
||||
exe = shutil.which("zstd") or ("/usr/bin/zstd" if os.path.exists("/usr/bin/zstd") else None)
|
||||
if not exe:
|
||||
return None, None, None
|
||||
|
||||
def run(args, data):
|
||||
return subprocess.run([exe, *args], input=data, capture_output=True, check=True).stdout
|
||||
return (lambda b: run(["-q", "-%d" % LEVEL, "-c"], b)), (lambda b: run(["-q", "-d", "-c"], b)), exe
|
||||
|
||||
|
||||
compress, decompress, ZSTD = _compressor()
|
||||
|
||||
|
||||
class Writer:
|
||||
"""Records the eye cameras into take_dir while recording is on.
|
||||
|
||||
start() starts the threads (not recording yet); record(True) and record(False) start and
|
||||
pause; stop() ends the open chunks, waits for the writer and returns stats(). error is set
|
||||
(and recording stops) when the disk gets full or the share is gone for good."""
|
||||
|
||||
def __init__(self, take_dir, share_path=None, want_path=None, log=None, min_free=MIN_FREE,
|
||||
chunk_frames=CHUNK_FRAMES):
|
||||
if compress is None:
|
||||
raise RuntimeError("no zstd: neither compression.zstd, zstandard nor the zstd command")
|
||||
self.dir, self.share_path, self.want_path = take_dir, share_path, want_path
|
||||
self.log = log or (lambda line: None)
|
||||
self.min_free, self.chunk_frames = min_free, chunk_frames
|
||||
self.error = None
|
||||
self._on = False
|
||||
self._lock = threading.Lock()
|
||||
self._stop = threading.Event()
|
||||
self._queue = queue.Queue()
|
||||
self._open = {c: [] for c in CAMERAS} # the chunk being filled: [(t, slot, bytes)]
|
||||
self._next = {c: 0 for c in CAMERAS} # the next chunk's number
|
||||
self._frames = {c: 0 for c in CAMERAS} # frames written (in chunks on disk)
|
||||
self._dropped = {c: 0 for c in CAMERAS} # frames lost: overwritten in the share, or a dropped chunk
|
||||
self._times = {c: deque() for c in CAMERAS}
|
||||
self._last = {c: None for c in CAMERAS}
|
||||
self._bytes = 0
|
||||
self._busy = 0.0 # s spent compressing and writing
|
||||
for c in CAMERAS:
|
||||
os.makedirs(os.path.join(take_dir, "cam%d" % c), exist_ok=True)
|
||||
self._index = open(os.path.join(take_dir, "index.tsv"), "a")
|
||||
if self._index.tell() == 0:
|
||||
self._index.write(INDEX_HEADER)
|
||||
# Numbering goes on after any chunks already there (a take resumed after a pause in another run).
|
||||
for c in CAMERAS:
|
||||
names = [n for n in os.listdir(os.path.join(take_dir, "cam%d" % c)) if n.endswith(".zst")]
|
||||
self._next[c] = max((int(n[:-4]) + 1 for n in names if n[:-4].isdigit()), default=0)
|
||||
self._reader = threading.Thread(target=self._read, name="eyerec-read", daemon=True)
|
||||
self._writer = threading.Thread(target=self._write, name="eyerec-write", daemon=True)
|
||||
|
||||
def start(self):
|
||||
self._reader.start()
|
||||
self._writer.start()
|
||||
return self
|
||||
|
||||
def record(self, on):
|
||||
with self._lock:
|
||||
if self._on and not on:
|
||||
for c in CAMERAS:
|
||||
self._end_chunk(c)
|
||||
self._on = bool(on) and self.error is None
|
||||
|
||||
@property
|
||||
def recording(self):
|
||||
return self._on
|
||||
|
||||
def stop(self):
|
||||
self.record(False)
|
||||
self._stop.set()
|
||||
self._reader.join(5)
|
||||
self._queue.put(None)
|
||||
self._writer.join(60)
|
||||
self._index.close()
|
||||
return self.stats()
|
||||
|
||||
def stats(self):
|
||||
"""{"frames": [right, left], "dropped": [...], "fps": [...], "bytes": on disk, "busy": s,
|
||||
"queued": chunks waiting, "last_t": [the newest frame's time or None], "error"}"""
|
||||
with self._lock:
|
||||
fps = []
|
||||
for c in CAMERAS:
|
||||
ts = self._times[c]
|
||||
fps.append(round((len(ts) - 1) / (ts[-1] - ts[0]), 1) if len(ts) > 1 and ts[-1] > ts[0] else 0.0)
|
||||
return {"frames": [self._frames[c] for c in CAMERAS], "dropped": [self._dropped[c] for c in CAMERAS],
|
||||
"fps": fps, "bytes": self._bytes, "busy": round(self._busy, 3), "queued": self._queue.qsize(),
|
||||
"last_t": [self._last[c] for c in CAMERAS], "error": self.error}
|
||||
|
||||
# -- the copy
|
||||
|
||||
def _read(self):
|
||||
share = eyes_share.open_share(SHARE_WAIT, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self._fail("no eye camera frames (%s): is frametop-eyegrab.service running?" % (self.share_path or eyes_share.CAMS))
|
||||
return
|
||||
seen = {c: share.count(c) for c in CAMERAS}
|
||||
touched = checked = time.monotonic()
|
||||
while not self._stop.is_set():
|
||||
got = False
|
||||
for c in CAMERAS:
|
||||
n = share.count(c)
|
||||
if n <= seen[c]: # nothing new (and never step back: that would copy frames twice)
|
||||
continue
|
||||
if n - seen[c] > share.slots: # fell behind: those are gone
|
||||
with self._lock:
|
||||
if self._on:
|
||||
self._dropped[c] += n - seen[c] - share.slots
|
||||
seen[c] = n - share.slots
|
||||
for i in range(seen[c], n):
|
||||
f = share.frame(c, i)
|
||||
got = True
|
||||
with self._lock:
|
||||
if f is None:
|
||||
if self._on:
|
||||
self._dropped[c] += 1
|
||||
continue
|
||||
self._seen(c, f[0])
|
||||
if self._on:
|
||||
self._open[c].append(f)
|
||||
if len(self._open[c]) >= self.chunk_frames:
|
||||
self._end_chunk(c)
|
||||
seen[c] = n
|
||||
now = time.monotonic()
|
||||
if now - touched >= 1.0:
|
||||
touched = now
|
||||
eyes_share.touch_want(self.want_path)
|
||||
if now - checked >= 2.0:
|
||||
checked = now
|
||||
if self._on and shutil.disk_usage(self.dir).free < self.min_free:
|
||||
self._fail("under %.1f GB free on the disk" % (self.min_free / 1e9))
|
||||
if share.replaced(): # ft-eyegrab restarted (SteamVR did): follow it
|
||||
self.log("eye frames: the share was replaced; following it")
|
||||
share.close()
|
||||
share = eyes_share.open_share(SHARE_WAIT, self.share_path, self.want_path)
|
||||
if share is None:
|
||||
self._fail("the eye camera frames stopped (frametop-eyegrab.service)")
|
||||
return
|
||||
seen = {c: share.count(c) for c in CAMERAS}
|
||||
if not got:
|
||||
time.sleep(POLL)
|
||||
share.close()
|
||||
|
||||
def _seen(self, c, t):
|
||||
ts = self._times[c]
|
||||
ts.append(t)
|
||||
while ts and ts[0] < t - RATE_WINDOW:
|
||||
ts.popleft()
|
||||
self._last[c] = t
|
||||
|
||||
def _end_chunk(self, c):
|
||||
"""Hand the open chunk to the writer (with the lock held)."""
|
||||
frames = self._open[c]
|
||||
if not frames:
|
||||
return
|
||||
self._open[c] = []
|
||||
if self._queue.qsize() >= MAX_QUEUED:
|
||||
self._dropped[c] += len(frames)
|
||||
self.log("eye frames: the writer is behind; dropped %d frames of camera %d" % (len(frames), c))
|
||||
return
|
||||
self._queue.put((c, self._next[c], frames))
|
||||
self._next[c] += 1
|
||||
|
||||
def _fail(self, why):
|
||||
with self._lock:
|
||||
if self.error is None:
|
||||
self.error = why
|
||||
self.log("eye frames: " + why)
|
||||
if self._on:
|
||||
for c in CAMERAS:
|
||||
self._end_chunk(c)
|
||||
self._on = False
|
||||
|
||||
# -- the writer
|
||||
|
||||
def _write(self):
|
||||
while True:
|
||||
item = self._queue.get()
|
||||
if item is None:
|
||||
return
|
||||
c, chunk, frames = item
|
||||
t0 = time.perf_counter()
|
||||
path = os.path.join(self.dir, "cam%d" % c, "%06d.zst" % chunk)
|
||||
try:
|
||||
data = compress(b"".join(f[2] for f in frames))
|
||||
with open(path + ".tmp", "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(path + ".tmp", path)
|
||||
except (OSError, subprocess.CalledProcessError) as e:
|
||||
self._fail("can't write %s: %s" % (path, e))
|
||||
with self._lock:
|
||||
self._dropped[c] += len(frames)
|
||||
continue
|
||||
self._index.write("".join("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, i, f[1], f[0]) for i, f in enumerate(frames)))
|
||||
self._index.flush()
|
||||
with self._lock:
|
||||
self._frames[c] += len(frames)
|
||||
self._bytes += len(data)
|
||||
self._busy += time.perf_counter() - t0
|
||||
|
||||
|
||||
def read_index(take_dir):
|
||||
"""index.tsv as [(camera, chunk, frame, slot, t), ...] in its order."""
|
||||
out = []
|
||||
try:
|
||||
with open(os.path.join(take_dir, "index.tsv")) as f:
|
||||
for line in f:
|
||||
if line.startswith("#") or not line.strip():
|
||||
continue
|
||||
c, chunk, i, slot, t = line.split("\t")
|
||||
out.append((int(c), int(chunk), int(i), int(slot), float(t)))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
def read_chunk(take_dir, cam, chunk):
|
||||
"""The frames of a chunk, as a list of 512x400 byte strings."""
|
||||
with open(os.path.join(take_dir, "cam%d" % cam, "%06d.zst" % chunk), "rb") as f:
|
||||
raw = decompress(f.read())
|
||||
if len(raw) % FRAME:
|
||||
raise ValueError("cam%d/%06d.zst: %d bytes isn't whole frames" % (cam, chunk, len(raw)))
|
||||
return [raw[i:i + FRAME] for i in range(0, len(raw), FRAME)]
|
||||
|
||||
|
||||
def iter_frames(take_dir):
|
||||
"""Every frame in index order: (camera, t, slot, bytes)."""
|
||||
cache = (None, None)
|
||||
for c, chunk, i, slot, t in read_index(take_dir):
|
||||
if cache[0] != (c, chunk):
|
||||
cache = ((c, chunk), read_chunk(take_dir, c, chunk))
|
||||
yield c, t, slot, cache[1][i]
|
||||
|
||||
|
||||
def main(argv):
|
||||
if not argv or argv[0] in ("-h", "--help"):
|
||||
sys.exit(__doc__)
|
||||
if argv[0] == "--read":
|
||||
n, ts = [0, 0], [[], []]
|
||||
for c, t, _slot, _data in iter_frames(argv[1]):
|
||||
n[c] += 1
|
||||
ts[c].append(t)
|
||||
for c in CAMERAS:
|
||||
span = max(ts[c]) - min(ts[c]) if len(ts[c]) > 1 else 0.0
|
||||
print("camera %d: %d frames over %.2f s" % (c, n[c], span))
|
||||
return
|
||||
secs = float(argv[1]) if len(argv) > 1 else 10.0
|
||||
w = Writer(argv[0], log=lambda line: print(line, file=sys.stderr)).start()
|
||||
w.record(True)
|
||||
end = time.monotonic() + secs
|
||||
try:
|
||||
while time.monotonic() < end and w.error is None:
|
||||
time.sleep(1)
|
||||
print(w.stats(), file=sys.stderr)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
print(w.stop())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1:])
|
||||
@@ -1,24 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Launch the Frametop Eye Recorder from a Plasma session on the Frame host (its app menu entry,
|
||||
# gaze/rec/install.sh).
|
||||
# The app runs in Frametop's container (PySide6, Kirigami and zstd come from Fedora there; a
|
||||
# release's image has the recorders' cryptography and huggingface_hub in /opt/frametop/venv, which
|
||||
# distrobox leaves off the front of PATH, so it goes first here).
|
||||
# podman needs the real XDG_RUNTIME_DIR and the real user bus (to reach systemd for
|
||||
# the container's cgroup; the Frametop session runs on a private bus from
|
||||
# dbus-run-session). The session's Wayland socket and bus go to the app itself.
|
||||
# Options go to ft_eyerec.py: --base DIR, --page NAME, and --dry-run (--hub-dry-run, --ignore-eyes) for testing.
|
||||
here=$(cd "$(dirname "$(readlink -f "$0")")" && pwd)
|
||||
wl=${WAYLAND_DISPLAY:-wayland-0}
|
||||
case $wl in /*) ;; *) wl="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}/$wl" ;; esac
|
||||
session_bus=${DBUS_SESSION_BUS_ADDRESS:-}
|
||||
export XDG_RUNTIME_DIR=/run/user/$(id -u)
|
||||
export DBUS_SESSION_BUS_ADDRESS=unix:path=$XDG_RUNTIME_DIR/bus
|
||||
# From the home folder: the app's host commands (distrobox-host-exec) run in its folder on the
|
||||
# host, and a folder only the container has, such as /run/host/tmp, makes them all fail.
|
||||
cd "$HOME" || exit 1
|
||||
exec "$here/../../scripts/in-box" env PATH="/opt/frametop/venv/bin:/opt/frametop/bin:$PATH:/usr/sbin:/sbin" \
|
||||
WAYLAND_DISPLAY="$wl" DISPLAY="${DISPLAY:-}" \
|
||||
XAUTHORITY="${XAUTHORITY:-}" DBUS_SESSION_BUS_ADDRESS="$session_bus" \
|
||||
QT_QPA_PLATFORM="wayland;xcb" \
|
||||
python3 "$here/ft_eyerec.py" "$@"
|
||||
@@ -1,9 +0,0 @@
|
||||
[Desktop Entry]
|
||||
Type=Application
|
||||
Name=Frametop Eye Recorder
|
||||
GenericName=Record your eyes for the eye dataset
|
||||
Comment=Fit the headset, then record, review and export eye recordings for Frametop's eye dataset
|
||||
Exec=@REPO@/gaze/rec/ft-eyerec
|
||||
Icon=camera-video
|
||||
Categories=Utility;
|
||||
Keywords=eyes;eye tracking;gaze;dataset;record;frametop;
|
||||
@@ -1,827 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Frametop Eye Recorder: record your eyes for the eye dataset (gaze/rec/DESIGN.md).
|
||||
|
||||
A Kirigami (QML) app with a Python backend, in the dev container, as the hand recorder
|
||||
(hands/rec/ft_handrec.py); what the two windows share is recorder/window.py:
|
||||
- Welcome: the consent text (CONSENT.md), the first time and again when its version changes.
|
||||
Agreeing writes profile.json with a random contributor id.
|
||||
- Bounties: the bounty board (recorder/bounties.py): what to record, the priority the
|
||||
maintainers gave it, and how often you've done it. Refresh fetches the dataset's board
|
||||
(hub.py board). Picking one goes on to the checklist.
|
||||
- Before you start: the checklist (glasses, eye makeup, face pad and strap, a measured IPD),
|
||||
kept in profile.json; the health questions behind their own tick, kept in BASE/health.json
|
||||
and encrypted in the export; free space; the eye camera check (frames from ft-eyegrab at
|
||||
90 fps, with the fix when SteamVR runs them at 15). Start hands the round to session.py.
|
||||
- Round: the runner's live status, the fit card while the headset is fitted, Next, Pause,
|
||||
Redo, Skip and Stop (Space, P, R, S and Esc while the window has focus; the button on the
|
||||
headset's right side too). The prompts and dots appear in the headset.
|
||||
- Review: rounds, their takes, and a viewer of both eye cameras a frame at a time, where
|
||||
ranges, takes and rounds can be deleted (takes.py).
|
||||
- Export: encrypt what's kept into exports/<round>/ at nice 19 (takes.py).
|
||||
- Upload: the Hugging Face login, and Upload: hub.py checks the export (validate.py) and
|
||||
uploads it as a pull request. Nothing reaches the real dataset until CONSENT.md and
|
||||
UPLOAD.md are reviewed (hub.py's gate); --hub-dry-run does everything but the network calls.
|
||||
Everything lives under ~/.local/share/frametop/eyes/contrib (--base). Launch with
|
||||
gaze/rec/ft-eyerec (host wrapper).
|
||||
"""
|
||||
import argparse
|
||||
import bisect
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from PySide6.QtCore import Property, Qt, QTimer, QUrl, Signal, Slot
|
||||
from PySide6.QtGui import QColor, QFont, QGuiApplication, QIcon, QImage, QPainter
|
||||
from PySide6.QtQml import QQmlApplicationEngine
|
||||
from PySide6.QtQuick import QQuickImageProvider
|
||||
from PySide6.QtQuickControls2 import QQuickStyle
|
||||
import shiboken6
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.dirname(HERE)
|
||||
REPO = os.path.dirname(GAZE)
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, REPO)
|
||||
import eyerec # noqa: E402 (next to this file; it imports gaze/tracker/eyes_share.py)
|
||||
import eyes_share # noqa: E402
|
||||
import hub # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from recorder import bounties # noqa: E402
|
||||
from recorder.common import host_command, write_json # noqa: E402
|
||||
from recorder.files import human_bytes, read_json # noqa: E402
|
||||
from recorder.window import RecorderBackend # noqa: E402
|
||||
|
||||
HUB_PATH = os.path.join(HERE, "hub.py")
|
||||
EYE_RATE = os.path.join(GAZE, "eye-rate.py")
|
||||
AWAKE_UNIT = "frametop-eyerec-awake.service"
|
||||
# A round of about 5 recorded minutes is about 6 GB (1.2 GB a minute: DESIGN.md, "Size"), and
|
||||
# its export as much again.
|
||||
ROUND_BYTES = 12 * 1000 ** 3
|
||||
ACTIVE_STATES = ("starting", "fitment", "ready", "countdown", "running", "paused")
|
||||
EYE_FPS_OK = 60 # under this the cameras run at SteamVR's 15 fps: recordings are worth little
|
||||
EYE_CHECK_S = 1.5 # how long the eye camera check counts frames
|
||||
# After the eye rate fix's SteamVR restart: check the cameras this often, for at most this long.
|
||||
RECHECK_S = 3
|
||||
RECHECK_FOR_S = 120
|
||||
FRAME_HEIGHT = 400 # the viewer's height for each camera (the cameras' own)
|
||||
THUMB_HEIGHT = 96
|
||||
# The checklist's choices (DESIGN.md, "Checklist"); the keys are what profile.json keeps.
|
||||
GLASSES = [("none", "No glasses"), ("glasses", "Glasses"), ("inserts", "Prescription lens inserts"),
|
||||
("contacts", "Contact lenses")]
|
||||
MAKEUP = [("mascara", "Mascara"), ("eyeliner", "Eyeliner"), ("lashes", "Lash extensions")]
|
||||
PADS = [("", "Rather not say"), ("stock", "The face pad it came with"), ("other", "Another face pad")]
|
||||
STRAPS = [("", "Rather not say"), ("stock", "The strap it came with"), ("other", "Another strap")]
|
||||
# The health questions: only with their own tick, and only ever in health.json(.age).
|
||||
HEALTH = [("surgery", "LASIK or other eye surgery"), ("strabismus", "A lazy or turned eye"),
|
||||
("ptosis", "A droopy eyelid"), ("nystagmus", "Nystagmus (eyes that move by themselves)"),
|
||||
("other", "Another eye condition")]
|
||||
GROUP_TITLES = {"ipd": "IPD sweep", "dots": "Nine dots"}
|
||||
STATUS_TEXT = {"running": "interrupted", "error": "ended with an error", "stopped": "stopped early"}
|
||||
|
||||
|
||||
def round_label(rid, title=""):
|
||||
"""20261014-120000-edges -> 2026-10-14 12:00 · View edges."""
|
||||
try:
|
||||
when = datetime.datetime.strptime(rid[:15], "%Y%m%d-%H%M%S").strftime("%Y-%m-%d %H:%M")
|
||||
except ValueError:
|
||||
return rid
|
||||
rest = rid[16:]
|
||||
n = rest.rsplit("-", 1)[1] if "-" in rest and rest.rsplit("-", 1)[1].isdigit() else ""
|
||||
return f"{when} · {title or rest}" + (f" ({n})" if n else "")
|
||||
|
||||
|
||||
def take_title(group):
|
||||
return GROUP_TITLES.get(group) or group.replace("-", " ").replace("_", " ").capitalize()
|
||||
|
||||
|
||||
def clock_text(seconds):
|
||||
return f"{int(seconds // 60)}:{seconds % 60:04.1f}"
|
||||
|
||||
|
||||
def eye_check(share_path=None, seconds=EYE_CHECK_S):
|
||||
"""Are the eye cameras' frames coming, and how fast? {"state": ok|slow|none, "fps": [right,
|
||||
left], "text"}. Counts frames in ft-eyegrab's share (read-only; it touches the want file,
|
||||
which wakes ft-eyegrab, as every reader does)."""
|
||||
share = eyes_share.open_share(3.0, share_path)
|
||||
if share is None:
|
||||
return {"state": "none", "fps": [0, 0],
|
||||
"text": "No eye camera frames: frametop-eyegrab.service isn't running. Frametop's gaze setup "
|
||||
"installs it (gaze/tracker/install.sh), and SteamVR has to be running."}
|
||||
try:
|
||||
time.sleep(0.5) # ft-eyegrab starts copying once it sees the want file
|
||||
eyes_share.touch_want()
|
||||
a, t0 = [share.count(c) for c in eyerec.CAMERAS], time.monotonic()
|
||||
time.sleep(seconds)
|
||||
b, t1 = [share.count(c) for c in eyerec.CAMERAS], time.monotonic()
|
||||
finally:
|
||||
share.close()
|
||||
fps = [round((y - x) / (t1 - t0), 1) for x, y in zip(a, b)]
|
||||
if max(fps) <= 0:
|
||||
return {"state": "none", "fps": fps,
|
||||
"text": "The eye cameras aren't sending frames. Is SteamVR running? If it is, restarting it "
|
||||
"starts the eye cameras again."}
|
||||
if min(fps) < EYE_FPS_OK:
|
||||
return {"state": "slow", "fps": fps,
|
||||
"text": f"The eye cameras run at {min(fps):.0f} frames a second; recordings need 90. SteamVR runs "
|
||||
"them at 90 with a setting Frametop can turn on, from SteamVR's next start."}
|
||||
return {"state": "ok", "fps": fps, "text": ""}
|
||||
|
||||
|
||||
class Timeline:
|
||||
"""A take's frames for the viewer: the camera with more frames sets the timeline, and the
|
||||
other camera's frame nearest in time goes beside it. Chunks are decompressed when needed
|
||||
and the last few kept."""
|
||||
KEEP = 4
|
||||
|
||||
def __init__(self, take_dir, clock):
|
||||
self.dir = take_dir
|
||||
self.clock = clock or []
|
||||
frames = {c: [] for c in eyerec.CAMERAS}
|
||||
for c, chunk, i, _slot, t in eyerec.read_index(take_dir):
|
||||
if c in frames:
|
||||
frames[c].append((t, chunk, i))
|
||||
for c in frames:
|
||||
frames[c].sort()
|
||||
self.main = max(eyerec.CAMERAS, key=lambda c: (len(frames[c]), -c))
|
||||
self.frames = frames
|
||||
self.times = {c: [f[0] for f in frames[c]] for c in frames}
|
||||
self._chunks = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def __len__(self):
|
||||
return len(self.frames[self.main])
|
||||
|
||||
def time(self, i):
|
||||
return self.frames[self.main][i][0]
|
||||
|
||||
def mono_ns(self, t_raw):
|
||||
"""A frame's RAW seconds as CLOCK_MONOTONIC ns (take.json's clock samples, turned round)."""
|
||||
raw = int(round(t_raw * 1e9))
|
||||
guess = raw - takes.raw_offset_ns(raw, self.clock)
|
||||
return raw - takes.raw_offset_ns(guess, self.clock)
|
||||
|
||||
def _chunk(self, cam, chunk):
|
||||
with self._lock:
|
||||
key = (cam, chunk)
|
||||
if key in self._chunks:
|
||||
self._chunks[key] = self._chunks.pop(key) # the newest at the end
|
||||
return self._chunks[key]
|
||||
frames = eyerec.read_chunk(self.dir, cam, chunk)
|
||||
with self._lock:
|
||||
self._chunks[key] = frames
|
||||
while len(self._chunks) > self.KEEP:
|
||||
self._chunks.pop(next(iter(self._chunks)))
|
||||
return frames
|
||||
|
||||
def pair(self, i):
|
||||
"""{camera: bytes} at timeline index i: the main camera's frame and the other's nearest."""
|
||||
out = {}
|
||||
t, chunk, k = self.frames[self.main][i]
|
||||
out[self.main] = self._chunk(self.main, chunk)[k]
|
||||
for c in eyerec.CAMERAS:
|
||||
if c == self.main or not self.frames[c]:
|
||||
continue
|
||||
j = bisect.bisect_left(self.times[c], t)
|
||||
j = min(range(max(0, j - 1), min(len(self.times[c]), j + 1)), key=lambda n: abs(self.times[c][n] - t))
|
||||
_, chunk, k = self.frames[c][j]
|
||||
out[c] = self._chunk(c, chunk)[k]
|
||||
return out
|
||||
|
||||
|
||||
class Timelines:
|
||||
"""Timelines by take, made again when the take's index.tsv or take.json changes."""
|
||||
|
||||
def __init__(self, store):
|
||||
self.store = store
|
||||
self._made = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def get(self, rnd, take):
|
||||
d = self.store.take_dir(rnd, take)
|
||||
stamp = tuple(os.path.getmtime(os.path.join(d, n)) if os.path.exists(os.path.join(d, n)) else 0
|
||||
for n in ("index.tsv", "take.json"))
|
||||
with self._lock:
|
||||
got = self._made.get((rnd, take))
|
||||
if got and got[0] == stamp:
|
||||
return got[1]
|
||||
tl = Timeline(d, read_json(os.path.join(d, "take.json")).get("clock"))
|
||||
with self._lock:
|
||||
self._made[(rnd, take)] = (stamp, tl)
|
||||
if len(self._made) > 8:
|
||||
self._made.pop(next(iter(self._made)))
|
||||
return tl
|
||||
|
||||
|
||||
def grey_image(data):
|
||||
img = QImage(data, eyerec.W, eyerec.H, eyerec.W, QImage.Format_Grayscale8)
|
||||
return img.copy()
|
||||
|
||||
|
||||
class EyeProvider(QQuickImageProvider):
|
||||
"""image://eyes/pair/ROUND/TAKE/INDEX: both eye cameras at a timeline index, the right eye
|
||||
(camera 0) first. image://eyes/thumb/ROUND/TAKE: the main camera mid-take, small. Anything
|
||||
after "?" is ignored (it makes QML load the image again)."""
|
||||
|
||||
def __init__(self, timelines):
|
||||
super().__init__(QQuickImageProvider.ImageType.Image)
|
||||
self.timelines = timelines
|
||||
|
||||
def requestImage(self, ident, size, requested):
|
||||
parts = ident.split("?", 1)[0].split("/")
|
||||
try:
|
||||
if parts[0] == "thumb" and len(parts) == 3:
|
||||
tl = self.timelines.get(parts[1], parts[2])
|
||||
if not len(tl):
|
||||
raise ValueError("no frames")
|
||||
data = tl.pair(len(tl) // 2)[tl.main]
|
||||
return grey_image(data).scaledToHeight(THUMB_HEIGHT, Qt.SmoothTransformation)
|
||||
if parts[0] == "pair" and len(parts) == 4:
|
||||
return self._pair(self.timelines.get(parts[1], parts[2]), int(parts[3]),
|
||||
requested.height() if requested.height() > 0 else FRAME_HEIGHT)
|
||||
except (OSError, ValueError, IndexError, KeyError) as e:
|
||||
print(f"ft-eyerec: image {ident}: {e}", file=sys.stderr)
|
||||
img = QImage(4, 3, QImage.Format_Grayscale8)
|
||||
img.fill(0)
|
||||
return img
|
||||
|
||||
def _pair(self, tl, i, height):
|
||||
frames = tl.pair(i)
|
||||
gap = 8
|
||||
scaled = [(c, grey_image(frames[c]).scaledToHeight(height, Qt.SmoothTransformation))
|
||||
for c in eyerec.CAMERAS if c in frames]
|
||||
out = QImage(sum(s.width() for _, s in scaled) + gap * (len(scaled) - 1), height, QImage.Format_RGB32)
|
||||
out.fill(QColor(30, 30, 30))
|
||||
p = QPainter(out)
|
||||
font = QFont()
|
||||
font.setPixelSize(max(12, height // 22))
|
||||
p.setFont(font)
|
||||
x = 0
|
||||
for c, img in scaled:
|
||||
p.drawImage(x, 0, img)
|
||||
p.setPen(QColor(255, 200, 80))
|
||||
p.drawText(x + 6, 6 + font.pixelSize(), ("Right eye" if c == 0 else "Left eye") + f" (camera {c})")
|
||||
x += img.width() + gap
|
||||
p.end()
|
||||
return out
|
||||
|
||||
|
||||
class Backend(RecorderBackend):
|
||||
"""The eye recorder's window backend: recorder/window.py's, plus the bounty board, the
|
||||
checklist and health answers, the eye camera check, the round and the review."""
|
||||
APP = "Frametop Eye Recorder"
|
||||
AWAKE_UNIT = AWAKE_UNIT
|
||||
AWAKE_WHY = "Exporting or uploading eye recordings"
|
||||
HUB_PATH = HUB_PATH
|
||||
CONSENT_PATH = hub.CONSENT_PATH
|
||||
UPLOAD_PATH = hub.UPLOAD_PATH
|
||||
ACTIVE_STATES = ACTIVE_STATES
|
||||
ROUND_BYTES = ROUND_BYTES
|
||||
PROFILE_KEEP = ("checklist", "bounties", "eyes_ipd")
|
||||
|
||||
# A property here can't notify with the base's signals (PySide6 6.11 leaves it without
|
||||
# one), so these relay them.
|
||||
profileRelay = Signal()
|
||||
sessionsRelay = Signal()
|
||||
boardChanged = Signal()
|
||||
healthChanged = Signal()
|
||||
eyesChanged = Signal()
|
||||
_eyesArrived = Signal(dict)
|
||||
_boardArrived = Signal(dict)
|
||||
|
||||
def __init__(self, store, session_options=None, hub_dry_run=False, ignore_eyes=False, timelines=None):
|
||||
super().__init__(store, hub, hub_dry_run)
|
||||
self.profileChanged.connect(self.profileRelay)
|
||||
self.sessionsChanged.connect(self.sessionsRelay)
|
||||
self.profileChanged.connect(self.boardChanged) # the board's done marks and needs
|
||||
self._session_options = session_options or {} # Session options: dry_run, speed, cams, button
|
||||
self._session_mod = None
|
||||
self._session_error = ""
|
||||
self._timelines = timelines or Timelines(store)
|
||||
self._ignore_eyes = ignore_eyes
|
||||
self._eyes = {}
|
||||
self._eyes_busy = False
|
||||
self._restarting = False
|
||||
self._board = {"state": "", "text": ""}
|
||||
self._board_busy = False
|
||||
self._eyesArrived.connect(self._on_eyes)
|
||||
self._boardArrived.connect(self._on_board)
|
||||
|
||||
# --- the session runner, imported when first needed
|
||||
def _runner(self):
|
||||
if self._session_mod is None and not self._session_error:
|
||||
try:
|
||||
import session as mod
|
||||
self._session_mod = mod
|
||||
except Exception as e: # missing, or broken: say so, the rest of the app still works
|
||||
self._session_error = f"The session runner (gaze/rec/session.py) can't be loaded: {e}"
|
||||
return self._session_mod
|
||||
|
||||
def _runner_error(self):
|
||||
self._runner()
|
||||
return self._session_error
|
||||
|
||||
def _texts_draft(self):
|
||||
return not hub.texts_ready()
|
||||
|
||||
def _upload_allowed(self):
|
||||
try:
|
||||
return hub.upload_allowed(hub.dataset_id())
|
||||
except hub.HubError:
|
||||
return False
|
||||
|
||||
def _upload_values(self, session, path):
|
||||
return {"ROUND": session}
|
||||
|
||||
@Property(str, constant=True)
|
||||
def toolVersion(self):
|
||||
return takes.tool_version()
|
||||
|
||||
# --- consent
|
||||
@Slot(bool, bool, bool)
|
||||
def acceptConsent(self, adult, region, agree):
|
||||
"""Write profile.json (recorder/window.py); its checklist and bounties stay."""
|
||||
optional = self.store.profile().get("optional")
|
||||
self._write_consent(adult, region, agree, optional if isinstance(optional, dict) else {})
|
||||
|
||||
# --- the bounty board
|
||||
def _checklist(self):
|
||||
c = self.store.profile().get("checklist")
|
||||
return c if isinstance(c, dict) else {}
|
||||
|
||||
def _board_now(self):
|
||||
mod = self._runner()
|
||||
return mod.load_board(self.store.base) if mod else None
|
||||
|
||||
@Property("QVariantList", notify=boardChanged)
|
||||
def bounties(self):
|
||||
"""The board's bounties (bounties.view), with "lastText" for the last round."""
|
||||
mod = self._runner()
|
||||
if not mod:
|
||||
return []
|
||||
rows = bounties.view(self._board_now(), self.store.profile(), self._checklist(), mod.have_round)
|
||||
for b in rows:
|
||||
b["lastText"] = round_label(b["last"]).split(" · ")[0] if b["last"] else ""
|
||||
b["minutesText"] = f"about {b['minutes']:g} min" if b["minutes"] else ""
|
||||
return rows
|
||||
|
||||
@Property(str, notify=boardChanged)
|
||||
def boardUpdated(self):
|
||||
board = self._board_now()
|
||||
return board["updated"] if board else ""
|
||||
|
||||
@Property("QVariantMap", notify=boardChanged)
|
||||
def boardFetch(self):
|
||||
"""The last fetch: {"state": ""|fetching|ok|error, "text"}."""
|
||||
return self._board
|
||||
|
||||
@Slot()
|
||||
def refreshBoard(self):
|
||||
"""hub.py board in a child process: the dataset's board into board-cache.json."""
|
||||
if self._board_busy or self._hub_dry_run:
|
||||
return
|
||||
self._board_busy = True
|
||||
self._board = {"state": "fetching", "text": "Getting the newest board"}
|
||||
self.boardChanged.emit()
|
||||
argv = self._hub_argv("board", "--json")
|
||||
|
||||
def run():
|
||||
try:
|
||||
r = subprocess.run(argv, capture_output=True, text=True, timeout=60)
|
||||
lines = [json.loads(x) for x in r.stdout.splitlines() if x.startswith("{")]
|
||||
result = lines[-1] if lines else {"error": {"text": (r.stderr.strip().splitlines() or ["hub.py failed"])[-1]}}
|
||||
except (OSError, ValueError, subprocess.TimeoutExpired) as e:
|
||||
result = {"error": {"text": f"The board didn't come: {e}"}}
|
||||
self._boardArrived.emit(result)
|
||||
self._thread(run)
|
||||
|
||||
def _on_board(self, result):
|
||||
self._board_busy = False
|
||||
if "done" in result:
|
||||
self._board = {"state": "ok", "text": f"The newest board, of {result['done'].get('updated', '?')}"}
|
||||
else:
|
||||
self._board = {"state": "error", "text": (result.get("error") or {}).get("text", "The board didn't come")
|
||||
+ " The board that came with the recorder is used."}
|
||||
self.boardChanged.emit()
|
||||
|
||||
@Slot(str, result=str)
|
||||
def planText(self, bounty_id):
|
||||
"""How long a bounty's round takes (session.plan_seconds), with the fit check before it."""
|
||||
mod = self._runner()
|
||||
b = next((b for b in self.bounties if b["id"] == bounty_id), None)
|
||||
if not mod or not b:
|
||||
return ""
|
||||
try:
|
||||
plan = mod.build_plan(mod.load_round(mod.round_path(b["script"])), 0)
|
||||
except (OSError, ValueError) as e:
|
||||
return f"Couldn't read the round: {e}"
|
||||
return (f"{len(plan)} steps, about {mod.plan_seconds(plan) / 60:.0f} min recorded, after putting the "
|
||||
"headset on")
|
||||
|
||||
# --- the checklist and health answers
|
||||
@Property("QVariantMap", notify=profileRelay)
|
||||
def checklist(self):
|
||||
c = self._checklist()
|
||||
return {"glasses": c.get("glasses") if c.get("glasses") in dict(GLASSES) else "none",
|
||||
"makeup": [m for m in c.get("makeup") or [] if m in dict(MAKEUP)],
|
||||
"pad": c.get("pad") if c.get("pad") in dict(PADS) else "",
|
||||
"strap": c.get("strap") if c.get("strap") in dict(STRAPS) else "",
|
||||
"ipd_mm": c.get("ipd_mm") if isinstance(c.get("ipd_mm"), (int, float)) else 0}
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def glassesChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in GLASSES]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def makeupChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in MAKEUP]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def padChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in PADS]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def strapChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in STRAPS]
|
||||
|
||||
@Property("QVariantList", constant=True)
|
||||
def healthChoices(self):
|
||||
return [{"value": k, "text": v} for k, v in HEALTH]
|
||||
|
||||
@Slot("QVariantMap")
|
||||
def saveChecklist(self, answers):
|
||||
"""Keep the checklist in profile.json: only the known answers. ipd_mm: 0 when not known,
|
||||
else 50 to 80 mm."""
|
||||
a = dict(answers)
|
||||
try:
|
||||
ipd = round(float(a.get("ipd_mm") or 0), 1)
|
||||
except (TypeError, ValueError):
|
||||
ipd = 0.0
|
||||
checklist = {"glasses": a.get("glasses") if a.get("glasses") in dict(GLASSES) else "none",
|
||||
"makeup": sorted({m for m in a.get("makeup") or [] if m in dict(MAKEUP)}),
|
||||
"pad": a.get("pad") if a.get("pad") in dict(PADS) else "",
|
||||
"strap": a.get("strap") if a.get("strap") in dict(STRAPS) else "",
|
||||
"ipd_mm": ipd if 50 <= ipd <= 80 else None}
|
||||
profile = self.store.profile()
|
||||
if not profile or profile.get("checklist") == checklist:
|
||||
return
|
||||
profile["checklist"] = checklist
|
||||
write_json(self.store.profile_path, profile)
|
||||
self.profileChanged.emit()
|
||||
|
||||
def _health_path(self):
|
||||
return os.path.join(self.store.base, "health.json")
|
||||
|
||||
def _health(self):
|
||||
h = read_json(self._health_path())
|
||||
return h if h.get("opt_in") is True else {}
|
||||
|
||||
@Property(bool, notify=healthChanged)
|
||||
def healthOptIn(self):
|
||||
return bool(self._health())
|
||||
|
||||
@Property("QVariantList", notify=healthChanged)
|
||||
def healthAnswers(self):
|
||||
return [k for k in self._health().get("conditions") or [] if k in dict(HEALTH)]
|
||||
|
||||
@Slot(bool, "QVariantList")
|
||||
def saveHealth(self, opt_in, conditions):
|
||||
"""The health answers, in BASE/health.json (only this user can read it), never in
|
||||
profile.json; each round gets a copy, which its export encrypts. Opting out deletes them."""
|
||||
path = self._health_path()
|
||||
if not opt_in:
|
||||
try:
|
||||
os.remove(path)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
else:
|
||||
data = {"schema": 1, "opt_in": True, "conditions": sorted({c for c in conditions if c in dict(HEALTH)})}
|
||||
if read_json(path) == data:
|
||||
return
|
||||
fd = os.open(path + ".tmp", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w") as f:
|
||||
json.dump(data, f, indent=1)
|
||||
f.write("\n")
|
||||
os.replace(path + ".tmp", path)
|
||||
self.healthChanged.emit()
|
||||
|
||||
# --- the eye camera check
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeState(self):
|
||||
"""ok, slow, none; checking while it runs; "" before the first check."""
|
||||
if self._eyes_busy and not self._eyes:
|
||||
return "checking"
|
||||
return self._eyes.get("state", "")
|
||||
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeText(self):
|
||||
return self._eyes.get("text", "")
|
||||
|
||||
@Property(str, notify=eyesChanged)
|
||||
def eyeFpsText(self):
|
||||
fps = self._eyes.get("fps") or []
|
||||
return " · ".join(f"{name} {f:.0f} fps" for name, f in zip(("right", "left"), fps))
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def eyeBusy(self):
|
||||
return self._eyes_busy
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def restartingSteamVR(self):
|
||||
return self._restarting
|
||||
|
||||
@Property(bool, constant=True)
|
||||
def eyesIgnored(self):
|
||||
return self._ignore_eyes or bool(self._session_options.get("dry_run"))
|
||||
|
||||
@Property(bool, notify=eyesChanged)
|
||||
def eyesBlockStart(self):
|
||||
return not self.eyesIgnored and self._eyes.get("state") != "ok"
|
||||
|
||||
def _check_now(self):
|
||||
if self.eyesIgnored:
|
||||
return {"state": "ok", "fps": [], "text": ""}
|
||||
try:
|
||||
return eye_check(self._session_options.get("cams"))
|
||||
except Exception as e:
|
||||
return {"state": "none", "fps": [], "text": f"The eye camera check failed: {e}"}
|
||||
|
||||
@Slot()
|
||||
def checkEyes(self):
|
||||
if self._eyes_busy or self.sessionActive:
|
||||
return
|
||||
self._eyes_busy = True
|
||||
self.eyesChanged.emit()
|
||||
self._thread(lambda: self._eyesArrived.emit(self._check_now()))
|
||||
|
||||
def _on_eyes(self, result):
|
||||
self._eyes = result
|
||||
self._eyes_busy = self._restarting
|
||||
self.eyesChanged.emit()
|
||||
|
||||
@Slot(bool)
|
||||
def restartSteamVR(self, fast):
|
||||
"""fast: first turn on SteamVR's 90 fps eye cameras (gaze/eye-rate.py on, on the host).
|
||||
Then systemctl --user restart --no-block steamvr.service (the job runs in systemd, so it
|
||||
finishes even when this window closes with the Frametop desktop), and the eye check every
|
||||
few seconds until the frames come at 90."""
|
||||
if self._restarting or self.sessionActive:
|
||||
return
|
||||
self._restarting = self._eyes_busy = True
|
||||
self.eyesChanged.emit()
|
||||
|
||||
def run():
|
||||
result = None
|
||||
try:
|
||||
if fast:
|
||||
r = subprocess.run(host_command("python3", EYE_RATE, "on"), capture_output=True, text=True,
|
||||
timeout=60)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError((r.stderr or r.stdout).strip() or "eye-rate.py: exit %d" % r.returncode)
|
||||
r = subprocess.run(host_command("systemctl", "--user", "restart", "--no-block", "steamvr.service"),
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError((r.stderr or r.stdout).strip() or "exit %d" % r.returncode)
|
||||
end = time.monotonic() + RECHECK_FOR_S
|
||||
time.sleep(RECHECK_S * 3)
|
||||
while time.monotonic() < end:
|
||||
result = self._check_now()
|
||||
if result.get("state") == "ok":
|
||||
break
|
||||
time.sleep(RECHECK_S)
|
||||
except Exception as e:
|
||||
result = {"state": "none", "fps": [], "text": f"Couldn't restart SteamVR: {e}"}
|
||||
self._restarting = False
|
||||
self._eyesArrived.emit(result or self._check_now())
|
||||
self._thread(run)
|
||||
|
||||
# --- the round
|
||||
@Slot(str, result=bool)
|
||||
def startSession(self, bounty_id):
|
||||
if self.sessionActive:
|
||||
return False
|
||||
mod = self._runner()
|
||||
if not mod:
|
||||
self.message.emit(self._session_error, True)
|
||||
return False
|
||||
if self.needsConsent:
|
||||
self.message.emit("Agree to the consent text first (Welcome page)", True)
|
||||
return False
|
||||
entry = next((b for b in self.bounties if b["id"] == bounty_id), None)
|
||||
if not entry:
|
||||
self.message.emit("Pick a bounty first", True)
|
||||
return False
|
||||
if entry["available"]:
|
||||
self.message.emit(entry["available"], True)
|
||||
return False
|
||||
if not self.eyesIgnored:
|
||||
self._eyes = self._check_now() # fresh: it takes two seconds
|
||||
self.eyesChanged.emit()
|
||||
if self._eyes.get("state") != "ok":
|
||||
self.message.emit(self._eyes.get("text", "The eye cameras aren't ready"), True)
|
||||
return False
|
||||
board = self._board_now() or {}
|
||||
try:
|
||||
self._session = mod.Session(self.store.base, entry["script"], bounty=dict(entry, updated=board.get("updated")),
|
||||
profile=self.store.profile(), checklist=self._checklist(), health=self._health(),
|
||||
on_status=lambda s: self._statusArrived.emit(dict(s)),
|
||||
profile_path=self.store.profile_path, **self._session_options)
|
||||
except Exception as e:
|
||||
self.message.emit(f"Couldn't set up the round: {e}", True)
|
||||
return False
|
||||
self._session_id = ""
|
||||
self._status = {"state": "starting", "bounty": bounty_id}
|
||||
self.statusChanged.emit()
|
||||
session = self._session
|
||||
|
||||
def run():
|
||||
try:
|
||||
session.start()
|
||||
except Exception as e:
|
||||
self._statusArrived.emit({"state": "error", "error": str(e)})
|
||||
self._thread(run)
|
||||
return True
|
||||
|
||||
def _on_status(self, status):
|
||||
super()._on_status(status)
|
||||
if status.get("state") == "done":
|
||||
self.profileChanged.emit() # the bounty is marked done, and the IPD kept
|
||||
|
||||
# --- review
|
||||
def _session_rows(self):
|
||||
out = []
|
||||
for r in self.store.rounds():
|
||||
r["label"] = round_label(r["id"], r["title"])
|
||||
r["sizeText"] = human_bytes(r["bytes"])
|
||||
r["exportText"] = human_bytes(r["export_bytes"]) if r["exported"] else ""
|
||||
r["active"] = self.sessionActive and r["id"] == self._session_id
|
||||
r["statusText"] = "" if r["active"] else STATUS_TEXT.get(r["status"], "")
|
||||
out.append(r)
|
||||
return out
|
||||
|
||||
@Slot(str, result="QVariantList")
|
||||
def takeList(self, rnd):
|
||||
try:
|
||||
rows = self.store.takes(rnd)
|
||||
except ValueError:
|
||||
return []
|
||||
for t in rows:
|
||||
t["title"] = take_title(t["group"])
|
||||
t["durationText"] = f"{int(t['duration_s'] // 60)}:{int(t['duration_s'] % 60):02d}"
|
||||
t["sizeText"] = human_bytes(t["bytes"])
|
||||
# Per eye, as the viewer steps (the camera with more frames)
|
||||
t["frameCount"] = max(t["frames"], default=0)
|
||||
t["deletedCount"] = max(t["deleted_frames"], default=0)
|
||||
t.pop("ranges")
|
||||
return rows
|
||||
|
||||
@Slot(str, str, result="QVariantMap")
|
||||
def takeInfo(self, rnd, take):
|
||||
"""For the viewer: {count, title, ranges: [[first, last] timeline indexes]}."""
|
||||
try:
|
||||
tl = self._timelines.get(rnd, take)
|
||||
ranges = self.store.ranges(rnd, take)
|
||||
meta = self.store.take_meta(rnd, take)
|
||||
except (OSError, ValueError):
|
||||
return {"count": 0, "title": take, "ranges": []}
|
||||
marks = []
|
||||
monos = [tl.mono_ns(t) for t in tl.times[tl.main]]
|
||||
for a, b in ranges:
|
||||
first, last = bisect.bisect_left(monos, a), bisect.bisect_right(monos, b) - 1
|
||||
marks.append([first, last] if first <= last else [-1, -1])
|
||||
return {"count": len(tl), "title": take_title(meta.get("group", take.split("-", 1)[-1])), "ranges": marks}
|
||||
|
||||
@Slot(str, str, int, result=str)
|
||||
def frameTime(self, rnd, take, i):
|
||||
"""Frame i's time from the take's first, m:ss.s."""
|
||||
try:
|
||||
tl = self._timelines.get(rnd, take)
|
||||
return clock_text(tl.time(i) - tl.time(0))
|
||||
except (OSError, ValueError, IndexError):
|
||||
return ""
|
||||
|
||||
@Slot(str, str, int, int)
|
||||
def deleteRange(self, rnd, take, first, last):
|
||||
"""Leave frames first..last (timeline indexes, either order) out of the export: both
|
||||
cameras', by time."""
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
tl = self._timelines.get(rnd, take)
|
||||
first, last = sorted((max(0, first), min(len(tl) - 1, last)))
|
||||
if first > last:
|
||||
return
|
||||
# Half a millisecond either side: index.tsv keeps whole microseconds, and the clocks'
|
||||
# conversion rounds; the frames are 11 ms apart.
|
||||
self.store.delete_range(rnd, take, tl.mono_ns(tl.time(first)) - 500000, tl.mono_ns(tl.time(last)) + 500000)
|
||||
self.message.emit(f"Frames {first + 1} to {last + 1} won't be exported", False)
|
||||
self.sessionsChanged.emit()
|
||||
|
||||
@Slot(str, str, int)
|
||||
def restoreRange(self, rnd, take, k):
|
||||
self.store.restore_range(rnd, take, k)
|
||||
self.message.emit("Range restored", False)
|
||||
self.sessionsChanged.emit()
|
||||
|
||||
@Slot(str, str)
|
||||
def deleteTake(self, rnd, take):
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
try:
|
||||
self.store.delete_take(rnd, take)
|
||||
self.message.emit(f"Deleted {take}", False)
|
||||
except (OSError, ValueError) as e:
|
||||
self.message.emit(f"Couldn't delete {take}: {e}", True)
|
||||
self.refreshSessions()
|
||||
|
||||
@Slot(str)
|
||||
def deleteSession(self, rnd):
|
||||
if self._active_guard(rnd):
|
||||
return
|
||||
try:
|
||||
self.store.delete_round(rnd)
|
||||
self.message.emit(f"Deleted the round of {round_label(rnd)}", False)
|
||||
except (OSError, ValueError) as e:
|
||||
self.message.emit(f"Couldn't delete the round: {e}", True)
|
||||
self.refreshSessions()
|
||||
|
||||
# --- export
|
||||
@Property(str, constant=True)
|
||||
def exportProblem(self):
|
||||
"""Why nothing can be exported here, or ""."""
|
||||
if eyerec.compress is None:
|
||||
return "zstd isn't installed in the dev container, so nothing can be exported. Run setup/dev-container.sh."
|
||||
try:
|
||||
import cryptography # noqa: F401 (recorder/age.py's)
|
||||
except ImportError:
|
||||
return ("The cryptography library isn't installed in the dev container, so nothing can be encrypted "
|
||||
"for export. Run setup/dev-container.sh.")
|
||||
return ""
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Frametop Eye Recorder")
|
||||
ap.add_argument("--base", default=takes.DEFAULT_BASE, help="where profile.json, rounds/ and exports/ go")
|
||||
ap.add_argument("--page", default="",
|
||||
help="open on this page: welcome, bounties, checklist, session, review, export, upload")
|
||||
ap.add_argument("--dry-run", action="store_true",
|
||||
help="test: rounds start no processes, make up the panel's replies and print its commands")
|
||||
ap.add_argument("--speed", type=float, default=1.0, help="test, with --dry-run: run rounds this much faster")
|
||||
ap.add_argument("--cams", help="test: read eye frames from this share (lab/ft-eyes-replay's)")
|
||||
ap.add_argument("--no-headset-button", action="store_true",
|
||||
help="rounds don't read the headset's button (Next, pause, resume)")
|
||||
ap.add_argument("--hub-dry-run", action="store_true",
|
||||
help="test: Upload checks the export and says what it would send, with no network calls")
|
||||
ap.add_argument("--ignore-eyes", action="store_true",
|
||||
help="start rounds even if the eye camera check finds no frames, or slow ones")
|
||||
ap.add_argument("--qml", default=os.path.join(HERE, "main.qml"),
|
||||
help="the window (default: this folder's main.qml, which needs Kirigami)")
|
||||
ap.add_argument("--style", default="org.kde.desktop", help="the Qt Quick Controls style")
|
||||
a, qt_args = ap.parse_known_args()
|
||||
app = QGuiApplication([sys.argv[0]] + qt_args)
|
||||
app.setApplicationName("ft-eyerec")
|
||||
app.setApplicationDisplayName("Frametop Eye Recorder")
|
||||
app.setDesktopFileName("ft-eyerec")
|
||||
if not QIcon.themeName():
|
||||
QIcon.setThemeName("breeze")
|
||||
QQuickStyle.setStyle(a.style)
|
||||
store = takes.Store(a.base)
|
||||
timelines = Timelines(store)
|
||||
engine = QQmlApplicationEngine()
|
||||
engine.addImageProvider("eyes", EyeProvider(timelines))
|
||||
options = {"dry_run": True, "speed": a.speed} if a.dry_run else {}
|
||||
if a.cams:
|
||||
options["cams"] = a.cams
|
||||
if a.no_headset_button or a.dry_run:
|
||||
options["button"] = False
|
||||
backend = Backend(store, options, hub_dry_run=a.hub_dry_run, ignore_eyes=a.ignore_eyes, timelines=timelines)
|
||||
app.aboutToQuit.connect(backend.shutdown)
|
||||
mod = None if a.dry_run else backend._runner()
|
||||
if mod: # Frametop's gaze service, if a round that was killed left it paused
|
||||
backend._thread(lambda: mod.resume_units(os.path.join(mod.run_dir(), mod.PAUSED_UNITS_FILE)))
|
||||
engine.rootContext().setContextProperty("backend", backend)
|
||||
engine.rootContext().setContextProperty("startPage", a.page)
|
||||
engine.load(QUrl.fromLocalFile(os.path.abspath(a.qml)))
|
||||
if not engine.rootObjects():
|
||||
sys.exit(1)
|
||||
|
||||
# SIGTERM and Ctrl+C quit as closing does, so a running round still stops cleanly. Python
|
||||
# runs signal handlers between bytecodes: the timer gives it some while Qt waits.
|
||||
def on_signal(*_):
|
||||
engine.rootObjects()[0].setProperty("quitting", True)
|
||||
app.quit()
|
||||
signal.signal(signal.SIGTERM, on_signal)
|
||||
signal.signal(signal.SIGINT, on_signal)
|
||||
tick = QTimer(interval=500, timeout=lambda: None)
|
||||
tick.start()
|
||||
code = app.exec()
|
||||
# The window goes before the backend (as the hand recorder: bindings would run against it).
|
||||
shiboken6.delete(engine)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
-280
@@ -1,280 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload an eye recorder export to the eye dataset on Hugging Face.
|
||||
|
||||
As the hand recorder's hub.py (the shared part is recorder/hub.py): the window runs it as a child
|
||||
process, it uses huggingface_hub's saved login (`hub.py login` is the browser's device code), and
|
||||
nobody types a token anywhere. An upload:
|
||||
1. checks the export with validate.py (against recipients.txt's keys), and stops on errors;
|
||||
2. stops if this export (same SHA256SUMS) was uploaded before, unless --again;
|
||||
3. stops unless CONSENT.md and UPLOAD.md are here and neither is a draft. FT_EYEREC_ALLOW_UPLOAD=1
|
||||
opens it for a rehearsal, and only to another dataset than the real one (FT_EYEREC_DATASET):
|
||||
the real one takes nothing before the texts are reviewed; and it stops for an export made under
|
||||
a draft consent text (agree to the current one, then export again);
|
||||
4. checks the login and access to the dataset, opens the pull request first, uploads the folder
|
||||
to contributions/<contributor>/<round> in it, and records it under "uploads" in session.json.
|
||||
--dry-run does all of it except the network calls and the records, and says what it would upload.
|
||||
|
||||
The images and health answers are encrypted already (takes.py's export): nothing here can read
|
||||
them, and the dataset's readers need one of its keys.
|
||||
|
||||
`hub.py board` fetches the bounty board (bounties.json) from the dataset into BASE/board-cache.json,
|
||||
with the saved login when there is one (the dataset is gated): the window's board then uses it
|
||||
when it's newer than the one shipped here (recorder/bounties.py checks what it may say).
|
||||
|
||||
usage: hub.py [--base DIR] whoami [--json]
|
||||
hub.py [--base DIR] login [--json]
|
||||
hub.py [--base DIR] board [--json]
|
||||
hub.py [--base DIR] upload ROUND [--dry-run] [--again] [--json]
|
||||
With --json, each line of output is one JSON object, as the hand recorder's hub.py.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
import takes # noqa: E402 (next to this file)
|
||||
from recorder import bounties # noqa: E402
|
||||
from recorder import hub as rhub # noqa: E402
|
||||
from recorder.common import write_json # noqa: E402
|
||||
from recorder.hub import HubError, export_sha, is_draft, now_iso # noqa: E402,F401
|
||||
|
||||
HF_DATASET = "DeeJanuz/frametop-eyes"
|
||||
DATASET_ENV = "FT_EYEREC_DATASET"
|
||||
ALLOW_ENV = "FT_EYEREC_ALLOW_UPLOAD"
|
||||
CONSENT_PATH = os.path.join(HERE, "CONSENT.md")
|
||||
UPLOAD_PATH = os.path.join(HERE, "UPLOAD.md")
|
||||
|
||||
|
||||
def dataset_id():
|
||||
"""HF_DATASET, or FT_EYEREC_DATASET when set."""
|
||||
return rhub.dataset_id(DATASET_ENV, HF_DATASET)
|
||||
|
||||
|
||||
def texts_ready():
|
||||
return all(os.path.isfile(p) and not is_draft(p) for p in (CONSENT_PATH, UPLOAD_PATH))
|
||||
|
||||
|
||||
def upload_allowed(repo):
|
||||
"""Real uploads: once the texts are reviewed, or for a rehearsal against another dataset."""
|
||||
return texts_ready() or (os.environ.get(ALLOW_ENV) == "1" and repo != HF_DATASET)
|
||||
|
||||
|
||||
def meta_path(store, rnd):
|
||||
return os.path.join(store.round_dir(rnd), "session.json")
|
||||
|
||||
|
||||
def describe(summary, report, sha, nbytes):
|
||||
"""(commit message, commit description) from validate's summary of the manifest."""
|
||||
s = summary
|
||||
message = f"Eyes: {s.get('bounty')} round {s.get('round')} from {s.get('contributor')}"
|
||||
lines = ["Contribution to the Frametop eye dataset, uploaded from the eye recorder. The eye images "
|
||||
"and any health answers are encrypted to the dataset's keys.", "",
|
||||
f"- Round: {s.get('round')}",
|
||||
f"- Bounty: {s.get('bounty')}",
|
||||
f"- Contributor: {s.get('contributor')}",
|
||||
f"- Takes: {s.get('takes')} ({s.get('minutes')} minutes, {s.get('frames')} frames)",
|
||||
f"- Glasses: {s.get('glasses') or 'not given'}",
|
||||
f"- Health answers: {'included, encrypted' if s.get('health') else 'none'}",
|
||||
f"- Consent version: {s.get('consent_version')}",
|
||||
f"- Tool: {s.get('tool')}",
|
||||
f"- Size: {takes.human_bytes(nbytes)}",
|
||||
f"- SHA256 of SHA256SUMS: {sha}", "",
|
||||
"Checked with gaze/rec/validate.py: no errors" + (f", {len(report.warnings)} warnings:"
|
||||
if report.warnings else ".")]
|
||||
lines += [f"- {w}" for w in report.warnings]
|
||||
return message, "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _hf():
|
||||
return rhub.hf_module("run setup/dev-container.sh (or: pip install --user huggingface_hub)")
|
||||
|
||||
|
||||
def whoami():
|
||||
return rhub.whoami(_hf(), dataset_id())
|
||||
|
||||
|
||||
def login(progress=None):
|
||||
_hf()
|
||||
return rhub.login(dataset_id(), progress, lambda: whoami(), "update the dev container (setup/dev-container.sh).")
|
||||
|
||||
|
||||
def board_url(repo):
|
||||
return f"{rhub.dataset_url(repo)}/resolve/main/bounties.json"
|
||||
|
||||
|
||||
def fetch_board(base):
|
||||
"""The dataset's bounties.json into base/board-cache.json (session.load_board reads it).
|
||||
Returns {"updated", "bounties"}; raises HubError."""
|
||||
repo = dataset_id()
|
||||
token = None
|
||||
try:
|
||||
token = _hf().get_token()
|
||||
except (HubError, AttributeError):
|
||||
pass # no huggingface_hub, or an old one: try without a login
|
||||
try:
|
||||
board = bounties.fetch(board_url(repo), os.path.join(base, "board-cache.json"), token)
|
||||
except ValueError as e:
|
||||
raise HubError("hub", f"The dataset's bounty board isn't valid: {e}") from None
|
||||
except OSError as e:
|
||||
status = getattr(e, "code", None)
|
||||
if status in (401, 403):
|
||||
raise HubError("terms", "Log in and accept the dataset's terms to get its newest bounty board.",
|
||||
rhub.dataset_url(repo)) from None
|
||||
if status == 404:
|
||||
raise HubError("not-found", f"{repo} has no bounty board yet.") from None
|
||||
raise HubError("network", f"Couldn't fetch the bounty board ({type(e).__name__}: {e}).") from None
|
||||
return {"updated": board["updated"], "bounties": len(board["bounties"])}
|
||||
|
||||
|
||||
def upload(store, rnd, dry_run=False, again=False, progress=None, log=None):
|
||||
"""Upload exports/<round> (the steps in this file's docstring). Returns the result; raises HubError."""
|
||||
tell = progress or (lambda phase, text, fraction=None, **extra: None)
|
||||
say = log or (lambda text: None)
|
||||
import validate
|
||||
repo = dataset_id()
|
||||
try:
|
||||
export = store.export_dir(rnd)
|
||||
except ValueError as e:
|
||||
raise HubError("no-export", str(e)) from None
|
||||
if not os.path.isfile(os.path.join(export, "manifest.json")):
|
||||
raise HubError("no-export", f"No export of round {rnd}: export it first")
|
||||
|
||||
tell("check", "Checking the export", 0.0)
|
||||
report = validate.validate(export, progress=lambda f, text: tell("check", text, f))
|
||||
if not report.ok:
|
||||
raise HubError("invalid", f"The export has {len(report.errors)} problems, so it can't be uploaded. Export "
|
||||
"the round again; if that doesn't help, report it.", errors=report.errors)
|
||||
summary = report.summary
|
||||
sha = export_sha(export)
|
||||
before = rhub.previous_upload(meta_path(store, rnd), sha)
|
||||
if before and not again:
|
||||
raise HubError("duplicate", f"This export was uploaded already, on {before.get('uploaded', '?')}: "
|
||||
f"{before.get('pr_url', '')}", before.get("pr_url", ""), extra={"previous": before})
|
||||
if not dry_run and not upload_allowed(repo):
|
||||
raise HubError("closed", "Eye contributions aren't open yet: the consent and upload texts are waiting for a "
|
||||
f"legal review. (A rehearsal: {ALLOW_ENV}=1 with {DATASET_ENV} set to a test dataset.)")
|
||||
if repo == HF_DATASET and str(summary.get("consent_version") or "").endswith("draft"):
|
||||
raise HubError("draft-consent", "This export was made when you'd agreed to a draft of the consent text. "
|
||||
"Agree to the current one on the Welcome page, then export the round again.")
|
||||
path_in_repo = f"contributions/{summary.get('contributor', '')}/{rnd}"
|
||||
message, description = describe(summary, report, sha, report.bytes)
|
||||
files = []
|
||||
for root, _, names in os.walk(export):
|
||||
files += [os.path.relpath(os.path.join(root, n), export) for n in names]
|
||||
plan = {"repo": repo, "repo_type": "dataset", "folder_path": export, "path_in_repo": path_in_repo,
|
||||
"create_pr": True, "commit_message": message, "commit_description": description,
|
||||
"files": len(files), "bytes": report.bytes, "warnings": report.warnings}
|
||||
|
||||
if dry_run:
|
||||
say(f"dry run: would check the login (whoami) and access to {repo} (auth_check)")
|
||||
say(f"dry run: would upload_folder {len(files)} files, {takes.human_bytes(report.bytes)}, "
|
||||
f"from {export} to datasets/{repo}/{path_in_repo}, as a pull request")
|
||||
for rel in sorted(files):
|
||||
say(f" {rel} {takes.human_bytes(os.path.getsize(os.path.join(export, rel)))}")
|
||||
say(f"dry run: commit message: {message}")
|
||||
say("dry run: commit description:\n" + description.rstrip())
|
||||
record = {"repo": repo, "pr_url": "", "uploaded": now_iso(), "export_sha": sha, "status": "done"}
|
||||
say(f"dry run: would record in session.json's uploads: {json.dumps(record)}")
|
||||
tell("done", "Dry run: nothing was uploaded", 1.0)
|
||||
return dict(plan, dry_run=True, pr_url="", record=record)
|
||||
|
||||
pr, who, record = rhub.push(_hf(), repo, export, path_in_repo, message, description, meta_path(store, rnd), sha,
|
||||
len(files), report.bytes, tell, write_json, takes.human_bytes)
|
||||
tell("done", "Uploaded", 1.0)
|
||||
return dict(plan, dry_run=False, pr_url=record["pr_url"], pr_num=pr.num, user=who.get("name", ""), record=record)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Upload an eye recorder export to the eye dataset on Hugging Face.")
|
||||
ap.add_argument("--base", default=takes.DEFAULT_BASE)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
w = sub.add_parser("whoami", help="show the saved Hugging Face login")
|
||||
w.add_argument("--json", action="store_true")
|
||||
li = sub.add_parser("login", help="log in to Hugging Face in the browser")
|
||||
li.add_argument("--json", action="store_true")
|
||||
b = sub.add_parser("board", help="fetch the dataset's bounty board")
|
||||
b.add_argument("--json", action="store_true")
|
||||
u = sub.add_parser("upload", help="upload exports/ROUND as a pull request")
|
||||
u.add_argument("round")
|
||||
u.add_argument("--dry-run", action="store_true", help="no network: say what would be uploaded")
|
||||
u.add_argument("--again", action="store_true", help="upload even if this export was uploaded before")
|
||||
u.add_argument("--json", action="store_true", help="JSON lines, for the window")
|
||||
a = ap.parse_args()
|
||||
|
||||
def emit(obj):
|
||||
print(json.dumps(obj), flush=True)
|
||||
|
||||
try:
|
||||
if a.cmd == "whoami":
|
||||
who = whoami()
|
||||
if a.json:
|
||||
emit({"done": who})
|
||||
else:
|
||||
print(f"logged in as {who['name']} (token role: {who['role'] or 'unknown'})")
|
||||
return 0
|
||||
if a.cmd == "board":
|
||||
got = fetch_board(os.path.abspath(os.path.expanduser(a.base)))
|
||||
if a.json:
|
||||
emit({"done": got})
|
||||
else:
|
||||
print(f"bounty board of {got['updated']}: {got['bounties']} bounties")
|
||||
return 0
|
||||
if a.cmd == "login":
|
||||
def code(phase, text, **extra):
|
||||
if a.json:
|
||||
emit(dict(extra, phase=phase, text=text))
|
||||
else:
|
||||
print(f"Open {extra['url']} and approve the code {extra['code']}. Waiting...", flush=True)
|
||||
who = login(code)
|
||||
if a.json:
|
||||
emit({"done": who})
|
||||
else:
|
||||
print(f"logged in as {who['name']}")
|
||||
return 0
|
||||
if a.json:
|
||||
def progress(phase, text, fraction=None, **extra):
|
||||
emit(dict(extra, phase=phase, text=text, fraction=fraction))
|
||||
|
||||
def log(text):
|
||||
emit({"log": text})
|
||||
else:
|
||||
last = {}
|
||||
|
||||
def progress(phase, text, fraction=None, **extra):
|
||||
if extra.get("pr_url"):
|
||||
print(f"pull request: {extra['pr_url']}", flush=True)
|
||||
line = text if fraction is None else f"{fraction * 100:5.1f}% {text}"
|
||||
if (phase, text) != last.get("key") or fraction in (0.0, 1.0):
|
||||
print(line, file=sys.stderr, flush=True)
|
||||
last["key"] = (phase, text)
|
||||
|
||||
def log(text):
|
||||
print(text, flush=True)
|
||||
if hasattr(os, "setpriority"):
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, 0, 19) # validation reads and hashes everything: stay out of VR's way
|
||||
except OSError:
|
||||
pass
|
||||
result = upload(takes.Store(a.base), a.round, dry_run=a.dry_run, again=a.again, progress=progress, log=log)
|
||||
if a.json:
|
||||
emit({"done": result})
|
||||
elif result["dry_run"]:
|
||||
print(f"dry run done: {result['files']} files would go to datasets/{result['repo']}/{result['path_in_repo']}")
|
||||
else:
|
||||
print(f"uploaded: {result['pr_url']}")
|
||||
return 0
|
||||
except HubError as e:
|
||||
if a.json:
|
||||
emit({"error": e.as_dict()})
|
||||
else:
|
||||
print(f"error ({e.kind}): {e.text}", file=sys.stderr)
|
||||
for line in e.errors:
|
||||
print(f" {line}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,46 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the eye recorder on a Frame that has Frametop (get.sh --experimental): bring the dev
|
||||
# container's packages up to date, build the gaze tools the round uses (gaze/build.sh: ft-gaze and
|
||||
# the headset panel ft-gazepanel; gaze/tracker/build.sh: our tracker ft-eyes, which the fit coach
|
||||
# runs) and add "Frametop Eye Recorder" to the app menu (for Frametop's desktop, as the hand
|
||||
# recorder's). It doesn't change Frametop's gaze setup; the window offers SteamVR's 90 fps eye
|
||||
# cameras (gaze/eye-rate.py) when they're slower.
|
||||
# A release (FRAME_RELEASE) brings the gaze tools built in its image (pack/Containerfile), so there it
|
||||
# only adds the menu entry; its install.sh offers this.
|
||||
# The recorder reads the eye cameras through our eye tracker's frame grabber
|
||||
# (frametop-eyegrab.service, gaze/tracker/install.sh): without it, the window says so.
|
||||
# Usage: gaze/rec/install.sh install or update
|
||||
# gaze/rec/install.sh uninstall remove the menu entry. Recordings stay where they are.
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
||||
. "$root/scripts/_env.sh"
|
||||
entry='~/.local/share/applications/frametop-eyerec.desktop'
|
||||
|
||||
case ${1:-install} in
|
||||
install)
|
||||
if [ "$FRAME_RELEASE" = 1 ]; then
|
||||
echo "== the menu entry"
|
||||
else
|
||||
echo "== 1/3 dev container packages"
|
||||
"$root/setup/dev-container.sh"
|
||||
echo "== 2/3 the gaze tools: ft-gaze, ft-gazepanel and ft-eyes"
|
||||
"$root/gaze/build.sh"
|
||||
"$root/gaze/tracker/build.sh"
|
||||
echo "== 3/3 the menu entry"
|
||||
fi
|
||||
fill_template "$root/gaze/rec/ft-eyerec.desktop" |
|
||||
on_frame "mkdir -p ~/.local/share/applications && cat > $entry"
|
||||
echo
|
||||
echo "Installed. On Frametop's desktop, open \"Frametop Eye Recorder\" from the app menu."
|
||||
echo "Recordings go to ~/.local/share/frametop/eyes/contrib."
|
||||
on_frame 'systemctl is-enabled -q frametop-eyegrab.service 2>/dev/null' ||
|
||||
echo "It needs our eye tracker's frame grabber, which isn't installed: gaze/tracker/install.sh"
|
||||
;;
|
||||
uninstall)
|
||||
on_frame "rm -f $entry"
|
||||
echo "Removed the menu entry."
|
||||
echo "Your recordings are still in ~/.local/share/frametop/eyes/contrib:"
|
||||
echo "delete that folder to remove them."
|
||||
;;
|
||||
*) echo "usage: $0 [install|uninstall]" >&2; exit 2 ;;
|
||||
esac
|
||||
-1811
File diff suppressed because it is too large.
Load diff
@@ -1,6 +0,0 @@
|
||||
# The eye dataset's recipients (gaze/rec/DESIGN.md "Files"): exports encrypt each chunk and health.json
|
||||
# to all of them (recorder/age.py). Key 1 is kept on the 7i, key 2 offline,
|
||||
# key 3 on dody (the GPU training machine, which decrypts pull requests itself).
|
||||
age1a0tc98pj2s7gdfgrv4gam6ah0qv69ymf3t2cc25hcwepmats69hst35z2e
|
||||
age1r0zlq7kcyg4dz4y3vkl9j36jw3zc6ly6reddh9ag5kfy6aw0fy6s5hepat
|
||||
age1e5vwgc03s9mk3dj5j39lcgxg6qyvkml77qh7g8svygar49r2kd7svldt70
|
||||
@@ -1,106 +0,0 @@
|
||||
# Building an eye bounty
|
||||
|
||||
A bounty is two things:
|
||||
|
||||
- **A board entry** in `gaze/rec/bounties.json`: id, title, blurb, minutes, priority, the script it runs,
|
||||
and what it needs from the checklist (`recorder/bounties.py` has the format).
|
||||
- **A round script**, `gaze/rec/rounds/<script>.json`: the steps (`session.py`'s docstring has the
|
||||
format of every step kind).
|
||||
|
||||
Scripts ship with the recorder. The dataset's copy of the board (`DeeJanuz/frametop-eyes`,
|
||||
`bounties.json`) can only describe, reorder, reprioritise or retire bounties, so a new bounty reaches
|
||||
people only with a Frametop release that carries its script.
|
||||
|
||||
`load_round` enforces the rules marked **(checked)**: a script that breaks one doesn't load, and the
|
||||
tests fail.
|
||||
|
||||
## What the round is for
|
||||
|
||||
Every recorded second becomes training data, labelled by where the dot was. So:
|
||||
|
||||
- **Know what each step labels.** In most steps the eyes are on a dot, and the dot's position is the
|
||||
label. Where they aren't (eyes shut, reading, looking around freely), the cue words or the step's
|
||||
kind say so, and training leaves those frames out or treats them as their own case.
|
||||
- **One edge case per bounty.** Say in the board's blurb and the DESIGN.md table which case it covers
|
||||
and why the trackers fail there.
|
||||
- **Rounds open the same way.** The recorder adds putting the headset on and the nine dots before every
|
||||
round. Don't repeat them. The group `dots` is theirs **(checked)**.
|
||||
|
||||
## Cues the person can't see get a beep
|
||||
|
||||
The panel can't cue someone whose eyes are shut. A cue that asks for that plays a beep
|
||||
(`gaze/rec/sounds.py`) as its third field: `["Close your eyes", 2.5, "close"]`.
|
||||
|
||||
- **Blink:** a cue whose word asks for a blink plays `blink` **(checked)**.
|
||||
- **Both eyes shut:** a cue asking to close or shut both eyes plays `close` **(checked)**, and the next
|
||||
cue plays `open` **(checked)**, so the person hears when to open them. A step never ends with the eyes
|
||||
shut.
|
||||
- **Say it in the step's text**, which is read before Next: "Close your eyes when you hear a beep, and
|
||||
then open your eyes when you hear the beep again." The words still show on the panel too.
|
||||
- **One eye or half-closed** (winks, squints, half-closed lids) keep words alone: the open eye reads
|
||||
them. They may beep too.
|
||||
- **No voice.** Beeps only; generated speech sounds odd in the headset. A new kind of cue sound goes
|
||||
into `sounds.SOUNDS` and the tests.
|
||||
|
||||
## Cue words are labels
|
||||
|
||||
Training reads the cue words in `prompts.jsonl` (`frame-eyes/train/data/ingest.py`, `shut_cams`):
|
||||
|
||||
- A word with **blink, close, shut, wink or yawn** marks the eyes as shut while it shows. **Left** or
|
||||
**right** limits that to one eye, and **half** means the lids are partly open, so the frames keep
|
||||
their labels.
|
||||
- Use those words only for that. "Open", "Look", "Relax" and "Normal" are the words for going back.
|
||||
- Keep words short (40 characters at most): they show big on the panel.
|
||||
|
||||
## Timing
|
||||
|
||||
- **Dots:** a hold of 1.2 s or more after a settle of about 0.6 s (the defaults). The eyes take that
|
||||
long to land on a dot and stay.
|
||||
- **Shut eyes:** 2 s or more closed, then 2 s or more open before anything else, so the eyes settle back
|
||||
on the dot.
|
||||
- **Blinks on a beep:** uneven gaps, 1.2 to 2 s, so the beep can't be predicted.
|
||||
- **Length:** about 4 to 6 minutes recorded. Check with `python3 gaze/rec/session.py <script> --plan`, and
|
||||
put that, rounded, in the board's `minutes`.
|
||||
|
||||
## Where things can go
|
||||
|
||||
- **Dots and paths stay on the panel** **(checked)**: about 30 degrees to the sides, 23 up and down, less
|
||||
in the corners (`off_panel`).
|
||||
- **Distance:** 0.25 to 10 m **(checked)**. 1.5 m unless distance is what the bounty is about.
|
||||
- **Head still,** unless the bounty is about moving it: say "Keep your head still" in the text of dot steps.
|
||||
|
||||
## Takes
|
||||
|
||||
- Steps of the same `group` share one take, and a group's steps must follow each other **(checked)**.
|
||||
Group steps that a reviewer would delete together.
|
||||
- A `reseat` step isn't recorded, and a round with one gets no shift check.
|
||||
- Every step waits for Next unless it has `"ready": false`. Only chain steps that follow on naturally.
|
||||
|
||||
## Comfort, safety and privacy
|
||||
|
||||
- Comfortable eye turns only: "as far as is comfortable", never "as far as you can".
|
||||
- No flashing: brightness changes are steps of a second or more (`light` levels), never a flicker.
|
||||
- Nothing that asks for sound, the outward cameras, or anything about the person beyond the checklist.
|
||||
Talking is fine (it moves the face), but the microphone is never recorded.
|
||||
- Text to read is neutral and made up: no names, places or questions about the person.
|
||||
|
||||
## Writing the text
|
||||
|
||||
- Plain words, one instruction per step, the action first: "Look at each dot until its ring fills."
|
||||
- `|` starts a new line on the panel, `||` leaves a blank line.
|
||||
- Say what to do, not why. The why goes in the round's `intro` or the board's blurb.
|
||||
|
||||
## Before it ships
|
||||
|
||||
1. `python3 gaze/rec/session.py <script> --plan`: the steps and their length.
|
||||
2. The tests, in the dev container: `cd gaze/rec && for t in tests/test_*.py; do python3 $t; done`.
|
||||
`test_shipped_rounds_load` loads every script; update its count.
|
||||
3. A dry run: `python3 gaze/rec/session.py <script> --dry-run --speed 20 --next-after 0.1`.
|
||||
4. A replay end to end: `frame-eyes/tools/eyerec-dryrun <bounty>` (record, export, validate), and the
|
||||
export through `eyes_hub.py check` and `ingest.py` on dody. Delete those rounds afterwards: their
|
||||
labels are wrong.
|
||||
5. A real round in the headset, with a note in `HEADSET-TESTS.md`: is every cue clear, and is the
|
||||
length right?
|
||||
6. Add the bounty to `gaze/rec/bounties.json` and a row to DESIGN.md's bounty table. After a release
|
||||
carries the script, put it on the dataset's board too (`frame-eyes/dataset-repo/bounties.json`,
|
||||
uploaded to `DeeJanuz/frametop-eyes`).
|
||||
@@ -1,245 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "blinks",
|
||||
"title": "Blinks and lids",
|
||||
"intro": "Blinks, closed and half-closed eyes, some of them on a beep. Keep looking at the dot whenever your eyes are open.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "blink-cue",
|
||||
"kind": "prompt",
|
||||
"text": "Look at the dot and blink once on each beep.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Blink",
|
||||
1.6,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.3,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
2.0,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.8,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.2,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.7,
|
||||
"blink"
|
||||
],
|
||||
[
|
||||
"Blink",
|
||||
1.5,
|
||||
"blink"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "closed",
|
||||
"kind": "prompt",
|
||||
"text": "Close your eyes when you hear a beep, and then open your eyes when you hear the beep again.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Look",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Close your eyes",
|
||||
2.5,
|
||||
"close"
|
||||
],
|
||||
[
|
||||
"Open your eyes",
|
||||
2,
|
||||
"open"
|
||||
],
|
||||
[
|
||||
"Close your eyes",
|
||||
2.5,
|
||||
"close"
|
||||
],
|
||||
[
|
||||
"Open your eyes",
|
||||
2,
|
||||
"open"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "winks",
|
||||
"kind": "prompt",
|
||||
"text": "Wink each eye in turn when it says so (as well as you can).",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Look",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your left eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your right eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your left eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
],
|
||||
[
|
||||
"Close your right eye",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Open",
|
||||
1.5
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "squint",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot while you change how open your eyes are.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Squint",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Eyes wide open",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Half closed",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Normal",
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "lids-down",
|
||||
"kind": "dots",
|
||||
"text": "Look down at each dot. Let your lids drop as they do when you look down.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
-12
|
||||
],
|
||||
[
|
||||
0,
|
||||
-18
|
||||
],
|
||||
[
|
||||
12,
|
||||
-18
|
||||
],
|
||||
[
|
||||
-12,
|
||||
-18
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
18,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-18,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "blink-dots",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot and blink once while its ring fills.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
10
|
||||
],
|
||||
[
|
||||
-15,
|
||||
10
|
||||
],
|
||||
[
|
||||
15,
|
||||
-10
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-10
|
||||
]
|
||||
],
|
||||
"hold": 2.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "depth",
|
||||
"title": "Depth",
|
||||
"intro": "Dots near and far. The panel moves closer and further away; keep the dot sharp.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "near",
|
||||
"kind": "dots",
|
||||
"text": "The panel is close now. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 0.5
|
||||
},
|
||||
{
|
||||
"id": "mid",
|
||||
"kind": "dots",
|
||||
"text": "The panel is at arm's length and a bit. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 1.5
|
||||
},
|
||||
{
|
||||
"id": "far",
|
||||
"kind": "dots",
|
||||
"text": "The panel is far now. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.5,
|
||||
"shuffle": true,
|
||||
"distance": 5.0
|
||||
},
|
||||
{
|
||||
"id": "near-far",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot as it comes near and goes far.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Far",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"Near",
|
||||
2.5
|
||||
]
|
||||
],
|
||||
"cue_distance": {
|
||||
"Far": 5.0,
|
||||
"Near": 0.5
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,129 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "edges",
|
||||
"title": "View edges",
|
||||
"intro": "Dots out at the edges of the panel and in its corners: turn only your eyes to them, as far as is comfortable.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "edges",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.|Some are far out: turn your eyes, not your head.",
|
||||
"dots": [
|
||||
[
|
||||
24,
|
||||
0
|
||||
],
|
||||
[
|
||||
-24,
|
||||
0
|
||||
],
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
18
|
||||
],
|
||||
[
|
||||
0,
|
||||
-18
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
22,
|
||||
16
|
||||
],
|
||||
[
|
||||
-22,
|
||||
16
|
||||
],
|
||||
[
|
||||
22,
|
||||
-16
|
||||
],
|
||||
[
|
||||
-22,
|
||||
-16
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "edges-sweep",
|
||||
"kind": "path",
|
||||
"text": "Follow the dot with your eyes as it goes round, far out.",
|
||||
"path": "circle 0 0 22 8",
|
||||
"seconds": 16
|
||||
},
|
||||
{
|
||||
"id": "edges-again",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "face",
|
||||
"title": "Face moves",
|
||||
"intro": "Faces move the headset and the lids. Keep looking at the dot while you make each face. The microphone isn't recorded.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "faces",
|
||||
"kind": "prompt",
|
||||
"text": "Keep looking at the dot and do what it says.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"cues": [
|
||||
[
|
||||
"Smile",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Raise your eyebrows",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Frown",
|
||||
3
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Yawn (or pretend to)",
|
||||
4
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
],
|
||||
[
|
||||
"Chew",
|
||||
4
|
||||
],
|
||||
[
|
||||
"Relax",
|
||||
2
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "talk",
|
||||
"kind": "prompt",
|
||||
"text": "Read this aloud while looking at the dot as much as you can:|\"The quick brown fox jumps over the lazy dog, then naps in the warm afternoon sun.\"",
|
||||
"dot": [
|
||||
0,
|
||||
10
|
||||
],
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "faces-dots",
|
||||
"kind": "dots",
|
||||
"text": "Smile and keep smiling. Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
15,
|
||||
15
|
||||
],
|
||||
"hold": 1.0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "glasses",
|
||||
"title": "Glasses",
|
||||
"intro": "With your glasses or inserts on. Wear them as you usually do in the headset.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "grid",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "edges",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.|Some are far out: turn your eyes, not your head.",
|
||||
"dots": [
|
||||
[
|
||||
30,
|
||||
0
|
||||
],
|
||||
[
|
||||
-30,
|
||||
0
|
||||
],
|
||||
[
|
||||
0,
|
||||
23
|
||||
],
|
||||
[
|
||||
0,
|
||||
-23
|
||||
],
|
||||
[
|
||||
30,
|
||||
20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
20
|
||||
],
|
||||
[
|
||||
30,
|
||||
-20
|
||||
],
|
||||
[
|
||||
-30,
|
||||
-20
|
||||
]
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
},
|
||||
{
|
||||
"id": "follow",
|
||||
"kind": "path",
|
||||
"text": "Follow the dot with your eyes.",
|
||||
"path": "eight 0 0 22 14 6",
|
||||
"seconds": 12
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "head",
|
||||
"title": "Head turning",
|
||||
"intro": "A dot that stays put in the room while you turn your head. Keep your eyes on it.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "turn-sides",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Turn your head slowly left and right.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 15
|
||||
},
|
||||
{
|
||||
"id": "turn-updown",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Nod slowly up and down.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "tilt",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Tilt your head side to side.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "shake",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "Keep looking at the dot. Shake your head a little, quickly.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-1.5
|
||||
],
|
||||
"seconds": 8
|
||||
},
|
||||
{
|
||||
"id": "walk",
|
||||
"kind": "world",
|
||||
"group": "head",
|
||||
"text": "If you have room: take a step to each side while looking at the dot.",
|
||||
"point": [
|
||||
0,
|
||||
0,
|
||||
-2.0
|
||||
],
|
||||
"seconds": 12
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,78 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "jumps",
|
||||
"title": "Jumps and pursuit",
|
||||
"intro": "A dot that jumps around, then one that glides. Follow it with your eyes, as quickly as you can, without moving your head.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "jumps",
|
||||
"kind": "jumps",
|
||||
"text": "Look at the dot wherever it jumps.",
|
||||
"count": 40,
|
||||
"range": [
|
||||
24,
|
||||
18
|
||||
],
|
||||
"each": [
|
||||
0.5,
|
||||
1.5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "circle-slow",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot with your eyes.",
|
||||
"path": "circle 0 0 15 6",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "circle-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot with your eyes. It's faster now.",
|
||||
"path": "circle 0 0 15 2.5",
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "line-slow",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot from side to side.",
|
||||
"path": "line -24 0 24 0 3",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "line-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot from side to side, faster.",
|
||||
"path": "line -24 0 24 0 1.2",
|
||||
"seconds": 8
|
||||
},
|
||||
{
|
||||
"id": "line-up",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot up and down.",
|
||||
"path": "line 0 -18 0 18 2.5",
|
||||
"seconds": 10
|
||||
},
|
||||
{
|
||||
"id": "eight",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot along the figure eight.",
|
||||
"path": "eight 0 0 22 12 6",
|
||||
"seconds": 12
|
||||
},
|
||||
{
|
||||
"id": "eight-fast",
|
||||
"kind": "path",
|
||||
"group": "pursuit",
|
||||
"text": "Follow the dot along the figure eight, faster.",
|
||||
"path": "eight 0 0 22 12 3",
|
||||
"seconds": 9
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "light",
|
||||
"title": "Light changes",
|
||||
"intro": "The panel goes dark and bright. Your pupils change size with it: that's what this records.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "steps",
|
||||
"kind": "light",
|
||||
"text": "Keep looking at the dot while the light changes.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"levels": [
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.9,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.33,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.9,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.33,
|
||||
4
|
||||
],
|
||||
[
|
||||
0.03,
|
||||
4
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "grid-dark",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.03
|
||||
},
|
||||
{
|
||||
"id": "grid-mid",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.33
|
||||
},
|
||||
{
|
||||
"id": "grid-bright",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
5,
|
||||
5,
|
||||
24,
|
||||
18
|
||||
],
|
||||
"hold": 1.0,
|
||||
"shuffle": true,
|
||||
"bg": 0.9
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "reading",
|
||||
"title": "Reading",
|
||||
"intro": "Reading and looking around, as in everyday use.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "read-1",
|
||||
"kind": "prompt",
|
||||
"group": "reading",
|
||||
"dot": null,
|
||||
"seconds": 20,
|
||||
"text": "Read this at your own pace.||The lighthouse keeper climbed the spiral stairs every evening, counting each of the one hundred and twelve steps. At the top, he polished the great lens until it shone, then lit the lamp and watched its beam sweep across the dark water. Ships far out at sea would see the flash and know where the rocks were."
|
||||
},
|
||||
{
|
||||
"id": "read-2",
|
||||
"kind": "prompt",
|
||||
"group": "reading",
|
||||
"dot": null,
|
||||
"seconds": 20,
|
||||
"text": "And this one.||In the morning the market filled with stalls of oranges, peppers and fresh bread. A boy on a bicycle wove between the crates, ringing his bell. Somewhere a radio played an old song, and an old woman hummed along while she weighed out a bag of cherries for a customer who couldn't decide."
|
||||
},
|
||||
{
|
||||
"id": "free",
|
||||
"kind": "prompt",
|
||||
"dot": null,
|
||||
"seconds": 15,
|
||||
"text": "Look around the panel freely: the corners, the edges, wherever you like."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,231 +0,0 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"id": "slips",
|
||||
"title": "Headset slips",
|
||||
"intro": "You'll nudge the headset on your face on purpose, then put it back. Use one hand, gently: a few millimetres is enough.",
|
||||
"steps": [
|
||||
{
|
||||
"id": "nudge-up",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Keep looking at the dot and push the headset up a little. Hold it there.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-up",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold the headset there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-down",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then pull the headset down a little. Hold it there.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-down",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-left",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then slide the headset a little to your left.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-left",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-right",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then slide it a little to your right.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-right",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "nudge-tilt",
|
||||
"kind": "prompt",
|
||||
"group": "nudges",
|
||||
"text": "Let go, then tilt the headset a little: one side up.",
|
||||
"dot": [
|
||||
0,
|
||||
0
|
||||
],
|
||||
"seconds": 5
|
||||
},
|
||||
{
|
||||
"id": "after-tilt",
|
||||
"kind": "dots",
|
||||
"group": "nudges",
|
||||
"ready": false,
|
||||
"text": "Hold it there. Look at each dot until its ring fills. Keep your head still.",
|
||||
"dots": [
|
||||
[
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
15,
|
||||
12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
15,
|
||||
-12
|
||||
],
|
||||
[
|
||||
-15,
|
||||
12
|
||||
]
|
||||
],
|
||||
"hold": 1.0
|
||||
},
|
||||
{
|
||||
"id": "reseat",
|
||||
"kind": "reseat",
|
||||
"text": "Take the headset off, put it back on as you usually do, then press the button."
|
||||
},
|
||||
{
|
||||
"id": "after-reseat",
|
||||
"kind": "dots",
|
||||
"text": "Look at each dot until its ring fills. Keep your head still.",
|
||||
"grid": [
|
||||
3,
|
||||
3,
|
||||
20,
|
||||
20
|
||||
],
|
||||
"hold": 1.2,
|
||||
"shuffle": true
|
||||
}
|
||||
]
|
||||
}
|
||||
-1659
File diff suppressed because it is too large.
Load diff
@@ -1,125 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's sound cues: plain beeps for steps done with the eyes shut, where the panel's
|
||||
words can't be seen. A round script names one as a cue's third field (session.py: "cues":
|
||||
[[word, seconds, sound]...]); it plays as the cue starts, and prompts.jsonl's "cue" line says which.
|
||||
|
||||
blink a short high beep: blink once
|
||||
close a beep: close your eyes
|
||||
open the same beep again: open them
|
||||
|
||||
No voice, no sound files: the beeps are made here (a sine with a short fade, so it doesn't click)
|
||||
into the run folder, and played with PipeWire's pw-play on the default output (the headset's
|
||||
speakers). A beep starts some tens of ms after its cue line (pw-play's start and its 20 ms
|
||||
latency); a person's reaction to it takes several times that.
|
||||
|
||||
sounds.py play each one, a second apart
|
||||
sounds.py NAME... play those
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import wave
|
||||
|
||||
RATE = 48000
|
||||
VOLUME = 0.6 # pw-play's stream volume; the beep itself is at half scale
|
||||
# name: (pitch Hz, seconds)
|
||||
SOUNDS = {
|
||||
"blink": (1000.0, 0.09),
|
||||
"close": (660.0, 0.18),
|
||||
"open": (660.0, 0.18),
|
||||
}
|
||||
FADE_S = 0.005
|
||||
|
||||
|
||||
def _dir():
|
||||
d = os.path.join(os.environ.get("XDG_RUNTIME_DIR") or "/run/user/%d" % os.getuid(), "frametop", "eyerec-sounds")
|
||||
os.makedirs(d, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def wav_path(name):
|
||||
"""The sound's WAV file, made the first time (and again when its pitch or length changed)."""
|
||||
hz, secs = SOUNDS[name]
|
||||
path = os.path.join(_dir(), "%s-%d-%d.wav" % (name, hz, secs * 1000))
|
||||
if not os.path.exists(path):
|
||||
n, fade = int(RATE * secs), int(RATE * FADE_S)
|
||||
frames = bytearray()
|
||||
for i in range(n):
|
||||
env = min(1.0, i / fade, (n - 1 - i) / fade)
|
||||
frames += struct.pack("<h", int(0.5 * 32767 * env * math.sin(2 * math.pi * hz * i / RATE)))
|
||||
with wave.open(path + ".tmp", "wb") as w:
|
||||
w.setnchannels(1)
|
||||
w.setsampwidth(2)
|
||||
w.setframerate(RATE)
|
||||
w.writeframes(bytes(frames))
|
||||
os.replace(path + ".tmp", path)
|
||||
return path
|
||||
|
||||
|
||||
class Player:
|
||||
"""Plays sounds without waiting for them. Off (dry runs, tests): it only says what it would play.
|
||||
Without pw-play, or when a sound fails, it logs that once and the round goes on silent."""
|
||||
|
||||
def __init__(self, on=True, log=None):
|
||||
self.log = log or (lambda text: None)
|
||||
self.pw_play = shutil.which("pw-play") if on else None
|
||||
self.procs = []
|
||||
self.warned = False
|
||||
if on and not self.pw_play:
|
||||
self._warn("no pw-play: the sound cues are off")
|
||||
if self.pw_play:
|
||||
for name in SOUNDS: # made now, not while a step runs
|
||||
wav_path(name)
|
||||
|
||||
def _warn(self, text):
|
||||
if not self.warned:
|
||||
self.warned = True
|
||||
self.log(text)
|
||||
|
||||
def play(self, name):
|
||||
if name not in SOUNDS:
|
||||
raise ValueError("no sound %r" % name)
|
||||
if not self.pw_play:
|
||||
return False
|
||||
self.procs = [p for p in self.procs if p.poll() is None]
|
||||
env = dict(os.environ, XDG_RUNTIME_DIR=os.environ.get("XDG_RUNTIME_DIR") or "/run/user/%d" % os.getuid())
|
||||
try:
|
||||
self.procs.append(subprocess.Popen(
|
||||
[self.pw_play, "--latency", "20ms", "--volume", str(VOLUME), "--media-role", "Notification",
|
||||
wav_path(name)], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
env=env))
|
||||
return True
|
||||
except OSError as e:
|
||||
self._warn("can't play the sound cues: %s" % e)
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
for p in self.procs:
|
||||
try:
|
||||
p.wait(timeout=1)
|
||||
except subprocess.TimeoutExpired:
|
||||
p.kill()
|
||||
self.procs = []
|
||||
|
||||
|
||||
def main(argv):
|
||||
names = argv or list(SOUNDS)
|
||||
bad = [n for n in names if n not in SOUNDS]
|
||||
if bad:
|
||||
sys.exit("no sound %s; there are: %s" % (", ".join(bad), ", ".join(SOUNDS)))
|
||||
p = Player(log=lambda text: print(text, file=sys.stderr))
|
||||
for i, name in enumerate(names):
|
||||
if i:
|
||||
time.sleep(1.0)
|
||||
print(name)
|
||||
p.play(name)
|
||||
p.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -1,472 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's rounds and takes on disk: listing, review, deleting, and the encrypted export.
|
||||
|
||||
The layout is DESIGN.md's "Files":
|
||||
BASE/profile.json
|
||||
BASE/rounds/<YYYYMMDD-HHMMSS>-<bounty>/session.json, fit.json, health.json, session.log
|
||||
BASE/rounds/<round>/takes/<NN>-<group>/cam0/*.zst, cam1/*.zst, index.tsv (eyerec.py),
|
||||
targets.jsonl, poses.jsonl, ipd.jsonl, own.jsonl, steamvr.jsonl, prompts.jsonl, take.json
|
||||
BASE/exports/<round>/
|
||||
|
||||
Deleted ranges live in take.json ("deleted": [[from_ns, to_ns], ...]) on CLOCK_MONOTONIC, the
|
||||
clock of prompts.jsonl, which the window's timeline shows. The frames and the other logs are on
|
||||
CLOCK_MONOTONIC_RAW seconds; take.json's clock samples ([mono_ns, raw - mono ns]) turn one into
|
||||
the other. The files keep every frame until export leaves the deleted ones out.
|
||||
|
||||
The export (exports/<round>/, built in <round>.partial and moved when it's whole):
|
||||
manifest.json the round, its profile and session, the recipients, each take
|
||||
fit.json the fit coach's readings, as recorded
|
||||
health.json.age the opt-in health answers, encrypted (when there are any)
|
||||
takes/<id>/take.json as recorded, plus "export": the deleted ranges in RAW seconds
|
||||
takes/<id>/index.tsv the kept frames; a chunk keeps its number, frames renumbered in it
|
||||
takes/<id>/cam0/NNNNNN.zst.age ... each chunk encrypted to every recipient (recorder/age.py)
|
||||
takes/<id>/targets.jsonl, ipd.jsonl, prompts.jsonl whole: they say what was shown and asked
|
||||
takes/<id>/own.jsonl, steamvr.jsonl, poses.jsonl rows in deleted ranges left out
|
||||
SHA256SUMS over every file above, the encrypted ones as they are
|
||||
A chunk none of whose frames are deleted is encrypted as it is; one with some deleted frames is
|
||||
decompressed, the rest compressed again (zstd 1, as eyerec.py) and encrypted; one with none left
|
||||
is dropped. Every chunk is decompressed once to check it holds the frames index.tsv says; a
|
||||
chunk that doesn't, or one index.tsv doesn't name (the recorder was killed before the index
|
||||
line), is left out and counted in the manifest. session.log isn't exported (it names paths).
|
||||
|
||||
Needs the cryptography library to export (recorder/age.py). The rest is standard library.
|
||||
|
||||
usage: takes.py [--base DIR] list | takes ROUND | export ROUND
|
||||
takes.py unpack EXPORT --identity KEYFILE --out DIR decrypt an export (the dataset's side)
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import threading
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REPO = os.path.dirname(os.path.dirname(HERE))
|
||||
sys.path.insert(0, REPO)
|
||||
sys.path.insert(0, HERE)
|
||||
import eyerec # noqa: E402 (gaze/rec/eyerec.py: the chunks' format)
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.common import tree_version, write_json # noqa: E402
|
||||
from recorder.files import (Cancelled, human_bytes, in_ranges, merge_ranges, read_json, sha256_file, # noqa: E402,F401
|
||||
tree_bytes, tree_stat, write_sums)
|
||||
|
||||
DEFAULT_BASE = os.path.expanduser("~/.local/share/frametop/eyes/contrib")
|
||||
RECIPIENTS = os.path.join(HERE, "recipients.txt")
|
||||
ROUND_RE = re.compile(r"^\d{8}-\d{6}-[a-z0-9][a-z0-9-]{0,39}$") # session.py: <time>-<bounty>[-N]
|
||||
TAKE_RE = re.compile(r"^\d{2,}-[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
CHUNK_RE = re.compile(r"^(\d{6})\.zst$")
|
||||
EXPORT_SCHEMA = 1
|
||||
WHOLE_LOGS = ("targets.jsonl", "ipd.jsonl", "prompts.jsonl")
|
||||
CUT_LOGS = ("own.jsonl", "steamvr.jsonl", "poses.jsonl") # "t" in RAW seconds
|
||||
GAP_S = 0.5 # frames further apart than this are a pause, not recorded time
|
||||
|
||||
|
||||
def tool_version():
|
||||
return "ft-eyerec " + (tree_version(REPO) or "unknown")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- clocks
|
||||
|
||||
def raw_offset_ns(t_ns, clock):
|
||||
"""RAW - MONOTONIC at t_ns (MONOTONIC), from take.json's clock samples: linear between the
|
||||
two around it, the nearest outside them, 0 without any."""
|
||||
pts = sorted((int(m), int(d)) for m, d in clock or [] if isinstance(m, (int, float)) and isinstance(d, (int, float)))
|
||||
if not pts:
|
||||
return 0
|
||||
if t_ns <= pts[0][0]:
|
||||
return pts[0][1]
|
||||
for (m0, d0), (m1, d1) in zip(pts, pts[1:]):
|
||||
if t_ns <= m1:
|
||||
return d0 + (d1 - d0) * (t_ns - m0) // max(1, m1 - m0)
|
||||
return pts[-1][1]
|
||||
|
||||
|
||||
def raw_ranges(ranges, clock):
|
||||
"""Deleted ranges (MONOTONIC ns) as [[from, to], ...] in RAW seconds."""
|
||||
return [[(a + raw_offset_ns(a, clock)) / 1e9, (b + raw_offset_ns(b, clock)) / 1e9] for a, b in merge_ranges(ranges)]
|
||||
|
||||
|
||||
def recorded_seconds(times):
|
||||
"""Time spent recording: the gaps under GAP_S between sorted frame times."""
|
||||
total = 0.0
|
||||
for a, b in zip(times, times[1:]):
|
||||
if b - a < GAP_S:
|
||||
total += b - a
|
||||
return total
|
||||
|
||||
|
||||
def cut_jsonl(src, dst, ranges):
|
||||
"""Copy a jsonl log, leaving out rows whose "t" (RAW seconds) is in ranges. Lines that don't
|
||||
parse are copied as they are (validate.py reports them)."""
|
||||
kept = cut = 0
|
||||
with open(src, encoding="utf-8", errors="replace") as f, open(dst, "w", encoding="utf-8", newline="\n") as out:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
t = json.loads(line).get("t")
|
||||
except (ValueError, AttributeError):
|
||||
t = None
|
||||
if isinstance(t, (int, float)) and in_ranges(t, ranges):
|
||||
cut += 1
|
||||
continue
|
||||
out.write(line if line.endswith("\n") else line + "\n")
|
||||
kept += 1
|
||||
return kept, cut
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the store
|
||||
|
||||
class Store:
|
||||
def __init__(self, base=DEFAULT_BASE):
|
||||
self.base = os.path.abspath(os.path.expanduser(base))
|
||||
self.rounds_dir = os.path.join(self.base, "rounds")
|
||||
self.exports_dir = os.path.join(self.base, "exports")
|
||||
self.profile_path = os.path.join(self.base, "profile.json")
|
||||
|
||||
# --- paths, checked: ids come from the window, so nothing may climb out of the base
|
||||
def round_dir(self, rnd):
|
||||
if not ROUND_RE.match(rnd or ""):
|
||||
raise ValueError(f"not a round id: {rnd!r}")
|
||||
return os.path.join(self.rounds_dir, rnd)
|
||||
|
||||
def take_dir(self, rnd, take):
|
||||
if not TAKE_RE.match(take or ""):
|
||||
raise ValueError(f"not a take id: {take!r}")
|
||||
return os.path.join(self.round_dir(rnd), "takes", take)
|
||||
|
||||
def export_dir(self, rnd):
|
||||
self.round_dir(rnd) # checks the id
|
||||
return os.path.join(self.exports_dir, rnd)
|
||||
|
||||
def profile(self):
|
||||
return read_json(self.profile_path)
|
||||
|
||||
# --- listing
|
||||
def rounds(self):
|
||||
"""Newest first: [{id, bounty, title, started, status, dry_run, takes, bytes, exported,
|
||||
export_stale (changed since), export_bytes}]."""
|
||||
out = []
|
||||
try:
|
||||
names = os.listdir(self.rounds_dir)
|
||||
except OSError:
|
||||
return out
|
||||
for rid in names:
|
||||
path = os.path.join(self.rounds_dir, rid)
|
||||
if not ROUND_RE.match(rid) or not os.path.isdir(path):
|
||||
continue
|
||||
meta = read_json(os.path.join(path, "session.json"))
|
||||
size, changed = tree_stat(path)
|
||||
manifest = os.path.join(self.export_dir(rid), "manifest.json")
|
||||
exported = os.path.getmtime(manifest) if os.path.isfile(manifest) else 0.0
|
||||
out.append({"id": rid, "bounty": meta.get("bounty", ""), "title": meta.get("title", ""),
|
||||
"started": meta.get("started", ""), "status": meta.get("status", ""),
|
||||
"dry_run": bool(meta.get("dry_run")), "takes": len(self.take_ids(rid)), "bytes": size,
|
||||
"exported": bool(exported), "export_stale": bool(exported) and changed > exported,
|
||||
"export_bytes": tree_bytes(self.export_dir(rid)) if exported else 0})
|
||||
return sorted(out, key=lambda r: r["id"], reverse=True)
|
||||
|
||||
def take_ids(self, rnd):
|
||||
d = os.path.join(self.round_dir(rnd), "takes")
|
||||
try:
|
||||
names = os.listdir(d)
|
||||
except OSError:
|
||||
return []
|
||||
return sorted(n for n in names if TAKE_RE.match(n) and os.path.isdir(os.path.join(d, n)))
|
||||
|
||||
def takes(self, rnd):
|
||||
"""[{id, group, status, frames: [right, left], deleted_frames: [..], ranges, duration_s,
|
||||
bytes, started_ns, ended_ns}]."""
|
||||
out = []
|
||||
for tid in self.take_ids(rnd):
|
||||
path = self.take_dir(rnd, tid)
|
||||
meta = read_json(os.path.join(path, "take.json"))
|
||||
ranges = merge_ranges(meta.get("deleted") or [])
|
||||
raw = raw_ranges(ranges, meta.get("clock"))
|
||||
times = {c: [] for c in eyerec.CAMERAS}
|
||||
for c, _chunk, _i, _slot, t in eyerec.read_index(path):
|
||||
if c in times:
|
||||
times[c].append(t)
|
||||
for c in times:
|
||||
times[c].sort()
|
||||
out.append({"id": tid, "group": meta.get("group", tid.split("-", 1)[-1]),
|
||||
"status": meta.get("status", "recording"),
|
||||
"frames": [len(times[c]) for c in eyerec.CAMERAS],
|
||||
"deleted_frames": [sum(1 for t in times[c] if in_ranges(t, raw)) for c in eyerec.CAMERAS],
|
||||
"ranges": ranges, "duration_s": max(recorded_seconds(times[c]) for c in eyerec.CAMERAS),
|
||||
"bytes": sum(tree_bytes(os.path.join(path, "cam%d" % c)) for c in eyerec.CAMERAS),
|
||||
"started_ns": meta.get("started_ns"), "ended_ns": meta.get("ended_ns")})
|
||||
return out
|
||||
|
||||
def take_meta(self, rnd, take):
|
||||
return read_json(os.path.join(self.take_dir(rnd, take), "take.json"))
|
||||
|
||||
# --- deleted ranges
|
||||
def ranges(self, rnd, take):
|
||||
return merge_ranges(self.take_meta(rnd, take).get("deleted") or [])
|
||||
|
||||
def _set_ranges(self, rnd, take, ranges):
|
||||
path = os.path.join(self.take_dir(rnd, take), "take.json")
|
||||
meta = read_json(path)
|
||||
if not meta:
|
||||
raise RuntimeError(f"no take {rnd}/{take}")
|
||||
meta["deleted"] = merge_ranges(ranges)
|
||||
write_json(path, meta)
|
||||
return meta["deleted"]
|
||||
|
||||
def delete_range(self, rnd, take, from_ns, to_ns):
|
||||
"""Leave the frames from from_ns to to_ns (CLOCK_MONOTONIC, inclusive) out of the export."""
|
||||
return self._set_ranges(rnd, take, self.ranges(rnd, take) + [[int(from_ns), int(to_ns)]])
|
||||
|
||||
def restore_range(self, rnd, take, i):
|
||||
"""Take deleted range i (in ranges()' order) back."""
|
||||
ranges = self.ranges(rnd, take)
|
||||
if 0 <= i < len(ranges):
|
||||
del ranges[i]
|
||||
return self._set_ranges(rnd, take, ranges)
|
||||
|
||||
# --- deleting files
|
||||
def delete_take(self, rnd, take):
|
||||
path = self.take_dir(rnd, take)
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
meta_path = os.path.join(self.round_dir(rnd), "session.json")
|
||||
meta = read_json(meta_path)
|
||||
if take in (meta.get("takes") or []):
|
||||
meta["takes"] = [t for t in meta["takes"] if t != take]
|
||||
meta.setdefault("deleted_takes", []).append(take)
|
||||
write_json(meta_path, meta)
|
||||
|
||||
def delete_round(self, rnd):
|
||||
"""The round and its export, if any."""
|
||||
for path in (self.round_dir(rnd), self.export_dir(rnd)):
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
|
||||
def delete_export(self, rnd):
|
||||
path = self.export_dir(rnd)
|
||||
if os.path.isdir(path):
|
||||
shutil.rmtree(path)
|
||||
|
||||
# --- export
|
||||
def export(self, rnd, progress=None, cancel=None, keep_notes=False, low_priority=True, recipients=None):
|
||||
"""Write exports/<round>/ (the module's docstring). progress(fraction 0..1, text) is
|
||||
called as it goes; cancel() (or cancel.is_set()) returning true stops it, raising
|
||||
Cancelled. Nothing is left behind on failure or cancel. Returns the export's path.
|
||||
recipients: "age1..." keys, default recipients.txt's. low_priority: run this thread at
|
||||
nice 19 (Linux nice is per thread)."""
|
||||
recipients = list(recipients) if recipients else age.read_recipients(RECIPIENTS)
|
||||
age._crypto() # fails now, not half way, without the cryptography library
|
||||
cancelled = getattr(cancel, "is_set", cancel) or (lambda: False)
|
||||
report = progress or (lambda fraction, text: None)
|
||||
src = self.round_dir(rnd)
|
||||
if not os.path.isdir(src):
|
||||
raise RuntimeError(f"no round {rnd}")
|
||||
if low_priority:
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, threading.get_native_id(), 19)
|
||||
except OSError:
|
||||
pass
|
||||
final = self.export_dir(rnd)
|
||||
work = final + ".partial"
|
||||
if os.path.isdir(work):
|
||||
shutil.rmtree(work)
|
||||
os.makedirs(work)
|
||||
try:
|
||||
self._export(rnd, src, work, recipients, report, cancelled, keep_notes)
|
||||
if os.path.isdir(final):
|
||||
shutil.rmtree(final)
|
||||
os.replace(work, final)
|
||||
except BaseException:
|
||||
shutil.rmtree(work, ignore_errors=True)
|
||||
raise
|
||||
report(1.0, "Done")
|
||||
return final
|
||||
|
||||
def _export(self, rnd, src, work, recipients, report, cancelled, keep_notes):
|
||||
profile = self.profile()
|
||||
meta = read_json(os.path.join(src, "session.json"))
|
||||
meta.pop("uploads", None) # earlier uploads' records (hub.py) aren't part of the contribution
|
||||
takes = self.takes(rnd)
|
||||
total = sum(t["bytes"] for t in takes) or 1
|
||||
done = 0
|
||||
entries = []
|
||||
for t in takes:
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
tdir = self.take_dir(rnd, t["id"])
|
||||
out = os.path.join(work, "takes", t["id"])
|
||||
os.makedirs(out)
|
||||
take_meta = read_json(os.path.join(tdir, "take.json"))
|
||||
raw = raw_ranges(t["ranges"], take_meta.get("clock"))
|
||||
logs = {}
|
||||
for name in WHOLE_LOGS:
|
||||
if os.path.isfile(os.path.join(tdir, name)):
|
||||
shutil.copyfile(os.path.join(tdir, name), os.path.join(out, name))
|
||||
for name in CUT_LOGS:
|
||||
if os.path.isfile(os.path.join(tdir, name)):
|
||||
kept, cut = cut_jsonl(os.path.join(tdir, name), os.path.join(out, name), raw)
|
||||
logs[name] = {"rows": kept, "deleted": cut}
|
||||
|
||||
def step(nbytes, title=t["id"]):
|
||||
nonlocal done
|
||||
done += nbytes
|
||||
report(min(done / total, 0.99), f"Encrypting {title}")
|
||||
chunks = self._chunks(tdir, out, raw, recipients, step, cancelled)
|
||||
if take_meta:
|
||||
take_meta["export"] = {"deleted_raw_s": raw}
|
||||
write_json(os.path.join(out, "take.json"), take_meta)
|
||||
entries.append({"id": t["id"], "group": t["group"], "status": t["status"],
|
||||
"frames": chunks["frames"], "frames_deleted": chunks["deleted"],
|
||||
"duration_s": round(t["duration_s"], 2), "chunks": chunks["chunks"],
|
||||
"raw_bytes": chunks["raw_bytes"], "left_out": chunks["left_out"], "logs": logs})
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
if os.path.isfile(os.path.join(src, "fit.json")):
|
||||
shutil.copyfile(os.path.join(src, "fit.json"), os.path.join(work, "fit.json"))
|
||||
health = os.path.join(src, "health.json")
|
||||
if os.path.isfile(health):
|
||||
age.encrypt_file(health, os.path.join(work, "health.json.age"), recipients)
|
||||
shown = json.loads(json.dumps(profile))
|
||||
shown.pop("health", None) # only ever encrypted, in health.json.age
|
||||
if not keep_notes and isinstance(shown.get("optional"), dict):
|
||||
shown["optional"].pop("notes", None)
|
||||
manifest = {"schema": EXPORT_SCHEMA, "tool": tool_version(),
|
||||
"exported": datetime.datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"round_id": rnd, "bounty": meta.get("bounty", ""),
|
||||
"contributor": profile.get("contributor", ""),
|
||||
"consent_version": (profile.get("consent") or {}).get("version", ""),
|
||||
"encryption": {"format": "age-encryption.org/v1", "recipients": recipients,
|
||||
"files": ["takes/*/cam*/*.zst.age"] + (["health.json.age"] if os.path.isfile(health) else [])},
|
||||
"frame": {"width": eyerec.W, "height": eyerec.H, "format": "u8 grey",
|
||||
"cameras": {"0": "right eye", "1": "left eye"}},
|
||||
"profile": shown, "session": meta, "takes": entries}
|
||||
write_json(os.path.join(work, "manifest.json"), manifest)
|
||||
report(0.995, "Writing checksums")
|
||||
write_sums(work, cancelled)
|
||||
|
||||
@staticmethod
|
||||
def _chunks(tdir, out, ranges, recipients, step, cancelled):
|
||||
"""Each camera's chunks into out/camN/*.zst.age and the kept frames into out/index.tsv."""
|
||||
rows = {}
|
||||
for c, chunk, i, slot, t in eyerec.read_index(tdir):
|
||||
rows.setdefault((c, chunk), []).append((i, slot, t))
|
||||
on_disk = set()
|
||||
for c in eyerec.CAMERAS:
|
||||
try:
|
||||
names = os.listdir(os.path.join(tdir, "cam%d" % c))
|
||||
except OSError:
|
||||
continue
|
||||
on_disk.update((c, int(m.group(1))) for m in map(CHUNK_RE.match, names) if m)
|
||||
res = {"frames": [0] * len(eyerec.CAMERAS), "deleted": [0] * len(eyerec.CAMERAS), "chunks": 0,
|
||||
"raw_bytes": 0, "left_out": {"not_indexed": 0, "missing": 0, "bad": 0}}
|
||||
res["left_out"]["not_indexed"] = len(on_disk - set(rows))
|
||||
lines = [eyerec.INDEX_HEADER]
|
||||
for c, chunk in sorted(rows):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
frames = sorted(rows[(c, chunk)])
|
||||
if c not in eyerec.CAMERAS:
|
||||
continue
|
||||
if (c, chunk) not in on_disk:
|
||||
res["left_out"]["missing"] += 1
|
||||
continue
|
||||
path = os.path.join(tdir, "cam%d" % c, "%06d.zst" % chunk)
|
||||
with open(path, "rb") as f:
|
||||
packed = f.read()
|
||||
step(len(packed))
|
||||
try:
|
||||
data = eyerec.decompress(packed)
|
||||
except Exception:
|
||||
data = b""
|
||||
if len(data) != len(frames) * eyerec.FRAME or [i for i, _, _ in frames] != list(range(len(frames))):
|
||||
res["left_out"]["bad"] += 1
|
||||
continue
|
||||
kept = [(i, slot, t) for i, slot, t in frames if not in_ranges(t, ranges)]
|
||||
res["deleted"][c] += len(frames) - len(kept)
|
||||
if not kept:
|
||||
continue
|
||||
if len(kept) < len(frames):
|
||||
packed = eyerec.compress(b"".join(data[i * eyerec.FRAME:(i + 1) * eyerec.FRAME] for i, _, _ in kept))
|
||||
os.makedirs(os.path.join(out, "cam%d" % c), exist_ok=True)
|
||||
with open(os.path.join(out, "cam%d" % c, "%06d.zst.age" % chunk), "wb") as f:
|
||||
f.write(age.encrypt_bytes(packed, recipients))
|
||||
lines.extend("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, k, slot, t) for k, (_, slot, t) in enumerate(kept))
|
||||
res["frames"][c] += len(kept)
|
||||
res["chunks"] += 1
|
||||
res["raw_bytes"] += len(kept) * eyerec.FRAME
|
||||
with open(os.path.join(out, "index.tsv"), "w") as f:
|
||||
f.writelines(lines)
|
||||
return res
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the dataset's side
|
||||
|
||||
def read_identity(path):
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if line.strip().startswith("AGE-SECRET-KEY-1"):
|
||||
return line.strip()
|
||||
raise ValueError(f"{path} has no age identity")
|
||||
|
||||
|
||||
def unpack(export, identity_path, dest, progress=None):
|
||||
"""Check an export's SHA256SUMS, then copy it to dest with every .age file decrypted (its
|
||||
name without .age). The output is a take folder eyerec.py can read. Returns dest."""
|
||||
identity = read_identity(identity_path)
|
||||
report = progress or (lambda fraction, text: None)
|
||||
sums = {}
|
||||
with open(os.path.join(export, "SHA256SUMS")) as f:
|
||||
for line in f:
|
||||
digest, rel = line.rstrip("\n").split(" ", 1)
|
||||
if rel.startswith("/") or ".." in rel.split("/"):
|
||||
raise ValueError(f"SHA256SUMS names {rel!r}")
|
||||
sums[rel] = digest
|
||||
for k, (rel, digest) in enumerate(sorted(sums.items())):
|
||||
src = os.path.join(export, rel)
|
||||
if sha256_file(src) != digest:
|
||||
raise ValueError(f"{rel}: checksum doesn't match")
|
||||
out = os.path.join(dest, rel[:-4] if rel.endswith(".age") else rel)
|
||||
os.makedirs(os.path.dirname(out), exist_ok=True)
|
||||
if rel.endswith(".age"):
|
||||
age.decrypt_file(src, out, identity)
|
||||
else:
|
||||
shutil.copyfile(src, out)
|
||||
report((k + 1) / len(sums), rel)
|
||||
return dest
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="List, export and unpack eye recorder rounds.")
|
||||
ap.add_argument("--base", default=DEFAULT_BASE)
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("list")
|
||||
sub.add_parser("takes").add_argument("round")
|
||||
sub.add_parser("export").add_argument("round")
|
||||
up = sub.add_parser("unpack", help="check and decrypt an export (needs one of the dataset's keys)")
|
||||
up.add_argument("export")
|
||||
up.add_argument("--identity", required=True, help="an age identity file")
|
||||
up.add_argument("--out", required=True)
|
||||
a = ap.parse_args()
|
||||
store = Store(a.base)
|
||||
if a.cmd == "list":
|
||||
for r in store.rounds():
|
||||
print(f"{r['id']} {r['status'] or '-'} {r['takes']} takes {human_bytes(r['bytes'])}"
|
||||
+ (" dry run" if r["dry_run"] else "") + (" exported" if r["exported"] else ""))
|
||||
elif a.cmd == "takes":
|
||||
for t in store.takes(a.round):
|
||||
print(f"{t['id']} {t['status']} frames {t['frames']} ({t['deleted_frames']} deleted) "
|
||||
f"{t['duration_s']:.1f} s {human_bytes(t['bytes'])}")
|
||||
elif a.cmd == "unpack":
|
||||
print(unpack(a.export, a.identity, a.out))
|
||||
else:
|
||||
path = store.export(a.round, progress=lambda f, text: print(f"\r{f * 100:5.1f}% {text:40.40}", end="",
|
||||
flush=True))
|
||||
print(f"\n{path} {human_bytes(tree_bytes(path))}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,253 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's frame writer (gaze/rec/eyerec.py) against a made-up share, laid out as
|
||||
ft-eyegrab's (gaze/tracker/eyes_share.py). No eye images: each frame is one byte value
|
||||
repeated, from its camera and number, so the round trip can be checked.
|
||||
|
||||
python3 gaze/rec/tests/test_eyerec.py
|
||||
"""
|
||||
import mmap
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
import eyerec # noqa: E402
|
||||
|
||||
W, H, SLOTS = eyerec.W, eyerec.H, 8
|
||||
ESIZE = 64 + W * H
|
||||
|
||||
|
||||
def frame_bytes(cam, n):
|
||||
return bytes([(cam * 128 + n) % 256]) * (W * H)
|
||||
|
||||
|
||||
class FakeShare:
|
||||
"""Writes frames into a share file at fps per camera, on a thread, as ft-eyegrab --share does."""
|
||||
|
||||
def __init__(self, path, fps=90.0):
|
||||
self.path, self.fps = path, fps
|
||||
self.count = [0, 0]
|
||||
self._stop = threading.Event()
|
||||
self._make()
|
||||
self._thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _make(self):
|
||||
size = 64 + 2 * SLOTS * ESIZE
|
||||
tmp = self.path + ".new"
|
||||
fd = os.open(tmp, os.O_RDWR | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
os.ftruncate(fd, size)
|
||||
self.mm = mmap.mmap(fd, size)
|
||||
os.close(fd)
|
||||
struct.pack_into("<6I2QI", self.mm, 0, 0x31434546, 1, W, H, SLOTS, ESIZE, 0, 0, os.getpid())
|
||||
os.replace(tmp, self.path)
|
||||
self.count = [0, 0]
|
||||
|
||||
def replace(self):
|
||||
"""As ft-eyegrab restarting: a new file, the counts from 0."""
|
||||
self._make()
|
||||
|
||||
def start(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
self._thread.join(2)
|
||||
|
||||
def put(self, cam):
|
||||
n = self.count[cam]
|
||||
off = 64 + (cam * SLOTS + n % SLOTS) * ESIZE
|
||||
seq = struct.unpack_from("<Q", self.mm, off)[0]
|
||||
struct.pack_into("<Q", self.mm, off, seq + 1)
|
||||
self.mm[off + 64:off + 64 + W * H] = frame_bytes(cam, n)
|
||||
struct.pack_into("<dQII", self.mm, off + 8, time.clock_gettime(time.CLOCK_MONOTONIC_RAW), n, cam, n % 4 + 4 * cam)
|
||||
struct.pack_into("<Q", self.mm, off, seq + 2)
|
||||
self.count[cam] = n + 1
|
||||
struct.pack_into("<Q", self.mm, 24 + 8 * cam, n + 1)
|
||||
|
||||
def _run(self):
|
||||
period = 1.0 / self.fps
|
||||
nxt = time.monotonic()
|
||||
while not self._stop.is_set():
|
||||
for c in (0, 1):
|
||||
self.put(c)
|
||||
nxt += period
|
||||
time.sleep(max(0.0, nxt - time.monotonic()))
|
||||
|
||||
|
||||
class WriterTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-test-")
|
||||
self.share_path = os.path.join(self.tmp, "cams")
|
||||
self.want = os.path.join(self.tmp, "want")
|
||||
self.take = os.path.join(self.tmp, "take")
|
||||
self.lines = []
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def writer(self, **kw):
|
||||
kw.setdefault("min_free", 0)
|
||||
return eyerec.Writer(self.take, share_path=self.share_path, want_path=self.want, log=self.lines.append, **kw)
|
||||
|
||||
def wait_frames(self, w, n, timeout=10):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
with w._lock:
|
||||
if all(len(w._times[c]) and w._last[c] is not None for c in (0, 1)) and \
|
||||
sum(len(w._open[c]) for c in (0, 1)) + sum(w._frames.values()) >= n:
|
||||
return
|
||||
time.sleep(0.02)
|
||||
self.fail("not enough frames")
|
||||
|
||||
def test_round_trip_with_a_pause(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(chunk_frames=20).start()
|
||||
time.sleep(0.2) # frames before recording aren't kept
|
||||
w.record(True)
|
||||
time.sleep(0.6)
|
||||
w.record(False)
|
||||
time.sleep(0.3) # nor while paused
|
||||
w.record(True)
|
||||
time.sleep(0.4)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIsNone(stats["error"])
|
||||
self.assertEqual(stats["dropped"], [0, 0])
|
||||
self.assertGreater(min(stats["frames"]), 50)
|
||||
self.assertGreater(min(stats["fps"]), 60)
|
||||
rows = eyerec.read_index(self.take)
|
||||
self.assertEqual(len(rows), sum(stats["frames"]))
|
||||
got = list(eyerec.iter_frames(self.take))
|
||||
for c in (0, 1):
|
||||
ts = [t for cam, t, _s, _d in got if cam == c]
|
||||
gaps = [b - a for a, b in zip(ts, ts[1:])]
|
||||
self.assertGreater(max(gaps), 0.25, "the pause leaves a gap")
|
||||
self.assertEqual(sorted(ts), ts)
|
||||
# Each frame is what the share had: a byte value from its camera and number, so
|
||||
# consecutive frames of a camera go up by one (mod 256) outside the gap.
|
||||
vals = [d[0] for cam, _t, _s, d in got if cam == c]
|
||||
steps = {(b - a) % 256 for a, b, g in zip(vals, vals[1:], gaps) if g < 0.05}
|
||||
self.assertEqual(steps, {1})
|
||||
self.assertTrue(all(d == bytes([d[0]]) * len(d) for cam, _t, _s, d in got if cam == c))
|
||||
# The pause ended the chunks: none spans it.
|
||||
for c in (0, 1):
|
||||
chunks = {}
|
||||
for cam, chunk, i, _slot, t in rows:
|
||||
if cam == c:
|
||||
chunks.setdefault(chunk, []).append(t)
|
||||
self.assertTrue(all(max(ts) - min(ts) < 0.25 for ts in chunks.values()))
|
||||
self.assertTrue(os.path.exists(self.want), "ft-eyegrab's want file is touched")
|
||||
# It compresses: a repeated byte is tiny.
|
||||
self.assertLess(stats["bytes"], sum(stats["frames"]) * W * H / 100)
|
||||
|
||||
def test_a_slow_writer_drops_chunks_not_frames_in_the_share(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
real = eyerec.compress
|
||||
|
||||
def slow(b):
|
||||
time.sleep(0.3)
|
||||
return real(b)
|
||||
try:
|
||||
with mock.patch.object(eyerec, "compress", slow), mock.patch.object(eyerec, "MAX_QUEUED", 1):
|
||||
w = self.writer(chunk_frames=5).start()
|
||||
w.record(True)
|
||||
time.sleep(1.0)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertGreater(sum(stats["dropped"]), 0)
|
||||
self.assertTrue(any("behind" in line for line in self.lines))
|
||||
rows = eyerec.read_index(self.take)
|
||||
self.assertEqual(len(rows), sum(stats["frames"]))
|
||||
for c, chunk, _i, _s, _t in rows: # every chunk the index names is there
|
||||
self.assertTrue(os.path.exists(os.path.join(self.take, "cam%d" % c, "%06d.zst" % chunk)))
|
||||
|
||||
def test_follows_a_replaced_share(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(chunk_frames=10).start()
|
||||
w.record(True)
|
||||
time.sleep(0.4)
|
||||
share.replace() # ft-eyegrab restarted with SteamVR
|
||||
time.sleep(2.8) # the reader looks every 2 s
|
||||
before = w.stats()["frames"]
|
||||
time.sleep(0.5)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIsNone(stats["error"])
|
||||
self.assertTrue(any("replaced" in line for line in self.lines))
|
||||
self.assertGreater(min(stats["frames"]), min(before))
|
||||
|
||||
def test_no_share(self):
|
||||
with mock.patch.object(eyerec, "SHARE_WAIT", 0.2):
|
||||
w = self.writer().start()
|
||||
w.record(True)
|
||||
time.sleep(0.6)
|
||||
stats = w.stop()
|
||||
self.assertIn("frametop-eyegrab", stats["error"])
|
||||
self.assertFalse(w.recording)
|
||||
|
||||
def test_stops_before_the_disk_fills(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
w = self.writer(min_free=1e18).start()
|
||||
w.record(True)
|
||||
time.sleep(2.5)
|
||||
self.assertFalse(w.recording)
|
||||
stats = w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
self.assertIn("free on the disk", stats["error"])
|
||||
w.record(True)
|
||||
self.assertFalse(w.recording, "it doesn't start again after an error")
|
||||
|
||||
def test_numbering_goes_on_in_a_take_already_there(self):
|
||||
share = FakeShare(self.share_path).start()
|
||||
try:
|
||||
for _ in range(2):
|
||||
w = self.writer(chunk_frames=10).start()
|
||||
w.record(True)
|
||||
time.sleep(0.3)
|
||||
w.stop()
|
||||
finally:
|
||||
share.stop()
|
||||
rows = eyerec.read_index(self.take)
|
||||
chunks = [chunk for c, chunk, _i, _s, _t in rows if c == 0]
|
||||
self.assertEqual(len(set(chunks)), len(os.listdir(os.path.join(self.take, "cam0"))))
|
||||
with open(os.path.join(self.take, "index.tsv")) as f:
|
||||
self.assertEqual(sum(1 for line in f if line.startswith("#")), 1)
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("zstd") or os.path.exists("/usr/bin/zstd"), "no zstd command")
|
||||
class CommandFallbackTest(unittest.TestCase):
|
||||
"""The zstd command, as on the SteamOS host's Python 3.12."""
|
||||
|
||||
def test_round_trip(self):
|
||||
real_import = __import__
|
||||
|
||||
def no_modules(name, *a, **kw):
|
||||
if name in ("compression", "zstandard"):
|
||||
raise ImportError(name)
|
||||
return real_import(name, *a, **kw)
|
||||
with mock.patch("builtins.__import__", no_modules):
|
||||
c, d, name = eyerec._compressor()
|
||||
self.assertTrue(name.endswith("zstd"))
|
||||
data = frame_bytes(0, 3) + frame_bytes(1, 4)
|
||||
self.assertEqual(d(c(data)), data)
|
||||
if eyerec.decompress is not None: # the module's and the command's are the same format
|
||||
self.assertEqual(eyerec.decompress(c(data)), data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,153 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's upload (gaze/rec/hub.py) without the network: the dry run, the gate that
|
||||
keeps the real dataset closed until the texts are reviewed, upload records, and the pull request
|
||||
flow against a fake Hub. The round is test_takes.make_round's random frames.
|
||||
|
||||
python3 gaze/rec/tests/test_hub.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import hub # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from test_takes import HAVE, RID, make_round # noqa: E402
|
||||
|
||||
|
||||
class Pr:
|
||||
def __init__(self, num, status="draft"):
|
||||
self.num, self.status, self.is_pull_request = num, status, True
|
||||
self.url = f"https://huggingface.co/datasets/a/b/discussions/{num}"
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class HubTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-hub-")
|
||||
base = os.path.join(self.tmp, "contrib")
|
||||
make_round(base)
|
||||
self.contributor = str(uuid.uuid4())
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": self.contributor,
|
||||
"consent": {"version": "test-1", "adult": True, "region_ok": True, "accepted": "2026-10-14"}}, f)
|
||||
self.store = takes.Store(base)
|
||||
meta = hub.meta_path(self.store, RID)
|
||||
with open(meta) as f:
|
||||
m = json.load(f)
|
||||
m.pop("uploads") # the fixture's, for the export's test
|
||||
with open(meta, "w") as f:
|
||||
json.dump(m, f)
|
||||
self.store.export(RID, low_priority=False) # to recipients.txt's keys, as validate wants
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_dry_run(self):
|
||||
logged = []
|
||||
with mock.patch.dict(os.environ, {hub.DATASET_ENV: "someone/test-eyes"}):
|
||||
result = hub.upload(self.store, RID, dry_run=True, log=logged.append)
|
||||
self.assertTrue(result["dry_run"])
|
||||
self.assertEqual(result["repo"], "someone/test-eyes")
|
||||
self.assertEqual(result["path_in_repo"], f"contributions/{self.contributor}/{RID}")
|
||||
self.assertIn("- Bounty: edges", result["commit_description"])
|
||||
self.assertIn("- Health answers: included, encrypted", result["commit_description"])
|
||||
self.assertTrue(any("health.json.age" in line for line in logged))
|
||||
self.assertEqual(hub.rhub.uploads(hub.meta_path(self.store, RID)), [])
|
||||
|
||||
def test_gate(self):
|
||||
self.assertTrue(hub.texts_ready(), "the texts are reviewed (2026-10-16)")
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "", hub.DATASET_ENV: ""}):
|
||||
self.assertTrue(hub.upload_allowed(hub.dataset_id()))
|
||||
# While a text is a draft: never the real dataset, a rehearsal only with ALLOW_ENV.
|
||||
with mock.patch.object(hub, "texts_ready", return_value=False):
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "1", hub.DATASET_ENV: ""}):
|
||||
self.assertFalse(hub.upload_allowed(hub.dataset_id()), "never the real dataset before the review")
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID)
|
||||
self.assertEqual(cm.exception.kind, "closed")
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "1", hub.DATASET_ENV: "a/test"}):
|
||||
self.assertTrue(hub.upload_allowed(hub.dataset_id()))
|
||||
with mock.patch.dict(os.environ, {hub.ALLOW_ENV: "", hub.DATASET_ENV: "a/test"}):
|
||||
self.assertFalse(hub.upload_allowed(hub.dataset_id()))
|
||||
|
||||
def test_draft_consent_blocks(self):
|
||||
# An export made under a draft consent text never goes to the real dataset.
|
||||
prof = os.path.join(self.store.base, "profile.json")
|
||||
with open(prof) as f:
|
||||
p = json.load(f)
|
||||
p["consent"]["version"] = "2026-10-14-draft"
|
||||
with open(prof, "w") as f:
|
||||
json.dump(p, f)
|
||||
shutil.rmtree(self.store.export_dir(RID))
|
||||
self.store.export(RID, low_priority=False)
|
||||
with mock.patch.dict(os.environ, {hub.DATASET_ENV: ""}):
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID, dry_run=True)
|
||||
self.assertEqual(cm.exception.kind, "draft-consent")
|
||||
|
||||
def test_invalid_blocks(self):
|
||||
with open(os.path.join(self.store.export_dir(RID), "health.json"), "w") as f:
|
||||
f.write("{}")
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID, dry_run=True)
|
||||
self.assertEqual(cm.exception.kind, "invalid")
|
||||
|
||||
def test_pull_request_first(self):
|
||||
calls = []
|
||||
|
||||
class Api:
|
||||
fail = True
|
||||
|
||||
def whoami(self):
|
||||
return {"name": "someone", "auth": {"accessToken": {"role": "write"}}}
|
||||
|
||||
def auth_check(self, repo, repo_type=None):
|
||||
pass
|
||||
|
||||
def create_pull_request(self, repo, title, description=None, repo_type=None):
|
||||
calls.append("create")
|
||||
return Pr(3)
|
||||
|
||||
def get_discussion_details(self, repo, num, repo_type=None):
|
||||
return Pr(num)
|
||||
|
||||
def upload_folder(self, **kw):
|
||||
calls.append(("upload", kw["revision"], kw["path_in_repo"]))
|
||||
if Api.fail:
|
||||
raise ConnectionResetError("reset")
|
||||
|
||||
def change_discussion_status(self, repo, num, status, repo_type=None):
|
||||
calls.append(("status", num, status))
|
||||
|
||||
env = {hub.DATASET_ENV: "a/b", hub.ALLOW_ENV: "1"}
|
||||
meta = hub.meta_path(self.store, RID)
|
||||
before = os.stat(meta).st_mtime_ns
|
||||
with mock.patch.dict(os.environ, env), mock.patch.object(hub, "_hf", return_value=mock.Mock(HfApi=Api)):
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID)
|
||||
self.assertEqual(cm.exception.kind, "network")
|
||||
[rec] = hub.rhub.uploads(meta)
|
||||
self.assertEqual((rec["status"], rec["pr_num"]), ("started", 3))
|
||||
Api.fail = False
|
||||
result = hub.upload(self.store, RID)
|
||||
with self.assertRaises(hub.HubError) as cm:
|
||||
hub.upload(self.store, RID, dry_run=True)
|
||||
self.assertEqual(cm.exception.kind, "duplicate")
|
||||
where = f"contributions/{self.contributor}/{RID}"
|
||||
self.assertEqual(calls, ["create", ("upload", "refs/pr/3", where), ("upload", "refs/pr/3", where),
|
||||
("status", 3, "open")])
|
||||
self.assertEqual(result["pr_num"], 3)
|
||||
self.assertEqual(os.stat(meta).st_mtime_ns, before, "the record doesn't make the export look stale")
|
||||
self.assertFalse(self.store.rounds()[0]["export_stale"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,173 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The gaze panel's rec mode (gaze/panel/ft-gazepanel.cpp, "show rec") without SteamVR (--no-vr,
|
||||
a made-up 90 Hz vsync): its commands, and targets.jsonl, the ground truth of where the dot was
|
||||
when. Needs the built panel (gaze/build.sh); skipped without it.
|
||||
|
||||
python3 gaze/rec/tests/test_panel.py
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PANEL = os.path.join(HERE, "..", "..", "build", "ft-gazepanel")
|
||||
IN_BOX = os.path.expanduser("~/.local/bin/distrobox")
|
||||
|
||||
|
||||
def command():
|
||||
"""The panel's argv: in the dev container on the Frame's host, where it was built."""
|
||||
if os.path.exists("/run/.containerenv") or not os.path.exists(IN_BOX):
|
||||
return [PANEL]
|
||||
return [IN_BOX, "enter", "dev", "--", PANEL]
|
||||
|
||||
|
||||
@unittest.skipUnless(os.path.exists(PANEL), "the panel isn't built (gaze/build.sh)")
|
||||
class RecPanelTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="gazepanel-test-")
|
||||
self.name = f"ft_gazepanel_test_{os.getpid()}"
|
||||
self.proc = subprocess.Popen(command() + ["--no-vr", "--watch-stdin", "--socket", self.name, "--dump",
|
||||
os.path.join(self.tmp, "pics")],
|
||||
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
self.sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.sock.bind("")
|
||||
self.sock.settimeout(1.0)
|
||||
end = time.monotonic() + 20
|
||||
while True:
|
||||
try:
|
||||
if self.ask("ping").startswith("ok"):
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() > end:
|
||||
self.fail("the panel didn't start")
|
||||
time.sleep(0.1)
|
||||
|
||||
def tearDown(self):
|
||||
self.proc.stdin.close()
|
||||
try:
|
||||
self.proc.wait(5)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
self.proc.stderr.close()
|
||||
self.sock.close()
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def ask(self, cmd):
|
||||
self.sock.sendto(cmd.encode(), "\0" + self.name)
|
||||
return self.sock.recv(4096).decode()
|
||||
|
||||
def rows(self):
|
||||
with open(os.path.join(self.tmp, "targets.jsonl")) as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
def test_rec_lines_and_errors(self):
|
||||
for cmd in ("show rec", "bg 0.3", "title Nine dots", "step Round 1 of 5", "rec on", "text Look at each dot|as it comes",
|
||||
"big 3", "action Ready? Press the side button", "note Your left eye is hard to see",
|
||||
"keys Space: go · P: pause", "countdown 0.4", "paused on", "paused off", "countdown off"):
|
||||
self.assertEqual(self.ask(cmd), "ok", cmd)
|
||||
self.assertTrue(self.ask("path spiral 0 0 1 2").startswith("error"))
|
||||
self.assertTrue(self.ask("path circle 0 0 10 0").startswith("error"), "seconds must be positive")
|
||||
self.assertTrue(self.ask("log start /nonexistent/dir").startswith("error"))
|
||||
self.assertEqual(self.ask("ipd"), "ok 0.06300")
|
||||
time.sleep(0.2)
|
||||
self.assertTrue(os.listdir(os.path.join(self.tmp, "pics")), "pictures were drawn")
|
||||
self.ask("show quick")
|
||||
self.assertTrue(self.ask("path circle 0 0 10 2").startswith("error"), "only in rec")
|
||||
|
||||
def test_fixed_dot_logs_its_changes(self):
|
||||
self.ask("show rec")
|
||||
self.assertEqual(self.ask(f"log start {self.tmp}"), "ok")
|
||||
self.ask("dot 10 -5 look")
|
||||
time.sleep(0.15)
|
||||
self.ask("dot 10 -5 capture 0.5")
|
||||
time.sleep(0.15)
|
||||
self.ask("bg 0.8")
|
||||
time.sleep(0.15)
|
||||
self.ask("dot 0 0 off")
|
||||
time.sleep(0.15)
|
||||
reply = self.ask("log stop").split()
|
||||
rows = self.rows()
|
||||
self.assertEqual(int(reply[1]), len(rows))
|
||||
on = [r for r in rows if r["dot"] == "fixed"]
|
||||
self.assertEqual([(r["yaw"], r["pitch"], r["state"]) for r in on],
|
||||
[(10, -5, "look"), (10, -5, "capture"), (10, -5, "capture")])
|
||||
self.assertEqual(on[-1]["bg"], 0.8)
|
||||
self.assertEqual(rows[-1]["dot"], "off")
|
||||
for r in rows:
|
||||
self.assertGreater(r["t"], r["set"], "the photons come after it's set")
|
||||
self.assertLess(r["t"] - r["set"], 0.03)
|
||||
# The RAW clock, as the eye frames' (this process's CLOCK_MONOTONIC_RAW).
|
||||
self.assertLess(abs(rows[-1]["set"] - time.clock_gettime(time.CLOCK_MONOTONIC_RAW)), 5)
|
||||
|
||||
def test_path_every_frame(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
self.assertEqual(self.ask("path circle 2 1 10 1.0"), "ok")
|
||||
time.sleep(1.2)
|
||||
self.ask("dot 0 0 off")
|
||||
self.ask("log stop")
|
||||
rows = [r for r in self.rows() if r["dot"] == "path"]
|
||||
ts = [r["t"] for r in rows]
|
||||
self.assertGreater(len(rows), 80, "about one a frame")
|
||||
gaps = [b - a for a, b in zip(ts, ts[1:])]
|
||||
self.assertTrue(all(g > 0 for g in gaps))
|
||||
self.assertAlmostEqual(sorted(gaps)[len(gaps) // 2], 1 / 90, delta=0.002)
|
||||
self.assertEqual(len({r["frame"] for r in rows}), len(rows), "one row a frame")
|
||||
for r in rows:
|
||||
self.assertAlmostEqual(math.hypot(r["yaw"] - 2, r["pitch"] - 1), 10, delta=1e-3)
|
||||
first = rows[0]
|
||||
self.assertAlmostEqual((first["yaw"], first["pitch"])[1], 11, delta=0.5, msg="it starts at the top")
|
||||
# A lap a second: back near the top a second later, having gone left (yaw up) first.
|
||||
later = min(rows, key=lambda r: abs(r["t"] - first["t"] - 1.0))
|
||||
self.assertAlmostEqual(later["pitch"], 11, delta=0.5)
|
||||
self.assertGreater(rows[5]["yaw"], 2)
|
||||
|
||||
def test_line_and_eight(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
self.ask("path line -10 0 10 0 0.5")
|
||||
time.sleep(0.8)
|
||||
self.ask("path eight 0 0 15 8 1")
|
||||
time.sleep(0.6)
|
||||
self.ask("log stop")
|
||||
rows = self.rows()
|
||||
line = [r for r in rows if r["dot"] == "path" and abs(r["pitch"]) < 1e-6 and abs(r["yaw"]) <= 10 + 1e-6]
|
||||
self.assertGreater(len(line), 60)
|
||||
self.assertGreater(max(r["yaw"] for r in line), 9)
|
||||
self.assertLess(min(r["yaw"] for r in line), -9.5)
|
||||
eight = [r for r in rows if r["t"] > line[-1]["t"]]
|
||||
self.assertTrue(all(abs(r["yaw"]) <= 15 + 1e-6 and abs(r["pitch"]) <= 8 + 1e-6 for r in eight if r["dot"] == "path"))
|
||||
|
||||
def test_world_dot_and_distance(self):
|
||||
self.ask("show rec")
|
||||
self.ask(f"log start {self.tmp}")
|
||||
reply = self.ask("world 0.5 0 -1")
|
||||
self.assertTrue(reply.startswith("ok"), reply)
|
||||
self.assertEqual([round(float(v), 3) for v in reply.split()[1:]], [0.5, 0.0, -1.0])
|
||||
time.sleep(0.2)
|
||||
again = self.ask("world 0.5 0 -1 capture 0.5")
|
||||
self.assertEqual(again, reply, "the same point again: only the state changes")
|
||||
time.sleep(0.1)
|
||||
self.ask("dot 0 0 look")
|
||||
self.assertEqual(self.ask("distance 0.5"), "ok")
|
||||
time.sleep(0.1)
|
||||
self.ask("log stop")
|
||||
rows = self.rows()
|
||||
world = [r for r in rows if r["dot"] == "world"]
|
||||
self.assertGreater(len(world), 10)
|
||||
# --no-vr's head is at the origin looking down -z: half a metre right of a metre ahead.
|
||||
self.assertAlmostEqual(world[0]["yaw"], -math.degrees(math.atan2(0.5, 1)), delta=0.01)
|
||||
self.assertAlmostEqual(world[0]["dist"], math.hypot(0.5, 1), delta=0.001)
|
||||
self.assertEqual(rows[-1]["dist"], 0.5)
|
||||
self.assertEqual(rows[-1]["dot"], "fixed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,72 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""main.qml against ft_eyerec.Backend: every backend.name(...) the window calls is a slot, and
|
||||
every backend.name it reads is a property or a slot. A method that lost its @Slot shows up in
|
||||
QML only as "is not a function" when its button is pressed (the hand recorder's Export, 2026-10-03).
|
||||
Needs PySide6 (the dev container); skipped without it. FT_EYEREC_QML=PATH checks another
|
||||
window against the same backend (the standalone Eye Recorder's, in its CI).
|
||||
|
||||
python3 gaze/rec/tests/test_qml_backend.py
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REC = os.path.dirname(HERE)
|
||||
sys.path.insert(0, REC)
|
||||
QML = os.environ.get("FT_EYEREC_QML") or os.path.join(REC, "main.qml")
|
||||
|
||||
|
||||
class QmlBackendTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
try:
|
||||
import ft_eyerec
|
||||
except ImportError as e:
|
||||
self.skipTest(f"no PySide6: {e}")
|
||||
meta = self.meta = ft_eyerec.Backend.staticMetaObject
|
||||
self.slots = {bytes(meta.method(i).name()).decode() for i in range(meta.methodCount())}
|
||||
self.props = {meta.property(i).name() for i in range(meta.propertyCount())}
|
||||
with open(QML) as f:
|
||||
self.qml = f.read()
|
||||
|
||||
def test_calls_are_slots(self):
|
||||
called = set(re.findall(r"\bbackend\.(\w+)\s*\(", self.qml))
|
||||
self.assertTrue(called)
|
||||
self.assertEqual(sorted(called - self.slots), [], "called from main.qml but not a slot")
|
||||
|
||||
def test_call_arity(self):
|
||||
"""Each backend.name(a, b) call passes as many arguments as some slot of that name takes
|
||||
(a decorator left at the old count fails only when the button is pressed)."""
|
||||
arity = {}
|
||||
meta = self.meta
|
||||
for i in range(meta.methodCount()):
|
||||
mm = meta.method(i)
|
||||
arity.setdefault(bytes(mm.name()).decode(), set()).add(mm.parameterCount())
|
||||
bad = []
|
||||
for m in re.finditer(r"\bbackend\.(\w+)\s*\(", self.qml):
|
||||
depth, args, k, seen = 1, 0, m.end(), False
|
||||
while depth and k < len(self.qml):
|
||||
c = self.qml[k]
|
||||
if c in "([{":
|
||||
depth += 1
|
||||
elif c in ")]}":
|
||||
depth -= 1
|
||||
elif c == "," and depth == 1:
|
||||
args += 1
|
||||
elif not c.isspace() and depth >= 1:
|
||||
seen = True
|
||||
k += 1
|
||||
n = args + 1 if seen else 0
|
||||
if m.group(1) in arity and n not in arity[m.group(1)]:
|
||||
bad.append("%s: %d arguments, slots take %s" % (m.group(1), n, sorted(arity[m.group(1)])))
|
||||
self.assertEqual(bad, [])
|
||||
|
||||
def test_reads_exist(self):
|
||||
read = set(re.findall(r"\bbackend\.(\w+)\b(?!\s*\()", self.qml))
|
||||
self.assertTrue(read)
|
||||
self.assertEqual(sorted(read - self.props - self.slots), [], "read in main.qml but not on Backend")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,343 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's session runner (gaze/rec/session.py) without the headset: the round
|
||||
scripts' checks, the plans, the control sockets' answers, and whole rounds in a dry run.
|
||||
|
||||
python3 gaze/rec/tests/test_session.py
|
||||
"""
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import session # noqa: E402
|
||||
|
||||
|
||||
def write_round(tmp, steps, **extra):
|
||||
d = dict({"schema": 1, "id": "t", "title": "Test", "steps": steps}, **extra)
|
||||
path = os.path.join(tmp, "t.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(d, f)
|
||||
return path
|
||||
|
||||
|
||||
class RoundTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-round-")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def bad(self, steps, want):
|
||||
with self.assertRaises(ValueError) as e:
|
||||
session.load_round(write_round(self.tmp, steps))
|
||||
self.assertIn(want, str(e.exception))
|
||||
|
||||
def test_shipped_rounds_load(self):
|
||||
paths = sorted(glob.glob(os.path.join(session.ROUNDS_DIR, "*.json")))
|
||||
self.assertEqual(len(paths), 10)
|
||||
for p in paths:
|
||||
r = session.load_round(p)
|
||||
self.assertEqual(r["id"], os.path.basename(p)[:-5])
|
||||
self.assertTrue(r["steps"][0]["ready"], p)
|
||||
|
||||
def test_panel_reach(self):
|
||||
self.assertFalse(session.off_panel(0, 0))
|
||||
self.assertFalse(session.off_panel(30, 20))
|
||||
self.assertTrue(session.off_panel(38, 0))
|
||||
self.assertTrue(session.off_panel(0, 26))
|
||||
self.assertTrue(session.off_panel(30, 23), "corners reach less far up")
|
||||
self.bad([{"id": "a", "kind": "dots", "dots": [[40, 0]]}], "off the panel")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "circle 0 0 26 4", "seconds": 4}], "off the panel")
|
||||
self.bad([{"id": "a", "kind": "jumps", "count": 3, "range": [40, 10], "each": [0.5, 1]}], "off the panel")
|
||||
|
||||
def test_bad_steps(self):
|
||||
self.bad([{"id": "a", "kind": "dance"}], "unknown kind")
|
||||
self.bad([{"id": "A b", "kind": "dots", "dots": [[0, 0]]}], "bad id")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "spiral 0 0 1 2", "seconds": 2}], "bad path")
|
||||
self.bad([{"id": "a", "kind": "path", "path": "circle 0 0 10 0", "seconds": 2}], "positive")
|
||||
self.bad([{"id": "a", "kind": "prompt"}], "seconds")
|
||||
self.bad([{"id": "a", "kind": "world", "point": [0, 0, 1], "seconds": 3}], "ahead")
|
||||
self.bad([{"id": "a", "kind": "dots", "dots": [[0, 0]]}, {"id": "a", "kind": "dots", "dots": [[0, 0]]}], "repeat")
|
||||
self.bad([{"id": "a", "kind": "dots", "group": "g", "dots": [[0, 0]]}, {"id": "b", "kind": "dots", "dots": [[0, 0]]},
|
||||
{"id": "c", "kind": "dots", "group": "g", "dots": [[0, 0]]}], "comes back")
|
||||
self.bad([{"id": "a", "kind": "dots", "group": "dots", "dots": [[0, 0]]}], "opener")
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Blink", 1, "honk"]]}], "sound: blink")
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Blink", 1, "blink", 2]]}], "cues want")
|
||||
# The eyes-shut rules (rounds/README.md)
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Blink", 1]]}], 'needs the sound "blink"')
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Close both eyes", 2], ["Open", 2]]}], 'needs the sound "close"')
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Shut your eyes", 2, "close"]]}], 'comes a cue with the sound "open"')
|
||||
self.bad([{"id": "a", "kind": "prompt", "cues": [["Close your eyes", 2, "close"], ["Open", 2]]}], '"open"')
|
||||
for word in ("Close your left eye", "Half closed", "Squint", "Open"):
|
||||
session.load_round(write_round(self.tmp, [{"id": "a", "kind": "prompt", "cues": [[word, 1]]}]))
|
||||
|
||||
def test_sound_cues(self):
|
||||
r = session.load_round(write_round(self.tmp, [{"id": "a", "kind": "prompt",
|
||||
"cues": [["Look", 1], ["Blink", 1.5, "blink"]]}]))
|
||||
self.assertEqual(r["steps"][0]["cues"], [["Look", 1.0], ["Blink", 1.5, "blink"]])
|
||||
self.assertEqual(session.step_seconds(r["steps"][0]), 2.5)
|
||||
|
||||
def test_grid_and_defaults(self):
|
||||
r = session.load_round(write_round(self.tmp, [{"id": "a", "kind": "dots", "grid": [3, 2, 10, 5], "ready": False},
|
||||
{"id": "b", "kind": "reseat", "text": "off and on"}]))
|
||||
a = r["steps"][0]
|
||||
self.assertEqual(a["dots"], [[10, 5], [0, 5], [-10, 5], [10, -5], [0, -5], [-10, -5]])
|
||||
self.assertTrue(a["ready"], "the first step always waits")
|
||||
self.assertEqual((a["bg"], a["distance"], a["group"]), (session.MID_BG, session.PANEL_DISTANCE, "a"))
|
||||
self.assertFalse(r["shift_check"], "a round that reseats on purpose has no shift check")
|
||||
|
||||
def test_plan(self):
|
||||
rnd = session.load_round(os.path.join(session.ROUNDS_DIR, "jumps.json"))
|
||||
a = session.build_plan(rnd, 7)
|
||||
self.assertEqual([s["group"] for s in a[:2]], ["dots", "jumps"])
|
||||
nine = a[0]["dots"]
|
||||
self.assertEqual((nine[0], nine[-1], len(nine)), ([0.0, 0.0], [0.0, 0.0], 10))
|
||||
self.assertEqual(sorted(map(tuple, nine[1:-1])), sorted(map(tuple, (d for d in session.grid(3, 3, 20, 20) if d != [0, 0]))))
|
||||
jumps = a[1]
|
||||
self.assertEqual(len(jumps["dots"]), 40)
|
||||
self.assertTrue(all(0.5 <= t <= 1.5 for t in jumps["times"]))
|
||||
self.assertTrue(all(not session.off_panel(*d) for d in jumps["dots"]))
|
||||
self.assertEqual(a, session.build_plan(rnd, 7), "the same seed, the same round")
|
||||
self.assertNotEqual(a[0]["dots"], session.build_plan(rnd, 8)[0]["dots"])
|
||||
self.assertGreater(session.plan_seconds(a), 60)
|
||||
|
||||
|
||||
class SocketTest(unittest.TestCase):
|
||||
"""The panel and ft-eyes answer every command in order: an answer waited for is the one
|
||||
after the answers of commands sent before it."""
|
||||
|
||||
def setUp(self):
|
||||
self.name = "eyerec_socket_test_%d_%s" % (os.getpid(), self.id().rsplit(".", 1)[-1])
|
||||
self.server = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.server.bind("\0" + self.name)
|
||||
self.server.settimeout(0.2)
|
||||
self.delay = 0.02
|
||||
self.drop = set()
|
||||
self.running = True
|
||||
threading.Thread(target=self.serve, daemon=True).start()
|
||||
|
||||
def tearDown(self):
|
||||
self.running = False
|
||||
self.server.close()
|
||||
|
||||
def serve(self):
|
||||
while self.running:
|
||||
try:
|
||||
data, addr = self.server.recvfrom(4096)
|
||||
except OSError:
|
||||
continue
|
||||
text = data.decode()
|
||||
time.sleep(self.delay)
|
||||
if text in self.drop:
|
||||
continue
|
||||
try:
|
||||
self.server.sendto(("ok " + text).encode(), addr)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_answers_in_order(self):
|
||||
s = session.Socket(self.name)
|
||||
for n in range(5):
|
||||
s.cmd("dot %d" % n)
|
||||
self.assertEqual(s.cmd("ipd", reply=True), "ok ipd")
|
||||
self.assertEqual(s.pending, 0)
|
||||
s.cmd("bg 1")
|
||||
self.assertEqual(s.cmd("log stop", reply=True), "ok log stop")
|
||||
s.close()
|
||||
|
||||
def test_lost_answer(self):
|
||||
s = session.Socket(self.name)
|
||||
self.drop.add("fit")
|
||||
self.assertIsNone(s.cmd("fit", reply=True, timeout=0.2))
|
||||
self.assertEqual(s.cmd("ipd", reply=True), "ok ipd", "counting starts afresh")
|
||||
s.close()
|
||||
|
||||
def test_errors_logged(self):
|
||||
logs = []
|
||||
s = session.Socket(self.name, log=logs.append)
|
||||
s.cmd("x")
|
||||
time.sleep(0.1)
|
||||
s._got("error bad")
|
||||
self.assertEqual(logs, ["%s: error bad" % self.name])
|
||||
s.close()
|
||||
|
||||
|
||||
class DryRunTest(unittest.TestCase):
|
||||
"""Whole rounds with no processes: the files, the fitment, the controls."""
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-dry-")
|
||||
self.out = []
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def run_round(self, script, steer=None, profile=None, **kw):
|
||||
path = os.path.join(self.tmp, "profile.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(profile or {}, f)
|
||||
states = []
|
||||
kw.setdefault("fit_coach", True) # the coach's own tests; test_fitment_coach_off has it off
|
||||
s = session.Session(self.tmp, script, profile=profile, dry_run=True, speed=kw.pop("speed", 40),
|
||||
next_after=kw.pop("next_after", 0.25), profile_path=path, button=False,
|
||||
on_status=lambda st: states.append(dict(st)), **kw)
|
||||
s.print = self.out.append
|
||||
s.start()
|
||||
if steer:
|
||||
steer(s, states)
|
||||
s.join(60)
|
||||
self.assertFalse(s._thread.is_alive())
|
||||
with open(path) as f:
|
||||
return s, states, json.load(f)
|
||||
|
||||
def files(self, s):
|
||||
with open(os.path.join(s.session_dir, "session.json")) as f:
|
||||
sj = json.load(f)
|
||||
with open(os.path.join(s.session_dir, "fit.json")) as f:
|
||||
fit = json.load(f)
|
||||
return sj, fit
|
||||
|
||||
def prompts(self, s, take):
|
||||
with open(os.path.join(s.session_dir, "takes", take, "prompts.jsonl")) as f:
|
||||
return [json.loads(line) for line in f]
|
||||
|
||||
def test_round_and_profile(self):
|
||||
s, states, profile = self.run_round("depth")
|
||||
self.assertEqual(s.state, "done")
|
||||
sj, fit = self.files(s)
|
||||
# No IPD step: the setting is read and shown at the fitment.
|
||||
self.assertEqual(sj["takes"], ["01-dots", "02-near", "03-mid", "04-far", "05-near-far"])
|
||||
self.assertEqual((sj["status"], sj["fit"], sj["ipd_start"]), ("done", "good", 0.063))
|
||||
self.assertNotIn("ipd_sweep", sj)
|
||||
self.assertIn(session.IPD_TEXT.format(mm=63.0), "\n".join(self.out))
|
||||
self.assertEqual(fit["fitment"]["result"], "good")
|
||||
self.assertIsNotNone(fit["baseline"])
|
||||
self.assertEqual(profile["bounties"]["depth"]["rounds"], 1)
|
||||
self.assertEqual(profile["eyes_ipd"], 0.063)
|
||||
for take in sj["takes"]:
|
||||
with open(os.path.join(s.session_dir, "takes", take, "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "complete", take)
|
||||
events = [e["event"] for e in self.prompts(s, "02-near")]
|
||||
self.assertEqual(events[:4], ["take", "ready", "rec", "step"])
|
||||
self.assertEqual(events[-3:], ["wait", "rec", "end"])
|
||||
self.assertIn("ft_eyerecpanel: distance 0.500", self.out)
|
||||
self.assertIn("ft_eyerecpanel: distance 5.000", self.out)
|
||||
s2, _, profile = self.run_round("depth", profile=profile)
|
||||
self.assertEqual(self.files(s2)[0]["takes"][0], "01-dots")
|
||||
self.assertEqual(profile["bounties"]["depth"]["rounds"], 2)
|
||||
|
||||
def test_blink_beeps(self):
|
||||
# The blinks round's beeps: each in prompts.jsonl's cue line, played as the cue starts.
|
||||
s, _, _ = self.run_round("blinks", profile={"eyes_ipd": 0.063})
|
||||
self.assertEqual(s.state, "done")
|
||||
played = [line[len("sound: "):] for line in self.out if line.startswith("sound: ")]
|
||||
self.assertEqual(played, ["blink"] * 8 + ["close", "open", "close", "open"])
|
||||
sj, _ = self.files(s)
|
||||
[take] = [t for t in sj["takes"] if t.endswith("blink-cue")]
|
||||
cues = [e for e in self.prompts(s, take) if e["event"] == "cue"]
|
||||
self.assertEqual([(e["word"], e.get("sound")) for e in cues], [("Blink", "blink")] * 8)
|
||||
[take] = [t for t in sj["takes"] if t.endswith("closed")]
|
||||
cues = [e.get("sound") for e in self.prompts(s, take) if e["event"] == "cue"]
|
||||
self.assertEqual(cues, [None, "close", "open", "close", "open"])
|
||||
|
||||
def test_fitment_warns_then_skips(self):
|
||||
# Next before the fit has been good for FIT_GOOD_S: a warning, then the round goes on.
|
||||
s, states, _ = self.run_round("reading", next_after=0.01, speed=20)
|
||||
sj, fit = self.files(s)
|
||||
self.assertEqual((sj["fit"], fit["fitment"]["result"]), ("skipped", "skipped"))
|
||||
self.assertTrue(any(st["note"] == session.FIT_SKIP_NOTE for st in states))
|
||||
self.assertIsNone(fit["baseline"])
|
||||
|
||||
def test_fitment_coach_off(self):
|
||||
# The default for now: no coaching or warning, the first Next goes on, and the coach's
|
||||
# readings are still kept; no baseline, so no slip checks.
|
||||
self.assertFalse(session.FIT_COACH)
|
||||
s, states, _ = self.run_round("reading", next_after=0.6, speed=20, fit_coach=False)
|
||||
sj, fit = self.files(s)
|
||||
self.assertEqual(s.state, "done")
|
||||
self.assertEqual((sj["fit"], fit["fitment"]["result"]), ("off", "off"))
|
||||
self.assertTrue(fit["readings"])
|
||||
self.assertIsNone(fit["baseline"])
|
||||
self.assertFalse(fit["slips"])
|
||||
self.assertFalse(any(st["note"] for st in states))
|
||||
self.assertFalse(any(st["fit_words"] or st["fit_card"] for st in states))
|
||||
self.assertTrue(all(st["fit_coach"] is False for st in states))
|
||||
|
||||
def test_skip_redo_stop(self):
|
||||
def steer(s, states):
|
||||
def wait_for(cond):
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and not cond():
|
||||
time.sleep(0.01)
|
||||
# In the jumps take: redo the step while it runs, then skip the rest of the take.
|
||||
# (Its ready first: the step's name comes one emit before the dots take's "running" goes.)
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "ready")
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "running")
|
||||
s.redo()
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "ready")
|
||||
wait_for(lambda: s._status["step"] == "jumps" and s._status["state"] == "running")
|
||||
s.skip()
|
||||
wait_for(lambda: s._status["step"] == "circle-slow" and s._status["state"] == "running")
|
||||
s.stop(wait=0)
|
||||
s, states, profile = self.run_round("jumps", steer=steer, speed=10, profile={"eyes_ipd": 0.063})
|
||||
self.assertEqual(s.state, "stopped")
|
||||
sj, _ = self.files(s)
|
||||
self.assertEqual(sj["takes"], ["01-dots", "02-jumps", "03-pursuit"])
|
||||
with open(os.path.join(s.session_dir, "takes", "02-jumps", "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "skipped")
|
||||
with open(os.path.join(s.session_dir, "takes", "03-pursuit", "take.json")) as f:
|
||||
self.assertEqual(json.load(f)["status"], "stopped")
|
||||
redo = [e for e in self.prompts(s, "02-jumps") if e["event"] == "redo"]
|
||||
self.assertEqual(len(redo), 1)
|
||||
self.assertLess(redo[0]["from"], redo[0]["to"])
|
||||
self.assertNotIn("bounties", profile, "a stopped round isn't counted")
|
||||
|
||||
def test_pause_stops_the_recording(self):
|
||||
def steer(s, states):
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and not (s._status["step"] == "dots" and s._status["state"] == "running"):
|
||||
time.sleep(0.01)
|
||||
s.pause()
|
||||
time.sleep(0.3)
|
||||
self.assertFalse(s._recording)
|
||||
s.resume()
|
||||
s, _, _ = self.run_round("face", steer=steer, speed=10, profile={"eyes_ipd": 0.063})
|
||||
self.assertEqual(s.state, "done")
|
||||
events = [e["event"] for e in self.prompts(s, "01-dots")]
|
||||
i = events.index("pause")
|
||||
self.assertEqual(events[i - 1:i + 3], ["rec", "pause", "resume", "rec"])
|
||||
|
||||
def test_shift_check(self):
|
||||
moved = {"on": False}
|
||||
|
||||
def steer(s, states):
|
||||
# Once the fitment is done, the eyes appear 12 degrees up: the headset slipped down.
|
||||
end = time.monotonic() + 30
|
||||
while time.monotonic() < end and s._status["state"] in ("starting", "fitment"):
|
||||
time.sleep(0.01)
|
||||
ref = s.ref
|
||||
summary = {str(cam): {"n": 90, "finds": 90, "found": 1.0,
|
||||
"x": ref.eyes[cam]["centre"][0] + ref.eyes[cam]["jacobian"][0][1] * 12,
|
||||
"y": ref.eyes[cam]["centre"][1] + ref.eyes[cam]["jacobian"][1][1] * 12,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0} for cam in (0, 1)}
|
||||
s._eyes.fake = lambda text: json.dumps(summary) if text == "fit" else "ok"
|
||||
moved["on"] = True
|
||||
s, states, _ = self.run_round("glasses", steer=steer, profile={"eyes_ipd": 0.063})
|
||||
self.assertTrue(moved["on"])
|
||||
_, fit = self.files(s)
|
||||
self.assertGreaterEqual(len(fit["slips"]), 2, "the nine dots' middle, twice, and the grid's")
|
||||
self.assertIn("up", fit["slips"][0]["hints"])
|
||||
self.assertTrue(any(st["note"].startswith("The headset moved") for st in states))
|
||||
self.assertTrue(any(e["event"] == "slip" for e in self.prompts(s, "01-dots")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""sounds.py: the beeps' WAV files, and a player that never stops a round."""
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import wave
|
||||
from unittest import mock
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import sounds # noqa: E402
|
||||
|
||||
|
||||
class SoundsTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-sounds-")
|
||||
self.env = mock.patch.dict(os.environ, {"XDG_RUNTIME_DIR": self.tmp})
|
||||
self.env.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.env.stop()
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_wavs(self):
|
||||
self.assertEqual(set(sounds.SOUNDS), {"blink", "close", "open"})
|
||||
self.assertEqual(sounds.SOUNDS["close"], sounds.SOUNDS["open"], "open is the same beep again")
|
||||
for name, (hz, secs) in sounds.SOUNDS.items():
|
||||
path = sounds.wav_path(name)
|
||||
self.assertTrue(path.startswith(self.tmp))
|
||||
with wave.open(path) as w:
|
||||
self.assertEqual((w.getnchannels(), w.getsampwidth(), w.getframerate()), (1, 2, sounds.RATE))
|
||||
self.assertEqual(w.getnframes(), int(sounds.RATE * secs))
|
||||
frames = w.readframes(w.getnframes())
|
||||
first, last = int.from_bytes(frames[:2], "little", signed=True), int.from_bytes(frames[-2:], "little", signed=True)
|
||||
self.assertEqual((first, last), (0, 0), "faded in and out: no click")
|
||||
|
||||
def test_player_off_and_missing(self):
|
||||
logged = []
|
||||
self.assertFalse(sounds.Player(on=False, log=logged.append).play("blink"))
|
||||
self.assertEqual(logged, [])
|
||||
with mock.patch.object(sounds.shutil, "which", return_value=None):
|
||||
p = sounds.Player(log=logged.append)
|
||||
self.assertFalse(p.play("close"))
|
||||
self.assertEqual(logged, ["no pw-play: the sound cues are off"])
|
||||
with self.assertRaises(ValueError):
|
||||
p.play("honk")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,227 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's store and encrypted export (gaze/rec/takes.py), on a made-up round of
|
||||
random frames (never real eye images): listing, deleted ranges across the two clocks, the export's
|
||||
chunks, logs and checksums, and unpacking it with a key. Needs zstd (Python 3.14's, as eyerec.py)
|
||||
and the cryptography library: the dev container has both.
|
||||
|
||||
python3 gaze/rec/tests/test_takes.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import eyerec # noqa: E402
|
||||
import takes # noqa: E402
|
||||
from recorder import age # noqa: E402
|
||||
|
||||
try:
|
||||
age._crypto()
|
||||
HAVE = eyerec.compress is not None
|
||||
except RuntimeError:
|
||||
HAVE = False
|
||||
|
||||
RID = "20261014-120000-edges"
|
||||
TID = "01-dots"
|
||||
MONO0 = 1000 * 10**9 # the take starts at 1000 s MONOTONIC
|
||||
OFF = 5 * 10**9 + 123 # RAW is 5 s ahead
|
||||
FPS = 90.0
|
||||
PER_CHUNK = 10
|
||||
CHUNKS = 3
|
||||
|
||||
|
||||
def raw_s(mono_ns):
|
||||
return (mono_ns + OFF) / 1e9
|
||||
|
||||
|
||||
def make_round(base, health=True):
|
||||
"""A round with one take: CHUNKS chunks of PER_CHUNK random frames per camera, frame k of
|
||||
camera c at MONO0 + k / FPS (plus 1 ms for camera 1). Returns {(c, k): frame bytes}."""
|
||||
rdir = os.path.join(base, "rounds", RID)
|
||||
tdir = os.path.join(rdir, "takes", TID)
|
||||
frames = {}
|
||||
lines = [eyerec.INDEX_HEADER]
|
||||
for c in eyerec.CAMERAS:
|
||||
os.makedirs(os.path.join(tdir, "cam%d" % c))
|
||||
for chunk in range(CHUNKS):
|
||||
data = []
|
||||
for i in range(PER_CHUNK):
|
||||
k = chunk * PER_CHUNK + i
|
||||
frames[(c, k)] = os.urandom(eyerec.FRAME)
|
||||
data.append(frames[(c, k)])
|
||||
lines.append("%d\t%d\t%d\t%d\t%.6f\n" % (c, chunk, i, k % 8, raw_s(MONO0 + int(k / FPS * 1e9)) + c * 0.001))
|
||||
with open(os.path.join(tdir, "cam%d" % c, "%06d.zst" % chunk), "wb") as f:
|
||||
f.write(eyerec.compress(b"".join(data)))
|
||||
with open(os.path.join(tdir, "index.tsv"), "w") as f:
|
||||
f.writelines(lines)
|
||||
end = MONO0 + int(CHUNKS * PER_CHUNK / FPS * 1e9)
|
||||
with open(os.path.join(tdir, "take.json"), "w") as f:
|
||||
json.dump({"group": "dots", "started_ns": MONO0, "ended_ns": end, "status": "complete", "steps": [],
|
||||
"deleted": [], "clock": [[MONO0, OFF], [end, OFF]], "notes": ""}, f)
|
||||
with open(os.path.join(tdir, "own.jsonl"), "w") as f:
|
||||
for k in range(CHUNKS * PER_CHUNK):
|
||||
f.write(json.dumps({"t": raw_s(MONO0 + int(k / FPS * 1e9)), "cam": 0, "x": 1.0}) + "\n")
|
||||
with open(os.path.join(tdir, "prompts.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": MONO0, "event": "take"}) + "\n")
|
||||
with open(os.path.join(tdir, "targets.jsonl"), "w") as f:
|
||||
f.write(json.dumps({"t": raw_s(MONO0), "dot": "on", "yaw": 0, "pitch": 0}) + "\n")
|
||||
with open(os.path.join(rdir, "session.json"), "w") as f:
|
||||
json.dump({"schema": 1, "round": RID, "bounty": "edges", "status": "done", "takes": [TID],
|
||||
"started": "2026-10-14T12:00:00+0000", "checklist": {"glasses": "none"},
|
||||
"uploads": [{"pr": 1}]}, f)
|
||||
with open(os.path.join(rdir, "fit.json"), "w") as f:
|
||||
json.dump({"fitment": {"state": "ok"}}, f)
|
||||
with open(os.path.join(rdir, "session.log"), "w") as f:
|
||||
f.write("a log that names /home/someone\n")
|
||||
if health:
|
||||
with open(os.path.join(rdir, "health.json"), "w") as f:
|
||||
json.dump({"conditions": ["test"]}, f)
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": "c0ffee", "consent": {"version": "test-1"}, "health": {"x": 1},
|
||||
"optional": {"notes": "private"}}, f)
|
||||
return frames
|
||||
|
||||
|
||||
def load(path, mode="r"):
|
||||
with open(path, mode) as f:
|
||||
return json.load(f) if path.endswith(".json") else f.read()
|
||||
|
||||
|
||||
def frame_mono(k):
|
||||
return MONO0 + int(k / FPS * 1e9)
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class TakesTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-takes-")
|
||||
self.base = os.path.join(self.tmp, "contrib")
|
||||
self.frames = make_round(self.base)
|
||||
self.store = takes.Store(self.base)
|
||||
self.ident, self.rec = age.keygen()
|
||||
self.key = os.path.join(self.tmp, "key.txt")
|
||||
with open(self.key, "w") as f:
|
||||
f.write("# test key\n" + self.ident + "\n")
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def test_ids(self):
|
||||
for bad in ("../x", "20261014-120000-../x", "", "x"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.store.round_dir(bad)
|
||||
for bad in ("..", "01-a/b", ".x"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.store.take_dir(RID, bad)
|
||||
self.store.round_dir("20261014-120000-edges-2")
|
||||
|
||||
def test_clock(self):
|
||||
clock = [[0, 100], [1000, 200]]
|
||||
self.assertEqual(takes.raw_offset_ns(-5, clock), 100)
|
||||
self.assertEqual(takes.raw_offset_ns(500, clock), 150)
|
||||
self.assertEqual(takes.raw_offset_ns(5000, clock), 200)
|
||||
self.assertEqual(takes.raw_offset_ns(5, []), 0)
|
||||
|
||||
def test_listing(self):
|
||||
[r] = self.store.rounds()
|
||||
self.assertEqual((r["id"], r["bounty"], r["takes"], r["exported"]), (RID, "edges", 1, False))
|
||||
[t] = self.store.takes(RID)
|
||||
self.assertEqual(t["frames"], [30, 30])
|
||||
self.assertEqual(t["deleted_frames"], [0, 0])
|
||||
self.assertAlmostEqual(t["duration_s"], 29 / FPS, places=2)
|
||||
|
||||
def test_ranges(self):
|
||||
# index.tsv's times are whole microseconds: the ranges have a millisecond to spare
|
||||
self.store.delete_range(RID, TID, frame_mono(12) - 10**6, frame_mono(25))
|
||||
self.store.delete_range(RID, TID, frame_mono(20), frame_mono(29) + 10**6)
|
||||
self.assertEqual(self.store.ranges(RID, TID), [[frame_mono(12) - 10**6, frame_mono(29) + 10**6]])
|
||||
[t] = self.store.takes(RID)
|
||||
self.assertEqual(t["deleted_frames"][0], 18)
|
||||
self.store.restore_range(RID, TID, 0)
|
||||
self.assertEqual(self.store.ranges(RID, TID), [])
|
||||
|
||||
def test_export_and_unpack(self):
|
||||
# frames 15..29 deleted: chunk 0 whole, chunk 1 half (10..14 kept), chunk 2 gone
|
||||
self.store.delete_range(RID, TID, frame_mono(15) - 1000, frame_mono(29) + 5 * 10**6)
|
||||
path = self.store.export(RID, recipients=[self.rec])
|
||||
self.assertFalse(os.path.exists(path + ".partial"))
|
||||
m = load(os.path.join(path, "manifest.json"))
|
||||
[e] = m["takes"]
|
||||
self.assertEqual(e["frames_deleted"][0], 15)
|
||||
self.assertEqual(e["frames"][0], 15)
|
||||
self.assertEqual(e["chunks"], 4)
|
||||
self.assertEqual(e["logs"]["own.jsonl"], {"rows": 15, "deleted": 15})
|
||||
self.assertNotIn("uploads", m["session"])
|
||||
self.assertNotIn("health", m["profile"])
|
||||
self.assertNotIn("notes", m["profile"]["optional"])
|
||||
self.assertEqual(m["encryption"]["recipients"], [self.rec])
|
||||
files = sorted(os.path.relpath(os.path.join(r, n), path) for r, _, ns in os.walk(path) for n in ns)
|
||||
self.assertIn("health.json.age", files)
|
||||
self.assertNotIn("health.json", files)
|
||||
self.assertNotIn("session.log", files)
|
||||
self.assertFalse([f for f in files if f.endswith(".zst")], "every chunk is encrypted")
|
||||
self.assertIn("takes/%s/cam0/000001.zst.age" % TID, files)
|
||||
self.assertNotIn("takes/%s/cam0/000002.zst.age" % TID, files)
|
||||
# chunk 0 untouched: its encrypted copy decrypts to the recorded file
|
||||
src = load(os.path.join(self.base, "rounds", RID, "takes", TID, "cam0", "000000.zst"), "rb")
|
||||
enc = load(os.path.join(path, "takes", TID, "cam0", "000000.zst.age"), "rb")
|
||||
self.assertEqual(age.decrypt_bytes(enc, self.ident), src)
|
||||
with open(os.path.join(path, "SHA256SUMS")) as f:
|
||||
self.assertEqual(sorted(line.split(" ", 1)[1].strip() for line in f), [f for f in files if f != "SHA256SUMS"])
|
||||
|
||||
out = takes.unpack(path, self.key, os.path.join(self.tmp, "unpacked"))
|
||||
self.assertEqual(load(os.path.join(out, "health.json")), {"conditions": ["test"]})
|
||||
got = {}
|
||||
tdir = os.path.join(out, "takes", TID)
|
||||
for c, t, slot, data in eyerec.iter_frames(tdir):
|
||||
got.setdefault(c, []).append(data)
|
||||
for c in eyerec.CAMERAS:
|
||||
self.assertEqual(got[c], [self.frames[(c, k)] for k in range(15)])
|
||||
self.assertEqual(load(os.path.join(tdir, "take.json"))["export"]["deleted_raw_s"][0][0],
|
||||
raw_s(frame_mono(15) - 1000))
|
||||
|
||||
[r] = self.store.rounds()
|
||||
self.assertTrue(r["exported"] and not r["export_stale"])
|
||||
|
||||
def test_default_recipients(self):
|
||||
path = self.store.export(RID)
|
||||
m = load(os.path.join(path, "manifest.json"))
|
||||
self.assertEqual(m["encryption"]["recipients"], age.read_recipients(takes.RECIPIENTS))
|
||||
with self.assertRaises(ValueError): # the test key isn't one of them
|
||||
takes.unpack(path, self.key, os.path.join(self.tmp, "no"))
|
||||
|
||||
def test_bad_chunks(self):
|
||||
tdir = os.path.join(self.base, "rounds", RID, "takes", TID)
|
||||
shutil.copyfile(os.path.join(tdir, "cam0", "000000.zst"), os.path.join(tdir, "cam0", "000007.zst"))
|
||||
with open(os.path.join(tdir, "cam1", "000001.zst"), "r+b") as f:
|
||||
f.truncate(100)
|
||||
os.remove(os.path.join(tdir, "cam1", "000002.zst"))
|
||||
path = self.store.export(RID, recipients=[self.rec])
|
||||
[e] = load(os.path.join(path, "manifest.json"))["takes"]
|
||||
self.assertEqual(e["left_out"], {"not_indexed": 1, "missing": 1, "bad": 1})
|
||||
self.assertEqual(e["frames"], [30, 10])
|
||||
with open(os.path.join(path, "takes", TID, "index.tsv")) as f:
|
||||
self.assertEqual(sum(1 for line in f if line.startswith("1\t")), 10)
|
||||
|
||||
def test_cancel(self):
|
||||
with self.assertRaises(takes.Cancelled):
|
||||
self.store.export(RID, recipients=[self.rec], cancel=lambda: True)
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID) + ".partial"))
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID)))
|
||||
|
||||
def test_delete(self):
|
||||
self.store.export(RID, recipients=[self.rec])
|
||||
self.store.delete_take(RID, TID)
|
||||
meta = load(os.path.join(self.base, "rounds", RID, "session.json"))
|
||||
self.assertEqual((meta["takes"], meta["deleted_takes"]), ([], [TID]))
|
||||
self.store.delete_round(RID)
|
||||
self.assertEqual(self.store.rounds(), [])
|
||||
self.assertFalse(os.path.exists(self.store.export_dir(RID)))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,150 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The eye recorder's export check (gaze/rec/validate.py) on takes.py's export of a made-up
|
||||
round of random frames (test_takes.make_round): a good one, with and without a key, and broken
|
||||
ones. Needs zstd and the cryptography library: the dev container has both.
|
||||
|
||||
python3 gaze/rec/tests/test_validate.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
sys.path.insert(0, os.path.join(HERE, ".."))
|
||||
import takes # noqa: E402
|
||||
import validate # noqa: E402
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.files import write_sums # noqa: E402
|
||||
from test_takes import HAVE, RID, TID, make_round # noqa: E402
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE, "needs zstd and the cryptography library (the dev container)")
|
||||
class ValidateTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="eyerec-validate-")
|
||||
base = os.path.join(self.tmp, "contrib")
|
||||
make_round(base)
|
||||
self.contributor = str(uuid.uuid4())
|
||||
with open(os.path.join(base, "profile.json"), "w") as f:
|
||||
json.dump({"contributor": self.contributor,
|
||||
"consent": {"version": "test-1", "adult": True, "region_ok": True, "accepted": "2026-10-14"},
|
||||
"bounties": {}}, f)
|
||||
self.ident, self.rec = age.keygen()
|
||||
self.keys = os.path.join(self.tmp, "recipients.txt")
|
||||
with open(self.keys, "w") as f:
|
||||
f.write("# test\n%s\n" % self.rec)
|
||||
self.store = takes.Store(base)
|
||||
self.path = self.store.export(RID, recipients=[self.rec])
|
||||
self.take = os.path.join(self.path, "takes", TID)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def check(self, identity=None, resum=False):
|
||||
if resum: # a change made on purpose, with checksums to match, so the other checks see it
|
||||
os.remove(os.path.join(self.path, "SHA256SUMS"))
|
||||
write_sums(self.path)
|
||||
return validate.validate(self.path, identity=identity, recipients_path=self.keys)
|
||||
|
||||
def has(self, report, text):
|
||||
self.assertTrue(any(text in e for e in report.errors), (text, report.errors))
|
||||
|
||||
def manifest(self, change):
|
||||
p = os.path.join(self.path, "manifest.json")
|
||||
with open(p) as f:
|
||||
m = json.load(f)
|
||||
change(m)
|
||||
with open(p, "w") as f:
|
||||
json.dump(m, f)
|
||||
|
||||
def test_good(self):
|
||||
r = self.check()
|
||||
self.assertEqual(r.errors, [])
|
||||
self.assertEqual(r.summary["frames"], 60)
|
||||
self.assertTrue(r.summary["health"])
|
||||
r = self.check(identity=self.ident)
|
||||
self.assertEqual(r.errors, [])
|
||||
|
||||
def test_shipped_keys(self):
|
||||
r = validate.validate(self.path) # recipients.txt's, not the test key
|
||||
self.has(r, "other keys than the dataset's")
|
||||
|
||||
def test_checksum(self):
|
||||
with open(os.path.join(self.take, "cam0", "000001.zst.age"), "r+b") as f:
|
||||
f.seek(200)
|
||||
b = f.read(1)
|
||||
f.seek(200)
|
||||
f.write(bytes([b[0] ^ 1]))
|
||||
self.has(self.check(), "checksum mismatch")
|
||||
|
||||
def test_unencrypted(self):
|
||||
shutil.copyfile(os.path.join(self.take, "index.tsv"), os.path.join(self.take, "cam0", "000009.zst"))
|
||||
with open(os.path.join(self.path, "health.json"), "w") as f:
|
||||
f.write("{}")
|
||||
r = self.check(resum=True)
|
||||
self.has(r, "cam0/000009.zst: not an allowed file (unencrypted")
|
||||
self.has(r, "health.json: not an allowed file (unencrypted")
|
||||
|
||||
def test_cut_short(self):
|
||||
# Without a key only some sizes show a cut (a last piece under a tag); the key finds any.
|
||||
p = os.path.join(self.take, "cam1", "000000.zst.age")
|
||||
with open(p, "rb") as f:
|
||||
data = f.read()
|
||||
body = data.index(b"\n", data.index(b"\n---") + 1) + 1 + 16
|
||||
whole = (len(data) - body) // (age.CHUNK + age.TAG)
|
||||
with open(p, "r+b") as f:
|
||||
f.truncate(body + whole * (age.CHUNK + age.TAG) + 5)
|
||||
self.has(self.check(resum=True), "cam1/000000.zst.age: a size no age file has")
|
||||
with open(p, "r+b") as f:
|
||||
f.truncate(len(data) - 65000)
|
||||
self.assertEqual(self.check(resum=True).errors, [])
|
||||
self.has(self.check(identity=self.ident, resum=True), "cam1/000000.zst.age: doesn't decrypt")
|
||||
|
||||
def test_wrong_content(self):
|
||||
with open(os.path.join(self.take, "cam0", "000000.zst.age"), "wb") as f:
|
||||
f.write(age.encrypt_bytes(os.urandom(1000), [self.rec]))
|
||||
self.assertEqual(self.check(resum=True).errors, [], "without a key it can't tell")
|
||||
self.has(self.check(identity=self.ident), "doesn't decompress")
|
||||
other, _ = age.keygen()
|
||||
self.has(self.check(identity=other), "doesn't decrypt")
|
||||
|
||||
def test_index(self):
|
||||
p = os.path.join(self.take, "index.tsv")
|
||||
with open(p) as f:
|
||||
lines = f.readlines()
|
||||
with open(p, "w") as f:
|
||||
f.writelines(lines[:3] + lines[4:]) # camera 0, chunk 0, frame 2 gone
|
||||
r = self.check(resum=True)
|
||||
self.has(r, "frames aren't numbered")
|
||||
self.has(r, "the manifest says [30, 30] frames, index.tsv has [29, 30]")
|
||||
os.remove(os.path.join(self.take, "cam1", "000001.zst.age"))
|
||||
self.has(self.check(resum=True), "cam1/000001.zst.age: in index.tsv but missing")
|
||||
|
||||
def test_manifest(self):
|
||||
def change(m):
|
||||
m["session"]["dry_run"] = True
|
||||
m["session"]["replay"] = "/dev/shm/x"
|
||||
m["profile"]["health"] = {"x": 1}
|
||||
m["profile"]["consent"]["adult"] = False
|
||||
m["contributor"] = m["profile"]["contributor"] = "c0ffee"
|
||||
self.manifest(change)
|
||||
r = self.check(resum=True)
|
||||
for text in ("dry-run round", "from a replay", "unexpected keys: health", "18 or older", "random uuid4"):
|
||||
self.has(r, text)
|
||||
|
||||
def test_two_recipients(self):
|
||||
_, second = age.keygen()
|
||||
with open(self.keys, "a") as f:
|
||||
f.write(second + "\n")
|
||||
self.has(self.check(), "other keys than the dataset's")
|
||||
self.manifest(lambda m: m["encryption"].update(recipients=[self.rec, second]))
|
||||
self.has(self.check(resum=True), "encrypted to 1 recipients, not 2")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,467 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check an eye recorder export before it's uploaded, or when it's received (takes.py's
|
||||
exports/<round>/).
|
||||
|
||||
The window runs it before an upload and refuses on errors; the maintainer runs it on each
|
||||
submission. The images are encrypted, so without a key it checks everything around them:
|
||||
- SHA256SUMS: every file listed and matching, nothing missing from it;
|
||||
- an allow-list of files: anything else (an unencrypted chunk, health.json, a symlink) is an error;
|
||||
- every .age file is age v1 to as many X25519 recipients as the manifest names, and its size
|
||||
is one a whole age file can have; the manifest's recipients are recipients.txt's;
|
||||
- manifest.json: the schema, required keys and types, the consent, a random uuid4 contributor,
|
||||
no dry run or replay, no health answers outside health.json.age;
|
||||
- each take's index.tsv against its chunks (every chunk named is there, every one there is
|
||||
named, frames numbered from 0, times going forward) and against the manifest's counts;
|
||||
- every .jsonl line is a JSON object with a time "t", and take.json and fit.json parse;
|
||||
- the total size.
|
||||
With --identity KEYFILE (one of the dataset's keys, the maintainer's side) it also decrypts each
|
||||
chunk in memory and checks it decompresses to the frames index.tsv gives it.
|
||||
|
||||
Standard library only without --identity (zstd and the cryptography library with it). Python
|
||||
3.12 or later, Linux or Windows.
|
||||
|
||||
usage: validate.py DIR [--json] [--identity KEYFILE]
|
||||
exit status 0: no errors, 1: errors, 2: not an export
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(HERE)))
|
||||
from recorder import age # noqa: E402
|
||||
from recorder.checks import (Cancelled, Report, check_sums, is_uuid4, listed, need, # noqa: E402,F401
|
||||
read_json_file)
|
||||
|
||||
EXPORT_SCHEMA = 1
|
||||
RECIPIENTS = os.path.join(HERE, "recipients.txt")
|
||||
TOP_FILES = {"manifest.json", "fit.json", "health.json.age", "SHA256SUMS"}
|
||||
TAKE_FILES = {"take.json", "index.tsv", "targets.jsonl", "ipd.jsonl", "prompts.jsonl", "own.jsonl",
|
||||
"steamvr.jsonl", "poses.jsonl"}
|
||||
CAM_DIRS = ("cam0", "cam1")
|
||||
ROUND_RE = re.compile(r"^\d{8}-\d{6}-[a-z0-9][a-z0-9-]{0,39}$")
|
||||
TAKE_RE = re.compile(r"^\d{2,}-[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
CHUNK_FILE_RE = re.compile(r"^(\d{6})\.zst\.age$")
|
||||
TAKE_STATUSES = ("complete", "stopped", "skipped")
|
||||
PROFILE_KEYS = {"schema", "contributor", "consent", "optional", "checklist", "bounties", "eyes_ipd"}
|
||||
INDEX_HEADER = "# camera\tchunk\tframe\tslot\tt_raw"
|
||||
FRAME = 512 * 400
|
||||
CHUNK_FRAMES_MAX = 1000 # eyerec.py writes 90 a chunk
|
||||
# One round is about 2.6 GB (2026-10-14 replay: 20 MB/s on disk); encryption adds next to nothing.
|
||||
WARN_BYTES = 6 * 1000 ** 3
|
||||
MAX_BYTES = 15 * 1000 ** 3
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the files
|
||||
|
||||
def walk_files(root, report):
|
||||
"""The export's files as relative paths, and allow-list errors on the way."""
|
||||
files = []
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
rel_dir = os.path.relpath(dirpath, root).replace(os.sep, "/")
|
||||
rel_dir = "" if rel_dir == "." else rel_dir
|
||||
parts = rel_dir.split("/") if rel_dir else []
|
||||
for d in list(dirnames):
|
||||
rel = f"{rel_dir}/{d}" if rel_dir else d
|
||||
ok = ((not parts and d == "takes") or (parts == ["takes"] and TAKE_RE.match(d))
|
||||
or (len(parts) == 2 and parts[0] == "takes" and d in CAM_DIRS))
|
||||
if os.path.islink(os.path.join(dirpath, d)):
|
||||
report.error(f"{rel}: a symlink, not allowed in an export")
|
||||
dirnames.remove(d)
|
||||
elif not ok:
|
||||
report.error(f"{rel}/: not an allowed folder")
|
||||
dirnames.remove(d)
|
||||
for name in filenames:
|
||||
rel = f"{rel_dir}/{name}" if rel_dir else name
|
||||
full = os.path.join(dirpath, name)
|
||||
if os.path.islink(full) or not os.path.isfile(full):
|
||||
report.error(f"{rel}: a symlink or special file, not allowed in an export")
|
||||
continue
|
||||
ok = ((not parts and name in TOP_FILES) or (len(parts) == 2 and name in TAKE_FILES)
|
||||
or (len(parts) == 3 and CHUNK_FILE_RE.match(name)))
|
||||
if not ok:
|
||||
hint = " (unencrypted: export it again)" if name.endswith(".zst") or name == "health.json" else ""
|
||||
report.error(f"{rel}: not an allowed file{hint}")
|
||||
files.append(rel)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
age_shape = age.shape # recorder/age.py: shared with the hand recorder's debug exports
|
||||
|
||||
|
||||
def check_jsonl(path, rel, report, cancelled):
|
||||
"""Every line a JSON object with a finite time "t" (RAW seconds, or MONOTONIC ns in prompts)."""
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
if n % 50000 == 0 and cancelled():
|
||||
raise Cancelled()
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except ValueError as e:
|
||||
report.error(f"{rel} line {n}: not valid JSON: {e}")
|
||||
return
|
||||
t = obj.get("t") if isinstance(obj, dict) else None
|
||||
if isinstance(t, bool) or not isinstance(t, (int, float)) or not math.isfinite(t):
|
||||
report.error(f"{rel} line {n}: not an object with a time \"t\"")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
report.error(f"{rel}: not UTF-8 text")
|
||||
except OSError as e:
|
||||
report.error(f"{rel}: can't read it: {e.strerror or e}")
|
||||
|
||||
|
||||
def read_index(path, rel, report):
|
||||
"""index.tsv as {(camera, chunk): [(frame, t)]}, or None when it doesn't parse."""
|
||||
out = {}
|
||||
with open(path, encoding="utf-8", errors="replace") as f:
|
||||
for n, line in enumerate(f, 1):
|
||||
line = line.rstrip("\n")
|
||||
if n == 1:
|
||||
if line != INDEX_HEADER:
|
||||
report.error(f"{rel}: the first line isn't the header")
|
||||
return None
|
||||
continue
|
||||
if not line:
|
||||
continue
|
||||
fields = line.split("\t")
|
||||
try:
|
||||
c, chunk, i, slot = (int(x) for x in fields[:4])
|
||||
t = float(fields[4])
|
||||
if len(fields) != 5 or not math.isfinite(t) or c not in (0, 1) or min(chunk, i, slot) < 0:
|
||||
raise ValueError
|
||||
except (ValueError, IndexError):
|
||||
report.error(f"{rel} line {n}: not camera, chunk, frame, slot, time")
|
||||
return None
|
||||
out.setdefault((c, chunk), []).append((i, t))
|
||||
return out
|
||||
|
||||
|
||||
def check_take(root, tid, entry, sizes, recipients, identity, report, step, cancelled):
|
||||
base = f"takes/{tid}/"
|
||||
for name in sorted(TAKE_FILES - {"take.json", "index.tsv"}):
|
||||
rel = base + name
|
||||
if rel in sizes:
|
||||
check_jsonl(os.path.join(root, rel), rel, report, cancelled)
|
||||
elif name in ("targets.jsonl", "prompts.jsonl", "own.jsonl"):
|
||||
report.warn(f"{rel} is missing")
|
||||
if base + "take.json" in sizes:
|
||||
tj = read_json_file(os.path.join(root, base + "take.json"), base + "take.json", report)
|
||||
if tj is not None and not isinstance(tj, dict):
|
||||
report.error(f"{base}take.json: not a JSON object")
|
||||
else:
|
||||
report.warn(f"{base}take.json is missing")
|
||||
chunks = {}
|
||||
for rel in sizes:
|
||||
if rel.startswith(base) and rel.count("/") == 3:
|
||||
m = CHUNK_FILE_RE.match(rel.rsplit("/", 1)[1])
|
||||
if m:
|
||||
chunks[(int(rel.split("/")[2][3:]), int(m.group(1)))] = rel
|
||||
if base + "index.tsv" not in sizes:
|
||||
if chunks:
|
||||
report.error(f"{base}index.tsv is missing, but there are {len(chunks)} chunks")
|
||||
return
|
||||
index = read_index(os.path.join(root, base + "index.tsv"), base + "index.tsv", report)
|
||||
if index is None:
|
||||
return
|
||||
frames = [0, 0]
|
||||
unnamed = sorted(set(chunks) - set(index))
|
||||
if unnamed:
|
||||
report.error(f"{base}: chunks index.tsv doesn't name: {listed(chunks[k] for k in unnamed)}")
|
||||
last = [-math.inf, -math.inf]
|
||||
for key in sorted(index):
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
c, chunk = key
|
||||
rows = index[key]
|
||||
frames[c] += len(rows)
|
||||
name = f"{base}cam{c}/{chunk:06d}.zst.age"
|
||||
if key not in chunks:
|
||||
report.error(f"{name}: in index.tsv but missing")
|
||||
continue
|
||||
if [i for i, _ in rows] != list(range(len(rows))) or len(rows) > CHUNK_FRAMES_MAX:
|
||||
report.error(f"{base}index.tsv: chunk {c}/{chunk:06d}'s frames aren't numbered 0 to {len(rows) - 1}")
|
||||
times = [t for _, t in rows]
|
||||
if any(b < a for a, b in zip(times, times[1:])) or times[0] < last[c]:
|
||||
report.error(f"{base}index.tsv: camera {c}'s times go back at chunk {chunk:06d}")
|
||||
last[c] = times[-1]
|
||||
problem = age_shape(os.path.join(root, name), recipients)
|
||||
if problem:
|
||||
report.error(f"{name}: {problem}")
|
||||
elif identity:
|
||||
problem = decrypted_problem(os.path.join(root, name), identity, len(rows))
|
||||
if problem:
|
||||
report.error(f"{name}: {problem}")
|
||||
step(sizes[name])
|
||||
if entry is not None and isinstance(entry.get("frames"), list) and entry["frames"] != frames:
|
||||
report.error(f"take {tid}: the manifest says {entry['frames']} frames, index.tsv has {frames}")
|
||||
|
||||
|
||||
def decrypted_problem(path, identity, nframes):
|
||||
sys.path.insert(0, HERE)
|
||||
import eyerec # zstd, as the recorder has it
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
data = eyerec.decompress(age.decrypt_bytes(f.read(), identity))
|
||||
except ValueError as e:
|
||||
return f"doesn't decrypt: {e}"
|
||||
except Exception as e:
|
||||
return f"doesn't decompress: {e}"
|
||||
if len(data) != nframes * FRAME:
|
||||
return f"{len(data)} bytes, but index.tsv gives it {nframes} frames"
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- the manifest
|
||||
|
||||
def check_manifest(m, report, root, shipped):
|
||||
"""The manifest's keys and types. Returns ({take id: entry}, the recipients)."""
|
||||
if not isinstance(m, dict):
|
||||
report.error("manifest: not a JSON object")
|
||||
return {}, []
|
||||
if m.get("schema") != EXPORT_SCHEMA:
|
||||
report.error(f"manifest: schema {m.get('schema')!r}, expected {EXPORT_SCHEMA}")
|
||||
tool = need(m, "tool", str, "", report)
|
||||
if tool is not None and not tool.startswith("ft-eyerec"):
|
||||
report.warn(f"manifest: tool is {tool!r}")
|
||||
need(m, "exported", str, "", report)
|
||||
need(m, "bounty", str, "", report)
|
||||
rid = need(m, "round_id", str, "", report)
|
||||
if rid is not None and not ROUND_RE.match(rid):
|
||||
report.error(f"manifest: round_id {rid!r} isn't a round id")
|
||||
if rid and ROUND_RE.match(os.path.basename(root)) and os.path.basename(root) != rid:
|
||||
report.warn(f"the folder is named {os.path.basename(root)}, the manifest's round is {rid}")
|
||||
contributor = need(m, "contributor", str, "", report)
|
||||
if contributor is not None and not is_uuid4(contributor):
|
||||
report.error(f"manifest: the contributor id {contributor!r} isn't a random uuid4")
|
||||
parent = os.path.basename(os.path.dirname(root))
|
||||
if contributor and is_uuid4(parent) and parent != contributor:
|
||||
report.error(f"the export sits under contributor {parent}, but its manifest says {contributor}")
|
||||
consent = need(m, "consent_version", str, "", report)
|
||||
if consent is not None and not consent.strip():
|
||||
report.error("manifest: the consent version is empty")
|
||||
|
||||
recipients = []
|
||||
enc = need(m, "encryption", dict, "", report)
|
||||
if enc is not None:
|
||||
if enc.get("format") != "age-encryption.org/v1":
|
||||
report.error(f"manifest: encryption.format {enc.get('format')!r}")
|
||||
r = enc.get("recipients")
|
||||
if not isinstance(r, list) or not r or not all(isinstance(x, str) for x in r):
|
||||
report.error("manifest: encryption.recipients isn't a list of keys")
|
||||
else:
|
||||
recipients = r
|
||||
try:
|
||||
for x in r:
|
||||
age.parse_recipient(x)
|
||||
except ValueError as e:
|
||||
report.error(f"manifest: encryption.recipients: {e}")
|
||||
if shipped is not None and sorted(r) != sorted(shipped):
|
||||
report.error("manifest: encrypted to other keys than the dataset's (recipients.txt)")
|
||||
|
||||
profile = need(m, "profile", dict, "", report)
|
||||
if profile is not None:
|
||||
if profile.get("contributor") != contributor:
|
||||
report.error("manifest: profile.contributor differs from contributor")
|
||||
pc = need(profile, "consent", dict, "profile.", report)
|
||||
if pc is not None:
|
||||
if pc.get("version") != consent:
|
||||
report.error("manifest: profile.consent.version differs from consent_version")
|
||||
if pc.get("adult") is not True:
|
||||
report.error("manifest: the contributor didn't confirm being 18 or older")
|
||||
if pc.get("region_ok") is not True:
|
||||
report.error("manifest: the contributor didn't confirm not living in Illinois, Texas or Washington")
|
||||
need(pc, "accepted", str, "profile.consent.", report)
|
||||
optional = profile.get("optional")
|
||||
if isinstance(optional, dict) and str(optional.get("notes") or "").strip():
|
||||
report.warn("the profile's notes are included: check they hold nothing private")
|
||||
extra = set(profile) - PROFILE_KEYS
|
||||
if extra:
|
||||
report.error(f"manifest: the profile has unexpected keys: {listed(sorted(extra))}")
|
||||
|
||||
session = need(m, "session", dict, "", report)
|
||||
if session is not None:
|
||||
for key, t in (("started", str), ("checklist", dict), ("takes", list), ("bounty", str)):
|
||||
need(session, key, t, "session.", report)
|
||||
if session.get("dry_run"):
|
||||
report.error("this is a dry-run round: it has no real recordings")
|
||||
if session.get("replay"):
|
||||
report.error("this round was recorded from a replay, not the headset's cameras")
|
||||
if session.get("status") not in ("done", None):
|
||||
report.warn(f"the round's status is {session.get('status')!r}")
|
||||
if "uploads" in session:
|
||||
report.warn("manifest: session.uploads is included (earlier uploads' links)")
|
||||
if "health" in session:
|
||||
report.error("manifest: health answers outside health.json.age")
|
||||
checklist = session.get("checklist")
|
||||
if isinstance(checklist, dict) and str(checklist.get("notes") or "").strip():
|
||||
report.warn("the checklist's notes are included: check they hold nothing private")
|
||||
if "/home/" in json.dumps(m):
|
||||
report.warn("manifest: it mentions a home folder path (a username may show)")
|
||||
|
||||
entries = {}
|
||||
takes = need(m, "takes", list, "", report)
|
||||
for i, t in enumerate(takes or []):
|
||||
where = f"takes[{i}]."
|
||||
if not isinstance(t, dict):
|
||||
report.error(f"manifest: takes[{i}] isn't an object")
|
||||
continue
|
||||
tid = need(t, "id", str, where, report)
|
||||
if tid is None:
|
||||
continue
|
||||
if not TAKE_RE.match(tid):
|
||||
report.error(f"manifest: take id {tid!r} isn't one")
|
||||
continue
|
||||
if tid in entries:
|
||||
report.error(f"manifest: take {tid} is listed twice")
|
||||
entries[tid] = t
|
||||
for key, typ in (("group", str), ("status", str), ("frames", list), ("frames_deleted", list),
|
||||
("duration_s", (int, float)), ("chunks", int)):
|
||||
need(t, key, typ, where, report)
|
||||
if t.get("status") not in TAKE_STATUSES:
|
||||
report.warn(f"manifest: take {tid} has status {t.get('status')!r}")
|
||||
left = t.get("left_out")
|
||||
if isinstance(left, dict) and any(left.values()):
|
||||
report.warn(f"take {tid}: chunks the export left out: {left}")
|
||||
return entries, recipients
|
||||
|
||||
|
||||
def summarize(m):
|
||||
"""What a pull request's description says about the export."""
|
||||
if not isinstance(m, dict):
|
||||
return {}
|
||||
session = m.get("session") if isinstance(m.get("session"), dict) else {}
|
||||
checklist = session.get("checklist") if isinstance(session.get("checklist"), dict) else {}
|
||||
takes = [t for t in m.get("takes") or [] if isinstance(t, dict)]
|
||||
seconds = sum(t.get("duration_s") or 0 for t in takes if isinstance(t.get("duration_s"), (int, float)))
|
||||
frames = sum(sum(f for f in t.get("frames") or [] if isinstance(f, int)) for t in takes
|
||||
if isinstance(t.get("frames"), list))
|
||||
enc = m.get("encryption") if isinstance(m.get("encryption"), dict) else {}
|
||||
return {"round": m.get("round_id", ""), "bounty": m.get("bounty", ""), "contributor": m.get("contributor", ""),
|
||||
"takes": len(takes), "frames": frames, "minutes": round(seconds / 60, 1),
|
||||
"glasses": checklist.get("glasses", ""), "health": "health.json.age" in (enc.get("files") or []),
|
||||
"consent_version": m.get("consent_version", ""), "tool": m.get("tool", "")}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- all of it
|
||||
|
||||
def validate(root, progress=None, cancel=None, identity=None, recipients_path=RECIPIENTS):
|
||||
"""Check the export at root. progress(fraction 0..1, text) is called as it goes; cancel()
|
||||
(or cancel.is_set()) returning true raises Cancelled. identity: an "AGE-SECRET-KEY-1..."
|
||||
string, to check inside the chunks. recipients_path: the keys the export must be encrypted
|
||||
to (None: don't compare). Returns a Report."""
|
||||
root = os.path.abspath(root)
|
||||
report = Report(root)
|
||||
cancelled = getattr(cancel, "is_set", cancel) or (lambda: False)
|
||||
tell = progress or (lambda fraction, text: None)
|
||||
if not os.path.isdir(root):
|
||||
report.error(f"{root}: no such folder")
|
||||
return report
|
||||
shipped = None
|
||||
if recipients_path:
|
||||
try:
|
||||
shipped = age.read_recipients(recipients_path)
|
||||
except (OSError, ValueError) as e:
|
||||
report.error(f"the dataset's keys ({recipients_path}): {e}")
|
||||
files = walk_files(root, report)
|
||||
sizes = {rel: os.path.getsize(os.path.join(root, rel)) for rel in files}
|
||||
report.bytes = sum(sizes.values())
|
||||
total = max(1, 2 * report.bytes) # checksums, then the chunks again
|
||||
done = 0
|
||||
|
||||
def step(n, text):
|
||||
nonlocal done
|
||||
done += n
|
||||
tell(min(done / total, 0.999), text)
|
||||
|
||||
tell(0.0, "Checking checksums")
|
||||
check_sums(root, files, report, lambda n: step(n, "Checking checksums"), cancelled)
|
||||
|
||||
manifest = None
|
||||
if "manifest.json" in files:
|
||||
manifest = read_json_file(os.path.join(root, "manifest.json"), "manifest.json", report)
|
||||
else:
|
||||
report.error("manifest.json is missing")
|
||||
entries, recipients = check_manifest(manifest, report, root, shipped) if manifest is not None else ({}, [])
|
||||
report.summary = summarize(manifest)
|
||||
|
||||
if "fit.json" in files:
|
||||
fit = read_json_file(os.path.join(root, "fit.json"), "fit.json", report)
|
||||
if fit is not None and not isinstance(fit, dict):
|
||||
report.error("fit.json: not a JSON object")
|
||||
else:
|
||||
report.warn("no fit.json: the round's fit readings are missing")
|
||||
enc_files = ((manifest or {}).get("encryption") or {}).get("files") if isinstance(manifest, dict) else None
|
||||
listed_health = isinstance(enc_files, list) and "health.json.age" in enc_files
|
||||
if ("health.json.age" in files) != listed_health:
|
||||
report.error("health.json.age and the manifest's encryption.files don't agree")
|
||||
if "health.json.age" in files and recipients:
|
||||
problem = age_shape(os.path.join(root, "health.json.age"), recipients)
|
||||
if problem:
|
||||
report.error(f"health.json.age: {problem}")
|
||||
|
||||
take_dirs = sorted({rel.split("/")[1] for rel in files if rel.startswith("takes/") and rel.count("/") >= 2})
|
||||
for tid in take_dirs:
|
||||
if tid not in entries:
|
||||
report.error(f"takes/{tid}/ isn't in the manifest")
|
||||
for tid, entry in entries.items():
|
||||
if tid not in take_dirs:
|
||||
report.error(f"take {tid} is in the manifest but has no folder")
|
||||
for tid in take_dirs:
|
||||
if cancelled():
|
||||
raise Cancelled()
|
||||
check_take(root, tid, entries.get(tid), sizes, recipients, identity, report,
|
||||
lambda n, tid=tid: step(n, f"Checking {tid}"), cancelled)
|
||||
|
||||
if report.bytes > MAX_BYTES:
|
||||
report.error(f"the export is {report.bytes / 1000 ** 3:.1f} GB, over the {MAX_BYTES / 1000 ** 3:.0f} GB limit")
|
||||
elif report.bytes > WARN_BYTES:
|
||||
report.warn(f"the export is large: {report.bytes / 1000 ** 3:.1f} GB")
|
||||
tell(1.0, "Checked")
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="Check an eye recorder export (an exports/<round>/ folder).")
|
||||
ap.add_argument("dir")
|
||||
ap.add_argument("--json", action="store_true", help="print the result as JSON")
|
||||
ap.add_argument("--identity", help="an age identity file: also decrypt and check every chunk")
|
||||
a = ap.parse_args()
|
||||
if not os.path.isdir(a.dir):
|
||||
print(f"{a.dir}: no such folder", file=sys.stderr)
|
||||
return 2
|
||||
if hasattr(os, "setpriority"): # not on Windows
|
||||
try:
|
||||
os.setpriority(os.PRIO_PROCESS, 0, 19) # stay out of VR's way
|
||||
except OSError:
|
||||
pass
|
||||
identity = None
|
||||
if a.identity:
|
||||
with open(a.identity) as f:
|
||||
identity = next((line.strip() for line in f if line.strip().startswith("AGE-SECRET-KEY-1")), None)
|
||||
if not identity:
|
||||
print(f"{a.identity}: no age identity in it", file=sys.stderr)
|
||||
return 2
|
||||
report = validate(a.dir, identity=identity)
|
||||
if a.json:
|
||||
print(json.dumps(report.as_dict(), indent=2))
|
||||
else:
|
||||
for e in report.errors:
|
||||
print(f"error: {e}")
|
||||
for w in report.warnings:
|
||||
print(f"warning: {w}")
|
||||
s = report.summary
|
||||
if s:
|
||||
print(f"{s.get('round')}: {s.get('takes')} takes, {s.get('frames')} frames, {s.get('minutes')} min, "
|
||||
f"{report.bytes / 1000 ** 2:.1f} MB, consent {s.get('consent_version') or '-'}")
|
||||
print("OK" if report.ok else f"{len(report.errors)} errors")
|
||||
return 0 if report.ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+6
-10
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install, start, stop, or inspect the gaze service (ft-gazed) on the Frame.
|
||||
# Usage: gaze/run.sh install|uninstall # user service, starts with SteamVR; install also keeps
|
||||
# # SteamVR's eye cameras at 90 fps (gaze/eye-rate.py)
|
||||
# Usage: gaze/run.sh install|uninstall # user service, starts with SteamVR
|
||||
# gaze/run.sh start|stop|restart|status|log [lines]
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
|
||||
@@ -10,15 +9,12 @@ frame="$root/scripts/frame.sh"
|
||||
unit=frametop-gaze.service
|
||||
case ${1:-status} in
|
||||
install)
|
||||
[ "$FRAME_RELEASE" = 1 ] || "$root/gaze/build.sh"
|
||||
"$root/gaze/build.sh"
|
||||
fill_template "$root/gaze/$unit" | on_frame "mkdir -p ~/.config/systemd/user && cat > ~/.config/systemd/user/$unit"
|
||||
on_frame "chmod +x gaze/ft-gazed gaze/ft-gazectl gaze/eye-rate.py"
|
||||
on_frame "/usr/bin/python3 gaze/eye-rate.py on" || true
|
||||
"$frame" --host "set -e; systemctl --user daemon-reload; systemctl --user enable $unit
|
||||
$(start_with_steamvr $unit)" ;;
|
||||
uninstall)
|
||||
"$frame" --host "systemctl --user disable --now $unit 2>/dev/null; rm -f ~/.config/systemd/user/$unit; systemctl --user daemon-reload; echo removed"
|
||||
on_frame "/usr/bin/python3 gaze/eye-rate.py off" || true ;;
|
||||
on_frame "chmod +x gaze/ft-gazed gaze/ft-gazectl"
|
||||
"$frame" --host "set -e; systemctl --user daemon-reload; systemctl --user enable --now $unit
|
||||
sleep 2; echo \"$unit: \$(systemctl --user is-active $unit)\"; journalctl --user -u $unit --no-pager -o cat -n 5" ;;
|
||||
uninstall) "$frame" --host "systemctl --user disable --now $unit 2>/dev/null; rm -f ~/.config/systemd/user/$unit; systemctl --user daemon-reload; echo removed" ;;
|
||||
start|stop|restart) "$frame" --host "systemctl --user $1 $unit; systemctl --user is-active $unit" ;;
|
||||
status) "$frame" --host "systemctl --user is-active $unit" || true; on_frame "gaze/ft-gazectl status" || true ;;
|
||||
log) "$frame" --host "journalctl --user -u $unit --no-pager -o cat -n ${2:-30}" ;;
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of gaze/eye-rate.py (SteamVR's eye cameras at 90 fps): on, off, and the camera
|
||||
rate, with SteamVR off (its settings file) and on (vrcmd). HOME is a temporary folder and
|
||||
vrcmd and pgrep are stand-ins, so the live SteamVR settings are never read or changed.
|
||||
|
||||
gaze/test/eye-rate-test.py
|
||||
"""
|
||||
import contextlib
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
tmp = Path(tempfile.mkdtemp(prefix="ft-eye-rate-test-"))
|
||||
os.environ["HOME"] = str(tmp) # eye-rate.py's paths come from it
|
||||
|
||||
loader = importlib.machinery.SourceFileLoader("eyerate", os.path.join(HERE, "..", "eye-rate.py"))
|
||||
er = importlib.util.module_from_spec(importlib.util.spec_from_loader("eyerate", loader))
|
||||
loader.exec_module(er)
|
||||
assert str(er.HOME) == str(tmp), "eye-rate.py must take HOME from the environment"
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(what, got, want):
|
||||
if got != want:
|
||||
failures.append(what)
|
||||
print(f"FAIL {what}: got {got!r}, want {want!r}", flush=True)
|
||||
|
||||
|
||||
def main_with(*args):
|
||||
"""eye-rate.py's main() with these arguments: (exit code, what it printed)."""
|
||||
er.sys.argv = ["eye-rate.py", *args]
|
||||
out = io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(out):
|
||||
code = er.main()
|
||||
return code, out.getvalue()
|
||||
|
||||
|
||||
settings = er.STEAMVR_SETTINGS[0]
|
||||
|
||||
|
||||
def file_value():
|
||||
return json.loads(settings.read_text()).get("steamvr", {}).get("gazeDepEyeToHeadEnable")
|
||||
|
||||
|
||||
# --- SteamVR off: the settings file ---
|
||||
er.steamvr_up = lambda: False
|
||||
settings.parent.mkdir(parents=True)
|
||||
settings.write_text(json.dumps({"steamvr": {"analogGain": 0.25}, "dashboard": {"x": 1}}, indent=3))
|
||||
check("off by default (not in the file)", er.get(), None)
|
||||
code, _ = main_with("on")
|
||||
check("on: exit 0", code, 0)
|
||||
check("on: written to the file", file_value(), True)
|
||||
check("on: SteamVR's other settings kept", json.loads(settings.read_text())["steamvr"]["analogGain"], 0.25)
|
||||
check("on: other sections kept", json.loads(settings.read_text())["dashboard"], {"x": 1})
|
||||
check("on: marked as Frametop's", er.MARK.exists(), True)
|
||||
code, out = main_with("on")
|
||||
check("on again: says it's on already", "already" in out, True)
|
||||
code, _ = main_with("off")
|
||||
check("off: written to the file", file_value(), False)
|
||||
check("off: mark gone", er.MARK.exists(), False)
|
||||
|
||||
# Someone turned it on themselves: off leaves it.
|
||||
settings.write_text(json.dumps({"steamvr": {"gazeDepEyeToHeadEnable": True}}))
|
||||
code, out = main_with("on")
|
||||
check("on, set by the user: no mark", er.MARK.exists(), False)
|
||||
code, out = main_with("off")
|
||||
check("off, set by the user: left on", file_value(), True)
|
||||
check("off, set by the user: says so", "left as they are" in out, True)
|
||||
|
||||
# No settings file yet (SteamVR never ran): on makes one.
|
||||
settings.unlink()
|
||||
code, _ = main_with("on")
|
||||
check("no file: on makes it", file_value(), True)
|
||||
main_with("off")
|
||||
|
||||
# A broken file isn't overwritten.
|
||||
settings.write_text("{not json")
|
||||
code, _ = main_with("on")
|
||||
check("broken file: on fails", code, 1)
|
||||
check("broken file: left alone", settings.read_text(), "{not json")
|
||||
check("broken file: no mark", er.MARK.exists(), False)
|
||||
settings.unlink()
|
||||
|
||||
# --- SteamVR on: vrcmd ---
|
||||
live = {"value": False}
|
||||
calls = []
|
||||
|
||||
|
||||
def fake_vrcmd(*args):
|
||||
calls.append(args)
|
||||
if args[0] == "--settings-bool":
|
||||
return f"steamvr.gazeDepEyeToHeadEnable={'true' if live['value'] else 'false'}\n"
|
||||
if args[0] == "--set-settings-bool":
|
||||
live["value"] = args[2] == "1" # as vrcmd does: "true" would read as 0
|
||||
return f"steamvr.gazeDepEyeToHeadEnable={args[2]}\n"
|
||||
return ""
|
||||
|
||||
|
||||
er.steamvr_up = lambda: True
|
||||
er.vrcmd = fake_vrcmd
|
||||
code, _ = main_with("on")
|
||||
check("SteamVR on: set through vrcmd", live["value"], True)
|
||||
check("SteamVR on: sets 1, never 'true'", [c for c in calls if c[0] == "--set-settings-bool"][-1][2], "1")
|
||||
check("SteamVR on: the file isn't touched", settings.exists(), False)
|
||||
check("SteamVR on: marked", er.MARK.exists(), True)
|
||||
code, _ = main_with("off")
|
||||
check("SteamVR on: off through vrcmd", live["value"], False)
|
||||
check("SteamVR on: mark gone", er.MARK.exists(), False)
|
||||
|
||||
# vrcmd that doesn't take it: on fails, no mark.
|
||||
er.vrcmd = lambda *args: "" if args[0] == "--set-settings-bool" else "steamvr.gazeDepEyeToHeadEnable=false\n"
|
||||
code, _ = main_with("on")
|
||||
check("vrcmd didn't take it: on fails", code, 1)
|
||||
check("vrcmd didn't take it: no mark", er.MARK.exists(), False)
|
||||
|
||||
# --- The camera rate, from eyetracking.txt ---
|
||||
check("no eyetracking.txt: unknown", er.camera_rate(), None)
|
||||
er.EYE_LOG.parent.mkdir(parents=True)
|
||||
head = "Sat Oct 10 2026 22:35:43.000000 [Info] - "
|
||||
er.EYE_LOG.write_text("\n".join([
|
||||
head + "eyetracking 2.18.2 startup with PID=1, config=x",
|
||||
head + "CStereoAdspCams: Set framerate 15.000000",
|
||||
head + "Eye tracking shutting down",
|
||||
head + "eyetracking 2.18.2 startup with PID=2, config=x",
|
||||
head + "Eye tracking started"]) + "\n")
|
||||
check("no rate set since the last start: 90", er.camera_rate(), 90.0)
|
||||
with er.EYE_LOG.open("a") as f:
|
||||
f.write(head + "CStereoAdspCams: Set framerate 15.000000\n")
|
||||
check("15 set since the last start", er.camera_rate(), 15.0)
|
||||
|
||||
print("eye-rate-test:", "FAILED " + ", ".join(failures) if failures else "all passed")
|
||||
raise SystemExit(1 if failures else 0)
|
||||
@@ -1,293 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of our own eye tracker's first calibration (gaze/gazecheck.py, "blind"): before
|
||||
it has a calibration, ft-eyes publishes no gaze, and the calibration must still open and take
|
||||
its dots. Until 2026-10-05 it couldn't: a fresh install that chose our tracker never calibrated.
|
||||
|
||||
Runs ft-gazed's Service with its sockets renamed and HOME in a temp folder (state and settings
|
||||
go there), a fake pointer helper (gaze mode on, headset worn), a fake ft-gaze (SteamVR sees both
|
||||
eyes; "own" is {"ok":0}, as with an uncalibrated ft-eyes), a fake ft-eyes control socket, and no
|
||||
panel (a stand-in process). The fake ft-eyes also answers late or not at all, as a slow one
|
||||
does: the service must keep running meanwhile (until 2026-10-09 it waited, up to 3 s a dot and
|
||||
10 s for the fit, and the helper's 3 s panel lease ran out). Nothing reaches the live gaze service, the pointer helper, ft-eyes,
|
||||
or SteamVR, so it's safe next to them.
|
||||
|
||||
gaze/test/first-calibration-test.py
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
HOME = tempfile.mkdtemp(prefix="ft-gaze-first-cal-test-")
|
||||
os.environ["HOME"] = HOME # before gazecal: its STATE, and frametop.conf, follow HOME
|
||||
|
||||
import importlib.machinery # noqa: E402
|
||||
import importlib.util # noqa: E402
|
||||
import json # noqa: E402
|
||||
import selectors # noqa: E402
|
||||
import shutil # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import sys # noqa: E402
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_first_cal_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL = f"\0{tag}_panel"
|
||||
gazed.EYES_SOCKET = gazecheck.EYES = f"\0{tag}_eyes"
|
||||
gazed.read_settings = lambda: ("own", "auto", "auto", 55.0)
|
||||
gazed.TrackedEye = lambda: lambda now=None: None # both eyes, whatever SteamVR's settings say
|
||||
DOTS = 4
|
||||
real_dots = gazecheck.check_dots
|
||||
gazecheck.check_dots = lambda kind, own: real_dots(kind, own)[:DOTS] # a short calibration
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, SteamVR sees both eyes, no gaze from ours.
|
||||
FAKE = os.path.join(HOME, "ft-gaze")
|
||||
with open(FAKE, "w") as f:
|
||||
f.write('''import json, os, select, sys, time
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
if not os.read(0, 4096):
|
||||
break
|
||||
print(json.dumps({"t": time.monotonic(), "src": {"mmap1": {"hy": 1.0, "hp": 2.0, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}, "own": {"ok": 0}}}), flush=True)
|
||||
''')
|
||||
SLEEPER = [sys.executable, "-c", "import sys; sys.stdin.read()"] # quits when its stdin closes
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
"""ft-gaze, straight from here instead of the dev container."""
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
|
||||
|
||||
def start_eyes(self):
|
||||
"""ft-eyes' process: a stand-in. Its control socket is the fake below."""
|
||||
self.eyes_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
|
||||
|
||||
def start_panel(self):
|
||||
self.panel_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.panel_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.panel_proc.stderr, selectors.EVENT_READ, "panel")
|
||||
|
||||
|
||||
gazed.Service.start_helper = start_helper
|
||||
gazed.Service.start_eyes = start_eyes
|
||||
gazecheck.Checks.start_panel = start_panel
|
||||
|
||||
# The fake ft-eyes: uncalibrated until calib-fit. "fail" answers the next calib-point with that;
|
||||
# "delay" holds calib-point's and calib-fit's answers that many seconds; "drop" leaves the next
|
||||
# calib-point unanswered.
|
||||
eyes_state = {"cal": None, "points": [], "fail": None, "delay": 0.0, "drop": False}
|
||||
eyes = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
eyes.bind(gazed.EYES_SOCKET)
|
||||
eyes.settimeout(0.2)
|
||||
|
||||
|
||||
def eyes_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = eyes.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
w = data.decode().split()
|
||||
if w[0] == "status":
|
||||
reply = json.dumps({"calibration": eyes_state["cal"], "calibrating": False, "dots": 0,
|
||||
"eyes": {"right": {"reseat": False}, "left": {"reseat": False}}})
|
||||
elif w[0] == "calib-start":
|
||||
eyes_state["points"] = []
|
||||
reply = "ok"
|
||||
elif w[0] == "calib-point":
|
||||
eyes_state["points"].append(tuple(map(float, w[1:5])))
|
||||
if eyes_state["drop"]:
|
||||
eyes_state["drop"] = False
|
||||
continue
|
||||
reply, eyes_state["fail"] = eyes_state["fail"] or "ok 50 50 1.00 1.00", None
|
||||
elif w[0] == "calib-fit":
|
||||
eyes_state["cal"] = {"made": "test", "dots": len(eyes_state["points"])}
|
||||
reply = f"ok {len(eyes_state['points'])} dots"
|
||||
else:
|
||||
reply = f"fail unknown command {w[0]}"
|
||||
if addr and w[0] in ("calib-point", "calib-fit") and eyes_state["delay"]:
|
||||
threading.Timer(eyes_state["delay"], send_late, (reply, addr)).start()
|
||||
elif addr:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
|
||||
|
||||
def send_late(reply, addr):
|
||||
try:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
except OSError:
|
||||
pass # the service gave up on it
|
||||
|
||||
|
||||
threading.Thread(target=eyes_answer, daemon=True).start()
|
||||
|
||||
helper_state = {"reply": "ok off worn", "heard": [], "calpanel": []} # calpanel: when "calpanel 1" came
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def helper_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(helper_state["reply"].encode(), addr)
|
||||
else:
|
||||
helper_state["heard"].append(data.decode())
|
||||
if data == b"calpanel 1":
|
||||
helper_state["calpanel"].append(time.monotonic())
|
||||
|
||||
|
||||
threading.Thread(target=helper_answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(3)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(65536).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
def check_state():
|
||||
return svc.checks.check or {}
|
||||
|
||||
|
||||
def take_dot(i):
|
||||
"""Wait for dot i to settle, then click: is it taken?"""
|
||||
wait(lambda: check_state().get("i") == i and not check_state().get("done_at"), 3)
|
||||
time.sleep(gazecheck.CHECK_SETTLE + gazecheck.CHECK_WINDOW + 0.1)
|
||||
before = check_state().get("captured", 0)
|
||||
ask("calaccept")
|
||||
return wait(lambda: check_state().get("captured", 0) > before, 2)
|
||||
|
||||
|
||||
check("gaze mode off, Gaze page open (wake): ours runs, uncalibrated", ask("wake 60"), "ok")
|
||||
check("the service knows ours has no calibration", wait(lambda: svc.checks.calibrated() is False, 6), True)
|
||||
# ft-eyes' status can come before the fake ft-gaze's first sample, and without one the refusal
|
||||
# below is "the headset is off" instead (failed about 1 run in 4 until 2026-10-06).
|
||||
check("SteamVR sees the eyes (the fake ft-gaze is sending)", wait(svc.checks.eyes_seen, 6), True)
|
||||
check("a quick check is refused while ours has no calibration", svc.checks.start("quick", "test"),
|
||||
"error our tracker isn't calibrated yet: use Calibrate")
|
||||
helper_state["reply"] = "ok on worn"
|
||||
check("gaze mode on: the calibration opens by itself", wait(lambda: check_state().get("kind") == "full", 6), True)
|
||||
check("it runs blind (no gaze from ours yet)", check_state().get("blind"), True)
|
||||
check("nothing says it can't open", any("can't open" in m for m in logs), False)
|
||||
# Live 2026-10-05: turning gaze mode on woke our tracker, and the calibration opened before
|
||||
# DON_DELAY of eyes had passed, so "the headset went on" came while it was open. That re-armed
|
||||
# it, and a second calibration opened as soon as the first ended.
|
||||
svc.checks.away, svc.checks.back_since = True, None
|
||||
|
||||
check("dot 1: a click takes it", take_dot(0), True)
|
||||
check("the headset went on while it was open",
|
||||
wait(lambda: not svc.checks.away, gazecheck.DON_DELAY + 2) and bool(svc.checks.check), True)
|
||||
t0, t1, yaw, pitch = eyes_state["points"][0]
|
||||
dot = gazecheck.check_dots("full", True)[0]
|
||||
check("its window is the look up to the click (about CHECK_WINDOW)",
|
||||
abs((t1 - t0) - gazecheck.CHECK_WINDOW) < 0.15, True)
|
||||
check("ft-eyes got that dot's direction", (round(yaw, 3), round(pitch, 3)), (round(dot[0], 3), round(dot[1], 3)))
|
||||
|
||||
eyes_state["fail"] = "fail the left eye was seen in only 3 frames"
|
||||
wait(lambda: check_state().get("i") == 1 and not check_state().get("done_at"), 3)
|
||||
time.sleep(gazecheck.CHECK_SETTLE + gazecheck.CHECK_WINDOW + 0.1)
|
||||
ask("calaccept")
|
||||
check("ft-eyes refusing a dot: its reason reaches the panel's note",
|
||||
wait(lambda: "left eye in only 3 frames" in check_state().get("note", ""), 2), True)
|
||||
check("dot 2, second try: taken", take_dot(1), True)
|
||||
|
||||
# A slow ft-eyes (2.5 s): the service goes on meanwhile, and a second click is ignored.
|
||||
eyes_state["delay"] = 2.5
|
||||
wait(lambda: check_state().get("i") == 2 and not check_state().get("done_at"), 3)
|
||||
time.sleep(gazecheck.CHECK_SETTLE + gazecheck.CHECK_WINDOW + 0.1)
|
||||
before = len(eyes_state["points"])
|
||||
ask("calaccept")
|
||||
check("dot 3, ft-eyes slow: the service waits for it", wait(lambda: svc.checks.asking is not None, 2), True)
|
||||
t = time.monotonic()
|
||||
ask("status")
|
||||
check("the service still answers meanwhile", time.monotonic() - t < 0.5, True)
|
||||
ask("calaccept")
|
||||
check("dot 3: taken once ft-eyes answers", wait(lambda: check_state().get("captured") == 3, 4), True)
|
||||
check("the helper's panel lease was renewed while ft-eyes took its time",
|
||||
sum(t < at < t + eyes_state["delay"] for at in helper_state["calpanel"]) >= 2, True)
|
||||
check("and the calibration is still open (a blocked service took that as the headset off)",
|
||||
check_state().get("kind"), "full")
|
||||
check("the second click asked ft-eyes nothing", len(eyes_state["points"]), before + 1)
|
||||
eyes_state["delay"] = 0.0
|
||||
|
||||
# No answer: the dot isn't taken, after calib-point's 3 s.
|
||||
eyes_state["drop"] = True
|
||||
wait(lambda: check_state().get("i") == 3 and not check_state().get("done_at"), 3)
|
||||
time.sleep(gazecheck.CHECK_SETTLE + gazecheck.CHECK_WINDOW + 0.1)
|
||||
ask("calaccept")
|
||||
check("dot 4, no answer from ft-eyes: not taken, and the panel says so",
|
||||
wait(lambda: "didn't answer" in check_state().get("note", ""), 5), True)
|
||||
eyes_state["delay"] = 2.5 # the fit too
|
||||
check("dot 4, second try: taken", take_dot(3) or wait(lambda: check_state().get("captured") == 4, 4), True)
|
||||
|
||||
check("while ours fits, the panel stays", wait(lambda: check_state().get("fitting") is True, 3), True)
|
||||
ask("calquit")
|
||||
check("and a right click doesn't close it", check_state().get("kind"), "full")
|
||||
check("all dots: ours fits its calibration (calib-fit)",
|
||||
wait(lambda: eyes_state["cal"] is not None and not svc.checks.check, 5), True)
|
||||
gaps = [b - a for a, b in zip(helper_state["calpanel"], helper_state["calpanel"][1:])]
|
||||
check("the helper's panel lease (3 s) never ran out", bool(gaps) and max(gaps) < 3.0, True)
|
||||
check("the service sees it calibrated", wait(lambda: svc.checks.calibrated() is True, 4), True)
|
||||
check("and gaze mode stays on", "gaze off" in helper_state["heard"], False)
|
||||
check("and no second calibration opens", wait(lambda: svc.checks.check is not None, 3), False)
|
||||
check("one calibration in the log", sum(m.startswith("full check:") for m in logs), 1)
|
||||
|
||||
print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True)
|
||||
svc.running = False
|
||||
time.sleep(0.7)
|
||||
shutil.rmtree(HOME, ignore_errors=True)
|
||||
os._exit(1 if failures else 0)
|
||||
@@ -1,247 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of the fit coach in the headset fit check (gaze/gazecheck.py, "fit"): with
|
||||
SteamVR's tracker in use, the check keeps ft-eyes running (the lease), asks it for the last
|
||||
second of pupil finds ("fit"), judges them only while the gaze is straight ahead, and puts the
|
||||
coach's hint first in the panel's hints.
|
||||
|
||||
The harness is first-calibration-test.py's: ft-gazed's Service with its sockets renamed, HOME in
|
||||
a temp folder, a fake pointer helper, a fake ft-gaze (its gaze from a file the test writes), a
|
||||
fake ft-eyes control socket (its finds from the shipped reference, offset as the test says),
|
||||
and the panel's commands recorded instead of sent. Safe next to the live gaze service.
|
||||
|
||||
gaze/test/fit-coach-check-test.py
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
HOME = tempfile.mkdtemp(prefix="ft-gaze-fit-coach-test-")
|
||||
os.environ["HOME"] = HOME # before gazecal: its STATE, and frametop.conf, follow HOME
|
||||
|
||||
import importlib.machinery # noqa: E402
|
||||
import importlib.util # noqa: E402
|
||||
import json # noqa: E402
|
||||
import selectors # noqa: E402
|
||||
import shutil # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import sys # noqa: E402
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import fitcoach # noqa: E402
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_fit_coach_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL = f"\0{tag}_panel"
|
||||
gazed.EYES_SOCKET = gazecheck.EYES = f"\0{tag}_eyes"
|
||||
gazed.read_settings = lambda: ("steam", "auto", "auto", 55.0) # SteamVR's tracker in use
|
||||
gazed.own_installed = lambda: True
|
||||
gazed.TrackedEye = lambda: lambda now=None: None
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, SteamVR sees both eyes, its gaze from LOOK.
|
||||
LOOK = os.path.join(HOME, "look.json")
|
||||
FAKE = os.path.join(HOME, "ft-gaze")
|
||||
|
||||
|
||||
def look(yaw, pitch):
|
||||
with open(LOOK + ".new", "w") as f:
|
||||
json.dump([yaw, pitch], f)
|
||||
os.replace(LOOK + ".new", LOOK)
|
||||
|
||||
|
||||
look(0.0, 0.0)
|
||||
with open(FAKE, "w") as f:
|
||||
f.write(f'''import json, os, select, sys, time
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
if not os.read(0, 4096):
|
||||
break
|
||||
try:
|
||||
hy, hp = json.load(open({LOOK!r}))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
print(json.dumps({{"t": time.monotonic(), "src": {{"mmap1": {{"hy": hy, "hp": hp, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}}}}}}), flush=True)
|
||||
''')
|
||||
SLEEPER = [sys.executable, "-c", "import sys; sys.stdin.read()"] # quits when its stdin closes
|
||||
started = {"eyes": 0}
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
|
||||
|
||||
def start_eyes(self):
|
||||
started["eyes"] += 1
|
||||
self.eyes_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
|
||||
|
||||
def start_panel(self):
|
||||
self.panel_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.panel_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.panel_proc.stderr, selectors.EVENT_READ, "panel")
|
||||
|
||||
|
||||
panel = []
|
||||
gazed.Service.start_helper = start_helper
|
||||
gazed.Service.start_eyes = start_eyes
|
||||
gazecheck.Checks.start_panel = start_panel
|
||||
gazecheck.Checks.to_panel = lambda self, command: panel.append(command)
|
||||
|
||||
# The fake ft-eyes: "fit" answers with finds where the shipped reference puts eyes offset by
|
||||
# eyes_state["offset"] (yaw, pitch degrees, both eyes alike).
|
||||
REF = fitcoach.Reference.load()
|
||||
eyes_state = {"offset": (0.0, 0.0), "asked": 0}
|
||||
eyes = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
eyes.bind(gazed.EYES_SOCKET)
|
||||
eyes.settimeout(0.2)
|
||||
|
||||
|
||||
def finds():
|
||||
yaw, pitch = eyes_state["offset"]
|
||||
out = {}
|
||||
for cam in (0, 1):
|
||||
(cx, cy), ((a, b), (c, d)) = REF.eyes[cam]["centre"], REF.eyes[cam]["jacobian"]
|
||||
out[cam] = {"n": 90, "finds": 90, "found": 1.0, "x": cx + a * yaw + b * pitch, "y": cy + c * yaw + d * pitch,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0}
|
||||
return out
|
||||
|
||||
|
||||
def eyes_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = eyes.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
w = data.decode().split()
|
||||
if w[0] == "status":
|
||||
reply = json.dumps({"calibration": None, "calibrating": False, "dots": 0,
|
||||
"eyes": {"right": {"reseat": False}, "left": {"reseat": False}}})
|
||||
elif w[0] == "fit":
|
||||
eyes_state["asked"] += 1
|
||||
reply = json.dumps(finds())
|
||||
else:
|
||||
reply = f"fail unknown command {w[0]}"
|
||||
if addr:
|
||||
try:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
except OSError:
|
||||
pass # the service gave up on it (as ft-eyes' serve() does)
|
||||
|
||||
|
||||
threading.Thread(target=eyes_answer, daemon=True).start()
|
||||
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def helper_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(b"ok off worn", addr)
|
||||
|
||||
|
||||
threading.Thread(target=helper_answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(3)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(65536).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
def coach_line():
|
||||
"""The first hint line the panel got last, if it's the coach's."""
|
||||
for cmd in reversed(panel):
|
||||
if cmd.startswith("hints "):
|
||||
first = cmd[len("hints "):].split("|")[0]
|
||||
return first if first.startswith("Headset position") else None
|
||||
return None
|
||||
|
||||
|
||||
ask("wake 60")
|
||||
check("the tracker sends", wait(lambda: time.monotonic() - svc.checks.sample_at < 1, 6), True)
|
||||
check("SteamVR's tracker in use: ft-eyes isn't running", svc.eyes_proc is None, True)
|
||||
check("fitcheck is taken", ask("fitcheck").startswith("ok"), True)
|
||||
check("the fit check opens", wait(lambda: (svc.checks.check or {}).get("kind") == "fit", 6), True)
|
||||
check("the fit check starts ft-eyes for the coach", wait(lambda: svc.eyes_proc is not None, 3), True)
|
||||
check("and asks it for finds", wait(lambda: eyes_state["asked"] > 0, 3), True)
|
||||
check("looking ahead, well fitted: good",
|
||||
wait(lambda: (coach_line() or "").startswith("Headset position: good."), 4), True)
|
||||
check("marked a first guess (the reference is from one face)", "first guess" in (coach_line() or ""), True)
|
||||
|
||||
eyes_state["offset"] = (0.0, 12.0) # both eyes appear to look up: the headset is low
|
||||
check("the headset low: push it up",
|
||||
wait(lambda: "Push the headset up" in (coach_line() or ""), 4), True)
|
||||
check("it's the first hint line", (coach_line() or "").startswith("Headset position: Push"), True)
|
||||
|
||||
look(25.0, 0.0) # looking away: the pupils move with the gaze, so no new reading
|
||||
check("looking away: the last centred reading, marked so",
|
||||
wait(lambda: "when you last looked straight ahead" in (coach_line() or ""), 4), True)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
time.sleep(1.5)
|
||||
check("and it isn't judged meanwhile", "Push the headset up" in (coach_line() or ""), True)
|
||||
look(0.0, 0.0)
|
||||
check("looking ahead again: judged again", wait(lambda: (coach_line() or "").startswith("Headset position: good."), 4), True)
|
||||
|
||||
ask("calquit")
|
||||
check("closing logs the last reading", wait(lambda: any(m.startswith("fit coach, last reading: ok") for m in logs), 2), True)
|
||||
check("the check closed", wait(lambda: svc.checks.check is None, 2), True)
|
||||
check("ft-eyes stops once its lease runs out",
|
||||
wait(lambda: svc.eyes_proc is None, gazecheck.COACH_LEASE + 4), True)
|
||||
|
||||
print("all passed" if not failures else f"{len(failures)} failed: {failures}", flush=True)
|
||||
shutil.rmtree(HOME, ignore_errors=True)
|
||||
os._exit(1 if failures else 0)
|
||||
@@ -1,316 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of the fit watch (gaze/gazecheck.py, GAZE_FIT_WATCH): the fit check keeping a
|
||||
good fit as yours, the put-on check in the quick check that opens when the headset goes on
|
||||
(holds the dot, says what to move, lets go once it's back or on a click), and the slip note
|
||||
with our tracker (ft-eyes' glint jump past the threshold for SLIP_HOLD; gone with the slip;
|
||||
not again within NOTE_COOLDOWN).
|
||||
|
||||
fit-coach-check-test.py's harness: ft-gazed's Service with its sockets renamed, HOME in a temp
|
||||
folder, a fake pointer helper, a fake ft-gaze, a fake ft-eyes control socket (its finds from
|
||||
the shipped reference, offset as the test says, and its status with the glint jump), and the
|
||||
panel's commands recorded instead of sent. Safe next to the live gaze service.
|
||||
|
||||
gaze/test/fit-watch-test.py
|
||||
"""
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
HOME = tempfile.mkdtemp(prefix="ft-gaze-fit-watch-test-")
|
||||
os.environ["HOME"] = HOME # before gazecal: its STATE, and frametop.conf, follow HOME
|
||||
|
||||
import importlib.machinery # noqa: E402
|
||||
import importlib.util # noqa: E402
|
||||
import json # noqa: E402
|
||||
import selectors # noqa: E402
|
||||
import shutil # noqa: E402
|
||||
import socket # noqa: E402
|
||||
import subprocess # noqa: E402
|
||||
import sys # noqa: E402
|
||||
import threading # noqa: E402
|
||||
import time # noqa: E402
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import fitcoach # noqa: E402
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_fit_watch_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL = f"\0{tag}_panel"
|
||||
gazed.EYES_SOCKET = gazecheck.EYES = f"\0{tag}_eyes"
|
||||
SETTINGS = {"tracker": "steam"} # SteamVR's tracker in use first, then ours
|
||||
gazed.read_settings = lambda: (SETTINGS["tracker"], "auto", "auto", 55.0)
|
||||
gazed.own_installed = lambda: True
|
||||
gazed.TrackedEye = lambda: lambda now=None: None
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, SteamVR sees both eyes, its gaze from LOOK.
|
||||
LOOK = os.path.join(HOME, "look.json")
|
||||
FAKE = os.path.join(HOME, "ft-gaze")
|
||||
|
||||
|
||||
def look(yaw, pitch):
|
||||
with open(LOOK + ".new", "w") as f:
|
||||
json.dump([yaw, pitch], f)
|
||||
os.replace(LOOK + ".new", LOOK)
|
||||
|
||||
|
||||
look(0.0, 0.0)
|
||||
with open(FAKE, "w") as f:
|
||||
f.write(f'''import json, os, select, sys, time
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
if not os.read(0, 4096):
|
||||
break
|
||||
try:
|
||||
hy, hp = json.load(open({LOOK!r}))
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
print(json.dumps({{"t": time.monotonic(), "src": {{"mmap1": {{"hy": hy, "hp": hp, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}}}}}}), flush=True)
|
||||
''')
|
||||
SLEEPER = [sys.executable, "-c", "import sys; sys.stdin.read()"] # quits when its stdin closes
|
||||
started = {"eyes": 0}
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
|
||||
|
||||
def start_eyes(self):
|
||||
started["eyes"] += 1
|
||||
self.eyes_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.eyes_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.eyes_proc.stderr, selectors.EVENT_READ, "eyes")
|
||||
|
||||
|
||||
def start_panel(self):
|
||||
self.panel_proc = subprocess.Popen(SLEEPER, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
os.set_blocking(self.panel_proc.stderr.fileno(), False)
|
||||
self.sel.register(self.panel_proc.stderr, selectors.EVENT_READ, "panel")
|
||||
|
||||
|
||||
panel = []
|
||||
gazed.Service.start_helper = start_helper
|
||||
gazed.Service.start_eyes = start_eyes
|
||||
gazecheck.Checks.start_panel = start_panel
|
||||
gazecheck.Checks.to_panel = lambda self, command: panel.append(command)
|
||||
|
||||
# The fake ft-eyes: "fit" answers with finds where the shipped reference puts eyes offset by
|
||||
# eyes_state["offset"] (yaw, pitch degrees, both eyes alike).
|
||||
REF = fitcoach.Reference.load()
|
||||
eyes_state = {"offset": (0.0, 0.0), "jump": (0.0, 0.0), "asked": 0}
|
||||
eyes = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
eyes.bind(gazed.EYES_SOCKET)
|
||||
eyes.settimeout(0.2)
|
||||
|
||||
|
||||
def finds():
|
||||
yaw, pitch = eyes_state["offset"]
|
||||
out = {}
|
||||
for cam in (0, 1):
|
||||
(cx, cy), ((a, b), (c, d)) = REF.eyes[cam]["centre"], REF.eyes[cam]["jacobian"]
|
||||
out[cam] = {"n": 90, "finds": 90, "found": 1.0, "x": cx + a * yaw + b * pitch, "y": cy + c * yaw + d * pitch,
|
||||
"a": 28.0, "spacing": 15.0, "extra": 0.0}
|
||||
return out
|
||||
|
||||
|
||||
def eyes_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = eyes.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
w = data.decode().split()
|
||||
if w[0] == "status":
|
||||
yaw, pitch = eyes_state["jump"] # the slip, as both pupils' move in degrees
|
||||
jump = {}
|
||||
for cam, name in ((0, "right"), (1, "left")):
|
||||
(a, b), (c, d) = REF.eyes[cam]["jacobian"]
|
||||
jump[name] = {"reseat": False, "jump": bool(yaw or pitch),
|
||||
"jump_px": [a * yaw + b * pitch, c * yaw + d * pitch]}
|
||||
reply = json.dumps({"calibration": {"dots": 27}, "calibrating": False, "dots": 27, "eyes": jump})
|
||||
elif w[0] == "fit":
|
||||
eyes_state["asked"] += 1
|
||||
reply = json.dumps(finds())
|
||||
else:
|
||||
reply = f"fail unknown command {w[0]}"
|
||||
if addr:
|
||||
try:
|
||||
eyes.sendto(reply.encode(), addr)
|
||||
except OSError:
|
||||
pass # the service gave up on it (as ft-eyes' serve() does)
|
||||
|
||||
|
||||
threading.Thread(target=eyes_answer, daemon=True).start()
|
||||
|
||||
HELPER = {"gaze": False}
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def helper_answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(b"ok on worn" if HELPER["gaze"] else b"ok off worn", addr)
|
||||
|
||||
|
||||
threading.Thread(target=helper_answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(3)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(65536).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
def since(n):
|
||||
"""The panel's commands from n on."""
|
||||
return panel[n:]
|
||||
|
||||
|
||||
def quick(label):
|
||||
"""Opens the quick check as the headset going on does; returns where its panel commands start."""
|
||||
n = len(panel)
|
||||
check(label, svc.checks.start("quick", gazecheck.PUTON_REASON), "ok")
|
||||
return n
|
||||
|
||||
|
||||
def puton():
|
||||
return ((svc.checks.check or {}).get("puton") or {}).get("state")
|
||||
|
||||
|
||||
def closed_with_capture(seconds=8):
|
||||
return wait(lambda: svc.checks.check is None and any(m.startswith("quick check done") for m in logs[mark:]), seconds)
|
||||
|
||||
|
||||
ask("wake 60")
|
||||
check("the tracker sends", wait(lambda: time.monotonic() - svc.checks.sample_at < 1, 6), True)
|
||||
check("the eyes count as seen", wait(lambda: svc.checks.eyes_seen(), 3), True)
|
||||
|
||||
# The fit watch off (the default): the quick check is as it was.
|
||||
mark = len(logs)
|
||||
quick("fit watch off: a quick check opens")
|
||||
check(" without a put-on check", puton(), None)
|
||||
check(" and takes its dot", closed_with_capture(), True)
|
||||
|
||||
# A good fit in the fit check is kept as yours.
|
||||
check("no saved fit yet", os.path.exists(fitcoach.BASELINE), False)
|
||||
check("fitcheck is taken", ask("fitcheck").startswith("ok"), True)
|
||||
check("the fit check opens", wait(lambda: (svc.checks.check or {}).get("kind") == "fit", 6), True)
|
||||
check("it reads a good fit", wait(lambda: ((svc.checks.check or {}).get("coach") or {}).get("good") is not None, 6), True)
|
||||
ask("calquit")
|
||||
check("closing it saves the fit as yours", wait(lambda: os.path.exists(fitcoach.BASELINE), 2), True)
|
||||
check("with both eyes", sorted(json.load(open(fitcoach.BASELINE))["eyes"]), ["0", "1"])
|
||||
|
||||
gazed.CONF.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(gazed.CONF, "w") as f:
|
||||
f.write("GAZE_FIT_WATCH=1\n")
|
||||
|
||||
# Put on as saved: the dot is held for the reading, then taken.
|
||||
mark = len(logs)
|
||||
quick("put on as saved: the quick check opens")
|
||||
check(" reading the fit first", puton(), "reading")
|
||||
check(" sits as saved", wait(lambda: any("sits as your saved fit" in m for m in logs[mark:]), 5), True)
|
||||
check(" then the dot is taken", closed_with_capture(), True)
|
||||
|
||||
# Put on low: it says to push it up, holds the dot past the quick check's timeout, lets go once back.
|
||||
eyes_state["offset"] = (0.0, 12.0)
|
||||
mark = len(logs)
|
||||
n = quick("put on low: the quick check opens")
|
||||
check(" the headset moved", wait(lambda: puton() == "moved", 5), True)
|
||||
check(" the panel widens", "show five" in since(n), True)
|
||||
check(" and says to push it up", wait(lambda: any(c.startswith("text Push the headset up") for c in since(n)), 2), True)
|
||||
time.sleep(gazecheck.QUICK_TIMEOUT + 1)
|
||||
check(" the dot waits past the quick check's timeout", (svc.checks.check or {}).get("captured"), 0)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
check(" pushed back: let go", wait(lambda: any("is back as your saved fit" in m for m in logs[mark:]), 5), True)
|
||||
check(" saying so", "note Good: it's back where it was" in since(n), True)
|
||||
check(" then the dot is taken", closed_with_capture(), True)
|
||||
|
||||
# Put on low, then a click: taken as it sits.
|
||||
eyes_state["offset"] = (0.0, 12.0)
|
||||
mark = len(logs)
|
||||
quick("put on low again")
|
||||
check(" the headset moved", wait(lambda: puton() == "moved", 5), True)
|
||||
ask("calaccept")
|
||||
check(" a click lets go", wait(lambda: any("a click" in m for m in logs[mark:]), 3), True)
|
||||
check(" and takes the dot", closed_with_capture(), True)
|
||||
eyes_state["offset"] = (0.0, 0.0)
|
||||
|
||||
# The slip note: our tracker, gaze mode on.
|
||||
SETTINGS["tracker"] = "own"
|
||||
HELPER["gaze"] = True
|
||||
with open(gazed.CONF, "w") as f:
|
||||
f.write("GAZE_FIT_WATCH=1\n# our tracker now\n")
|
||||
check("our tracker in use", wait(lambda: svc.kind == "own" and svc.eyes_proc is not None, 6), True)
|
||||
check("gaze mode on", wait(lambda: svc.checks.gaze_on is True, 4), True)
|
||||
check("ft-eyes' status comes", wait(lambda: time.monotonic() - svc.own_at < 2, 4), True)
|
||||
n = len(panel)
|
||||
time.sleep(2)
|
||||
check("no slip: no note", "show note" in since(n), False)
|
||||
eyes_state["jump"] = (0.0, -12.0) # the pupils dropped: the headset rode up
|
||||
start = time.monotonic()
|
||||
check("a slip: the note", wait(lambda: "show note" in since(n), gazecheck.SLIP_HOLD + 4), True)
|
||||
check(" not before SLIP_HOLD", time.monotonic() - start >= gazecheck.SLIP_HOLD, True)
|
||||
check(" saying so", "title Your headset slipped" in since(n), True)
|
||||
check(" and which way", any(c.startswith("text Let the headset down") for c in since(n)), True)
|
||||
check(" no pointer hiding for it", any(c == "calpanel 1" for c in since(n)), False)
|
||||
n = len(panel)
|
||||
eyes_state["jump"] = (0.0, 0.0)
|
||||
check("the slip gone: the note goes", wait(lambda: "hide" in since(n), 4), True)
|
||||
n = len(panel)
|
||||
eyes_state["jump"] = (0.0, -12.0)
|
||||
time.sleep(gazecheck.SLIP_HOLD + 3)
|
||||
check("again within NOTE_COOLDOWN: no note", "show note" in since(n), False)
|
||||
eyes_state["jump"] = (0.0, 0.0)
|
||||
|
||||
print("all passed" if not failures else f"{len(failures)} failed: {failures}", flush=True)
|
||||
shutil.rmtree(HOME, ignore_errors=True)
|
||||
os._exit(1 if failures else 0)
|
||||
@@ -1,238 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The fit coach (gaze/fitcoach.py) on made-up readings: the reference fit, which way each hint
|
||||
points with both cameras' orientations, the state's hysteresis, lost eyes, and a baseline.
|
||||
|
||||
python3 gaze/test/fitcoach-test.py
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, os.path.dirname(HERE))
|
||||
import fitcoach # noqa: E402
|
||||
|
||||
# About the maintainer's headset's (gaze/fit-reference.json): camera 1's image is turned 180
|
||||
# degrees from camera 0's, so its signs are the other way.
|
||||
J = {0: [[1.27, -0.42], [-0.48, -1.34]], 1: [[-1.34, -0.53], [-0.48, 1.48]]}
|
||||
CENTRE = {0: [341.0, 221.4], 1: [334.7, 173.5]}
|
||||
|
||||
|
||||
def ref(follow=None, faces=1):
|
||||
return fitcoach.Reference({c: {"centre": CENTRE[c], "jacobian": J[c]} for c in (0, 1)}, ipd=0.065,
|
||||
faces=faces, cameras_follow_lenses=follow)
|
||||
|
||||
|
||||
def pupil(cam, yaw, pitch, centre=None):
|
||||
"""Where camera cam sees the pupil when the eye appears to look (yaw, pitch) degrees."""
|
||||
cx, cy = centre or CENTRE[cam]
|
||||
(a, b), (c, d) = J[cam]
|
||||
return cx + a * yaw + b * pitch, cy + c * yaw + d * pitch
|
||||
|
||||
|
||||
def summary(right=(0, 0), left=(0, 0), found=(1.0, 1.0), spacing=(14.0, 16.0), extra=(0.0, 0.0)):
|
||||
out = {}
|
||||
for cam, (yaw, pitch) in ((0, right), (1, left)):
|
||||
x, y = pupil(cam, yaw, pitch)
|
||||
out[cam] = {"n": 90, "finds": int(90 * found[cam]), "found": found[cam], "x": x, "y": y, "a": 28.0,
|
||||
"spacing": spacing[cam], "extra": extra[cam]}
|
||||
return out
|
||||
|
||||
|
||||
def keys(reading):
|
||||
return [h["key"] for h in reading["hints"]]
|
||||
|
||||
|
||||
class ReferenceTest(unittest.TestCase):
|
||||
def test_fit_recovers_both_orientations(self):
|
||||
rng = random.Random(1)
|
||||
rows = []
|
||||
for made in ("a", "b"):
|
||||
for _ in range(27):
|
||||
yaw, pitch = rng.uniform(-20, 20), rng.uniform(-15, 15)
|
||||
rows.append({"made": made, "truth": [yaw, pitch],
|
||||
"eye": {str(c): {"pupil": [v + rng.gauss(0, 0.5) for v in pupil(c, yaw, pitch)]} for c in (0, 1)}})
|
||||
# A third calibration that fits badly is left out.
|
||||
for _ in range(27):
|
||||
yaw, pitch = rng.uniform(-20, 20), rng.uniform(-15, 15)
|
||||
rows.append({"made": "bad", "truth": [yaw, pitch],
|
||||
"eye": {str(c): {"pupil": [v + rng.gauss(0, 15) for v in pupil(c, yaw, pitch)]} for c in (0, 1)}})
|
||||
r = fitcoach.Reference.from_dots(rows)
|
||||
for c in (0, 1):
|
||||
self.assertEqual(r.eyes[c]["fits"], 2)
|
||||
for i in range(2):
|
||||
self.assertAlmostEqual(r.eyes[c]["centre"][i], CENTRE[c][i], delta=0.5)
|
||||
for j in range(2):
|
||||
self.assertAlmostEqual(r.eyes[c]["jacobian"][i][j], J[c][i][j], delta=0.05)
|
||||
# degrees() undoes the Jacobian.
|
||||
x, y = pupil(1, 7.0, -3.0)
|
||||
yaw, pitch = r.degrees(1, x - r.eyes[1]["centre"][0], y - r.eyes[1]["centre"][1])
|
||||
self.assertAlmostEqual(yaw, 7.0, delta=0.2)
|
||||
self.assertAlmostEqual(pitch, -3.0, delta=0.2)
|
||||
|
||||
def test_json_round_trip(self):
|
||||
r = ref(follow=True, faces=4)
|
||||
back = fitcoach.Reference.from_json(r.to_json())
|
||||
self.assertEqual(back.eyes[1]["jacobian"], J[1])
|
||||
self.assertTrue(back.cameras_follow_lenses)
|
||||
self.assertFalse(back.provisional)
|
||||
self.assertTrue(ref().provisional)
|
||||
|
||||
def test_the_shipped_reference_loads(self):
|
||||
r = fitcoach.Reference.load()
|
||||
self.assertIsNotNone(r, fitcoach.REFERENCE)
|
||||
self.assertEqual(sorted(r.eyes), [0, 1])
|
||||
self.assertTrue(r.provisional)
|
||||
|
||||
|
||||
class AssessTest(unittest.TestCase):
|
||||
def test_centred_is_ok(self):
|
||||
r = fitcoach.assess(summary(), ref())
|
||||
self.assertEqual(r["state"], "ok")
|
||||
self.assertEqual(r["hints"], [])
|
||||
self.assertTrue(r["provisional"])
|
||||
|
||||
def test_eyes_high_means_push_up(self):
|
||||
r = fitcoach.assess(summary(right=(0, 12), left=(0, 12)), ref())
|
||||
self.assertEqual(r["state"], "adjust")
|
||||
self.assertEqual(keys(r), ["up"])
|
||||
r = fitcoach.assess(summary(right=(0, -12), left=(0, -12)), ref())
|
||||
self.assertEqual(keys(r), ["down"])
|
||||
|
||||
def test_eyes_left_means_slide_left(self):
|
||||
r = fitcoach.assess(summary(right=(11, 0), left=(11, 0)), ref())
|
||||
self.assertEqual(keys(r), ["left"])
|
||||
self.assertAlmostEqual(r["terms"]["left"], 11, delta=0.1)
|
||||
r = fitcoach.assess(summary(right=(-11, 0), left=(-11, 0)), ref())
|
||||
self.assertEqual(keys(r), ["right"])
|
||||
|
||||
def test_ipd_only_advised_once_the_cameras_are_known_to_follow_the_lenses(self):
|
||||
apart = summary(right=(-10, 0), left=(10, 0)) # each eye outward of its lens
|
||||
r = fitcoach.assess(apart, ref(follow=None))
|
||||
self.assertAlmostEqual(r["terms"]["ipd"], 10, delta=0.1)
|
||||
self.assertNotIn("ipd-wider", keys(r))
|
||||
self.assertFalse(r["ipd_advice"])
|
||||
self.assertEqual(r["state"], "ok", "the common terms are centred")
|
||||
r = fitcoach.assess(apart, ref(follow=True))
|
||||
self.assertEqual(keys(r), ["ipd-wider"])
|
||||
self.assertEqual(r["state"], "adjust")
|
||||
r = fitcoach.assess(summary(right=(10, 0), left=(-10, 0)), ref(follow=True))
|
||||
self.assertEqual(keys(r), ["ipd-narrower"])
|
||||
|
||||
def test_tilt(self):
|
||||
r = fitcoach.assess(summary(right=(0, -5), left=(0, 5)), ref())
|
||||
self.assertEqual(keys(r), ["tilt-left-up"])
|
||||
r = fitcoach.assess(summary(right=(0, 5), left=(0, -5)), ref())
|
||||
self.assertEqual(keys(r), ["tilt-right-up"])
|
||||
|
||||
def test_hysteresis_between_ok_and_adjust(self):
|
||||
mid = summary(right=(0, 6.5), left=(0, 6.5))
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "ok"})["state"], "ok")
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "adjust"})["state"], "adjust")
|
||||
self.assertEqual(fitcoach.assess(mid, ref())["state"], "adjust")
|
||||
self.assertEqual(fitcoach.assess(mid, ref(), previous={"state": "lost"})["state"], "adjust")
|
||||
# Adjust without a term past ADJUST_DEG: the biggest one's hint, so there's something to do.
|
||||
self.assertEqual(keys(fitcoach.assess(mid, ref())), ["up"])
|
||||
self.assertEqual(keys(fitcoach.assess(mid, ref(), previous={"state": "ok"})), [])
|
||||
|
||||
def test_a_lost_eye_comes_first(self):
|
||||
r = fitcoach.assess(summary(right=(0, 12), left=(0, 12), found=(1.0, 0.3)), ref())
|
||||
self.assertEqual(r["state"], "lost")
|
||||
self.assertEqual(keys(r)[0], "lost")
|
||||
self.assertIn("left eye", r["hints"][0]["text"])
|
||||
|
||||
def test_waiting_without_frames(self):
|
||||
r = fitcoach.assess({0: {"n": 0}, 1: {"n": 0}}, ref())
|
||||
self.assertEqual(r["state"], "waiting")
|
||||
self.assertEqual(fitcoach.assess({}, ref())["state"], "waiting")
|
||||
|
||||
def test_reflections(self):
|
||||
r = fitcoach.assess(summary(extra=(0.0, 0.8)), ref())
|
||||
self.assertEqual(keys(r), ["reflections"])
|
||||
self.assertIn("left", r["hints"][0]["text"])
|
||||
|
||||
|
||||
class BaselineTest(unittest.TestCase):
|
||||
def test_against_the_baseline(self):
|
||||
good = summary(right=(4, -3), left=(4, -3)) # this face's good fit isn't the reference's
|
||||
base = fitcoach.make_baseline(good, ipd=0.065)
|
||||
r = fitcoach.assess(good, ref(), ipd=0.065, baseline=base)
|
||||
self.assertEqual((r["state"], r["against"]), ("ok", "baseline"))
|
||||
self.assertFalse(fitcoach.moved(r))
|
||||
# Back on, 7 degrees lower on the face: moved, and which way to put it back.
|
||||
slipped = summary(right=(4, 4), left=(4, 4))
|
||||
r = fitcoach.assess(slipped, ref(), ipd=0.065, baseline=base)
|
||||
self.assertTrue(fitcoach.moved(r))
|
||||
self.assertAlmostEqual(r["terms"]["up"], 7, delta=0.1)
|
||||
|
||||
def test_eye_relief_and_ipd_change(self):
|
||||
base = fitcoach.make_baseline(summary(), ipd=0.065)
|
||||
near = summary(spacing=(14.0 * 1.12, 16.0 * 1.12))
|
||||
r = fitcoach.assess(near, ref(), ipd=0.0665, baseline=base)
|
||||
self.assertEqual(keys(r), ["ipd-changed", "nearer"])
|
||||
self.assertIn("65.0 mm, now 66.5 mm", r["hints"][0]["text"])
|
||||
self.assertTrue(fitcoach.moved(r))
|
||||
far = summary(spacing=(14.0 * 0.9, 16.0 * 0.9))
|
||||
self.assertEqual(keys(fitcoach.assess(far, ref(), ipd=0.065, baseline=base)), ["further"])
|
||||
|
||||
|
||||
class ObservationsTest(unittest.TestCase):
|
||||
def test_window_and_medians(self):
|
||||
obs = fitcoach.Observations(window=1.0)
|
||||
# A pair 14 px apart and four stray ones: MANY_GLINTS.
|
||||
glints = [(300.0, 200.0), (301.0, 214.0), (250.0, 120.0), (260.0, 120.0), (270.0, 120.0), (280.0, 120.0)]
|
||||
for i in range(60):
|
||||
t = i / 30
|
||||
obs.add(0, t, {"x": 300 + (i % 3), "y": 210.0, "a": 25.0, "glints": glints})
|
||||
obs.add(1, t, None if i % 2 else {"x": 320.0, "y": 170.0, "a": 30.0, "glints": []})
|
||||
s = obs.summary()
|
||||
self.assertEqual(s[0]["n"], 31) # the last second only
|
||||
self.assertEqual(s[0]["x"], 301)
|
||||
self.assertAlmostEqual(s[0]["spacing"], math.hypot(1, 14), places=5)
|
||||
self.assertEqual(s[0]["extra"], 1.0)
|
||||
self.assertAlmostEqual(s[1]["found"], 15 / 31, places=3)
|
||||
self.assertIsNone(s[1]["spacing"])
|
||||
|
||||
def test_glint_spacing_matches_eyes_pupil(self):
|
||||
try:
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(HERE), "tracker"))
|
||||
import eyes_pupil
|
||||
except ImportError:
|
||||
self.skipTest("no numpy or OpenCV here")
|
||||
rng = random.Random(3)
|
||||
for _ in range(200):
|
||||
p = {"x": 300.0, "y": 200.0, "glints": [(rng.uniform(270, 330), rng.uniform(170, 230)) for _ in range(rng.randint(0, 5))]}
|
||||
pair = eyes_pupil.glint_pair(p)
|
||||
want = None if pair is None else math.hypot(pair[1][0] - pair[0][0], pair[1][1] - pair[0][1])
|
||||
got = fitcoach.glint_spacing(p)
|
||||
if want is None:
|
||||
self.assertIsNone(got)
|
||||
else:
|
||||
self.assertAlmostEqual(got, want, places=6)
|
||||
|
||||
|
||||
class SlipTest(unittest.TestCase):
|
||||
"""The slip notice's reading: ft-eyes' glint jumps (pupil px) as position terms and a hint."""
|
||||
|
||||
def jumps(self, yaw, pitch):
|
||||
# Both pupils moved as if the eyes looked (yaw, pitch): each camera's own direction.
|
||||
return {c: (pupil(c, yaw, pitch)[0] - CENTRE[c][0], pupil(c, yaw, pitch)[1] - CENTRE[c][1]) for c in (0, 1)}
|
||||
|
||||
def test_terms(self):
|
||||
t = fitcoach.slip_terms(self.jumps(2.0, 9.0), ref())
|
||||
self.assertAlmostEqual(t["up"], 9.0, places=6)
|
||||
self.assertAlmostEqual(t["left"], 2.0, places=6)
|
||||
self.assertEqual(fitcoach.slip_terms({}, ref()), {})
|
||||
self.assertEqual(fitcoach.slip_terms({0: (3.0, 4.0)}, None), {})
|
||||
|
||||
def test_hint(self):
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(0, 9.0), ref()))["key"], "up")
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(0, -9.0), ref()))["key"], "down")
|
||||
self.assertEqual(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(-7.0, 6.0), ref()))["key"], "right")
|
||||
self.assertIsNone(fitcoach.position_hint(fitcoach.slip_terms(self.jumps(3.0, -4.0), ref())))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,179 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline test of the gaze service idling (gaze/ft-gazed, gaze/gazecheck.py): ft-gaze runs only
|
||||
while the gaze is in use, and a check asked for while it idles waits for the tracker.
|
||||
|
||||
Runs ft-gazed's Service with its sockets renamed, a fake pointer helper (answers "gaze ?
|
||||
headset" as the test says), and a fake ft-gaze (prints samples, quits when its stdin closes).
|
||||
SteamVR's tracker is the one in use, so our own isn't started; the panel isn't either. Nothing
|
||||
reaches the live gaze service, the pointer helper, or SteamVR, so it's safe next to them.
|
||||
|
||||
gaze/test/idle-test.py
|
||||
"""
|
||||
import importlib.machinery
|
||||
import importlib.util
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
GAZE = os.path.join(HERE, "..")
|
||||
sys.path.insert(0, GAZE)
|
||||
loader = importlib.machinery.SourceFileLoader("ftgazed", os.path.join(GAZE, "ft-gazed"))
|
||||
gazed = importlib.util.module_from_spec(importlib.util.spec_from_loader("ftgazed", loader))
|
||||
loader.exec_module(gazed)
|
||||
import gazecheck # noqa: E402 (the module ft-gazed imported)
|
||||
|
||||
tag = f"ft_gaze_idle_test_{os.getpid()}"
|
||||
gazed.ME = f"\0{tag}_gazed"
|
||||
gazed.POINTER = gazecheck.POINTER = f"\0{tag}_helper"
|
||||
gazecheck.SCREENS = f"\0{tag}_screens"
|
||||
gazecheck.PANEL_PROG = gazed.REPO / "nonexistent-panel" # "isn't built": no panel
|
||||
gazed.IDLE_AFTER, gazed.WAKE_SETTLE = 1.0, 3.0
|
||||
gazed.read_settings = lambda: ("steam", "steam", "auto", 55.0)
|
||||
gazed.TrackedEye = lambda: lambda now=None: None # both eyes, whatever SteamVR's settings say
|
||||
logs = []
|
||||
gazed.log = gazecheck.log = lambda msg: logs.append(msg)
|
||||
|
||||
# The fake ft-gaze: 90 samples a second, both eyes seen, until its stdin closes. It logs the
|
||||
# sources it was asked for: "argv LIST" (--sources), then "line LIST" for each "sources LIST".
|
||||
tmp = tempfile.mkdtemp(prefix="ft-gaze-idle-test-")
|
||||
FAKE = os.path.join(tmp, "ft-gaze")
|
||||
SOURCES_LOG = os.path.join(tmp, "sources.log")
|
||||
with open(FAKE, "w") as f:
|
||||
f.write('''import json, os, select, sys, time
|
||||
log = open(sys.argv[1], "a", buffering=1)
|
||||
log.write("argv " + (sys.argv[sys.argv.index("--sources") + 1] if "--sources" in sys.argv else "-") + "\\n")
|
||||
buf = b""
|
||||
while True:
|
||||
if select.select([sys.stdin], [], [], 1 / 90)[0]:
|
||||
data = os.read(0, 4096)
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
while b"\\n" in buf:
|
||||
line, buf = buf.split(b"\\n", 1)
|
||||
if line.startswith(b"sources "):
|
||||
log.write("line " + line[8:].decode() + "\\n")
|
||||
eye = {"hy": 1.0, "hp": 2.0}
|
||||
print(json.dumps({"t": time.monotonic(), "src": {"mmap1": {"hy": 1.0, "hp": 2.0, "unc": [0.001, 0.001],
|
||||
"open": [0.8, 0.8]}, "left": eye, "right": eye}}), flush=True)
|
||||
''')
|
||||
started = []
|
||||
|
||||
|
||||
def start_helper(self):
|
||||
"""ft-gaze, straight from here instead of the dev container."""
|
||||
import selectors
|
||||
self.proc = subprocess.Popen([sys.executable, FAKE, SOURCES_LOG, "--sources", self.wanted_sources()],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
self.proc_sources = self.wanted_sources()
|
||||
os.set_blocking(self.proc.stdout.fileno(), False)
|
||||
os.set_blocking(self.proc.stderr.fileno(), False)
|
||||
self.sel.register(self.proc.stdout, selectors.EVENT_READ, "stdout")
|
||||
self.sel.register(self.proc.stderr, selectors.EVENT_READ, "stderr")
|
||||
self.buf = b""
|
||||
started.append(time.monotonic())
|
||||
|
||||
|
||||
gazed.Service.start_helper = start_helper
|
||||
|
||||
# The fake pointer helper.
|
||||
helper_state = {"reply": "ok off worn"}
|
||||
helper = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
helper.bind(gazed.POINTER)
|
||||
helper.settimeout(0.2)
|
||||
|
||||
|
||||
def answer():
|
||||
while True:
|
||||
try:
|
||||
data, addr = helper.recvfrom(512)
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
return
|
||||
if data.startswith(b"gaze ?") and addr:
|
||||
helper.sendto(helper_state["reply"].encode(), addr)
|
||||
|
||||
|
||||
threading.Thread(target=answer, daemon=True).start()
|
||||
svc = gazed.Service(None, False, gazed.POINTER)
|
||||
threading.Thread(target=svc.run, daemon=True).start()
|
||||
ctl = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
ctl.bind("")
|
||||
ctl.settimeout(2)
|
||||
|
||||
|
||||
def ask(cmd):
|
||||
ctl.sendto(cmd.encode(), gazed.ME)
|
||||
return ctl.recv(4096).decode()
|
||||
|
||||
|
||||
failures = []
|
||||
|
||||
|
||||
def check(label, got, want):
|
||||
ok = got == want
|
||||
print(("ok " if ok else "FAIL ") + label + ("" if ok else f": got {got!r}, want {want!r}"), flush=True)
|
||||
if not ok:
|
||||
failures.append(label)
|
||||
|
||||
|
||||
def wait(cond, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
if cond():
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return cond()
|
||||
|
||||
|
||||
time.sleep(2.5)
|
||||
check("gaze mode off: idle, no ft-gaze", (svc.awake, svc.proc is None), (False, True))
|
||||
check("status says why", ask("status").count('"idle": "gaze mode is off"'), 1)
|
||||
|
||||
helper_state["reply"] = "ok on worn"
|
||||
check("gaze mode on, headset worn: awake within 2 s", wait(lambda: svc.awake and svc.proc is not None, 2.5), True)
|
||||
check("samples come in", wait(lambda: svc.counts["samples"] > 20, 2), True)
|
||||
|
||||
helper_state["reply"] = "ok on away"
|
||||
check("headset off: still awake for IDLE_AFTER", wait(lambda: not svc.awake, 0.5), False)
|
||||
check("then idle, ft-gaze stopped", wait(lambda: not svc.awake and svc.proc is None, 3.5), True)
|
||||
check("why: nobody wears it", ask("status").count('"idle": "nobody is wearing the headset"'), 1)
|
||||
|
||||
helper_state["reply"] = "ok on"
|
||||
check("an older helper (no headset word): awake with gaze mode on", wait(lambda: svc.awake, 2.5), True)
|
||||
helper_state["reply"] = "ok off worn"
|
||||
check("gaze mode off again: idle", wait(lambda: not svc.awake and svc.proc is None, 4.5), True)
|
||||
|
||||
check("wake lease", ask("wake 2"), "ok")
|
||||
check("wake: awake at once", (svc.awake, wait(lambda: svc.proc is not None, 1)), (True, True))
|
||||
check("lease over (2 s + IDLE_AFTER): idle", wait(lambda: not svc.awake, 4.5), True)
|
||||
|
||||
time.sleep(0.5)
|
||||
logs.clear()
|
||||
open(SOURCES_LOG, "w").close()
|
||||
count = len(started)
|
||||
check("quick check while idle: queued, waking", ask("quickcal"), "ok waking the eye tracker first")
|
||||
check("it woke", wait(lambda: svc.awake and len(started) > count, 1.5), True)
|
||||
check("once the tracker sends, it ran (and says why it couldn't open)",
|
||||
wait(lambda: any("quickcal, asked for while idle: the panel isn't running" in m for m in logs), 3), True)
|
||||
check("nothing left pending", svc.checks.pending, None)
|
||||
sources = open(SOURCES_LOG).read().split("\n")
|
||||
check("ft-gaze started with every source for the check", sources[0], "argv all")
|
||||
in_use = svc.wanted_sources()
|
||||
check("then only those in use (SteamVR's tracker: not the action, not own)",
|
||||
(f"line {in_use}" in sources, in_use != "all", "action" in in_use, "own" in in_use), (True, True, False, False))
|
||||
check("idle again after", wait(lambda: not svc.awake, 3), True)
|
||||
|
||||
print("FAILED: " + ", ".join(failures) if failures else "all passed", flush=True)
|
||||
svc.running = False
|
||||
time.sleep(0.7)
|
||||
os.remove(FAKE)
|
||||
os.remove(SOURCES_LOG)
|
||||
os.rmdir(tmp)
|
||||
os._exit(1 if failures else 0)
|
||||
Loaded 100 of 500 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user