From 7a99a2003e59dc63283dfa6e1a298914c2478a37 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 26 Sep 2026 19:50:32 -0600 Subject: [PATCH 1/9] Local voice dictation and "Hey Scarlet" wake word for the Steam Frame Dictation runs fully on the headset: Moonshine (sherpa-onnx, CPU) types words live as two decodes agree, whisper.cpp (Vulkan) handles wake-word commands, and a uinput keyboard types into whatever has focus, including SteamVR and gamescope apps. Push-to-talk works from any keyboard or mouse button and from Steam Frame controller buttons read through vrserver, set in a GTK 4 settings app. Runs as systemd user services. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 4 + README.md | 108 ++++ bindings.py | 67 +++ config.example.toml | 39 ++ hooks/on-command | 14 + scarlet-config | 4 + scarlet_config.py | 360 +++++++++++ scarlet_voice.py | 982 +++++++++++++++++++++++++++++++ scarlet_vr_input.py | 152 +++++ scarletctl | 22 + systemd/scarlet-voice.service | 22 + systemd/scarlet-vr-input.service | 14 + systemd/scarlet-whisper.service | 12 + 13 files changed, 1800 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 bindings.py create mode 100644 config.example.toml create mode 100755 hooks/on-command create mode 100755 scarlet-config create mode 100755 scarlet_config.py create mode 100755 scarlet_voice.py create mode 100755 scarlet_vr_input.py create mode 100755 scarletctl create mode 100644 systemd/scarlet-voice.service create mode 100644 systemd/scarlet-vr-input.service create mode 100644 systemd/scarlet-whisper.service diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af50285 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.venv/ +vendor/ +models/ +__pycache__/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..db4b60f --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# scarlet-voice + +Local voice input for the Steam Frame. Nothing leaves the headset. + +- **Dictation**: hold a key, speak, release. The text is typed into whatever has focus (desktop apps, search bars, terminals, game-mode windows) through a virtual uinput keyboard. +- **Wake word**: say "Hey Scarlet", then a command. The command is transcribed and handed to a hook script and to any event subscribers. This is the entry point for the future Scarlet assistant. + +## Pieces + +| Part | What it does | +| --- | --- | +| `scarlet-whisper.service` | `whisper-server` (whisper.cpp, Vulkan on the Adreno 750) with `ggml-small.en-q5_1` loaded, on `127.0.0.1:8178` | +| `scarlet-voice.service` | `scarlet_voice.py`: mic capture, hotkey, wake word, typing, control socket | +| `scarlet-vr-input.service` | `scarlet_vr_input.py`: controller push-to-talk, reads left d-pad down from SteamVR's raw input stream | +| `scarlet-config` | Settings app (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. In the app menu as *Scarlet Voice Settings* | +| `scarletctl` | CLI for the control socket | +| `hooks/on-command` | Runs after "Hey Scarlet ..." with the command text as `$1` | + +The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx open-vocabulary KWS, about 5% of one core) listens continuously. Silero VAD ends commands on silence and trims clips before transcription. + +## Triggers + +| Where | How | +| --- | --- | +| Keyboard or mouse | Tap **Right Ctrl** (default) to start, tap again to stop. Shortcuts with it (Ctrl+C) don't start or stop dictation. Hold mode is available in the settings app. | +| Steam Frame controllers | Hold left **d-pad down** (default), speak, release. Works in games, the Steam UI and the desktop with no per-app setup. The button is observed, not captured, so whatever else it is bound to still happens. | +| Desktop mode shortcut | KDE System Settings > Shortcuts > add a command shortcut running `scarletctl toggle`. | +| Scripts, the assistant | `scarletctl toggle`, `start`, `stop`, `cancel`, `type "text"` | +| Voice | "Hey Scarlet, ..." runs `hooks/on-command`. "Hey Scarlet, type hello world" types "hello world" (example in the hook). | + +Recordings shorter than 2 s (`min_seconds`) are discarded with the cancel tone, so brief or accidental presses never type anything. + +Change the buttons in *Scarlet Voice Settings* (`./scarlet-config`): **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger and switches live typing, sounds and the wake word. Everything lives in `~/.config/scarlet-voice/bindings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. + +Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. At a 0.8 s pause, or at the quietest point once the window reaches 8 s, the window is decoded one last time and its remaining words are typed. On test audio the first words appeared at about 2.3 s, then every 1-1.5 s, and the text was complete 0.1-0.2 s after release. Nothing is cut while it is still being decoded, so words aren't split or doubled at the edges the way fixed pieces did. Moonshine adds fewer commas than whisper, and a long pause can still end a sentence with a period. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). The "Hey Scarlet" command path still uses whisper. + +Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) + +Audio cues are off by default (`[sounds] enabled = true` turns them on): rising tone = recording, falling tone = processing, single high tone = wake word heard, low tone = cancelled or nothing heard. + +## Control and events + +``` +./scarletctl status +./scarletctl wake off # stop wake word listening; the mic fully closes when idle +./scarletctl wake on +./scarletctl wake # act as if "Hey Scarlet" was heard +./scarletctl type "text" # type into the focused app +./scarletctl events # stream JSON events: wake, command, dictation, command_timeout, error +``` + +The socket is `$XDG_RUNTIME_DIR/scarlet-voice/control.sock`, mode 0600. It can type into any focused window, so keep it owner-only. + +An assistant can either replace `hooks/on-command` or subscribe with `scarletctl events` and act on `{"event": "command", "text": ...}`. + +## Config + +Copy `config.example.toml` to `~/.config/scarlet-voice/config.toml`. Every key is optional. Then `systemctl --user restart scarlet-voice`. + +Wake phrase tuning: raise `threshold` for fewer false triggers, raise `boost` if it misses you. Phrases can be anything spelled in English words; they are tokenized at startup, no training needed. + +## Services + +``` +systemctl --user status scarlet-whisper scarlet-voice scarlet-vr-input +journalctl --user -u scarlet-voice -f +systemctl --user stop scarlet-voice scarlet-whisper scarlet-vr-input # the units are linked, so `disable` removes the links +``` + +## Build notes + +whisper.cpp is in `vendor/whisper.cpp`, built with: + +``` +cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_NATIVE=OFF \ + -DGGML_CPU_ARM_ARCH=armv9.2-a+dotprod+i8mm+sve2+fp16 \ + -DCMAKE_PREFIX_PATH=$PWD/../prefix \ + -DCMAKE_C_FLAGS="-mtune=cortex-a720 -I$PWD/../prefix/include" \ + -DCMAKE_CXX_FLAGS="-mtune=cortex-a720 -I$PWD/../prefix/include" +cmake --build build -j5 --target whisper-server whisper-cli whisper-bench +``` + +`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `scarlet-config` launcher sets that up. + +The whisper service sets `DISABLE_GAMESCOPE_WSI=1`. Without it the Gamescope Vulkan layer aborts whisper-server at startup. + +Typing goes through a uinput keyboard that SteamVR must have open: SteamVR opens `/dev/input` devices only when it starts, and gamescope gets keyboard input from SteamVR. `scarlet-voice.service` is ordered `Before=steamvr.service` and parks the device's fd in systemd's fd store, so daemon restarts keep the same device. After the very first install, restart SteamVR once. The device is a full keyboard (fd store name `typer2`; the first version, `typer`, had only the typing keys and is replaced automatically, which also needs one SteamVR restart). Multidesk's input relay picks the device up immediately for its desktop screens. To drop the device: `systemctl --user clean --what=fdstore scarlet-voice`. + +Controller input: SteamVR sends OpenVR actions only to the app with input focus, and while the 2D Steam or desktop screens are up the compositor's laser mouse holds focus, so a background or overlay OpenVR app never sees the button there. `scarlet_vr_input.py` instead subscribes to vrserver's local web socket (`ws://127.0.0.1:27062`, mailbox `input_server`, `request_input_state_updates`), the stream the controller binding page uses for its live view. It is undocumented and could change with a SteamVR update. `scarlet-vr-input.service` has no `After=steamvr.service`: with `WantedBy=default.target` that is an ordering cycle at boot. + +## Measurements (2026-09-25, jfk.wav) + +Server with the model loaded, `small.en-q5_1` on Vulkan: + +| Clip | Latency | +| --- | --- | +| 3.2 s phrase | 0.4 s | +| 11 s sentence | 1.2 s | + +Whisper always encodes a 30 s window. The daemon shrinks `audio_ctx` to the clip length (floor 512), which is most of that speedup. The CPU backend took over 20 s. This process only gets CPUs 0-4; the faster cores are reserved. + +Bigger models (2026-09-26, 9 real dictations replayed, 512 window, VR running): `medium.en-q5_0` and `large-v3-turbo-q5_0` were no more accurate than `small.en` on this voice (the one difference: "T3 Code Write" vs "right"), took 2-3x longer (about 3 s and 3.5 s per piece versus 1 s), and loading them next to VR made the headset stutter. Turbo also added stray "..." on streamed pieces. Below a 512 window both loop and take 10-20 s. `small.en` stays the default; both bigger models are still in `models/` (about 1.1 GB). + +## Limits + +- Typing assumes a US keyboard layout and ASCII. Accents are stripped, other characters dropped. +- Transcription uses the GPU. A heavy VR game may stutter briefly while a clip transcribes. `ggml-base.en.bin` is faster and lighter if that matters. +- Not yet verified: typing into SteamVR dashboard/overlay text fields and into apps inside VR games. diff --git a/bindings.py b/bindings.py new file mode 100644 index 0000000..3e40b19 --- /dev/null +++ b/bindings.py @@ -0,0 +1,67 @@ +"""Push-to-talk bindings and settings shared by scarlet-voice, scarlet-vr-input and scarlet-config. + +~/.config/scarlet-voice/bindings.json (written by the scarlet-config app): + {"keyboard": ["KEY_RIGHTCTRL", "BTN_SIDE"], + "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], + "keyboard_mode": "toggle", "controller_mode": "hold", + "live_typing": true, "sounds": false, "wake_word": true} + +keyboard: evdev key or mouse button names, read from every physical keyboard and mouse. +controller: a SteamVR device path and one of its boolean inputs, read from vrserver. +*_mode: "hold" records while held; "toggle" starts on one press and stops on the next. +The options override config.toml. +""" +import json +import os +from pathlib import Path + +PATH = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / "scarlet-voice/bindings.json" +DEFAULT = { + "keyboard": ["KEY_RIGHTCTRL"], + "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], + # Toggle for keyboards: live text can't be typed while a modifier like Ctrl is held. + "keyboard_mode": "toggle", + "controller_mode": "hold", + "live_typing": True, + "sounds": False, + "wake_word": True, +} +MODES = ("hold", "toggle") +OPTIONS = ("live_typing", "sounds", "wake_word") +# Held modifiers turn typed letters into shortcuts, so live typing waits until they are released. +MODIFIERS = {"KEY_LEFTCTRL", "KEY_RIGHTCTRL", "KEY_LEFTALT", "KEY_RIGHTALT", + "KEY_LEFTSHIFT", "KEY_RIGHTSHIFT", "KEY_LEFTMETA", "KEY_RIGHTMETA"} + + +def load(): + try: + data = json.loads(PATH.read_text()) + except (FileNotFoundError, ValueError): + data = {} + out = json.loads(json.dumps(DEFAULT)) + if isinstance(data.get("keyboard"), list): + out["keyboard"] = [k for k in data["keyboard"] if isinstance(k, str)] + if isinstance(data.get("controller"), list): + out["controller"] = [c for c in data["controller"] if isinstance(c, dict) + and isinstance(c.get("device"), str) and isinstance(c.get("input"), str)] + for key in ("keyboard_mode", "controller_mode"): + if data.get(key) in MODES: + out[key] = data[key] + for key in OPTIONS: + if isinstance(data.get(key), bool): + out[key] = data[key] + return out + + +def save(data): + PATH.parent.mkdir(parents=True, exist_ok=True) + tmp = PATH.with_suffix(".tmp") + tmp.write_text(json.dumps(data, indent=2) + "\n") + tmp.replace(PATH) + + +def mtime(): + try: + return PATH.stat().st_mtime_ns + except FileNotFoundError: + return 0 diff --git a/config.example.toml b/config.example.toml new file mode 100644 index 0000000..271e03b --- /dev/null +++ b/config.example.toml @@ -0,0 +1,39 @@ +# Copy to ~/.config/scarlet-voice/config.toml and edit. Every key is optional. +# Push-to-talk buttons, hold/tap modes, live typing, sounds and the wake word on/off +# are set in the Scarlet Voice Settings app (scarlet-config), saved in bindings.json. + +[audio] +source = "" # PipeWire node name; empty = default mic + +[whisper] +url = "http://127.0.0.1:8178/inference" +language = "en" +prompt = "Scarlet, SteamOS, Steam Frame, T3 Code, Godot." # vocabulary hints +min_audio_ctx = 512 # smallest encoder window; lower makes medium/turbo loop on short clips + +[dictation] +hold_delay = 0.0 # seconds held before recording starts (0 = on press) +min_seconds = 2.0 # recordings shorter than this are discarded +max_seconds = 120 +key_delay_ms = 4 +trailing_space = true +streaming = true # transcribe pieces while you talk; only the last one waits for release +engine = "moonshine" # live engine: moonshine (CPU, fast) or whisper (GPU server) +moonshine_threads = 2 +step_seconds = 0.8 # re-decode the current window this often +pause_seconds = 0.8 # a pause this long closes the window +max_window_seconds = 8.0 # without a pause, close at the quietest point by this length (moonshine max ~10) +save_clips = 0 # keep the last N dictations' audio in ~/.local/state/scarlet-voice/clips + +[wake] +phrases = ["HEY SCARLET"] +threshold = 0.25 # higher = fewer false triggers, more misses +boost = 1.5 # higher = easier to trigger +command_timeout = 5.0 # seconds to start speaking after the wake chime +end_silence = 0.8 # silence that ends a command +max_command_seconds = 20 +wake_hook = "" # run with the keyword as $1 as soon as the wake word fires +command_hook = "hooks/on-command" # run with the transcribed command as $1 + +[sounds] +volume = 0.5 diff --git a/hooks/on-command b/hooks/on-command new file mode 100755 index 0000000..26fc16b --- /dev/null +++ b/hooks/on-command @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Called after "Hey Scarlet ..." with the transcribed command as $1. +# Replace this with a call into the assistant. Runs as your user, no shell interpolation of $1. +set -euo pipefail +text="$1" +log_dir="${XDG_STATE_HOME:-$HOME/.local/state}/scarlet-voice" +mkdir -p "$log_dir" +printf '%s\t%s\n' "$(date -Is)" "$text" >> "$log_dir/commands.log" + +# Built-in example: "Hey Scarlet, type hello world" types into the focused field. +shopt -s nocasematch +if [[ "$text" =~ ^(type|dictate)[[:space:],:]+(.*)$ ]]; then + exec "$(dirname "$0")/../scarletctl" type "${BASH_REMATCH[2]}" +fi diff --git a/scarlet-config b/scarlet-config new file mode 100755 index 0000000..a247a99 --- /dev/null +++ b/scarlet-config @@ -0,0 +1,4 @@ +#!/bin/sh +# GTK comes from the system Python; evdev and websockets from the project venv. +dir=$(dirname "$(readlink -f "$0")") +PYTHONPATH="$dir:$dir/.venv/lib/python3.12/site-packages" exec /usr/bin/python3 "$dir/scarlet_config.py" "$@" diff --git a/scarlet_config.py b/scarlet_config.py new file mode 100755 index 0000000..ec6f751 --- /dev/null +++ b/scarlet_config.py @@ -0,0 +1,360 @@ +#!/usr/bin/env python3 +"""Scarlet Voice settings (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. + +"Add" waits for a controller button (read from vrserver's input stream, like +scarlet-vr-input) or a keyboard key / mouse button (read from evdev, like scarlet-voice) +and saves ~/.config/scarlet-voice/bindings.json. Both services pick changes up on their +own. Run through the `scarlet-config` launcher: GTK comes from the system Python, evdev +and websockets from the project venv. +""" +import asyncio +import json +import os +import secrets +import select +import socket +import sys +import threading +import time +import urllib.request +from pathlib import Path + +import gi + +gi.require_version("Gtk", "4.0") +gi.require_version("Adw", "1") +import evdev # noqa: E402 +import websockets # noqa: E402 +from evdev import ecodes as e # noqa: E402 +from gi.repository import Adw, GLib, Gtk # noqa: E402 + +import bindings # noqa: E402 + +SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +VRSERVER = "http://127.0.0.1:27062" +VR_HEADERS = {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} +TYPER_NAME = "scarlet-voice keyboard" +CAPTURE_SECONDS = 15 + +# Clicking "Add" would otherwise bind the mouse button that clicked it. +IGNORED_KEYS = {e.BTN_LEFT, e.BTN_RIGHT, e.BTN_TOUCH, e.BTN_TOOL_FINGER, e.BTN_TOOL_PEN, + e.BTN_TOOL_DOUBLETAP, e.BTN_TOOL_TRIPLETAP, e.BTN_STYLUS} +# The system button opens the SteamVR dashboard; ft_pointer is Frametop's virtual pointer (its 3D mouse). +IGNORED_INPUTS = {"/input/system/click"} +IGNORED_CONTROLLERS = {"ft_pointer"} + +INPUT_NAMES = {"dpad_up": "D-pad up", "dpad_down": "D-pad down", "dpad_left": "D-pad left", + "dpad_right": "D-pad right", "thumbstick": "Thumbstick press", "trackpad": "Trackpad press", + "trigger": "Trigger", "grip": "Grip", "bumper": "Bumper", "thumbrest": "Thumb rest", + "view": "View", "menu": "Menu", "system": "System"} +KEY_NAMES = {"CTRL": "Ctrl", "ALT": "Alt", "SHIFT": "Shift", "META": "Super", "SIDE": "side button", + "EXTRA": "extra button", "FORWARD": "forward button", "BACK": "back button", + "MIDDLE": "middle button", "CAPSLOCK": "Caps Lock", "SCROLLLOCK": "Scroll Lock", + "PAUSE": "Pause", "ESC": "Esc", "SPACE": "Space", "ENTER": "Enter"} + + +def key_label(name): + base = name.split("_", 1)[1] if "_" in name else name + if name.startswith("BTN_"): + return "Mouse " + KEY_NAMES.get(base, base.lower().replace("_", " ")) + for side in ("LEFT", "RIGHT"): + rest = base[len(side):] + if base.startswith(side) and rest in KEY_NAMES: + return f"{side.title()} {KEY_NAMES[rest]}" + return KEY_NAMES.get(base, base.title() if len(base) > 1 else base) + + +def controller_label(device, input_path): + hand = device.rsplit("/", 1)[-1] + part = input_path.split("/")[2] if input_path.count("/") >= 2 else input_path + name = INPUT_NAMES.get(part, part.upper() if len(part) == 1 else part.replace("_", " ").capitalize()) + return f"{hand.title() if hand in ('left', 'right') else device} controller: {name}" + + +MODE_LABELS = {"toggle": "Tap to start, tap to stop", "hold": "Hold to talk"} +MODE_HINTS = {"toggle": "Tap to start, tap again to stop.", "hold": "Hold to talk, release to stop."} + + +def key_note(name, mode): + if name in bindings.MODIFIERS: + if mode == "hold": + return "While it's held, text waits for release (it would turn letters into shortcuts)" + return "Shortcuts with it (like Ctrl+C) don't start or stop dictation" + if name.startswith("KEY_") and (len(name) == 5 or name in ("KEY_SPACE", "KEY_ENTER")): + return "This key also types into apps" + return name + + +def vr_devices(): + """Connected controllers as SteamVR reports them, or None when SteamVR is not running.""" + req = urllib.request.Request(f"{VRSERVER}/input/getstate.json", headers=VR_HEADERS) + try: + with urllib.request.urlopen(req, timeout=3) as resp: + state = json.load(resp) + except OSError: + return None + return [{"device": d["root_path"], "controller_type": d["controller_type"], + "label": f"{d['root_path'].rsplit('/', 1)[-1]} {d['controller_type'].replace('_', ' ').title().replace('Frame', 'Steam Frame')}"} + for d in state.get("devices", []) + if d.get("class") == "TrackedDeviceClass_Controller" and d.get("root_path") + and d.get("controller_type") not in IGNORED_CONTROLLERS] + + +def notify_daemon(): + try: + with socket.socket(socket.AF_UNIX) as sock: + sock.settimeout(2) + sock.connect(str(SOCKET_PATH)) + sock.sendall(b"reload\n") + sock.recv(4096) + return True + except OSError: + return False + + +class Capture: + """One capture at a time; a new request or /cancel ends the previous one.""" + + def __init__(self): + self.lock = threading.Lock() + self.cancel = threading.Event() + + def begin(self): + self.cancel.set() + self.lock.acquire() + self.cancel = threading.Event() + return self.cancel + + def end(self): + self.lock.release() + + def keyboard(self): + cancel = self.begin() + devices = [] + try: + for path in evdev.list_devices(): + try: + dev = evdev.InputDevice(path) + except OSError: + continue + if dev.name == TYPER_NAME or e.EV_KEY not in dev.capabilities(): + dev.close() + continue + devices.append(dev) + for dev in devices: # drop anything already queued + try: + while dev.read_one(): + pass + except OSError: + pass + deadline = time.monotonic() + CAPTURE_SECONDS + while time.monotonic() < deadline and not cancel.is_set(): + ready, _, _ = select.select(devices, [], [], 0.2) + for dev in ready: + try: + events = list(dev.read()) + except OSError: + continue + for ev in events: + if ev.type != e.EV_KEY or ev.value != 1 or ev.code in IGNORED_KEYS: + continue + if ev.code == e.KEY_ESC: + return {"cancelled": True} + name = e.KEY.get(ev.code) or e.BTN.get(ev.code) + if isinstance(name, list): + name = name[0] + if name: + return {"code": name, "label": key_label(name), "source": dev.name} + return {"cancelled": True} if cancel.is_set() else {"timeout": True} + finally: + for dev in devices: + dev.close() + self.end() + + def controller(self): + cancel = self.begin() + try: + devices = vr_devices() + if devices is None: + return {"error": "SteamVR is not running."} + if not devices: + return {"error": "No controller is connected. Wake a controller and try again."} + return asyncio.run(self._controller(cancel, devices)) + except (OSError, websockets.WebSocketException) as err: + return {"error": f"Could not read controllers from SteamVR: {err}"} + finally: + self.end() + + async def _controller(self, cancel, devices): + mailbox = f"scarlet_config_{os.getpid()}_{secrets.token_hex(3)}" + types = {d["device"]: d["controller_type"] for d in devices} + async with websockets.connect(VRSERVER.replace("http", "ws"), origin=VRSERVER, max_size=None, + additional_headers=VR_HEADERS) as ws: + await ws.send(f"mailbox_open {mailbox}") + for device in types: + message = {"type": "request_input_state_updates", "device_path": device, "returnAddress": mailbox} + await ws.send(f"mailbox_send input_server {json.dumps(message)}") + released = {} # device -> inputs seen up; a button held when capture starts doesn't count + deadline = time.monotonic() + CAPTURE_SECONDS + while time.monotonic() < deadline and not cancel.is_set(): + try: + message = json.loads(await asyncio.wait_for(ws.recv(), 0.2)) + except asyncio.TimeoutError: + continue + device = message.get("device") + if message.get("type") != "update_component_states" or device not in types: + continue + seen = released.setdefault(device, set()) + for name, value in message.get("components", {}).items(): + if not name.endswith("/click") or name in IGNORED_INPUTS: + continue + if not value: + seen.add(name) + elif name in seen: + return {"device": device, "input": name, "controller_type": types[device], + "label": controller_label(device, name)} + return {"cancelled": True} if cancel.is_set() else {"timeout": True} + + +class Window(Adw.ApplicationWindow): + def __init__(self, app): + super().__init__(application=app, title="Scarlet Voice", default_width=560, default_height=640) + self.capture = Capture() + self.data = bindings.load() + self.rows = [] + self.dialog = None + + self.controller_status = "Checking for controllers…" + self.controller_group = Adw.PreferencesGroup(title="Controller") + self.keyboard_group = Adw.PreferencesGroup(title="Keyboard and mouse") + for group, kind in ((self.controller_group, "controller"), (self.keyboard_group, "keyboard")): + add = Gtk.Button(label="Add", css_classes=["flat"]) + add.connect("clicked", lambda _b, k=kind: self.start_capture(k)) + group.set_header_suffix(add) + page = Adw.PreferencesPage() + for group in (self.controller_group, self.keyboard_group, self._behavior_group()): + page.add(group) + self.toasts = Adw.ToastOverlay(child=page) + view = Adw.ToolbarView(content=self.toasts) + view.add_top_bar(Adw.HeaderBar()) + self.set_content(view) + self.refresh() + threading.Thread(target=self._load_controllers, daemon=True).start() + + def _behavior_group(self): + group = Adw.PreferencesGroup(title="Behavior", + description="Recordings shorter than 2 seconds are discarded.") + modes = list(MODE_LABELS) + for key, title in (("keyboard_mode", "Keyboard and mouse"), ("controller_mode", "Controller")): + row = Adw.ComboRow(title=title, model=Gtk.StringList.new([MODE_LABELS[m] for m in modes]), + selected=modes.index(self.data[key])) + row.connect("notify::selected", lambda r, _p, k=key: self.set_option(k, modes[r.get_selected()])) + group.add(row) + for key, title, subtitle in ( + ("live_typing", "Type while talking", "Text appears every few seconds instead of after you stop"), + ("sounds", "Sounds", "Tones when recording starts and stops"), + ("wake_word", "\u201cHey Scarlet\u201d wake word", "Listen for the wake word when not dictating")): + row = Adw.SwitchRow(title=title, subtitle=subtitle, active=self.data[key]) + row.connect("notify::active", lambda r, _p, k=key: self.set_option(k, r.get_active())) + group.add(row) + return group + + def set_option(self, key, value): + self.data[key] = value + self.save("Saved.") + + def _load_controllers(self): + devices = vr_devices() + if devices is None: + text = "SteamVR is not running." + elif not devices: + text = "No controller connected. Wake one to add its buttons." + else: + text = "Connected: " + ", ".join(d["label"] for d in devices) + "." + text = text[:11] + text[11].upper() + text[12:] + self.controller_status = text + GLib.idle_add(self.refresh) + + def refresh(self): + for group, row in self.rows: + group.remove(row) + self.rows = [] + entries = [(self.controller_group, controller_label(c["device"], c["input"]), f"{c['device']} {c['input']}", ("controller", i)) + for i, c in enumerate(self.data["controller"])] + self.controller_group.set_description(MODE_HINTS[self.data["controller_mode"]] + " " + self.controller_status) + self.keyboard_group.set_description(MODE_HINTS[self.data["keyboard_mode"]]) + entries += [(self.keyboard_group, key_label(k), key_note(k, self.data["keyboard_mode"]), ("keyboard", i)) + for i, k in enumerate(self.data["keyboard"])] + for group, title, subtitle, (kind, index) in entries: + row = Adw.ActionRow(title=title, subtitle=subtitle) + remove = Gtk.Button(icon_name="user-trash-symbolic", valign=Gtk.Align.CENTER, + css_classes=["flat"], tooltip_text="Remove") + remove.connect("clicked", lambda _b, k=kind, i=index: self.remove(k, i)) + row.add_suffix(remove) + group.add(row) + self.rows.append((group, row)) + for group, kind in ((self.controller_group, "controller"), (self.keyboard_group, "keyboard")): + if not self.data[kind]: + row = Adw.ActionRow(title="Nothing bound", subtitle="Use Add to pick a button", sensitive=False) + group.add(row) + self.rows.append((group, row)) + + def save(self, message): + bindings.save(self.data) + live = notify_daemon() + self.refresh() + self.toast(message + (" Active now." if live else " scarlet-voice is not running.")) + + def toast(self, text): + self.toasts.add_toast(Adw.Toast(title=text, timeout=3)) + + def remove(self, kind, index): + del self.data[kind][index] + self.save("Removed.") + + def start_capture(self, kind): + what = "controller button" if kind == "controller" else "key or mouse button" + self.dialog = Adw.AlertDialog(heading=f"Press a {what}", + body=f"Press the {what} you want to use for dictation.") + self.dialog.set_extra_child(Gtk.Spinner(spinning=True, height_request=32)) + self.dialog.add_response("cancel", "Cancel") + self.dialog.connect("response", lambda *_: self.capture.cancel.set()) + self.dialog.present(self) + + def work(): + result = self.capture.controller() if kind == "controller" else self.capture.keyboard() + GLib.idle_add(self.capture_done, kind, result) + threading.Thread(target=work, daemon=True).start() + + def capture_done(self, kind, result): + if self.dialog: + self.dialog.force_close() + self.dialog = None + if "error" in result: + self.toast(result["error"]) + elif "timeout" in result: + self.toast("Nothing pressed.") + elif kind == "controller" and "input" in result: + entry = {"device": result["device"], "input": result["input"]} + if entry in self.data["controller"]: + self.toast(f"{result['label']} is already bound.") + else: + self.data["controller"].append(entry) + self.save(f"Added {result['label']}.") + elif kind == "keyboard" and "code" in result: + if result["code"] in self.data["keyboard"]: + self.toast(f"{result['label']} is already bound.") + else: + self.data["keyboard"].append(result["code"]) + self.save(f"Added {result['label']}.") + return False + + +def main(): + app = Adw.Application(application_id="local.scarletvoice.Config") + app.connect("activate", lambda a: (a.get_active_window() or Window(a)).present()) + return app.run(sys.argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scarlet_voice.py b/scarlet_voice.py new file mode 100755 index 0000000..a6d0037 --- /dev/null +++ b/scarlet_voice.py @@ -0,0 +1,982 @@ +#!/usr/bin/env python3 +"""scarlet-voice: local push-to-talk dictation and "Hey Scarlet" wake word daemon. + +One PipeWire capture stream feeds two consumers: + * dictation: hotkey or socket command -> record -> whisper-server -> type via uinput + * wake word: sherpa-onnx keyword spotter -> capture the spoken command -> hooks + events +""" +import asyncio +import io +import json +import logging +import os +import re +import signal +import socket +import struct +import time +import tomllib +import unicodedata +import urllib.request +import uuid +import wave +from pathlib import Path + +import evdev + +import bindings +import numpy as np +import sentencepiece +import sherpa_onnx +from evdev import ecodes as e + +ROOT = Path(__file__).resolve().parent +RATE = 16000 +CHUNK = 1536 # 96 ms of 16 kHz mono: a whole number of 512-sample VAD windows +MODIFIER_CODES = {e.ecodes[name] for name in bindings.MODIFIERS} +TYPER_NAME = "scarlet-voice keyboard" +KWS_DIR = ROOT / "models/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01" +VAD_MODEL = ROOT / "models/silero_vad.onnx" +RUNTIME_DIR = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice" +CLIP_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "scarlet-voice/clips" +SOCKET_PATH = RUNTIME_DIR / "control.sock" + +log = logging.getLogger("scarlet-voice") + +DEFAULTS = { + "audio": {"source": ""}, + "whisper": { + "url": "http://127.0.0.1:8178/inference", + "language": "en", + "prompt": "Scarlet, SteamOS, Steam Frame, T3 Code, Godot.", + # Smallest encoder window (50 per second of audio). medium and large-v3-turbo loop + # and take 10-20 s on short clips below 512; 512 covers pieces up to ~10 s. + "min_audio_ctx": 512, + }, + "dictation": { + "hold_delay": 0.0, # seconds the hotkey must be held before recording starts + "min_seconds": 2.0, # shorter recordings are discarded, not transcribed + "max_seconds": 120, + "key_delay_ms": 4, + "trailing_space": True, + # Transcribe while recording: pieces split at pauses go to whisper as soon as they end, + # so only the last piece is left after release. + "streaming": True, + # Live text by agreement: the current window of speech is re-decoded every + # step_seconds and words two decodes in a row agree on are typed. A window closes at a + # pause (its last words are typed then) or at its quietest point by max_window_seconds. + # moonshine runs on the CPU (~0.1-0.3 s per decode, max ~10 s of audio); "whisper" uses + # the whisper-server on the GPU (better punctuation, ~0.5-0.8 s per decode). + "engine": "moonshine", + "moonshine_model": "models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27", + "moonshine_threads": 2, + "step_seconds": 0.8, + "pause_seconds": 0.8, # shorter pauses inside a sentence ended it with "?"/"." in testing + "max_window_seconds": 8.0, + # Keep the audio of the last N dictations in ~/.local/state/scarlet-voice/clips for + # comparing models and settings. 0 = never write audio to disk. + "save_clips": 0, + }, + "wake": { + "phrases": ["HEY SCARLET"], + "threshold": 0.25, + "boost": 1.5, + "command_timeout": 5.0, + "end_silence": 0.8, + "max_command_seconds": 20, + "wake_hook": "", + "command_hook": "hooks/on-command", + }, + "sounds": {"volume": 0.5}, # on/off is in bindings.json (scarlet-config) +} + + +def load_config(): + path = Path(os.environ.get("SCARLET_VOICE_CONFIG", "~/.config/scarlet-voice/config.toml")).expanduser() + cfg = {k: dict(v) for k, v in DEFAULTS.items()} + if path.exists(): + with open(path, "rb") as f: + for section, values in tomllib.load(f).items(): + cfg.setdefault(section, {}).update(values) + log.info("config: %s", path) + return cfg + + +def resolve(p): + p = Path(p).expanduser() + return p if p.is_absolute() else ROOT / p + + +# ---------------------------------------------------------------- typing + +def _build_keymap(): + km = {} + for c in "abcdefghijklmnopqrstuvwxyz": + code = getattr(e, f"KEY_{c.upper()}") + km[c], km[c.upper()] = (code, False), (code, True) + for c, shifted in zip("1234567890", "!@#$%^&*()"): + code = getattr(e, f"KEY_{c}") + km[c], km[shifted] = (code, False), (code, True) + for plain, shifted, name in [ + ("-", "_", "MINUS"), ("=", "+", "EQUAL"), ("[", "{", "LEFTBRACE"), ("]", "}", "RIGHTBRACE"), + ("\\", "|", "BACKSLASH"), (";", ":", "SEMICOLON"), ("'", '"', "APOSTROPHE"), + (",", "<", "COMMA"), (".", ">", "DOT"), ("/", "?", "SLASH"), ("`", "~", "GRAVE"), + ]: + code = getattr(e, f"KEY_{name}") + km[plain], km[shifted] = (code, False), (code, True) + km[" "] = (e.KEY_SPACE, False) + return km + + +KEYMAP = _build_keymap() +ASCII_FIXUPS = str.maketrans({"‘": "'", "’": "'", "“": '"', "”": '"', + "–": "-", "—": "-", "…": "...", "\n": " ", "\t": " "}) + + +INPUT_EVENT = struct.Struct("llHHi") # struct input_event on 64-bit + + +def inherited_fd(name): + """An fd systemd handed back from its fd store (LISTEN_FDS protocol), or None.""" + if os.environ.get("LISTEN_PID") != str(os.getpid()): + return None + names = os.environ.get("LISTEN_FDNAMES", "").split(":") + count = int(os.environ.get("LISTEN_FDS", "0")) + return next((3 + i for i in range(count) if i < len(names) and names[i] == name), None) + + +def sd_notify(message, fds=()): + addr = os.environ.get("NOTIFY_SOCKET") + if not addr: + return False + if addr.startswith("@"): + addr = "\0" + addr[1:] + ancillary = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, struct.pack(f"{len(fds)}i", *fds))] if fds else [] + with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock: + # socket.send_fds() ignores its address argument on Python 3.12. + sock.sendmsg([message.encode()], ancillary, 0, addr) + return True + + +class Typer: + """Types text through a virtual uinput keyboard. Assumes a US layout on the receiving side. + + SteamVR opens /dev/input devices only when it starts, and gamescope gets its keyboard + input from SteamVR. So the device must exist before SteamVR starts and must never be + recreated: under systemd its fd is parked in the unit's fd store and reused on restart. + """ + + FDNAME = "typer2" # v2 is a full keyboard; v1 ("typer") had only the typing keys + + def __init__(self, key_delay_ms): + self.delay = key_delay_ms / 1000 + self.ui = None + old = inherited_fd("typer") + if old is not None: + sd_notify("FDSTOREREMOVE=1\nFDNAME=typer") + os.close(old) + log.warning("typer: replaced the v1 virtual keyboard; restart SteamVR once so it opens the new one") + self.fd = inherited_fd(self.FDNAME) + if self.fd is not None: + log.info("typer: reusing virtual keyboard from the systemd fd store") + return + self.ui = evdev.UInput({e.EV_KEY: list(range(e.KEY_ESC, e.KEY_MICMUTE + 1))}, name=TYPER_NAME) + self.fd = self.ui.fd + if sd_notify(f"FDSTORE=1\nFDNAME={self.FDNAME}", [self.fd]): + log.info("typer: new virtual keyboard stored in the systemd fd store") + self.ui = None # never UI_DEV_DESTROY it; the fd store owns the device now + else: + log.warning("typer: not under systemd; SteamVR only sees this keyboard if it starts after us") + + def _emit(self, type_, code, value): + os.write(self.fd, INPUT_EVENT.pack(0, 0, type_, code, value)) + + def _key(self, code, value): + self._emit(e.EV_KEY, code, value) + self._emit(e.EV_SYN, e.SYN_REPORT, 0) + time.sleep(self.delay) + + def type(self, text): + text = unicodedata.normalize("NFKD", text.translate(ASCII_FIXUPS)) + for ch in text: + if ch not in KEYMAP: + continue # drops combining accents and anything outside the US keymap + code, shift = KEYMAP[ch] + if shift: + self._key(e.KEY_LEFTSHIFT, 1) + self._key(code, 1) + self._key(code, 0) + if shift: + self._key(e.KEY_LEFTSHIFT, 0) + + def close(self): + if self.ui: # only when not handed to systemd + self.ui.close() + + +def drop_repeats(text): + """Collapse sentences whisper repeated back to back ("Ask not. Ask not. Ask not.").""" + out = [] + for sentence in re.split(r"(?<=[.!?])\s+", text): + if not out or sentence.strip(" .!?,").lower() != out[-1].strip(" .!?,").lower(): + out.append(sentence) + return " ".join(out) + + +def common_prefix(a, b): + """How many leading words two hypotheses share, ignoring case and punctuation.""" + n = 0 + for x, y in zip(a, b): + if re.sub(r"[^\w']", "", x.lower()) != re.sub(r"[^\w']", "", y.lower()): + break + n += 1 + return n + + +def feed_vad(vad, samples): + """Silero VAD only handles input correctly in whole 512-sample windows.""" + size = vad.config.silero_vad.window_size + for i in range(0, len(samples) - size + 1, size): + vad.accept_waveform(samples[i:i + size]) + + +# ---------------------------------------------------------------- sounds + +def make_sounds(volume): + """Short tones so state changes are audible inside VR. Returns name -> wav path.""" + specs = {"start": [660, 880], "stop": [880, 660], "wake": [988], "cancel": [330, 262]} + out = {} + RUNTIME_DIR.mkdir(parents=True, exist_ok=True) + for name, freqs in specs.items(): + parts = [] + for f in freqs: + t = np.arange(int(RATE * 0.11)) / RATE + env = np.minimum(1, np.minimum(t, t[::-1]) * 200) + parts.append(np.sin(2 * np.pi * f * t) * env * volume) + path = RUNTIME_DIR / f"{name}.wav" + path.write_bytes(to_wav(np.concatenate(parts).astype(np.float32))) + out[name] = path + return out + + +def to_wav(samples): + buf = io.BytesIO() + with wave.open(buf, "wb") as w: + w.setnchannels(1) + w.setsampwidth(2) + w.setframerate(RATE) + w.writeframes((np.clip(samples, -1, 1) * 32767).astype(np.int16).tobytes()) + return buf.getvalue() + + +# ---------------------------------------------------------------- daemon + +class ScarletVoice: + def __init__(self, cfg): + self.cfg = cfg + self.dcfg, self.wcfg = cfg["dictation"], cfg["wake"] + self.hotkeys = {} # evdev code -> name, from load_bindings() + self.typer = Typer(self.dcfg["key_delay_ms"]) + self.type_lock = asyncio.Lock() + self.sounds = {} # apply_options() makes them when enabled + self.subscribers = set() + + self.recording = None # list of chunks while dictating + self.recording_source = None + self.stream = None # streaming state of the current dictation, see start_dictation + self.whisper_lock = asyncio.Lock() # one request at a time, in order + self.command = None # dict while capturing a post-wake command + self.wake_enabled = bindings.DEFAULT["wake_word"] # apply_options() sets the saved value + self.mic_task = None + self.mic_proc = None + self.audio_session_ready = False + self.tasks = set() # keep references so fire-and-forget tasks are not collected early + + self.hk_down = False + self.hk_chord = False + self.kb_mode = bindings.DEFAULT["keyboard_mode"] + self.applied = {} # bindings.json options last applied, so a save only changes what changed + self.mods_down = set() # (device path, code) of modifiers held on any keyboard + self.mods_released = asyncio.Event() + self.mods_released.set() + self.hk_timer = None + + self.kws = self._make_kws() + self.kws_stream = self.kws.create_stream() + self.cmd_vad = self._make_vad(self.wcfg["end_silence"]) + self.clip_vad = self._make_vad(0.5) + self.piece_vad = self._make_vad(0.1) # per-chunk speech/silence labels for live windows + self.moonshine = None + if self.dcfg["engine"] == "moonshine": + m = resolve(self.dcfg["moonshine_model"]) + self.moonshine = sherpa_onnx.OfflineRecognizer.from_moonshine_v2( + encoder=str(m / "encoder_model.ort"), decoder=str(m / "decoder_model_merged.ort"), + tokens=str(m / "tokens.txt"), num_threads=self.dcfg["moonshine_threads"]) + log.info("live dictation engine: moonshine (%s)", m.name) + self.load_bindings() + + # --- models + + def _make_kws(self): + sp = sentencepiece.SentencePieceProcessor(model_file=str(KWS_DIR / "bpe.model")) + lines = [] + for phrase in self.wcfg["phrases"]: + phrase = phrase.upper().strip() + tokens = " ".join(sp.encode(phrase, out_type=str)) + lines.append(f"{tokens} :{self.wcfg['boost']} #{self.wcfg['threshold']} @{phrase.replace(' ', '_')}") + RUNTIME_DIR.mkdir(parents=True, exist_ok=True) + kw_file = RUNTIME_DIR / "keywords.txt" + kw_file.write_text("\n".join(lines) + "\n") + log.info("wake phrases: %s", ", ".join(self.wcfg["phrases"])) + return sherpa_onnx.KeywordSpotter( + tokens=str(KWS_DIR / "tokens.txt"), + encoder=str(KWS_DIR / "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"), + decoder=str(KWS_DIR / "decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"), + joiner=str(KWS_DIR / "joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx"), + keywords_file=str(kw_file), + num_threads=1, + ) + + def _make_vad(self, min_silence): + config = sherpa_onnx.VadModelConfig() + config.silero_vad.model = str(VAD_MODEL) + config.silero_vad.min_silence_duration = min_silence + config.silero_vad.min_speech_duration = 0.2 + config.sample_rate = RATE + return sherpa_onnx.VoiceActivityDetector(config, buffer_size_in_seconds=180) + + # --- state + + def state(self): + return { + "dictating": self.recording is not None, + "awaiting_command": self.command is not None, + "wake_enabled": self.wake_enabled, + "mic_open": self.mic_proc is not None and self.mic_proc.returncode is None, + } + + def broadcast(self, event, **fields): + msg = (json.dumps({"event": event, "time": time.time(), **fields}) + "\n").encode() + for w in list(self.subscribers): + try: + w.write(msg) + except Exception: + self.subscribers.discard(w) + + def chime(self, name): + if name in self.sounds: + # SteamOS runs every playback stream through a per-stream spatializer that takes + # ~0.35 s to set up, late enough to bury a start tone under the first word. Its + # WirePlumber script skips streams named like filter chains, so use such a name. + props = '{ node.name = "scarlet-voice-chime-filter-chain" media.role = "Notification" }' + task = asyncio.create_task(self._run(["pw-play", "-P", props, str(self.sounds[name])], quiet=True)) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + async def _run(self, argv, env=None, quiet=False): + try: + proc = await asyncio.create_subprocess_exec( + *argv, env={**os.environ, **(env or {})}, + stdout=asyncio.subprocess.DEVNULL if quiet else None, + stderr=asyncio.subprocess.DEVNULL if quiet else None) + rc = await proc.wait() + if rc and not quiet: + log.warning("%s exited %s", argv[0], rc) + except OSError as err: + log.warning("could not run %s: %s", argv[0], err) + + def run_hook(self, key, *args, env=None): + hook = self.wcfg.get(key) + if not hook: + return + path = resolve(hook) + if not os.access(path, os.X_OK): + log.warning("hook %s not executable: %s", key, path) + return + asyncio.create_task(self._run([str(path), *args], env=env)) + + # --- microphone + + def mic_needed(self): + return self.wake_enabled or self.recording is not None or self.command is not None + + def update_mic(self): + if self.mic_needed() and (self.mic_task is None or self.mic_task.done()): + self.mic_task = asyncio.create_task(self._capture()) + elif not self.mic_needed() and self.mic_proc and self.mic_proc.returncode is None: + self.mic_proc.terminate() # fully close the mic when nothing is listening + + async def _wait_for_audio_session(self): + """Don't open the mic while SteamOS is still building its mic filter chain. + + At boot this daemon starts before SteamVR, and WirePlumber only after it. A capture + stream that connects mid-setup can leave the echo-cancel link stuck, and the mic then + reads silence until PipeWire restarts. This unit can't be ordered after WirePlumber + (cycle through steamvr.service), so wait here instead. + """ + while True: + proc = await asyncio.create_subprocess_exec( + "systemctl", "--user", "show", "wireplumber", "-p", "ActiveState", "-p", "ActiveEnterTimestampMonotonic", + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL) + props = dict(line.split("=", 1) for line in (await proc.communicate())[0].decode().split()) + if props.get("ActiveState") == "active": + started = int(props.get("ActiveEnterTimestampMonotonic", "0")) / 1e6 + settle = started + 10 - time.clock_gettime(time.CLOCK_MONOTONIC) + if settle > 0: + await asyncio.sleep(settle) + return + await asyncio.sleep(1) + + async def _capture(self): + if not self.audio_session_ready: + log.info("waiting for WirePlumber before opening the mic") + await self._wait_for_audio_session() + self.audio_session_ready = True + while self.mic_needed(): + argv = ["pw-record", "--raw", "--rate", str(RATE), "--channels", "1", + "--format", "s16", "--latency", "50ms", "-"] + if self.cfg["audio"]["source"]: + argv[1:1] = ["--target", self.cfg["audio"]["source"]] + self.mic_proc = await asyncio.create_subprocess_exec( + *argv, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL) + log.info("mic open") + try: + while True: + data = await self.mic_proc.stdout.readexactly(CHUNK * 2) + self.on_audio(np.frombuffer(data, np.int16).astype(np.float32) / 32768) + except asyncio.IncompleteReadError: + pass + await self.mic_proc.wait() + log.info("mic closed") + if self.mic_needed(): + await asyncio.sleep(1) + + def on_audio(self, chunk): + if self.recording is not None: + self.recording.append(chunk) + if self.stream: + feed_vad(self.piece_vad, chunk) + self.stream["speech"].append(self.piece_vad.is_speech_detected()) + self.stream["level"].append(float(np.sqrt(np.mean(chunk ** 2)))) + while not self.piece_vad.empty(): + self.piece_vad.pop() + self.stream["wake"].set() + if len(self.recording) * CHUNK / RATE >= self.dcfg["max_seconds"]: + asyncio.create_task(self.stop_dictation()) + elif self.command is not None: + self._command_audio(chunk) + elif self.wake_enabled: + self.kws_stream.accept_waveform(RATE, chunk) + while self.kws.is_ready(self.kws_stream): + self.kws.decode_stream(self.kws_stream) + keyword = self.kws.get_result(self.kws_stream) + if keyword: + self.kws.reset_stream(self.kws_stream) + self.on_wake(keyword.replace("_", " ")) + break + + # --- transcription + + def trim_to_speech(self, samples): + """Cut leading/trailing silence. Returns None when there is no speech at all.""" + vad = self.clip_vad + vad.reset() + feed_vad(vad, samples) + vad.flush() + first = last = None + while not vad.empty(): + seg = vad.front + first = seg.start if first is None else first + last = seg.start + len(seg.samples) + vad.pop() + if first is None: + return None + pad = int(0.25 * RATE) + return samples[max(0, first - pad):last + pad] + + async def transcribe(self, samples): + speech = self.trim_to_speech(samples) + if speech is None: + log.info("no speech detected; skipping transcription") + return "" + return await self.transcribe_speech(speech) + + async def transcribe_speech(self, speech, context="", quiet=False): + """Transcribe audio that is already speech. `context` is the text said just before.""" + dur = len(speech) / RATE + # Whisper's encoder always runs a 30 s window (1500 frames, 50/s). Shrinking it to the + # clip length is the main latency win on the Adreno GPU. It must still cover the whole + # clip, and very small contexts make the decoder loop, hence the floor. + audio_ctx = 0 if dur > 28 else min(1500, max(self.cfg["whisper"]["min_audio_ctx"], int(dur * 50 * 1.1) + 64)) + async with self.whisper_lock: + t0 = time.monotonic() + text = await asyncio.to_thread(self._post, to_wav(speech), audio_ctx, context) + text = re.sub(r"\s+", " ", re.sub(r"\[[^\]]*\]|\([^)]*\)", "", text)).strip() + text = drop_repeats(text) + (log.debug if quiet else log.info)("transcribed %.1fs audio in %.2fs: %r", dur, time.monotonic() - t0, text) + return text + + def _post(self, wav_bytes, audio_ctx, context=""): + w = self.cfg["whisper"] + # temperature_inc 0: no fallback re-decodes, which cost 10+ s on short pieces with turbo. + fields = {"response_format": "json", "temperature": "0", "temperature_inc": "0", + "no_timestamps": "true", "language": w["language"], "audio_ctx": str(audio_ctx)} + prompt = " ".join(filter(None, [w["prompt"], context])) + if prompt: + fields["prompt"] = prompt + boundary = uuid.uuid4().hex + parts = [f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode() + for k, v in fields.items()] + parts.append(f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="clip.wav"\r\n' + f"Content-Type: audio/wav\r\n\r\n".encode() + wav_bytes + b"\r\n") + parts.append(f"--{boundary}--\r\n".encode()) + req = urllib.request.Request(w["url"], data=b"".join(parts), headers={ + "Content-Type": f"multipart/form-data; boundary={boundary}"}) + with urllib.request.urlopen(req, timeout=120) as resp: + return json.load(resp).get("text", "") + + async def type_text(self, text): + async with self.type_lock: + await asyncio.to_thread(self.typer.type, text) + + # --- dictation + + def start_dictation(self, source): + if self.recording is not None: + return + if self.command is not None: + self.command = None # dictation wins over a pending wake command + self.recording, self.recording_source = [], source + if self.dcfg["streaming"]: + self.piece_vad.reset() + self.stream = {"source": source, "t0": time.monotonic(), "chunks": self.recording, + "speech": [], "level": [], "wake": asyncio.Event(), "final": False, + "win": 0, "decoded_at": 0, "prev": [], "wdone": [], + "words": [], "typed": 0} + self.stream["worker"] = asyncio.create_task(self._stream_worker(self.stream)) + self.update_mic() + self.chime("start") + self.broadcast("dictation_start", source=source) + log.info("dictation start (%s)", source) + + def cancel_dictation(self, chime=True): + if self.recording is None: + return + self.recording = self.recording_source = None + if self.stream: + self.stream["worker"].cancel() + self.stream = None + self.update_mic() + if chime: + self.chime("cancel") + self.broadcast("dictation_cancel") + log.info("dictation cancelled") + + async def stop_dictation(self): + if self.recording is None: + return + chunks, self.recording, self.recording_source = self.recording, None, None + stream, self.stream = self.stream, None + self.update_mic() + seconds = len(chunks) * CHUNK / RATE + if seconds < self.dcfg["min_seconds"]: + if stream: + stream["worker"].cancel() + log.info("dictation discarded: %.1fs is under the %.1fs minimum", seconds, self.dcfg["min_seconds"]) + self.chime("cancel") + self.broadcast("dictation_cancel", reason="too_short", seconds=seconds) + return + self.chime("stop") + if stream: + per = CHUNK / RATE + log.info("recorded %.1fs: %.1fs labeled speech, peak level %.4f", seconds, + sum(stream["speech"]) * per, max(stream["level"], default=0)) + if self.dcfg["save_clips"]: + self.save_clip(np.concatenate(chunks)) + try: + if stream: + stream["final"] = True + stream["wake"].set() + await stream["worker"] + text = " ".join(stream["words"]) + if not text: + log.info("no speech detected") + else: + text = await self.transcribe(np.concatenate(chunks)) + except Exception as err: + log.error("transcription failed: %s", err) + self.chime("cancel") + self.broadcast("error", message=str(err)) + return + self.broadcast("dictation", text=text) + if stream: + await self._flush_typing(stream, final=True) + elif text: + await self.type_text(text + (" " if self.dcfg["trailing_space"] else "")) + + def save_clip(self, samples): + CLIP_DIR.mkdir(parents=True, exist_ok=True) + (CLIP_DIR / time.strftime("%Y%m%d-%H%M%S.wav")).write_bytes(to_wav(samples)) + for old in sorted(CLIP_DIR.glob("*.wav"))[:-self.dcfg["save_clips"]]: + old.unlink() + + async def _decode(self, audio, context): + """Text for a window of audio from the live engine, without the "..." whisper puts on + text that stops mid-sentence.""" + if self.moonshine: + async with self.whisper_lock: + text = await asyncio.to_thread(self._moonshine_decode, audio) + text = drop_repeats(re.sub(r"\s+", " ", text).strip()) + else: + text = await self.transcribe_speech(audio, context, quiet=True) + text = re.sub(r"([,;:!?])\1+", r"\1", text) # "so,," -> "so," + return re.sub(r"^(\.\.\.|\u2026)\s*|\s*(\.\.\.|\u2026)$", "", text) + + def _moonshine_decode(self, audio): + # Quiet speech is scaled up to a fixed peak first, as Faceclaw does (helped at low levels). + peak = float(np.abs(audio).max()) or 1.0 + audio = audio * min(10.0, 0.5 / peak) + stream = self.moonshine.create_stream() + stream.accept_waveform(RATE, audio) + self.moonshine.decode_stream(stream) + return stream.result.text + + async def _stream_worker(self, stream): + """Live transcription by agreement. + + The current window (speech since the last pause) is re-decoded every step_seconds of + new audio. Words that two decodes in a row agree on are committed and typed, except + the last agreed word, whose punctuation can still change. At a pause, or at the + window's quietest point once it is max_window_seconds long, the window closes: it is + decoded once more and the rest of its words are committed. Nothing is cut + mid-sentence while it is still being decoded, so there are no split or doubled words + at piece edges. + """ + per = CHUNK / RATE + step = max(1, round(self.dcfg["step_seconds"] / per)) + while True: + await stream["wake"].wait() + stream["wake"].clear() + speech, win = stream["speech"], stream["win"] + end = len(speech) + window = speech[win:end] + if stream["final"]: + if any(window): + await self._close_window(stream, end) + return + if not any(window): + if len(window) * per > 1.0: + stream["win"] = stream["decoded_at"] = end - int(0.3 / per) # keep a lead-in + continue + silence = len(window) - (max(i for i, v in enumerate(window) if v) + 1) + if silence * per >= self.dcfg["pause_seconds"]: + await self._close_window(stream, end - silence + int(0.15 / per)) + elif len(window) * per >= self.dcfg["max_window_seconds"]: + # No pause: close at the quietest chunk of the second half, preferring ones + # the VAD calls non-speech (a breath between words rather than inside one). + level = stream["level"] + first = win + int(len(window) / 2) + cut = min(range(first, end), key=lambda i: (speech[i], level[i], -i)) + 1 + await self._close_window(stream, cut) + elif end - stream["decoded_at"] >= step: + stream["decoded_at"] = end + hyp = (await self._decode(np.concatenate(stream["chunks"][win:end]), self._context(stream))).split() + agreed = common_prefix(stream["prev"], hyp) + stream["prev"] = hyp + done = len(stream["wdone"]) + if agreed - 1 > done: + self._commit(stream, hyp[done:agreed - 1]) + + async def _close_window(self, stream, cut): + win = stream["win"] + words = (await self._decode(np.concatenate(stream["chunks"][win:cut]), self._context(stream))).split() + # Words already typed stay as typed; the rest of the final decode follows them. + self._commit(stream, words[len(stream["wdone"]):]) + log.info("window %.1fs: %s", (cut - win) * CHUNK / RATE, " ".join(words)) + stream.update(win=cut, decoded_at=cut, prev=[], wdone=[]) + + def _context(self, stream): + return " ".join(stream["words"][-30:]) + + def _commit(self, stream, words): + if not words: + return + skip = 0 + if not stream["wdone"]: + # A window's first words can repeat the end of the previous one when it closed + # inside a phrase. + skip = max((k for k in range(1, 4) if common_prefix([w for w in stream["words"][-k:]], words[:k]) == k + and len(stream["words"]) >= k), default=0) + stream["wdone"].extend(words) + words = words[skip:] + if not words: + return + stream["words"].extend(words) + self.broadcast("dictation_piece", text=" ".join(words)) + task = asyncio.create_task(self._flush_typing(stream)) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + + async def _flush_typing(self, stream, final=False): + """Type committed words not typed yet: live while recording when allowed, else at the end.""" + if not final: + live = (self.dcfg["live_typing"] and stream is self.stream + and time.monotonic() - stream["t0"] >= self.dcfg["min_seconds"]) + if not live or self.mods_down: + return # a held Ctrl/Alt/Super would turn letters into shortcuts; retried on release + elif self.mods_down: + await self.mods_released.wait() + new = stream["words"][stream["typed"]:] + text = (" " if stream["typed"] and new else "") + " ".join(new) + stream["typed"] += len(new) + if final and stream["words"] and self.dcfg["trailing_space"]: + text += " " + if text: + if not final: + log.info("typing %d words live (%.1fs in)", len(new), time.monotonic() - stream["t0"]) + await self.type_text(text) + + async def toggle_dictation(self, source): + if self.recording is None: + self.start_dictation(source) + else: + await self.stop_dictation() + + # --- wake word + + def on_wake(self, keyword): + log.info("wake word: %s", keyword) + self.chime("wake") + self.broadcast("wake", keyword=keyword) + self.run_hook("wake_hook", keyword, env={"SCARLET_EVENT": "wake"}) + self.command = {"chunks": [], "t0": time.monotonic(), "heard": False} + self.cmd_vad.reset() + self.update_mic() + + def _command_audio(self, chunk): + cmd = self.command + cmd["chunks"].append(chunk) + feed_vad(self.cmd_vad, chunk) + while not self.cmd_vad.empty(): + self.cmd_vad.pop() + speaking = self.cmd_vad.is_speech_detected() + cmd["heard"] |= speaking + elapsed = time.monotonic() - cmd["t0"] + if cmd["heard"] and not speaking or elapsed > self.wcfg["max_command_seconds"]: + self.command = None + asyncio.create_task(self.finish_command(np.concatenate(cmd["chunks"]))) + elif not cmd["heard"] and elapsed > self.wcfg["command_timeout"]: + self.command = None + log.info("wake: no command heard") + self.chime("cancel") + self.broadcast("command_timeout") + if self.command is None: + self.kws.reset_stream(self.kws_stream) + self.update_mic() + + async def finish_command(self, samples): + self.chime("stop") + try: + text = await self.transcribe(samples) + except Exception as err: + log.error("transcription failed: %s", err) + self.broadcast("error", message=str(err)) + return + self.broadcast("command", text=text) + if text: + self.run_hook("command_hook", text, env={"SCARLET_EVENT": "command"}) + + # --- hotkey + + def on_key(self, dev, code, value): + loop = asyncio.get_running_loop() + if code in MODIFIER_CODES: + self._track_modifier(dev, code, value) + if code in self.hotkeys: + if value == 1: + self.hk_down, self.hk_chord = True, False + if self.kb_mode == "hold": + self.hk_timer = loop.call_later(self.dcfg["hold_delay"], self._hold_start) + elif value == 0: + self.hk_down = False + if self.hk_timer: + self.hk_timer.cancel() + if self.kb_mode == "hold": + if self.recording_source == "hotkey": + asyncio.create_task(self.stop_dictation()) + elif not self.hk_chord: + asyncio.create_task(self.toggle_dictation("hotkey")) + elif value == 1 and self.hk_down: + # Hotkey used as a modifier (e.g. Ctrl+C): not a dictation gesture. + self.hk_chord = True + if self.hk_timer: + self.hk_timer.cancel() + if self.kb_mode == "hold" and self.recording_source == "hotkey": + self.cancel_dictation(chime=False) + + def _track_modifier(self, dev, code, value): + key = (dev.path, code) + if value: + self.mods_down.add(key) + self.mods_released.clear() + else: + self.mods_down.discard(key) + self._modifiers_changed() + + def _modifiers_changed(self): + if not self.mods_down: + self.mods_released.set() + if self.stream: # type words committed while a modifier was held + asyncio.create_task(self._flush_typing(self.stream)) + + def _hold_start(self): + if self.hk_down and not self.hk_chord: + self.start_dictation("hotkey") + + async def watch_devices(self): + watching = {} + while True: + for path in evdev.list_devices(): + if path in watching and not watching[path].done(): + continue + try: + dev = evdev.InputDevice(path) + except OSError: + continue + keys = set(dev.capabilities().get(e.EV_KEY, [])) + if dev.name == TYPER_NAME or not (self.hotkeys.keys() | MODIFIER_CODES) & keys: + dev.close() + continue + log.info("hotkey: watching %s (%s)", dev.name, path) + watching[path] = asyncio.create_task(self._read_device(dev)) + if bindings.mtime() != self.bindings_mtime: + self.load_bindings() + continue # rescan now for devices with the new keys + await asyncio.sleep(2) + + def load_bindings(self): + self.bindings_mtime = bindings.mtime() + b = bindings.load() # defaults when the file doesn't exist yet + self.hotkeys = {} + for name in b["keyboard"]: + code = e.ecodes.get(name) + if isinstance(code, int): + self.hotkeys[code] = name + else: + log.warning("bindings: unknown key %r", name) + log.info("keyboard/mouse push-to-talk: %s", ", ".join(self.hotkeys.values()) or "none") + self.apply_options(b) + + def apply_options(self, b): + """Settings from the config app. Only changed ones are applied, so a save doesn't + undo e.g. `scarletctl wake off`.""" + changed = {k: b[k] for k in ("keyboard_mode", *bindings.OPTIONS) if self.applied.get(k) != b[k]} + self.applied.update(changed) + if "keyboard_mode" in changed: + self.kb_mode = changed["keyboard_mode"] + if "live_typing" in changed: + self.dcfg["live_typing"] = changed["live_typing"] + if "sounds" in changed: + self.sounds = make_sounds(self.cfg["sounds"]["volume"]) if changed["sounds"] else {} + if "wake_word" in changed and changed["wake_word"] != self.wake_enabled: + self.wake_enabled = changed["wake_word"] + self.kws.reset_stream(self.kws_stream) + try: + asyncio.get_running_loop() + self.update_mic() + except RuntimeError: + pass # at startup; run() opens the mic + if changed: + log.info("settings: %s", ", ".join(f"{k}={v}" for k, v in changed.items())) + + async def _read_device(self, dev): + try: + async for ev in dev.async_read_loop(): + if ev.type == e.EV_KEY: + self.on_key(dev, ev.code, ev.value) + except OSError: + log.info("hotkey: lost %s", dev.name) + finally: + self.mods_down = {m for m in self.mods_down if m[0] != dev.path} + self._modifiers_changed() + dev.close() + + # --- control socket + + async def handle_client(self, reader, writer): + try: + while line := await reader.readline(): + cmd, _, arg = line.decode().strip().partition(" ") + reply = await self.command_reply(cmd, arg, writer) + if reply is None: + return # subscribed to events; keep the connection open + writer.write((json.dumps(reply) + "\n").encode()) + await writer.drain() + except (ConnectionError, UnicodeDecodeError): + pass + finally: + if writer not in self.subscribers: + writer.close() + + async def command_reply(self, cmd, arg, writer): + match cmd: + case "toggle": + await self.toggle_dictation(arg or "socket") + case "start": + self.start_dictation(arg or "socket") + case "stop": + await self.stop_dictation() + case "cancel": + self.cancel_dictation() + case "type": + await self.type_text(arg) + case "wake": + if arg in ("on", "off", "toggle"): + self.wake_enabled = {"on": True, "off": False}.get(arg, not self.wake_enabled) + self.kws.reset_stream(self.kws_stream) + self.update_mic() + else: + self.on_wake("manual") + case "status": + pass + case "reload": + self.load_bindings() + case "events": + self.subscribers.add(writer) + writer.write((json.dumps({"event": "subscribed", **self.state()}) + "\n").encode()) + return None + case _: + return {"ok": False, "error": f"unknown command {cmd!r}"} + return {"ok": True, **self.state()} + + async def run(self): + RUNTIME_DIR.mkdir(parents=True, exist_ok=True, mode=0o700) + SOCKET_PATH.unlink(missing_ok=True) + old_umask = os.umask(0o177) # socket is owner-only: it can type into any focused app + server = await asyncio.start_unix_server(self.handle_client, path=str(SOCKET_PATH)) + os.umask(old_umask) + log.info("control socket: %s", SOCKET_PATH) + sd_notify("READY=1") + log.info("dictation: keyboard %s, min %.1fs", self.kb_mode, self.dcfg["min_seconds"]) + self.update_mic() + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, stop.set) + watcher = asyncio.create_task(self.watch_devices()) + await stop.wait() + watcher.cancel() + server.close() + if self.mic_proc and self.mic_proc.returncode is None: + self.mic_proc.terminate() + self.typer.close() + SOCKET_PATH.unlink(missing_ok=True) + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + asyncio.run(ScarletVoice(load_config()).run()) + + +if __name__ == "__main__": + main() diff --git a/scarlet_vr_input.py b/scarlet_vr_input.py new file mode 100755 index 0000000..acfe510 --- /dev/null +++ b/scarlet_vr_input.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Controller push-to-talk for scarlet-voice: hold a bound button (or tap it, in toggle mode). + +SteamVR only gives controller input to the app that has focus, and while the 2D Steam +or desktop screens are up the compositor's laser mouse holds it, so an OpenVR action +never fires there. Instead this reads raw component state from vrserver's local web +socket (127.0.0.1:27062), the same stream the controller binding page uses for its live +view. It sees the button whatever has focus. Press -> `start vr`, release -> `stop` on +the scarlet-voice control socket; the daemon applies the 2 s minimum. + +The button is observed, not captured: whatever else is bound to it still happens. +Buttons come from ~/.config/scarlet-voice/bindings.json (see bindings.py and the +scarlet-config app); changes apply within a second. +""" +import asyncio +import json +import logging +import os +import socket +import time +import urllib.request +from pathlib import Path + +import websockets + +import bindings + +SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +VRSERVER = "http://127.0.0.1:27062" +MAILBOX = f"scarlet_voice_ptt_{os.getpid()}" +VR_HEADERS = {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} +DEVICE_POLL = 5 # seconds between checks for controllers connecting or leaving + +log = logging.getLogger("scarlet-vr-input") + + +def send(command): + try: + with socket.socket(socket.AF_UNIX) as sock: + sock.settimeout(2) + sock.connect(str(SOCKET_PATH)) + sock.sendall((command + "\n").encode()) + sock.recv(4096) + except OSError as err: + log.warning("scarlet-voice not reachable (%s): %s", command, err) + + +async def connect(): + # vrserver serves the binding page, and its socket, to local clients from that page. + return await websockets.connect(VRSERVER.replace("http", "ws"), origin=VRSERVER, max_size=None, + additional_headers=VR_HEADERS) + + +def connected(): + """Root paths of the controllers SteamVR has right now (vrserver only streams changes, + so a quiet stream says nothing about whether a controller is still there).""" + req = urllib.request.Request(f"{VRSERVER}/input/getstate.json", headers=VR_HEADERS) + with urllib.request.urlopen(req, timeout=3) as resp: + return {d["root_path"] for d in json.load(resp).get("devices", []) if d.get("root_path")} + + +async def subscribe(ws, device, mailbox, on=True): + kind = "request_input_state_updates" if on else "cancel_input_state_updates" + message = {"type": kind, "device_path": device, "returnAddress": mailbox} + await ws.send(f"mailbox_send input_server {json.dumps(message)}") + + +def wanted(): + """device -> set of inputs, from bindings.json.""" + out = {} + for b in bindings.load()["controller"]: + out.setdefault(b["device"], set()).add(b["input"]) + return out + + +async def run_session(): + """One connection to vrserver. Returns or raises when SteamVR goes away.""" + async with await connect() as ws: + await ws.send(f"mailbox_open {MAILBOX}") + stamp, inputs, mode = bindings.mtime(), wanted(), bindings.load()["controller_mode"] + for device in inputs: + await subscribe(ws, device, MAILBOX) + log.info("connected to SteamVR, watching %s (%s)", describe(inputs), mode) + held = False + pressed = set() # (device, input) currently down + present = await asyncio.to_thread(connected) + next_poll = time.monotonic() + DEVICE_POLL + try: + while True: + try: + raw = await asyncio.wait_for(ws.recv(), 1) + except asyncio.TimeoutError: + raw = None + now = time.monotonic() + if raw is not None: + message = json.loads(raw) + device = message.get("device") + if message.get("type") == "update_component_states" and device in inputs: + components = message.get("components", {}) + for name in inputs[device]: + if components.get(name): + pressed.add((device, name)) + elif name in components: + pressed.discard((device, name)) + if bindings.mtime() != stamp: + stamp, new, mode = bindings.mtime(), wanted(), bindings.load()["controller_mode"] + for device in inputs.keys() - new.keys(): + await subscribe(ws, device, MAILBOX, on=False) + for device in new.keys() - inputs.keys(): + await subscribe(ws, device, MAILBOX) + inputs = new + pressed = {(d, i) for d, i in pressed if i in inputs.get(d, ())} + log.info("bindings changed, watching %s (%s)", describe(inputs), mode) + if now >= next_poll: + next_poll = now + DEVICE_POLL + current = await asyncio.to_thread(connected) + for device in inputs.keys() & (current - present): + log.info("%s connected", device) + await subscribe(ws, device, MAILBOX, on=False) + await subscribe(ws, device, MAILBOX) + for device in inputs.keys() & (present - current): + log.info("%s disconnected", device) + pressed = {(d, i) for d, i in pressed if d != device} + present = current + if pressed and not held: + held = True + send("toggle vr" if mode == "toggle" else "start vr") + elif held and not pressed: + held = False + if mode == "hold": + send("stop") + finally: + if held and mode == "hold": + send("stop") + + +def describe(inputs): + return ", ".join(f"{d} {i}" for d, names in inputs.items() for i in sorted(names)) or "nothing" + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + while True: + try: + asyncio.run(run_session()) + except (OSError, websockets.WebSocketException) as err: + log.info("SteamVR not reachable (%s); retrying", err) + time.sleep(5) + + +if __name__ == "__main__": + main() diff --git a/scarletctl b/scarletctl new file mode 100755 index 0000000..fed2db8 --- /dev/null +++ b/scarletctl @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +"""Control the scarlet-voice daemon: scarletctl toggle|start|stop|cancel|status|type TEXT|wake [on|off|toggle]|events""" +import os +import socket +import sys +from pathlib import Path + +path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): + sys.exit(__doc__) +sock = socket.socket(socket.AF_UNIX) +try: + sock.connect(str(path)) +except OSError as err: + sys.exit(f"scarlet-voice not reachable at {path}: {err}") +sock.sendall((" ".join(sys.argv[1:]) + "\n").encode()) +f = sock.makefile() +if sys.argv[1] == "events": + for line in f: + print(line, end="", flush=True) +else: + print(f.readline(), end="") diff --git a/systemd/scarlet-voice.service b/systemd/scarlet-voice.service new file mode 100644 index 0000000..0979da5 --- /dev/null +++ b/systemd/scarlet-voice.service @@ -0,0 +1,22 @@ +[Unit] +Description=scarlet-voice dictation and wake word daemon +Wants=scarlet-whisper.service pipewire.service +After=scarlet-whisper.service pipewire.service +# SteamVR opens input devices only at startup, and gamescope gets keyboard input through +# SteamVR, so the typing keyboard must exist before SteamVR starts. +Before=steamvr.service + +[Service] +# READY=1 is sent after the virtual keyboard exists. +Type=notify +NotifyAccess=main +ExecStart=%h/Desktop/Projects/scarlet-voice/.venv/bin/python %h/Desktop/Projects/scarlet-voice/scarlet_voice.py +Restart=on-failure +RestartSec=2 +# systemd keeps the virtual keyboard's fd across restarts, so SteamVR keeps the same device. +# Clear with: systemctl --user clean --what=fdstore scarlet-voice +FileDescriptorStoreMax=1 +FileDescriptorStorePreserve=yes + +[Install] +WantedBy=default.target steamvr.service diff --git a/systemd/scarlet-vr-input.service b/systemd/scarlet-vr-input.service new file mode 100644 index 0000000..aad3808 --- /dev/null +++ b/systemd/scarlet-vr-input.service @@ -0,0 +1,14 @@ +[Unit] +Description=scarlet-voice controller push-to-talk (left d-pad down) +# No After=steamvr.service: default.target is ordered after its wants and steamvr after +# the graphical session, so that would be an ordering cycle. The script retries until +# SteamVR is up. +After=scarlet-voice.service + +[Service] +ExecStart=%h/Desktop/Projects/scarlet-voice/.venv/bin/python %h/Desktop/Projects/scarlet-voice/scarlet_vr_input.py +Restart=always +RestartSec=5 + +[Install] +WantedBy=default.target steamvr.service diff --git a/systemd/scarlet-whisper.service b/systemd/scarlet-whisper.service new file mode 100644 index 0000000..02fb459 --- /dev/null +++ b/systemd/scarlet-whisper.service @@ -0,0 +1,12 @@ +[Unit] +Description=whisper.cpp server for scarlet-voice + +[Service] +# The Gamescope WSI layer asserts on compute-only Vulkan devices; whisper needs no swapchain. +Environment=DISABLE_GAMESCOPE_WSI=1 +ExecStart=%h/Desktop/Projects/scarlet-voice/vendor/whisper.cpp/build/bin/whisper-server -m %h/Desktop/Projects/scarlet-voice/models/ggml-small.en-q5_1.bin --host 127.0.0.1 --port 8178 -t 3 -fa +Restart=on-failure +Nice=5 + +[Install] +WantedBy=default.target From 128d5a20351a51a293c81a74267e432b8b3ed3b2 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:06:06 -0600 Subject: [PATCH 2/9] Name the project frame-voice Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index db4b60f..487e6ed 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ -# scarlet-voice +# frame-voice Local voice input for the Steam Frame. Nothing leaves the headset. +The services and commands are named `scarlet-*` after Scarlet, the assistant this was built for. + - **Dictation**: hold a key, speak, release. The text is typed into whatever has focus (desktop apps, search bars, terminals, game-mode windows) through a virtual uinput keyboard. - **Wake word**: say "Hey Scarlet", then a command. The command is transcribed and handed to a hook script and to any event subscribers. This is the entry point for the future Scarlet assistant. From ef10d84e2f58e0632dc9761094658dac33564815 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:16:49 -0600 Subject: [PATCH 3/9] Rename to frame-voice; make wake words configurable Everything named scarlet is now frame-voice: scripts, services (frame-voice, frame-voice-whisper, frame-voice-vr-input), the frame-voice-ctl and frame-voice-config commands, paths under ~/.config, ~/.local/state and the runtime dir, and the virtual keyboard. Bindings and options move to ~/.config/frame-voice/settings.json. Wake words are a list in settings.json, managed in the settings app (add, remove, listening on/off). The daemon rebuilds the keyword spotter when the list changes; any English words work. The default is "Hey Computer". The whisper service sets LD_LIBRARY_PATH, since the binary's RUNPATH is the directory it was built in. Co-Authored-By: Claude Opus 5.5 --- README.md | 58 +++++----- config.example.toml | 11 +- scarlet-config => frame-voice-config | 2 +- scarletctl => frame-voice-ctl | 6 +- scarlet_voice.py => frame_voice.py | 106 +++++++++++------- scarlet_config.py => frame_voice_config.py | 75 +++++++++---- ...let_vr_input.py => frame_voice_vr_input.py | 30 ++--- hooks/on-command | 8 +- bindings.py => settings.py | 20 +++- ...t.service => frame-voice-vr-input.service} | 6 +- systemd/frame-voice-whisper.service | 15 +++ ...rlet-voice.service => frame-voice.service} | 10 +- systemd/scarlet-whisper.service | 12 -- 13 files changed, 213 insertions(+), 146 deletions(-) rename scarlet-config => frame-voice-config (79%) rename scarletctl => frame-voice-ctl (69%) rename scarlet_voice.py => frame_voice.py (92%) rename scarlet_config.py => frame_voice_config.py (82%) rename scarlet_vr_input.py => frame_voice_vr_input.py (86%) rename bindings.py => settings.py (73%) rename systemd/{scarlet-vr-input.service => frame-voice-vr-input.service} (57%) create mode 100644 systemd/frame-voice-whisper.service rename systemd/{scarlet-voice.service => frame-voice.service} (60%) delete mode 100644 systemd/scarlet-whisper.service diff --git a/README.md b/README.md index 487e6ed..d4d4072 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,19 @@ Local voice input for the Steam Frame. Nothing leaves the headset. -The services and commands are named `scarlet-*` after Scarlet, the assistant this was built for. - -- **Dictation**: hold a key, speak, release. The text is typed into whatever has focus (desktop apps, search bars, terminals, game-mode windows) through a virtual uinput keyboard. -- **Wake word**: say "Hey Scarlet", then a command. The command is transcribed and handed to a hook script and to any event subscribers. This is the entry point for the future Scarlet assistant. +- **Dictation**: tap a key (or hold a controller button) and speak. Words are typed as you talk into whatever has focus (desktop apps, search bars, terminals, game-mode windows) through a virtual uinput keyboard, transcribed on the headset by Moonshine. +- **Wake words**: say one (default "Hey Computer"; add your own in the settings app), then a command. The command is transcribed and handed to a hook script and to any event subscribers. This is the entry point for a voice assistant. ## Pieces | Part | What it does | | --- | --- | -| `scarlet-whisper.service` | `whisper-server` (whisper.cpp, Vulkan on the Adreno 750) with `ggml-small.en-q5_1` loaded, on `127.0.0.1:8178` | -| `scarlet-voice.service` | `scarlet_voice.py`: mic capture, hotkey, wake word, typing, control socket | -| `scarlet-vr-input.service` | `scarlet_vr_input.py`: controller push-to-talk, reads left d-pad down from SteamVR's raw input stream | -| `scarlet-config` | Settings app (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. In the app menu as *Scarlet Voice Settings* | -| `scarletctl` | CLI for the control socket | -| `hooks/on-command` | Runs after "Hey Scarlet ..." with the command text as `$1` | +| `frame-voice-whisper.service` | `whisper-server` (whisper.cpp, Vulkan on the Adreno 750) with `ggml-small.en-q5_1` loaded, on `127.0.0.1:8178` | +| `frame-voice.service` | `frame_voice.py`: mic capture, hotkey, wake word, typing, control socket | +| `frame-voice-vr-input.service` | `frame_voice_vr_input.py`: controller push-to-talk, reads left d-pad down from SteamVR's raw input stream | +| `frame-voice-config` | Settings app (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. In the app menu as *Frame Voice Settings* | +| `frame-voice-ctl` | CLI for the control socket | +| `hooks/on-command` | Runs after a wake word with the command text as `$1` | The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx open-vocabulary KWS, about 5% of one core) listens continuously. Silero VAD ends commands on silence and trims clips before transcription. @@ -26,15 +24,15 @@ The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx op | --- | --- | | Keyboard or mouse | Tap **Right Ctrl** (default) to start, tap again to stop. Shortcuts with it (Ctrl+C) don't start or stop dictation. Hold mode is available in the settings app. | | Steam Frame controllers | Hold left **d-pad down** (default), speak, release. Works in games, the Steam UI and the desktop with no per-app setup. The button is observed, not captured, so whatever else it is bound to still happens. | -| Desktop mode shortcut | KDE System Settings > Shortcuts > add a command shortcut running `scarletctl toggle`. | -| Scripts, the assistant | `scarletctl toggle`, `start`, `stop`, `cancel`, `type "text"` | -| Voice | "Hey Scarlet, ..." runs `hooks/on-command`. "Hey Scarlet, type hello world" types "hello world" (example in the hook). | +| Desktop mode shortcut | KDE System Settings > Shortcuts > add a command shortcut running `frame-voice-ctl toggle`. | +| Scripts, the assistant | `frame-voice-ctl toggle`, `start`, `stop`, `cancel`, `type "text"` | +| Voice | ", ..." runs `hooks/on-command`. "Hey Computer, type hello world" types "hello world" (example in the hook). | Recordings shorter than 2 s (`min_seconds`) are discarded with the cancel tone, so brief or accidental presses never type anything. -Change the buttons in *Scarlet Voice Settings* (`./scarlet-config`): **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger and switches live typing, sounds and the wake word. Everything lives in `~/.config/scarlet-voice/bindings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. +Change the buttons in *Frame Voice Settings* (`./frame-voice-config`): **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. -Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. At a 0.8 s pause, or at the quietest point once the window reaches 8 s, the window is decoded one last time and its remaining words are typed. On test audio the first words appeared at about 2.3 s, then every 1-1.5 s, and the text was complete 0.1-0.2 s after release. Nothing is cut while it is still being decoded, so words aren't split or doubled at the edges the way fixed pieces did. Moonshine adds fewer commas than whisper, and a long pause can still end a sentence with a period. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). The "Hey Scarlet" command path still uses whisper. +Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. At a 0.8 s pause, or at the quietest point once the window reaches 8 s, the window is decoded one last time and its remaining words are typed. On test audio the first words appeared at about 2.3 s, then every 1-1.5 s, and the text was complete 0.1-0.2 s after release. Nothing is cut while it is still being decoded, so words aren't split or doubled at the edges the way fixed pieces did. Moonshine adds fewer commas than whisper, and a long pause can still end a sentence with a period. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). Wake-word commands still use whisper. Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) @@ -43,30 +41,30 @@ Audio cues are off by default (`[sounds] enabled = true` turns them on): rising ## Control and events ``` -./scarletctl status -./scarletctl wake off # stop wake word listening; the mic fully closes when idle -./scarletctl wake on -./scarletctl wake # act as if "Hey Scarlet" was heard -./scarletctl type "text" # type into the focused app -./scarletctl events # stream JSON events: wake, command, dictation, command_timeout, error +./frame-voice-ctl status +./frame-voice-ctl wake off # stop wake word listening; the mic fully closes when idle +./frame-voice-ctl wake on +./frame-voice-ctl wake # act as if a wake word was heard +./frame-voice-ctl type "text" # type into the focused app +./frame-voice-ctl events # stream JSON events: wake, command, dictation, command_timeout, error ``` -The socket is `$XDG_RUNTIME_DIR/scarlet-voice/control.sock`, mode 0600. It can type into any focused window, so keep it owner-only. +The socket is `$XDG_RUNTIME_DIR/frame-voice/control.sock`, mode 0600. It can type into any focused window, so keep it owner-only. -An assistant can either replace `hooks/on-command` or subscribe with `scarletctl events` and act on `{"event": "command", "text": ...}`. +An assistant can either replace `hooks/on-command` or subscribe with `frame-voice-ctl events` and act on `{"event": "command", "text": ...}`. ## Config -Copy `config.example.toml` to `~/.config/scarlet-voice/config.toml`. Every key is optional. Then `systemctl --user restart scarlet-voice`. +Copy `config.example.toml` to `~/.config/frame-voice/config.toml`. Every key is optional. Then `systemctl --user restart frame-voice`. Wake phrase tuning: raise `threshold` for fewer false triggers, raise `boost` if it misses you. Phrases can be anything spelled in English words; they are tokenized at startup, no training needed. ## Services ``` -systemctl --user status scarlet-whisper scarlet-voice scarlet-vr-input -journalctl --user -u scarlet-voice -f -systemctl --user stop scarlet-voice scarlet-whisper scarlet-vr-input # the units are linked, so `disable` removes the links +systemctl --user status frame-voice-whisper frame-voice frame-voice-vr-input +journalctl --user -u frame-voice -f +systemctl --user stop frame-voice frame-voice-whisper frame-voice-vr-input # the units are linked, so `disable` removes the links ``` ## Build notes @@ -82,13 +80,13 @@ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_NATIV cmake --build build -j5 --target whisper-server whisper-cli whisper-bench ``` -`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `scarlet-config` launcher sets that up. +`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `frame-voice-config` launcher sets that up. The whisper service sets `DISABLE_GAMESCOPE_WSI=1`. Without it the Gamescope Vulkan layer aborts whisper-server at startup. -Typing goes through a uinput keyboard that SteamVR must have open: SteamVR opens `/dev/input` devices only when it starts, and gamescope gets keyboard input from SteamVR. `scarlet-voice.service` is ordered `Before=steamvr.service` and parks the device's fd in systemd's fd store, so daemon restarts keep the same device. After the very first install, restart SteamVR once. The device is a full keyboard (fd store name `typer2`; the first version, `typer`, had only the typing keys and is replaced automatically, which also needs one SteamVR restart). Multidesk's input relay picks the device up immediately for its desktop screens. To drop the device: `systemctl --user clean --what=fdstore scarlet-voice`. +Typing goes through a uinput keyboard that SteamVR must have open: SteamVR opens `/dev/input` devices only when it starts, and gamescope gets keyboard input from SteamVR. `frame-voice.service` is ordered `Before=steamvr.service` and parks the device's fd in systemd's fd store, so daemon restarts keep the same device. After the very first install, restart SteamVR once. The device is a full keyboard (fd store name `typer2`; the first version, `typer`, had only the typing keys and is replaced automatically, which also needs one SteamVR restart). Multidesk's input relay picks the device up immediately for its desktop screens. To drop the device: `systemctl --user clean --what=fdstore frame-voice`. -Controller input: SteamVR sends OpenVR actions only to the app with input focus, and while the 2D Steam or desktop screens are up the compositor's laser mouse holds focus, so a background or overlay OpenVR app never sees the button there. `scarlet_vr_input.py` instead subscribes to vrserver's local web socket (`ws://127.0.0.1:27062`, mailbox `input_server`, `request_input_state_updates`), the stream the controller binding page uses for its live view. It is undocumented and could change with a SteamVR update. `scarlet-vr-input.service` has no `After=steamvr.service`: with `WantedBy=default.target` that is an ordering cycle at boot. +Controller input: SteamVR sends OpenVR actions only to the app with input focus, and while the 2D Steam or desktop screens are up the compositor's laser mouse holds focus, so a background or overlay OpenVR app never sees the button there. `frame_voice_vr_input.py` instead subscribes to vrserver's local web socket (`ws://127.0.0.1:27062`, mailbox `input_server`, `request_input_state_updates`), the stream the controller binding page uses for its live view. It is undocumented and could change with a SteamVR update. `frame-voice-vr-input.service` has no `After=steamvr.service`: with `WantedBy=default.target` that is an ordering cycle at boot. ## Measurements (2026-09-25, jfk.wav) diff --git a/config.example.toml b/config.example.toml index 271e03b..9ce75f1 100644 --- a/config.example.toml +++ b/config.example.toml @@ -1,6 +1,6 @@ -# Copy to ~/.config/scarlet-voice/config.toml and edit. Every key is optional. -# Push-to-talk buttons, hold/tap modes, live typing, sounds and the wake word on/off -# are set in the Scarlet Voice Settings app (scarlet-config), saved in bindings.json. +# Copy to ~/.config/frame-voice/config.toml and edit. Every key is optional. +# Push-to-talk buttons, hold/tap modes, live typing, sounds, wake words and wake on/off +# are set in the Frame Voice Settings app (frame-voice-config), saved in settings.json. [audio] source = "" # PipeWire node name; empty = default mic @@ -8,7 +8,7 @@ source = "" # PipeWire node name; empty = default mic [whisper] url = "http://127.0.0.1:8178/inference" language = "en" -prompt = "Scarlet, SteamOS, Steam Frame, T3 Code, Godot." # vocabulary hints +prompt = "SteamOS, Steam Frame, Godot." # vocabulary hints min_audio_ctx = 512 # smallest encoder window; lower makes medium/turbo loop on short clips [dictation] @@ -23,10 +23,9 @@ moonshine_threads = 2 step_seconds = 0.8 # re-decode the current window this often pause_seconds = 0.8 # a pause this long closes the window max_window_seconds = 8.0 # without a pause, close at the quietest point by this length (moonshine max ~10) -save_clips = 0 # keep the last N dictations' audio in ~/.local/state/scarlet-voice/clips +save_clips = 0 # keep the last N dictations' audio in ~/.local/state/frame-voice/clips [wake] -phrases = ["HEY SCARLET"] threshold = 0.25 # higher = fewer false triggers, more misses boost = 1.5 # higher = easier to trigger command_timeout = 5.0 # seconds to start speaking after the wake chime diff --git a/scarlet-config b/frame-voice-config similarity index 79% rename from scarlet-config rename to frame-voice-config index a247a99..f591d84 100755 --- a/scarlet-config +++ b/frame-voice-config @@ -1,4 +1,4 @@ #!/bin/sh # GTK comes from the system Python; evdev and websockets from the project venv. dir=$(dirname "$(readlink -f "$0")") -PYTHONPATH="$dir:$dir/.venv/lib/python3.12/site-packages" exec /usr/bin/python3 "$dir/scarlet_config.py" "$@" +PYTHONPATH="$dir:$dir/.venv/lib/python3.12/site-packages" exec /usr/bin/python3 "$dir/frame_voice_config.py" "$@" diff --git a/scarletctl b/frame-voice-ctl similarity index 69% rename from scarletctl rename to frame-voice-ctl index fed2db8..1900cf4 100755 --- a/scarletctl +++ b/frame-voice-ctl @@ -1,18 +1,18 @@ #!/usr/bin/env python3 -"""Control the scarlet-voice daemon: scarletctl toggle|start|stop|cancel|status|type TEXT|wake [on|off|toggle]|events""" +"""Control the frame-voice daemon: frame-voice-ctl toggle|start|stop|cancel|status|type TEXT|wake [on|off|toggle]|events""" import os import socket import sys from pathlib import Path -path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice/control.sock" if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): sys.exit(__doc__) sock = socket.socket(socket.AF_UNIX) try: sock.connect(str(path)) except OSError as err: - sys.exit(f"scarlet-voice not reachable at {path}: {err}") + sys.exit(f"frame-voice not reachable at {path}: {err}") sock.sendall((" ".join(sys.argv[1:]) + "\n").encode()) f = sock.makefile() if sys.argv[1] == "events": diff --git a/scarlet_voice.py b/frame_voice.py similarity index 92% rename from scarlet_voice.py rename to frame_voice.py index a6d0037..0917f64 100755 --- a/scarlet_voice.py +++ b/frame_voice.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""scarlet-voice: local push-to-talk dictation and "Hey Scarlet" wake word daemon. +"""frame-voice: local push-to-talk dictation and wake word daemon. One PipeWire capture stream feeds two consumers: * dictation: hotkey or socket command -> record -> whisper-server -> type via uinput @@ -24,7 +24,7 @@ from pathlib import Path import evdev -import bindings +import settings import numpy as np import sentencepiece import sherpa_onnx @@ -33,22 +33,22 @@ from evdev import ecodes as e ROOT = Path(__file__).resolve().parent RATE = 16000 CHUNK = 1536 # 96 ms of 16 kHz mono: a whole number of 512-sample VAD windows -MODIFIER_CODES = {e.ecodes[name] for name in bindings.MODIFIERS} -TYPER_NAME = "scarlet-voice keyboard" +MODIFIER_CODES = {e.ecodes[name] for name in settings.MODIFIERS} +TYPER_NAME = "frame-voice keyboard" KWS_DIR = ROOT / "models/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01" VAD_MODEL = ROOT / "models/silero_vad.onnx" -RUNTIME_DIR = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice" -CLIP_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "scarlet-voice/clips" +RUNTIME_DIR = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice" +CLIP_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "frame-voice/clips" SOCKET_PATH = RUNTIME_DIR / "control.sock" -log = logging.getLogger("scarlet-voice") +log = logging.getLogger("frame-voice") DEFAULTS = { "audio": {"source": ""}, "whisper": { "url": "http://127.0.0.1:8178/inference", "language": "en", - "prompt": "Scarlet, SteamOS, Steam Frame, T3 Code, Godot.", + "prompt": "SteamOS, Steam Frame, Godot.", # Smallest encoder window (50 per second of audio). medium and large-v3-turbo loop # and take 10-20 s on short clips below 512; 512 covers pieces up to ~10 s. "min_audio_ctx": 512, @@ -73,12 +73,11 @@ DEFAULTS = { "step_seconds": 0.8, "pause_seconds": 0.8, # shorter pauses inside a sentence ended it with "?"/"." in testing "max_window_seconds": 8.0, - # Keep the audio of the last N dictations in ~/.local/state/scarlet-voice/clips for + # Keep the audio of the last N dictations in ~/.local/state/frame-voice/clips for # comparing models and settings. 0 = never write audio to disk. "save_clips": 0, }, "wake": { - "phrases": ["HEY SCARLET"], "threshold": 0.25, "boost": 1.5, "command_timeout": 5.0, @@ -87,12 +86,12 @@ DEFAULTS = { "wake_hook": "", "command_hook": "hooks/on-command", }, - "sounds": {"volume": 0.5}, # on/off is in bindings.json (scarlet-config) + "sounds": {"volume": 0.5}, # on/off is in settings.json (frame-voice-config) } def load_config(): - path = Path(os.environ.get("SCARLET_VOICE_CONFIG", "~/.config/scarlet-voice/config.toml")).expanduser() + path = Path(os.environ.get("FRAME_VOICE_CONFIG", "~/.config/frame-voice/config.toml")).expanduser() cfg = {k: dict(v) for k, v in DEFAULTS.items()} if path.exists(): with open(path, "rb") as f: @@ -271,11 +270,11 @@ def to_wav(samples): # ---------------------------------------------------------------- daemon -class ScarletVoice: +class FrameVoice: def __init__(self, cfg): self.cfg = cfg self.dcfg, self.wcfg = cfg["dictation"], cfg["wake"] - self.hotkeys = {} # evdev code -> name, from load_bindings() + self.hotkeys = {} # evdev code -> name, from load_settings() self.typer = Typer(self.dcfg["key_delay_ms"]) self.type_lock = asyncio.Lock() self.sounds = {} # apply_options() makes them when enabled @@ -286,7 +285,7 @@ class ScarletVoice: self.stream = None # streaming state of the current dictation, see start_dictation self.whisper_lock = asyncio.Lock() # one request at a time, in order self.command = None # dict while capturing a post-wake command - self.wake_enabled = bindings.DEFAULT["wake_word"] # apply_options() sets the saved value + self.wake_enabled = settings.DEFAULT["wake_word"] # apply_options() sets the saved value self.mic_task = None self.mic_proc = None self.audio_session_ready = False @@ -294,15 +293,14 @@ class ScarletVoice: self.hk_down = False self.hk_chord = False - self.kb_mode = bindings.DEFAULT["keyboard_mode"] - self.applied = {} # bindings.json options last applied, so a save only changes what changed + self.kb_mode = settings.DEFAULT["keyboard_mode"] + self.applied = {} # settings.json options last applied, so a save only changes what changed self.mods_down = set() # (device path, code) of modifiers held on any keyboard self.mods_released = asyncio.Event() self.mods_released.set() self.hk_timer = None - self.kws = self._make_kws() - self.kws_stream = self.kws.create_stream() + self.kws = self.kws_stream = None # built from the wake phrases in settings.json self.cmd_vad = self._make_vad(self.wcfg["end_silence"]) self.clip_vad = self._make_vad(0.5) self.piece_vad = self._make_vad(0.1) # per-chunk speech/silence labels for live windows @@ -313,21 +311,25 @@ class ScarletVoice: encoder=str(m / "encoder_model.ort"), decoder=str(m / "decoder_model_merged.ort"), tokens=str(m / "tokens.txt"), num_threads=self.dcfg["moonshine_threads"]) log.info("live dictation engine: moonshine (%s)", m.name) - self.load_bindings() + self.load_settings() # --- models - def _make_kws(self): + def _make_kws(self, phrases): + """Open-vocabulary spotter for the phrases: they are spelled with the model's BPE + tokens at load time, so any English words work without training.""" + if not phrases: + log.info("wake phrases: none") + return None sp = sentencepiece.SentencePieceProcessor(model_file=str(KWS_DIR / "bpe.model")) lines = [] - for phrase in self.wcfg["phrases"]: - phrase = phrase.upper().strip() + for phrase in phrases: tokens = " ".join(sp.encode(phrase, out_type=str)) lines.append(f"{tokens} :{self.wcfg['boost']} #{self.wcfg['threshold']} @{phrase.replace(' ', '_')}") RUNTIME_DIR.mkdir(parents=True, exist_ok=True) kw_file = RUNTIME_DIR / "keywords.txt" kw_file.write_text("\n".join(lines) + "\n") - log.info("wake phrases: %s", ", ".join(self.wcfg["phrases"])) + log.info("wake phrases: %s", ", ".join(phrases)) return sherpa_onnx.KeywordSpotter( tokens=str(KWS_DIR / "tokens.txt"), encoder=str(KWS_DIR / "encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx"), @@ -368,7 +370,7 @@ class ScarletVoice: # SteamOS runs every playback stream through a per-stream spatializer that takes # ~0.35 s to set up, late enough to bury a start tone under the first word. Its # WirePlumber script skips streams named like filter chains, so use such a name. - props = '{ node.name = "scarlet-voice-chime-filter-chain" media.role = "Notification" }' + props = '{ node.name = "frame-voice-chime-filter-chain" media.role = "Notification" }' task = asyncio.create_task(self._run(["pw-play", "-P", props, str(self.sounds[name])], quiet=True)) self.tasks.add(task) task.add_done_callback(self.tasks.discard) @@ -398,7 +400,16 @@ class ScarletVoice: # --- microphone def mic_needed(self): - return self.wake_enabled or self.recording is not None or self.command is not None + return (self.wake_enabled and self.kws is not None) or self.recording is not None or self.command is not None + + def reset_kws(self): + if self.kws: + self.kws.reset_stream(self.kws_stream) + + async def _rebuild_kws(self, phrases): + kws = await asyncio.to_thread(self._make_kws, phrases) + self.kws, self.kws_stream = kws, kws.create_stream() if kws else None + self.update_mic() def update_mic(self): if self.mic_needed() and (self.mic_task is None or self.mic_task.done()): @@ -465,13 +476,13 @@ class ScarletVoice: asyncio.create_task(self.stop_dictation()) elif self.command is not None: self._command_audio(chunk) - elif self.wake_enabled: + elif self.wake_enabled and self.kws: self.kws_stream.accept_waveform(RATE, chunk) while self.kws.is_ready(self.kws_stream): self.kws.decode_stream(self.kws_stream) keyword = self.kws.get_result(self.kws_stream) if keyword: - self.kws.reset_stream(self.kws_stream) + self.reset_kws() self.on_wake(keyword.replace("_", " ")) break @@ -748,7 +759,7 @@ class ScarletVoice: log.info("wake word: %s", keyword) self.chime("wake") self.broadcast("wake", keyword=keyword) - self.run_hook("wake_hook", keyword, env={"SCARLET_EVENT": "wake"}) + self.run_hook("wake_hook", keyword, env={"FRAME_VOICE_EVENT": "wake"}) self.command = {"chunks": [], "t0": time.monotonic(), "heard": False} self.cmd_vad.reset() self.update_mic() @@ -771,7 +782,7 @@ class ScarletVoice: self.chime("cancel") self.broadcast("command_timeout") if self.command is None: - self.kws.reset_stream(self.kws_stream) + self.reset_kws() self.update_mic() async def finish_command(self, samples): @@ -784,7 +795,7 @@ class ScarletVoice: return self.broadcast("command", text=text) if text: - self.run_hook("command_hook", text, env={"SCARLET_EVENT": "command"}) + self.run_hook("command_hook", text, env={"FRAME_VOICE_EVENT": "command"}) # --- hotkey @@ -849,28 +860,28 @@ class ScarletVoice: continue log.info("hotkey: watching %s (%s)", dev.name, path) watching[path] = asyncio.create_task(self._read_device(dev)) - if bindings.mtime() != self.bindings_mtime: - self.load_bindings() + if settings.mtime() != self.settings_mtime: + self.load_settings() continue # rescan now for devices with the new keys await asyncio.sleep(2) - def load_bindings(self): - self.bindings_mtime = bindings.mtime() - b = bindings.load() # defaults when the file doesn't exist yet + def load_settings(self): + self.settings_mtime = settings.mtime() + b = settings.load() # defaults when the file doesn't exist yet self.hotkeys = {} for name in b["keyboard"]: code = e.ecodes.get(name) if isinstance(code, int): self.hotkeys[code] = name else: - log.warning("bindings: unknown key %r", name) + log.warning("settings: unknown key %r", name) log.info("keyboard/mouse push-to-talk: %s", ", ".join(self.hotkeys.values()) or "none") self.apply_options(b) def apply_options(self, b): """Settings from the config app. Only changed ones are applied, so a save doesn't - undo e.g. `scarletctl wake off`.""" - changed = {k: b[k] for k in ("keyboard_mode", *bindings.OPTIONS) if self.applied.get(k) != b[k]} + undo e.g. `frame-voice-ctl wake off`.""" + changed = {k: b[k] for k in ("keyboard_mode", "wake_phrases", *settings.OPTIONS) if self.applied.get(k) != b[k]} self.applied.update(changed) if "keyboard_mode" in changed: self.kb_mode = changed["keyboard_mode"] @@ -880,12 +891,21 @@ class ScarletVoice: self.sounds = make_sounds(self.cfg["sounds"]["volume"]) if changed["sounds"] else {} if "wake_word" in changed and changed["wake_word"] != self.wake_enabled: self.wake_enabled = changed["wake_word"] - self.kws.reset_stream(self.kws_stream) + self.reset_kws() try: asyncio.get_running_loop() self.update_mic() except RuntimeError: pass # at startup; run() opens the mic + if "wake_phrases" in changed: + try: + asyncio.get_running_loop() + task = asyncio.create_task(self._rebuild_kws(changed["wake_phrases"])) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + except RuntimeError: # at startup + self.kws = self._make_kws(changed["wake_phrases"]) + self.kws_stream = self.kws.create_stream() if self.kws else None if changed: log.info("settings: %s", ", ".join(f"{k}={v}" for k, v in changed.items())) @@ -933,14 +953,14 @@ class ScarletVoice: case "wake": if arg in ("on", "off", "toggle"): self.wake_enabled = {"on": True, "off": False}.get(arg, not self.wake_enabled) - self.kws.reset_stream(self.kws_stream) + self.reset_kws() self.update_mic() else: self.on_wake("manual") case "status": pass case "reload": - self.load_bindings() + self.load_settings() case "events": self.subscribers.add(writer) writer.write((json.dumps({"event": "subscribed", **self.state()}) + "\n").encode()) @@ -975,7 +995,7 @@ class ScarletVoice: def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - asyncio.run(ScarletVoice(load_config()).run()) + asyncio.run(FrameVoice(load_config()).run()) if __name__ == "__main__": diff --git a/scarlet_config.py b/frame_voice_config.py similarity index 82% rename from scarlet_config.py rename to frame_voice_config.py index ec6f751..8f4ba01 100755 --- a/scarlet_config.py +++ b/frame_voice_config.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""Scarlet Voice settings (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. +"""Frame Voice settings (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. "Add" waits for a controller button (read from vrserver's input stream, like -scarlet-vr-input) or a keyboard key / mouse button (read from evdev, like scarlet-voice) -and saves ~/.config/scarlet-voice/bindings.json. Both services pick changes up on their -own. Run through the `scarlet-config` launcher: GTK comes from the system Python, evdev +frame-voice-vr-input) or a keyboard key / mouse button (read from evdev, like frame-voice) +and saves ~/.config/frame-voice/settings.json. Both services pick changes up on their +own. Run through the `frame-voice-config` launcher: GTK comes from the system Python, evdev and websockets from the project venv. """ import asyncio @@ -28,12 +28,12 @@ import websockets # noqa: E402 from evdev import ecodes as e # noqa: E402 from gi.repository import Adw, GLib, Gtk # noqa: E402 -import bindings # noqa: E402 +import settings # noqa: E402 -SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice/control.sock" VRSERVER = "http://127.0.0.1:27062" VR_HEADERS = {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} -TYPER_NAME = "scarlet-voice keyboard" +TYPER_NAME = "frame-voice keyboard" CAPTURE_SECONDS = 15 # Clicking "Add" would otherwise bind the mouse button that clicked it. @@ -76,7 +76,7 @@ MODE_HINTS = {"toggle": "Tap to start, tap again to stop.", "hold": "Hold to tal def key_note(name, mode): - if name in bindings.MODIFIERS: + if name in settings.MODIFIERS: if mode == "hold": return "While it's held, text waits for release (it would turn letters into shortcuts)" return "Shortcuts with it (like Ctrl+C) don't start or stop dictation" @@ -85,6 +85,13 @@ def key_note(name, mode): return name +def device_label(root_path, controller_type): + name = controller_type.replace("_", " ").title().replace("Frame", "Steam Frame") + hand = root_path.rsplit("/", 1)[-1] + # A controller without a hand role (Frametop's pointer holds one) is known by its serial. + return f"{hand.title()} {name}" if hand in ("left", "right") else f"{name} ({hand})" + + def vr_devices(): """Connected controllers as SteamVR reports them, or None when SteamVR is not running.""" req = urllib.request.Request(f"{VRSERVER}/input/getstate.json", headers=VR_HEADERS) @@ -94,7 +101,7 @@ def vr_devices(): except OSError: return None return [{"device": d["root_path"], "controller_type": d["controller_type"], - "label": f"{d['root_path'].rsplit('/', 1)[-1]} {d['controller_type'].replace('_', ' ').title().replace('Frame', 'Steam Frame')}"} + "label": device_label(d["root_path"], d["controller_type"])} for d in state.get("devices", []) if d.get("class") == "TrackedDeviceClass_Controller" and d.get("root_path") and d.get("controller_type") not in IGNORED_CONTROLLERS] @@ -186,7 +193,7 @@ class Capture: self.end() async def _controller(self, cancel, devices): - mailbox = f"scarlet_config_{os.getpid()}_{secrets.token_hex(3)}" + mailbox = f"frame_voice_config_{os.getpid()}_{secrets.token_hex(3)}" types = {d["device"]: d["controller_type"] for d in devices} async with websockets.connect(VRSERVER.replace("http", "ws"), origin=VRSERVER, max_size=None, additional_headers=VR_HEADERS) as ws: @@ -218,9 +225,9 @@ class Capture: class Window(Adw.ApplicationWindow): def __init__(self, app): - super().__init__(application=app, title="Scarlet Voice", default_width=560, default_height=640) + super().__init__(application=app, title="Frame Voice", default_width=560, default_height=640) self.capture = Capture() - self.data = bindings.load() + self.data = settings.load() self.rows = [] self.dialog = None @@ -232,7 +239,8 @@ class Window(Adw.ApplicationWindow): add.connect("clicked", lambda _b, k=kind: self.start_capture(k)) group.set_header_suffix(add) page = Adw.PreferencesPage() - for group in (self.controller_group, self.keyboard_group, self._behavior_group()): + self.wake_group = self._wake_group() + for group in (self.controller_group, self.keyboard_group, self.wake_group, self._behavior_group()): page.add(group) self.toasts = Adw.ToastOverlay(child=page) view = Adw.ToolbarView(content=self.toasts) @@ -241,6 +249,30 @@ class Window(Adw.ApplicationWindow): self.refresh() threading.Thread(target=self._load_controllers, daemon=True).start() + def _wake_group(self): + group = Adw.PreferencesGroup( + title="Wake words", + description="Say one, then a command; the command goes to hooks/on-command and the event " + "stream. Any English words work. Two or more words trigger by accident less.") + switch = Adw.SwitchRow(title="Listen for wake words", subtitle="While not dictating", + active=self.data["wake_word"]) + switch.connect("notify::active", lambda r, _p: self.set_option("wake_word", r.get_active())) + group.add(switch) + self.wake_entry = Adw.EntryRow(title="Add a wake word, like \u201cHey Computer\u201d", show_apply_button=True) + self.wake_entry.connect("apply", self.add_wake_phrase) + return group + + def add_wake_phrase(self, entry): + phrase = settings.clean_phrase(entry.get_text()) + entry.set_text("") + if not phrase: + self.toast("Use letters and spaces.") + elif phrase in self.data["wake_phrases"]: + self.toast(f"\u201c{phrase.title()}\u201d is already a wake word.") + else: + self.data["wake_phrases"].append(phrase) + self.save(f"Added \u201c{phrase.title()}\u201d.") + def _behavior_group(self): group = Adw.PreferencesGroup(title="Behavior", description="Recordings shorter than 2 seconds are discarded.") @@ -252,8 +284,7 @@ class Window(Adw.ApplicationWindow): group.add(row) for key, title, subtitle in ( ("live_typing", "Type while talking", "Text appears every few seconds instead of after you stop"), - ("sounds", "Sounds", "Tones when recording starts and stops"), - ("wake_word", "\u201cHey Scarlet\u201d wake word", "Listen for the wake word when not dictating")): + ("sounds", "Sounds", "Tones when recording starts and stops")): row = Adw.SwitchRow(title=title, subtitle=subtitle, active=self.data[key]) row.connect("notify::active", lambda r, _p, k=key: self.set_option(k, r.get_active())) group.add(row) @@ -271,7 +302,6 @@ class Window(Adw.ApplicationWindow): text = "No controller connected. Wake one to add its buttons." else: text = "Connected: " + ", ".join(d["label"] for d in devices) + "." - text = text[:11] + text[11].upper() + text[12:] self.controller_status = text GLib.idle_add(self.refresh) @@ -285,8 +315,11 @@ class Window(Adw.ApplicationWindow): self.keyboard_group.set_description(MODE_HINTS[self.data["keyboard_mode"]]) entries += [(self.keyboard_group, key_label(k), key_note(k, self.data["keyboard_mode"]), ("keyboard", i)) for i, k in enumerate(self.data["keyboard"])] + entries += [(self.wake_group, p.title(), "", ("wake_phrases", i)) for i, p in enumerate(self.data["wake_phrases"])] for group, title, subtitle, (kind, index) in entries: - row = Adw.ActionRow(title=title, subtitle=subtitle) + row = Adw.ActionRow(title=title) + if subtitle: + row.set_subtitle(subtitle) remove = Gtk.Button(icon_name="user-trash-symbolic", valign=Gtk.Align.CENTER, css_classes=["flat"], tooltip_text="Remove") remove.connect("clicked", lambda _b, k=kind, i=index: self.remove(k, i)) @@ -298,12 +331,14 @@ class Window(Adw.ApplicationWindow): row = Adw.ActionRow(title="Nothing bound", subtitle="Use Add to pick a button", sensitive=False) group.add(row) self.rows.append((group, row)) + self.wake_group.add(self.wake_entry) # keep the entry below the phrases + self.rows.append((self.wake_group, self.wake_entry)) def save(self, message): - bindings.save(self.data) + settings.save(self.data) live = notify_daemon() self.refresh() - self.toast(message + (" Active now." if live else " scarlet-voice is not running.")) + self.toast(message + (" Active now." if live else " frame-voice is not running.")) def toast(self, text): self.toasts.add_toast(Adw.Toast(title=text, timeout=3)) @@ -351,7 +386,7 @@ class Window(Adw.ApplicationWindow): def main(): - app = Adw.Application(application_id="local.scarletvoice.Config") + app = Adw.Application(application_id="local.framevoice.Config") app.connect("activate", lambda a: (a.get_active_window() or Window(a)).present()) return app.run(sys.argv) diff --git a/scarlet_vr_input.py b/frame_voice_vr_input.py similarity index 86% rename from scarlet_vr_input.py rename to frame_voice_vr_input.py index acfe510..ff132e8 100755 --- a/scarlet_vr_input.py +++ b/frame_voice_vr_input.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -"""Controller push-to-talk for scarlet-voice: hold a bound button (or tap it, in toggle mode). +"""Controller push-to-talk for frame-voice: hold a bound button (or tap it, in toggle mode). SteamVR only gives controller input to the app that has focus, and while the 2D Steam or desktop screens are up the compositor's laser mouse holds it, so an OpenVR action never fires there. Instead this reads raw component state from vrserver's local web socket (127.0.0.1:27062), the same stream the controller binding page uses for its live view. It sees the button whatever has focus. Press -> `start vr`, release -> `stop` on -the scarlet-voice control socket; the daemon applies the 2 s minimum. +the frame-voice control socket; the daemon applies the 2 s minimum. The button is observed, not captured: whatever else is bound to it still happens. -Buttons come from ~/.config/scarlet-voice/bindings.json (see bindings.py and the -scarlet-config app); changes apply within a second. +Buttons come from ~/.config/frame-voice/settings.json (see settings.py and the +frame-voice-config app); changes apply within a second. """ import asyncio import json @@ -23,15 +23,15 @@ from pathlib import Path import websockets -import bindings +import settings -SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "scarlet-voice/control.sock" +SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice/control.sock" VRSERVER = "http://127.0.0.1:27062" -MAILBOX = f"scarlet_voice_ptt_{os.getpid()}" +MAILBOX = f"frame_voice_ptt_{os.getpid()}" VR_HEADERS = {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} DEVICE_POLL = 5 # seconds between checks for controllers connecting or leaving -log = logging.getLogger("scarlet-vr-input") +log = logging.getLogger("frame-voice-vr-input") def send(command): @@ -42,7 +42,7 @@ def send(command): sock.sendall((command + "\n").encode()) sock.recv(4096) except OSError as err: - log.warning("scarlet-voice not reachable (%s): %s", command, err) + log.warning("frame-voice not reachable (%s): %s", command, err) async def connect(): @@ -66,9 +66,9 @@ async def subscribe(ws, device, mailbox, on=True): def wanted(): - """device -> set of inputs, from bindings.json.""" + """device -> set of inputs, from settings.json.""" out = {} - for b in bindings.load()["controller"]: + for b in settings.load()["controller"]: out.setdefault(b["device"], set()).add(b["input"]) return out @@ -77,7 +77,7 @@ async def run_session(): """One connection to vrserver. Returns or raises when SteamVR goes away.""" async with await connect() as ws: await ws.send(f"mailbox_open {MAILBOX}") - stamp, inputs, mode = bindings.mtime(), wanted(), bindings.load()["controller_mode"] + stamp, inputs, mode = settings.mtime(), wanted(), settings.load()["controller_mode"] for device in inputs: await subscribe(ws, device, MAILBOX) log.info("connected to SteamVR, watching %s (%s)", describe(inputs), mode) @@ -102,15 +102,15 @@ async def run_session(): pressed.add((device, name)) elif name in components: pressed.discard((device, name)) - if bindings.mtime() != stamp: - stamp, new, mode = bindings.mtime(), wanted(), bindings.load()["controller_mode"] + if settings.mtime() != stamp: + stamp, new, mode = settings.mtime(), wanted(), settings.load()["controller_mode"] for device in inputs.keys() - new.keys(): await subscribe(ws, device, MAILBOX, on=False) for device in new.keys() - inputs.keys(): await subscribe(ws, device, MAILBOX) inputs = new pressed = {(d, i) for d, i in pressed if i in inputs.get(d, ())} - log.info("bindings changed, watching %s (%s)", describe(inputs), mode) + log.info("settings changed, watching %s (%s)", describe(inputs), mode) if now >= next_poll: next_poll = now + DEVICE_POLL current = await asyncio.to_thread(connected) diff --git a/hooks/on-command b/hooks/on-command index 26fc16b..8457148 100755 --- a/hooks/on-command +++ b/hooks/on-command @@ -1,14 +1,14 @@ #!/usr/bin/env bash -# Called after "Hey Scarlet ..." with the transcribed command as $1. +# Called after a wake word with the transcribed command as $1. # Replace this with a call into the assistant. Runs as your user, no shell interpolation of $1. set -euo pipefail text="$1" -log_dir="${XDG_STATE_HOME:-$HOME/.local/state}/scarlet-voice" +log_dir="${XDG_STATE_HOME:-$HOME/.local/state}/frame-voice" mkdir -p "$log_dir" printf '%s\t%s\n' "$(date -Is)" "$text" >> "$log_dir/commands.log" -# Built-in example: "Hey Scarlet, type hello world" types into the focused field. +# Built-in example: ", type hello world" types into the focused field. shopt -s nocasematch if [[ "$text" =~ ^(type|dictate)[[:space:],:]+(.*)$ ]]; then - exec "$(dirname "$0")/../scarletctl" type "${BASH_REMATCH[2]}" + exec "$(dirname "$0")/../frame-voice-ctl" type "${BASH_REMATCH[2]}" fi diff --git a/bindings.py b/settings.py similarity index 73% rename from bindings.py rename to settings.py index 3e40b19..59d227f 100644 --- a/bindings.py +++ b/settings.py @@ -1,21 +1,24 @@ -"""Push-to-talk bindings and settings shared by scarlet-voice, scarlet-vr-input and scarlet-config. +"""Push-to-talk bindings and settings shared by frame-voice, frame-voice-vr-input and frame-voice-config. -~/.config/scarlet-voice/bindings.json (written by the scarlet-config app): +~/.config/frame-voice/settings.json (written by the frame-voice-config app): {"keyboard": ["KEY_RIGHTCTRL", "BTN_SIDE"], "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], "keyboard_mode": "toggle", "controller_mode": "hold", - "live_typing": true, "sounds": false, "wake_word": true} + "live_typing": true, "sounds": false, "wake_word": true, + "wake_phrases": ["HEY COMPUTER"]} keyboard: evdev key or mouse button names, read from every physical keyboard and mouse. controller: a SteamVR device path and one of its boolean inputs, read from vrserver. *_mode: "hold" records while held; "toggle" starts on one press and stops on the next. +wake_phrases: what to say before a command; any English words, spelled out at load time. The options override config.toml. """ import json import os +import re from pathlib import Path -PATH = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / "scarlet-voice/bindings.json" +PATH = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / "frame-voice/settings.json" DEFAULT = { "keyboard": ["KEY_RIGHTCTRL"], "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], @@ -25,6 +28,7 @@ DEFAULT = { "live_typing": True, "sounds": False, "wake_word": True, + "wake_phrases": ["HEY COMPUTER"], } MODES = ("hold", "toggle") OPTIONS = ("live_typing", "sounds", "wake_word") @@ -33,6 +37,11 @@ MODIFIERS = {"KEY_LEFTCTRL", "KEY_RIGHTCTRL", "KEY_LEFTALT", "KEY_RIGHTALT", "KEY_LEFTSHIFT", "KEY_RIGHTSHIFT", "KEY_LEFTMETA", "KEY_RIGHTMETA"} +def clean_phrase(text): + """A wake phrase as the keyword spotter wants it: uppercase words, letters and apostrophes.""" + return " ".join(re.sub(r"[^A-Z' ]", " ", text.upper()).split()) + + def load(): try: data = json.loads(PATH.read_text()) @@ -50,6 +59,9 @@ def load(): for key in OPTIONS: if isinstance(data.get(key), bool): out[key] = data[key] + if isinstance(data.get("wake_phrases"), list): + phrases = (clean_phrase(p) for p in data["wake_phrases"] if isinstance(p, str)) + out["wake_phrases"] = list(dict.fromkeys(p for p in phrases if p)) return out diff --git a/systemd/scarlet-vr-input.service b/systemd/frame-voice-vr-input.service similarity index 57% rename from systemd/scarlet-vr-input.service rename to systemd/frame-voice-vr-input.service index aad3808..b30bcb0 100644 --- a/systemd/scarlet-vr-input.service +++ b/systemd/frame-voice-vr-input.service @@ -1,12 +1,12 @@ [Unit] -Description=scarlet-voice controller push-to-talk (left d-pad down) +Description=frame-voice controller push-to-talk # No After=steamvr.service: default.target is ordered after its wants and steamvr after # the graphical session, so that would be an ordering cycle. The script retries until # SteamVR is up. -After=scarlet-voice.service +After=frame-voice.service [Service] -ExecStart=%h/Desktop/Projects/scarlet-voice/.venv/bin/python %h/Desktop/Projects/scarlet-voice/scarlet_vr_input.py +ExecStart=%h/Desktop/Projects/frame-voice/.venv/bin/python %h/Desktop/Projects/frame-voice/frame_voice_vr_input.py Restart=always RestartSec=5 diff --git a/systemd/frame-voice-whisper.service b/systemd/frame-voice-whisper.service new file mode 100644 index 0000000..63d6daf --- /dev/null +++ b/systemd/frame-voice-whisper.service @@ -0,0 +1,15 @@ +[Unit] +Description=whisper.cpp server for frame-voice + +[Service] +# The Gamescope WSI layer asserts on compute-only Vulkan devices; whisper needs no swapchain. +Environment=DISABLE_GAMESCOPE_WSI=1 +# The binary's RUNPATH is the directory it was built in; this keeps it working if the +# project folder moves. +Environment=LD_LIBRARY_PATH=%h/Desktop/Projects/frame-voice/vendor/whisper.cpp/build/bin +ExecStart=%h/Desktop/Projects/frame-voice/vendor/whisper.cpp/build/bin/whisper-server -m %h/Desktop/Projects/frame-voice/models/ggml-small.en-q5_1.bin --host 127.0.0.1 --port 8178 -t 3 -fa +Restart=on-failure +Nice=5 + +[Install] +WantedBy=default.target diff --git a/systemd/scarlet-voice.service b/systemd/frame-voice.service similarity index 60% rename from systemd/scarlet-voice.service rename to systemd/frame-voice.service index 0979da5..3d96f34 100644 --- a/systemd/scarlet-voice.service +++ b/systemd/frame-voice.service @@ -1,7 +1,7 @@ [Unit] -Description=scarlet-voice dictation and wake word daemon -Wants=scarlet-whisper.service pipewire.service -After=scarlet-whisper.service pipewire.service +Description=frame-voice dictation and wake word daemon +Wants=frame-voice-whisper.service pipewire.service +After=frame-voice-whisper.service pipewire.service # SteamVR opens input devices only at startup, and gamescope gets keyboard input through # SteamVR, so the typing keyboard must exist before SteamVR starts. Before=steamvr.service @@ -10,11 +10,11 @@ Before=steamvr.service # READY=1 is sent after the virtual keyboard exists. Type=notify NotifyAccess=main -ExecStart=%h/Desktop/Projects/scarlet-voice/.venv/bin/python %h/Desktop/Projects/scarlet-voice/scarlet_voice.py +ExecStart=%h/Desktop/Projects/frame-voice/.venv/bin/python %h/Desktop/Projects/frame-voice/frame_voice.py Restart=on-failure RestartSec=2 # systemd keeps the virtual keyboard's fd across restarts, so SteamVR keeps the same device. -# Clear with: systemctl --user clean --what=fdstore scarlet-voice +# Clear with: systemctl --user clean --what=fdstore frame-voice FileDescriptorStoreMax=1 FileDescriptorStorePreserve=yes diff --git a/systemd/scarlet-whisper.service b/systemd/scarlet-whisper.service deleted file mode 100644 index 02fb459..0000000 --- a/systemd/scarlet-whisper.service +++ /dev/null @@ -1,12 +0,0 @@ -[Unit] -Description=whisper.cpp server for scarlet-voice - -[Service] -# The Gamescope WSI layer asserts on compute-only Vulkan devices; whisper needs no swapchain. -Environment=DISABLE_GAMESCOPE_WSI=1 -ExecStart=%h/Desktop/Projects/scarlet-voice/vendor/whisper.cpp/build/bin/whisper-server -m %h/Desktop/Projects/scarlet-voice/models/ggml-small.en-q5_1.bin --host 127.0.0.1 --port 8178 -t 3 -fa -Restart=on-failure -Nice=5 - -[Install] -WantedBy=default.target From 712bc10af5ec7d196deebe55f23e171f90189982 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:12:39 -0600 Subject: [PATCH 4/9] Read keys of keyboards Frametop grabs, from its input relay Frametop's input relay now grabs pass-through keyboards while typing goes to its VR desktop, so gamescope doesn't type them into the Steam app too. A grabbed keyboard can't be read here, so the Right Ctrl hotkey stopped working in the desktop. The relay sends those keys to @frametop_keys as "key " datagrams; frame-voice listens there and feeds them to the same hotkey and modifier handling. The socket is abstract, so only datagrams from our own user are accepted (SO_PASSCRED). Keys from our own typing keyboard, which the relay grabs too, are skipped: seen as user input, they turned the stop tap into a Ctrl shortcut. When live typing waits for a held modifier, the log now says which key on which device, to track down one that looks stuck. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++- frame_voice.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4d4072..f5338e7 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ Change the buttons in *Frame Voice Settings* (`./frame-voice-config`): **Add** w Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. At a 0.8 s pause, or at the quietest point once the window reaches 8 s, the window is decoded one last time and its remaining words are typed. On test audio the first words appeared at about 2.3 s, then every 1-1.5 s, and the text was complete 0.1-0.2 s after release. Nothing is cut while it is still being decoded, so words aren't split or doubled at the edges the way fixed pieces did. Moonshine adds fewer commas than whisper, and a long pause can still end a sentence with a period. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). Wake-word commands still use whisper. -Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) +Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) When typing holds back, the log says which key on which device it waits for (`typing waits for release of ...`). + +Keyboards grabbed by Frametop's input relay (while typing goes to its VR desktop) can't be read directly, so the relay sends their keys to the abstract socket `@frametop_keys`, and frame-voice treats them like keys from any other keyboard. It accepts them only from your own user, and skips its own typing keyboard, which the relay grabs too. Audio cues are off by default (`[sounds] enabled = true` turns them on): rising tone = recording, falling tone = processing, single high tone = wake word heard, low tone = cancelled or nothing heard. diff --git a/frame_voice.py b/frame_voice.py index 0917f64..33ae480 100755 --- a/frame_voice.py +++ b/frame_voice.py @@ -40,6 +40,9 @@ VAD_MODEL = ROOT / "models/silero_vad.onnx" RUNTIME_DIR = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice" CLIP_DIR = Path(os.environ.get("XDG_STATE_HOME", Path.home() / ".local/state")) / "frame-voice/clips" SOCKET_PATH = RUNTIME_DIR / "control.sock" +# Keys of keyboards Frametop's input relay has grabbed for its VR desktop: a grabbed +# keyboard can't be read here, so the relay sends its keys ("key "). +FRAMETOP_KEYS = "\0frametop_keys" log = logging.getLogger("frame-voice") @@ -157,6 +160,12 @@ def sd_notify(message, fds=()): return True +class RelayKeyboard: + """Stands in for an evdev device in on_key() for keys from Frametop's input relay.""" + path = "frametop-relay" + name = "Frametop relay" + + class Typer: """Types text through a virtual uinput keyboard. Assumes a US layout on the receiving side. @@ -298,6 +307,7 @@ class FrameVoice: self.mods_down = set() # (device path, code) of modifiers held on any keyboard self.mods_released = asyncio.Event() self.mods_released.set() + self._held_logged = None # the modifiers last logged as holding typing back self.hk_timer = None self.kws = self.kws_stream = None # built from the wake phrases in settings.json @@ -734,8 +744,11 @@ class FrameVoice: live = (self.dcfg["live_typing"] and stream is self.stream and time.monotonic() - stream["t0"] >= self.dcfg["min_seconds"]) if not live or self.mods_down: + if live: + self._log_held_modifiers() return # a held Ctrl/Alt/Super would turn letters into shortcuts; retried on release elif self.mods_down: + self._log_held_modifiers() await self.mods_released.wait() new = stream["words"][stream["typed"]:] text = (" " if stream["typed"] and new else "") + " ".join(new) @@ -834,9 +847,18 @@ class FrameVoice: self.mods_down.discard(key) self._modifiers_changed() + def _log_held_modifiers(self): + """Say which modifier holds typing back, once per set of held keys.""" + held = sorted(self.mods_down) + if held != self._held_logged: + self._held_logged = held + log.info("typing waits for release of %s", + ", ".join(f"{e.KEY.get(code, code)} on {path}" for path, code in held)) + def _modifiers_changed(self): if not self.mods_down: self.mods_released.set() + self._held_logged = None if self.stream: # type words committed while a modifier was held asyncio.create_task(self._flush_typing(self.stream)) @@ -909,6 +931,42 @@ class FrameVoice: if changed: log.info("settings: %s", ", ".join(f"{k}={v}" for k, v in changed.items())) + def listen_frametop_keys(self): + """Keys from Frametop's input relay, for the keyboards it grabs. Only from our own + user: the socket is abstract, so any local user could send to it.""" + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_NONBLOCK) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_PASSCRED, 1) # the kernel adds the sender's uid + try: + sock.bind(FRAMETOP_KEYS) + except OSError as err: + log.warning("frametop keys: can't listen (%s)", err) + sock.close() + return None + relay = RelayKeyboard() + creds = struct.Struct("3i") # struct ucred: pid, uid, gid + + def readable(): + while True: + try: + data, anc, _, _ = sock.recvmsg(64, socket.CMSG_SPACE(creds.size)) + except BlockingIOError: + return + uid = next((creds.unpack(d[:creds.size])[1] for level, kind, d in anc + if level == socket.SOL_SOCKET and kind == socket.SCM_CREDENTIALS), None) + words = data.split(maxsplit=3) + if uid != os.getuid() or len(words) < 3 or words[0] != b"key": + continue + if len(words) > 3 and words[3].decode(errors="replace") == TYPER_NAME: + continue # our own typing, grabbed along with the real keyboards + try: + code, value = int(words[1]), int(words[2]) + except ValueError: + continue + self.on_key(relay, code, value) + + asyncio.get_running_loop().add_reader(sock.fileno(), readable) + return sock + async def _read_device(self, dev): try: async for ev in dev.async_read_loop(): @@ -984,8 +1042,12 @@ class FrameVoice: for sig in (signal.SIGTERM, signal.SIGINT): loop.add_signal_handler(sig, stop.set) watcher = asyncio.create_task(self.watch_devices()) + frametop_keys = self.listen_frametop_keys() await stop.wait() watcher.cancel() + if frametop_keys: + asyncio.get_running_loop().remove_reader(frametop_keys.fileno()) + frametop_keys.close() server.close() if self.mic_proc and self.mic_proc.returncode is None: self.mic_proc.terminate() From 95574d60d50e4da7bd922984218a23daa5479935 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:43:38 -0600 Subject: [PATCH 5/9] Join live dictation windows so pauses don't break sentences Each window of live text was decoded on its own, so every pause read as a finished sentence ("increase the. Width"), and the seam logic lost or doubled words. Now: - Each window is decoded after the last 1.5 s of the previous one (context_seconds), so a sentence that goes on is heard as one. - Decodes are lined up with the typed words by content, not word count. This replaces the 1-3 word duplicate skip, which deleted a phrase said twice across a pause ("reset the leash pose. The leash pose ..."). - Punctuation after a pause waits for the next decode; a period is dropped when the next decode runs on and the word can't end a sentence. - One word that flips between decodes no longer stalls live typing. - A window cut without a pause commits only agreed words and carries the rest, since Moonshine garbles cut-off audio ("on a dot on a dot"). - Onset padding is 0.5 s plus any unflagged sound in the last second, and no decode passes Moonshine's ~9 s limit. On TTS clips with pauses mid-sentence and between sentences, replayed in real time, no mid-sentence seam got a stray period (previously 4 of 6), and the last word of a phrase is typed 0.3-1.8 s after it is spoken. Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +- config.example.toml | 3 +- frame_voice.py | 279 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 231 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index f5338e7..36cfd77 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,11 @@ Recordings shorter than 2 s (`min_seconds`) are discarded with the cancel tone, Change the buttons in *Frame Voice Settings* (`./frame-voice-config`): **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. -Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. At a 0.8 s pause, or at the quietest point once the window reaches 8 s, the window is decoded one last time and its remaining words are typed. On test audio the first words appeared at about 2.3 s, then every 1-1.5 s, and the text was complete 0.1-0.2 s after release. Nothing is cut while it is still being decoded, so words aren't split or doubled at the edges the way fixed pieces did. Moonshine adds fewer commas than whisper, and a long pause can still end a sentence with a period. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). Wake-word commands still use whisper. +Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. One word the two decodes disagree on (a filler heard as "a" once and "uh" the next time) doesn't hold back the words after it once three of them agree again. At a 0.8 s pause, or at the quietest point once a decode would pass 8 s of audio (Moonshine returns nothing past about 9 s), the window is decoded one last time and its remaining words are typed. + +Windows join up like one recording. Each one is decoded after the last 1.5 s of the one before it (`context_seconds`), so Moonshine hears a sentence that goes on after a pause as the continuation it is. Every decode is lined up with the words already typed, by content rather than by word count, and only what comes after them is new; a phrase you really say twice stays twice. The punctuation after a pause waits until the next window is heard: a period Moonshine put there only because the audio stopped is dropped when the next decode runs on and the word can't end a sentence ("the", "to", "we are"), so "increase the. Width" comes out as "increase the width". A window cut without a pause ends mid-speech, where Moonshine can garble or repeat the cut-off words, so only words the last live decode agrees with are typed then; the rest (up to 5) go to the next window, whose context is made long enough to hold them. After a silence, 0.5 s before the speech is kept, and anything loud in the last second the speech detector didn't flag (a short "I" before a gap), since its speech flag comes about 0.27 s late. + +On test clips (two TTS voices, 1 s pauses placed mid-sentence and between sentences, replayed in real time) every mid-sentence seam came out without a stray period or capital. The previous version put a period in 4 of the 6, deleted the phrase said twice, doubled words where a window was cut without a pause, and in one run dropped a whole phrase. The last word of each phrase is typed 0.3-1.8 s after you say it, about the same as before. Moonshine adds fewer commas than whisper. `engine = "whisper"` switches live text to the whisper-server (it dropped words when re-decoded this way in testing). Wake-word commands still use whisper. Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) When typing holds back, the log says which key on which device it waits for (`typing waits for release of ...`). diff --git a/config.example.toml b/config.example.toml index 9ce75f1..bc6955b 100644 --- a/config.example.toml +++ b/config.example.toml @@ -22,7 +22,8 @@ engine = "moonshine" # live engine: moonshine (CPU, fast) or whisper (GPU moonshine_threads = 2 step_seconds = 0.8 # re-decode the current window this often pause_seconds = 0.8 # a pause this long closes the window -max_window_seconds = 8.0 # without a pause, close at the quietest point by this length (moonshine max ~10) +max_window_seconds = 8.0 # without a pause, close at the quietest point by this much audio, context included (moonshine max ~9) +context_seconds = 1.5 # decode each window after this much of the previous one, so a sentence can carry on across a pause save_clips = 0 # keep the last N dictations' audio in ~/.local/state/frame-voice/clips [wake] diff --git a/frame_voice.py b/frame_voice.py index 33ae480..3f12c38 100755 --- a/frame_voice.py +++ b/frame_voice.py @@ -43,6 +43,10 @@ SOCKET_PATH = RUNTIME_DIR / "control.sock" # Keys of keyboards Frametop's input relay has grabbed for its VR desktop: a grabbed # keyboard can't be read here, so the relay sends its keys ("key "). FRAMETOP_KEYS = "\0frametop_keys" +# Audio kept before speech that starts after a silence: the VAD flags speech ~0.27 s late. +LEAD_IN_SECONDS = 0.5 +# Committed words a window's decode is lined up against: more than its context can hold. +ALIGN_WORDS = 12 log = logging.getLogger("frame-voice") @@ -68,14 +72,17 @@ DEFAULTS = { # Live text by agreement: the current window of speech is re-decoded every # step_seconds and words two decodes in a row agree on are typed. A window closes at a # pause (its last words are typed then) or at its quietest point by max_window_seconds. - # moonshine runs on the CPU (~0.1-0.3 s per decode, max ~10 s of audio); "whisper" uses - # the whisper-server on the GPU (better punctuation, ~0.5-0.8 s per decode). + # moonshine runs on the CPU (~0.1-0.3 s per decode, returns nothing past ~9 s of audio); + # "whisper" uses the whisper-server on the GPU (better punctuation, ~0.5-0.8 s per decode). "engine": "moonshine", "moonshine_model": "models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27", "moonshine_threads": 2, "step_seconds": 0.8, "pause_seconds": 0.8, # shorter pauses inside a sentence ended it with "?"/"." in testing - "max_window_seconds": 8.0, + "max_window_seconds": 8.0, # audio per decode, context included + # Each window is decoded after this much of the previous one's end, so a sentence that + # goes on after a pause isn't heard as a new one (no stray "." and capital at the seam). + "context_seconds": 1.5, # Keep the audio of the last N dictations in ~/.local/state/frame-voice/clips for # comparing models and settings. 0 = never write audio to disk. "save_clips": 0, @@ -231,16 +238,87 @@ def drop_repeats(text): return " ".join(out) +SAME_WORD = {"okay": "ok"} # spellings Moonshine switches between from one decode to the next + + +def norm(word): + word = re.sub(r"[^\w']", "", word.lower()) + return SAME_WORD.get(word, word) + + +def split_marks(word): + """("the.", ...) -> ("the", "."): a word and the punctuation after it.""" + m = re.fullmatch(r"(.*?)([.,!?;:]*)", word) + return m[1], m[2] + + +SENTENCE_END = (".", "?", "!") +# Words a sentence rarely ends on, so a pause after one is a pause inside a sentence. +DANGLING = set("""a an the to of for in on at by with from into onto about as than and or but nor + because if when while where whether that which whose my our your their its i we they he she + i'm we're you're they're it's are were be been being am have has had will would can + could should shall may might must very""".split()) + + +def settle_mark(word, held, heard): + """Punctuation after `word`, the last one before a pause. held: what the window decoded + alone put there (every window it ends reads as a finished sentence). heard: what the next + decode put there after hearing what followed (None if it didn't line up), which in turn + tends to run finished sentences together.""" + if heard is None: + return held + if heard.endswith(SENTENCE_END) or norm(word) in DANGLING: + return heard + return held or heard + + def common_prefix(a, b): """How many leading words two hypotheses share, ignoring case and punctuation.""" n = 0 for x, y in zip(a, b): - if re.sub(r"[^\w']", "", x.lower()) != re.sub(r"[^\w']", "", y.lower()): + if norm(x) != norm(y): break n += 1 return n +def agreement(prev, cur): + """How many leading words of `cur` the previous hypothesis backs up. One word that + differs (a filler heard as "A" one time and "Uh," the next) doesn't hold back the rest + when the three words after it agree again; cur's version of it wins.""" + i = j = 0 + while True: + k = common_prefix(prev[i:], cur[j:]) + i, j = i + k, j + k + if i >= len(prev) or j >= len(cur): + return j + for di, dj in ((1, 1), (0, 1), (1, 0)): + if common_prefix(prev[i + di:], cur[j + dj:]) >= 3: + i, j = i + di, j + dj + break + else: + return j + + +def align(ref, hyp): + """Where the words after `ref` start in `hyp`, a decode whose audio starts inside the + audio of ref's last words. Lines ref's end up with hyp's start (same word +2, anything + else -1, ref may start anywhere); on a tie the earlier start wins, so a phrase said twice + in a row isn't taken for the copy already typed. Returns (start, hyp's form of ref's last + word or None when that word didn't line up), or None when nothing lines up.""" + a, b = [norm(w) for w in ref], [norm(w) for w in hyp] + row = [-j for j in range(len(b) + 1)] + for i in range(1, len(a) + 1): + nxt = [0] + for j in range(1, len(b) + 1): + nxt.append(max(row[j - 1] + (2 if a[i - 1] == b[j - 1] else -1), row[j] - 1, nxt[j - 1] - 1)) + row = nxt + start = max(range(len(b) + 1), key=lambda j: (row[j], -j)) + if not a or row[start] < 2: + return None + return start, hyp[start - 1] if start and b[start - 1] == a[-1] else None + + def feed_vad(vad, samples): """Silero VAD only handles input correctly in whole 512-sample windows.""" size = vad.config.silero_vad.window_size @@ -570,10 +648,15 @@ class FrameVoice: self.recording, self.recording_source = [], source if self.dcfg["streaming"]: self.piece_vad.reset() + # win: first chunk of the current window. lead: the end of the previous window, + # decoded ahead of it. deferred: lead holds a word not committed yet (a window cut + # without a pause). words: committed, text: the same as typed, held: punctuation + # after the last word, waiting to hear if the sentence goes on. typed: chars typed. self.stream = {"source": source, "t0": time.monotonic(), "chunks": self.recording, "speech": [], "level": [], "wake": asyncio.Event(), "final": False, - "win": 0, "decoded_at": 0, "prev": [], "wdone": [], - "words": [], "typed": 0} + "win": 0, "decoded_at": 0, "lead": np.zeros(0, np.float32), + "deferred": False, "prev": [], "base": 0, + "words": [], "text": "", "held": None, "typed": 0} self.stream["worker"] = asyncio.create_task(self._stream_worker(self.stream)) self.update_mic() self.chime("start") @@ -619,7 +702,9 @@ class FrameVoice: stream["final"] = True stream["wake"].set() await stream["worker"] - text = " ".join(stream["words"]) + stream["text"] += stream["held"] or "" + stream["held"] = None + text = stream["text"] if not text: log.info("no speech detected") else: @@ -651,6 +736,7 @@ class FrameVoice: else: text = await self.transcribe_speech(audio, context, quiet=True) text = re.sub(r"([,;:!?])\1+", r"\1", text) # "so,," -> "so," + text = re.sub(r"(?= self.dcfg["max_window_seconds"] if stream["final"]: - if any(window): - await self._close_window(stream, end) + if last is None: + return + if full: # Moonshine returns nothing for more than ~9 s of audio + await self._close_window(stream, self._quiet_cut(stream, end), forced=True) + stream["wake"].set() + continue + await self._close_window(stream, end) return - if not any(window): + if last is None: if len(window) * per > 1.0: - stream["win"] = stream["decoded_at"] = end - int(0.3 / per) # keep a lead-in + stream["win"] = stream["decoded_at"] = self._lead_in(stream, end) continue - silence = len(window) - (max(i for i, v in enumerate(window) if v) + 1) + silence = len(window) - (last + 1) if silence * per >= self.dcfg["pause_seconds"]: await self._close_window(stream, end - silence + int(0.15 / per)) - elif len(window) * per >= self.dcfg["max_window_seconds"]: - # No pause: close at the quietest chunk of the second half, preferring ones - # the VAD calls non-speech (a breath between words rather than inside one). - level = stream["level"] - first = win + int(len(window) / 2) - cut = min(range(first, end), key=lambda i: (speech[i], level[i], -i)) + 1 - await self._close_window(stream, cut) + elif full: + await self._close_window(stream, self._quiet_cut(stream, end), forced=True) elif end - stream["decoded_at"] >= step: stream["decoded_at"] = end - hyp = (await self._decode(np.concatenate(stream["chunks"][win:end]), self._context(stream))).split() - agreed = common_prefix(stream["prev"], hyp) + hyp = (await self._decode(self._window_audio(stream, end), self._context(stream))).split() + found = self._uncommitted(stream, hyp) + if found is None: + log.info("decode skipped, doesn't line up with the words typed: %s", " ".join(hyp)) + continue + new, boundary = found + before = self._uncommitted(stream, stream["prev"]) stream["prev"] = hyp - done = len(stream["wdone"]) - if agreed - 1 > done: - self._commit(stream, hyp[done:agreed - 1]) + agreed = agreement(before[0], new) if before else 0 + if agreed > 1: + self._commit(stream, new[:agreed - 1], boundary) - async def _close_window(self, stream, cut): - win = stream["win"] - words = (await self._decode(np.concatenate(stream["chunks"][win:cut]), self._context(stream))).split() - # Words already typed stay as typed; the rest of the final decode follows them. - self._commit(stream, words[len(stream["wdone"]):]) - log.info("window %.1fs: %s", (cut - win) * CHUNK / RATE, " ".join(words)) - stream.update(win=cut, decoded_at=cut, prev=[], wdone=[]) + async def _close_window(self, stream, cut, forced=False): + """Decode the window once more and commit the rest of its words. After a pause the last + word's punctuation is held for the next window. Without one (forced), the audio stops + mid-speech, and Moonshine can garble or repeat the cut-off words ("on a dot on a dot"), + so only words the last live decode agrees on are committed; the rest (up to 5) are + carried in the next window's context, sized to hold them.""" + audio = self._window_audio(stream, cut) + hyp = (await self._decode(audio, self._context(stream))).split() + found = self._uncommitted(stream, hyp) + if found is None: + # Go by position within this window's own audio, as before context was added. + log.info("window decode doesn't line up with the words typed: %s", " ".join(hyp)) + hyp = (await self._decode(np.concatenate(stream["chunks"][stream["win"]:cut]), self._context(stream))).split() + found = hyp[len(stream["words"]) - stream["base"]:], None + new, boundary = found + start = len(hyp) - len(new) + context = self.dcfg["context_seconds"] + if forced: + before = self._uncommitted(stream, stream["prev"]) + keep = max(0, agreement(before[0], new) - 1 if before else 0, len(new) - 5) + context = max(context, min(3.5, 0.45 * (len(new) - keep + 2))) # ~0.3-0.4 s per word + new = new[:keep] + self._commit(stream, new, boundary, hold=not forced) + log.info("window %.1fs + %.1fs context: %s", (cut - stream["win"]) * CHUNK / RATE, + len(stream["lead"]) / RATE, " ".join(hyp[:start] + ["|"] + hyp[start:]) if start else " ".join(hyp)) + stream.update(win=cut, decoded_at=cut, lead=self._lead(audio, context), deferred=forced, prev=[], + base=len(stream["words"])) + + def _window_audio(self, stream, end): + return np.concatenate([stream["lead"], *stream["chunks"][stream["win"]:end]]) + + def _lead(self, audio, seconds): + """Context for the next window: about `seconds` from the end of this one, from the + quietest 30 ms nearby, so it rarely starts inside a word.""" + n = int(seconds * RATE) + if n <= 0: + return audio[:0] + if len(audio) <= n: + return audio + frame = RATE * 30 // 1000 + starts = range(max(0, len(audio) - n - RATE // 2), len(audio) - n + RATE // 4, frame) + return audio[min(starts, key=lambda i: float(np.mean(audio[i:i + frame] ** 2))):] + + def _lead_in(self, stream, end): + """Start of a window with no speech yet: LEAD_IN_SECONDS before now, or before a sound + in the last second the VAD didn't flag (a lone short word like "I" is too brief).""" + per = CHUNK / RATE + win, level = stream["win"], stream["level"] + start = end - round(LEAD_IN_SECONDS / per) + floor = float(np.percentile(level[win:end], 20)) + loud = next((i for i in range(max(win, end - round(1.0 / per)), end) + if level[i] > max(3 * floor, 0.005)), None) + if loud is not None: + start = min(start, loud - round(0.2 / per)) + return max(win, start) + + def _quiet_cut(self, stream, end): + """Where to close a window with no pause: after the quietest chunk of its second half, + preferring ones the VAD calls non-speech (a breath between words, not inside one).""" + speech, level = stream["speech"], stream["level"] + first = stream["win"] + (end - stream["win"]) // 2 + return min(range(first, end), key=lambda i: (speech[i], level[i], -i)) + 1 + + def _uncommitted(self, stream, hyp): + """hyp's words after the committed ones, and hyp's form of the last committed word + (None when it didn't line up). None when hyp doesn't line up with them at all.""" + # Without context, only words committed in this window are in hyp. With it, a decode too + # short to line up ("Post. The" for "...pose. The") is skipped rather than guessed at. + ref = stream["words"][max(0, stream["base"] - ALIGN_WORDS) if len(stream["lead"]) else stream["base"]:] + if not ref: + return hyp, None + found = align(ref, hyp) + return (hyp[found[0]:], found[1]) if found else None def _context(self, stream): return " ".join(stream["words"][-30:]) - def _commit(self, stream, words): - if not words: - return - skip = 0 - if not stream["wdone"]: - # A window's first words can repeat the end of the previous one when it closed - # inside a phrase. - skip = max((k for k in range(1, 4) if common_prefix([w for w in stream["words"][-k:]], words[:k]) == k - and len(stream["words"]) >= k), default=0) - stream["wdone"].extend(words) - words = words[skip:] + def _commit(self, stream, words, boundary=None, hold=False): + """Queue words for typing. boundary: this decode's form of the last committed word, + which settles punctuation held back at a pause. hold: keep the last new word's + punctuation back until the next window hears whether the sentence goes on.""" if not words: return + piece = "" + if stream["held"] is not None: + heard = None if boundary is None else split_marks(boundary)[1] + mark = settle_mark(stream["words"][-1] if stream["words"] else "", stream["held"], heard) + piece, stream["held"] = mark, None + if mark.endswith(SENTENCE_END) and words[0][:1].islower(): + words = [words[0][:1].upper() + words[0][1:], *words[1:]] + if hold: + word, stream["held"] = split_marks(words[-1]) + words = words[:-1] + ([word] if word else []) + if words: + piece += (" " if stream["text"] else "") + " ".join(words) stream["words"].extend(words) - self.broadcast("dictation_piece", text=" ".join(words)) + stream["text"] += piece + if not piece: + return + self.broadcast("dictation_piece", text=piece.strip()) task = asyncio.create_task(self._flush_typing(stream)) self.tasks.add(task) task.add_done_callback(self.tasks.discard) async def _flush_typing(self, stream, final=False): - """Type committed words not typed yet: live while recording when allowed, else at the end.""" + """Type committed text not typed yet: live while recording when allowed, else at the end.""" if not final: live = (self.dcfg["live_typing"] and stream is self.stream and time.monotonic() - stream["t0"] >= self.dcfg["min_seconds"]) @@ -750,14 +920,13 @@ class FrameVoice: elif self.mods_down: self._log_held_modifiers() await self.mods_released.wait() - new = stream["words"][stream["typed"]:] - text = (" " if stream["typed"] and new else "") + " ".join(new) - stream["typed"] += len(new) - if final and stream["words"] and self.dcfg["trailing_space"]: + text = stream["text"][stream["typed"]:] + stream["typed"] = len(stream["text"]) + if final and stream["text"] and self.dcfg["trailing_space"]: text += " " if text: if not final: - log.info("typing %d words live (%.1fs in)", len(new), time.monotonic() - stream["t0"]) + log.info("typing %d words live (%.1fs in)", len(text.split()), time.monotonic() - stream["t0"]) await self.type_text(text) async def toggle_dictation(self, source): From 5cc7da193c5b20f82a1642eb39bc4b314d570190 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:43:38 -0600 Subject: [PATCH 6/9] Start, stop and autostart from the settings app; work in Frametop The settings app gets a Service group: Running starts or stops all three units, Start at system startup enables or disables them. Disabling keeps the repo links and never lets systemd see the unit missing (disable and link with --no-reload, then daemon-reload): a plain disable drops frame-voice's fd store, and with it the typing keyboard SteamVR opened, even while the service runs. Frametop's desktop sets its own XDG_RUNTIME_DIR, session bus and XDG_CONFIG_HOME. Launched there, the app couldn't reach systemd or the control socket, and saved settings to a copy the daemon never read. The app and frame-voice-ctl now use /run/user/, and settings live in ~/.config/frame-voice/settings.json (FRAME_VOICE_SETTINGS overrides it). Co-Authored-By: Claude Opus 5.5 --- README.md | 20 +++++-- frame-voice-ctl | 3 +- frame_voice_config.py | 133 +++++++++++++++++++++++++++++++++++++++++- settings.py | 4 +- 4 files changed, 151 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 36cfd77..c4569e9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Local voice input for the Steam Frame. Nothing leaves the headset. | `frame-voice-whisper.service` | `whisper-server` (whisper.cpp, Vulkan on the Adreno 750) with `ggml-small.en-q5_1` loaded, on `127.0.0.1:8178` | | `frame-voice.service` | `frame_voice.py`: mic capture, hotkey, wake word, typing, control socket | | `frame-voice-vr-input.service` | `frame_voice_vr_input.py`: controller push-to-talk, reads left d-pad down from SteamVR's raw input stream | -| `frame-voice-config` | Settings app (GTK 4 / libadwaita): pick push-to-talk buttons by pressing them. In the app menu as *Frame Voice Settings* | +| `frame-voice-config` | Settings app (GTK 4 / libadwaita): start and stop the services, start at system startup, pick push-to-talk buttons by pressing them. In the app menu as *Frame Voice Settings* | | `frame-voice-ctl` | CLI for the control socket | | `hooks/on-command` | Runs after a wake word with the command text as `$1` | @@ -30,7 +30,7 @@ The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx op Recordings shorter than 2 s (`min_seconds`) are discarded with the cancel tone, so brief or accidental presses never type anything. -Change the buttons in *Frame Voice Settings* (`./frame-voice-config`): **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. +*Frame Voice Settings* (`./frame-voice-config`) starts and stops all three services (**Running**) and sets whether they start at system startup (see Services). Change the buttons there too: **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. One word the two decodes disagree on (a filler heard as "a" once and "uh" the next time) doesn't hold back the words after it once three of them agree again. At a 0.8 s pause, or at the quietest point once a decode would pass 8 s of audio (Moonshine returns nothing past about 9 s), the window is decoded one last time and its remaining words are typed. @@ -55,7 +55,7 @@ Audio cues are off by default (`[sounds] enabled = true` turns them on): rising ./frame-voice-ctl events # stream JSON events: wake, command, dictation, command_timeout, error ``` -The socket is `$XDG_RUNTIME_DIR/frame-voice/control.sock`, mode 0600. It can type into any focused window, so keep it owner-only. +The socket is `/run/user//frame-voice/control.sock` (systemd's runtime dir; the clients ignore `$XDG_RUNTIME_DIR`, which Frametop's desktop points elsewhere), mode 0600. It can type into any focused window, so keep it owner-only. An assistant can either replace `hooks/on-command` or subscribe with `frame-voice-ctl events` and act on `{"event": "command", "text": ...}`. @@ -67,12 +67,24 @@ Wake phrase tuning: raise `threshold` for fewer false triggers, raise `boost` if ## Services +**Running** in the settings app starts or stops all three units; **Start at system startup** enables or disables them. Or by hand: + ``` systemctl --user status frame-voice-whisper frame-voice frame-voice-vr-input journalctl --user -u frame-voice -f -systemctl --user stop frame-voice frame-voice-whisper frame-voice-vr-input # the units are linked, so `disable` removes the links +systemctl --user stop frame-voice frame-voice-whisper frame-voice-vr-input ``` +The units are linked from `systemd/`. Don't turn startup off with a plain `systemctl --user disable`: it removes those links too, and when a unit file goes missing systemd drops the typing keyboard (see below), even while the service runs. The app does what this does: + +``` +systemctl --user disable --no-reload frame-voice-whisper frame-voice frame-voice-vr-input +systemctl --user link --no-reload ~/Desktop/Projects/frame-voice/systemd/*.service +systemctl --user daemon-reload +``` + +Startup matters for typing into game-mode windows: SteamVR only uses the typing keyboard if it existed when SteamVR started (see Build notes). Enabled, `frame-voice` starts before SteamVR, and stopping and starting it keeps the same keyboard. Disabled, it starts only when you start it, after SteamVR, so restart SteamVR once after each start to type into game-mode windows: systemd keeps a stopped unit's fd store only while the unit is enabled, so each start makes a new keyboard. Typing into Frametop's desktop works either way. + ## Build notes whisper.cpp is in `vendor/whisper.cpp`, built with: diff --git a/frame-voice-ctl b/frame-voice-ctl index 1900cf4..147f154 100755 --- a/frame-voice-ctl +++ b/frame-voice-ctl @@ -5,7 +5,8 @@ import socket import sys from pathlib import Path -path = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice/control.sock" +# Not $XDG_RUNTIME_DIR: Frametop's desktop points it at its own directory. +path = Path(f"/run/user/{os.getuid()}") / "frame-voice/control.sock" if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"): sys.exit(__doc__) sock = socket.socket(socket.AF_UNIX) diff --git a/frame_voice_config.py b/frame_voice_config.py index 8f4ba01..21132e2 100755 --- a/frame_voice_config.py +++ b/frame_voice_config.py @@ -4,7 +4,8 @@ "Add" waits for a controller button (read from vrserver's input stream, like frame-voice-vr-input) or a keyboard key / mouse button (read from evdev, like frame-voice) and saves ~/.config/frame-voice/settings.json. Both services pick changes up on their -own. Run through the `frame-voice-config` launcher: GTK comes from the system Python, evdev +own. The Service switches start, stop, enable and disable the three systemd user units. +Run through the `frame-voice-config` launcher: GTK comes from the system Python, evdev and websockets from the project venv. """ import asyncio @@ -13,6 +14,7 @@ import os import secrets import select import socket +import subprocess import sys import threading import time @@ -30,7 +32,10 @@ from gi.repository import Adw, GLib, Gtk # noqa: E402 import settings # noqa: E402 -SOCKET_PATH = Path(os.environ.get("XDG_RUNTIME_DIR", f"/run/user/{os.getuid()}")) / "frame-voice/control.sock" +# systemd's runtime dir, where the services and their socket live. Not $XDG_RUNTIME_DIR: Frametop's +# desktop points that (and the session bus) at its own, where systemctl --user can't reach them. +USER_RUNTIME = Path(f"/run/user/{os.getuid()}") +SOCKET_PATH = USER_RUNTIME / "frame-voice/control.sock" VRSERVER = "http://127.0.0.1:27062" VR_HEADERS = {"Referer": f"{VRSERVER}/dashboard/controllerbinding.html"} TYPER_NAME = "frame-voice keyboard" @@ -107,6 +112,48 @@ def vr_devices(): and d.get("controller_type") not in IGNORED_CONTROLLERS] +UNIT_DIR = Path(__file__).resolve().parent / "systemd" +UNITS = ("frame-voice-whisper.service", "frame-voice.service", "frame-voice-vr-input.service") + + +def systemctl(*args): + env = {**os.environ, "XDG_RUNTIME_DIR": str(USER_RUNTIME), "DBUS_SESSION_BUS_ADDRESS": f"unix:path={USER_RUNTIME}/bus"} + try: + return subprocess.run(["systemctl", "--user", *args], capture_output=True, text=True, timeout=90, env=env) + except (OSError, subprocess.TimeoutExpired) as err: + return subprocess.CompletedProcess(args, 1, "", str(err)) + + +def service_state(): + """unit -> {"LoadState", "ActiveState", "UnitFileState"}, in UNITS order.""" + out = systemctl("show", "-p", "Id,LoadState,ActiveState,UnitFileState", *UNITS).stdout + blocks = [dict(line.split("=", 1) for line in block.splitlines() if "=" in line) for block in out.split("\n\n")] + return {unit: next((b for b in blocks if b.get("Id") == unit), {}) for unit in UNITS} + + +def service_action(action): + """systemctl start, stop, enable or disable all three units. Returns an error or None. + + SteamVR only sees the typing keyboard if it existed when SteamVR started. It lives in + frame-voice's fd store, which systemd keeps across a stop only while the unit is enabled, + and drops when the unit file goes missing, even while running. The units are linked from + the repo and `systemctl disable` removes those links too, so they are linked again before + systemd reloads.""" + steps = [(action, *UNITS)] + if action in ("start", "enable"): # links missing (never installed, or removed by hand) + state = service_state() + missing = [str(UNIT_DIR / u) for u in UNITS if state[u].get("LoadState") != "loaded"] + steps[:0] = [("link", *missing)] if missing else [] + elif action == "disable": + steps = [("disable", "--no-reload", *UNITS), ("link", "--no-reload", *(str(UNIT_DIR / u) for u in UNITS)), + ("daemon-reload",)] + for args in steps: + result = systemctl(*args) + if result.returncode: + return (result.stderr.strip() or f"systemctl {args[0]} failed").splitlines()[0] + return None + + def notify_daemon(): try: with socket.socket(socket.AF_UNIX) as sock: @@ -231,6 +278,10 @@ class Window(Adw.ApplicationWindow): self.rows = [] self.dialog = None + self.service_busy = False # a start/stop/enable/disable is running + self.polling = False + self.syncing = False # switches being set from the service state, not by the user + self.controller_status = "Checking for controllers…" self.controller_group = Adw.PreferencesGroup(title="Controller") self.keyboard_group = Adw.PreferencesGroup(title="Keyboard and mouse") @@ -240,7 +291,8 @@ class Window(Adw.ApplicationWindow): group.set_header_suffix(add) page = Adw.PreferencesPage() self.wake_group = self._wake_group() - for group in (self.controller_group, self.keyboard_group, self.wake_group, self._behavior_group()): + for group in (self._service_group(), self.controller_group, self.keyboard_group, self.wake_group, + self._behavior_group()): page.add(group) self.toasts = Adw.ToastOverlay(child=page) view = Adw.ToolbarView(content=self.toasts) @@ -248,6 +300,81 @@ class Window(Adw.ApplicationWindow): self.set_content(view) self.refresh() threading.Thread(target=self._load_controllers, daemon=True).start() + self._poll_service() + GLib.timeout_add_seconds(2, self._poll_service) + + def _service_group(self): + group = Adw.PreferencesGroup(title="Service") + self.running_row = Adw.SwitchRow(title="Running", subtitle="Checking…") + self.running_row.connect("notify::active", lambda r, _p: self.syncing or self.run_service( + "start" if r.get_active() else "stop")) + self.startup_row = Adw.SwitchRow(title="Start at system startup") + self.startup_row.connect("notify::active", lambda r, _p: self.syncing or self.run_service( + "enable" if r.get_active() else "disable")) + group.add(self.running_row) + group.add(self.startup_row) + return group + + def _poll_service(self): + if not self.polling and not self.service_busy: + self.polling = True + + def work(): + state = service_state() + GLib.idle_add(self._polled, state) + threading.Thread(target=work, daemon=True).start() + return True # keep polling: systemctl or a reboot can change it too + + def _polled(self, state): + self.polling = False + self.show_service(state) + + def show_service(self, state): + if self.service_busy: + return + active = {u: state[u].get("ActiveState", "unknown") for u in UNITS} + on = active["frame-voice.service"] in ("active", "activating", "reloading") + failed = [u for u in UNITS if active[u] == "failed"] + stopped = [u for u in UNITS if active[u] not in ("active", "activating", "reloading")] + if failed: + subtitle = f"{failed[0].removesuffix('.service')} failed. Log: journalctl --user -u {failed[0]}" + elif on and stopped: + subtitle = "Partly running: " + ", ".join(u.removesuffix(".service") for u in stopped) + " stopped" + elif on: + subtitle = "Dictation, wake words and controller push-to-talk are on" + else: + subtitle = "Stopped" + enabled = state["frame-voice.service"].get("UnitFileState") == "enabled" + self.syncing = True + self.running_row.set_active(on) + self.startup_row.set_active(enabled) + self.syncing = False + self.running_row.set_subtitle(subtitle) + self.startup_row.set_subtitle( + "Starts before SteamVR, so typing reaches game-mode windows" if enabled else + "Off: restart SteamVR after starting Frame Voice to type into game-mode windows") + + def run_service(self, action): + self.service_busy = True + row = self.running_row if action in ("start", "stop") else self.startup_row + row.set_subtitle({"start": "Starting…", "stop": "Stopping…", "enable": "Enabling…", "disable": "Disabling…"}[action]) + self.running_row.set_sensitive(False) + self.startup_row.set_sensitive(False) + + def work(): + error = service_action(action) + GLib.idle_add(self._service_done, action, error, service_state()) + threading.Thread(target=work, daemon=True).start() + + def _service_done(self, action, error, state): + self.service_busy = False + self.running_row.set_sensitive(True) + self.startup_row.set_sensitive(True) + self.show_service(state) + done = {"start": "Frame Voice started.", "stop": "Frame Voice stopped.", + "enable": "Frame Voice will start at system startup.", + "disable": "Frame Voice won't start at system startup."} + self.toast(f"Couldn't {action} Frame Voice: {error}" if error else done[action]) def _wake_group(self): group = Adw.PreferencesGroup( diff --git a/settings.py b/settings.py index 59d227f..b4e4582 100644 --- a/settings.py +++ b/settings.py @@ -18,7 +18,9 @@ import os import re from pathlib import Path -PATH = Path(os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config")) / "frame-voice/settings.json" +# Not under $XDG_CONFIG_HOME: Frametop's desktop points it at ~/.config/frametop, and the +# services (started by systemd) and the settings app must read the same file. +PATH = Path(os.environ.get("FRAME_VOICE_SETTINGS", "~/.config/frame-voice/settings.json")).expanduser() DEFAULT = { "keyboard": ["KEY_RIGHTCTRL"], "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], From a658b80aeed5af537799c8243be73b3f5685ff1d Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:52:49 -0600 Subject: [PATCH 7/9] README: the Frametop key relay is optional and needs SHARE_KEYS=1 Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c4569e9..147095f 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ On test clips (two TTS voices, 1 s pauses placed mid-sentence and between senten Live text never types while Ctrl, Alt, Shift or Super is held on any keyboard: the letters would reach apps as shortcuts. It waits for the release. That is why the keyboard default is tap mode: holding Right Ctrl to talk would hold back the text until you let go. (Pressing and releasing Ctrl on the virtual keyboard to "unhold" it did not work: letters still arrived as Ctrl shortcuts in the Frametop desktop.) When typing holds back, the log says which key on which device it waits for (`typing waits for release of ...`). -Keyboards grabbed by Frametop's input relay (while typing goes to its VR desktop) can't be read directly, so the relay sends their keys to the abstract socket `@frametop_keys`, and frame-voice treats them like keys from any other keyboard. It accepts them only from your own user, and skips its own typing keyboard, which the relay grabs too. +Only if you use Frametop: keyboards grabbed by Frametop's input relay (while typing goes to its VR desktop) can't be read directly, so the relay sends their keys to the abstract socket `@frametop_keys`, and frame-voice treats them like keys from any other keyboard. The relay only does this with `SHARE_KEYS=1` in `~/.config/frametop.conf`; without it, the push-to-talk key does nothing while typing goes to the desktop. frame-voice accepts these keys only from your own user, and skips its own typing keyboard, which the relay grabs too. With `SHARE_KEYS=1`, everything typed into the desktop, passwords included, goes to that socket, where any local process could listen while frame-voice isn't running. Without Frametop nothing sends to the socket; keyboards aren't grabbed, and frame-voice reads them directly. Audio cues are off by default (`[sounds] enabled = true` turns them on): rising tone = recording, falling tone = processing, single high tone = wake word heard, low tone = cancelled or nothing heard. From 9df7df5f91167e57dc47ecc769ee307895951bf2 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:54:01 -0600 Subject: [PATCH 8/9] Run programs from wake word commands that start with a set phrase A "commands" list in settings.json maps phrases to an argv. After the wake word, a command whose words start with a phrase runs that program (longest phrase wins) instead of the command hook, with the rest of the words as one extra argument. No shell is involved. The command event gains "matched" and "ran". Entries are validated and reload live. The settings app now writes only the keys it manages and keeps every other key in the file, so hand-edited commands survive its saves. Adds a small unittest suite for the matcher and the settings round trip. Co-Authored-By: Claude Opus 5.5 --- README.md | 41 ++++++++++++-- frame_voice.py | 45 ++++++++++++--- settings.py | 20 ++++++- tests/__init__.py | 0 tests/test_commands.py | 123 +++++++++++++++++++++++++++++++++++++++++ voice_commands.py | 92 ++++++++++++++++++++++++++++++ 6 files changed, 307 insertions(+), 14 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_commands.py create mode 100644 voice_commands.py diff --git a/README.md b/README.md index 147095f..bcf8731 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Local voice input for the Steam Frame. Nothing leaves the headset. - **Dictation**: tap a key (or hold a controller button) and speak. Words are typed as you talk into whatever has focus (desktop apps, search bars, terminals, game-mode windows) through a virtual uinput keyboard, transcribed on the headset by Moonshine. -- **Wake words**: say one (default "Hey Computer"; add your own in the settings app), then a command. The command is transcribed and handed to a hook script and to any event subscribers. This is the entry point for a voice assistant. +- **Wake words**: say one (default "Hey Computer"; add your own in the settings app), then a command. The command is transcribed and handed to a hook script and to any event subscribers, or runs a program you set up for it (see Voice commands). This is the entry point for a voice assistant. ## Pieces @@ -26,11 +26,11 @@ The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx op | Steam Frame controllers | Hold left **d-pad down** (default), speak, release. Works in games, the Steam UI and the desktop with no per-app setup. The button is observed, not captured, so whatever else it is bound to still happens. | | Desktop mode shortcut | KDE System Settings > Shortcuts > add a command shortcut running `frame-voice-ctl toggle`. | | Scripts, the assistant | `frame-voice-ctl toggle`, `start`, `stop`, `cancel`, `type "text"` | -| Voice | ", ..." runs `hooks/on-command`. "Hey Computer, type hello world" types "hello world" (example in the hook). | +| Voice | ", ..." runs a matching voice command (see Voice commands), otherwise `hooks/on-command`. "Hey Computer, type hello world" types "hello world" (example in the hook). | Recordings shorter than 2 s (`min_seconds`) are discarded with the cancel tone, so brief or accidental presses never type anything. -*Frame Voice Settings* (`./frame-voice-config`) starts and stops all three services (**Running**) and sets whether they start at system startup (see Services). Change the buttons there too: **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`; both services apply changes without a restart. Delete the file to go back to the defaults. +*Frame Voice Settings* (`./frame-voice-config`) starts and stops all three services (**Running**) and sets whether they start at system startup (see Services). Change the buttons there too: **Add** waits for you to press a controller button, key or mouse button (not left/right click, not the controller's system button). The app also sets hold or tap mode per trigger, switches live typing and sounds, and manages wake words: add any phrase in English words (they're spelled out for the spotter when loaded, no training), remove them, or turn listening off. Changes apply within a couple of seconds. Everything lives in `~/.config/frame-voice/settings.json`, along with voice commands (see Voice commands); both services apply changes without a restart. Delete the file to go back to the defaults. Live text works by agreement, with the approach and model borrowed from the Faceclaw app for Even Realities G2 glasses (Moonshine base through sherpa-onnx, on the CPU): the speech since your last pause is re-decoded every 0.8 s, and words two decodes in a row agree on are typed, except the last one, whose punctuation can still change. One word the two decodes disagree on (a filler heard as "a" once and "uh" the next time) doesn't hold back the words after it once three of them agree again. At a 0.8 s pause, or at the quietest point once a decode would pass 8 s of audio (Moonshine returns nothing past about 9 s), the window is decoded one last time and its remaining words are typed. @@ -59,6 +59,39 @@ The socket is `/run/user//frame-voice/control.sock` (systemd's runtime dir; An assistant can either replace `hooks/on-command` or subscribe with `frame-voice-ctl events` and act on `{"event": "command", "text": ...}`. +## Voice commands + +A command that starts with a set phrase can run a program. Add a `"commands"` list to `~/.config/frame-voice/settings.json` by hand (the settings app doesn't edit it, and keeps it when it saves). It is empty by default. Example: "Hey Computer, flag that" or "Hey Computer, flag that, fingers crossed" runs a hand tracking tool's flag command. + +```json + "commands": [ + { + "say": ["flag that", "flag this", "flag it", "flag"], + "run": ["~/.local/bin/ft-handsctl", "flag"] + } + ] +``` + +| Key | Meaning | +| --- | --- | +| `say` | Phrases. Case and punctuation don't matter. | +| `run` | The program and its arguments. `~` is expanded. It runs directly, never through a shell, so spoken words can't become shell syntax. | +| `rest` | Optional. `"arg"` (default): the words after the phrase are added as one more argument when there are any. `"none"`: they never are. | + +Matching works on whole words. A command matches when its words equal a phrase or start with it, so "Flag that." and "Flag that, fingers crossed." match "flag that", but "Flagging that" and "Please flag that" don't. If phrases of several entries match, the longest wins, then the first entry. The rest keeps your words as transcribed, without the punctuation that follows the phrase or a final period: the second example runs `ft-handsctl flag "fingers crossed"`. + +A matched command runs instead of `hooks/on-command`. The `command` event is still sent, with `"matched"` (the phrase) and `"ran"` (the argv) added. The program gets these environment variables: + +| Variable | Value | +| --- | --- | +| `FRAME_VOICE_EVENT` | `voice_command` | +| `FRAME_VOICE_TEXT` | The whole transcribed command | +| `FRAME_VOICE_PHRASE` | The phrase that matched, as written in `say` | +| `FRAME_VOICE_REST` | The words after it, or empty | +| `FRAME_VOICE_WAKE_UNIX` | When the wake word was heard, in Unix seconds (e.g. `1791582811.212`) | + +frame-voice doesn't wait for the program. A malformed entry is skipped with a warning in the log (`journalctl --user -u frame-voice`). Changes to the list apply within a couple of seconds, like the other settings. Updating frame-voice from a version without voice commands needs one `systemctl --user restart frame-voice`. + ## Config Copy `config.example.toml` to `~/.config/frame-voice/config.toml`. Every key is optional. Then `systemctl --user restart frame-voice`. @@ -98,7 +131,7 @@ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_NATIV cmake --build build -j5 --target whisper-server whisper-cli whisper-bench ``` -`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `frame-voice-config` launcher sets that up. +`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. Tests: `python3 -m unittest` in the repo (no audio stack needed). The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `frame-voice-config` launcher sets that up. The whisper service sets `DISABLE_GAMESCOPE_WSI=1`. Without it the Gamescope Vulkan layer aborts whisper-server at startup. diff --git a/frame_voice.py b/frame_voice.py index 3f12c38..b6d677d 100755 --- a/frame_voice.py +++ b/frame_voice.py @@ -3,7 +3,8 @@ One PipeWire capture stream feeds two consumers: * dictation: hotkey or socket command -> record -> whisper-server -> type via uinput - * wake word: sherpa-onnx keyword spotter -> capture the spoken command -> hooks + events + * wake word: sherpa-onnx keyword spotter -> capture the spoken command -> voice command, + or hooks + events """ import asyncio import io @@ -25,6 +26,7 @@ from pathlib import Path import evdev import settings +import voice_commands import numpy as np import sentencepiece import sherpa_onnx @@ -372,6 +374,7 @@ class FrameVoice: self.stream = None # streaming state of the current dictation, see start_dictation self.whisper_lock = asyncio.Lock() # one request at a time, in order self.command = None # dict while capturing a post-wake command + self.voice_commands = [] # from "commands" in settings.json, see voice_commands.py self.wake_enabled = settings.DEFAULT["wake_word"] # apply_options() sets the saved value self.mic_task = None self.mic_proc = None @@ -475,6 +478,11 @@ class FrameVoice: except OSError as err: log.warning("could not run %s: %s", argv[0], err) + def spawn(self, coro): + task = asyncio.create_task(coro) + self.tasks.add(task) + task.add_done_callback(self.tasks.discard) + def run_hook(self, key, *args, env=None): hook = self.wcfg.get(key) if not hook: @@ -942,7 +950,7 @@ class FrameVoice: self.chime("wake") self.broadcast("wake", keyword=keyword) self.run_hook("wake_hook", keyword, env={"FRAME_VOICE_EVENT": "wake"}) - self.command = {"chunks": [], "t0": time.monotonic(), "heard": False} + self.command = {"chunks": [], "t0": time.monotonic(), "heard": False, "wake_unix": time.time()} self.cmd_vad.reset() self.update_mic() @@ -957,7 +965,7 @@ class FrameVoice: elapsed = time.monotonic() - cmd["t0"] if cmd["heard"] and not speaking or elapsed > self.wcfg["max_command_seconds"]: self.command = None - asyncio.create_task(self.finish_command(np.concatenate(cmd["chunks"]))) + asyncio.create_task(self.finish_command(np.concatenate(cmd["chunks"]), cmd["wake_unix"])) elif not cmd["heard"] and elapsed > self.wcfg["command_timeout"]: self.command = None log.info("wake: no command heard") @@ -967,7 +975,7 @@ class FrameVoice: self.reset_kws() self.update_mic() - async def finish_command(self, samples): + async def finish_command(self, samples, wake_unix): self.chime("stop") try: text = await self.transcribe(samples) @@ -975,9 +983,23 @@ class FrameVoice: log.error("transcription failed: %s", err) self.broadcast("error", message=str(err)) return - self.broadcast("command", text=text) - if text: - self.run_hook("command_hook", text, env={"FRAME_VOICE_EVENT": "command"}) + found = voice_commands.match(text, self.voice_commands) if text else None + if found is None: + self.broadcast("command", text=text) + if text: + self.run_hook("command_hook", text, env={"FRAME_VOICE_EVENT": "command"}) + return + # A voice command runs instead of the command hook. The argv goes straight to + # exec, so the transcribed words never reach a shell. + log.info("voice command %r: %s", found.phrase, found.argv) + self.broadcast("command", text=text, matched=found.phrase, ran=found.argv) + self.spawn(self._run(found.argv, env={ + "FRAME_VOICE_EVENT": "voice_command", + "FRAME_VOICE_TEXT": text, + "FRAME_VOICE_REST": found.rest, + "FRAME_VOICE_PHRASE": found.phrase, + "FRAME_VOICE_WAKE_UNIX": f"{wake_unix:.3f}", + })) # --- hotkey @@ -1072,8 +1094,15 @@ class FrameVoice: def apply_options(self, b): """Settings from the config app. Only changed ones are applied, so a save doesn't undo e.g. `frame-voice-ctl wake off`.""" - changed = {k: b[k] for k in ("keyboard_mode", "wake_phrases", *settings.OPTIONS) if self.applied.get(k) != b[k]} + changed = {k: b[k] for k in ("keyboard_mode", "wake_phrases", "commands", *settings.OPTIONS) + if self.applied.get(k) != b[k]} self.applied.update(changed) + if "commands" in changed: + self.voice_commands, warnings = voice_commands.parse(changed.pop("commands")) + for warning in warnings: + log.warning("settings: %s", warning) + log.info("voice commands: %s", ", ".join( + f"{c.phrases[0][0]!r} -> {c.run[0]}" for c in self.voice_commands) or "none") if "keyboard_mode" in changed: self.kb_mode = changed["keyboard_mode"] if "live_typing" in changed: diff --git a/settings.py b/settings.py index b4e4582..4295a3b 100644 --- a/settings.py +++ b/settings.py @@ -5,12 +5,15 @@ "controller": [{"device": "/user/hand/left", "input": "/input/dpad_down/click"}], "keyboard_mode": "toggle", "controller_mode": "hold", "live_typing": true, "sounds": false, "wake_word": true, - "wake_phrases": ["HEY COMPUTER"]} + "wake_phrases": ["HEY COMPUTER"], + "commands": [{"say": ["flag that", "flag"], "run": ["~/.local/bin/my-tool", "flag"]}]} keyboard: evdev key or mouse button names, read from every physical keyboard and mouse. controller: a SteamVR device path and one of its boolean inputs, read from vrserver. *_mode: "hold" records while held; "toggle" starts on one press and stops on the next. wake_phrases: what to say before a command; any English words, spelled out at load time. +commands: wake word commands that run a program (see voice_commands.py). Edited by hand; +the settings app keeps them as they are. The options override config.toml. """ import json @@ -49,6 +52,8 @@ def load(): data = json.loads(PATH.read_text()) except (FileNotFoundError, ValueError): data = {} + if not isinstance(data, dict): + data = {} out = json.loads(json.dumps(DEFAULT)) if isinstance(data.get("keyboard"), list): out["keyboard"] = [k for k in data["keyboard"] if isinstance(k, str)] @@ -64,13 +69,24 @@ def load(): if isinstance(data.get("wake_phrases"), list): phrases = (clean_phrase(p) for p in data["wake_phrases"] if isinstance(p, str)) out["wake_phrases"] = list(dict.fromkeys(p for p in phrases if p)) + # Checked by voice_commands.parse(), which warns about each bad entry. + out["commands"] = data.get("commands", []) return out def save(data): + """Write the keys the settings app manages (DEFAULT). Every other key, "commands" + included, keeps its value from the file, so hand edits survive a save from the app.""" + try: + on_disk = json.loads(PATH.read_text()) + except (FileNotFoundError, ValueError): + on_disk = {} + if not isinstance(on_disk, dict): + on_disk = {} + merged = {**on_disk, **{k: data[k] for k in DEFAULT if k in data}} PATH.parent.mkdir(parents=True, exist_ok=True) tmp = PATH.with_suffix(".tmp") - tmp.write_text(json.dumps(data, indent=2) + "\n") + tmp.write_text(json.dumps(merged, indent=2) + "\n") tmp.replace(PATH) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..094b0eb --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,123 @@ +"""Voice command matching and the settings round trip. Run: python3 -m unittest""" +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import voice_commands as vc # noqa: E402 + +FLAG = {"say": ["flag that", "flag this", "flag it", "flag"], "run": ["~/.local/bin/tool", "flag"]} +TOOL = os.path.expanduser("~/.local/bin/tool") + + +def parsed(*entries): + commands, warnings = vc.parse(list(entries)) + return commands + + +class Normalise(unittest.TestCase): + def test_case_punctuation_space(self): + self.assertEqual(vc.normalise(" Flag that, fingers-crossed! "), "flag that fingers crossed") + self.assertEqual(vc.normalise("..."), "") + + +class Matching(unittest.TestCase): + def setUp(self): + self.commands = parsed(FLAG) + + def check(self, text, phrase, rest, argv): + m = vc.match(text, self.commands) + self.assertIsNotNone(m, text) + self.assertEqual((m.phrase, m.rest, m.argv), (phrase, rest, argv)) + + def test_whisper_forms(self): + self.check("Flag that.", "flag that", "", [TOOL, "flag"]) + self.check(" flag that", "flag that", "", [TOOL, "flag"]) + self.check("FLAG THAT!", "flag that", "", [TOOL, "flag"]) + + def test_rest_is_one_argument_with_user_words(self): + self.check("Flag that, fingers crossed.", "flag that", "fingers crossed", + [TOOL, "flag", "fingers crossed"]) + self.check("Flag that: the left hand is off...", "flag that", "the left hand is off", + [TOOL, "flag", "the left hand is off"]) + self.check("Flag that, is it wrong?", "flag that", "is it wrong?", [TOOL, "flag", "is it wrong?"]) + + def test_longest_phrase_wins(self): + self.check("Flag it.", "flag it", "", [TOOL, "flag"]) + self.check("Flag, the pinch.", "flag", "the pinch", [TOOL, "flag", "the pinch"]) + long = parsed(FLAG, {"say": ["flag that hand"], "run": ["other"]}) + self.assertEqual(vc.match("Flag that hand now.", long).argv, ["other", "now"]) + self.assertEqual(vc.match("Flag that, now.", long).argv, [TOOL, "flag", "now"]) + + def test_ties_go_to_first_entry(self): + both = parsed({"say": ["go"], "run": ["a"]}, {"say": ["Go!"], "run": ["b"]}) + self.assertEqual(vc.match("go", both).argv, ["a"]) + + def test_no_match(self): + for text in ("Flagging that.", "Please flag that.", "", "fla", "Type hello world."): + self.assertIsNone(vc.match(text, self.commands), text) + self.assertIsNone(vc.match("flag", [])) + + def test_rest_none(self): + cmds = parsed({**FLAG, "rest": "none"}) + m = vc.match("Flag that, fingers crossed.", cmds) + self.assertEqual((m.rest, m.argv), ("fingers crossed", [TOOL, "flag"])) + + def test_shell_text_stays_one_argument(self): + m = vc.match("flag that; rm -rf $HOME `x`", self.commands) + self.assertEqual(m.argv, [TOOL, "flag", "rm -rf $HOME `x`"]) + + +class Parsing(unittest.TestCase): + def test_malformed_entries_are_skipped_with_warnings(self): + entries = [ + "flag", + {"run": ["x"]}, + {"say": "flag", "run": "x --y"}, + {"say": ["flag"], "run": []}, + {"say": ["flag"], "run": ["", "x"]}, + {"say": ["flag"], "run": ["x", 3]}, + {"say": [1, None], "run": ["x"]}, + {"say": ["...", " "], "run": ["x"]}, + {"say": ["flag"], "run": ["x"], "rest": "all"}, + {"say": "single", "run": ["ok"]}, + ] + commands, warnings = vc.parse(entries) + self.assertEqual(len(commands), 1) + self.assertEqual(len(warnings), len(entries) - 1) + self.assertEqual(vc.match("Single.", commands).argv, ["ok"]) + + def test_not_a_list(self): + self.assertEqual(vc.parse(None), ([], [])) + commands, warnings = vc.parse({"say": ["x"]}) + self.assertEqual((commands, len(warnings)), ([], 1)) + + +class SettingsRoundTrip(unittest.TestCase): + def test_app_save_keeps_commands_and_unknown_keys(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "settings.json" + path.write_text(json.dumps({"wake_phrases": ["HEY COMPUTER"], "commands": [FLAG], "extra": 1})) + os.environ["FRAME_VOICE_SETTINGS"] = str(path) + sys.modules.pop("settings", None) + import settings + try: + data = settings.load() + self.assertEqual(data["commands"], [FLAG]) + data["sounds"] = True + data["commands"] = [] # a stale copy must not overwrite the file's list + settings.save(data) + saved = json.loads(path.read_text()) + self.assertEqual((saved["commands"], saved["extra"], saved["sounds"]), ([FLAG], 1, True)) + path.write_text("{}") + self.assertEqual(settings.load()["commands"], []) + finally: + del os.environ["FRAME_VOICE_SETTINGS"] + sys.modules.pop("settings", None) + + +if __name__ == "__main__": + unittest.main() diff --git a/voice_commands.py b/voice_commands.py new file mode 100644 index 0000000..98f4058 --- /dev/null +++ b/voice_commands.py @@ -0,0 +1,92 @@ +"""Voice commands: run a program when a wake word command starts with a set phrase. + +From "commands" in ~/.config/frame-voice/settings.json: + [{"say": ["flag that", "flag"], "run": ["~/.local/bin/my-tool", "flag"], "rest": "arg"}] + +say: phrases. A command matches when its words equal a phrase or start with it. + Case and punctuation don't matter. The longest matching phrase of all entries wins. +run: the program and its arguments, run directly (never through a shell). "~" is expanded. +rest: "arg" (default) appends the words after the phrase as one extra argument when there + are any. "none" never appends them. + +Pure standard library, so the tests can import it without the audio stack. +""" +import os +import re +from typing import NamedTuple + +REST_MODES = ("arg", "none") +WORD = re.compile(r"\w+") + + +class Command(NamedTuple): + phrases: tuple # (as written, normalised word tuple) + run: tuple + rest: str + + +class Match(NamedTuple): + phrase: str # as written in settings.json + rest: str # the user's words after the phrase, "" when none + argv: list + + +def words(text): + """Lowercase words with punctuation as separators: "Flag that, OK." -> ("flag", "that", "ok").""" + return tuple(w.lower() for w in WORD.findall(text)) + + +def normalise(text): + return " ".join(words(text)) + + +def parse(entries): + """Valid commands from the settings list, and a warning for each entry skipped.""" + commands, warnings = [], [] + if not isinstance(entries, list): + if entries is not None: + warnings.append(f"commands: expected a list, got {type(entries).__name__}") + return commands, warnings + for i, entry in enumerate(entries): + problem = None + if not isinstance(entry, dict): + problem = "not an object" + else: + say = entry.get("say") + if isinstance(say, str): + say = [say] + run = entry.get("run") + rest = entry.get("rest", "arg") + if not isinstance(say, list) or not all(isinstance(p, str) for p in say): + problem = '"say" must be a list of phrases' + elif not isinstance(run, list) or not run or not all(isinstance(a, str) for a in run) or not run[0]: + problem = '"run" must be a list: program, then arguments' + elif rest not in REST_MODES: + problem = f'"rest" must be one of {", ".join(REST_MODES)}' + else: + phrases = tuple((p.strip(), words(p)) for p in say if words(p)) + if not phrases: + problem = '"say" has no words' + if problem: + warnings.append(f"commands[{i}] skipped: {problem}") + continue + commands.append(Command(phrases, tuple(os.path.expanduser(a) for a in run), rest)) + return commands, warnings + + +def match(text, commands): + """The command `text` starts with, or None. Ties go to the earlier entry.""" + found = list(WORD.finditer(text or "")) + heard = tuple(m.group().lower() for m in found) + best = None + for cmd in commands: + for phrase, pw in cmd.phrases: + if heard[:len(pw)] == pw and (best is None or len(pw) > len(best[2])): + best = (cmd, phrase, pw) + if best is None: + return None + cmd, phrase, pw = best + rest = text[found[len(pw) - 1].end():] + rest = re.sub(r"^\W+", "", rest).rstrip(" \t\n.") + argv = list(cmd.run) + ([rest] if rest and cmd.rest == "arg" else []) + return Match(phrase, rest, argv) From 1d88af4c6eea6f3fd0ad7d025224440bcc2879d5 Mon Sep 17 00:00:00 2001 From: DeeJanuz <45082401+DeeJanuz@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:26:08 -0600 Subject: [PATCH 9/9] Push-to-talk key counts only on its own, not with another modifier held A modifier already held when the key went down (Shift, then Right Ctrl) still started or stopped dictation; only keys pressed during the hold counted as a shortcut. Now any other modifier held on any keyboard at the press makes it a shortcut too, in tap and hold mode. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- frame_voice.py | 10 +++- frame_voice_config.py | 2 +- tests/test_hotkey.py | 121 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 tests/test_hotkey.py diff --git a/README.md b/README.md index bcf8731..61f0ba2 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The mic is captured once with `pw-record`. The wake word spotter (sherpa-onnx op | Where | How | | --- | --- | -| Keyboard or mouse | Tap **Right Ctrl** (default) to start, tap again to stop. Shortcuts with it (Ctrl+C) don't start or stop dictation. Hold mode is available in the settings app. | +| Keyboard or mouse | Tap **Right Ctrl** (default) to start, tap again to stop. It counts only on its own: shortcuts with it (Ctrl+C), or with another modifier held when you press it (Shift+Right Ctrl), don't start or stop dictation. Hold mode is available in the settings app. | | Steam Frame controllers | Hold left **d-pad down** (default), speak, release. Works in games, the Steam UI and the desktop with no per-app setup. The button is observed, not captured, so whatever else it is bound to still happens. | | Desktop mode shortcut | KDE System Settings > Shortcuts > add a command shortcut running `frame-voice-ctl toggle`. | | Scripts, the assistant | `frame-voice-ctl toggle`, `start`, `stop`, `cancel`, `type "text"` | @@ -131,7 +131,7 @@ cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON -DGGML_NATIV cmake --build build -j5 --target whisper-server whisper-cli whisper-bench ``` -`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. Tests: `python3 -m unittest` in the repo (no audio stack needed). The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `frame-voice-config` launcher sets that up. +`vendor/prefix` holds SPIRV-Headers, which SteamOS does not ship. Tests: `.venv/bin/python -m unittest` in the repo (no audio stack needed; with the system `python3` the push-to-talk key tests are skipped, since they need `evdev`). The Python side lives in `.venv` (`evdev sherpa-onnx numpy sentencepiece websockets`). `models/sherpa-onnx-moonshine-base-en-quantized-2026-02-27` holds the Moonshine files from huggingface.co/csukuangfj2 (`encoder_model.ort`, `decoder_model_merged.ort`, `tokens.txt`). The settings app uses the system Python for GTK (`gi`) and the venv for the rest; the `frame-voice-config` launcher sets that up. The whisper service sets `DISABLE_GAMESCOPE_WSI=1`. Without it the Gamescope Vulkan layer aborts whisper-server at startup. diff --git a/frame_voice.py b/frame_voice.py index b6d677d..274e841 100755 --- a/frame_voice.py +++ b/frame_voice.py @@ -1009,8 +1009,14 @@ class FrameVoice: self._track_modifier(dev, code, value) if code in self.hotkeys: if value == 1: - self.hk_down, self.hk_chord = True, False - if self.kb_mode == "hold": + # Only the key on its own: a modifier already held (Shift, then Right Ctrl, or + # Ctrl, then a mouse button) makes it a shortcut, not a dictation gesture. + others = sorted({c for _, c in self.mods_down if c != code}) + self.hk_down, self.hk_chord = True, bool(others) + if others: + log.info("hotkey: %s with %s held, not dictation", self.hotkeys[code], + "+".join(e.KEY.get(c, str(c)) for c in others)) + elif self.kb_mode == "hold": self.hk_timer = loop.call_later(self.dcfg["hold_delay"], self._hold_start) elif value == 0: self.hk_down = False diff --git a/frame_voice_config.py b/frame_voice_config.py index 21132e2..61759d3 100755 --- a/frame_voice_config.py +++ b/frame_voice_config.py @@ -84,7 +84,7 @@ def key_note(name, mode): if name in settings.MODIFIERS: if mode == "hold": return "While it's held, text waits for release (it would turn letters into shortcuts)" - return "Shortcuts with it (like Ctrl+C) don't start or stop dictation" + return "Counts only on its own: not in shortcuts like Ctrl+C, or with another modifier held" if name.startswith("KEY_") and (len(name) == 5 or name in ("KEY_SPACE", "KEY_ENTER")): return "This key also types into apps" return name diff --git a/tests/test_hotkey.py b/tests/test_hotkey.py new file mode 100644 index 0000000..b7e1b6e --- /dev/null +++ b/tests/test_hotkey.py @@ -0,0 +1,121 @@ +"""The push-to-talk key starts or stops dictation only when pressed on its own. +Needs the venv's Python (evdev): .venv/bin/python -m unittest""" +import asyncio +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +try: + import frame_voice as fv # noqa: E402 +except ImportError as err: + raise unittest.SkipTest(f"needs the venv's Python: {err}") + +e = fv.e + + +class Dev: + def __init__(self, path): + self.path = path + + +KEYBOARD, OTHER, MOUSE = Dev("/dev/input/event3"), Dev("/dev/input/event7"), Dev("/dev/input/event9") + + +class Daemon(fv.FrameVoice): + """on_key() and the state it reads, without models, mic or typing.""" + + def __init__(self, hotkeys, mode): + self.hotkeys = {e.ecodes[name]: name for name in hotkeys} + self.kb_mode = mode + self.dcfg = {"hold_delay": 0.0} + self.hk_down = self.hk_chord = False + self.hk_timer = None + self.mods_down = set() + self.mods_released = asyncio.Event() + self._held_logged = None + self.stream = None + self.recording_source = None + self.calls = [] + + async def toggle_dictation(self, source): + self.calls.append("toggle") + + def start_dictation(self, source): + self.calls.append("start") + self.recording_source = source + + async def stop_dictation(self): + self.calls.append("stop") + self.recording_source = None + + def cancel_dictation(self, chime=True): + self.calls.append("cancel") + self.recording_source = None + + +class Hotkey(unittest.IsolatedAsyncioTestCase): + async def press(self, daemon, *events): + """events: (device, key name, 1 for down or 0 for up).""" + for dev, name, value in events: + daemon.on_key(dev, e.ecodes[name], value) + await asyncio.sleep(0) + await asyncio.sleep(0.01) # the tasks and the hold timer it started + return daemon.calls + + async def test_tap_alone_toggles(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 1), (KEYBOARD, "KEY_RIGHTCTRL", 0)), + ["toggle"]) + + async def test_modifier_held_before(self): + for other in ("KEY_LEFTSHIFT", "KEY_RIGHTSHIFT", "KEY_LEFTALT", "KEY_LEFTMETA", "KEY_LEFTCTRL"): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + calls = await self.press(d, (KEYBOARD, other, 1), (KEYBOARD, "KEY_RIGHTCTRL", 1), + (KEYBOARD, "KEY_RIGHTCTRL", 0), (KEYBOARD, other, 0)) + self.assertEqual(calls, [], other) + + async def test_modifier_released_first(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_LEFTSHIFT", 1), (KEYBOARD, "KEY_RIGHTCTRL", 1), + (KEYBOARD, "KEY_LEFTSHIFT", 0), (KEYBOARD, "KEY_RIGHTCTRL", 0)), []) + + async def test_modifier_held_on_another_keyboard(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + self.assertEqual(await self.press(d, (OTHER, "KEY_LEFTALT", 1), (KEYBOARD, "KEY_RIGHTCTRL", 1), + (KEYBOARD, "KEY_RIGHTCTRL", 0), (OTHER, "KEY_LEFTALT", 0)), []) + + async def test_modifier_pressed_during(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 1), (KEYBOARD, "KEY_LEFTSHIFT", 1), + (KEYBOARD, "KEY_LEFTSHIFT", 0), (KEYBOARD, "KEY_RIGHTCTRL", 0)), []) + + async def test_shortcut_with_the_key(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 1), (KEYBOARD, "KEY_C", 1), + (KEYBOARD, "KEY_C", 0), (KEYBOARD, "KEY_RIGHTCTRL", 0)), []) + + async def test_tap_alone_after_a_shortcut(self): + d = Daemon(["KEY_RIGHTCTRL"], "toggle") + await self.press(d, (KEYBOARD, "KEY_LEFTSHIFT", 1), (KEYBOARD, "KEY_RIGHTCTRL", 1), + (KEYBOARD, "KEY_RIGHTCTRL", 0), (KEYBOARD, "KEY_LEFTSHIFT", 0)) + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 1), (KEYBOARD, "KEY_RIGHTCTRL", 0)), + ["toggle"]) + + async def test_mouse_button_with_ctrl(self): + d = Daemon(["BTN_SIDE"], "toggle") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_LEFTCTRL", 1), (MOUSE, "BTN_SIDE", 1), + (MOUSE, "BTN_SIDE", 0), (KEYBOARD, "KEY_LEFTCTRL", 0)), []) + self.assertEqual(await self.press(d, (MOUSE, "BTN_SIDE", 1), (MOUSE, "BTN_SIDE", 0)), ["toggle"]) + + async def test_hold_mode(self): + d = Daemon(["KEY_RIGHTCTRL"], "hold") + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_LEFTSHIFT", 1), (KEYBOARD, "KEY_RIGHTCTRL", 1)), []) + await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 0), (KEYBOARD, "KEY_LEFTSHIFT", 0)) + self.assertEqual(d.calls, []) + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 1)), ["start"]) + self.assertEqual(await self.press(d, (KEYBOARD, "KEY_RIGHTCTRL", 0)), ["start", "stop"]) + + +if __name__ == "__main__": + unittest.main()