Merge remote-tracking branch 'origin/main' into ci/frame-smoke-source-commit

Co-authored-by: SAPHID <saphid@users.noreply.github.com>

.github/workflows/checks.yml

@@ -77,6 +77,11 @@ jobs:
       - name: OBS lifecycle tests
         run: node app/test/obs-source.test.js
       - name: Server tests
+        # The dedicated wireless-windows job runs the live inventory on a fresh
+        # Windows runner; do not repeat it after the full server suite has
+        # consumed this runner's probe budget.
+        env:
+          FRAME_SKIP_WINDOWS_WIRELESS_LIVE: "1"
         run: python -m unittest discover -s tests -v
 
   # Fast, PC-only Windows query validation, without capture libraries or a headset.

docs/releasing.md

@@ -57,6 +57,16 @@ count. So a build reaches people only when you publish it, after testing it.
    publish unless every required job succeeded; it cannot prove physical
    headset or installer behavior on its own.
 
+   Supply the saved smoke result to the publisher. Both its dry run and publish
+   command reject a result from another or dirty commit, incomplete title or
+   cleanup steps, a skip or failure, or an unconfirmed headset lock release:
+
+   ```sh
+   export FRAME_SMOKE_RESULT=/path/to/smoke-result.json
+   python3 scripts/verify-frame-smoke-result.py "$FRAME_SMOKE_RESULT" \
+     "$(git rev-parse 'v0.4.0^{commit}')"
+   ```
+
    ```sh
    scripts/publish-release.sh --dry-run v0.4.0   # checks and shows update.json, changes nothing
    scripts/publish-release.sh v0.4.0

docs/testing.md

@@ -215,7 +215,10 @@ acquires the shared `/tmp/frame-test.lock` with a unique owner token. It reads
 SteamVR activity, proximity and battery while locked. An awake headset, unknown
 activity, battery below 20%, or a busy lock stops the run without installing
 anything. It releases only its own lock after cleanup. The JSON result includes
-the read-only preflight for completed smoke runs.
+the read-only preflight and the source checkout's commit and clean/dirty state.
+For a release, run it from a clean checkout of the exact release commit;
+`scripts/verify-frame-smoke-result.py` rejects a different or dirty checkout,
+missing steps, skipped or failed launches, and an unconfirmed lock release.
 
 It checks `properties.json` and the status, then installs, launches and
 removes three tiny titles built from bytes by `tests/smoke/tiny_programs.py`

scripts/publish-release.sh

@@ -3,7 +3,7 @@
 # (docs/releasing.md). Checks every installer is attached with a SHA-256
 # digest first, since the app's updater refuses assets without one, then
 # attaches update.json, the manifest the updater reads.
-# Usage: scripts/publish-release.sh [--dry-run] v0.4.0
+# Usage: FRAME_SMOKE_RESULT=/path/to/result.json scripts/publish-release.sh [--dry-run] v0.4.0
 #
 # Everything goes through the REST API (gh api), not `gh release view/edit`:
 # those look a draft up by its tag, and a draft can show tag_name
@@ -34,7 +34,7 @@ gh api "repos/$repo/git/ref/tags/$tag" >/dev/null 2>&1 \
 # A complete asset list is insufficient: a cancelled installer build can leave
 # old uploaded bytes behind. Check the exact tagged commit's main/e2e jobs and
 # the latest exact-tag release attempt before any release mutation.
-python3 "$(dirname "$0")/verify-release-gates.py" "$tag"
+python3 "$(dirname "$0")/verify-release-gates.py" "$tag" "${FRAME_SMOKE_RESULT:-}"
 
 # Every release, drafts included, one JSON object per line.
 gh api --paginate "repos/$repo/releases?per_page=100" --jq '.[]' > "$tmp/releases"

scripts/verify-frame-smoke-result.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Check a locked Frame smoke result against the exact release commit."""
+
+import json
+import re
+import sys
+from pathlib import Path
+
+
+REQUIRED_STEPS = {
+    'cleanup: leftovers from earlier runs',
+    'devkit service: properties.json',
+    'status',
+    'cleanup: everything this run installed',
+    *(f'{kind}: {action}' for kind in ('arm64', 'x86_64', 'exe')
+      for action in ('install', 'launch', 'remove')),
+}
+
+
+def verify(result, commit):
+    if not isinstance(result, dict):
+        raise ValueError('smoke result is not an object')
+    if not re.fullmatch(r'[0-9a-f]{40}', commit):
+        raise ValueError('expected a full lowercase commit SHA-1')
+    if result.get('source') != {'commit': commit, 'clean': True}:
+        raise ValueError('smoke source is not the exact clean release commit')
+    if result.get('lock_released') is not True:
+        raise ValueError('smoke lock release was not recorded')
+    steps = result.get('steps')
+    if not isinstance(steps, list) or not steps:
+        raise ValueError('smoke steps are missing')
+    names = [step.get('step') for step in steps if isinstance(step, dict)]
+    if (len(names) != len(steps) or any(not isinstance(name, str) for name in names) or
+            len(names) != len(set(names)) or not REQUIRED_STEPS.issubset(names)):
+        raise ValueError('smoke steps are missing or duplicated')
+    passed = sum(step.get('ok') is True and not step.get('skipped') for step in steps)
+    if (passed != len(steps) or result.get('passed') != passed or
+            result.get('skipped') != 0 or result.get('failed') != 0):
+        raise ValueError('smoke contains a failed, skipped, or inconsistent step')
+    state = result.get('preflight') or {}
+    if (not isinstance(state, dict) or state.get('activity') != 3 or
+            not isinstance(state.get('proximity'), list) or not state['proximity'] or
+            any(type(value) not in (int, float) or value < 0 or value > 10
+                for value in state['proximity']) or
+            not isinstance(state.get('battery'), list) or not state['battery'] or
+            any(type(value) is not int or value < 20 or value > 100
+                for value in state['battery'])):
+        raise ValueError('locked standby preflight is missing')
+
+
+def main(argv):
+    if len(argv) != 3:
+        raise ValueError('usage: verify-frame-smoke-result.py RESULT.json RELEASE_COMMIT_SHA')
+    result = json.loads(Path(argv[1]).read_text(encoding='utf-8'))
+    verify(result, argv[2])
+    print(f'Frame smoke passed for clean release commit {argv[2]}')
+
+
+if __name__ == '__main__':
+    try:
+        main(sys.argv)
+    except (OSError, ValueError) as error:
+        sys.exit(f'Frame smoke gate failed: {error}')

scripts/verify-release-gates.py

@@ -1,13 +1,14 @@
 #!/usr/bin/env python3
-"""Refuse publication unless the exact tag commit and tagged installers passed CI.
+"""Refuse publication unless CI, tagged installers, and exact Frame smoke passed.
 
 This checks GitHub's current run and job records, not a prior PR's check marks.
-The real Frame smoke and independent package checks remain release-procedure
+Independent package checks and release verification remain release-procedure
 gates documented in docs/releasing.md.
 """
 import json
 import subprocess
 import sys
+from pathlib import Path
 from urllib.parse import quote
 
 
@@ -98,18 +99,26 @@ def successful_jobs(run, required):
     print(f"gate passed: {run.get('html_url')} ({len(latest_jobs)} jobs, {attempt} attempts)")
 
 
-def main(tag):
+def main(tag, smoke_result):
     if not tag.startswith("v") or not tag[1:]:
         raise GateError("expected a version tag beginning with v")
+    if not smoke_result:
+        raise GateError("FRAME_SMOKE_RESULT is required for the locked real Frame smoke gate")
     sha = tag_commit(tag)
     successful_jobs(exact_run("checks.yml", "main", sha, "push"), MAIN_JOBS)
     successful_jobs(exact_run("decky-runtime.yml", "main", sha, "push"), DECKY_JOBS)
     successful_jobs(exact_run("release.yml", tag, sha, "push"), RELEASE_JOBS)
+    command = [sys.executable, str(Path(__file__).with_name("verify-frame-smoke-result.py")),
+               smoke_result, sha]
+    checked = subprocess.run(command, text=True, capture_output=True, check=False)
+    if checked.returncode:
+        raise GateError(checked.stderr.strip() or "Frame smoke result verification failed")
+    print(checked.stdout.strip())
     print(f"exact release commit: {sha}")
 
 
 if __name__ == "__main__":
     try:
-        main(sys.argv[1])
+        main(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else None)
     except (IndexError, GateError) as error:
         sys.exit(f"release gate failed: {error}")

tests/smoke/frame_smoke.py

@@ -35,6 +35,7 @@ import argparse
 import json
 import os
 import platform
+import re
 import secrets
 import shlex
 import subprocess
@@ -61,6 +62,20 @@ PREFIX = 'fc_smoke_'
 LOCK = '/tmp/frame-test.lock'
 
 
+def source_revision():
+    """Record which checkout produced the smoke result for release review."""
+    try:
+        head = subprocess.run(['git', '-C', str(ROOT), 'rev-parse', 'HEAD'],
+                              capture_output=True, text=True, timeout=10, check=True).stdout.strip()
+        status = subprocess.run(['git', '-C', str(ROOT), 'status', '--porcelain'],
+                                capture_output=True, text=True, timeout=10, check=True).stdout
+    except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired):
+        return {'commit': None, 'clean': False}
+    if not isinstance(head, str) or not re.fullmatch(r'[0-9a-f]{40}', head):
+        return {'commit': None, 'clean': False}
+    return {'commit': head, 'clean': not bool(status.strip())}
+
+
 def remote(command, *, input=None):
     """Use a bounded SSH call for shared-lock operations."""
     return subprocess.run(['ssh', '-o', 'BatchMode=yes', '-o', 'ConnectTimeout=8', ALIAS, command],
@@ -420,12 +435,14 @@ def run_smoke(args, before):
     failed = len(smoke.steps) - passed - skipped
     RESULTS.mkdir(exist_ok=True)
     out = RESULTS / f"{started.replace(':', '')}-{build}.json"
-    out.write_text(json.dumps({'started': started, 'alias': ALIAS, 'preflight': before,
+    out.write_text(json.dumps({'started': started, 'alias': ALIAS, 'source': source_revision(),
+                               'preflight': before,
                                'os_release': release, 'paired': args.pair,
                                'passed': passed, 'skipped': skipped, 'failed': failed,
+                               'lock_released': False,
                                'devkit_utils_there_before': had_utils, 'steps': smoke.steps}, indent=1))
     print(f'==> {passed} passed, {skipped} skipped, {failed} failed on build {build}; results in {out}')
-    return 0 if failed == 0 and skipped == 0 else 1
+    return (0 if failed == 0 and skipped == 0 else 1), out
 
 
 def main():
@@ -440,12 +457,12 @@ def main():
     if not locked:
         print('Shared Frame lock busy or unreachable; no headset status read or smoke run.', file=sys.stderr)
         return 2
-    result = 1
+    result, output = 1, None
     try:
         try:
             before = preflight()
             print('Locked read-only preflight: ' + json.dumps(before), flush=True)
-            result = run_smoke(args, before)
+            result, output = run_smoke(args, before)
         except (OSError, ValueError, RuntimeError, subprocess.TimeoutExpired) as failure:
             print(str(failure), file=sys.stderr)
             result = 2
@@ -457,6 +474,14 @@ def main():
         if not released:
             print('ERROR: could not release owned Frame lock.', file=sys.stderr)
             result = 1
+        elif output is not None:
+            try:
+                saved = json.loads(output.read_text())
+                saved['lock_released'] = True
+                output.write_text(json.dumps(saved, indent=1))
+            except (OSError, ValueError) as failure:
+                print(f'ERROR: could not record Frame lock release: {failure}', file=sys.stderr)
+                result = 1
     return result
 
 

tests/test_publish_release.py

@@ -38,15 +38,35 @@ def draft(tag_name="untagged-c6ddfed7f75d67db2e99", name="Frame Control 9.8.7",
 
 class PublishRelease(unittest.TestCase):
     def run_script(self, releases, *args, tags="v9.8.7", gates=None,
-                   corrupt_manifest_readback=False):
+                   corrupt_manifest_readback=False, smoke=None):
         d = Path(tempfile.mkdtemp())
         (d / "releases.json").write_text(json.dumps(releases))
         (d / "gates.json").write_text(json.dumps(gates or {}))
+        steps = ["cleanup: leftovers from earlier runs", "devkit service: properties.json",
+                 "status", "cleanup: everything this run installed"] + [
+                     f"{kind}: {action}" for kind in ("arm64", "x86_64", "exe")
+                     for action in ("install", "launch", "remove")]
+        clean_smoke = {"source": {"commit": "a" * 40, "clean": True},
+                       "lock_released": True,
+                       "steps": [{"step": name, "ok": True} for name in steps],
+                       "passed": len(steps), "skipped": 0, "failed": 0,
+                       "preflight": {"activity": 3, "proximity": [2.3125], "battery": [100]}}
+        if smoke == "skipped":
+            clean_smoke["steps"][0].update(ok=False, skipped=True)
+            clean_smoke.update(passed=len(steps) - 1, skipped=1)
+        elif smoke == "unreleased":
+            clean_smoke["lock_released"] = False
+        if smoke is not False:
+            (d / "smoke.json").write_text(json.dumps(
+                clean_smoke if smoke is None or isinstance(smoke, str) else smoke))
         env = dict(os.environ, PATH="%s:%s" % (FAKEGH, os.environ["PATH"]),
                    FAKEGH_RELEASES=str(d / "releases.json"), FAKEGH_TAGS=tags,
                    FAKEGH_LOG=str(d / "log"), FAKEGH_UPLOAD=str(d / "upload.json"),
                    FAKEGH_GATES=str(d / "gates.json"),
                    FAKEGH_CORRUPT_MANIFEST_READBACK="1" if corrupt_manifest_readback else "0")
+        env.pop("FRAME_SMOKE_RESULT", None)
+        if smoke is not False:
+            env["FRAME_SMOKE_RESULT"] = str(d / "smoke.json")
         p = subprocess.run(["sh", str(SCRIPT), *args], env=env, capture_output=True, text=True, timeout=30)
         log = [json.loads(line) for line in (d / "log").read_text().splitlines()] if (d / "log").exists() else []
         upload = json.loads((d / "upload.json").read_text()) if (d / "upload.json").exists() else None
@@ -156,6 +176,18 @@ class PublishRelease(unittest.TestCase):
             self.assertIsNone(upload)
             self.assertFalse(any("-X" in call for call in log))
 
+    def test_refuses_missing_or_skipped_frame_smoke_before_mutation(self):
+        for smoke in (False,
+                      {"source": {"commit": "b" * 40, "clean": True}},
+                      {"source": {"commit": "a" * 40, "clean": False}},
+                      "skipped", "unreleased"):
+            with self.subTest(smoke=smoke):
+                p, log, upload = self.run_script([draft()], "v9.8.7", smoke=smoke)
+                self.assertNotEqual(p.returncode, 0)
+                self.assertIn("release gate failed", p.stderr)
+                self.assertIsNone(upload)
+                self.assertFalse(any("-X" in call for call in log))
+
     def test_refuses_a_prior_cancelled_installer_after_successful_rerun(self):
         gates = {"release_attempts": 2,
                  "prior_bad_job": "build (macos-latest, dist, app/dist/*.dmg app/dist/*.zip)",

tests/test_smoke_cleanup.py

@@ -95,6 +95,18 @@ class RealProcesses(unittest.TestCase):
 
 
 class Safety(unittest.TestCase):
+    def test_source_revision_records_clean_checkout_and_rejects_invalid_head(self):
+        head = 'a' * 40
+        with mock.patch.object(frame_smoke.subprocess, 'run', side_effect=[
+                mock.Mock(stdout=head + '\n'), mock.Mock(stdout='')]):
+            self.assertEqual(frame_smoke.source_revision(), {'commit': head, 'clean': True})
+        with mock.patch.object(frame_smoke.subprocess, 'run', side_effect=[
+                mock.Mock(stdout=head + '\n'), mock.Mock(stdout=' M tests/smoke/frame_smoke.py\n')]):
+            self.assertEqual(frame_smoke.source_revision(), {'commit': head, 'clean': False})
+        with mock.patch.object(frame_smoke.subprocess, 'run', side_effect=[
+                mock.Mock(stdout='not-a-commit\n'), mock.Mock(stdout='')]):
+            self.assertEqual(frame_smoke.source_revision(), {'commit': None, 'clean': False})
+
     def test_locked_preflight_rejects_active_or_unknown_proximity_before_titles(self):
         for proximity in ([], [88.0625], [4.6875, 88.0625], [None]):
             with self.subTest(proximity=proximity), mock.patch.object(
@@ -147,6 +159,8 @@ class Safety(unittest.TestCase):
                 else:
                     smoke.step(f'{kind}: launch', lambda: {'outcome': 'running'})
             with mock.patch.object(frame_smoke, 'RESULTS', output), \
+                 mock.patch.object(frame_smoke, 'source_revision', return_value={
+                     'commit': 'a' * 40, 'clean': True}), \
                  mock.patch.object(frame_smoke, 'os_release', return_value={
                      'BUILD_ID': 'fake', 'VERSION_ID': 'fake', 'VARIANT_ID': 'fake'}), \
                  mock.patch.object(frame_smoke, 'ssh', return_value='yes'), \
@@ -155,10 +169,14 @@ class Safety(unittest.TestCase):
                  mock.patch.object(frame_smoke, 'status', return_value={}), \
                  mock.patch.object(frame_smoke, 'title_cycle', side_effect=cycle), \
                  mock.patch.object(frame_smoke.tempfile, 'mkdtemp', return_value=str(folder)):
-                self.assertEqual(frame_smoke.run_smoke(mock.Mock(pair=False),
-                                 {'activity': 3, 'proximity': [], 'battery': [100]}), 1)
+                code, saved_path = frame_smoke.run_smoke(mock.Mock(pair=False),
+                                 {'activity': 3, 'proximity': [], 'battery': [100]})
+                self.assertEqual(code, 1)
             result = json.loads(next(output.glob('*-fake.json')).read_text())
+            self.assertEqual(saved_path.parent, output)
             self.assertEqual((result['passed'], result['skipped'], result['failed']), (6, 1, 0))
+            self.assertEqual(result['source'], {'commit': 'a' * 40, 'clean': True})
+            self.assertFalse(result['lock_released'])
             self.assertEqual(next(s for s in result['steps'] if s['step'] == 'x86_64: launch')
                              ['detail']['outcome'], 'runtime missing')
 
@@ -207,13 +225,27 @@ class Safety(unittest.TestCase):
                 mock.patch.object(frame_smoke, 'cleanup', side_effect=RuntimeError('cannot stop')), \
                 mock.patch.object(frame_smoke, 'title_cycle') as cycle, \
                 mock.patch('builtins.print'):
-            self.assertEqual(frame_smoke.run_smoke(mock.Mock(pair=False),
-                             {'activity':3,'proximity':[],'battery':[100]}), 1)
+            code, _ = frame_smoke.run_smoke(mock.Mock(pair=False),
+                             {'activity':3,'proximity':[],'battery':[100]})
+            self.assertEqual(code, 1)
             cycle.assert_not_called()
             self.assertEqual(len(list(Path(folder).glob('*-fake.json'))), 1)
 
 
 class SharedSmokeLock(unittest.TestCase):
+    def test_result_marks_release_only_after_owned_lock_is_removed(self):
+        with tempfile.TemporaryDirectory() as folder:
+            result_path = Path(folder) / 'smoke.json'
+            for released in (False, True):
+                result_path.write_text('{"lock_released": false}')
+                with mock.patch.object(frame_smoke.sys, 'argv', ['frame_smoke.py']), \
+                     mock.patch.object(frame_smoke, 'acquire_lock', return_value=True), \
+                     mock.patch.object(frame_smoke, 'preflight', return_value={}), \
+                     mock.patch.object(frame_smoke, 'run_smoke', return_value=(0, result_path)), \
+                     mock.patch.object(frame_smoke, 'release_lock', return_value=released):
+                    self.assertEqual(frame_smoke.main(), 0 if released else 1)
+                self.assertIs(json.loads(result_path.read_text())['lock_released'], released)
+
     def test_busy_lock_prevents_any_status_read_or_title_change(self):
         with mock.patch.object(frame_smoke.sys, 'argv', ['frame_smoke.py']), \
              mock.patch.object(frame_smoke, 'acquire_lock', return_value=False), \

tests/test_verify_frame_smoke_result.py

@@ -0,0 +1,62 @@
+"""The release smoke gate requires a complete result from its exact checkout."""
+
+import importlib.util
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+SPEC = importlib.util.spec_from_file_location(
+    'verify_frame_smoke_result', ROOT / 'scripts' / 'verify-frame-smoke-result.py')
+gate = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(gate)
+SHA = 'a' * 40
+
+
+def result():
+    steps = [{'step': name, 'ok': True} for name in sorted(gate.REQUIRED_STEPS)]
+    return {'source': {'commit': SHA, 'clean': True}, 'lock_released': True, 'steps': steps,
+            'passed': len(steps), 'skipped': 0, 'failed': 0,
+            'preflight': {'activity': 3, 'proximity': [2.3125], 'battery': [100]}}
+
+
+class VerifyFrameSmokeResult(unittest.TestCase):
+    def test_accepts_complete_clean_result_from_exact_commit(self):
+        gate.verify(result(), SHA)
+
+    def test_rejects_wrong_commit_dirty_checkout_or_missing_step(self):
+        for changed in ('commit', 'clean', 'step'):
+            with self.subTest(changed=changed):
+                data = result()
+                if changed == 'commit':
+                    data['source']['commit'] = 'b' * 40
+                elif changed == 'clean':
+                    data['source']['clean'] = False
+                else:
+                    data['steps'].pop()
+                    data['passed'] -= 1
+                with self.assertRaises(ValueError):
+                    gate.verify(data, SHA)
+
+    def test_rejects_skipped_launch_even_when_counts_claim_success(self):
+        data = result()
+        launch = next(step for step in data['steps'] if step['step'] == 'x86_64: launch')
+        launch.update(ok=False, skipped=True)
+        data.update(passed=len(data['steps']) - 1, skipped=1)
+        with self.assertRaisesRegex(ValueError, 'failed, skipped'):
+            gate.verify(data, SHA)
+
+    def test_rejects_missing_locked_preflight(self):
+        data = result()
+        data['preflight']['activity'] = 1
+        with self.assertRaisesRegex(ValueError, 'preflight'):
+            gate.verify(data, SHA)
+
+    def test_rejects_missing_lock_release_record(self):
+        data = result()
+        data['lock_released'] = False
+        with self.assertRaisesRegex(ValueError, 'lock release'):
+            gate.verify(data, SHA)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/test_wireless.py

@@ -7,6 +7,7 @@ import base64
 import copy
 import http.client
 import json
+import os
 import shutil
 import subprocess
 import sys
@@ -189,6 +190,8 @@ class Probe(unittest.TestCase):
         finally:
             wireless._lock.release()
 
+    @unittest.skipIf(os.environ.get('FRAME_SKIP_WINDOWS_WIRELESS_LIVE') == '1',
+                     'dedicated wireless-windows CI job owns the live inventory')
     @unittest.skipUnless(wireless.frame_host.WINDOWS, 'Live read-only Windows inventory runs on Windows CI')
     def test_live_windows_inventory(self):
         r = wireless.diagnose()