Reject prior failed tagged release runs before publishing

Bind Frame smoke evidence to release commit and lock cleanup

docs/releasing.md

@@ -74,9 +74,9 @@ count. So a build reaches people only when you publish it, after testing it.
 
    The script checks that the tag is on GitHub, that exact-commit main CI
    (including e2e), native amd64/ARM64 Decky runtime checks, and the tagged
-   release workflow are green with no failed,
-   cancelled, skipped or running jobs in any run attempt, and that all ten installers
-   are attached, each with the SHA-256 digest GitHub records. It also checks
+   release workflow are green with no failed, cancelled, skipped or running
+   jobs in any exact-tag push run or attempt, and that all ten installers are
+   attached, each with the SHA-256 digest GitHub records. It also checks
    that the draft does not begin with a `WITHDRAWN` notice, so the preserved
    v0.4.48 draft cannot accidentally be offered again. It checks
    that the uploaded `SHA256SUMS` digest matches a fresh list of those ten

scripts/publish-release.sh

@@ -33,7 +33,7 @@ gh api "repos/$repo/git/ref/tags/$tag" >/dev/null 2>&1 \
 
 # A complete asset list is insufficient: a cancelled installer build can leave
 # old uploaded bytes behind. Check the exact tagged commit's main/e2e jobs and
-# the latest exact-tag release attempt before any release mutation.
+# every exact-tag push run and attempt before any release mutation.
 python3 "$(dirname "$0")/verify-release-gates.py" "$tag" "${FRAME_SMOKE_RESULT:-}"
 
 # Every release, drafts included, one JSON object per line.

scripts/verify-release-gates.py

@@ -61,14 +61,29 @@ def tag_commit(tag):
     raise GateError(f"{tag} has too many nested tag objects")
 
 
-def exact_run(workflow, branch, sha, event):
+def exact_runs(workflow, branch, sha, event):
     query = f"actions/workflows/{workflow}/runs?branch={quote(branch, safe='')}&head_sha={sha}&per_page=100"
-    runs = api(query).get("workflow_runs") or []
+    runs = []
+    for page in range(1, 21):
+        data = api(f"{query}&page={page}")
+        batch = data.get("workflow_runs")
+        total = data.get("total_count")
+        if not isinstance(batch, list) or type(total) is not int or total < 0:
+            raise GateError(f"invalid {workflow} run list for {branch} at {sha}")
+        runs.extend(batch)
+        if len(runs) >= total or len(batch) < 100:
+            break
+    if len(runs) != total or len({run.get('id') for run in runs}) != len(runs):
+        raise GateError(f"incomplete or duplicate {workflow} run list for {branch} at {sha}")
     matching = [run for run in runs if run.get("head_sha") == sha
                 and run.get("head_branch") == branch and run.get("event") == event]
     if not matching:
         raise GateError(f"no exact {workflow} {event} run for {branch} at {sha}")
-    run = max(matching, key=lambda item: item["id"])
+    return sorted(matching, key=lambda item: item["id"])
+
+
+def exact_run(workflow, branch, sha, event):
+    run = exact_runs(workflow, branch, sha, event)[-1]
     if run.get("status") != "completed" or run.get("conclusion") != "success":
         raise GateError(f"{workflow} run {run['id']} is {run.get('status')}/{run.get('conclusion')}")
     return run
@@ -107,7 +122,12 @@ def main(tag, smoke_result):
     sha = tag_commit(tag)
     successful_jobs(exact_run("checks.yml", "main", sha, "push"), MAIN_JOBS)
     successful_jobs(exact_run("decky-runtime.yml", "main", sha, "push"), DECKY_JOBS)
-    successful_jobs(exact_run("release.yml", tag, sha, "push"), RELEASE_JOBS)
+    # A fresh run must not hide a failed, cancelled, or skipped installer in an
+    # earlier separate push run for the same tag and commit.
+    for run in exact_runs("release.yml", tag, sha, "push"):
+        if run.get("status") != "completed" or run.get("conclusion") != "success":
+            raise GateError(f"release.yml run {run['id']} is {run.get('status')}/{run.get('conclusion')}")
+        successful_jobs(run, RELEASE_JOBS)
     command = [sys.executable, str(Path(__file__).with_name("verify-frame-smoke-result.py")),
                smoke_result, sha]
     checked = subprocess.run(command, text=True, capture_output=True, check=False)

tests/fakegh/gh

@@ -96,15 +96,26 @@ elif method == "GET" and endpoint.startswith(repo + "actions/workflows/"):
            "status": gate.get("decky_status" if run_id == 503 else ("main_status" if is_main else "release_status"), "completed"),
            "conclusion": gate.get("decky_conclusion" if run_id == 503 else ("main_conclusion" if is_main else "release_conclusion"), "success"),
            "html_url": "https://github.com/saphid/frame-control/actions/runs/%s" % run_id}
-    print(json.dumps({"workflow_runs": [run]}))
+    runs = [run]
+    if run_id == 502 and gate.get("prior_release_run_bad_job"):
+        runs.append(dict(run, id=500, run_attempt=1, status="completed",
+                         conclusion=gate.get("prior_release_run_conclusion", "failure"),
+                         html_url="https://github.com/saphid/frame-control/actions/runs/500"))
+    if run_id == 502 and gate.get("release_run_list_incomplete"):
+        print(json.dumps({"total_count": len(runs) + 1, "workflow_runs": runs}))
+    else:
+        print(json.dumps({"total_count": len(runs), "workflow_runs": runs}))
 elif method == "GET" and endpoint.startswith(repo + "actions/runs/"):
     parts = endpoint[len(repo + "actions/runs/"):].split("/")
     run_id = parts[0]
     attempt = int(parts[2])
-    names = {"501": main_jobs, "502": release_jobs, "503": decky_jobs}[run_id]
+    names = {"500": release_jobs, "501": main_jobs, "502": release_jobs,
+             "503": decky_jobs}[run_id]
     names = [name for name in names if name != gate.get("omit_job")]
     jobs = [{"name": name, "status": "completed",
-             "conclusion": (gate.get("prior_bad_job_conclusion", "failure")
+             "conclusion": (gate.get("prior_release_run_bad_conclusion", "cancelled")
+                            if run_id == "500" and name == gate.get("prior_release_run_bad_job")
+                            else gate.get("prior_bad_job_conclusion", "failure")
                             if run_id == "502" and attempt == 1 and name == gate.get("prior_bad_job")
                             else gate.get("bad_job_conclusion", "success") if name == gate.get("bad_job")
                             else "success")}

tests/test_publish_release.py

@@ -199,6 +199,30 @@ class PublishRelease(unittest.TestCase):
         self.assertIsNone(upload)
         self.assertFalse(any("-X" in call for call in log))
 
+    def test_refuses_bad_installer_in_separate_prior_tag_run(self):
+        for run_conclusion, job_conclusion, expected in (
+                ("failure", "cancelled", "failure"),
+                ("success", "skipped", "skipped")):
+            with self.subTest(run_conclusion=run_conclusion, job_conclusion=job_conclusion):
+                gates = {"prior_release_run_bad_job":
+                         "build (macos-latest, dist, app/dist/*.dmg app/dist/*.zip)",
+                         "prior_release_run_conclusion": run_conclusion,
+                         "prior_release_run_bad_conclusion": job_conclusion}
+                p, log, upload = self.run_script([draft()], "v9.8.7", gates=gates)
+                self.assertNotEqual(p.returncode, 0)
+                self.assertIn("run 500", p.stderr)
+                self.assertIn(expected, p.stderr)
+                self.assertIsNone(upload)
+                self.assertFalse(any("-X" in call for call in log))
+
+    def test_refuses_incomplete_tag_run_list(self):
+        p, log, upload = self.run_script([draft()], "v9.8.7",
+                                         gates={"release_run_list_incomplete": True})
+        self.assertNotEqual(p.returncode, 0)
+        self.assertIn("incomplete", p.stderr)
+        self.assertIsNone(upload)
+        self.assertFalse(any("-X" in call for call in log))
+
     def test_refuses_missing_or_nonexact_gate_results(self):
         for gates in ({"omit_job": "e2e"}, {"main_sha": "c" * 40},
                       {"decky_conclusion": "failure"}, {"release_conclusion": "failure"},