coordinator: Rerun a master push with only no-runner jobs

.claude/skills/obs-org-coordinator/SKILL.md

@@ -52,6 +52,7 @@ coordinator holds, and orders them:
 | Kind | What it means | What you do |
 |---|---|---|
 | `fix-master` | a default branch's last push run failed (its PRs' `fix-ci` tasks are hidden until it is green) | find the PR that broke it; fix or revert first. Queued PRs fail until it is green. Needs an admin (token, secret, rule)? One `human-review-needed` issue with the log line |
+| `rerun-master` | a default branch's last push run failed only on jobs that got no runner (its PRs' `fix-ci` tasks stay on the list) | no PR broke it and the log is empty: `gh run rerun <run> -R <repo> --failed`. Do not revert or escalate |
 | `requeue` | a green PR now conflicts (ejected from the queue) | rebase, keep both sides of list conflicts, `--force-with-lease`, `gh pr merge --auto` |
 | `fix-ci` | a PR's checks fail | §3 |
 | `enqueue` | green and mergeable, not queued | `gh pr merge <N> -R <repo> --auto`, then confirm (gh pr view has no queue field): `gh api graphql -f query='{repository(owner:"obs-rust",name:"<repo>"){pullRequest(number:<N>){isInMergeQueue}}}'` |
@@ -105,6 +106,10 @@ already be caught by the local gate, so this is the exception path:
 1. Read the failed job log (`gh run view <run> --log-failed`).
 2. **Flake** (timeout, runner died, network, a timing test the diff doesn't
    touch that passed in the sibling job): `gh run rerun <run> --failed`.
+   **A job with no runner** is one too: `cancelled`, no steps, "not acquired by
+   Runner of type hosted", and `--log-failed` returns nothing. Rerun it. During
+   a GitHub incident (the work list prints them first), do not revert or
+   escalate.
 3. **Mechanical** (lint, format, a test count, a features-ledger line, a
    stale reference): push a `fixup:` commit and say so in a comment.
 4. **Master was red**: fix master first (`fix-master`); re-queuing only

WORKFLOW.md

@@ -119,7 +119,7 @@ Ejections, rebases and CI repairs after that are the coordinator's.
 every tick and clears the list top to bottom. Each kind comes with its
 instructions:
 
-`fix-master` → `requeue` → `fix-ci` → `enqueue` → `rebase` → `review` →
+`fix-master` → `rerun-master` → `requeue` → `fix-ci` → `enqueue` → `rebase` → `review` →
 `decide` → `triage` → `stalled` → `investigating` → `escalated`
 
 Several coordinators can run at once. Before acting on a task:

tests/test_coordinator.py

@@ -7,11 +7,14 @@
 
 from __future__ import annotations
 
+import io
+import json
 import tempfile
 import unittest
 from datetime import timedelta
 
 from workflow import coordinator
+from workflow.gh import github_incidents, only_no_runner
 from workflow.sim import World
 
 R = "obs-studio"
@@ -24,7 +27,10 @@ def setUp(self):
 
     def work(self):
         return coordinator.tasks(
-            self.w.gh.items("obs-rust"), self.w.gh.red_branches("obs-rust"), self.w.now
+            self.w.gh.items("obs-rust"),
+            self.w.gh.red_branches("obs-rust"),
+            self.w.now,
+            no_runner=self.w.gh.no_runner_runs("obs-rust"),
         )
 
     def kinds(self, number=None):
@@ -329,6 +335,115 @@ def test_closed_issue_leaves_the_list(self):
         self.assertEqual(self.kinds(n), [])
 
 
+def no_runner_job(name="macos"):
+    # As GitHub reports it: "The job was not acquired by Runner of type hosted".
+    return {"name": name, "conclusion": "cancelled", "runner_id": 0, "runner_name": "", "steps": []}
+
+
+def passed_job(name="linux"):
+    return {
+        "name": name,
+        "conclusion": "success",
+        "runner_id": 7,
+        "runner_name": "GitHub Actions 7",
+        "steps": [{"name": "build"}],
+    }
+
+
+class NoRunnerPush(Case):
+    """obs-rust/.github#29: four master pushes in a row were red on 2026-11-04
+    only because macOS jobs got no runner. No PR broke them, the failed-job log
+    was empty, and every other repo's fix-ci stayed hidden behind fix-master."""
+
+    def test_gives_a_rerun_task_and_no_fix_master(self):
+        self.w.no_runner_push(R, run=41874818392)
+        work = self.work()
+        self.assertEqual([t.kind for t in work], ["rerun-master"])
+        self.assertIn("41874818392", work[0].title)
+
+    def test_keeps_fix_ci_on_the_list(self):
+        n = self.w.open_pr(R, "alice")
+        self.w.ci(R, n, passed=False)
+        self.w.no_runner_push(R, run=1)
+        self.assertEqual(self.kinds(n), ["fix-ci"])
+        self.assertIn("rerun-master", self.kinds())
+
+    def test_a_green_push_clears_it(self):
+        self.w.no_runner_push(R, run=1)
+        self.w.master(R, red=False)
+        self.assertEqual(self.kinds(), [])
+
+    def test_report_says_how(self):
+        self.w.no_runner_push(R, run=1)
+        text = coordinator.report(self.work())
+        self.assertIn("## rerun-master (1)", text)
+        self.assertIn("gh run rerun", text)
+
+
+class NoRunnerClassification(unittest.TestCase):
+    def test_only_no_runner_jobs(self):
+        self.assertTrue(only_no_runner([passed_job(), no_runner_job(), no_runner_job("swift")]))
+
+    def test_one_job_that_ran_and_failed_stays_red(self):
+        failed = {**passed_job("cross"), "conclusion": "failure"}
+        self.assertFalse(only_no_runner([no_runner_job(), failed]))
+
+    def test_a_cancelled_job_that_had_a_runner_stays_red(self):
+        cancelled = {**passed_job("cross"), "conclusion": "cancelled"}
+        self.assertFalse(only_no_runner([cancelled]))
+
+    def test_a_cancelled_job_with_steps_stays_red(self):
+        self.assertFalse(only_no_runner([{**no_runner_job(), "steps": [{"name": "build"}]}]))
+
+    def test_no_failed_job_is_not_this_case(self):
+        self.assertFalse(only_no_runner([passed_job()]))
+        self.assertFalse(only_no_runner([]))
+
+
+class GitHubIncidents(Case):
+    def test_incident_prints_at_the_top(self):
+        self.green_pr()
+        text = coordinator.report(self.work(), incidents=["Actions degraded (investigating)"])
+        self.assertTrue(text.startswith("## GitHub incidents"), text)
+        self.assertIn("- Actions degraded (investigating)", text)
+
+    def test_incident_prints_on_a_quiet_list(self):
+        text = coordinator.report([], incidents=["Actions degraded (investigating)"])
+        self.assertTrue(text.startswith("## GitHub incidents"), text)
+        self.assertIn("quiet: nothing to do", text)
+
+    def test_failed_read_does_not_stop_the_list(self):
+        self.green_pr()
+        text = coordinator.report(self.work(), incidents=None)
+        self.assertIn("could not read", text)
+        self.assertIn("## enqueue (1)", text)
+
+    def test_no_incident_adds_nothing(self):
+        self.assertEqual(coordinator.report([], incidents=[]), "quiet: nothing to do")
+
+    def test_read_failure_gives_none(self):
+        def broken(url, timeout):
+            raise OSError("no route")
+
+        self.assertIsNone(github_incidents(broken))
+
+    def test_read_formats_each_incident(self):
+        body = io.BytesIO(
+            json.dumps(
+                {"incidents": [{"name": "Actions degraded", "status": "investigating"}]}
+            ).encode()
+        )
+        self.assertEqual(
+            github_incidents(lambda url, timeout: body), ["Actions degraded (investigating)"]
+        )
+
+    def test_a_malformed_reply_gives_none(self):
+        self.assertIsNone(github_incidents(lambda url, timeout: io.BytesIO(b"<html>")))
+
+    def test_sim_reads_no_status(self):
+        self.assertEqual(self.w.gh.incidents(), [])
+
+
 class OrderAndReport(Case):
     def test_red_master_comes_first(self):
         self.green_pr()
@@ -354,6 +469,7 @@ def at(repo, number):
         self.assertEqual(at(R, n), ["fix-ci"])
 
     def test_full_order(self):
+        self.w.no_runner_push("obs-rust/obs-deps", run=1)
         conflict = self.w.open_pr(R, "alice")
         self.w.label(R, conflict, "claim-conflict")
         self.w.master(R, red=True)

workflow/coordinator.py

@@ -17,6 +17,7 @@
 
 Order, most urgent first:
   fix-master     the default branch is red: fix it before queuing anything
+  rerun-master   the default branch's push failed only on jobs with no runner
   requeue        a PR left the merge queue with a conflict: rebase, re-queue
   fix-ci         a ready PR's checks fail: rerun a flake, fix a mechanical
                  failure, or comment the cause for the author
@@ -51,6 +52,7 @@
 
 ORDER = [
     "fix-master",
+    "rerun-master",
     "claim-conflict",
     "requeue",
     "fix-ci",
@@ -73,6 +75,10 @@
     "and fix or revert it first. Queued PRs fail until it is green. A failure only an "
     "admin can fix (token, secret, branch rule): open or update one `human-review-needed` "
     "issue in that repo with the log line, and report it.",
+    "rerun-master": "Every failed job of this push run got no runner (`cancelled`, no steps), "
+    "so no PR broke it and `--log-failed` is empty. Do not look for a culprit, revert or "
+    "escalate: `gh run rerun <run> -R obs-rust/<repo> --failed`. Rerun it once per tick; "
+    "during a GitHub incident, rerun it after the incident ends.",
     "requeue": "UNMERGEABLE in the queue: read its merge-group run; a semantic clash "
     "with master (a new lint, a renamed helper) needs a fixup: dequeue, push, `--auto`. "
     "The queue ejected it (or it conflicts): rebase on the default branch, "
@@ -180,11 +186,20 @@ def issue_task(item: Item, now: datetime) -> str | None:
 
 
 def tasks(
-    items: list[Item], red: list[str], now: datetime, held: dict[str, str] | None = None
+    items: list[Item],
+    red: list[str],
+    now: datetime,
+    held: dict[str, str] | None = None,
+    no_runner: dict[str, int] | None = None,
 ) -> list[Task]:
-    """The work list, leaving out tasks another coordinator holds."""
+    """The work list, leaving out tasks another coordinator holds. `no_runner`
+    maps a repo to its push run that failed only on jobs with no runner."""
     held = held or {}
     out = [Task("fix-master", repo, 0, "default branch is red", 0.0) for repo in red]
+    out += [
+        Task("rerun-master", repo, 0, f"push run {run} failed only on jobs with no runner", 0.0)
+        for repo, run in (no_runner or {}).items()
+    ]
     for item in items:
         if item.state != "OPEN":
             continue
@@ -197,12 +212,21 @@ def tasks(
     return sorted(out, key=lambda t: t.rank)
 
 
-def report(work: list[Task], per_kind: int = 15) -> str:
-    """The list for a human or an agent: each kind with its instructions,
-    oldest first, at most `per_kind` lines each."""
+def incident_lines(incidents: list[str] | None) -> list[str]:
+    """GitHub's unresolved incidents; None means the read failed."""
+    if incidents is None:
+        return ["GitHub status: could not read it, see https://www.githubstatus.com"]
+    if not incidents:
+        return []
+    return ["## GitHub incidents", *(f"- {i}" for i in incidents)]
+
+
+def report(work: list[Task], per_kind: int = 15, incidents: list[str] | None = ()) -> str:
+    """The list for a human or an agent: GitHub's incidents, then each kind
+    with its instructions, oldest first, at most `per_kind` lines each."""
+    lines = incident_lines(incidents)
     if not work:
-        return "quiet: nothing to do"
-    lines: list[str] = []
+        return "\n".join([*lines, "", "quiet: nothing to do"]).strip()
     for kind in ORDER:
         mine = [t for t in work if t.kind == kind]
         if not mine:
@@ -267,8 +291,14 @@ def main() -> int:
             repo = item.repo if "/" in item.repo else f"{args.org}/{item.repo}"
             if pr_claims.conflicts(gh, repo, gh.pull_request(repo, item.number)):
                 item.labels.add(pr_claims.LABEL)
-    work = tasks(items, gh.red_branches(args.org), now, held)
-    print(json.dumps([asdict(t) for t in work], indent=1) if args.json else report(work))
+    work = tasks(items, gh.red_branches(args.org), now, held, gh.no_runner_runs(args.org))
+    incidents = gh.incidents()
+    if args.json:
+        for line in incident_lines(incidents):
+            print(line, file=sys.stderr)
+        print(json.dumps([asdict(t) for t in work], indent=1))
+    else:
+        print(report(work, incidents=incidents))
     return 0
 
 

workflow/gh.py

@@ -5,6 +5,8 @@
 import json
 import subprocess
 import time
+import urllib.request
+from collections.abc import Callable
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timedelta, timezone
 from typing import Any
@@ -55,6 +57,32 @@ def graphql(query: str, **variables: Any) -> dict[str, Any]:
     return json.loads(run(args))["data"]
 
 
+def only_no_runner(jobs: list[dict]) -> bool:
+    """Every job that did not pass got no runner: `cancelled`, no runner and
+    no steps, as GitHub reports "The job was not acquired by Runner of type
+    hosted even after multiple attempts". Any other failed job is a real
+    failure (obs-rust/.github#29)."""
+    failed = [j for j in jobs if j.get("conclusion") not in ("success", "skipped", "neutral")]
+    return bool(failed) and all(
+        j.get("conclusion") == "cancelled" and not j.get("runner_id") and not j.get("steps")
+        for j in failed
+    )
+
+
+INCIDENTS_URL = "https://www.githubstatus.com/api/v2/incidents/unresolved.json"
+INCIDENTS_TIMEOUT = 5  # seconds; the work list must not wait on a status page
+
+
+def github_incidents(fetch: Callable[..., Any] = urllib.request.urlopen) -> list[str] | None:
+    """GitHub's unresolved incidents, or None when they cannot be read."""
+    try:
+        with fetch(INCIDENTS_URL, timeout=INCIDENTS_TIMEOUT) as reply:
+            incidents = json.load(reply)["incidents"]
+        return [f"{i['name']} ({i['status']})" for i in incidents]
+    except (OSError, ValueError, KeyError, TypeError):
+        return None
+
+
 LEASE_REPO = "obs-rust/.github"
 
 
@@ -409,12 +437,15 @@ def check_counts(self, repo: str, number: int) -> tuple[int, int, int]:
             failed += state in ("failure", "error")
         return passed, failed, pending
 
-    def red_branches(self, org: str) -> list[str]:
-        """Repos whose latest finished push run on the default branch, in the
-        last RED_DAYS, failed. Older failures are dormant copies of upstream
-        repos, not a master anyone is merging into."""
+    def _failed_pushes(self, org: str) -> dict[str, dict]:
+        """Each repo's latest finished push run on the default branch, if it
+        failed in the last RED_DAYS. Older failures are dormant copies of
+        upstream repos, not a master anyone is merging into."""
+        cached = getattr(self, "_failed_pushes_cache", None)
+        if cached is not None:
+            return cached
 
-        def red(repo: str) -> bool:
+        def failed(repo: str) -> dict | None:
             meta = rest(f"repos/{org}/{repo}", paginate=False)
             runs = rest(
                 f"repos/{org}/{repo}/actions/runs?branch={meta['default_branch']}"
@@ -423,13 +454,41 @@ def red(repo: str) -> bool:
             )
             latest = runs.get("workflow_runs") or []
             if not latest or latest[0]["conclusion"] != "failure":
-                return False
+                return None
             age = datetime.now(timezone.utc) - ts(latest[0]["created_at"])
-            return age < timedelta(days=RED_DAYS)
+            return latest[0] if age < timedelta(days=RED_DAYS) else None
+
+        def jobs(repo: str, run: dict) -> list[dict]:
+            page = rest(
+                f"repos/{org}/{repo}/actions/runs/{run['id']}/jobs?per_page=100", paginate=False
+            )
+            # More jobs than one page: not all seen, so it stays a red master.
+            return page["jobs"] if page["total_count"] <= len(page["jobs"]) else []
 
         repos = self._repos(org)
         with ThreadPoolExecutor(READ_WORKERS) as pool:
-            return [r for r, bad in zip(repos, pool.map(red, repos)) if bad]
+            runs = {r: run for r, run in zip(repos, pool.map(failed, repos)) if run}
+            for (repo, run), run_jobs in zip(
+                runs.items(), pool.map(lambda kv: jobs(*kv), runs.items())
+            ):
+                run["only_no_runner"] = only_no_runner(run_jobs)
+        self._failed_pushes_cache = runs
+        return runs
+
+    def red_branches(self, org: str) -> list[str]:
+        """Repos whose latest push run failed, except a run that only lacked a
+        runner: no PR broke that one (no_runner_runs)."""
+        return [r for r, run in self._failed_pushes(org).items() if not run["only_no_runner"]]
+
+    def no_runner_runs(self, org: str) -> dict[str, int]:
+        """Repo to run id, for a latest push run that failed only because jobs
+        got no runner."""
+        return {
+            r: run["id"] for r, run in self._failed_pushes(org).items() if run["only_no_runner"]
+        }
+
+    def incidents(self) -> list[str] | None:
+        return github_incidents()
 
     def board(self, org: str, number: int) -> Board:
         item_ids: dict[str, str] = {}