diff --git a/.claude/skills/obs-org-coordinator/SKILL.md b/.claude/skills/obs-org-coordinator/SKILL.md index 9367892..eba913c 100644 --- a/.claude/skills/obs-org-coordinator/SKILL.md +++ b/.claude/skills/obs-org-coordinator/SKILL.md @@ -52,6 +52,7 @@ coordinator holds, and orders them: | Kind | What it means | What you do | |---|---|---| | `fix-master` | a default branch's last push run failed (its PRs' `fix-ci` tasks are hidden until it is green) | find the PR that broke it; fix or revert first. Queued PRs fail until it is green. Needs an admin (token, secret, rule)? One `human-review-needed` issue with the log line | +| `rerun-master` | a default branch's last push run failed only on jobs that got no runner (its PRs' `fix-ci` tasks stay on the list) | no PR broke it and the log is empty: `gh run rerun -R --failed`. Do not revert or escalate | | `requeue` | a green PR now conflicts (ejected from the queue) | rebase, keep both sides of list conflicts, `--force-with-lease`, `gh pr merge --auto` | | `fix-ci` | a PR's checks fail | §3 | | `enqueue` | green and mergeable, not queued | `gh pr merge -R --auto`, then confirm (gh pr view has no queue field): `gh api graphql -f query='{repository(owner:"obs-rust",name:""){pullRequest(number:){isInMergeQueue}}}'` | @@ -105,6 +106,10 @@ already be caught by the local gate, so this is the exception path: 1. Read the failed job log (`gh run view --log-failed`). 2. **Flake** (timeout, runner died, network, a timing test the diff doesn't touch that passed in the sibling job): `gh run rerun --failed`. + **A job with no runner** is one too: `cancelled`, no steps, "not acquired by + Runner of type hosted", and `--log-failed` returns nothing. Rerun it. During + a GitHub incident (the work list prints them first), do not revert or + escalate. 3. **Mechanical** (lint, format, a test count, a features-ledger line, a stale reference): push a `fixup:` commit and say so in a comment. 4. **Master was red**: fix master first (`fix-master`); re-queuing only diff --git a/WORKFLOW.md b/WORKFLOW.md index ea3bc5a..6c3f626 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -119,7 +119,7 @@ Ejections, rebases and CI repairs after that are the coordinator's. every tick and clears the list top to bottom. Each kind comes with its instructions: -`fix-master` → `requeue` → `fix-ci` → `enqueue` → `rebase` → `review` → +`fix-master` → `rerun-master` → `requeue` → `fix-ci` → `enqueue` → `rebase` → `review` → `decide` → `triage` → `stalled` → `investigating` → `escalated` Several coordinators can run at once. Before acting on a task: diff --git a/tests/test_coordinator.py b/tests/test_coordinator.py index f0ff710..f94dd47 100644 --- a/tests/test_coordinator.py +++ b/tests/test_coordinator.py @@ -7,11 +7,14 @@ from __future__ import annotations +import io +import json import tempfile import unittest from datetime import timedelta from workflow import coordinator +from workflow.gh import github_incidents, only_no_runner from workflow.sim import World R = "obs-studio" @@ -24,7 +27,10 @@ def setUp(self): def work(self): return coordinator.tasks( - self.w.gh.items("obs-rust"), self.w.gh.red_branches("obs-rust"), self.w.now + self.w.gh.items("obs-rust"), + self.w.gh.red_branches("obs-rust"), + self.w.now, + no_runner=self.w.gh.no_runner_runs("obs-rust"), ) def kinds(self, number=None): @@ -329,6 +335,115 @@ def test_closed_issue_leaves_the_list(self): self.assertEqual(self.kinds(n), []) +def no_runner_job(name="macos"): + # As GitHub reports it: "The job was not acquired by Runner of type hosted". + return {"name": name, "conclusion": "cancelled", "runner_id": 0, "runner_name": "", "steps": []} + + +def passed_job(name="linux"): + return { + "name": name, + "conclusion": "success", + "runner_id": 7, + "runner_name": "GitHub Actions 7", + "steps": [{"name": "build"}], + } + + +class NoRunnerPush(Case): + """obs-rust/.github#29: four master pushes in a row were red on 2026-11-04 + only because macOS jobs got no runner. No PR broke them, the failed-job log + was empty, and every other repo's fix-ci stayed hidden behind fix-master.""" + + def test_gives_a_rerun_task_and_no_fix_master(self): + self.w.no_runner_push(R, run=41874818392) + work = self.work() + self.assertEqual([t.kind for t in work], ["rerun-master"]) + self.assertIn("41874818392", work[0].title) + + def test_keeps_fix_ci_on_the_list(self): + n = self.w.open_pr(R, "alice") + self.w.ci(R, n, passed=False) + self.w.no_runner_push(R, run=1) + self.assertEqual(self.kinds(n), ["fix-ci"]) + self.assertIn("rerun-master", self.kinds()) + + def test_a_green_push_clears_it(self): + self.w.no_runner_push(R, run=1) + self.w.master(R, red=False) + self.assertEqual(self.kinds(), []) + + def test_report_says_how(self): + self.w.no_runner_push(R, run=1) + text = coordinator.report(self.work()) + self.assertIn("## rerun-master (1)", text) + self.assertIn("gh run rerun", text) + + +class NoRunnerClassification(unittest.TestCase): + def test_only_no_runner_jobs(self): + self.assertTrue(only_no_runner([passed_job(), no_runner_job(), no_runner_job("swift")])) + + def test_one_job_that_ran_and_failed_stays_red(self): + failed = {**passed_job("cross"), "conclusion": "failure"} + self.assertFalse(only_no_runner([no_runner_job(), failed])) + + def test_a_cancelled_job_that_had_a_runner_stays_red(self): + cancelled = {**passed_job("cross"), "conclusion": "cancelled"} + self.assertFalse(only_no_runner([cancelled])) + + def test_a_cancelled_job_with_steps_stays_red(self): + self.assertFalse(only_no_runner([{**no_runner_job(), "steps": [{"name": "build"}]}])) + + def test_no_failed_job_is_not_this_case(self): + self.assertFalse(only_no_runner([passed_job()])) + self.assertFalse(only_no_runner([])) + + +class GitHubIncidents(Case): + def test_incident_prints_at_the_top(self): + self.green_pr() + text = coordinator.report(self.work(), incidents=["Actions degraded (investigating)"]) + self.assertTrue(text.startswith("## GitHub incidents"), text) + self.assertIn("- Actions degraded (investigating)", text) + + def test_incident_prints_on_a_quiet_list(self): + text = coordinator.report([], incidents=["Actions degraded (investigating)"]) + self.assertTrue(text.startswith("## GitHub incidents"), text) + self.assertIn("quiet: nothing to do", text) + + def test_failed_read_does_not_stop_the_list(self): + self.green_pr() + text = coordinator.report(self.work(), incidents=None) + self.assertIn("could not read", text) + self.assertIn("## enqueue (1)", text) + + def test_no_incident_adds_nothing(self): + self.assertEqual(coordinator.report([], incidents=[]), "quiet: nothing to do") + + def test_read_failure_gives_none(self): + def broken(url, timeout): + raise OSError("no route") + + self.assertIsNone(github_incidents(broken)) + + def test_read_formats_each_incident(self): + body = io.BytesIO( + json.dumps( + {"incidents": [{"name": "Actions degraded", "status": "investigating"}]} + ).encode() + ) + self.assertEqual( + github_incidents(lambda url, timeout: body), ["Actions degraded (investigating)"] + ) + + def test_a_malformed_reply_gives_none(self): + self.assertIsNone(github_incidents(lambda url, timeout: io.BytesIO(b""))) + + def test_sim_reads_no_status(self): + self.assertEqual(self.w.gh.incidents(), []) + + class OrderAndReport(Case): def test_red_master_comes_first(self): self.green_pr() @@ -354,6 +469,7 @@ def at(repo, number): self.assertEqual(at(R, n), ["fix-ci"]) def test_full_order(self): + self.w.no_runner_push("obs-rust/obs-deps", run=1) conflict = self.w.open_pr(R, "alice") self.w.label(R, conflict, "claim-conflict") self.w.master(R, red=True) diff --git a/workflow/coordinator.py b/workflow/coordinator.py index f1d6bdc..b1a3dd3 100644 --- a/workflow/coordinator.py +++ b/workflow/coordinator.py @@ -17,6 +17,7 @@ Order, most urgent first: fix-master the default branch is red: fix it before queuing anything + rerun-master the default branch's push failed only on jobs with no runner requeue a PR left the merge queue with a conflict: rebase, re-queue fix-ci a ready PR's checks fail: rerun a flake, fix a mechanical failure, or comment the cause for the author @@ -51,6 +52,7 @@ ORDER = [ "fix-master", + "rerun-master", "claim-conflict", "requeue", "fix-ci", @@ -73,6 +75,10 @@ "and fix or revert it first. Queued PRs fail until it is green. A failure only an " "admin can fix (token, secret, branch rule): open or update one `human-review-needed` " "issue in that repo with the log line, and report it.", + "rerun-master": "Every failed job of this push run got no runner (`cancelled`, no steps), " + "so no PR broke it and `--log-failed` is empty. Do not look for a culprit, revert or " + "escalate: `gh run rerun -R obs-rust/ --failed`. Rerun it once per tick; " + "during a GitHub incident, rerun it after the incident ends.", "requeue": "UNMERGEABLE in the queue: read its merge-group run; a semantic clash " "with master (a new lint, a renamed helper) needs a fixup: dequeue, push, `--auto`. " "The queue ejected it (or it conflicts): rebase on the default branch, " @@ -180,11 +186,20 @@ def issue_task(item: Item, now: datetime) -> str | None: def tasks( - items: list[Item], red: list[str], now: datetime, held: dict[str, str] | None = None + items: list[Item], + red: list[str], + now: datetime, + held: dict[str, str] | None = None, + no_runner: dict[str, int] | None = None, ) -> list[Task]: - """The work list, leaving out tasks another coordinator holds.""" + """The work list, leaving out tasks another coordinator holds. `no_runner` + maps a repo to its push run that failed only on jobs with no runner.""" held = held or {} out = [Task("fix-master", repo, 0, "default branch is red", 0.0) for repo in red] + out += [ + Task("rerun-master", repo, 0, f"push run {run} failed only on jobs with no runner", 0.0) + for repo, run in (no_runner or {}).items() + ] for item in items: if item.state != "OPEN": continue @@ -197,12 +212,21 @@ def tasks( return sorted(out, key=lambda t: t.rank) -def report(work: list[Task], per_kind: int = 15) -> str: - """The list for a human or an agent: each kind with its instructions, - oldest first, at most `per_kind` lines each.""" +def incident_lines(incidents: list[str] | None) -> list[str]: + """GitHub's unresolved incidents; None means the read failed.""" + if incidents is None: + return ["GitHub status: could not read it, see https://www.githubstatus.com"] + if not incidents: + return [] + return ["## GitHub incidents", *(f"- {i}" for i in incidents)] + + +def report(work: list[Task], per_kind: int = 15, incidents: list[str] | None = ()) -> str: + """The list for a human or an agent: GitHub's incidents, then each kind + with its instructions, oldest first, at most `per_kind` lines each.""" + lines = incident_lines(incidents) if not work: - return "quiet: nothing to do" - lines: list[str] = [] + return "\n".join([*lines, "", "quiet: nothing to do"]).strip() for kind in ORDER: mine = [t for t in work if t.kind == kind] if not mine: @@ -267,8 +291,14 @@ def main() -> int: repo = item.repo if "/" in item.repo else f"{args.org}/{item.repo}" if pr_claims.conflicts(gh, repo, gh.pull_request(repo, item.number)): item.labels.add(pr_claims.LABEL) - work = tasks(items, gh.red_branches(args.org), now, held) - print(json.dumps([asdict(t) for t in work], indent=1) if args.json else report(work)) + work = tasks(items, gh.red_branches(args.org), now, held, gh.no_runner_runs(args.org)) + incidents = gh.incidents() + if args.json: + for line in incident_lines(incidents): + print(line, file=sys.stderr) + print(json.dumps([asdict(t) for t in work], indent=1)) + else: + print(report(work, incidents=incidents)) return 0 diff --git a/workflow/gh.py b/workflow/gh.py index 5f648a3..3ae334e 100644 --- a/workflow/gh.py +++ b/workflow/gh.py @@ -5,6 +5,8 @@ import json import subprocess import time +import urllib.request +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime, timedelta, timezone from typing import Any @@ -55,6 +57,32 @@ def graphql(query: str, **variables: Any) -> dict[str, Any]: return json.loads(run(args))["data"] +def only_no_runner(jobs: list[dict]) -> bool: + """Every job that did not pass got no runner: `cancelled`, no runner and + no steps, as GitHub reports "The job was not acquired by Runner of type + hosted even after multiple attempts". Any other failed job is a real + failure (obs-rust/.github#29).""" + failed = [j for j in jobs if j.get("conclusion") not in ("success", "skipped", "neutral")] + return bool(failed) and all( + j.get("conclusion") == "cancelled" and not j.get("runner_id") and not j.get("steps") + for j in failed + ) + + +INCIDENTS_URL = "https://www.githubstatus.com/api/v2/incidents/unresolved.json" +INCIDENTS_TIMEOUT = 5 # seconds; the work list must not wait on a status page + + +def github_incidents(fetch: Callable[..., Any] = urllib.request.urlopen) -> list[str] | None: + """GitHub's unresolved incidents, or None when they cannot be read.""" + try: + with fetch(INCIDENTS_URL, timeout=INCIDENTS_TIMEOUT) as reply: + incidents = json.load(reply)["incidents"] + return [f"{i['name']} ({i['status']})" for i in incidents] + except (OSError, ValueError, KeyError, TypeError): + return None + + LEASE_REPO = "obs-rust/.github" @@ -409,12 +437,15 @@ def check_counts(self, repo: str, number: int) -> tuple[int, int, int]: failed += state in ("failure", "error") return passed, failed, pending - def red_branches(self, org: str) -> list[str]: - """Repos whose latest finished push run on the default branch, in the - last RED_DAYS, failed. Older failures are dormant copies of upstream - repos, not a master anyone is merging into.""" + def _failed_pushes(self, org: str) -> dict[str, dict]: + """Each repo's latest finished push run on the default branch, if it + failed in the last RED_DAYS. Older failures are dormant copies of + upstream repos, not a master anyone is merging into.""" + cached = getattr(self, "_failed_pushes_cache", None) + if cached is not None: + return cached - def red(repo: str) -> bool: + def failed(repo: str) -> dict | None: meta = rest(f"repos/{org}/{repo}", paginate=False) runs = rest( f"repos/{org}/{repo}/actions/runs?branch={meta['default_branch']}" @@ -423,13 +454,41 @@ def red(repo: str) -> bool: ) latest = runs.get("workflow_runs") or [] if not latest or latest[0]["conclusion"] != "failure": - return False + return None age = datetime.now(timezone.utc) - ts(latest[0]["created_at"]) - return age < timedelta(days=RED_DAYS) + return latest[0] if age < timedelta(days=RED_DAYS) else None + + def jobs(repo: str, run: dict) -> list[dict]: + page = rest( + f"repos/{org}/{repo}/actions/runs/{run['id']}/jobs?per_page=100", paginate=False + ) + # More jobs than one page: not all seen, so it stays a red master. + return page["jobs"] if page["total_count"] <= len(page["jobs"]) else [] repos = self._repos(org) with ThreadPoolExecutor(READ_WORKERS) as pool: - return [r for r, bad in zip(repos, pool.map(red, repos)) if bad] + runs = {r: run for r, run in zip(repos, pool.map(failed, repos)) if run} + for (repo, run), run_jobs in zip( + runs.items(), pool.map(lambda kv: jobs(*kv), runs.items()) + ): + run["only_no_runner"] = only_no_runner(run_jobs) + self._failed_pushes_cache = runs + return runs + + def red_branches(self, org: str) -> list[str]: + """Repos whose latest push run failed, except a run that only lacked a + runner: no PR broke that one (no_runner_runs).""" + return [r for r, run in self._failed_pushes(org).items() if not run["only_no_runner"]] + + def no_runner_runs(self, org: str) -> dict[str, int]: + """Repo to run id, for a latest push run that failed only because jobs + got no runner.""" + return { + r: run["id"] for r, run in self._failed_pushes(org).items() if run["only_no_runner"] + } + + def incidents(self) -> list[str] | None: + return github_incidents() def board(self, org: str, number: int) -> Board: item_ids: dict[str, str] = {} diff --git a/workflow/sim.py b/workflow/sim.py index d23b25f..95cc6a3 100644 --- a/workflow/sim.py +++ b/workflow/sim.py @@ -51,6 +51,7 @@ UNIQUE(content, field)); CREATE TABLE IF NOT EXISTS log (at TEXT, actor TEXT, what TEXT); CREATE TABLE IF NOT EXISTS red (repo TEXT PRIMARY KEY); +CREATE TABLE IF NOT EXISTS no_runner (repo TEXT PRIMARY KEY, run INTEGER); CREATE TABLE IF NOT EXISTS noqueue (repo TEXT PRIMARY KEY); CREATE TABLE IF NOT EXISTS qstate (item TEXT PRIMARY KEY, state TEXT); CREATE TABLE IF NOT EXISTS leases (key TEXT PRIMARY KEY, owner TEXT, at TEXT); @@ -323,6 +324,12 @@ def check_counts(self, repo: str, number: int) -> tuple[int, int, int]: def red_branches(self, org: str) -> list[str]: return [r for (r,) in self.q("SELECT repo FROM red")] + def no_runner_runs(self, org: str) -> dict[str, int]: + return {repo: run for repo, run in self.q("SELECT repo, run FROM no_runner")} + + def incidents(self) -> list[str] | None: + return [] # a simulated GitHub has no status page to read + def board(self, org: str, number: int) -> Board: fields = {} for name, kind, options in self.q("SELECT name, type, options FROM fields"): @@ -494,9 +501,14 @@ def no_merge_queue(self, repo: str) -> None: def master(self, repo: str, red: bool) -> None: self.gh.x("DELETE FROM red WHERE repo=?", repo) + self.gh.x("DELETE FROM no_runner WHERE repo=?", repo) if red: self.gh.x("INSERT INTO red VALUES (?)", repo) + def no_runner_push(self, repo: str, run: int) -> None: + self.master(repo, red=False) + self.gh.x("INSERT INTO no_runner VALUES (?, ?)", repo, run) + def ready_for_review(self, repo: str, pr: int) -> None: self.gh.x("UPDATE items SET draft=0 WHERE id=?", self.gh.node(repo, pr))