From c4a0b5b2da3d32bf145cefa2321d71374058246a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:00:46 -0500 Subject: [PATCH 1/4] fix(ci): count a lane cancelled after its steps finished as passed GitHub can hold a job open after Complete job until its timeout-minutes ends, then record it cancelled; ci-status then fails and the merge queue evicts a healthy PR. ci-status now re-reads such a lane's jobs and counts it as passed only when every step finished and the job ended at least 10 s after Complete job. Each cancelled lane writes a notice to count. Closes #12520 Closes #12506 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VQb9Q1Asjw331T5WUQHJyF --- .github/workflows/pr-require-checks.yml | 34 ++++++++++ scripts/ci-ordered-skips.test.sh | 84 +++++++++++++++++++++++++ 2 files changed, 118 insertions(+) diff --git a/.github/workflows/pr-require-checks.yml b/.github/workflows/pr-require-checks.yml index d55b4c4514b..c14672ebe60 100644 --- a/.github/workflows/pr-require-checks.yml +++ b/.github/workflows/pr-require-checks.yml @@ -6511,12 +6511,46 @@ jobs: # through as it is, so a skip that hid work stays `skipped` and the # aggregate fails it. Pinned by scripts/ci-ordered-skips.test.sh, which # runs this body. + # + # A CANCELLED LANE WHOSE STEPS ALL FINISHED COUNTS AS PASSED. GitHub can + # leave a job open after `Complete job` until its `timeout-minutes` ends, + # then record it `cancelled`: 31 jobs in 29 of 55 failed or cancelled + # merge-queue runs (2026-11-02). This job does not run on a run-level cancel, so a + # cancelled lane here reached its limit. The lane counts as passed only when + # every step succeeded or skipped, `Complete job` is last, and the job ended + # at least 10 s after it: a limit that lands between two steps skips the rest + # and ends the job within seconds, which looks the same but for that gap + # (3 to 5 s on 4 of those 31, which stay red). Any read failure leaves the + # lane cancelled. Each cancelled lane writes a `cancelled-after-success` + # notice, the count to compare against that baseline. - name: Check each skipped lane against scope id: lanes if: ${{ !cancelled() }} env: LANES: ${{ toJSON(needs) }} + GH_TOKEN: ${{ github.token }} run: | + if jq -e 'any(.[]; .result == "cancelled")' <<<"$LANES" >/dev/null 2>&1; then + jobs_file=$(mktemp) + gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" --jq '.jobs[]' >"$jobs_file" || : >"$jobs_file" + recovered=$(jq -r --slurpfile jobs "$jobs_file" ' + def finished: + .steps as $s + | ($s | length) > 0 and $s[-1].name == "Complete job" + and all($s[]; .conclusion == "success" or .conclusion == "skipped") + and ((.completed_at | fromdate) - ($s[-1].completed_at | fromdate)) >= 10; + to_entries[] | select(.value.result == "cancelled") | .key + | . as $lane + | select([$jobs[] | select(.name == $lane or (.name | startswith($lane + " (")))] + | length > 0 and all(.conclusion == "success" or .conclusion == "skipped" + or (.conclusion == "cancelled" and finished)))' <<<"$LANES") || recovered= + for lane in $(jq -r 'to_entries[] | select(.value.result == "cancelled") | .key' <<<"$LANES"); do + if grep -qxF "$lane" <<<"$recovered"; then counted=true; else counted=false; fi + echo "::notice title=cancelled-after-success::lane=$lane counted-as-passed=$counted" + done + LANES=$(jq --arg names "$recovered" '($names | split("\n")) as $n + | with_entries(if (.key as $k | $n | index($k)) then .value.result = "success" else . end)' <<<"$LANES") + fi results=$(jq -r ' .["select-tests"] as $s | to_entries diff --git a/scripts/ci-ordered-skips.test.sh b/scripts/ci-ordered-skips.test.sh index d5e8e3da54e..8d3710fe810 100755 --- a/scripts/ci-ordered-skips.test.sh +++ b/scripts/ci-ordered-skips.test.sh @@ -86,4 +86,88 @@ expect "a contract-only run passes every skip through" 0 "skipped skipped skippe "$(lanes skipped "" "" "" skipped skipped skipped "" skipped | sed 's/"lint-repo":{"result":"success"/"lint-repo":{"result":"skipped"/')" expect "unreadable lanes fail and write no results" nonzero "" "not json" +# A lane cancelled after its steps finished. The stub stands in for the jobs read: it prints +# GH_FIXTURE, or fails on GH_FIXTURE=fail, and records that it was called. +mkdir -p "$TMP_ROOT/bin" +cat >"$TMP_ROOT/bin/gh" <<'STUB' +#!/usr/bin/env bash +echo called >>"$GH_CALLS" +[[ "$GH_FIXTURE" == fail ]] && exit 1 +printf '%s\n' "$GH_FIXTURE" +STUB +chmod +x "$TMP_ROOT/bin/gh" +export GH_CALLS="$TMP_ROOT/gh-calls" GITHUB_REPOSITORY=o/r GITHUB_RUN_ID=1 GITHUB_RUN_ATTEMPT=1 + +# job +job() { + printf '{"name":"%s","conclusion":"%s","completed_at":"%s","steps":[{"name":"Run","conclusion":"%s","completed_at":"2026-11-02T20:00:00Z"},{"name":"%s","conclusion":"success","completed_at":"%s"}]}\n' "$1" "$2" "$3" "$6" "$4" "$5" +} + +# expect_cancelled