Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 58 additions & 0 deletions .github/workflows/pr-require-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -870,6 +870,8 @@ jobs:
done
echo "::error::could not mark ci-lanes pending on $HEAD_SHA; a contract-only run could carry an older verdict forward while this run is in flight."
exit 1
- name: Lane finished
run: 'true'

select-tests:
# The contract-only gate, verbatim on every job but `ci-status` (see the
Expand Down Expand Up @@ -1214,6 +1216,8 @@ jobs:
with:
tokens: github-token
outside: ${{ github.event_name == 'pull_request' && 'detect-changes:pulls/files' || '' }}
- name: Lane finished
run: 'true'

# Public repository: every lane runs on GitHub-hosted ubuntu-24.04 (free for
# public repositories); the local-runner selector is not permitted here.
Expand Down Expand Up @@ -1971,6 +1975,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

# THE SHELL LINTERS: ShellCheck, and the repo-local gates that read shell
# source (GNU-only constructs, piped early-exit greps, temp-path cleanup,
Expand Down Expand Up @@ -2245,6 +2251,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

# PLUGIN CONTRACTS ACROSS FILES: the manifest schemas and duplicate keys, the
# catalog presence and enablement, option docs, the changelog and vendor-bump
Expand Down Expand Up @@ -2778,6 +2786,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

# THE PLUGIN CHECKS THAT READ THE ROOT NPM INSTALL: `claude plugin validate`
# over the plugins and the catalog (with the contract, catalog, cheatsheet
Expand Down Expand Up @@ -2920,6 +2930,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

# SKILL AND EVAL CONTRACTS: check 25 and the changed-skills gate, the
# portability lint, leaf names, spoke paths, the listing budgets, the eval
Expand Down Expand Up @@ -3292,6 +3304,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

# THE TEST LANES RUN WHAT THE SELECTION SAYS. `select-tests` plans them once
# (scripts/plan-test-lanes.sh): the shell suites the change selects go to
Expand Down Expand Up @@ -3607,6 +3621,8 @@ jobs:
exit 1
fi
scripts/run-plugin-tests.sh --jobs 3 --suites-from "$RUNNER_TEMP/leg-suites.txt"
- name: Lane finished
run: 'true'

# THE PYTHON SUITES THE CHANGE SELECTS, one pytest process per module, on one
# to nine legs of about 180 suite-seconds (one module at a time). Every module
Expand Down Expand Up @@ -3755,6 +3771,8 @@ jobs:
[ "$suite_rc" = 0 ] || rc=1
done <"$RUNNER_TEMP/leg-suites.txt"
exit "$rc"
- name: Lane finished
run: 'true'

# THE NODE PACKAGES THE CHANGE REACHES: the five sub-projects with their own
# install, typecheck, lint and test chains (the miro MCP server, the
Expand Down Expand Up @@ -3937,6 +3955,8 @@ jobs:
with:
path: ~/.npm
key: ${{ steps.npm_cache.outputs.cache-primary-key }}
- name: Lane finished
run: 'true'

# A pull request that hand-edits a file the standards sync-manifest manages
# fails. A standards-sync pull request (melodic-standards-sync[bot], head
Expand Down Expand Up @@ -3969,6 +3989,8 @@ jobs:
fetch-depth: 0
- name: Guard managed files
uses: melodic-software/ci-workflows/.github/actions/check-managed-files@2476d5dde8979eab34374150b7e4b0a2b99f4e88 # 2476d5d 2026-10-29
- name: Lane finished
run: 'true'

# THE WHOLE-TREE CAN'T-FAIL SCAN, WHENEVER THERE IS NO DIFF BASE: the
# schedule, a dispatch, and a pull request or merge group that
Expand Down Expand Up @@ -4011,6 +4033,8 @@ jobs:
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
- name: Lane finished
run: 'true'

ci-status:
needs:
Expand Down Expand Up @@ -6511,12 +6535,46 @@ jobs:
# through as it is, so a skip that hid work stays `skipped` and the
# aggregate fails it. Pinned by scripts/ci-ordered-skips.test.sh, which
# runs this body.
#
# A CANCELLED LANE WHOSE STEPS ALL FINISHED COUNTS AS PASSED. GitHub can
# leave a job open after `Complete job` until its `timeout-minutes` ends,
# then record it `cancelled`: 31 jobs in 29 of 55 failed or cancelled
# merge-queue runs (2026-11-02). This job does not run on a run-level cancel, so a
# cancelled lane here reached its limit. A limit that lands between two steps
# skips the rest, so `success` and `skipped` alone do not show the work ran.
# Each lane's last step, `Lane finished`, runs only when no earlier step
# failed or was cancelled, so the lane counts as passed only when that step
# succeeded, no step did worse than skip, and `Complete job` is last. Any read
# failure leaves the lane cancelled. Each cancelled lane writes a
# `cancelled-after-success` notice, the count to compare against that baseline.
- name: Check each skipped lane against scope
id: lanes
if: ${{ !cancelled() }}
env:
LANES: ${{ toJSON(needs) }}
GH_TOKEN: ${{ github.token }}
run: |
if jq -e 'any(.[]; .result == "cancelled")' <<<"$LANES" >/dev/null 2>&1; then
jobs_file=$(mktemp)
gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT/jobs?per_page=100" --jq '.jobs[]' >"$jobs_file" || : >"$jobs_file"
recovered=$(jq -r --slurpfile jobs "$jobs_file" '
def finished:
.steps as $s
| ($s | length) > 0 and $s[-1].name == "Complete job"
and all($s[]; .conclusion == "success" or .conclusion == "skipped")
and any($s[]; .name == "Lane finished" and .conclusion == "success");
to_entries[] | select(.value.result == "cancelled") | .key
| . as $lane
| select([$jobs[] | select(.name == $lane or (.name | startswith($lane + " (")))]
| length > 0 and all(.conclusion == "success"
or (.conclusion == "cancelled" and finished)))' <<<"$LANES") || recovered=
for lane in $(jq -r 'to_entries[] | select(.value.result == "cancelled") | .key' <<<"$LANES"); do
if grep -qxF "$lane" <<<"$recovered"; then counted=true; else counted=false; fi
echo "::notice title=cancelled-after-success::lane=$lane counted-as-passed=$counted"
done
LANES=$(jq --arg names "$recovered" '($names | split("\n")) as $n
| with_entries(if (.key as $k | $n | index($k)) then .value.result = "success" else . end)' <<<"$LANES")
fi
results=$(jq -r '
.["select-tests"] as $s
| to_entries
Expand Down
82 changes: 82 additions & 0 deletions scripts/ci-ordered-skips.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -86,4 +86,86 @@ expect "a contract-only run passes every skip through" 0 "skipped skipped skippe
"$(lanes skipped "" "" "" skipped skipped skipped "" skipped | sed 's/"lint-repo":{"result":"success"/"lint-repo":{"result":"skipped"/')"
expect "unreadable lanes fail and write no results" nonzero "" "not json"

# A lane cancelled after its steps finished. The stub stands in for the jobs read: it prints
# GH_FIXTURE, or fails on GH_FIXTURE=fail, and records that it was called.
mkdir -p "$TMP_ROOT/bin"
cat >"$TMP_ROOT/bin/gh" <<'STUB'
#!/usr/bin/env bash
echo called >>"$GH_CALLS"
[[ "$GH_FIXTURE" == fail ]] && exit 1
printf '%s\n' "$GH_FIXTURE"
STUB
chmod +x "$TMP_ROOT/bin/gh"
export GH_CALLS="$TMP_ROOT/gh-calls" GITHUB_REPOSITORY=o/r GITHUB_RUN_ID=1 GITHUB_RUN_ATTEMPT=1

# job <name> <conclusion> <other-step> <lane-finished-step> [<last-step-name>]
job() {
printf '{"name":"%s","conclusion":"%s","steps":[{"name":"Run","conclusion":"%s"},{"name":"Lane finished","conclusion":"%s"},{"name":"%s","conclusion":"success"}]}\n' "$1" "$2" "$3" "$4" "${5-Complete job}"
}

# expect_cancelled <label> <want-results> <lanes-json> <fixture> [<want-notice>]
expect_cancelled() {
local out got
: >"$TMP_ROOT/out"
out="$(PATH="$TMP_ROOT/bin:$PATH" GH_FIXTURE="$4" LANES="$3" GITHUB_OUTPUT="$TMP_ROOT/out" bash -e "$TMP_ROOT/body.sh" 2>&1)" || {
fail "$1: step failed: $out"
return
}
got="$(sed -n 's/^results=//p' "$TMP_ROOT/out")"
if [[ "$got" != "$2" ]]; then
fail "$1: expected results '$2' got '$got'"
elif [[ -n "${5-}" && "$out" != *"$5"* ]]; then
fail "$1: no notice '$5' in '$out'"
else
ok "$1"
fi
}

cancelled_python="$(lanes success false true false skipped cancelled skipped false skipped)"
passed="success success success success success success"
kept="success success success success cancelled success"
expect_cancelled "every step finished: the lane counts as passed, with a notice" "$passed" "$cancelled_python" \
"$(job 'test-python (0)' cancelled success success)" "lane=test-python counted-as-passed=true"
expect_cancelled "a limit between two steps skips Lane finished: the lane stays cancelled" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled success skipped)" "lane=test-python counted-as-passed=false"
expect_cancelled "a later step skipped on its own condition still counts as passed" "$passed" "$cancelled_python" \
"$(job 'test-python (0)' cancelled skipped success)"
expect_cancelled "a job with no Lane finished step stays cancelled" "$kept" "$cancelled_python" \
'{"name":"test-python (0)","conclusion":"cancelled","steps":[{"name":"Run","conclusion":"success"},{"name":"Complete job","conclusion":"success"}]}'
expect_cancelled "a skipped job with no steps stays cancelled" "$kept" "$cancelled_python" \
'{"name":"test-python (0)","conclusion":"skipped","steps":[]}'
expect_cancelled "a step cancelled mid-run stays cancelled" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled cancelled skipped)"
expect_cancelled "a failed step stays cancelled" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled failure success)"
expect_cancelled "no Complete job stays cancelled" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled success success 'Post Run')"
expect_cancelled "a matrix sibling that failed keeps the lane cancelled" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled success success; job 'test-python (1)' failure failure skipped)"
expect_cancelled "every cancelled matrix sibling must have finished" "$kept" "$cancelled_python" \
"$(job 'test-python (0)' cancelled success success; job 'test-python (1)' cancelled success skipped)"
expect_cancelled "a job read that fails leaves the lane cancelled" "$kept" "$cancelled_python" fail \
"lane=test-python counted-as-passed=false"
expect_cancelled "a lane with no job in the read stays cancelled" "$kept" "$cancelled_python" \
"$(job 'test-node' cancelled success success)"
expect_cancelled "another lane's name prefix does not match" "$kept" "$cancelled_python" \
"$(job 'test-python-extra' cancelled success success)"
# select-tests finished but hung: its dependents never ran, so the run stays red on them.
expect_cancelled "a recovered select-tests does not excuse the lanes it blocked" \
"success skipped skipped skipped skipped skipped" \
"$(lanes cancelled "" "" "" skipped skipped skipped "" skipped | sed 's/"lint-repo":{"result":"success"/"lint-repo":{"result":"skipped"/')" \
"$(job select-tests cancelled success success)"

: >"$GH_CALLS"
expect_cancelled "no cancelled lane makes no API call" \
"success success failure failure success success" \
"$(lanes success true false false failure skipped skipped true failure)" "$(job x success success success)"
if [[ -s "$GH_CALLS" ]]; then fail "gh was called with no cancelled lane"; else ok "gh is not called with no cancelled lane"; fi

# Every lane ci-status needs ends in Lane finished, so the rule above can pass it.
while read -r lane; do
last="$(awk -v job=" $lane:" '$0 == job { on = 1; next } on && /^ [a-z]/ { exit } on && /^ - name: / { sub(/^ - name: /, ""); name = $0 } END { print name }' "$WORKFLOW")"
if [[ "$last" == "Lane finished" ]]; then ok "$lane ends in Lane finished"; else fail "$lane ends in '$last'"; fi
done < <(awk '/^ ci-status:$/ { on = 1; next } on && /^ needs:$/ { n = 1; next } n && /^ - / { print $2; next } n { exit }' "$WORKFLOW")

test_harness::report
Loading