From 07fb202ec2f29acefa4030f2e13bd6c4a34fc6da Mon Sep 17 00:00:00 2001 From: Nirvedh Meshram Date: Tue, 6 Oct 2026 14:12:15 -0500 Subject: [PATCH 1/3] Cancel a PR's GPU CI when the PR is merged or closed The concurrency group only cancels a PR's in-flight run when a newer run of the same PR starts. Merging or closing starts none, so the old run kept holding the GPU runners. Trigger the three GPU workflows on pull_request closed too: that run joins the PR's group, cancels the in-flight run, and skips its own jobs. Also key the group on the PR number rather than the head branch, so two forks' same-named branches (e.g. patch-1) no longer cancel each other's runs, which closing a PR would otherwise now do as well. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/iris-external-validation-test.yml | 9 ++++++++- .github/workflows/iris-performance-regression-test.yml | 8 +++++++- .github/workflows/iris-tests.yml | 9 ++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/iris-external-validation-test.yml b/.github/workflows/iris-external-validation-test.yml index aa12811e1..5b4b1198c 100644 --- a/.github/workflows/iris-external-validation-test.yml +++ b/.github/workflows/iris-external-validation-test.yml @@ -5,10 +5,15 @@ on: branches: [ main ] pull_request: branches: [ main ] + # closed: the run below joins this PR's concurrency group, which cancels the + # PR's in-flight run instead of letting it hold the GPU runners after merge. + # Its jobs are skipped. + types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + # PR number, not head branch: two forks' "patch-1" branches must not cancel each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: @@ -21,6 +26,7 @@ jobs: external-validation-test: name: External Validation Test runs-on: [linux-mi350-8gpu-iris] + if: github.event.action != 'closed' timeout-minutes: 180 steps: @@ -77,6 +83,7 @@ jobs: external-gluon-validation-test: name: External Gluon Validation Test runs-on: [linux-mi350-8gpu-iris] + if: github.event.action != 'closed' steps: - name: Checkout repository diff --git a/.github/workflows/iris-performance-regression-test.yml b/.github/workflows/iris-performance-regression-test.yml index 96e694493..5b3bae047 100644 --- a/.github/workflows/iris-performance-regression-test.yml +++ b/.github/workflows/iris-performance-regression-test.yml @@ -5,10 +5,15 @@ on: branches: [ main ] pull_request: branches: [ main ] + # closed: the run below joins this PR's concurrency group, which cancels the + # PR's in-flight run instead of letting it hold the GPU runners after merge. + # Its jobs are skipped. + types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + # PR number, not head branch: two forks' "patch-1" branches must not cancel each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: @@ -21,6 +26,7 @@ jobs: performance-test: name: ${{ matrix.example_name }} runs-on: [linux-mi350-8gpu-iris] + if: github.event.action != 'closed' timeout-minutes: 180 strategy: fail-fast: false diff --git a/.github/workflows/iris-tests.yml b/.github/workflows/iris-tests.yml index c635b986a..19d6a1eeb 100644 --- a/.github/workflows/iris-tests.yml +++ b/.github/workflows/iris-tests.yml @@ -5,10 +5,15 @@ on: branches: [ main ] pull_request: branches: [ main ] + # closed: the run below joins this PR's concurrency group, which cancels the + # PR's in-flight run instead of letting it hold the GPU runners after merge. + # Its jobs are skipped. + types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + # PR number, not head branch: two forks' "patch-1" branches must not cancel each other. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} env: @@ -21,6 +26,7 @@ jobs: test-git: name: Test ${{ matrix.test_dir }} (${{ matrix.num_ranks }} ranks, git install) runs-on: [linux-mi350-8gpu-iris] + if: github.event.action != 'closed' timeout-minutes: 180 strategy: fail-fast: false @@ -246,6 +252,7 @@ jobs: test-new-examples: name: New examples (${{ matrix.num_ranks }} ranks, ${{ matrix.install_method }}) runs-on: [linux-mi350-8gpu-iris] + if: github.event.action != 'closed' timeout-minutes: 180 permissions: contents: read From ef8f96d1112ffe2dfd64a07a9dbffed12e0891bc Mon Sep 17 00:00:00 2001 From: Nirvedh Meshram Date: Tue, 6 Oct 2026 14:21:04 -0500 Subject: [PATCH 2/3] Cancel closed PRs' runs from a pull_request_target workflow The previous commit's pull_request closed trigger had two gaps. On merge, GITHUB_REF is refs/heads/main, so cancel-in-progress evaluated to false and nothing was cancelled in the one case that matters. And a fork contributor's closed-event run waits for the same maintainer approval as their other runs. pull_request_target runs from main regardless of approval settings and GITHUB_REF. The new workflow cancels the PR's queued and in-progress runs by head commit, branch and repo through the API. It only reads the event payload and never checks out PR code, so its write token is not exposed to it. The GPU workflows keep only the PR-number concurrency key. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/cancel-closed-pr-ci.yml | 49 +++++++++++++++++++ .../iris-external-validation-test.yml | 6 --- .../iris-performance-regression-test.yml | 5 -- .github/workflows/iris-tests.yml | 6 --- 4 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/cancel-closed-pr-ci.yml diff --git a/.github/workflows/cancel-closed-pr-ci.yml b/.github/workflows/cancel-closed-pr-ci.yml new file mode 100644 index 000000000..76e1b4392 --- /dev/null +++ b/.github/workflows/cancel-closed-pr-ci.yml @@ -0,0 +1,49 @@ +name: Cancel CI for closed PRs + +# A PR's concurrency group only cancels its in-flight runs when a newer run of +# the same PR starts. Merging or closing starts none, so without this the runs +# keep holding the GPU runners. +# +# pull_request_target runs from main and is never held for fork-PR approval, so +# this also covers first-time contributors. It gets a write token even for fork +# PRs: it must never check out or run PR code. It only reads the event payload. +on: + pull_request_target: + types: [closed] + +permissions: + actions: write + +jobs: + cancel: + runs-on: ubuntu-latest + steps: + - name: Cancel the PR's queued and running workflow runs + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + run: | + # Older commits' runs were already cancelled by concurrency when the + # next commit was pushed, so the head commit's runs are all that's left. + # Branch and repo also have to match: two PRs can share a head commit. + runs=$(gh api "repos/$REPO/actions/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" \ + --jq '.workflow_runs[] + | select(.status != "completed" + and .head_branch == env.HEAD_REF + and .head_repository.full_name == env.HEAD_REPO) + | "\(.id)\t\(.name)"') + if [ -z "$runs" ]; then + echo "No in-flight runs for $HEAD_REPO:$HEAD_REF@$HEAD_SHA" + exit 0 + fi + while IFS=$'\t' read -r id name; do + # A run can finish between listing and cancelling; that's fine. + if gh api -X POST "repos/$REPO/actions/runs/$id/cancel" > /dev/null 2>&1; then + echo "Cancelled $name ($id)" + else + echo "Could not cancel $name ($id); it has probably just finished" + fi + done <<< "$runs" diff --git a/.github/workflows/iris-external-validation-test.yml b/.github/workflows/iris-external-validation-test.yml index 5b4b1198c..81a9b22ac 100644 --- a/.github/workflows/iris-external-validation-test.yml +++ b/.github/workflows/iris-external-validation-test.yml @@ -5,10 +5,6 @@ on: branches: [ main ] pull_request: branches: [ main ] - # closed: the run below joins this PR's concurrency group, which cancels the - # PR's in-flight run instead of letting it hold the GPU runners after merge. - # Its jobs are skipped. - types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: @@ -26,7 +22,6 @@ jobs: external-validation-test: name: External Validation Test runs-on: [linux-mi350-8gpu-iris] - if: github.event.action != 'closed' timeout-minutes: 180 steps: @@ -83,7 +78,6 @@ jobs: external-gluon-validation-test: name: External Gluon Validation Test runs-on: [linux-mi350-8gpu-iris] - if: github.event.action != 'closed' steps: - name: Checkout repository diff --git a/.github/workflows/iris-performance-regression-test.yml b/.github/workflows/iris-performance-regression-test.yml index 5b3bae047..d760065dd 100644 --- a/.github/workflows/iris-performance-regression-test.yml +++ b/.github/workflows/iris-performance-regression-test.yml @@ -5,10 +5,6 @@ on: branches: [ main ] pull_request: branches: [ main ] - # closed: the run below joins this PR's concurrency group, which cancels the - # PR's in-flight run instead of letting it hold the GPU runners after merge. - # Its jobs are skipped. - types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: @@ -26,7 +22,6 @@ jobs: performance-test: name: ${{ matrix.example_name }} runs-on: [linux-mi350-8gpu-iris] - if: github.event.action != 'closed' timeout-minutes: 180 strategy: fail-fast: false diff --git a/.github/workflows/iris-tests.yml b/.github/workflows/iris-tests.yml index 19d6a1eeb..7ea57ef58 100644 --- a/.github/workflows/iris-tests.yml +++ b/.github/workflows/iris-tests.yml @@ -5,10 +5,6 @@ on: branches: [ main ] pull_request: branches: [ main ] - # closed: the run below joins this PR's concurrency group, which cancels the - # PR's in-flight run instead of letting it hold the GPU runners after merge. - # Its jobs are skipped. - types: [opened, synchronize, reopened, closed] workflow_dispatch: concurrency: @@ -26,7 +22,6 @@ jobs: test-git: name: Test ${{ matrix.test_dir }} (${{ matrix.num_ranks }} ranks, git install) runs-on: [linux-mi350-8gpu-iris] - if: github.event.action != 'closed' timeout-minutes: 180 strategy: fail-fast: false @@ -252,7 +247,6 @@ jobs: test-new-examples: name: New examples (${{ matrix.num_ranks }} ranks, ${{ matrix.install_method }}) runs-on: [linux-mi350-8gpu-iris] - if: github.event.action != 'closed' timeout-minutes: 180 permissions: contents: read From e579909aee89ead768dcd2f3b4c26e5c6ddf1951 Mon Sep 17 00:00:00 2001 From: Nirvedh Meshram Date: Tue, 6 Oct 2026 14:30:53 -0500 Subject: [PATCH 3/3] Sweep closed PRs' runs on a schedule instead of pull_request_target pull_request_target is a HIGH dangerous-triggers finding in the PR security scan. A scheduled run is equally immune to fork-approval holds and to GITHUB_REF being main on merge, and needs no scanner exception. Every 10 minutes it lists in-flight pull_request runs, finds each run's PR by head branch (fork runs carry no PR number) and cancels it only if every PR whose head is that commit is closed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .github/workflows/cancel-closed-pr-ci.yml | 68 +++++++++++++++-------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/.github/workflows/cancel-closed-pr-ci.yml b/.github/workflows/cancel-closed-pr-ci.yml index 76e1b4392..e52227fcc 100644 --- a/.github/workflows/cancel-closed-pr-ci.yml +++ b/.github/workflows/cancel-closed-pr-ci.yml @@ -2,48 +2,68 @@ name: Cancel CI for closed PRs # A PR's concurrency group only cancels its in-flight runs when a newer run of # the same PR starts. Merging or closing starts none, so without this the runs -# keep holding the GPU runners. +# keep holding the GPU runners until they finish. # -# pull_request_target runs from main and is never held for fork-PR approval, so -# this also covers first-time contributors. It gets a write token even for fork -# PRs: it must never check out or run PR code. It only reads the event payload. +# Why a sweep rather than reacting to the close event: +# - pull_request: closed sets GITHUB_REF to refs/heads/main on merge, so the +# workflows' cancel-in-progress is false in exactly the case that matters, +# and a fork contributor's run of it waits for maintainer approval. +# - pull_request_target is never held for approval, but it is a HIGH +# dangerous-triggers finding in the security scan. +# A scheduled run comes from main, is never held for approval, and needs no +# exception. The cost is up to one interval before a closed PR's runs stop. on: - pull_request_target: - types: [closed] + schedule: + - cron: "7-59/10 * * * *" + workflow_dispatch: permissions: actions: write + pull-requests: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false jobs: cancel: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - - name: Cancel the PR's queued and running workflow runs + - name: Cancel in-flight runs of closed PRs env: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - HEAD_REF: ${{ github.event.pull_request.head.ref }} - HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} run: | - # Older commits' runs were already cancelled by concurrency when the - # next commit was pushed, so the head commit's runs are all that's left. - # Branch and repo also have to match: two PRs can share a head commit. - runs=$(gh api "repos/$REPO/actions/runs?head_sha=$HEAD_SHA&event=pull_request&per_page=100" \ - --jq '.workflow_runs[] - | select(.status != "completed" - and .head_branch == env.HEAD_REF - and .head_repository.full_name == env.HEAD_REPO) - | "\(.id)\t\(.name)"') - if [ -z "$runs" ]; then - echo "No in-flight runs for $HEAD_REPO:$HEAD_REF@$HEAD_SHA" + set -euo pipefail + inflight=$(for status in queued in_progress waiting pending; do + gh api --paginate "repos/$REPO/actions/runs?event=pull_request&status=$status&per_page=100" \ + --jq '.workflow_runs[] | [.id, .name, .head_repository.full_name, .head_branch, .head_sha] | @tsv' + done) + if [ -z "$inflight" ]; then + echo "No in-flight PR runs" exit 0 fi - while IFS=$'\t' read -r id name; do + + # Branch names come from PR authors: they are only ever data here, + # URL-encoded for the query and compared, never evaluated. + declare -A pr_state + while IFS=$'\t' read -r id name head_repo branch sha; do + key="$head_repo:$branch:$sha" + if [ -z "${pr_state[$key]+set}" ]; then + head=$(printf '%s:%s' "${head_repo%%/*}" "$branch" | jq -sRr @uri) + # Fork runs carry no PR number, so find the PR by its head branch. + # Only a PR whose head is this commit counts, and any open one wins. + pr_state[$key]=$(SHA="$sha" gh api "repos/$REPO/pulls?state=all&head=$head&per_page=100" \ + --jq '[.[] | select(.head.sha == env.SHA) | .state] | if index("open") then "open" elif length > 0 then "closed" else "none" end') + fi + if [ "${pr_state[$key]}" != "closed" ]; then + continue + fi # A run can finish between listing and cancelling; that's fine. if gh api -X POST "repos/$REPO/actions/runs/$id/cancel" > /dev/null 2>&1; then - echo "Cancelled $name ($id)" + echo "Cancelled $name ($id) from closed PR $head_repo:$branch" else echo "Could not cancel $name ($id); it has probably just finished" fi - done <<< "$runs" + done <<< "$inflight"