diff --git a/.github/scripts/acquire_gpus.sh b/.github/scripts/acquire_gpus.sh deleted file mode 100755 index 8aaded388..000000000 --- a/.github/scripts/acquire_gpus.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: MIT -# Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. -# -# Acquire GPUs for CI workflows - to be called as a workflow step -# Usage: acquire_gpus.sh -# -# Exports GPU_DEVICES environment variable to $GITHUB_ENV for use in subsequent steps - -set -e - -NUM_GPUS=$1 - -if [ -z "$NUM_GPUS" ]; then - echo "[ERROR] Missing required argument" - echo "Usage: $0 " - exit 1 -fi - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -echo "[ACQUIRE-GPUS] Acquiring $NUM_GPUS GPU(s)" -source "$SCRIPT_DIR/gpu_allocator.sh" -acquire_gpus "$NUM_GPUS" - -echo "[ACQUIRE-GPUS] Allocated GPUs: $GPU_DEVICES" -echo "[ACQUIRE-GPUS] GPU allocation details:" -echo " GPU_DEVICES=$GPU_DEVICES" -echo " ALLOCATED_GPU_BITMAP=$ALLOCATED_GPU_BITMAP" - -# Export to GITHUB_ENV so subsequent steps can use these variables -if [ -n "$GITHUB_ENV" ]; then - { - echo "GPU_DEVICES=$GPU_DEVICES" - echo "ALLOCATED_GPU_BITMAP=$ALLOCATED_GPU_BITMAP" - } >> "$GITHUB_ENV" - echo "[ACQUIRE-GPUS] Exported variables to GITHUB_ENV" -else - echo "[ACQUIRE-GPUS] WARNING: GITHUB_ENV not set, variables not exported" -fi diff --git a/.github/scripts/gpu_task_queue.py b/.github/scripts/gpu_task_queue.py index 6204b0e29..3e38815e4 100644 --- a/.github/scripts/gpu_task_queue.py +++ b/.github/scripts/gpu_task_queue.py @@ -13,9 +13,14 @@ Usage: gpu_task_queue.py --install git unittests:8 unittests:4 unittests:2 unittests:1 gpu_task_queue.py --install editable new_examples:8 new_examples:2 + gpu_task_queue.py --task 8 "GEMM All-Scatter" "bash .github/scripts/run_perf_benchmark.sh ..." -Each task is :. "new_examples" runs run_new_examples.sh; -anything else runs run_tests.sh on tests/. +A positional task is :. "new_examples" runs +run_new_examples.sh; anything else runs run_tests.sh on tests/. + +--task RANKS NAME COMMAND runs COMMAND with bash, for jobs that are not a test +directory. Every task runs in a private copy of the checkout with GPU_DEVICES +set to its GPUs, and must pass them on (container_exec.sh --gpus). Scheduling: largest task first, smaller tasks backfill whatever GPUs are free. If the largest pending task has waited RESERVE_AFTER seconds, it posts a @@ -58,14 +63,14 @@ def log(msg): class Task: - def __init__(self, spec, install): - test_dir, _, ranks = spec.rpartition(":") - if not test_dir or not ranks.isdigit() or not 1 <= int(ranks) <= MAX_GPUS: - raise SystemExit("bad task '{}': expected :<1-{}>".format(spec, MAX_GPUS)) + def __init__(self, ranks, name, test_dir=None, install=None, shell=None): + if not ranks.isdigit() or not 1 <= int(ranks) <= MAX_GPUS: + raise SystemExit("bad task '{}': ranks must be 1-{}, got '{}'".format(name, MAX_GPUS, ranks)) self.test_dir = test_dir self.ranks = int(ranks) self.install = install - self.name = "{} ({} ranks, {})".format(test_dir, self.ranks, install) + self.shell = shell + self.name = name self.proc = None self.mask = 0 self.gpus = "" @@ -77,7 +82,17 @@ def __init__(self, spec, install): self.ended_at = None self.rc = None + @classmethod + def from_spec(cls, spec, install): + """A positional : task.""" + test_dir, _, ranks = spec.rpartition(":") + if not test_dir or not ranks.isdigit() or not 1 <= int(ranks) <= MAX_GPUS: + raise SystemExit("bad task '{}': expected :<1-{}>".format(spec, MAX_GPUS)) + return cls(ranks, "{} ({} ranks, {})".format(test_dir, ranks, install), test_dir=test_dir, install=install) + def command(self): + if self.shell is not None: + return ["bash", "-c", self.shell] scripts = os.path.join(self.workdir, ".github", "scripts") if self.test_dir == "new_examples": return ["bash", os.path.join(scripts, "run_new_examples.sh"), str(self.ranks), self.install] @@ -135,7 +150,6 @@ def __init__(self, tasks): self.pending = sorted(tasks, key=lambda t: -t.ranks) self.running = [] self.done = [] - self.head_waiting_since = None self.reserved_for = None def held_mask(self): @@ -183,14 +197,11 @@ def try_start(self): alloc.write_bitmap(bitmap) self.record_held() - # Anti-starvation: track how long the largest pending task has waited. - # The head only changes when it launches, since nothing is added later. - if head is None or head in to_launch: - self.head_waiting_since = None - else: - if self.head_waiting_since is None: - self.head_waiting_since = time.time() - waited = time.time() - self.head_waiting_since + # Anti-starvation: the largest pending task reserves the node once it + # has been queued for RESERVE_AFTER. Counted from when the job queued + # it, so a task that already waited behind a sibling does not restart. + if head is not None and head not in to_launch: + waited = time.time() - head.queued_at if self.reserved_for is head: alloc.reserve(head.ranks) # refresh elif self.reserved_for is None and waited >= RESERVE_AFTER and (not foreign or head.ranks > res[1]): @@ -330,11 +341,25 @@ def summary(self): def main(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument("--install", required=True, choices=["git", "editable", "install"]) - parser.add_argument("tasks", nargs="+", metavar="TEST_DIR:RANKS") + parser.add_argument("--install", choices=["git", "editable", "install"], help="required for TEST_DIR:RANKS tasks") + parser.add_argument( + "--task", + nargs=3, + action="append", + default=[], + metavar=("RANKS", "NAME", "COMMAND"), + help="run COMMAND with bash on RANKS GPUs; repeatable", + ) + parser.add_argument("tasks", nargs="*", metavar="TEST_DIR:RANKS") args = parser.parse_args() - - queue = Queue([Task(spec, args.install) for spec in args.tasks]) + if not args.tasks and not args.task: + parser.error("no tasks given") + if args.tasks and not args.install: + parser.error("--install is required for TEST_DIR:RANKS tasks") + + tasks = [Task(ranks, name, shell=command) for ranks, name, command in args.task] + tasks += [Task.from_spec(spec, args.install) for spec in args.tasks] + queue = Queue(tasks) def on_signal(signum, _frame): raise SystemExit(128 + signum) diff --git a/.github/scripts/release_gpus.sh b/.github/scripts/release_gpus.sh index 5eda98feb..428b772ad 100755 --- a/.github/scripts/release_gpus.sh +++ b/.github/scripts/release_gpus.sh @@ -5,7 +5,7 @@ # Release GPUs for CI workflows - to be called as a workflow step with if: always() # Usage: release_gpus.sh # -# Reads GPU allocation details from environment variables set by acquire_gpus.sh +# Returns whatever GPUs gpu_task_queue.py still holds if it was killed mid-run. set -e diff --git a/.github/scripts/run_external_validation.sh b/.github/scripts/run_external_validation.sh new file mode 100755 index 000000000..16645001a --- /dev/null +++ b/.github/scripts/run_external_validation.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Install Iris from this commit and run an external validation test on 2 ranks. +# Usage: run_external_validation.sh +# +# The tests live in gists, outside the repo, so they exercise Iris the way an +# external user would: a pip install from git rather than the checkout. + +set -e + +case "$1" in + triton) + TEST_URL="https://gist.githubusercontent.com/mawad-amd/6375dc078e39e256828f379e03310ec7/raw/0827d023eaf8e9755b17cbe8ab06f2ce258e746a/test_iris_distributed.py" + ;; + gluon) + TEST_URL="https://gist.githubusercontent.com/mawad-amd/2666dde8ebe2755eb0c4f2108709fcd5/raw/c5544943e2832c75252160bd9084600bf01a6b06/test_iris_gluon_distributed.py" + ;; + *) + echo "[ERROR] Usage: $0 " + exit 1 + ;; +esac + +if [ -z "$GPU_DEVICES" ]; then + echo "[ERROR] GPU_DEVICES is not set; run this through gpu_task_queue.py" + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO=${GITHUB_REPOSITORY:-"ROCm/iris"} +SHA=${GITHUB_SHA:-"HEAD"} +TEST_FILE=$(basename "$TEST_URL") + +"$SCRIPT_DIR/container_exec.sh" --gpus "$GPU_DEVICES" " + set -e + cd /iris_workspace + pip install git+https://github.com/${REPO}.git@${SHA} + wget -O ${TEST_FILE} ${TEST_URL} + torchrun --rdzv-backend=c10d --rdzv-endpoint=localhost:0 --nnodes=1 --nproc_per_node=2 ${TEST_FILE} +" + +echo "✅ External $1 validation test passed!" diff --git a/.github/scripts/run_new_examples.sh b/.github/scripts/run_new_examples.sh index b54724d82..9767f26c4 100755 --- a/.github/scripts/run_new_examples.sh +++ b/.github/scripts/run_new_examples.sh @@ -14,7 +14,7 @@ INSTALL_METHOD=${2:-"editable"} SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# GPU_DEVICES should be set by the workflow-level acquire_gpus.sh step +# GPU_DEVICES is set by gpu_task_queue.py GPU_ARG="" if [ -n "$GPU_DEVICES" ]; then GPU_ARG="--gpus $GPU_DEVICES" diff --git a/.github/scripts/run_tests.sh b/.github/scripts/run_tests.sh index 9e3b82e97..f5ad694a8 100755 --- a/.github/scripts/run_tests.sh +++ b/.github/scripts/run_tests.sh @@ -11,6 +11,9 @@ # - "git": pip install git+https://github.com/${{ github.repository }}.git@${{ github.sha }} # - "editable": pip install -e . # - "install": pip install . +# +# CONTAINER_IMAGE, if set, selects the container image (e.g. the nightly Triton +# image) instead of the one container_build.sh produced. set -e @@ -43,18 +46,21 @@ fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# GPU_DEVICES should be provided by workflow-level acquire_gpus.sh step -# or via command-line argument for backward compatibility +# GPU_DEVICES is set by gpu_task_queue.py, or passed as an argument if [ -z "$GPU_DEVICES" ]; then echo "[RUN-TESTS] WARNING: No GPUs allocated. GPU_DEVICES not set." echo "[RUN-TESTS] Tests may fail if they require GPUs." fi -# Build GPU argument +# Build GPU and image arguments GPU_ARG="" if [ -n "$GPU_DEVICES" ]; then GPU_ARG="--gpus $GPU_DEVICES" fi +IMAGE_ARG=() +if [ -n "$CONTAINER_IMAGE" ]; then + IMAGE_ARG=(--image "$CONTAINER_IMAGE") +fi # Build install command based on method INSTALL_CMD="" @@ -72,9 +78,10 @@ fi # Run tests in container EXIT_CODE=0 # shellcheck disable=SC2086 -"$SCRIPT_DIR/container_exec.sh" $GPU_ARG " +"$SCRIPT_DIR/container_exec.sh" "${IMAGE_ARG[@]}" $GPU_ARG " set -e + echo \"Triton version: \$(pip show triton 2>/dev/null | grep Version || echo unknown)\" echo \"Installing iris using method: $INSTALL_METHOD\" $INSTALL_CMD diff --git a/.github/workflows/iris-ccl-benchmark.yml b/.github/workflows/iris-ccl-benchmark.yml index de4e04da3..2ea97c09f 100644 --- a/.github/workflows/iris-ccl-benchmark.yml +++ b/.github/workflows/iris-ccl-benchmark.yml @@ -83,26 +83,23 @@ jobs: run: | bash .github/scripts/container_build.sh - - name: Acquire GPUs + - name: Run the sweep run: | - # Enough GPUs for the largest rank count; smaller ones run inside it. - # Validate first: this runs before run_ccl_benchmark.sh checks inputs. + # Validate first: num_ranks sizes the GPU request below, before + # run_ccl_benchmark.sh checks its inputs. if ! [[ "$NUM_RANKS" =~ ^[1-8](,[1-8])*$ ]]; then echo "[ERROR] num_ranks must be comma-separated integers 1-8, got '$NUM_RANKS'" exit 1 fi + # Enough GPUs for the largest rank count; smaller ones run inside it. MAX_RANKS=$(echo "$NUM_RANKS" | tr ',' '\n' | sort -n | tail -1) - bash .github/scripts/acquire_gpus.sh "$MAX_RANKS" - - - name: Run the sweep - run: | - set -e - bash .github/scripts/run_ccl_benchmark.sh \ - "$NUM_RANKS" \ - "$DTYPES" \ - "$M_VALUES" \ - "$N_VALUES" \ - "$N_REPEAT" + # Single-quoted: the task's shell expands the inputs from the + # environment, so they are never spliced into the command text. The + # task runs in a private copy of the checkout that is deleted when it + # ends, so copy the results out for the steps below, even on failure. + python3 .github/scripts/gpu_task_queue.py --task "$MAX_RANKS" "CCL sweep" \ + 'bash .github/scripts/run_ccl_benchmark.sh "$NUM_RANKS" "$DTYPES" "$M_VALUES" "$N_VALUES" "$N_REPEAT"; rc=$? + cp -r ccl_results "$GITHUB_WORKSPACE/" 2>/dev/null; exit $rc' - name: Publish the comparison table if: always() diff --git a/.github/workflows/iris-external-validation-test.yml b/.github/workflows/iris-external-validation-test.yml index aa12811e1..112fd57fc 100644 --- a/.github/workflows/iris-external-validation-test.yml +++ b/.github/workflows/iris-external-validation-test.yml @@ -19,7 +19,7 @@ permissions: jobs: external-validation-test: - name: External Validation Test + name: External Validation Tests (Triton and Gluon) runs-on: [linux-mi350-8gpu-iris] timeout-minutes: 180 @@ -44,85 +44,20 @@ jobs: run: | bash .github/scripts/container_build.sh - - name: Acquire GPUs - run: | - bash .github/scripts/acquire_gpus.sh 2 - - - name: Report environment versions - run: | - bash .github/scripts/report_versions.sh - - - name: Run External Validation Test - run: | - set -e - - echo "::group::Running external validation test" - bash .github/scripts/container_exec.sh " - set -e - - cd /iris_workspace - pip install git+https://github.com/${{ github.repository }}.git@${{ github.sha }} - wget -O test_iris_distributed.py https://gist.githubusercontent.com/mawad-amd/6375dc078e39e256828f379e03310ec7/raw/0827d023eaf8e9755b17cbe8ab06f2ce258e746a/test_iris_distributed.py - torchrun --rdzv-backend=c10d --rdzv-endpoint=localhost:0 --nnodes=1 --nproc_per_node=2 test_iris_distributed.py - " - echo "::endgroup::" - - echo "✅ External validation test passed!" - - - name: Release GPUs - if: always() - run: | - bash .github/scripts/release_gpus.sh - - external-gluon-validation-test: - name: External Gluon Validation Test - runs-on: [linux-mi350-8gpu-iris] - - steps: - - name: Checkout repository - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - with: - persist-credentials: false - - - name: Setup Apptainer (if not available) - run: | - if ! command -v apptainer &> /dev/null && ! command -v docker &> /dev/null; then - echo "Neither Apptainer nor Docker found, installing Apptainer..." - apt-get update && apt-get install -y software-properties-common - add-apt-repository -y ppa:apptainer/ppa - apt-get update && apt-get install -y apptainer - else - echo "Container runtime already available" - fi - - - name: Build Iris container - run: | - bash .github/scripts/container_build.sh - - - name: Acquire GPUs - run: | - bash .github/scripts/acquire_gpus.sh 2 - - name: Report environment versions run: | bash .github/scripts/report_versions.sh - - name: Run External Gluon Validation Test + # Both tests share one job and run side by side on 2 GPUs each, so the + # pair holds one runner slot instead of two while waiting for GPUs. + - name: Run external validation tests + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SHA: ${{ github.sha }} run: | - set -e - - echo "::group::Running external gluon validation test" - bash .github/scripts/container_exec.sh --gpus "$GPU_DEVICES" " - set -e - - cd /iris_workspace - pip install git+https://github.com/${{ github.repository }}.git@${{ github.sha }} - wget -O test_iris_gluon_distributed.py https://gist.githubusercontent.com/mawad-amd/2666dde8ebe2755eb0c4f2108709fcd5/raw/c5544943e2832c75252160bd9084600bf01a6b06/test_iris_gluon_distributed.py - torchrun --rdzv-backend=c10d --rdzv-endpoint=localhost:0 --nnodes=1 --nproc_per_node=2 test_iris_gluon_distributed.py - " - echo "::endgroup::" - - echo "✅ External gluon validation test passed!" + python3 .github/scripts/gpu_task_queue.py \ + --task 2 "External Validation Test" "bash .github/scripts/run_external_validation.sh triton" \ + --task 2 "External Gluon Validation Test" "bash .github/scripts/run_external_validation.sh gluon" - name: Release GPUs if: always() diff --git a/.github/workflows/iris-nightly-triton-test.yml b/.github/workflows/iris-nightly-triton-test.yml index e5223e718..f23dab27b 100644 --- a/.github/workflows/iris-nightly-triton-test.yml +++ b/.github/workflows/iris-nightly-triton-test.yml @@ -15,53 +15,14 @@ permissions: jobs: test-nightly: - name: Test ${{ matrix.test_dir }} (${{ matrix.num_ranks }} ranks, nightly Triton) + name: Test ${{ matrix.test_dir }} (1/2/4/8 ranks, nightly Triton) runs-on: [linux-mi350-8gpu-iris] timeout-minutes: 180 strategy: fail-fast: false matrix: - include: - - test_dir: examples - num_ranks: 1 - - test_dir: examples - num_ranks: 2 - - test_dir: examples - num_ranks: 4 - - test_dir: examples - num_ranks: 8 - - test_dir: unittests - num_ranks: 1 - - test_dir: unittests - num_ranks: 2 - - test_dir: unittests - num_ranks: 4 - - test_dir: unittests - num_ranks: 8 - - test_dir: ccl - num_ranks: 1 - - test_dir: ccl - num_ranks: 2 - - test_dir: ccl - num_ranks: 4 - - test_dir: ccl - num_ranks: 8 - - test_dir: context - num_ranks: 1 - - test_dir: context - num_ranks: 2 - - test_dir: context - num_ranks: 4 - - test_dir: context - num_ranks: 8 - - test_dir: ops - num_ranks: 1 - - test_dir: ops - num_ranks: 2 - - test_dir: ops - num_ranks: 4 - - test_dir: ops - num_ranks: 8 + # One job per directory runs every rank count, as in iris-tests.yml. + test_dir: [examples, unittests, ccl, context, ops] steps: - name: Checkout repository @@ -102,48 +63,18 @@ jobs: ) 200>"$LOCK" echo "NIGHTLY_SIF=$NIGHTLY_SIF" >> "$GITHUB_ENV" - - - name: Acquire GPUs - run: | - bash .github/scripts/acquire_gpus.sh "${{ matrix.num_ranks }}" + # run_tests.sh runs in this image instead of the container_build.sh one. + echo "CONTAINER_IMAGE=$NIGHTLY_SIF" >> "$GITHUB_ENV" - name: Report environment versions run: | bash .github/scripts/report_versions.sh - - name: Run ${{ matrix.test_dir }} tests with ${{ matrix.num_ranks }} ranks (nightly Triton) + - name: Run ${{ matrix.test_dir }} tests with 1, 2, 4 and 8 ranks (nightly Triton) run: | - set -e - echo "::group::Running ${{ matrix.test_dir }} tests with ${{ matrix.num_ranks }} ranks (nightly Triton)" - - # Build GPU argument (GPU_DEVICES set by acquire_gpus.sh) - GPU_ARG="" - if [ -n "$GPU_DEVICES" ]; then - GPU_ARG="--gpus $GPU_DEVICES" - fi - - # Run tests in pre-built nightly container (Triton from main already installed) - bash .github/scripts/container_exec.sh --image "$NIGHTLY_SIF" $GPU_ARG " - set -e - - echo \"Triton version: \$(pip show triton 2>/dev/null | grep Version || echo unknown)\" - - # Install iris in editable mode - echo \"Installing iris in editable mode\" - pip install -e . - - # Run tests in the specified directory - for test_file in tests/${{ matrix.test_dir }}/test_*.py; do - if [ -f \"\$test_file\" ]; then - echo \"Testing: \$test_file with ${{ matrix.num_ranks }} ranks (nightly Triton)\" - torchrun --rdzv-backend=c10d --rdzv-endpoint=localhost:0 \ - --nnodes=1 --nproc_per_node=${{ matrix.num_ranks }} \ - tests/run_tests_distributed.py \"\$test_file\" -v --tb=short --durations=10 - fi - done - " - echo "::endgroup::" - echo "✅ ${{ matrix.test_dir }} tests with ${{ matrix.num_ranks }} ranks (nightly Triton) passed!" + python3 .github/scripts/gpu_task_queue.py --install editable \ + "${{ matrix.test_dir }}:8" "${{ matrix.test_dir }}:4" \ + "${{ matrix.test_dir }}:2" "${{ matrix.test_dir }}:1" - name: Release GPUs if: always() diff --git a/.github/workflows/iris-performance-regression-test.yml b/.github/workflows/iris-performance-regression-test.yml index 96e694493..62e8a2693 100644 --- a/.github/workflows/iris-performance-regression-test.yml +++ b/.github/workflows/iris-performance-regression-test.yml @@ -19,36 +19,9 @@ permissions: jobs: performance-test: - name: ${{ matrix.example_name }} + name: GEMM All-Scatter benchmarks (8 ranks) runs-on: [linux-mi350-8gpu-iris] timeout-minutes: 180 - strategy: - fail-fast: false - matrix: - # Performance baselines measured on AMD Instinct MI325X (8 GPUs), before CI - # moved to the MI350X runners -- re-measure before tightening thresholds. - include: - - example_name: "GEMM All-Scatter WG Specialization" - example_path: "10_gemm_all_scatter_wg_specialization" - tflops_threshold: 1440 # Actual: ~1802 TFLOPs (80% regression threshold) - # gemm_sms must stay below the CU count -- the remainder is what runs the - # communication path. 256 left zero of it on a 256-CU part. - benchmark_args: "-m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 192" - - - example_name: "GEMM All-Scatter" - example_path: "07_gemm_all_scatter" - tflops_threshold: 1000 # Actual: ~1407 TFLOPs - benchmark_args: "-m 16384 -n 16384 -k 16384 --BLK_M 256 --BLK_N 64 --BLK_K 64 --gsize_m 6 --gemm_sms 256" - - - example_name: "GEMM All-Scatter Producer-Consumer" - example_path: "11_gemm_all_scatter_producer_consumer" - tflops_threshold: 1600 # Actual: ~2190 TFLOPs - benchmark_args: "-m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 256 --comm_sms 48" - - - example_name: "GEMM All-Scatter Bulk Synchronous" - example_path: "12_gemm_all_scatter_bulk_synchronous" - tflops_threshold: 900 # Actual: ~1262 TFLOPs - benchmark_args: "-m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 256" steps: - name: Checkout repository @@ -71,27 +44,32 @@ jobs: run: | bash .github/scripts/container_build.sh - - name: Acquire GPUs - run: | - bash .github/scripts/acquire_gpus.sh 8 - - name: Report environment versions run: | bash .github/scripts/report_versions.sh - - name: Run ${{ matrix.example_name }} Benchmark (8 ranks) + # One job runs all four benchmarks back to back. Each needs the whole node + # for about 75 s, but as separate jobs each held a runner slot while it + # waited for 8 free GPUs -- for up to 2.4 h. gpu_task_queue.py takes the + # GPUs per benchmark and reserves the node so they are not starved by + # smaller tasks. + # + # Baselines were measured on AMD Instinct MI325X (8 GPUs), before CI moved + # to the MI350X runners -- re-measure before tightening thresholds. + # Thresholds are ~80% of the measured TFLOPs noted alongside. + - name: Run benchmarks run: | - set -e - - echo "::group::Running performance benchmark" - bash .github/scripts/run_perf_benchmark.sh \ - "${{ matrix.example_path }}" \ - "${{ matrix.tflops_threshold }}" \ - ${{ matrix.benchmark_args }} - echo "::endgroup::" + python3 .github/scripts/gpu_task_queue.py \ + --task 8 "GEMM All-Scatter WG Specialization" \ + "bash .github/scripts/run_perf_benchmark.sh 10_gemm_all_scatter_wg_specialization 1440 -m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 192" \ + --task 8 "GEMM All-Scatter" \ + "bash .github/scripts/run_perf_benchmark.sh 07_gemm_all_scatter 1000 -m 16384 -n 16384 -k 16384 --BLK_M 256 --BLK_N 64 --BLK_K 64 --gsize_m 6 --gemm_sms 256" \ + --task 8 "GEMM All-Scatter Producer-Consumer" \ + "bash .github/scripts/run_perf_benchmark.sh 11_gemm_all_scatter_producer_consumer 1600 -m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 256 --comm_sms 48" \ + --task 8 "GEMM All-Scatter Bulk Synchronous" \ + "bash .github/scripts/run_perf_benchmark.sh 12_gemm_all_scatter_bulk_synchronous 900 -m 16384 -n 16384 -k 16384 --BLK_M 128 --BLK_N 128 --BLK_K 64 --gsize_m 6 --gemm_sms 256" - name: Release GPUs if: always() run: | bash .github/scripts/release_gpus.sh -