From a4c110b437681bb690b4d864433311eb7d6def83 Mon Sep 17 00:00:00 2001 From: "Marcus D. Hanwell" Date: Tue, 20 Oct 2026 14:52:31 -0400 Subject: [PATCH 01/10] Initial pass at GPU testing for cutile-rs --- .github/workflows/cutile-rs.yml | 107 ++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 11d0496217..d9da7ffda9 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -184,6 +184,113 @@ jobs: - name: CPU only tests run: ./scripts/run_cpu_tests.sh + # nextest archives carry test executables, Cargo metadata, and build + # outputs needed to run on another machine without rebuilding Rust. + # Keep both workspaces separate, then bundle them in one downloadable tar. + - name: Install cargo-nextest + run: cargo install cargo-nextest --version 0.9.146 --locked + + - name: Archive Tile runtime test packages + run: | + mkdir -p "$RUNNER_TEMP/cutile-gpu-tests" + cargo nextest archive --locked --features experimental-tune \ + --archive-file "$RUNNER_TEMP/cutile-gpu-tests/tile-tests.tar.zst" + + - name: Archive shared-host runtime test packages + working-directory: ${{ github.workspace }} + run: | + cargo nextest archive --locked \ + -p cuda-bindings -p cuda-core -p cuda-async \ + --archive-file "$RUNNER_TEMP/cutile-gpu-tests/host-tests.tar.zst" + + - name: Bundle runtime test archives + run: | + cp "$(command -v cargo-nextest)" "$RUNNER_TEMP/cutile-gpu-tests/cargo-nextest" + git rev-parse HEAD > "$RUNNER_TEMP/cutile-gpu-tests/revision.txt" + cargo nextest --version > "$RUNNER_TEMP/cutile-gpu-tests/nextest-version.txt" + tar -cf "$RUNNER_TEMP/cutile-gpu-tests.tar" \ + -C "$RUNNER_TEMP/cutile-gpu-tests" . + + - name: Upload runtime test packages + uses: actions/upload-artifact@v4 + with: + name: cutile-gpu-tests + path: ${{ runner.temp }}/cutile-gpu-tests.tar + if-no-files-found: error + compression-level: 0 # The nested nextest archives are already compressed. + retention-days: 7 + + gpu-tests: + needs: build + runs-on: nv-gpu-amd64-rtxpro6000-1gpu + timeout-minutes: 30 + container: + image: nvidia/cuda:13.3.0-devel-ubuntu24.04 + options: -u root --security-opt seccomp=unconfined --shm-size 16g + env: + NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} + env: + CUDA_TOOLKIT_PATH: /usr/local/cuda + CUTILE_TILEIRAS_PATH: /usr/local/cuda/bin/tileiras + steps: + - uses: actions/checkout@v7 + + - uses: actions/download-artifact@v4 + with: + name: cutile-gpu-tests + path: ${{ runner.temp }}/gpu-tests + + - name: Extract runtime test packages + run: | + tar -xf "$RUNNER_TEMP/gpu-tests/cutile-gpu-tests.tar" -C "$RUNNER_TEMP/gpu-tests" + test "$(cat "$RUNNER_TEMP/gpu-tests/revision.txt")" = "$GITHUB_SHA" + echo "$RUNNER_TEMP/gpu-tests" >> "$GITHUB_PATH" + nvidia-smi + "$CUTILE_TILEIRAS_PATH" --version + + # Mirror scripts/run_gpu_tests.sh's runtime selection. Archive mode uses + # the checked-out sources for fixtures and remaps the two workspace roots. + # Serial execution prevents unrelated GPU tests competing for resources. + - name: Tile GPU runtime tests + run: | + cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ + --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ + -E 'package(=cutile) & ( + binary(/^(arange|autotune|do_bench|dtype_float_ops|gpu_execution_ops|nested_partition_mut|slice_non_divisible|tensor_reinterpret|tensor_views|warmup_suite|gpu)$/) + | (binary(=control_flow_ops) & not test(compile_)) + | (binary(/^(tensor_and_matrix_ops|type_conversion_ops)$/) & test(execute_)) + | (binary(=specialization_bits) & test(raw_pointer_launch)) + )' + + - name: Submission lifetime tests (default, reactor, spin) + run: | + for budget in default 0 200000; do + echo "CUDA_ASYNC_SPIN_BUDGET_US=$budget" + if [[ "$budget" == default ]]; then + unset CUDA_ASYNC_SPIN_BUDGET_US + else + export CUDA_ASYNC_SPIN_BUDGET_US="$budget" + fi + cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ + --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ + -E 'package(=cutile) & binary(=submission_lifetimes)' + done + + - name: Cross-file debug source kernel + run: | + cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ + --workspace-remap "$PWD" --test-threads 1 --run-ignored only \ + -E 'package(=cutile) & binary(=debug_info) & test(=kernel_executes_in_all_debug_modes)' + + - name: Shared-host GPU runtime tests + working-directory: ${{ github.workspace }} + run: | + cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/host-tests.tar.zst" \ + --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ + -E 'package(=cuda-bindings) + | (package(=cuda-core) & binary(/^(vmm|vmm_multicast|borrow_raw|simt_context_limits|simt_context_sync_policy|simt_device_buffer|simt_device_buffer_leaks|simt_pinned_host_buffer|simt_stream_priority|simt_stream_query|simt_vmm_multicast|simt_vmm_p2p)$/)) + | (package(=cuda-async) & (kind(lib) | binary(/^(concurrent_capture|cuda_graph|device_fault|drop_in_flight|execute_once|execution_lock|pool_allocation|reactor_correctness)$/)))' + # Breaking API changes must ship with the right version bump: compare each # publishable crate's API against its latest crates.io release and fail the # PR when a break arrives without one (the 0.3.0 -> 0.3.1 lesson). Same From 6104358cf203e663eccc60a30a5fc2b324636c9c Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 22 Oct 2026 16:05:17 +0200 Subject: [PATCH 02/10] Use taiki-e/install-action to install nextest Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index d9da7ffda9..0a23ee351e 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -187,8 +187,10 @@ jobs: # nextest archives carry test executables, Cargo metadata, and build # outputs needed to run on another machine without rebuilding Rust. # Keep both workspaces separate, then bundle them in one downloadable tar. - - name: Install cargo-nextest - run: cargo install cargo-nextest --version 0.9.146 --locked + - uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2.87.16 + name: Install cargo-nextest + with: + tool: cargo-nextest - name: Archive Tile runtime test packages run: | From 383d584e5c77576bde9e4093bc35884ff21a36d7 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 22 Oct 2026 16:09:57 +0200 Subject: [PATCH 03/10] Fix git checkout by using new $ syntax for local actions Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 0a23ee351e..bfd41e4b69 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -155,8 +155,8 @@ jobs: command -v tileiras tileiras --help | grep -q "sm_120" + - uses: $/cutile-rs/.github/actions/setup - uses: actions/checkout@v7 - - uses: ./cutile-rs/.github/actions/setup # Cheapest checks first so a formatting or lint slip fails in seconds. - name: Format check From 1cb65530805bb316bee9fd8b26fcb075381e60e4 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 22 Oct 2026 16:38:42 +0200 Subject: [PATCH 04/10] Modify setup action to work pre- and post- checkout Signed-off-by: Jacob Wujciak-Jens --- cutile-rs/.github/actions/setup/action.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cutile-rs/.github/actions/setup/action.yml b/cutile-rs/.github/actions/setup/action.yml index 004f7e0fdc..0706d39695 100644 --- a/cutile-rs/.github/actions/setup/action.yml +++ b/cutile-rs/.github/actions/setup/action.yml @@ -19,8 +19,8 @@ runs: using: composite steps: - name: Validate inputs + working-directory: ${{ github.workspace }} shell: bash - working-directory: cutile-rs env: RUST_COMPONENTS: ${{ inputs.components }} RUST_CACHE: ${{ inputs.cache }} @@ -35,8 +35,8 @@ runs: fi - name: Install build dependencies + working-directory: ${{ github.workspace }} shell: bash - working-directory: cutile-rs run: | apt-get update -qq apt-get install -y -qq --no-install-recommends build-essential curl \ @@ -47,7 +47,8 @@ runs: # nightly lives in cuda-oxide/rust-toolchain.toml. - name: Install Rust shell: bash - working-directory: cutile-rs + # relative path to land at the rust-toolchain.toml regardless of use of $ or ./ syntax + working-directory: ${{ github.action_path }}/../../.. env: RUST_COMPONENTS: ${{ inputs.components }} run: | From 063b3a8b67f372946b997ae7370ad18b9f4f7f65 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 22 Oct 2026 16:58:21 +0200 Subject: [PATCH 05/10] Mark checkout as safe git dir Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index bfd41e4b69..5e1e67619a 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -158,6 +158,9 @@ jobs: - uses: $/cutile-rs/.github/actions/setup - uses: actions/checkout@v7 + - name: Mark checkout as safe + run: git config --global --add safe.directory /__w/cuda-rust/cuda-rust + # Cheapest checks first so a formatting or lint slip fails in seconds. - name: Format check run: cargo fmt -- --check From 67d1214db1a303e3650275befbda17f32e95ed0c Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 23 Oct 2026 15:24:25 +0200 Subject: [PATCH 06/10] Update test runner label Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 5e1e67619a..34da1659ec 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -227,7 +227,7 @@ jobs: gpu-tests: needs: build - runs-on: nv-gpu-amd64-rtxpro6000-1gpu + runs-on: linux-amd64-gpu-rtxpro6000-latest-1 timeout-minutes: 30 container: image: nvidia/cuda:13.3.0-devel-ubuntu24.04 From ba76459dc8380cdd7c76c0d93dd2c4073396c8d2 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Fri, 23 Oct 2026 15:25:02 +0200 Subject: [PATCH 07/10] Try running job without custom options Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 34da1659ec..323330ec77 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -231,9 +231,6 @@ jobs: timeout-minutes: 30 container: image: nvidia/cuda:13.3.0-devel-ubuntu24.04 - options: -u root --security-opt seccomp=unconfined --shm-size 16g - env: - NVIDIA_VISIBLE_DEVICES: ${{ env.NVIDIA_VISIBLE_DEVICES }} env: CUDA_TOOLKIT_PATH: /usr/local/cuda CUTILE_TILEIRAS_PATH: /usr/local/cuda/bin/tileiras From 8bec57545b3be050056d304db8e9ade5bc724e87 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Mon, 26 Oct 2026 18:04:37 +0100 Subject: [PATCH 08/10] Reduce retention days Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 323330ec77..5d4096e60b 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -223,7 +223,7 @@ jobs: path: ${{ runner.temp }}/cutile-gpu-tests.tar if-no-files-found: error compression-level: 0 # The nested nextest archives are already compressed. - retention-days: 7 + retention-days: 1 gpu-tests: needs: build From 4ef965d2df3f3707b953bb3ff4517378ca641b88 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Mon, 26 Oct 2026 18:05:52 +0100 Subject: [PATCH 09/10] Use run_gpu_tests.sh in CI to avoid drift Local behavior is unchanged but we drop the custom bash that mirrored the script in CI. Signed-off-by: Jacob Wujciak-Jens --- .github/workflows/cutile-rs.yml | 45 +----- CONTRIBUTING.md | 11 ++ cutile-rs/scripts/run_gpu_tests.sh | 215 ++++++++++++++++++++++++----- 3 files changed, 194 insertions(+), 77 deletions(-) diff --git a/.github/workflows/cutile-rs.yml b/.github/workflows/cutile-rs.yml index 5d4096e60b..521a54ef34 100644 --- a/.github/workflows/cutile-rs.yml +++ b/.github/workflows/cutile-rs.yml @@ -246,52 +246,11 @@ jobs: run: | tar -xf "$RUNNER_TEMP/gpu-tests/cutile-gpu-tests.tar" -C "$RUNNER_TEMP/gpu-tests" test "$(cat "$RUNNER_TEMP/gpu-tests/revision.txt")" = "$GITHUB_SHA" - echo "$RUNNER_TEMP/gpu-tests" >> "$GITHUB_PATH" nvidia-smi "$CUTILE_TILEIRAS_PATH" --version - # Mirror scripts/run_gpu_tests.sh's runtime selection. Archive mode uses - # the checked-out sources for fixtures and remaps the two workspace roots. - # Serial execution prevents unrelated GPU tests competing for resources. - - name: Tile GPU runtime tests - run: | - cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ - --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ - -E 'package(=cutile) & ( - binary(/^(arange|autotune|do_bench|dtype_float_ops|gpu_execution_ops|nested_partition_mut|slice_non_divisible|tensor_reinterpret|tensor_views|warmup_suite|gpu)$/) - | (binary(=control_flow_ops) & not test(compile_)) - | (binary(/^(tensor_and_matrix_ops|type_conversion_ops)$/) & test(execute_)) - | (binary(=specialization_bits) & test(raw_pointer_launch)) - )' - - - name: Submission lifetime tests (default, reactor, spin) - run: | - for budget in default 0 200000; do - echo "CUDA_ASYNC_SPIN_BUDGET_US=$budget" - if [[ "$budget" == default ]]; then - unset CUDA_ASYNC_SPIN_BUDGET_US - else - export CUDA_ASYNC_SPIN_BUDGET_US="$budget" - fi - cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ - --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ - -E 'package(=cutile) & binary(=submission_lifetimes)' - done - - - name: Cross-file debug source kernel - run: | - cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/tile-tests.tar.zst" \ - --workspace-remap "$PWD" --test-threads 1 --run-ignored only \ - -E 'package(=cutile) & binary(=debug_info) & test(=kernel_executes_in_all_debug_modes)' - - - name: Shared-host GPU runtime tests - working-directory: ${{ github.workspace }} - run: | - cargo-nextest nextest run --archive-file "$RUNNER_TEMP/gpu-tests/host-tests.tar.zst" \ - --workspace-remap "$PWD" --test-threads 1 --no-fail-fast \ - -E 'package(=cuda-bindings) - | (package(=cuda-core) & binary(/^(vmm|vmm_multicast|borrow_raw|simt_context_limits|simt_context_sync_policy|simt_device_buffer|simt_device_buffer_leaks|simt_pinned_host_buffer|simt_stream_priority|simt_stream_query|simt_vmm_multicast|simt_vmm_p2p)$/)) - | (package(=cuda-async) & (kind(lib) | binary(/^(concurrent_capture|cuda_graph|device_fault|drop_in_flight|execute_once|execution_lock|pool_allocation|reactor_correctness)$/)))' + - name: GPU runtime tests + run: ./scripts/run_gpu_tests.sh --archive-dir "$RUNNER_TEMP/gpu-tests" # Breaking API changes must ship with the right version bump: compare each # publishable crate's API against its latest crates.io release and fail the diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93c51116b7..ab909717cc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,6 +192,17 @@ cutile-rs/scripts/run_all.sh # both of the above, then the examples and Run `cutile-rs/scripts/run_all.sh` before marking a Tile pull request ready for review. + +The GPU script also accepts `--archive-dir DIR` to run prebuilt tests without +Cargo or a Rust toolchain. The directory must contain `cargo-nextest`, +`tile-tests.tar.zst`, and `host-tests.tar.zst`, as bundled by CI. This mode runs +tests serially and uses the checkout for source fixtures. +See the archive, bundle, and extract steps in +[the Tile CI workflow](.github/workflows/cutile-rs.yml) for how to produce this +directory. Extract `cutile-gpu-tests.tar` first and use a checkout matching its +`revision.txt`; `cargo-nextest` must be executable. The script's opening comments +describe the directory layout and the contents of each test archive. + CI runs the same suites in `.github/workflows/cutile-rs.yml`. Because the Tile lanes use NVIDIA self-hosted runners, that workflow triggers on the `pull-request/` branches that copy-pr-bot mirrors rather than on the pull diff --git a/cutile-rs/scripts/run_gpu_tests.sh b/cutile-rs/scripts/run_gpu_tests.sh index b40b2254a4..7a9311b81f 100755 --- a/cutile-rs/scripts/run_gpu_tests.sh +++ b/cutile-rs/scripts/run_gpu_tests.sh @@ -7,6 +7,153 @@ set -u source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/test_runner_common.sh" +# With no arguments, build and run locally. Archive mode needs only the bundled +# cargo-nextest executable, the two archives, and this checkout for fixtures. +# After extracting CI's cutile-gpu-tests.tar, --archive-dir points to: +# DIR/cargo-nextest executable used to create the archives +# DIR/tile-tests.tar.zst Tile workspace, with experimental-tune enabled +# DIR/host-tests.tar.zst cuda-bindings, cuda-core, and cuda-async workspace tests +# CI also bundles revision.txt and nextest-version.txt; the workflow checks the +# revision against the checkout before invoking this script. See the archive, +# bundle, and extract steps in .github/workflows/cutile-rs.yml at the git root. +# Both archives use nextest's standard format, including target/nextest metadata. +usage() { + echo "Usage: $0 [--archive-dir DIR]" +} + +ARCHIVE_DIR="" +case "$#" in + 0) ;; + 1) + if [[ "$1" == --help || "$1" == -h ]]; then + usage + exit 0 + fi + usage >&2 + exit 2 + ;; + 2) + if [[ "$1" != --archive-dir || ! -d "$2" ]]; then + usage >&2 + exit 2 + fi + ARCHIVE_DIR="$(cd "$2" && pwd)" + for archive in tile-tests.tar.zst host-tests.tar.zst; do + if [[ ! -f "$ARCHIVE_DIR/$archive" ]]; then + echo "Missing test archive: $ARCHIVE_DIR/$archive" >&2 + exit 2 + fi + done + if [[ ! -x "$ARCHIVE_DIR/cargo-nextest" ]]; then + echo "Missing executable: $ARCHIVE_DIR/cargo-nextest" >&2 + exit 2 + fi + ;; + *) usage >&2; exit 2 ;; +esac + +if [[ -n "$ARCHIVE_DIR" ]]; then + # Extract each workspace once, rather than unpacking it for every test target. + EXTRACT_DIR="$(mktemp -d)" || exit 1 + trap 'rm -rf "$EXTRACT_DIR"' EXIT + for workspace in tile host; do + workspace_root="$REPO_ROOT" + if [[ "$workspace" == host ]]; then + workspace_root="$GIT_ROOT" + fi + "$ARCHIVE_DIR/cargo-nextest" nextest list --list-type binaries-only \ + --archive-file "$ARCHIVE_DIR/$workspace-tests.tar.zst" \ + --extract-to "$EXTRACT_DIR/$workspace" \ + --workspace-remap "$workspace_root" > /dev/null || exit 1 + done +fi + +# Keep suite selection below independent of the runner. Target is an integration +# test name, "lib" for unit tests, or "all" for the whole package. +# shellcheck disable=SC2329 # Invoked through run_gpu_step/run_step. +gpu_test() ( + local workspace="$1" package="$2" target="$3" + shift 3 + local workspace_root="$REPO_ROOT" + if [[ "$workspace" == host ]]; then + workspace_root="$GIT_ROOT" + fi + cd "$workspace_root" || return 1 + + local expression="package(=$package)" + local cargo_args=(test -p "$package") + local libtest_args=() + local nextest_args=(--test-threads 1 --no-fail-fast) + case "$target" in + lib) cargo_args+=(--lib); expression+=" & kind(lib)" ;; + all) ;; + *) cargo_args+=(--test "$target"); expression+=" & binary(=$target)" ;; + esac + while [[ $# -gt 0 ]]; do + case "$1" in + --filter) + cargo_args+=("$2") + expression+=" & test($2)" + shift 2 ;; + --skip) + libtest_args+=(--skip "$2") + expression+=" & not test($2)" + shift 2 ;; + --ignored) + libtest_args+=(--ignored) + nextest_args+=(--run-ignored only) + shift ;; + --serial) + libtest_args+=(--test-threads=1) + shift ;; + --features) + cargo_args+=(--features "$2") + shift 2 ;; + *) echo "Unknown GPU test option: $1" >&2; return 2 ;; + esac + done + + if [[ -n "$ARCHIVE_DIR" ]]; then + # Serialize GPU tests so unrelated processes cannot compete for resources. + local target_dir="$EXTRACT_DIR/$workspace/target" + "$ARCHIVE_DIR/cargo-nextest" nextest run \ + --cargo-metadata "$target_dir/nextest/cargo-metadata.json" \ + --binaries-metadata "$target_dir/nextest/binaries-metadata.json" \ + --target-dir-remap "$target_dir" \ + --workspace-remap "$workspace_root" "${nextest_args[@]}" -E "$expression" + else + if [[ ${#libtest_args[@]} -gt 0 ]]; then + cargo_args+=(-- "${libtest_args[@]}") + fi + cargo "${cargo_args[@]}" + fi +) + +# shellcheck disable=SC2329 # Invoked through run_gpu_step/run_step. +submission_test() ( + if [[ "$1" == default ]]; then + # Local runs honor developer overrides; CI exercises the actual default. + if [[ -n "$ARCHIVE_DIR" ]]; then + unset CUDA_ASYNC_SPIN_BUDGET_US + fi + else + export CUDA_ASYNC_SPIN_BUDGET_US="$1" + fi + gpu_test tile cutile submission_lifetimes --serial +) + +run_gpu_step() { + if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::group::$1" + fi + local status=0 + run_step "$@" || status=$? + if [[ "${GITHUB_ACTIONS:-}" == true ]]; then + echo "::endgroup::" + fi + return "$status" +} + print_header "Running GPU tests" for test_target in \ @@ -20,34 +167,34 @@ for test_target in \ tensor_reinterpret \ tensor_views do - run_step \ + run_gpu_step \ "cutile GPU integration test ${test_target}" \ - cargo test -p cutile --features experimental-tune --test "$test_target" + gpu_test tile cutile "$test_target" --features experimental-tune done -run_step \ +run_gpu_step \ "cutile GPU integration test control_flow_ops runtime cases" \ - cargo test -p cutile --test control_flow_ops -- --skip compile_ + gpu_test tile cutile control_flow_ops --skip compile_ -run_step \ +run_gpu_step \ "cutile GPU integration test tensor_and_matrix_ops runtime cases" \ - cargo test -p cutile --test tensor_and_matrix_ops execute_ + gpu_test tile cutile tensor_and_matrix_ops --filter execute_ -run_step \ +run_gpu_step \ "cutile GPU integration test type_conversion_ops runtime cases" \ - cargo test -p cutile --test type_conversion_ops execute_ + gpu_test tile cutile type_conversion_ops --filter execute_ -run_step \ +run_gpu_step \ "cutile GPU integration test specialization_bits runtime cases" \ - cargo test -p cutile --test specialization_bits raw_pointer_launch + gpu_test tile cutile specialization_bits --filter raw_pointer_launch -run_step \ +run_gpu_step \ "cutile GPU warmup and disk-cache tests" \ - cargo test -p cutile --test warmup_suite + gpu_test tile cutile warmup_suite -run_step \ +run_gpu_step \ "cutile GPU aggregate tests" \ - cargo test -p cutile --test gpu + gpu_test tile cutile gpu # Submission-lifetime tests: their own binary, single-threaded. Each test gates # a stream with a blocking host function, and any context-wide synchronize in @@ -59,29 +206,29 @@ run_step \ # in the inline cuStreamQuery spin; 0 forces the reactor path, a large budget # forces the spin path to resolve against the tests' blocking Gates. Under a # Gate, `Pending` is the only correct first-poll outcome on either path. -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests" \ - cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test default -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests (reactor path, CUDA_ASYNC_SPIN_BUDGET_US=0)" \ - env CUDA_ASYNC_SPIN_BUDGET_US=0 cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test 0 -run_step \ +run_gpu_step \ "cutile GPU submission-lifetime tests (spin path, CUDA_ASYNC_SPIN_BUDGET_US=200000)" \ - env CUDA_ASYNC_SPIN_BUDGET_US=200000 cargo test -p cutile --test submission_lifetimes -- --test-threads=1 + submission_test 200000 -run_step \ +run_gpu_step \ "cutile cross-file debug source kernel" \ - cargo test -p cutile --test debug_info kernel_executes_in_all_debug_modes -- --ignored + gpu_test tile cutile debug_info --filter kernel_executes_in_all_debug_modes --ignored -run_step \ +run_gpu_step \ "cuda-core GPU integration test vmm" \ - host_cargo test -p cuda-core --test vmm + gpu_test host cuda-core vmm -run_step \ +run_gpu_step \ "cuda-core GPU integration test vmm_multicast" \ - host_cargo test -p cuda-core --test vmm_multicast + gpu_test host cuda-core vmm_multicast # The SIMT integration tests carried over from cuda-oxide. simt_vmm_p2p # exercises the real two-GPU P2P path (it self-skips on single-device @@ -99,20 +246,20 @@ for test_target in \ simt_vmm_multicast \ simt_vmm_p2p do - run_step \ + run_gpu_step \ "cuda-core GPU integration test ${test_target}" \ - host_cargo test -p cuda-core --test "$test_target" + gpu_test host cuda-core "$test_target" done -run_step \ +run_gpu_step \ "cuda-async unit tests (with live driver)" \ - host_cargo test -p cuda-async --lib + gpu_test host cuda-async lib # Unfiltered: includes the GPU-requiring `cuda_tests` module that the CPU # path skips. -run_step \ +run_gpu_step \ "cuda-bindings tests (with live driver)" \ - host_cargo test -p cuda-bindings + gpu_test host cuda-bindings all for test_target in \ concurrent_capture \ @@ -124,9 +271,9 @@ for test_target in \ pool_allocation \ reactor_correctness do - run_step \ + run_gpu_step \ "cuda-async GPU integration test ${test_target}" \ - host_cargo test -p cuda-async --test "$test_target" + gpu_test host cuda-async "$test_target" done print_summary_and_exit \ From 3d45eb06f11bb41f578363cc22fa29f2cf60b8c4 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Mon, 26 Oct 2026 18:32:00 +0100 Subject: [PATCH 10/10] Make sure extraction path exists This is required by nextest Signed-off-by: Jacob Wujciak-Jens --- cutile-rs/scripts/run_gpu_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/cutile-rs/scripts/run_gpu_tests.sh b/cutile-rs/scripts/run_gpu_tests.sh index 7a9311b81f..cb94864231 100755 --- a/cutile-rs/scripts/run_gpu_tests.sh +++ b/cutile-rs/scripts/run_gpu_tests.sh @@ -57,6 +57,7 @@ if [[ -n "$ARCHIVE_DIR" ]]; then EXTRACT_DIR="$(mktemp -d)" || exit 1 trap 'rm -rf "$EXTRACT_DIR"' EXIT for workspace in tile host; do + mkdir -p "$EXTRACT_DIR/$workspace" || exit 1 workspace_root="$REPO_ROOT" if [[ "$workspace" == host ]]; then workspace_root="$GIT_ROOT"