diff --git a/.github/workflows/host-gates.yml b/.github/workflows/host-gates.yml new file mode 100644 index 00000000..b60bd6f1 --- /dev/null +++ b/.github/workflows/host-gates.yml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Fresh-checkout host gates (issue #90). +# +# Composition, so this file stays small: +# - make test is already the host-tests job in ci.yml. It is not repeated here. +# - make test-runtime-api is runtime-api.yml. Issue #189 owns that job. +# - x86-64 make test is the x86-host-tests job in .github/workflows/host-x86.yml. This file does not add it. +# The AArch64 job below is the documented Linux host path with no GPU device opened. +# Budgets: make -j4, 30 minutes on the arm runner, 5 minutes for the figure check. +name: host-gates +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +concurrency: + group: host-gates-${{ github.ref }} + cancel-in-progress: true +jobs: + figures: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Checker negative controls + run: bash tools/ci/host-gates.sh --self-test + - name: Figure check + run: bash tools/ci/host-gates.sh --figures + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: host-gates-figures-${{ github.sha }} + path: build/host-gates/ + if-no-files-found: warn + fresh-checkout: + runs-on: ubuntu-24.04-arm + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Checker negative controls + run: bash tools/ci/host-gates.sh --self-test + - name: Host gates + env: + HOST_GATES_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: bash tools/ci/host-gates.sh + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: host-gates-${{ github.sha }} + path: build/host-gates/ + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 479c406e..50a47402 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,7 +75,7 @@ make hello-gpu # runs make doctor, then examples/he - Status words in `README.md` and `ROADMAP.md` are PASS (receipt on a named commit), PARTIAL, MISSING, BUILDING. Do not upgrade a status without the receipt. - New source files carry `SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception`. Keep third-party notices (`THIRD_PARTY.md`, `NOTICE`). - Plain English, short sentences, every number with the command that produced it. -- Pull request CI runs the host-only `make test` suite, pinned Omega binding regeneration and mutants, SBOM generation and GitHub CodeQL. Its disposable arm64 host-test runner installs only the checksum-verified NVIDIA `nvdisasm` 13.0.85 executable and the bridge source pinned by `om/omega-bridge.lock`. The archive URL and SHA-256 are fixed in `.github/workflows/ci.yml`, from NVIDIA's `redistrib_13.0.2.json` manifest. The oracle is verification-only: no CUDA driver, runtime or compiler is installed, no NVIDIA binary is committed, and no GPU is opened. Provisioning errors and missing-oracle checks remain failures. Put local results in the pull request with their exact source revision; all-green host CI is not physical qualification. +- Pull request CI runs the host-only `make test` suite, pinned Omega binding regeneration and mutants, SBOM generation and GitHub CodeQL. Its disposable arm64 host-test runner installs only the checksum-verified NVIDIA `nvdisasm` 13.0.85 executable and the bridge source pinned by `om/omega-bridge.lock`. The archive URL and SHA-256 are fixed in `.github/workflows/ci.yml`, from NVIDIA's `redistrib_13.0.2.json` manifest. The oracle is verification-only: no CUDA driver, runtime or compiler is installed, no NVIDIA binary is committed, and no GPU is opened. Provisioning errors and missing-oracle checks remain failures. Put local results in the pull request with their exact source revision; all-green host CI is not physical qualification. `.github/workflows/host-gates.yml` runs `make -j4`, `make test-native-artifact`, `make zero-cuda-gate`, `build/hello_gpu --sim`, read-only `make doctor`, and `python3 tools/figures/make_figures.py --check`. `make test` stays in the `host-tests` job. The rules are `docs/CI_ACCEPTANCE.md`. ## Safety and approvals diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c17f0f4..2c45c16c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,8 @@ build/hello_gpu --sim # simulator, not a hardware result make doctor # diagnostic; may report NOT READY on other hosts ``` +Pull requests run those host commands. The acceptance rules are [docs/CI_ACCEPTANCE.md](docs/CI_ACCEPTANCE.md). A green check is a host or simulator result for that commit. It is not a GB10 result. + Only on a qualified GB10 when it is safe to use the GPU (never interrupt a live service): ```sh diff --git a/docs/CI_ACCEPTANCE.md b/docs/CI_ACCEPTANCE.md new file mode 100644 index 00000000..145a7129 --- /dev/null +++ b/docs/CI_ACCEPTANCE.md @@ -0,0 +1,50 @@ +# CI acceptance + +A pull-request check is a result for the commit GitHub built. A green host check is not a GB10 qualification. Commands that did not run are `NOT_RUN`. + +The workflow is `.github/workflows/host-gates.yml`. It composes with the jobs that already exist: + +- `make test` is the `host-tests` job in `.github/workflows/ci.yml`. That job judges the process exit code and the last line, `CUDA++ HOST TESTS PASS ( targets)`. The log keeps intentional `FAIL` and `ERROR` lines from mutants and negative controls. This policy does not start a second copy of that suite. +- `make test-runtime-api` is `.github/workflows/runtime-api.yml`. Issue #189 owns any further change to that job. +- x86-64 `make test` is the `x86-host-tests` job in `.github/workflows/host-x86.yml`. This policy does not add that job. + +## What each gate is + +| Command | Where it runs | What a pass is | +| --- | --- | --- | +| `make -j4` | `host-gates` / `fresh-checkout`, `ubuntu-24.04-arm`, 30 minute limit | Exit 0. `build/libomega_gpu.a` and `build/libcudapp.a` exist. Their sha256 is uploaded. | +| `make test` | `ci.yml` / `host-tests`, same arm runner | Exit 0 and the final `CUDA++ HOST TESTS PASS` line. Needs the pinned `nvdisasm` 13.0.85, full git history, and the pinned Omega tools named in that workflow. | +| `make test-native-artifact` | `host-gates` / `fresh-checkout` | Exit 0 and a last summary `test_gpu_artifact_export: checks, 0 failed`. | +| `make zero-cuda-gate` | `host-gates` / `fresh-checkout` | Exit 0. The log still shows `ZERO_CUDA FAIL` for the stub `libcudart` probe, then `ZERO_CUDA SELF-TEST PASS`, then `ZERO_CUDA PASS` for `build/hello_gpu` and `build/gpu_matmul_api_test`. | +| `build/hello_gpu --sim` | `host-gates` / `fresh-checkout` | Exit 0, the host-simulator sentence, the 1048576-element check, the work line, the pinned output sha256, and the same hashes on a second run. This is the warp simulator. | +| `make doctor` | `host-gates` / `fresh-checkout` | The runner has no NVIDIA module and no GPU node, so `make` exits 2 and the log contains `NOT READY:` plus the five refusal lines and `NOT_RUN` for the seeded target table. A line `READY. Next: make hello-gpu` fails the job. Chip qualification stays `NOT_RUN`. | +| `python3 tools/figures/make_figures.py --check` | `host-gates` / `figures`, `ubuntu-latest`, 5 minute limit | Exit 0 and `figures current: `. `docs/img/stack.svg` is one of those files. | + +`build/hello_gpu --sim --bug=past-end` must exit 1, print the `FAULT` line, and name `0 bytes past the end of buffer a`. That path does not print `PASS`. `build/hello_gpu --bug=past-end` must exit 2 and refuse to run off the simulator. Those two are negative controls. `make zero-cuda-gate` also prints `ZERO_CUDA FAIL` for its stub probe. That text stays in the log. The job is judged by its exit code and its last line, `CUDA++ HOST GATES PASS`, the same way `make test` is judged. + +The checker itself has a negative control: `tools/ci/host-gates.sh --self-test`. A log that contains `ZERO_CUDA FAIL` and still exits 0 must be accepted. A doctor log that says `NOT READY` must be accepted. A doctor log that also says `READY. Next: make hello-gpu` must be rejected. A figure log that says `stale figures` must be rejected. + +## Classes + +- **Host-only.** `make -j4`, `make test`, `make test-native-artifact`, `make zero-cuda-gate`. No device node is opened. +- **Simulator-only.** `build/hello_gpu --sim`. A simulator pass is not hardware qualification and not a performance result. +- **Target-specific compile.** AArch64 Linux is the native host. Production ordering there stays `dsb sy` (`nvrm/cpu_barrier.h`). On x86-64 the portable suite is the one in `docs/OMEGA_HOST_TESTS.md`: `make test` with gcc, no NVIDIA tool and no sibling checkout. The x86-64 device open returns `ENOTSUP` and the text `requires AArch64 Linux`. That is `NOT SUPPORTED` for a GPU run. The receipt `evidence/HOST-X86-20261030/` is for its own commit. On that VM the default `CUDAPP_MEM_RESERVE_MIB` of 4096 made `make test` exit 2, and the suite passed only with `CUDAPP_MEM_RESERVE_MIB=0`. The default is unchanged. This workflow does not set the reserve and does not run the x86-64 suite. +- **GPU-only.** `make hello-gpu` and `./build/gpu_*_test` on a GB10. GitHub runners have no GB10 and no `/dev/aien-gpu`. Those commands are `NOT_RUN`. A missing GPU run is never recorded as a pass. + +## Figures + +`tools/figures/make_figures.py` is the source of the README pictures. `--check` compares its bytes with `docs/img/`. On `522ee0d` it printed `figures current: 10`, so `docs/img/stack.svg` already matched the generator. A later change that moves the generator must commit the regenerated SVG in the same change. A hand edit that makes the SVG disagree with the generator fails the check. The rendered legend test (`tools/figures/test_stack_legend.py`) needs Inkscape. The hosted figure job does not install Inkscape, so that test is `NOT_RUN` there. When Inkscape is on `PATH`, the same job runs it. + +## Toolchain + +`fresh-checkout` uses the `ubuntu-24.04-arm` image's `gcc`, `make` and `python3`. It installs no packages and no CUDA toolkit. `make -j4` is the compile budget. The job records `git rev-parse HEAD`, `uname`, the compiler line and the archive sha256 in the `host-gates-` artifact. The figure job uploads `host-gates-figures-`. No vendor binary, model weight or home-directory path is committed. + +`make test` on the arm runner still installs the checksum-verified `nvdisasm` named in `ci.yml`. That oracle is verification-only. The host-gates job does not need it. + +## What a green pull request does not prove + +The active ruleset `main-verified-promotion` blocks deletion and non-fast-forward pushes. It does not list required status checks. The workflows start on every pull request. Marking a check required is an administrator change and is `NOT_RUN` here. + +An outside engineer following `docs/START_HERE.md` on a machine this repository does not control is `NOT_RUN`. + +`make hello-gpu`, every chip test, and any open of `/dev/nvidia*` or `/dev/aien-gpu` are `NOT_RUN` on these runners. diff --git a/docs/START_HERE.md b/docs/START_HERE.md index 24ad2112..3a51f3c3 100644 --- a/docs/START_HERE.md +++ b/docs/START_HERE.md @@ -90,9 +90,11 @@ maps and receipts remain unchanged. The obsolete [consumer PR #618](https://github.com/aien-dev/omega-mv/pull/618) used an older pin and different source-check contract; it is not the canonical cutover base. -On x86-64, `make test` is the portable host suite. Production `dsb sy` stays on AArch64 (`nvrm/cpu_barrier.h`). The x86-64 publication barrier is a real sequentially consistent fence. A target-only check that names its requirement and exits 2 is not a pass of that check. The two lists are in [docs/OMEGA_HOST_TESTS.md](OMEGA_HOST_TESTS.md) ([#15](https://github.com/aien-dev/CUDApp/issues/15)). +On x86-64, `make test` is the portable host suite. Production `dsb sy` stays on AArch64 (`nvrm/cpu_barrier.h`). The x86-64 publication barrier is a real sequentially consistent fence. A target-only check that names its requirement and exits 2 is not a pass of that check. The two lists are in [docs/OMEGA_HOST_TESTS.md](OMEGA_HOST_TESTS.md) ([#15](https://github.com/aien-dev/CUDApp/issues/15)). The x86-64 device path returns `ENOTSUP` and the text `requires AArch64 Linux`. That path is not a supported GPU run. -Optional: `python3 tools/figures/make_figures.py --check` verifies that README diagrams match the generator. Check it on your exact commit. +`python3 tools/figures/make_figures.py --check` verifies that the README diagrams, including `docs/img/stack.svg`, match `tools/figures/make_figures.py`. It must print `figures current:` and exit 0. Pull requests run that check. When the generator changes, commit the regenerated SVG in the same change. A diagram that disagrees with the generator fails the check. + +Pull requests also run `make -j4`, `make test-native-artifact`, `make zero-cuda-gate`, `build/hello_gpu --sim` and read-only `make doctor` on `ubuntu-24.04-arm`. `make test` runs as the `host-tests` job. None of those jobs open a GPU. What a green check means is [docs/CI_ACCEPTANCE.md](CI_ACCEPTANCE.md). ## 3. Diagnose a native GPU machine diff --git a/tools/ci/host-gates.sh b/tools/ci/host-gates.sh new file mode 100755 index 00000000..a92b2392 --- /dev/null +++ b/tools/ci/host-gates.sh @@ -0,0 +1,539 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# Fresh-checkout host gates. Reads no GPU device node and does not run make hello-gpu. +# +# tools/ci/host-gates.sh --self-test checker negative controls (no compile) +# tools/ci/host-gates.sh --figures README figure byte check +# tools/ci/host-gates.sh make -j4 and the host commands below +# +# make test is the host-tests job in .github/workflows/ci.yml. This script does not repeat it. +# make test-runtime-api is .github/workflows/runtime-api.yml (#189). An x86-64 host-tests +# job is #196. Neither is started here. +set -euo pipefail +export LC_ALL=C + +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +cd "$root" +out=build/host-gates +mkdir -p "$out" + +# Simulator output of c = a + b over 2^20 floats. Measured by build/hello_gpu --sim. +# Pinned so a silent answer change fails even when the program still prints PASS. +output_sha='output sha256 72d0c1b2928779e83518f0b4c7d101d933cf89f210cd4627e138c33c9ce9b6a6' + +fail=0 +summary=$out/summary.txt + +note() { printf '%s\n' "$*" | tee -a "$summary"; } + +run_log() { + local log=$1 + shift + set +e + "$@" >"$log" 2>&1 + local rc=$? + set -e + cat "$log" + printf '%s\n' "$rc" >"$log.rc" +} + +rc_of() { cat "$1.rc"; } + +archives_ok() { + local rc=$1 dir=$2 + [ "$rc" -eq 0 ] || return 1 + [ -f "$dir/libomega_gpu.a" ] && [ -f "$dir/libcudapp.a" ] +} + +artifact_ok() { + local rc=$1 log=$2 line + [ "$rc" -eq 0 ] || return 1 + line=$(grep -E '^test_gpu_artifact_export: ' "$log" | tail -n 1 || true) + [[ "$line" =~ ^test_gpu_artifact_export:\ [0-9]+\ checks,\ 0\ failed$ ]] +} + +zero_cuda_ok() { + local rc=$1 log=$2 + [ "$rc" -eq 0 ] || return 1 + grep -qF 'ZERO_CUDA SELF-TEST PASS (CUDA probe rejected, clean library accepted)' "$log" || return 1 + grep -qF 'ZERO_CUDA FAIL' "$log" || return 1 + grep -qF 'libcudart.so' "$log" || return 1 + grep -qF 'cudaMalloc' "$log" || return 1 + grep -qF 'ZERO_CUDA PASS build/hello_gpu needs=' "$log" || return 1 + grep -qF 'ZERO_CUDA PASS build/gpu_matmul_api_test needs=' "$log" || return 1 + grep -qF 'ZERO_CUDA SELF-TEST BROKEN' "$log" && return 1 + return 0 +} + +sim_ok() { + local rc=$1 log=$2 + [ "$rc" -eq 0 ] || return 1 + grep -qF 'on the host simulator (no GPU)' "$log" || return 1 + grep -qF 'checked 1048576 of 1048576 elements: 0 wrong, 0 never written' "$log" || return 1 + grep -qF 'work: 12582912 bytes moved (8388608 read, 4194304 written), 1048576 floating-point operations' "$log" || return 1 + grep -qF "$output_sha" "$log" || return 1 + grep -qxF 'PASS' "$log" || return 1 + grep -qE '^machine code sha256 [0-9a-f]{64}$' "$log" || return 1 +} + +# The fault path exits 1 from inside the launch loop. It prints FAULT and does not print PASS. +past_end_ok() { + local rc=$1 log=$2 + [ "$rc" -eq 1 ] || return 1 + grep -qF 'FAULT at launch 8 of 8 of kernel vecadd (simulator): global load outside every buffer' "$log" || return 1 + grep -qF '0 bytes past the end of buffer a' "$log" || return 1 + grep -qF '0 wrong, 0 never written' "$log" && return 1 + grep -qxF 'PASS' "$log" && return 1 + return 0 +} + +gpu_bug_refused_ok() { + local rc=$1 log=$2 + [ "$rc" -eq 2 ] || return 1 + grep -qF -e '--bug=past-end is only for --sim: on the GPU it would write past a buffer' "$log" || return 1 +} + +# make's exit status is the decision. Exit 2 is the recipe refusal. Every other exit fails. +# The words "Error 1" in make's message are not part of the check. +# Every line matching ^[[:space:]]*FAIL([[:space:]]|:|$) is a problem line. +# It must equal one of the five missing-hardware lines, whole line, and it counts toward the five. +# A longer line, a prefix, or any other FAIL line fails. There must be exactly those five. +# The summary line must start with "NOT READY: 5 problems above." +doctor_fail_line() { + case $1 in + ' FAIL NVIDIA kernel module: not loaded (/proc/driver/nvidia/version does not exist)') + printf '%s\n' module ;; + ' FAIL device files: /dev/nvidiactl is missing') + printf '%s\n' ctl ;; + ' FAIL device files: /dev/nvidia0 is missing') + printf '%s\n' gpu0 ;; + ' FAIL device files: /dev/nvidia-uvm is missing') + printf '%s\n' uvm ;; + ' FAIL GPU: no NVIDIA GPU found in /sys/bus/pci/devices') + printf '%s\n' gpu ;; + *) + return 1 + ;; + esac +} + +doctor_ok() { + local rc=$1 log=$2 line kind n=0 module=0 ctl=0 gpu0=0 uvm=0 gpu=0 + local ready_n any_ready + [ "$rc" -eq 2 ] || return 1 + grep -qF 'cudapp doctor: is this machine ready for CUDA++?' "$log" || return 1 + grep -qF 'NOT_RUN: the target table is seeded Omega' "$log" || return 1 + if grep -qxF 'READY. Next: make hello-gpu' "$log"; then + return 1 + fi + while IFS= read -r line || [ -n "$line" ]; do + if [[ "$line" =~ ^[[:space:]]*FAIL([[:space:]]|:|$) ]]; then + kind=$(doctor_fail_line "$line") || return 1 + n=$((n + 1)) + case $kind in + module) module=$((module + 1)) ;; + ctl) ctl=$((ctl + 1)) ;; + gpu0) gpu0=$((gpu0 + 1)) ;; + uvm) uvm=$((uvm + 1)) ;; + gpu) gpu=$((gpu + 1)) ;; + *) return 1 ;; + esac + fi + done <"$log" + [ "$n" -eq 5 ] || return 1 + [ "$module" -eq 1 ] && [ "$ctl" -eq 1 ] && [ "$gpu0" -eq 1 ] && [ "$uvm" -eq 1 ] && [ "$gpu" -eq 1 ] || return 1 + ready_n=$(grep -cE '^NOT READY: 5 problems above\.' "$log" || true) + any_ready=$(grep -cE '^NOT READY:' "$log" || true) + [ "$ready_n" -eq 1 ] || return 1 + [ "$any_ready" -eq 1 ] || return 1 + return 0 +} + +# RUN when the binary can be executed. FAIL when make -j4 exited 0 and it is missing. +# NOT_RUN only when that build already failed. +hello_sim_rows() { + local make_rc=$1 hello=$2 + if [ -x "$hello" ]; then + printf '%s\n' RUN + return 0 + fi + if [ "$make_rc" -eq 0 ]; then + printf '%s\n' FAIL + return 0 + fi + printf '%s\n' NOT_RUN +} + +figures_ok() { + local rc=$1 log=$2 + [ "$rc" -eq 0 ] || return 1 + grep -qE '^figures current: [0-9]+$' "$log" || return 1 + grep -qF 'stale figures' "$log" && return 1 + return 0 +} + +expect_ok() { + local name=$1 + shift + if "$@"; then + echo "PASS $name" + return 0 + fi + echo "FAIL $name" + fail=1 + return 0 +} + +expect_reject() { + local name=$1 + shift + if "$@"; then + echo "FAIL $name (checker accepted a case that must be rejected)" + fail=1 + return 0 + fi + echo "PASS $name" +} + +self_test() { + local t log + t=$(mktemp -d) + log=$t/log + + mkdir -p "$t/both" + : >"$t/both/libomega_gpu.a" + : >"$t/both/libcudapp.a" + expect_ok archives-both archives_ok 0 "$t/both" + expect_reject archives-missing archives_ok 0 "$t" + expect_reject archives-bad-exit archives_ok 2 "$t/both" + + printf '%s\n' 'test_gpu_artifact_export: 47 checks, 0 failed' >"$log" + expect_ok artifact-clean artifact_ok 0 "$log" + printf '%s\n' 'FAIL tamper caught' 'test_gpu_artifact_export: 47 checks, 0 failed' >"$log" + expect_ok artifact-fail-word-kept artifact_ok 0 "$log" + printf '%s\n' 'test_gpu_artifact_export: 47 checks, 1 failed' >"$log" + expect_reject artifact-failed-count artifact_ok 0 "$log" + printf '%s\n' 'test_gpu_artifact_export: 47 checks, 0 failed' >"$log" + expect_reject artifact-bad-exit artifact_ok 1 "$log" + + cat >"$log" <<'EOF' +ZERO_CUDA FAIL /tmp/probe.so needs: libcudart.so.13 +ZERO_CUDA FAIL /tmp/probe.so imports: cudaMalloc +ZERO_CUDA SELF-TEST PASS (CUDA probe rejected, clean library accepted) +ZERO_CUDA PASS build/hello_gpu needs=[libm.so.6 libc.so.6] +ZERO_CUDA PASS build/gpu_matmul_api_test needs=[libc.so.6] +EOF + expect_ok zero-cuda-with-visible-fail zero_cuda_ok 0 "$log" + expect_reject zero-cuda-bad-exit zero_cuda_ok 1 "$log" + printf '%s\n' 'ZERO_CUDA PASS build/hello_gpu needs=[libc.so.6]' >"$log" + expect_reject zero-cuda-hidden-control zero_cuda_ok 0 "$log" + + cat >"$log" <"$log" + expect_reject sim-pass-only sim_ok 0 "$log" + cat >"$log" <"$log" <<'EOF' +FAULT at launch 8 of 8 of kernel vecadd (simulator): global load outside every buffer + address: 0x1000 (4 bytes), 0 bytes past the end of buffer a = [0x0, 0x1000) (4194304 bytes) +EOF + expect_ok past-end past_end_ok 1 "$log" + printf '%s\n' \ + 'FAULT at launch 8 of 8 of kernel vecadd (simulator): global load outside every buffer' \ + '0 bytes past the end of buffer a' \ + 'checked 1048576 of 1048576 elements: 0 wrong, 0 never written' \ + 'PASS' >"$log" + expect_reject past-end-continued past_end_ok 1 "$log" + expect_reject past-end-exit0 past_end_ok 0 "$log" + + printf '%s\n' '--bug=past-end is only for --sim: on the GPU it would write past a buffer' >"$log" + expect_ok gpu-bug-refused gpu_bug_refused_ok 2 "$log" + expect_reject gpu-bug-exit0 gpu_bug_refused_ok 0 "$log" + + cat >"$log" <<'EOF' +cudapp doctor: is this machine ready for CUDA++? + FAIL NVIDIA kernel module: not loaded (/proc/driver/nvidia/version does not exist) + FAIL device files: /dev/nvidiactl is missing + FAIL device files: /dev/nvidia0 is missing + FAIL device files: /dev/nvidia-uvm is missing + FAIL GPU: no NVIDIA GPU found in /sys/bus/pci/devices +NOT READY: 5 problems above. Fix the first FAIL line and run make doctor again. + NOT_RUN: the target table is seeded Omega and needs the pinned omc. +EOF + cp "$log" "$t/doctor-good" + expect_ok doctor-not-ready doctor_ok 2 "$log" + expect_reject doctor-exit0 doctor_ok 0 "$log" + printf '%s\n' 'make: *** [mk/dx-doctor.mk:14: doctor] Error 1' >>"$log" + expect_reject doctor-error-text-exit0 doctor_ok 0 "$log" + printf '%s\n' 'READY. Next: make hello-gpu' >>"$log" + expect_reject doctor-ready-line doctor_ok 2 "$log" + grep -qF 'NOT READY:' "$log" || { echo "FAIL doctor fixture lost NOT READY"; fail=1; } + + cp "$t/doctor-good" "$t/doctor-extra" + printf '%s\n' ' FAIL device files: /dev/nvidia0 exists but this user cannot open it for reading and writing' >>"$t/doctor-extra" + expect_reject doctor-extra-fail doctor_ok 2 "$t/doctor-extra" + sed 's/^NOT READY: 5 problems above\./NOT READY: 6 problems above./' "$t/doctor-good" >"$t/doctor-count6" + expect_reject doctor-count-6 doctor_ok 2 "$t/doctor-count6" + grep -vxF ' FAIL device files: /dev/nvidia-uvm is missing' "$t/doctor-good" >"$t/doctor-missing" + expect_reject doctor-missing-allowlisted-line doctor_ok 2 "$t/doctor-missing" + expect_reject doctor-exit1 doctor_ok 1 "$t/doctor-good" + expect_reject doctor-exit3 doctor_ok 3 "$t/doctor-good" + cp "$t/doctor-good" "$t/doctor-noindent" + printf '%s\n' 'FAIL x' >>"$t/doctor-noindent" + expect_reject doctor-fail-noindent doctor_ok 2 "$t/doctor-noindent" + cp "$t/doctor-good" "$t/doctor-tab" + printf '\tFAIL x\n' >>"$t/doctor-tab" + expect_reject doctor-fail-tab doctor_ok 2 "$t/doctor-tab" + + saved_fail=$fail + saved_summary=$summary + fail=0 + summary=$t/missing-hello-summary.txt + : >"$summary" + record_hello_or_fail 0 "$t/no-such-hello" || true + rows=$(grep -c ' FAIL ' "$summary" || true) + if [ "$fail" -eq 1 ] && [ "$rows" -eq 3 ]; then + echo "PASS missing-hello-after-make" + else + echo "FAIL missing-hello-after-make (fail=$fail rows=$rows)" + saved_fail=1 + fi + fail=0 + : >"$summary" + : >"$t/hello_gpu" + chmod +x "$t/hello_gpu" + if record_hello_or_fail 0 "$t/hello_gpu"; then + echo "PASS hello-present-is-run" + else + echo "FAIL hello-present-is-run" + saved_fail=1 + fi + fail=0 + : >"$summary" + record_hello_or_fail 2 "$t/no-such-hello" || true + rows=$(grep -c ' NOT_RUN ' "$summary" || true) + bad_rows=$(grep -c ' FAIL ' "$summary" || true) + if [ "$fail" -eq 0 ] && [ "$rows" -eq 3 ] && [ "$bad_rows" -eq 0 ]; then + echo "PASS missing-hello-after-failed-make" + else + echo "FAIL missing-hello-after-failed-make (fail=$fail not_run=$rows fail_rows=$bad_rows)" + saved_fail=1 + fi + fail=$saved_fail + summary=$saved_summary + + printf '%s\n' 'figures current: 10' >"$log" + expect_ok figures-current figures_ok 0 "$log" + printf '%s\n' 'stale figures (run python3 tools/figures/make_figures.py): stack.svg' >"$log" + expect_reject figures-stale figures_ok 1 "$log" + printf '%s\n' 'figures current: 10' >"$log" + expect_reject figures-bad-exit figures_ok 1 "$log" + + rm -rf "$t" + if [ "$fail" -ne 0 ]; then + echo "CUDA++ HOST GATES SELF-TEST FAIL" | tee "$out/self-test.txt" + exit 1 + fi + echo "CUDA++ HOST GATES SELF-TEST PASS" | tee "$out/self-test.txt" + exit 0 +} + +finish() { + echo "----- summary -----" + cat "$summary" + if [ "$fail" -ne 0 ]; then + echo "CUDA++ HOST GATES FAIL" + exit 1 + fi + echo "CUDA++ HOST GATES PASS" + exit 0 +} + +record() { + # name, exit, result, detail + note "$(printf '%s\texit=%s\t%s\t%s' "$1" "$2" "$3" "$4")" + if [ "$3" = FAIL ]; then + fail=1 + fi +} + +# Returns 0 only when the three simulator commands should run. +# A successful make -j4 with no build/hello_gpu records three FAILs. +record_hello_or_fail() { + local make_rc=$1 hello=$2 how + how=$(hello_sim_rows "$make_rc" "$hello") + if [ "$how" = RUN ]; then + return 0 + fi + if [ "$how" = FAIL ]; then + record 'build/hello_gpu --sim' '-' FAIL 'make -j4 exited 0 and build/hello_gpu is missing' + record 'build/hello_gpu --sim --bug=past-end' '-' FAIL 'make -j4 exited 0 and build/hello_gpu is missing' + record 'build/hello_gpu --bug=past-end' '-' FAIL 'make -j4 exited 0 and build/hello_gpu is missing' + return 1 + fi + record 'build/hello_gpu --sim' '-' NOT_RUN 'blocked: build/hello_gpu was not built' + record 'build/hello_gpu --sim --bug=past-end' '-' NOT_RUN 'blocked: build/hello_gpu was not built' + record 'build/hello_gpu --bug=past-end' '-' NOT_RUN 'blocked: build/hello_gpu was not built' + return 1 +} + +figures_mode() { + local log=$out/figures.log rc + : >"$summary" + run_log "$log" python3 tools/figures/make_figures.py --check + rc=$(rc_of "$log") + if figures_ok "$rc" "$log"; then + record 'python3 tools/figures/make_figures.py --check' "$rc" PASS "$(grep -E '^figures current: ' "$log")" + else + record 'python3 tools/figures/make_figures.py --check' "$rc" FAIL 'checker rejected the figure output' + fi + if command -v inkscape >/dev/null 2>&1; then + local legend=$out/stack-legend.log + run_log "$legend" python3 tools/figures/test_stack_legend.py -v + rc=$(rc_of "$legend") + if [ "$rc" -eq 0 ]; then + record 'python3 tools/figures/test_stack_legend.py' "$rc" PASS 'rendered legend' + else + record 'python3 tools/figures/test_stack_legend.py' "$rc" FAIL 'legend test failed' + fi + else + record 'python3 tools/figures/test_stack_legend.py' '-' NOT_RUN 'inkscape is not installed' + fi + finish +} + +identity() { + { + echo "source_sha $(git rev-parse HEAD)" + if [ -n "${HOST_GATES_HEAD_SHA:-}" ]; then + echo "head_sha ${HOST_GATES_HEAD_SHA}" + fi + echo "uname $(uname -s -m -r)" + if [ -r /etc/os-release ]; then + # shellcheck disable=SC1091 + . /etc/os-release + echo "os ${PRETTY_NAME:-unknown}" + fi + echo "gcc $(gcc --version | head -n 1)" + echo "make $(make --version | head -n 1)" + echo "python3 $(python3 --version 2>&1)" + echo "jobs 4" + echo "utc $(date -u +%Y-%m-%dT%H:%M:%SZ)" + } | tee "$out/identity.txt" +} + +gates() { + local log rc make_rc + : >"$summary" + identity + note 'make test exit=- ENFORCED_ELSEWHERE CI job host-tests in .github/workflows/ci.yml' + note 'make test-runtime-api exit=- ENFORCED_ELSEWHERE .github/workflows/runtime-api.yml (#189)' + note 'x86-64 make test exit=- ENFORCED_ELSEWHERE job x86-host-tests in .github/workflows/host-x86.yml' + note 'make hello-gpu exit=- NOT_RUN no GB10 and no /dev/aien-gpu on this runner' + + log=$out/make-j4.log + run_log "$log" make -j4 + make_rc=$(rc_of "$log") + rc=$make_rc + if archives_ok "$rc" build; then + sha256sum build/libomega_gpu.a build/libcudapp.a | tee "$out/archives.sha256" + record 'make -j4' "$rc" PASS "$(sed -n '1,2p' "$out/archives.sha256" | tr '\n' ' ')" + else + record 'make -j4' "$rc" FAIL 'library archives were not produced' + fi + + if [ -f build/libomega_gpu.a ]; then + log=$out/native-artifact.log + run_log "$log" make -j4 test-native-artifact + rc=$(rc_of "$log") + if artifact_ok "$rc" "$log"; then + record 'make test-native-artifact' "$rc" PASS "$(grep -E '^test_gpu_artifact_export: ' "$log" | tail -n 1)" + else + record 'make test-native-artifact' "$rc" FAIL 'exit code or summary line rejected' + fi + + log=$out/zero-cuda.log + run_log "$log" make -j4 zero-cuda-gate + rc=$(rc_of "$log") + if zero_cuda_ok "$rc" "$log"; then + record 'make zero-cuda-gate' "$rc" PASS 'self-test FAIL lines kept; binaries PASS' + else + record 'make zero-cuda-gate' "$rc" FAIL 'exit code or ZERO_CUDA lines rejected' + fi + else + record 'make test-native-artifact' '-' NOT_RUN 'blocked: make -j4 did not produce the library' + record 'make zero-cuda-gate' '-' NOT_RUN 'blocked: make -j4 did not produce the library' + fi + + if record_hello_or_fail "$make_rc" build/hello_gpu; then + log=$out/hello-sim.log + run_log "$log" build/hello_gpu --sim + rc=$(rc_of "$log") + local log2=$out/hello-sim-2.log + run_log "$log2" build/hello_gpu --sim + local rc2 + rc2=$(rc_of "$log2") + local h1 h2 + h1=$(grep -E '^(output|machine code) sha256 ' "$log" || true) + h2=$(grep -E '^(output|machine code) sha256 ' "$log2" || true) + if sim_ok "$rc" "$log" && sim_ok "$rc2" "$log2" && [ -n "$h1" ] && [ "$h1" = "$h2" ]; then + record 'build/hello_gpu --sim' "$rc" PASS 'two runs, same output and machine-code hashes' + else + record 'build/hello_gpu --sim' "$rc" FAIL "repeat exit $rc2; hashes differ or checks missed" + fi + + log=$out/hello-past-end.log + run_log "$log" build/hello_gpu --sim --bug=past-end + rc=$(rc_of "$log") + if past_end_ok "$rc" "$log"; then + record 'build/hello_gpu --sim --bug=past-end' "$rc" PASS 'negative control: FAULT, exit 1, no PASS' + else + record 'build/hello_gpu --sim --bug=past-end' "$rc" FAIL 'negative control did not fail closed' + fi + + log=$out/hello-bug-refused.log + run_log "$log" build/hello_gpu --bug=past-end + rc=$(rc_of "$log") + if gpu_bug_refused_ok "$rc" "$log"; then + record 'build/hello_gpu --bug=past-end' "$rc" PASS 'refused off the simulator' + else + record 'build/hello_gpu --bug=past-end' "$rc" FAIL 'GPU bug path was not refused' + fi + fi + + log=$out/doctor.log + run_log "$log" make -j4 doctor + rc=$(rc_of "$log") + if doctor_ok "$rc" "$log"; then + record 'make doctor' "$rc" PASS 'NOT READY on this runner; chip qualification NOT_RUN' + else + record 'make doctor' "$rc" FAIL 'expected the no-GPU refusal, not READY' + fi + finish +} + +case "${1:-}" in +--self-test) self_test ;; +--figures) figures_mode ;; +'') gates ;; +*) + echo "usage: $0 [--self-test|--figures]" >&2 + exit 2 + ;; +esac