From 482611a44319691ed5b0a480213548b46717f0f6 Mon Sep 17 00:00:00 2001 From: Martin Kuntz Jacobsen Date: Mon, 9 Nov 2026 16:59:46 +0100 Subject: [PATCH] ci: compiler probe tests fail normally; scan ci.yml for swallowed failures (part of #10060) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Xut2hwESun1HL2eSVd1d5B --- .github/workflows/ci.yml | 72 +++++++++++++------ tools/ci/aggregate_gates_test.rs | 119 +++++++++++++++++++++++++++++-- tools/ci/ci_workflow/src/lib.rs | 31 ++++++++ 3 files changed, 197 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11271c7f9e..6e692f3646 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,7 +53,6 @@ jobs: with: cache-save: ${{ github.event_name != 'pull_request' }} - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - continue-on-error: true with: pattern: dx-reports-coverage-* path: ${{ runner.temp }}/coverage-reports @@ -183,28 +182,45 @@ jobs: while read -r label; do pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}" mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name" - cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true + [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" done < "$RUNNER_TEMP/hermetic-targets" + - name: Run the compiler probe tests + id: compiler-probes + if: ${{ !cancelled() }} + run: | + # shellcheck disable=SC2086 + set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS + bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin + facts="$(bazel info "$@" bazel-bin)/tools/docs/support_matrix/facts.json" + jq -r '.rows[] | select(.section == "compilers") | .identity.probe, .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ' > "$RUNNER_TEMP/compiler-targets" + targets="$(cat "$RUNNER_TEMP/compiler-targets")" + # shellcheck disable=SC2086 + bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets + status=0 + # shellcheck disable=SC2086 + bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets && status=0 || status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then + exit "$status" + fi - name: Record compiler evidence if: always() run: | + if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then + echo "::notice::The compiler probe tests did not run; no compiler evidence recorded." + exit 0 + fi # shellcheck disable=SC2086 set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS out="$RUNNER_TEMP/compiler-evidence/linux_x86_64.jsonl" mkdir -p "$(dirname "$out")" : > "$out" - bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin DX_BIN="$(bazel info "$@" bazel-bin)" DX_TESTLOGS="$(bazel info "$@" bazel-testlogs)" DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin" [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe" export DX_BIN DX_TESTLOGS DX_PRODUCERS facts="${DX_BIN}/tools/docs/support_matrix/facts.json" - targets="$(jq -r '.rows[] | select(.section == "compilers") | .identity.probe, .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ')" - # shellcheck disable=SC2086 - bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets || true - # shellcheck disable=SC2086 - bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets || true + targets="$(cat "$RUNNER_TEMP/compiler-targets")" DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets" export DX_EVIDENCE_COMMAND tools/ci/compiler_evidence.sh "$out" linux_x86_64 "$facts" @@ -286,7 +302,7 @@ jobs: while read -r label; do pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}" mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name" - cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true + [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" done < "$RUNNER_TEMP/hermetic-targets" - name: Check the upload holds no secret id: scan-hermetic-evidence @@ -337,7 +353,7 @@ jobs: while read -r label; do pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}" mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name" - cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true + [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" done < "$RUNNER_TEMP/hermetic-targets" - name: Check the upload holds no secret id: scan-hermetic-evidence @@ -410,7 +426,7 @@ jobs: while read -r label; do pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}" mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name" - cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true + [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" done < "$RUNNER_TEMP/hermetic-targets" # The test copies its manifest into its outputs; bazel-bin on Windows does not hold it. - name: Verify the evidence @@ -629,28 +645,47 @@ jobs: echo "::error::Every starter consumer test was skipped on ${PLATFORM}, shard ${SHARD}." exit 1 fi + - name: Run the compiler probe tests + id: compiler-probes + if: ${{ !cancelled() && matrix.shard == 0 }} + env: + PLATFORM: ${{ matrix.platform }} + run: | + # shellcheck disable=SC2086 + set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS + bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin + facts="$PWD/bazel-bin/tools/docs/support_matrix/facts.json" + jq -r --arg platform "$PLATFORM" '.rows[] | select(.section == "compilers") | (.identity.platform_probes[$platform] // .identity.probe), .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ' > "$RUNNER_TEMP/compiler-targets" + targets="$(cat "$RUNNER_TEMP/compiler-targets")" + # shellcheck disable=SC2086 + bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets + status=0 + # shellcheck disable=SC2086 + bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets && status=0 || status=$? + if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then + exit "$status" + fi - name: Record compiler evidence if: always() && matrix.shard == 0 env: PLATFORM: ${{ matrix.platform }} run: | + if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then + echo "::notice::The compiler probe tests did not run; no compiler evidence recorded." + exit 0 + fi # shellcheck disable=SC2086 set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS out="$RUNNER_TEMP/compiler-evidence/starter-${PLATFORM}.jsonl" mkdir -p "$(dirname "$out")" : > "$out" - bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin DX_BIN="$PWD/bazel-bin" DX_TESTLOGS="$PWD/bazel-testlogs" DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin" [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe" export DX_BIN DX_TESTLOGS DX_PRODUCERS facts="${DX_BIN}/tools/docs/support_matrix/facts.json" - targets="$(jq -r --arg platform "$PLATFORM" '.rows[] | select(.section == "compilers") | (.identity.platform_probes[$platform] // .identity.probe), .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ')" - # shellcheck disable=SC2086 - bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets || true - # shellcheck disable=SC2086 - bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets || true + targets="$(cat "$RUNNER_TEMP/compiler-targets")" DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets" export DX_EVIDENCE_COMMAND tools/ci/compiler_evidence.sh "$out" "$PLATFORM" "$facts" @@ -1026,14 +1061,12 @@ jobs: persist-credentials: false - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 if: always() - continue-on-error: true with: pattern: compiler-evidence-* merge-multiple: true path: ${{ runner.temp }}/compiler-evidence - name: Merge compiler evidence if: always() - continue-on-error: true run: | mkdir -p "$RUNNER_TEMP/compiler-evidence" { @@ -1048,7 +1081,6 @@ jobs: path: ${{ runner.temp }}/compiler-evidence-all.jsonl - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: ${{ always() && steps.scan-compiler-evidence.outcome == 'success' }} - continue-on-error: true with: name: compiler-evidence-all path: ${{ runner.temp }}/compiler-evidence-all.jsonl diff --git a/tools/ci/aggregate_gates_test.rs b/tools/ci/aggregate_gates_test.rs index b146ae8ce5..498a12f99a 100644 --- a/tools/ci/aggregate_gates_test.rs +++ b/tools/ci/aggregate_gates_test.rs @@ -1,6 +1,6 @@ use serde_json::{Value, json}; -use ci_workflow::{document, jobs, steps, triggers}; +use ci_workflow::{document, jobs, steps, swallows, triggers}; use dx_testing::read_runfiles; const CI_WORKFLOW: &str = ".github/workflows/ci.yml"; @@ -268,10 +268,7 @@ fn the_native_matrix_keeps_all_five_platforms() { #[test] fn every_gate_workflow_runs_on_merge_group_checks_requested() { - for workflow in [ - CI_WORKFLOW, - "examples/docs-ci/caller.yml", - ] { + for workflow in [CI_WORKFLOW, "examples/docs-ci/caller.yml"] { let on = triggers(workflow); assert_eq!( on["merge_group"]["types"], @@ -500,3 +497,115 @@ fn no_other_workflow_job_reports_the_gate_context() { } } } + +/// Failures ci.yml may swallow: (job, step, kind, reason). +const ALLOWED_SWALLOWS: [(&str, &str, &str, &str); 3] = [ + ( + "preflight", + "Record the Windows rmeta diagnostics (#4850)", + "|| true", + "diagnostics only; the build under diagnosis reports its own failure", + ), + ( + "preflight", + "Record the runner image", + "|| true", + "the image record is informational", + ), + ( + "scheduled-report", + "Open, update or close the tracking issue", + "|| true", + "a failed issue lookup must not fail the report", + ), +]; + +#[test] +fn ci_swallows_no_failure_outside_the_allow_list() { + let found = swallows(CI_WORKFLOW); + for (job, step, kind) in &found { + assert!( + ALLOWED_SWALLOWS + .iter() + .any(|(j, s, k, _)| j == job && s == step && k == kind), + "{CI_WORKFLOW} job {job} step {step:?} swallows a failure with {kind}" + ); + } + for (job, step, kind, reason) in ALLOWED_SWALLOWS { + assert!( + found + .iter() + .any(|(j, s, k)| j == job && s == step && k == kind), + "stale allow-list entry: {job} {step:?} {kind} ({reason})" + ); + } +} + +#[test] +fn the_swallow_scanner_finds_entries() { + assert!( + swallows(CONSUMER_WORKFLOW) + .iter() + .any(|(_, _, kind)| kind == "|| true"), + "{CONSUMER_WORKFLOW} swallows failures the scanner must report" + ); +} + +#[test] +fn compiler_probes_run_before_the_evidence_step_and_fail_normally() { + for job in ["hermetic", "starter-consumer"] { + let all = steps(CI_WORKFLOW, job); + let position = |name: &str| { + all.iter() + .position(|step| step["name"] == json!(name)) + .unwrap_or_else(|| panic!("{job} has no step {name:?}")) + }; + let probe = position("Run the compiler probe tests"); + let evidence = position("Record compiler evidence"); + assert!(probe < evidence, "{job} probe step must precede evidence"); + let probe_run = all[probe]["run"].as_str().unwrap_or_default(); + assert!(!probe_run.contains("|| true"), "{job} probe step swallows"); + assert!(probe_run.contains("bazel test"), "{job} probe runs no test"); + let evidence_run = all[evidence]["run"].as_str().unwrap_or_default(); + assert!( + !evidence_run + .lines() + .any(|line| line.trim_start().starts_with("bazel test")), + "{job} evidence step runs tests" + ); + assert!( + all[evidence]["if"] + .as_str() + .unwrap_or_default() + .starts_with("always()"), + "{job} evidence step must run always" + ); + } +} + +#[test] +fn the_gate_job_merges_evidence_without_swallowing() { + let all = steps(CI_WORKFLOW, GATE); + for step in all.iter().filter(|step| { + let uses = step["uses"].as_str().unwrap_or_default(); + step["name"] == json!("Merge compiler evidence") + || uses.starts_with("actions/download-artifact@") + || uses.starts_with("actions/upload-artifact@") + }) { + assert!( + step.get("continue-on-error").is_none(), + "{GATE} step swallows: {step}" + ); + assert!( + step["if"] + .as_str() + .unwrap_or_default() + .starts_with("always()") + || step["if"] + .as_str() + .unwrap_or_default() + .starts_with("${{ always()"), + "{GATE} step must run always: {step}" + ); + } +} diff --git a/tools/ci/ci_workflow/src/lib.rs b/tools/ci/ci_workflow/src/lib.rs index d63bf05e20..f14dbf93d6 100644 --- a/tools/ci/ci_workflow/src/lib.rs +++ b/tools/ci/ci_workflow/src/lib.rs @@ -122,3 +122,34 @@ pub fn strings(path: &str) -> Vec { walk(&document(path), &mut out); out } + +/// Returns (job id, step name or `uses`, "|| true" or "continue-on-error") for every failure the workflow at `path` swallows. +#[track_caller] +pub fn swallows(path: &str) -> Vec<(String, String, String)> { + let mut found = Vec::new(); + for (id, job) in jobs(path) { + if job.get("continue-on-error").is_some() { + found.push((id.clone(), String::new(), "continue-on-error".to_owned())); + } + let Some(steps) = job["steps"].as_array() else { + continue; + }; + for step in steps { + let label = step["name"] + .as_str() + .or_else(|| step["uses"].as_str()) + .unwrap_or_default() + .to_owned(); + if step.get("continue-on-error").is_some() { + found.push((id.clone(), label.clone(), "continue-on-error".to_owned())); + } + if step["run"] + .as_str() + .is_some_and(|run| run.contains("|| true")) + { + found.push((id.clone(), label, "|| true".to_owned())); + } + } + } + found +}