ci: compiler probe tests fail normally; scan ci.yml for swallowed failures

.github/workflows/ci.yml

@@ -53,7 +53,6 @@ jobs:
         with:
           cache-save: ${{ github.event_name != 'pull_request' }}
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
-        continue-on-error: true
         with:
           pattern: dx-reports-coverage-*
           path: ${{ runner.temp }}/coverage-reports
@@ -183,28 +182,45 @@ jobs:
           while read -r label; do
             pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
             mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true
+            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
           done < "$RUNNER_TEMP/hermetic-targets"
+      - name: Run the compiler probe tests
+        id: compiler-probes
+        if: ${{ !cancelled() }}
+        run: |
+          # shellcheck disable=SC2086
+          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
+          bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin
+          facts="$(bazel info "$@" bazel-bin)/tools/docs/support_matrix/facts.json"
+          jq -r '.rows[] | select(.section == "compilers") | .identity.probe, .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ' > "$RUNNER_TEMP/compiler-targets"
+          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
+          # shellcheck disable=SC2086
+          bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets
+          status=0
+          # shellcheck disable=SC2086
+          bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets && status=0 || status=$?
+          if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then
+            exit "$status"
+          fi
       - name: Record compiler evidence
         if: always()
         run: |
+          if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then
+            echo "::notice::The compiler probe tests did not run; no compiler evidence recorded."
+            exit 0
+          fi
           # shellcheck disable=SC2086
           set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
           out="$RUNNER_TEMP/compiler-evidence/linux_x86_64.jsonl"
           mkdir -p "$(dirname "$out")"
           : > "$out"
-          bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin
           DX_BIN="$(bazel info "$@" bazel-bin)"
           DX_TESTLOGS="$(bazel info "$@" bazel-testlogs)"
           DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin"
           [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe"
           export DX_BIN DX_TESTLOGS DX_PRODUCERS
           facts="${DX_BIN}/tools/docs/support_matrix/facts.json"
-          targets="$(jq -r '.rows[] | select(.section == "compilers") | .identity.probe, .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ')"
-          # shellcheck disable=SC2086
-          bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets || true
-          # shellcheck disable=SC2086
-          bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets || true
+          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
           DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets"
           export DX_EVIDENCE_COMMAND
           tools/ci/compiler_evidence.sh "$out" linux_x86_64 "$facts"
@@ -286,7 +302,7 @@ jobs:
           while read -r label; do
             pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
             mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true
+            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
           done < "$RUNNER_TEMP/hermetic-targets"
       - name: Check the upload holds no secret
         id: scan-hermetic-evidence
@@ -337,7 +353,7 @@ jobs:
           while read -r label; do
             pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
             mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true
+            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
           done < "$RUNNER_TEMP/hermetic-targets"
       - name: Check the upload holds no secret
         id: scan-hermetic-evidence
@@ -410,7 +426,7 @@ jobs:
           while read -r label; do
             pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
             mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/" || true
+            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
           done < "$RUNNER_TEMP/hermetic-targets"
       # The test copies its manifest into its outputs; bazel-bin on Windows does not hold it.
       - name: Verify the evidence
@@ -629,28 +645,47 @@ jobs:
             echo "::error::Every starter consumer test was skipped on ${PLATFORM}, shard ${SHARD}."
             exit 1
           fi
+      - name: Run the compiler probe tests
+        id: compiler-probes
+        if: ${{ !cancelled() && matrix.shard == 0 }}
+        env:
+          PLATFORM: ${{ matrix.platform }}
+        run: |
+          # shellcheck disable=SC2086
+          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
+          bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin
+          facts="$PWD/bazel-bin/tools/docs/support_matrix/facts.json"
+          jq -r --arg platform "$PLATFORM" '.rows[] | select(.section == "compilers") | (.identity.platform_probes[$platform] // .identity.probe), .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ' > "$RUNNER_TEMP/compiler-targets"
+          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
+          # shellcheck disable=SC2086
+          bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets
+          status=0
+          # shellcheck disable=SC2086
+          bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets && status=0 || status=$?
+          if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then
+            exit "$status"
+          fi
       - name: Record compiler evidence
         if: always() && matrix.shard == 0
         env:
           PLATFORM: ${{ matrix.platform }}
         run: |
+          if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then
+            echo "::notice::The compiler probe tests did not run; no compiler evidence recorded."
+            exit 0
+          fi
           # shellcheck disable=SC2086
           set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
           out="$RUNNER_TEMP/compiler-evidence/starter-${PLATFORM}.jsonl"
           mkdir -p "$(dirname "$out")"
           : > "$out"
-          bazel build --noshow_progress "$@" //tools/docs/support_matrix:facts //tools/ci/producers:producers_bin
           DX_BIN="$PWD/bazel-bin"
           DX_TESTLOGS="$PWD/bazel-testlogs"
           DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin"
           [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe"
           export DX_BIN DX_TESTLOGS DX_PRODUCERS
           facts="${DX_BIN}/tools/docs/support_matrix/facts.json"
-          targets="$(jq -r --arg platform "$PLATFORM" '.rows[] | select(.section == "compilers") | (.identity.platform_probes[$platform] // .identity.probe), .ci_tests[]' "$facts" | tr -d '\r' | sort -u | grep . | tr '\n' ' ')"
-          # shellcheck disable=SC2086
-          bazel build --noshow_progress --skip_incompatible_explicit_targets --output_groups=starlark_manifest "$@" $targets || true
-          # shellcheck disable=SC2086
-          bazel test --noshow_progress --test_output=errors --skip_incompatible_explicit_targets "$@" $targets || true
+          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
           DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets"
           export DX_EVIDENCE_COMMAND
           tools/ci/compiler_evidence.sh "$out" "$PLATFORM" "$facts"
@@ -1026,14 +1061,12 @@ jobs:
           persist-credentials: false
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         if: always()
-        continue-on-error: true
         with:
           pattern: compiler-evidence-*
           merge-multiple: true
           path: ${{ runner.temp }}/compiler-evidence
       - name: Merge compiler evidence
         if: always()
-        continue-on-error: true
         run: |
           mkdir -p "$RUNNER_TEMP/compiler-evidence"
           {
@@ -1048,7 +1081,6 @@ jobs:
           path: ${{ runner.temp }}/compiler-evidence-all.jsonl
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         if: ${{ always() && steps.scan-compiler-evidence.outcome == 'success' }}
-        continue-on-error: true
         with:
           name: compiler-evidence-all
           path: ${{ runner.temp }}/compiler-evidence-all.jsonl

tools/ci/aggregate_gates_test.rs

@@ -1,6 +1,6 @@
 use serde_json::{Value, json};
 
-use ci_workflow::{document, jobs, steps, triggers};
+use ci_workflow::{document, jobs, steps, swallows, triggers};
 use dx_testing::read_runfiles;
 
 const CI_WORKFLOW: &str = ".github/workflows/ci.yml";
@@ -268,10 +268,7 @@ fn the_native_matrix_keeps_all_five_platforms() {
 
 #[test]
 fn every_gate_workflow_runs_on_merge_group_checks_requested() {
-    for workflow in [
-        CI_WORKFLOW,
-        "examples/docs-ci/caller.yml",
-    ] {
+    for workflow in [CI_WORKFLOW, "examples/docs-ci/caller.yml"] {
         let on = triggers(workflow);
         assert_eq!(
             on["merge_group"]["types"],
@@ -500,3 +497,115 @@ fn no_other_workflow_job_reports_the_gate_context() {
         }
     }
 }
+
+/// Failures ci.yml may swallow: (job, step, kind, reason).
+const ALLOWED_SWALLOWS: [(&str, &str, &str, &str); 3] = [
+    (
+        "preflight",
+        "Record the Windows rmeta diagnostics (#4850)",
+        "|| true",
+        "diagnostics only; the build under diagnosis reports its own failure",
+    ),
+    (
+        "preflight",
+        "Record the runner image",
+        "|| true",
+        "the image record is informational",
+    ),
+    (
+        "scheduled-report",
+        "Open, update or close the tracking issue",
+        "|| true",
+        "a failed issue lookup must not fail the report",
+    ),
+];
+
+#[test]
+fn ci_swallows_no_failure_outside_the_allow_list() {
+    let found = swallows(CI_WORKFLOW);
+    for (job, step, kind) in &found {
+        assert!(
+            ALLOWED_SWALLOWS
+                .iter()
+                .any(|(j, s, k, _)| j == job && s == step && k == kind),
+            "{CI_WORKFLOW} job {job} step {step:?} swallows a failure with {kind}"
+        );
+    }
+    for (job, step, kind, reason) in ALLOWED_SWALLOWS {
+        assert!(
+            found
+                .iter()
+                .any(|(j, s, k)| j == job && s == step && k == kind),
+            "stale allow-list entry: {job} {step:?} {kind} ({reason})"
+        );
+    }
+}
+
+#[test]
+fn the_swallow_scanner_finds_entries() {
+    assert!(
+        swallows(CONSUMER_WORKFLOW)
+            .iter()
+            .any(|(_, _, kind)| kind == "|| true"),
+        "{CONSUMER_WORKFLOW} swallows failures the scanner must report"
+    );
+}
+
+#[test]
+fn compiler_probes_run_before_the_evidence_step_and_fail_normally() {
+    for job in ["hermetic", "starter-consumer"] {
+        let all = steps(CI_WORKFLOW, job);
+        let position = |name: &str| {
+            all.iter()
+                .position(|step| step["name"] == json!(name))
+                .unwrap_or_else(|| panic!("{job} has no step {name:?}"))
+        };
+        let probe = position("Run the compiler probe tests");
+        let evidence = position("Record compiler evidence");
+        assert!(probe < evidence, "{job} probe step must precede evidence");
+        let probe_run = all[probe]["run"].as_str().unwrap_or_default();
+        assert!(!probe_run.contains("|| true"), "{job} probe step swallows");
+        assert!(probe_run.contains("bazel test"), "{job} probe runs no test");
+        let evidence_run = all[evidence]["run"].as_str().unwrap_or_default();
+        assert!(
+            !evidence_run
+                .lines()
+                .any(|line| line.trim_start().starts_with("bazel test")),
+            "{job} evidence step runs tests"
+        );
+        assert!(
+            all[evidence]["if"]
+                .as_str()
+                .unwrap_or_default()
+                .starts_with("always()"),
+            "{job} evidence step must run always"
+        );
+    }
+}
+
+#[test]
+fn the_gate_job_merges_evidence_without_swallowing() {
+    let all = steps(CI_WORKFLOW, GATE);
+    for step in all.iter().filter(|step| {
+        let uses = step["uses"].as_str().unwrap_or_default();
+        step["name"] == json!("Merge compiler evidence")
+            || uses.starts_with("actions/download-artifact@")
+            || uses.starts_with("actions/upload-artifact@")
+    }) {
+        assert!(
+            step.get("continue-on-error").is_none(),
+            "{GATE} step swallows: {step}"
+        );
+        assert!(
+            step["if"]
+                .as_str()
+                .unwrap_or_default()
+                .starts_with("always()")
+                || step["if"]
+                    .as_str()
+                    .unwrap_or_default()
+                    .starts_with("${{ always()"),
+            "{GATE} step must run always: {step}"
+        );
+    }
+}

tools/ci/ci_workflow/src/lib.rs

@@ -122,3 +122,34 @@ pub fn strings(path: &str) -> Vec<String> {
     walk(&document(path), &mut out);
     out
 }
+
+/// Returns (job id, step name or `uses`, "|| true" or "continue-on-error") for every failure the workflow at `path` swallows.
+#[track_caller]
+pub fn swallows(path: &str) -> Vec<(String, String, String)> {
+    let mut found = Vec::new();
+    for (id, job) in jobs(path) {
+        if job.get("continue-on-error").is_some() {
+            found.push((id.clone(), String::new(), "continue-on-error".to_owned()));
+        }
+        let Some(steps) = job["steps"].as_array() else {
+            continue;
+        };
+        for step in steps {
+            let label = step["name"]
+                .as_str()
+                .or_else(|| step["uses"].as_str())
+                .unwrap_or_default()
+                .to_owned();
+            if step.get("continue-on-error").is_some() {
+                found.push((id.clone(), label.clone(), "continue-on-error".to_owned()));
+            }
+            if step["run"]
+                .as_str()
+                .is_some_and(|run| run.contains("|| true"))
+            {
+                found.push((id.clone(), label, "|| true".to_owned()));
+            }
+        }
+    }
+    found
+}