ci: one hermetic-evidence composite action for the hermetic jobs

.github/actions/compiler-evidence/action.yml

@@ -0,0 +1,66 @@
+name: compiler-evidence
+description: Record the evidence of the compiler probe tests listed in $RUNNER_TEMP/compiler-targets, and upload it.
+inputs:
+  platform:
+    description: Platform the evidence is recorded for.
+    required: true
+  name:
+    description: Basename of the evidence file and suffix of the uploaded artifact.
+    required: true
+  bin-from-bazel-info:
+    description: Resolve bazel-bin and bazel-testlogs with bazel info instead of the workspace symlinks.
+    required: false
+    default: "false"
+runs:
+  using: composite
+  steps:
+    - name: Record compiler evidence
+      if: always()
+      shell: bash
+      env:
+        PLATFORM: ${{ inputs.platform }}
+        EVIDENCE_NAME: ${{ inputs.name }}
+        BIN_FROM_INFO: ${{ inputs.bin-from-bazel-info }}
+      run: |
+        # shellcheck disable=SC2086
+        set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
+        out="$RUNNER_TEMP/compiler-evidence/${EVIDENCE_NAME}.jsonl"
+        if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then
+          echo "::notice::The compiler probe tests did not run; no compiler evidence recorded."
+          exit 0
+        fi
+        mkdir -p "$(dirname "$out")"
+        : > "$out"
+        if [ "$BIN_FROM_INFO" = true ]; then
+          DX_BIN="$(bazel info "$@" bazel-bin)"
+          DX_TESTLOGS="$(bazel info "$@" bazel-testlogs)"
+        else
+          DX_BIN="$PWD/bazel-bin"
+          DX_TESTLOGS="$PWD/bazel-testlogs"
+        fi
+        DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin"
+        [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe"
+        export DX_BIN DX_TESTLOGS DX_PRODUCERS
+        facts="${DX_BIN}/tools/docs/support_matrix/facts.json"
+        targets="$(cat "$RUNNER_TEMP/compiler-targets")"
+        DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets"
+        export DX_EVIDENCE_COMMAND
+        tools/ci/compiler_evidence.sh "$out" "$PLATFORM" "$facts"
+        {
+          echo "#### Compiler evidence"
+          echo "| row | version | probe compiler | target | observed producers | ran/total | outcome |"
+          echo "| --- | --- | --- | --- | --- | --- | --- |"
+          jq -r '"| \(.row) | \(.version) | \(.compiler) | \(.target) | \(.observed_producers | join("; ")) | \(.ran)/\(.total) | \(.outcome) |"' "$out"
+        } >> "$GITHUB_STEP_SUMMARY"
+    - name: Check the upload holds no secret
+      id: scan-compiler-evidence
+      if: always()
+      uses: ./.github/actions/assert-no-secret
+      with:
+        path: ${{ runner.temp }}/compiler-evidence
+    - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+      if: ${{ always() && steps.scan-compiler-evidence.outcome == 'success' }}
+      with:
+        name: compiler-evidence-${{ inputs.name }}
+        path: ${{ runner.temp }}/compiler-evidence
+        retention-days: 3

.github/actions/hermetic-evidence/action.yml

@@ -0,0 +1,55 @@
+name: hermetic-evidence
+description: Collect, verify and upload the evidence of the hermetic qualification targets listed in $RUNNER_TEMP/hermetic-targets.
+inputs:
+  platform:
+    description: Platform suffix of the uploaded artifact name.
+    required: true
+  manifest-from:
+    description: Where the manifest sits, bazel-bin or evidence (the collected outputs).
+    required: false
+    default: bazel-bin
+runs:
+  using: composite
+  steps:
+    - name: Collect the evidence
+      if: always()
+      shell: bash
+      run: |
+        mkdir -p "$RUNNER_TEMP/hermetic-evidence"
+        while read -r label; do
+          pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
+          mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
+          cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
+        done < "$RUNNER_TEMP/hermetic-targets"
+    - name: Verify the evidence
+      if: ${{ job.status == 'success' }}
+      shell: bash
+      env:
+        DX_MANIFEST_FROM: ${{ inputs.manifest-from }}
+      run: |
+        # shellcheck disable=SC2086
+        set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
+        while read -r label; do
+          pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
+          if [ "$DX_MANIFEST_FROM" = evidence ]; then
+            dir="$(cygpath -m "$RUNNER_TEMP/hermetic-evidence/$name")"
+            evidence="$dir/evidence.json"
+            manifest="$dir/manifest.json"
+          else
+            evidence="$PWD/bazel-testlogs/$pkg/$name/test.outputs/evidence.json"
+            manifest="$PWD/bazel-bin/$pkg/${name}_manifest.json"
+          fi
+          bazel run --noshow_progress "$@" //tools/testing/hermetic:verify -- --evidence "$evidence" --commit "$(git rev-parse HEAD)" --manifest "$manifest"
+        done < "$RUNNER_TEMP/hermetic-targets"
+    - name: Check the upload holds no secret
+      id: scan-hermetic-evidence
+      if: always()
+      uses: ./.github/actions/assert-no-secret
+      with:
+        path: ${{ runner.temp }}/hermetic-evidence
+    - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+      if: ${{ always() && steps.scan-hermetic-evidence.outcome == 'success' }}
+      with:
+        name: hermetic-evidence-${{ inputs.platform }}
+        path: ${{ runner.temp }}/hermetic-evidence
+        retention-days: 3

.github/workflows/ci.yml

@@ -167,23 +167,10 @@ jobs:
           echo "$targets" > "$RUNNER_TEMP/hermetic-targets"
           # shellcheck disable=SC2086
           bazel test --noshow_progress --nozip_undeclared_test_outputs --test_output=errors --test_timeout=3000 "--test_env=DX_HERMETIC_BAZELISK=$(command -v bazel)" "$@" //tools/testing/hermetic:isolation_probe_test $targets
-      - name: Verify the evidence
-        run: |
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            bazel run --noshow_progress "$@" //tools/testing/hermetic:verify -- --evidence "$PWD/bazel-testlogs/$pkg/$name/test.outputs/evidence.json" --commit "$(git rev-parse HEAD)" --manifest "$PWD/bazel-bin/$pkg/${name}_manifest.json"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Collect the evidence
+      - uses: ./.github/actions/hermetic-evidence
         if: always()
-        run: |
-          mkdir -p "$RUNNER_TEMP/hermetic-evidence"
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
-          done < "$RUNNER_TEMP/hermetic-targets"
+        with:
+          platform: linux_x86_64
       - name: Run the compiler probe tests
         id: compiler-probes
         if: ${{ !cancelled() }}
@@ -202,58 +189,12 @@ jobs:
           if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then
             exit "$status"
           fi
-      - name: Record compiler evidence
-        if: always()
-        run: |
-          if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then
-            echo "::notice::The compiler probe tests did not run; no compiler evidence recorded."
-            exit 0
-          fi
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          out="$RUNNER_TEMP/compiler-evidence/linux_x86_64.jsonl"
-          mkdir -p "$(dirname "$out")"
-          : > "$out"
-          DX_BIN="$(bazel info "$@" bazel-bin)"
-          DX_TESTLOGS="$(bazel info "$@" bazel-testlogs)"
-          DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin"
-          [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe"
-          export DX_BIN DX_TESTLOGS DX_PRODUCERS
-          facts="${DX_BIN}/tools/docs/support_matrix/facts.json"
-          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
-          DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets"
-          export DX_EVIDENCE_COMMAND
-          tools/ci/compiler_evidence.sh "$out" linux_x86_64 "$facts"
-          {
-            echo "#### Compiler evidence"
-            echo "| row | version | probe compiler | target | observed producers | ran/total | outcome |"
-            echo "| --- | --- | --- | --- | --- | --- | --- |"
-            jq -r '"| \(.row) | \(.version) | \(.compiler) | \(.target) | \(.observed_producers | join("; ")) | \(.ran)/\(.total) | \(.outcome) |"' "$out"
-          } >> "$GITHUB_STEP_SUMMARY"
-      - name: Check the upload holds no secret
-        id: scan-compiler-evidence
-        if: always()
-        uses: ./.github/actions/assert-no-secret
-        with:
-          path: ${{ runner.temp }}/compiler-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-compiler-evidence.outcome == 'success' }}
-        with:
-          name: compiler-evidence-linux_x86_64
-          path: ${{ runner.temp }}/compiler-evidence
-          retention-days: 3
-      - name: Check the upload holds no secret
-        id: scan-hermetic-evidence
+      - uses: ./.github/actions/compiler-evidence
         if: always()
-        uses: ./.github/actions/assert-no-secret
         with:
-          path: ${{ runner.temp }}/hermetic-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-hermetic-evidence.outcome == 'success' }}
-        with:
-          name: hermetic-evidence-linux_x86_64
-          path: ${{ runner.temp }}/hermetic-evidence
-          retention-days: 3
+          platform: linux_x86_64
+          name: linux_x86_64
+          bin-from-bazel-info: "true"
       - name: Write job summary
         if: always()
         env:
@@ -287,35 +228,10 @@ jobs:
           set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
           # shellcheck disable=SC2046
           bazel test --noshow_progress --nozip_undeclared_test_outputs --test_output=errors --test_timeout=3000 "--test_env=DX_HERMETIC_BAZELISK=$(command -v bazel)" "$@" $(cat "$RUNNER_TEMP/hermetic-targets")
-      - name: Verify the evidence
-        run: |
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            bazel run --noshow_progress "$@" //tools/testing/hermetic:verify -- --evidence "$PWD/bazel-testlogs/$pkg/$name/test.outputs/evidence.json" --commit "$(git rev-parse HEAD)" --manifest "$PWD/bazel-bin/$pkg/${name}_manifest.json"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Collect the evidence
-        if: always()
-        run: |
-          mkdir -p "$RUNNER_TEMP/hermetic-evidence"
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Check the upload holds no secret
-        id: scan-hermetic-evidence
+      - uses: ./.github/actions/hermetic-evidence
         if: always()
-        uses: ./.github/actions/assert-no-secret
         with:
-          path: ${{ runner.temp }}/hermetic-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-hermetic-evidence.outcome == 'success' }}
-        with:
-          name: hermetic-evidence-linux_arm64
-          path: ${{ runner.temp }}/hermetic-evidence
-          retention-days: 3
+          platform: linux_arm64
   hermetic-macos:
     name: hermetic-macos (macOS-compatible consumer qualifications on macos_arm64)
     timeout-minutes: 90
@@ -338,35 +254,10 @@ jobs:
           set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
           # shellcheck disable=SC2046
           bazel test --noshow_progress --nozip_undeclared_test_outputs --test_output=errors --test_timeout=3000 "--test_env=DX_HERMETIC_BAZELISK=$(command -v bazel)" "$@" //tools/testing/hermetic:isolation_probe_test $(cat "$RUNNER_TEMP/hermetic-targets")
-      - name: Verify the evidence
-        run: |
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            bazel run --noshow_progress "$@" //tools/testing/hermetic:verify -- --evidence "$PWD/bazel-testlogs/$pkg/$name/test.outputs/evidence.json" --commit "$(git rev-parse HEAD)" --manifest "$PWD/bazel-bin/$pkg/${name}_manifest.json"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Collect the evidence
-        if: always()
-        run: |
-          mkdir -p "$RUNNER_TEMP/hermetic-evidence"
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Check the upload holds no secret
-        id: scan-hermetic-evidence
+      - uses: ./.github/actions/hermetic-evidence
         if: always()
-        uses: ./.github/actions/assert-no-secret
         with:
-          path: ${{ runner.temp }}/hermetic-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-hermetic-evidence.outcome == 'success' }}
-        with:
-          name: hermetic-evidence-macos_arm64
-          path: ${{ runner.temp }}/hermetic-evidence
-          retention-days: 3
+          platform: macos_arm64
   hermetic-windows:
     name: hermetic-windows (MSVC consumer fixtures on ${{ matrix.platform }})
     timeout-minutes: 90
@@ -419,36 +310,11 @@ jobs:
           if [ "$PLATFORM" = windows_x86_64 ]; then
             bazel test --noshow_progress --nozip_undeclared_test_outputs --test_output=errors --test_timeout=3000 "$@" --//cc/toolchains:compiler=msvc //deploy/rules:pypi_native_msvc_verify
           fi
-      - name: Collect the evidence
+      - uses: ./.github/actions/hermetic-evidence
         if: always()
-        run: |
-          mkdir -p "$RUNNER_TEMP/hermetic-evidence"
-          while read -r label; do
-            pkg="${label%%:*}"; pkg="${pkg#//}"; name="${label##*:}"
-            mkdir -p "$RUNNER_TEMP/hermetic-evidence/$name"
-            [ ! -d "bazel-testlogs/$pkg/$name/test.outputs" ] || cp -R "bazel-testlogs/$pkg/$name/test.outputs/." "$RUNNER_TEMP/hermetic-evidence/$name/"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      # The test copies its manifest into its outputs; bazel-bin on Windows does not hold it.
-      - name: Verify the evidence
-        run: |
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          while read -r label; do
-            dir="$(cygpath -m "$RUNNER_TEMP/hermetic-evidence/${label##*:}")"
-            bazel run --noshow_progress "$@" //tools/testing/hermetic:verify -- --evidence "$dir/evidence.json" --commit "$(git rev-parse HEAD)" --manifest "$dir/manifest.json"
-          done < "$RUNNER_TEMP/hermetic-targets"
-      - name: Check the upload holds no secret
-        id: scan-hermetic-evidence
-        if: always()
-        uses: ./.github/actions/assert-no-secret
         with:
-          path: ${{ runner.temp }}/hermetic-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-hermetic-evidence.outcome == 'success' }}
-        with:
-          name: hermetic-evidence-${{ matrix.platform }}
-          path: ${{ runner.temp }}/hermetic-evidence
-          retention-days: 3
+          platform: ${{ matrix.platform }}
+          manifest-from: evidence
   gcc-cross-build:
     name: gcc-cross-build (linux_x86_64 -> linux_arm64 fixtures)
     timeout-minutes: 45
@@ -665,48 +531,11 @@ jobs:
           if [ "$status" -ne 0 ] && [ "$status" -ne 4 ]; then
             exit "$status"
           fi
-      - name: Record compiler evidence
+      - uses: ./.github/actions/compiler-evidence
         if: always() && matrix.shard == 0
-        env:
-          PLATFORM: ${{ matrix.platform }}
-        run: |
-          if [ ! -f "$RUNNER_TEMP/compiler-targets" ]; then
-            echo "::notice::The compiler probe tests did not run; no compiler evidence recorded."
-            exit 0
-          fi
-          # shellcheck disable=SC2086
-          set -- --lockfile_mode=error $BAZEL_CONFIG $BB_ARGS
-          out="$RUNNER_TEMP/compiler-evidence/starter-${PLATFORM}.jsonl"
-          mkdir -p "$(dirname "$out")"
-          : > "$out"
-          DX_BIN="$PWD/bazel-bin"
-          DX_TESTLOGS="$PWD/bazel-testlogs"
-          DX_PRODUCERS="$DX_BIN/tools/ci/producers/producers_bin"
-          [ -f "$DX_PRODUCERS" ] || DX_PRODUCERS="$DX_PRODUCERS.exe"
-          export DX_BIN DX_TESTLOGS DX_PRODUCERS
-          facts="${DX_BIN}/tools/docs/support_matrix/facts.json"
-          targets="$(cat "$RUNNER_TEMP/compiler-targets")"
-          DX_EVIDENCE_COMMAND="bazel test --skip_incompatible_explicit_targets $* $targets"
-          export DX_EVIDENCE_COMMAND
-          tools/ci/compiler_evidence.sh "$out" "$PLATFORM" "$facts"
-          {
-            echo "#### Compiler evidence"
-            echo "| row | version | probe compiler | target | observed producers | ran/total | outcome |"
-            echo "| --- | --- | --- | --- | --- | --- | --- |"
-            jq -r '"| \(.row) | \(.version) | \(.compiler) | \(.target) | \(.observed_producers | join("; ")) | \(.ran)/\(.total) | \(.outcome) |"' "$out"
-          } >> "$GITHUB_STEP_SUMMARY"
-      - name: Check the upload holds no secret
-        id: scan-compiler-evidence
-        if: always() && matrix.shard == 0
-        uses: ./.github/actions/assert-no-secret
-        with:
-          path: ${{ runner.temp }}/compiler-evidence
-      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
-        if: ${{ always() && steps.scan-compiler-evidence.outcome == 'success' }}
         with:
-          name: compiler-evidence-starter-${{ matrix.platform }}
-          path: ${{ runner.temp }}/compiler-evidence
-          retention-days: 3
+          platform: ${{ matrix.platform }}
+          name: starter-${{ matrix.platform }}
       - name: Write job summary
         if: always()
         env: