diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 9967d0190..b77d9b05e 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -7,7 +7,7 @@ - `ci.yml`: this repository's own `ci` gate. A failed Windows preflight build uploads `rmeta-diagnostics-` (#4850). - `issue-labels.yml`: daily warning for open issues older than one day, except pinned ones, that have no P0 to P3 label. Writes a job summary. Runs on `ubuntu-slim`. A second job warns about labels named in `CONTRIBUTING.md` or issue template front matter that do not exist in the repository (#6643). -- `evidence.yml`: native evidence on hosted runners, dispatched per run. The `evidence-` artifact holds `evidence.log` and each test's log, XML and undeclared outputs. Set `platforms` to `["macos_arm64"]` for Apple runs. Set `android` to boot an emulator and run the Rust Android device scenarios on `linux_x86_64`. Set `qualify` on `windows_x86_64` to run `//cc/context:qualify` for llvm and msvc. Set `relocate` (test or build) to rerun the task under a fresh `--output_base` with the disk cache off and remote cache hits refused; the second run logs to `relocated.log`, and `bazel-testlogs` then points at the relocated base. With `task=build` it also hashes every output of `scope` in both bases, writes `match` or `differ ` lines to `determinism.txt`, copies each differing file into `determinism/first/` and `determinism/relocated/`, and fails when any differ. The first build may hit the remote cache and the relocated one cannot, so a `first` copy can come from the cache. Set `lock_refresh` to regenerate `MODULE.bazel.lock` on the platform before the scope task and fail when it differs from the checkout; the refreshed lock and `lock.diff` land in `lock/` of the artifact, so a lock entry that only changes on Windows can be copied back. Set `release_cell` to build and qualify the platform's release cell first with `tools/ci/release_cell.sh`, the script `release-cells.yml` runs, uncached and without RBE; the files land in `release-cell/` of the artifact. +- `evidence.yml`: native evidence on hosted runners, dispatched per run. The run posts the commit status `evidence///` on the dispatched commit: pending at dispatch, then success, failure or error. The `evidence-` artifact holds `evidence.log` and each test's log, XML and undeclared outputs. Set `platforms` to `["macos_arm64"]` for Apple runs. Set `android` to boot an emulator and run the Rust Android device scenarios on `linux_x86_64`. Set `qualify` on `windows_x86_64` to run `//cc/context:qualify` for llvm and msvc. Set `relocate` (test or build) to rerun the task under a fresh `--output_base` with the disk cache off and remote cache hits refused; the second run logs to `relocated.log`, and `bazel-testlogs` then points at the relocated base. With `task=build` it also hashes every output of `scope` in both bases, writes `match` or `differ ` lines to `determinism.txt`, copies each differing file into `determinism/first/` and `determinism/relocated/`, and fails when any differ. The first build may hit the remote cache and the relocated one cannot, so a `first` copy can come from the cache. Set `lock_refresh` to regenerate `MODULE.bazel.lock` on the platform before the scope task and fail when it differs from the checkout; the refreshed lock and `lock.diff` land in `lock/` of the artifact, so a lock entry that only changes on Windows can be copied back. Set `release_cell` to build and qualify the platform's release cell first with `tools/ci/release_cell.sh`, the script `release-cells.yml` runs, uncached and without RBE; the files land in `release-cell/` of the artifact. - `evidence-command.yml`: a comment on an issue or pull request whose first line starts with `/evidence`, from an owner, member or collaborator, dispatches `evidence.yml` and replies with the run URL. On a pull request it runs on the PR branch, otherwise on `main`. Arguments are `platforms=`, `task=`, `scope=`, `test_timeout=`, `displays=`, `android=`, `qualify=`, `relocate=` and `release_cell=`, for example `/evidence platforms=["macos_arm64"] task=test scope=//haskell/...`. `evidence.yml` validates them and a rejected run fails there. - `dx-update.yml`: weekly `tools/dx update --pr` for dependency updates. `dx init` writes the same file. Where `lock-refresh.yml` exists, it then dispatches that workflow for each open `dx-update/` branch whose `MODULE.bazel.lock` changes the `rules_python` `python` or `rules_go` `go_sdk` entries. - `pin-android-maven.yml`: dispatch-only. Runs `REPIN=1 bazel run @rules_android_maven//:pin` on `ubuntu-latest` and uploads `rules_android_maven_install.json` as an artifact to commit after a rules_android or rules_flutter bump, so the lock is not resolved through Coursier on a throttled network (#7184). @@ -24,6 +24,6 @@ The platform table is `PLATFORMS` in `tools/ci/input_validation/src/lib.rs`; `input_validation_test` pins every other copy to it. -Land a PR with `gh pr checks --required --watch --fail-fast`, then `gh pr merge --squash`, never `--admin`. +Land a PR with `gh pr checks --watch --fail-fast --required`, then `gh pr merge --squash`, never `--admin`. The consumer contract is [GitHub CI](../../docs/github-ci.md). diff --git a/.github/workflows/evidence.yml b/.github/workflows/evidence.yml index a430bee92..5e4acceeb 100644 --- a/.github/workflows/evidence.yml +++ b/.github/workflows/evidence.yml @@ -63,9 +63,37 @@ env: MSYS2_ARG_CONV_EXCL: "//" DX_ANDROID_API: "34" DX_ANDROID_ABI: "x86_64" - BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} jobs: + pending: + name: evidence status (pending) + runs-on: ubuntu-slim + timeout-minutes: 5 + permissions: + statuses: write + steps: + - name: Post pending statuses + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + DX_SCOPE: ${{ inputs.scope }} + DX_TASK: ${{ inputs.task }} + DX_PLATFORMS: ${{ inputs.platforms }} + run: | + set -o pipefail + task="$(printf '%s' "$DX_TASK" | tr -c 'a-z0-9_' '_' | cut -c1-40)" + scope="$(printf '%s' "$DX_SCOPE" | tr -c 'A-Za-z0-9_.-' '_' | cut -c1-60)" + description="$(printf '%s %s' "$task" "$DX_SCOPE" | tr '\n' ' ' | cut -c1-140)" + url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + printf '%s' "$DX_PLATFORMS" | jq -r '.[]? | strings' 2>/dev/null | while read -r platform; do + [[ "$platform" =~ ^[a-z0-9_]{1,40}$ ]] || continue + gh api "repos/$GH_REPO/statuses/$GITHUB_SHA" \ + -f state=pending \ + -f "context=evidence/$platform/$task/$scope" \ + -f "target_url=$url" \ + -f "description=$description" + done + evidence: name: evidence (${{ matrix.platform }}) timeout-minutes: ${{ inputs.release_cell && 90 || 60 }} @@ -77,6 +105,8 @@ jobs: matrix: platform: ${{ fromJSON(inputs.platforms) }} runs-on: ${{ fromJSON('{"linux_x86_64":"ubuntu-latest","linux_arm64":"ubuntu-24.04-arm","macos_arm64":"macos-15","windows_x86_64":"windows-latest","windows_arm64":"windows-11-arm"}')[matrix.platform] || 'ubuntu-latest' }} + env: + BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }} permissions: contents: read steps: @@ -523,3 +553,41 @@ jobs: printf '```\n' fi } >> "$GITHUB_STEP_SUMMARY" + + finish: + name: evidence status (final) + needs: [pending, evidence] + if: ${{ always() && needs.pending.result == 'success' }} + runs-on: ubuntu-slim + timeout-minutes: 10 + permissions: + actions: read + statuses: write + steps: + - name: Post final statuses + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + DX_SCOPE: ${{ inputs.scope }} + DX_TASK: ${{ inputs.task }} + DX_PLATFORMS: ${{ inputs.platforms }} + run: | + set -o pipefail + task="$(printf '%s' "$DX_TASK" | tr -c 'a-z0-9_' '_' | cut -c1-40)" + scope="$(printf '%s' "$DX_SCOPE" | tr -c 'A-Za-z0-9_.-' '_' | cut -c1-60)" + url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + jobs="$(gh api "repos/$GH_REPO/actions/runs/$GITHUB_RUN_ID/jobs" --paginate --jq '.jobs[] | {name, conclusion}' | jq -s '.')" + printf '%s' "$DX_PLATFORMS" | jq -r '.[]? | strings' 2>/dev/null | while read -r platform; do + [[ "$platform" =~ ^[a-z0-9_]{1,40}$ ]] || continue + result="$(printf '%s' "$jobs" | jq -r --arg name "evidence ($platform)" '[.[] | select(.name == $name) | .conclusion][0] // "missing"')" + case "$result" in + success) state=success; description="evidence passed" ;; + failure) state=failure; description="evidence failed" ;; + *) state=error; description="run ended without a result" ;; + esac + gh api "repos/$GH_REPO/statuses/$GITHUB_SHA" \ + -f "state=$state" \ + -f "context=evidence/$platform/$task/$scope" \ + -f "target_url=$url" \ + -f "description=$description" + done diff --git a/AGENTS.md b/AGENTS.md index 8d449dd46..4890b09b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,7 +11,7 @@ This repository uses Bazel. - Three exceptions only: `--help` text stays one short line per flag, `LCOV_EXCL_*` markers keep their required short `reason:` plus `issue:`, and `# floor: ` stays above each `bazel_dep`. Tools parse it. - Write plain and human. Short sentences. Commands first. No filler, no essays, no AI tone. If a paragraph explains why something exists, delete it, except the "What rules_dx replaces" section in `README.md`. - Don't wait on upstream. Patch it here, or fetch the fixed rule set in rules_dx's own module extension, so consumers get the fix now. Then fix it upstream too: open an upstream issue first, then a small PR that follows that repo's contributing guide and style and includes a regression test. Link both from the rules_dx issue. Delete our patch once a release carries the fix. -- No `actions: write`? Comment `/evidence platforms=["macos_arm64"] task=test scope=//pkg/...` on the issue or PR. It dispatches `evidence.yml` and replies with the run URL. +- No `actions: write`? Comment `/evidence platforms=["macos_arm64"] task=test scope=//pkg/...` on the issue or PR. It dispatches `evidence.yml` and replies with the run URL. Wait with `gh pr checks --watch`. - A PR that touches `.github/review-paths.txt` paths gets `needs-review` and a second-session `Review:` comment before merge. See CONTRIBUTING.md. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 019732718..f647b3f09 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,10 @@ tools/dx test --affected -- --config=bb `--config=bb` reads the key from the environment through `tools/buildbuddy-credential-helper.sh` on Linux and macOS. Never put the key in `user.bazelrc` or any other rc file, and never pass `--announce_rc`. +## Evidence status + +Each `evidence.yml` run posts the commit status `evidence///` on the dispatched commit: pending at dispatch, then success, failure or error. `gh pr checks --watch` shows it. A second run for the same platform, task and scope on one commit replaces the status. + ## Issues and labels Use the bug template. Paste `dx status`. diff --git a/tools/ci/BUILD.bazel b/tools/ci/BUILD.bazel index fc442fbeb..5d0659f31 100644 --- a/tools/ci/BUILD.bazel +++ b/tools/ci/BUILD.bazel @@ -117,6 +117,28 @@ rust_test( ], ) +rust_test( + name = "evidence_status_test", + srcs = ["evidence_status_test.rs"], + aliases = aliases( + package_name = "tools/ci_tests", + normal = True, + normal_dev = True, + ), + crate_name = "evidence_status_test", + crate_root = "evidence_status_test.rs", + data = ["//.github:workflows/evidence.yml"], + deps = all_crate_deps( + package_name = "tools/ci_tests", + normal = True, + normal_dev = True, + ) + [ + "//tools/testing:dx_testing", + "@crates//:serde_json", + "@crates//:yaml_serde", + ], +) + rust_test( name = "docs_scope_test", srcs = ["docs_scope_test.rs"], diff --git a/tools/ci/evidence_status_test.rs b/tools/ci/evidence_status_test.rs new file mode 100644 index 000000000..43caf412c --- /dev/null +++ b/tools/ci/evidence_status_test.rs @@ -0,0 +1,107 @@ +use dx_testing::read_runfiles; +use serde_json::{Value, json}; + +fn workflow() -> Value { + yaml_serde::from_str(&read_runfiles(".github/workflows/evidence.yml")) + .expect("evidence.yml is valid YAML") +} + +fn run_text(workflow: &Value, job: &str) -> String { + workflow["jobs"][job]["steps"] + .as_array() + .unwrap_or_else(|| panic!("job {job} has steps")) + .iter() + .filter_map(|step| step["run"].as_str()) + .collect::>() + .join("\n") +} + +#[test] +fn the_only_trigger_is_workflow_dispatch() { + let workflow = workflow(); + let on = workflow + .get("on") + .and_then(Value::as_object) + .expect("evidence.yml has an on map"); + assert_eq!(on.keys().collect::>(), ["workflow_dispatch"]); +} + +#[test] +fn only_the_status_jobs_hold_statuses_write() { + let workflow = workflow(); + assert_eq!( + workflow["jobs"]["pending"]["permissions"], + json!({"statuses": "write"}) + ); + assert_eq!( + workflow["jobs"]["finish"]["permissions"], + json!({"actions": "read", "statuses": "write"}) + ); + assert_eq!( + workflow["jobs"]["evidence"]["permissions"], + json!({"contents": "read"}) + ); + assert_eq!(workflow["permissions"], json!({"contents": "read"})); +} + +#[test] +fn finish_always_runs_after_the_evidence_job() { + let workflow = workflow(); + let finish = &workflow["jobs"]["finish"]; + assert_eq!(finish["needs"], json!(["pending", "evidence"])); + let condition = finish["if"].as_str().expect("finish has an if"); + assert!(condition.contains("always()"), "{condition}"); + assert!(workflow["jobs"]["pending"].get("needs").is_none()); +} + +#[test] +fn statuses_use_the_commit_context_and_run_url() { + let workflow = workflow(); + for job in ["pending", "finish"] { + let text = run_text(&workflow, job); + assert!( + text.contains("context=evidence/$platform/$task/$scope"), + "{job}" + ); + assert!(text.contains("statuses/$GITHUB_SHA"), "{job}"); + assert!(text.contains("actions/runs/$GITHUB_RUN_ID"), "{job}"); + assert!(!text.contains("${{"), "{job} puts an expression in run"); + assert!(text.contains("set -o pipefail"), "{job}"); + assert!(!text.contains("|| true"), "{job} swallows a failed post"); + } +} + +#[test] +fn the_matrix_job_name_matches_what_finish_reads() { + let workflow = workflow(); + assert_eq!( + workflow["jobs"]["evidence"]["name"], + "evidence (${{ matrix.platform }})" + ); + assert!(run_text(&workflow, "finish").contains(r#""evidence ($platform)""#)); +} + +#[test] +fn only_the_evidence_job_receives_secrets() { + let workflow = workflow(); + assert!( + workflow + .get("env") + .is_none_or(|env| !env.to_string().contains("secrets.")) + ); + for job in ["pending", "finish"] { + let job_env = workflow["jobs"][job].get("env"); + assert!( + job_env.is_none_or(|env| !env.to_string().contains("secrets.")), + "{job} has secrets in env" + ); + for step in workflow["jobs"][job]["steps"].as_array().expect("steps") { + assert!( + step.get("env") + .is_none_or(|env| !env.to_string().contains("secrets.")), + "{job} step has secrets in env" + ); + } + } + assert!(workflow["jobs"]["evidence"]["env"]["BUILDBUDDY_API_KEY"].is_string()); +} diff --git a/tools/ci/secret_flags_test.rs b/tools/ci/secret_flags_test.rs index 8f69337cd..c74739dba 100644 --- a/tools/ci/secret_flags_test.rs +++ b/tools/ci/secret_flags_test.rs @@ -224,10 +224,11 @@ fn every_upload_follows_a_matching_scan() { let mut uploads = 0; for path in dx_testing::workflow_files() { let document = parse(&path); - if document["env"]["BUILDBUDDY_API_KEY"].is_null() { - continue; - } + let workflow_key = !document["env"]["BUILDBUDDY_API_KEY"].is_null(); for (id, steps) in jobs(&document) { + if !workflow_key && document["jobs"][&id]["env"]["BUILDBUDDY_API_KEY"].is_null() { + continue; + } for (index, step) in steps.iter().enumerate() { if !is_upload(step) { continue; @@ -392,7 +393,10 @@ fn helper_action_writes_a_working_helper() { let step = named_step(SETUP_ACTION, HELPER_STEP); let script = step["run"].as_str().expect("helper run"); let key = ("BUILDBUDDY_API_KEY", "k3y"); - let cache = ("DX_REMOTE_CACHE_ARG", "--remote_cache=grpcs://remote.example.com"); + let cache = ( + "DX_REMOTE_CACHE_ARG", + "--remote_cache=grpcs://remote.example.com", + ); let header = ("REMOTE_CACHE_HEADER", "x-t=s3cr3t"); let own_host = ( "DX_REMOTE_CACHE_ARG", @@ -410,7 +414,9 @@ fn helper_action_writes_a_working_helper() { "key with a custom cache", &[key, cache], 0, - Some("--remote_cache=grpcs://remote.example.com --credential_helper=remote.buildbuddy.io="), + Some( + "--remote_cache=grpcs://remote.example.com --credential_helper=remote.buildbuddy.io=", + ), Some(r#"{"headers":{"x-buildbuddy-api-key":["k3y"]}}"#), ), ( @@ -427,9 +433,21 @@ fn helper_action_writes_a_working_helper() { Some("--credential_helper=cache.example.com="), Some(r#"{"headers":{"x-t":["s3cr3t"]}}"#), ), - ("no equals", &[("REMOTE_CACHE_HEADER", "novalue")], 1, None, None), + ( + "no equals", + &[("REMOTE_CACHE_HEADER", "novalue")], + 1, + None, + None, + ), ("quote", &[("REMOTE_CACHE_HEADER", "x=a\"b")], 1, None, None), - ("percent", &[("REMOTE_CACHE_HEADER", "x=a%b")], 1, None, None), + ( + "percent", + &[("REMOTE_CACHE_HEADER", "x=a%b")], + 1, + None, + None, + ), ("nothing set", &[], 0, None, None), ]; for (name, env, status, args, printed) in cases { diff --git a/tools/ci/workflow_permissions_test.rs b/tools/ci/workflow_permissions_test.rs index 6e6186356..dd0e54696 100644 --- a/tools/ci/workflow_permissions_test.rs +++ b/tools/ci/workflow_permissions_test.rs @@ -39,6 +39,7 @@ const MARKERS: &[(&str, &[&str])] = &[ ("issues", &["gh issue ", "gh pr comment"]), ("packages", &["docker push", "deploy --apply"]), ("pages", &["actions/deploy-pages@"]), + ("statuses", &["/statuses/"]), ( "pull-requests", &[