From 17a9256555f7402c0930011339f4d4224b68de74 Mon Sep 17 00:00:00 2001 From: kubbot Date: Thu, 27 Aug 2026 20:42:03 +0800 Subject: [PATCH] Add fail-closed Slack to Codex control plane --- .../workflows/automation-control-plane.yml | 32 ++++ AUTOMATION.md | 19 ++- automation/codex-targets.yml | 57 +++++++ automation/codex-task-contract.md | 60 ++++++++ automation/prompts/daily-ops.md | 27 ++-- automation/prompts/weekly-eval.md | 4 +- automation/scripts/codex-review-health.sh | 121 +++++++++++++++ .../scripts/codex-review-health.test.sh | 82 ++++++++++ automation/scripts/codex-task-lint.sh | 144 ++++++++++++++++++ automation/scripts/codex-task-lint.test.sh | 111 ++++++++++++++ automation/scripts/registry.test.sh | 46 +++++- automation/slack-routing.md | 21 ++- automation/templates/codex-task.md | 30 ++++ 13 files changed, 734 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/automation-control-plane.yml create mode 100644 automation/codex-targets.yml create mode 100644 automation/codex-task-contract.md create mode 100755 automation/scripts/codex-review-health.sh create mode 100755 automation/scripts/codex-review-health.test.sh create mode 100755 automation/scripts/codex-task-lint.sh create mode 100755 automation/scripts/codex-task-lint.test.sh create mode 100644 automation/templates/codex-task.md diff --git a/.github/workflows/automation-control-plane.yml b/.github/workflows/automation-control-plane.yml new file mode 100644 index 0000000..66fed20 --- /dev/null +++ b/.github/workflows/automation-control-plane.yml @@ -0,0 +1,32 @@ +name: Automation control plane + +on: + pull_request: + paths: + - .github/workflows/automation-control-plane.yml + - AUTOMATION.md + - automation/** + push: + branches: [main] + paths: + - .github/workflows/automation-control-plane.yml + - AUTOMATION.md + - automation/** + +permissions: + contents: read + +concurrency: + group: automation-control-plane-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + contracts: + name: Validate automation contracts + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - name: Run control-plane tests + run: automation/scripts/registry.test.sh diff --git a/AUTOMATION.md b/AUTOMATION.md index 81fe474..7c465e9 100644 --- a/AUTOMATION.md +++ b/AUTOMATION.md @@ -48,6 +48,13 @@ evidence location, CI workflow (when present), acceptance statement, privacy boundary, and explicit coverage gap. These fields guide discovery; the product repository and immutable run artifacts remain the source of truth. +The desired one-environment/one-repository mapping for Slack-driven Codex work +lives in [`automation/codex-targets.yml`](automation/codex-targets.yml). A +target marked `pending_verification` or `paused` is not runnable. Every request +must pass the fail-closed +[`automation/codex-task-contract.md`](automation/codex-task-contract.md) before +an operator mentions `@Codex`. + ## Operator entry points - Run `automation/scripts/portfolio-health.sh` for a read-only JSON Lines @@ -74,8 +81,16 @@ repository and immutable run artifacts remain the source of truth. shell (including shell process substitution); scan failures and matches are configuration drift rather than a silent clean result. - Run `automation/scripts/registry.test.sh` after changing repository - lifecycle or Eval metadata. Every active product must keep a complete, - machine-readable Eval discovery contract. + lifecycle, Eval metadata, or Codex targets. Every active product must keep a + complete, machine-readable Eval discovery contract and exactly one desired + Codex environment mapping. +- Run `automation/scripts/codex-task-lint.sh TASK.md` before dispatching a + Slack task to Codex. It rejects ambiguous, secret-bearing, unapproved, or + unverified targets and requires the source thread, acceptance evidence, + recovery path, and human boundary. +- Run `automation/scripts/codex-review-health.sh` for a read-only JSON Lines + inventory of Codex environment readiness, applicable `AGENTS.md` files, and + repository-owned `## Code Review Rules` coverage. - Run `automation/scripts/eval-health.sh` before the weekly Eval review. It proves the declared suite and workflow exist on the default branch, finds a successful run where the exact Eval job and step actually executed, verifies diff --git a/automation/codex-targets.yml b/automation/codex-targets.yml new file mode 100644 index 0000000..7cb993d --- /dev/null +++ b/automation/codex-targets.yml @@ -0,0 +1,57 @@ +version: 1 + +slack: + channel: build + task_thread_required: true + completion_posting: link_only_preferred_if_workspace_supports_it + +policy: + repository_scope: selected_public_repositories + environment_scope: one_environment_with_one_repository + ambiguous_target: reject + unverified_target: reject + review_rollout: manual_before_automatic + approval_boundary: explicit_human_message + +targets: + talent-signal: + repository: getyak/talent-signal + desired_environment: getyak-talent-signal + repo_map: [getyak/talent-signal] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + daypage: + repository: getyak/daypage + desired_environment: getyak-daypage + repo_map: [getyak/daypage] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + FitCoach: + repository: getyak/FitCoach + desired_environment: getyak-fitcoach + repo_map: [getyak/FitCoach] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + apply-agent: + repository: getyak/apply-agent + desired_environment: getyak-apply-agent + repo_map: [getyak/apply-agent] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + telepace-next: + repository: getyak/telepace-next + desired_environment: getyak-telepace-next + repo_map: [getyak/telepace-next] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + solo-compass: + repository: getyak/solo-compass + desired_environment: getyak-solo-compass + repo_map: [getyak/solo-compass] + integration_state: pending_verification + review_mode: manual_p0_p1_after_setup + +excluded: + - private signing repositories + - credential and secret repositories + - repositories outside getyak diff --git a/automation/codex-task-contract.md b/automation/codex-task-contract.md new file mode 100644 index 0000000..f214e7b --- /dev/null +++ b/automation/codex-task-contract.md @@ -0,0 +1,60 @@ +# Slack to Codex Task Contract + +This contract makes a Slack request safe to hand to Codex. It is intentionally +fail closed: a task is not ready merely because it is understandable to a +person. + +Codex in Slack can use the most recently used environment when a request is +ambiguous, and a chat runs against the default branch of the first repository +in an environment's repository map. Therefore every Getyak Codex environment +has one repository and every request names both values explicitly. The desired +mapping and its verified state live in +[`codex-targets.yml`](codex-targets.yml). This defensive rule follows the +[official Codex in Slack behavior](https://learn.chatgpt.com/docs/third-party/slack). + +## Required envelope + +Start from [`templates/codex-task.md`](templates/codex-task.md). A task must +contain: + +- exactly one approved public Getyak repository; +- that repository's exact desired Codex environment; +- the originating `#build` Slack thread permalink; +- one stable fingerprint for retries and follow-up; +- one observable outcome and one durable GitHub evidence URL; +- explicit constraints, acceptance checks, and rollback or recovery steps; +- the human boundary for deploys, secrets, permissions, OAuth, external + communication, and other consequential actions. + +Run the local validator before mentioning `@Codex`: + +```bash +automation/scripts/codex-task-lint.sh path/to/task.md +``` + +The validator rejects an environment whose `integration_state` is not `ready`. +`CODEX_TASK_ALLOW_PENDING_TARGET=true` exists only for testing the contract +before an integration is configured; it must not be used to dispatch a real +Slack task. + +The resulting pull request is the durable artifact. Diagnosis, approval, and +completion remain replies in the originating Slack thread. A completion link +does not prove acceptance: verify the named checks and evidence before closing +the task. + +## Review rollout + +Begin with an explicit `@codex review` on selected pull requests. Add two or +three concise, outcome-focused rules under `## Code Review Rules` in the +repository's applicable `AGENTS.md`. Mechanical formatting and test checks stay +in CI. Enable automatic review only after the manual sample demonstrates useful +signal and an acceptable false-positive rate. + +The rollout follows the +[official Codex code-review guidance](https://learn.chatgpt.com/docs/third-party/github): +keep repository-specific review instructions concise, and use CI for +mechanical checks and enforcement. + +Use `automation/scripts/codex-review-health.sh` for a read-only inventory of +environment readiness and review-rule coverage. A pending integration or +missing review rule is planned setup debt, not an hourly incident. diff --git a/automation/prompts/daily-ops.md b/automation/prompts/daily-ops.md index 991a52c..b81f862 100644 --- a/automation/prompts/daily-ops.md +++ b/automation/prompts/daily-ops.md @@ -6,7 +6,11 @@ Run once on weekdays in `Asia/Shanghai`. This is a read-mostly control loop. `automation/slack-routing.md` from `getyak/.github`. 2. Run `automation/scripts/portfolio-health.sh` or perform equivalent read-only checks with GitHub CLI for every public, non-archived repository. -3. Review `configuration_drift`. Secret scanning, push protection, Dependabot +3. Run `automation/scripts/codex-review-health.sh`. Treat an environment that + changed from `ready` to another state or a newly missing review rule as + configuration drift. Known `pending_verification` setup is planned debt and + must not create repeated hourly or daily incidents. +4. Review `configuration_drift`. Secret scanning, push protection, Dependabot security updates, read-only default workflow permissions, and immutable Action SHA enforcement are the expected public-repository baseline. Each repository must allow only GitHub-owned Actions plus the exact external @@ -18,26 +22,29 @@ Run once on weekdays in `Asia/Shanghai`. This is a read-mostly control loop. `workflow_supply_chain` scan, an unpinned external Action, or any remote script piped directly into a shell as configuration drift requiring inspection. -4. Inspect only newly failing or still-unresolved workflow runs, deployment +5. Inspect only newly failing or still-unresolved workflow runs, deployment failures, high/critical Dependabot alerts, code-scanning alerts, and missing security coverage. Treat `recent_failed_runs` as default-branch candidates, not proof of an incident, and inspect failure steps before classifying them. -5. Check the automation host for disk pressure, memory pressure, unhealthy +6. Check the automation host for disk pressure, memory pressure, unhealthy containers, stopped required services, and expiring certificates. Read the cached macOS update inventory and reject it as stale when its successful scan is older than 48 hours. A restart update is maintenance debt, not an incident; include it only when its version differs from the last version recorded after a verified digest post. Never include secret values or private data in output. -6. Compare each finding with the latest `#ops` threads. Reuse an unresolved +7. Compare each finding with the latest `#ops` threads. Reuse an unresolved thread with the same fingerprint; do not create a duplicate root. -7. Post immediately only for new SEV-1/2 findings. Otherwise create at most one +8. Post immediately only for new SEV-1/2 findings. Otherwise create at most one concise daily digest in `#ops`. If there is no actionable change, post nothing. -8. Put scoped engineering work in `#build`, preferably as a reply linking the - original `#ops` thread. Do not deploy, rotate secrets, change permissions, - grant OAuth access, install system updates, restart the host, or send external - communications without explicit human approval naming the action and target. -9. When evidence confirms recovery, reply once to the original incident and +9. Put scoped engineering work in `#build`, preferably as a reply linking the + original `#ops` thread. Build the request from + `automation/templates/codex-task.md` and require + `automation/scripts/codex-task-lint.sh` to pass before mentioning `@Codex`. + Do not deploy, rotate secrets, change permissions, grant OAuth access, + install system updates, restart the host, or send external communications + without explicit human approval naming the action and target. +10. When evidence confirms recovery, reply once to the original incident and mark the state resolved. Never infer recovery from elapsed time. The final task result must state what changed since the previous run, what was diff --git a/automation/prompts/weekly-eval.md b/automation/prompts/weekly-eval.md index b229fef..81bc900 100644 --- a/automation/prompts/weekly-eval.md +++ b/automation/prompts/weekly-eval.md @@ -20,7 +20,9 @@ Run once each Friday in `Asia/Shanghai`. a coverage state of `complete`, `partial`, or `invalid`. 6. Route real regressions or meaningful improvements to one weekly `#signals` digest. Put implementation work in `#build` only when it can be expressed as - a bounded task with acceptance evidence. + a bounded task with acceptance evidence. Build it from + `automation/templates/codex-task.md` and do not mention `@Codex` unless + `automation/scripts/codex-task-lint.sh` passes against a `ready` target. 7. Escalate to `#hq` only when a result changes a product decision, release decision, safety boundary, or portfolio priority. Do not copy the full `#signals` digest. diff --git a/automation/scripts/codex-review-health.sh b/automation/scripts/codex-review-health.sh new file mode 100755 index 0000000..6da6ef9 --- /dev/null +++ b/automation/scripts/codex-review-health.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +set -euo pipefail + +for required_command in gh jq ruby base64 grep; do + command -v "$required_command" >/dev/null || { + echo "$required_command is required" >&2 + exit 2 + } +done + +repository_root="$(cd "$(dirname "$BASH_SOURCE")/../.." && pwd)" +targets_path="${CODEX_TARGETS_PATH:-$repository_root/automation/codex-targets.yml}" +retry_delay="${CODEX_REVIEW_API_RETRY_DELAY_SECONDS:-1}" + +api_json() { + local endpoint="$1" + local attempt output + for attempt in 1 2 3; do + if output="$(gh api "$endpoint" 2>/dev/null)" && jq -e . >/dev/null 2>&1 <<<"$output"; then + printf '%s\n' "$output" + return 0 + fi + if ((attempt < 3)); then + sleep "$retry_delay" + fi + done + return 1 +} + +target_records="$({ + ruby - "$targets_path" <<'RUBY' +require "base64" +require "json" +require "yaml" + +YAML.load_file(ARGV.fetch(0)).fetch("targets").each do |name, target| + puts Base64.strict_encode64(JSON.generate( + name: name, + repository: target.fetch("repository"), + desired_environment: target.fetch("desired_environment"), + integration_state: target.fetch("integration_state"), + review_mode: target.fetch("review_mode") + )) +end +RUBY +} )" + +while IFS= read -r encoded_record; do + [[ -n "$encoded_record" ]] || continue + record="$(printf '%s' "$encoded_record" | base64 --decode)" + repository="$(jq -r '.repository' <<<"$record")" + desired_environment="$(jq -r '.desired_environment' <<<"$record")" + integration_state="$(jq -r '.integration_state' <<<"$record")" + review_mode="$(jq -r '.review_mode' <<<"$record")" + default_branch="" + repository_state="unavailable" + agents_files='[]' + review_rules_files='[]' + review_debt='[]' + + if repository_json="$(api_json "repos/$repository")"; then + default_branch="$(jq -r '.default_branch // empty' <<<"$repository_json")" + repository_state="available" + else + review_debt='["repository_unavailable"]' + fi + + if [[ -n "$default_branch" ]]; then + if tree_json="$(api_json "repos/$repository/git/trees/$default_branch?recursive=1")" && + jq -e '.truncated != true and (.tree | type == "array")' >/dev/null 2>&1 <<<"$tree_json"; then + agents_files="$(jq -c '[.tree[]? | select(.type == "blob" and (.path | test("(^|/)AGENTS\\.md$"))) | .path] | sort' <<<"$tree_json")" + + while IFS=$'\t' read -r agents_path blob_sha; do + [[ -n "$agents_path" && -n "$blob_sha" ]] || continue + if blob_json="$(api_json "repos/$repository/git/blobs/$blob_sha")"; then + content="$(jq -r '.content // empty' <<<"$blob_json" | tr -d '\n' | base64 --decode 2>/dev/null || true)" + if grep -Eq '^##[[:space:]]+Code Review Rules[[:space:]]*$' <<<"$content"; then + review_rules_files="$(jq -c --arg path "$agents_path" '. + [$path]' <<<"$review_rules_files")" + fi + else + review_debt="$(jq -c '. + ["agents_file_unavailable"] | unique' <<<"$review_debt")" + fi + done < <(jq -r '.tree[]? | select(.type == "blob" and (.path | test("(^|/)AGENTS\\.md$"))) | [.path, .sha] | @tsv' <<<"$tree_json") + else + review_debt="$(jq -c '. + ["default_branch_tree_unavailable"] | unique' <<<"$review_debt")" + fi + fi + + if [[ "$integration_state" != "ready" ]]; then + review_debt="$(jq -c '. + ["codex_slack_environment_not_ready"] | unique' <<<"$review_debt")" + fi + if [[ "$(jq 'length' <<<"$agents_files")" == "0" ]]; then + review_debt="$(jq -c '. + ["agents_instructions_missing"] | unique' <<<"$review_debt")" + elif [[ "$(jq 'length' <<<"$review_rules_files")" == "0" ]]; then + review_debt="$(jq -c '. + ["code_review_rules_missing"] | unique' <<<"$review_debt")" + fi + + jq -cn \ + --arg observed_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg repository "$repository" \ + --arg repository_state "$repository_state" \ + --arg default_branch "$default_branch" \ + --arg desired_environment "$desired_environment" \ + --arg integration_state "$integration_state" \ + --arg review_mode "$review_mode" \ + --argjson agents_files "$agents_files" \ + --argjson review_rules_files "$review_rules_files" \ + --argjson review_debt "$review_debt" \ + '{ + observed_at: $observed_at, + repository: $repository, + repository_state: $repository_state, + default_branch: ($default_branch | if length > 0 then . else null end), + desired_environment: $desired_environment, + integration_state: $integration_state, + review_mode: $review_mode, + agents_files: $agents_files, + code_review_rules_files: $review_rules_files, + review_debt: $review_debt + }' +done <<<"$target_records" diff --git a/automation/scripts/codex-review-health.test.sh b/automation/scripts/codex-review-health.test.sh new file mode 100755 index 0000000..475651c --- /dev/null +++ b/automation/scripts/codex-review-health.test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v jq >/dev/null || { + echo "jq is required" >&2 + exit 2 +} + +repository_root="$(cd "$(dirname "$BASH_SOURCE")/../.." && pwd)" +test_root="$(mktemp -d "${TMPDIR:-/tmp}/codex-review-health-test.XXXXXX")" +trap 'rm -rf "$test_root"' EXIT +mkdir -p "$test_root/bin" + +cat >"$test_root/targets.yml" <<'YAML' +version: 1 +targets: + example: + repository: getyak/example + desired_environment: getyak-example + repo_map: [getyak/example] + integration_state: ready + review_mode: manual_p0_p1_after_setup +YAML + +cat >"$test_root/bin/gh" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail + +[[ "$1" == "api" ]] || exit 1 +endpoint="$2" +case "$endpoint" in + repos/getyak/example) + printf '%s\n' '{"default_branch":"main"}' + ;; + 'repos/getyak/example/git/trees/main?recursive=1') + printf '%s\n' '{"truncated":false,"tree":[{"path":"AGENTS.md","type":"blob","sha":"agents-sha"},{"path":"src/app.ts","type":"blob","sha":"app-sha"}]}' + ;; + repos/getyak/example/git/blobs/agents-sha) + if [[ "${MOCK_NO_RULES:-0}" == "1" ]]; then + content='# Instructions\n\nKeep tests green.\n' + else + content='# Instructions\n\n## Code Review Rules\n\n- Flag unsafe migrations.\n' + fi + encoded="$(printf '%b' "$content" | base64 | tr -d '\n')" + printf '{"content":"%s"}\n' "$encoded" + ;; + *) + exit 1 + ;; +esac +MOCK +chmod +x "$test_root/bin/gh" + +snapshot="$( + PATH="$test_root/bin:$PATH" \ + CODEX_TARGETS_PATH="$test_root/targets.yml" \ + CODEX_REVIEW_API_RETRY_DELAY_SECONDS=0 \ + "$repository_root/automation/scripts/codex-review-health.sh" +)" +jq -e ' + .repository == "getyak/example" and + .repository_state == "available" and + .default_branch == "main" and + .integration_state == "ready" and + .agents_files == ["AGENTS.md"] and + .code_review_rules_files == ["AGENTS.md"] and + .review_debt == [] +' <<<"$snapshot" >/dev/null + +missing_rules_snapshot="$( + PATH="$test_root/bin:$PATH" \ + MOCK_NO_RULES=1 \ + CODEX_TARGETS_PATH="$test_root/targets.yml" \ + CODEX_REVIEW_API_RETRY_DELAY_SECONDS=0 \ + "$repository_root/automation/scripts/codex-review-health.sh" +)" +jq -e ' + .code_review_rules_files == [] and + .review_debt == ["code_review_rules_missing"] +' <<<"$missing_rules_snapshot" >/dev/null + +echo "Codex review health tests passed" diff --git a/automation/scripts/codex-task-lint.sh b/automation/scripts/codex-task-lint.sh new file mode 100755 index 0000000..8ff7304 --- /dev/null +++ b/automation/scripts/codex-task-lint.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +command -v ruby >/dev/null || { + echo "ruby is required" >&2 + exit 2 +} + +repository_root="$(cd "$(dirname "$BASH_SOURCE")/../.." && pwd)" +targets_path="${CODEX_TARGETS_PATH:-$repository_root/automation/codex-targets.yml}" +task_path="${1:--}" + +temporary_task="" +if [[ "$task_path" == "-" ]]; then + temporary_task="$(mktemp "${TMPDIR:-/tmp}/codex-task.XXXXXX")" + trap 'rm -f "$temporary_task"' EXIT + task_path="$temporary_task" + cat >"$task_path" +elif [[ ! -f "$task_path" ]]; then + echo "task file not found: $task_path" >&2 + exit 2 +fi + +ruby - "$targets_path" "$task_path" <<'RUBY' +require "json" +require "yaml" + +targets_path = ARGV.fetch(0) +task_path = ARGV.fetch(1) +document = File.read(task_path) +config = YAML.load_file(targets_path) +targets = config.fetch("targets") +errors = [] + +if document.bytesize > 32_768 + errors << "task envelope exceeds 32 KiB" +end + +secret_patterns = { + "private key" => /-----BEGIN [A-Z0-9 ]*PRIVATE KEY-----/, + "GitHub token" => /\b(?:github_pat_|gh[oprsu]_[A-Za-z0-9]{20,})/, + "OpenAI-style key" => /\bsk-[A-Za-z0-9_-]{20,}/, + "Slack token" => /\bxox[baprs]-[A-Za-z0-9-]{10,}/, + "AWS access key" => /\bAKIA[0-9A-Z]{16}\b/, + "bearer credential" => /\bBearer\s+[A-Za-z0-9._~+\/-]{20,}={0,2}/i +} +secret_patterns.each do |label, pattern| + errors << "task envelope contains a possible #{label}" if document.match?(pattern) +end + +fields = {} +%w[Repository Environment Fingerprint Outcome Evidence].each do |name| + matches = document.scan(/^#{Regexp.escape(name)}:\s*(.+?)\s*$/i).flatten + if matches.length != 1 + errors << "#{name} must appear exactly once" + else + fields[name.downcase] = matches.first.strip + end +end + +channel_matches = document.scan(/^Slack channel:\s*(.+?)\s*$/i).flatten +if channel_matches.length != 1 || channel_matches.first.strip != "#build" + errors << "Slack channel must appear exactly once and be #build" +end + +source_matches = document.scan(/^Source thread:\s*(.+?)\s*$/i).flatten +if source_matches.length != 1 + errors << "Source thread must appear exactly once" +else + source_thread = source_matches.first.strip + unless source_thread.match?(%r{\Ahttps://getyak\.slack\.com/archives/C[A-Z0-9]+/p[0-9]+(?:\?thread_ts=[0-9.]+&cid=C[A-Z0-9]+)?\z}) + errors << "Source thread must be a getyak Slack message permalink" + end +end + +sections = {} +heading_matches = document.to_enum(:scan, /^##\s+(.+?)\s*$/).map do + [Regexp.last_match.begin(0), Regexp.last_match.end(0), Regexp.last_match(1).downcase] +end +heading_matches.each_with_index do |(_start_at, body_start, name), index| + body_end = heading_matches.fetch(index + 1, [document.length]).first + sections[name] = document[body_start...body_end] +end + +required_sections = ["constraints", "acceptance", "rollback / recovery", "human boundary"] +required_sections.each do |name| + heading_count = heading_matches.count { |_start_at, _body_start, heading| heading == name } + errors << "section ## #{name.split.map(&:capitalize).join(' ')} must appear exactly once" unless heading_count == 1 + body = sections[name].to_s + errors << "section ## #{name.split.map(&:capitalize).join(' ')} must contain a list item" unless body.match?(/^\s*-\s+\S/) +end + +human_boundary = sections["human boundary"].to_s +{ + "deploy" => /deploy/i, + "secret or credential" => /secret|credential/i, + "permission or OAuth" => /permission|oauth/i, + "external communication" => /external|communicat/i, + "explicit human approval" => /explicit human|human (?:message|approval)/i +}.each do |label, pattern| + errors << "human boundary must cover #{label}" unless human_boundary.match?(pattern) +end + +repository = fields["repository"] +environment = fields["environment"] +target_name, target = targets.find { |_name, value| value["repository"] == repository } + +if repository && target.nil? + errors << "Repository is not an approved Codex target" +elsif target + errors << "Environment does not match the repository's desired environment" unless environment == target["desired_environment"] + errors << "Codex target must map exactly one repository" unless target["repo_map"] == [repository] + if target["integration_state"] != "ready" && ENV["CODEX_TASK_ALLOW_PENDING_TARGET"] != "true" + errors << "Codex target is #{target["integration_state"].inspect}, not ready" + end + + evidence = fields["evidence"] + durable_evidence = %r{\Ahttps://github\.com/#{Regexp.escape(repository)}/(?:issues/[0-9]+|pull/[0-9]+|actions/runs/[0-9]+|commit/[0-9a-fA-F]{7,40}|blob/[0-9a-fA-F]{40}/[^?#]+)(?:[?#].*)?\z} + errors << "Evidence must be an issue, pull request, run, commit, or immutable blob URL inside the named GitHub repository" unless evidence&.match?(durable_evidence) + + fingerprint = fields["fingerprint"] + unless fingerprint&.match?(%r{\A#{Regexp.escape(target_name)}:[a-z0-9][a-z0-9_-]*:[a-zA-Z0-9][a-zA-Z0-9._-]*\z}) + errors << "Fingerprint must use REPOSITORY:CLASS:STABLE_ID" + end +end + +unless fields["outcome"].to_s.match?(/\S/) && fields["outcome"].to_s.length >= 12 + errors << "Outcome must describe one observable result" +end + +unless errors.empty? + warn errors.uniq.join("\n") + exit 1 +end + +puts JSON.generate( + valid: true, + repository: repository, + environment: environment, + target: target_name, + integration_state: target.fetch("integration_state"), + fingerprint: fields.fetch("fingerprint") +) +RUBY diff --git a/automation/scripts/codex-task-lint.test.sh b/automation/scripts/codex-task-lint.test.sh new file mode 100755 index 0000000..2e6a140 --- /dev/null +++ b/automation/scripts/codex-task-lint.test.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +for required_command in jq sed; do + command -v "$required_command" >/dev/null || { + echo "$required_command is required" >&2 + exit 2 + } +done + +repository_root="$(cd "$(dirname "$BASH_SOURCE")/../.." && pwd)" +test_root="$(mktemp -d "${TMPDIR:-/tmp}/codex-task-lint-test.XXXXXX")" +trap 'rm -rf "$test_root"' EXIT + +write_valid_task() { + cat >"$1" <<'TASK' +# Codex task + +Repository: getyak/daypage +Environment: getyak-daypage +Slack channel: #build +Source thread: https://getyak.slack.com/archives/C0123456789/p1234567890123456 +Fingerprint: daypage:task:issue-904 +Outcome: Make the backend static checks runner-portable. +Evidence: https://github.com/getyak/daypage/issues/904 + +## Constraints + +- Preserve the current test semantics. + +## Acceptance + +- The backend static check and pull-request checks pass. + +## Rollback / recovery + +- Revert the pull request if the check changes semantics. + +## Human boundary + +- Do not deploy, expose secrets or credentials, change permissions or OAuth, + communicate externally, or proceed without an explicit human message. +TASK +} + +assert_rejected() { + local task_path="$1" + local expected="$2" + if CODEX_TASK_ALLOW_PENDING_TARGET=true \ + "$repository_root/automation/scripts/codex-task-lint.sh" "$task_path" \ + >"$test_root/stdout" 2>"$test_root/stderr"; then + echo "expected task to be rejected: $expected" >&2 + exit 1 + fi + grep -Fq "$expected" "$test_root/stderr" +} + +valid_task="$test_root/valid.md" +write_valid_task "$valid_task" + +if "$repository_root/automation/scripts/codex-task-lint.sh" "$valid_task" \ + >"$test_root/stdout" 2>"$test_root/stderr"; then + echo "pending target unexpectedly passed without test override" >&2 + exit 1 +fi +grep -Fq 'not ready' "$test_root/stderr" + +valid_output="$(CODEX_TASK_ALLOW_PENDING_TARGET=true \ + "$repository_root/automation/scripts/codex-task-lint.sh" "$valid_task")" +jq -e ' + .valid == true and + .repository == "getyak/daypage" and + .environment == "getyak-daypage" and + .integration_state == "pending_verification" +' <<<"$valid_output" >/dev/null + +stdin_output="$(CODEX_TASK_ALLOW_PENDING_TARGET=true \ + "$repository_root/automation/scripts/codex-task-lint.sh" - <"$valid_task")" +jq -e '.valid == true and .repository == "getyak/daypage"' <<<"$stdin_output" >/dev/null + +environment_task="$test_root/environment.md" +cp "$valid_task" "$environment_task" +sed -i.bak 's/Environment: getyak-daypage/Environment: getyak-other/' "$environment_task" +assert_rejected "$environment_task" "Environment does not match" + +rollback_task="$test_root/rollback.md" +cp "$valid_task" "$rollback_task" +sed -i.bak '/^## Rollback \/ recovery$/,/^## Human boundary$/ { /^-/d; }' "$rollback_task" +assert_rejected "$rollback_task" "Rollback / Recovery must contain a list item" + +secret_task="$test_root/secret.md" +cp "$valid_task" "$secret_task" +sed -i.bak 's/Preserve the current test semantics./Preserve token github_pat_ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890./' "$secret_task" +assert_rejected "$secret_task" "possible GitHub token" + +source_task="$test_root/source.md" +cp "$valid_task" "$source_task" +sed -i.bak 's#https://getyak.slack.com/archives/C0123456789/p1234567890123456#https://example.invalid/thread#' "$source_task" +assert_rejected "$source_task" "getyak Slack message permalink" + +mutable_evidence_task="$test_root/mutable-evidence.md" +cp "$valid_task" "$mutable_evidence_task" +sed -i.bak 's#https://github.com/getyak/daypage/issues/904#https://github.com/getyak/daypage/blob/main/README.md#' "$mutable_evidence_task" +assert_rejected "$mutable_evidence_task" "issue, pull request, run, commit, or immutable blob" + +excluded_task="$test_root/excluded.md" +cp "$valid_task" "$excluded_task" +sed -i.bak 's#getyak/daypage#getyak/signing#g' "$excluded_task" +assert_rejected "$excluded_task" "not an approved Codex target" + +echo "Codex task lint tests passed" diff --git a/automation/scripts/registry.test.sh b/automation/scripts/registry.test.sh index aa80897..2bff468 100755 --- a/automation/scripts/registry.test.sh +++ b/automation/scripts/registry.test.sh @@ -3,10 +3,13 @@ set -euo pipefail repository_root="$(cd "$(dirname "$BASH_SOURCE")/../.." && pwd)" -ruby - "$repository_root/automation/registry.yml" <<'RUBY' +ruby - \ + "$repository_root/automation/registry.yml" \ + "$repository_root/automation/codex-targets.yml" <<'RUBY' require "yaml" registry = YAML.load_file(ARGV.fetch(0)) +codex_targets = YAML.load_file(ARGV.fetch(1)) repositories = registry.fetch("repositories") required_fields = %w[ maturity @@ -55,8 +58,47 @@ repositories.each do |name, repository| end end +active_repositories = repositories.each_with_object([]) do |(name, repository), values| + values << name if repository["lifecycle"] == "active" +end.sort +targets = codex_targets.fetch("targets") +unless targets.keys.sort == active_repositories + errors << "Codex targets must exactly match active repositories" +end + +allowed_integration_states = %w[pending_verification ready paused] +allowed_review_modes = %w[manual_p0_p1_after_setup automatic_p0_p1 paused] +desired_environments = [] +targets.each do |name, target| + expected_repository = "getyak/#{name}" + errors << "#{name}: Codex repository must be #{expected_repository}" unless target["repository"] == expected_repository + errors << "#{name}: Codex repo_map must contain only #{expected_repository}" unless target["repo_map"] == [expected_repository] + if target["desired_environment"].to_s.empty? + errors << "#{name}: desired_environment is missing" + else + desired_environments << target["desired_environment"] + end + unless allowed_integration_states.include?(target["integration_state"]) + errors << "#{name}: unsupported integration_state #{target["integration_state"].inspect}" + end + unless allowed_review_modes.include?(target["review_mode"]) + errors << "#{name}: unsupported review_mode #{target["review_mode"].inspect}" + end +end +errors << "Codex desired environments must be unique" unless desired_environments.uniq.length == desired_environments.length +errors << "Codex Slack route must be build" unless codex_targets.dig("slack", "channel") == "build" +errors << "Codex ambiguous targets must be rejected" unless codex_targets.dig("policy", "ambiguous_target") == "reject" +errors << "Codex unverified targets must be rejected" unless codex_targets.dig("policy", "unverified_target") == "reject" + abort(errors.join("\n")) unless errors.empty? -puts "registry Eval contracts passed" +puts "registry Eval and Codex target contracts passed" RUBY bash -n "$repository_root/automation/scripts/eval-health.sh" +bash -n "$repository_root/automation/scripts/codex-task-lint.sh" +bash -n "$repository_root/automation/scripts/codex-task-lint.test.sh" +bash -n "$repository_root/automation/scripts/codex-review-health.sh" +bash -n "$repository_root/automation/scripts/codex-review-health.test.sh" + +"$repository_root/automation/scripts/codex-task-lint.test.sh" +"$repository_root/automation/scripts/codex-review-health.test.sh" diff --git a/automation/slack-routing.md b/automation/slack-routing.md index 1873d05..1308448 100644 --- a/automation/slack-routing.md +++ b/automation/slack-routing.md @@ -103,8 +103,10 @@ message that names the action and target. A reaction alone is insufficient. ## Codex and ChatGPT roles - Use **Codex in `#build`** for repository-scoped diagnosis, implementation, - tests, and pull requests. Start from a message containing the repository, - desired outcome, constraints, and evidence URL. + tests, and pull requests. Start from a task that passes + `automation/scripts/codex-task-lint.sh`; the repository, environment, source + thread, fingerprint, outcome, evidence, constraints, acceptance, recovery, + and human boundary are all mandatory. - Use **ChatGPT** for cross-channel synthesis, canvases, and decision summaries. Keep it out of private or data-heavy channels unless the task requires that context and the access is explicitly approved. @@ -116,6 +118,9 @@ message that names the action and target. A reaction alone is insufficient. - Install the Codex Slack app only in `#build`. Its GitHub App and Cloud environments may access public product repositories listed in the registry, but never private signing or credential repositories. +- Give every active product its own Codex environment with exactly the one + repository declared in `automation/codex-targets.yml`. Never rely on a recent + environment or the first repository in a multi-repository map. - Keep ChatGPT available for deliberate synthesis and Canvas work. Do not add automatic ChatGPT responses to incident channels. - Link the GitHub Slack app to the public Getyak repositories for rich previews @@ -124,9 +129,15 @@ message that names the action and target. A reaction alone is insufficient. - Do not subscribe `#ops` to unfiltered `workflows`, `commits`, or every repository event. The control loop owns failure-only workflow routing because the native workflow subscription reports successful runs too. -- A Codex request must name one repository and one verifiable outcome. The - resulting pull request is the durable artifact; discussion and approval stay - in the originating Slack thread. +- A Codex request must name one repository, its exact environment, one + verifiable outcome, and the originating thread. Reject the request if the + declared target is not `ready`. The resulting pull request is the durable + artifact; discussion and approval stay in the originating Slack thread. +- Begin code review with deliberate `@codex review` requests. Enable automatic + review only after a representative sample shows useful P0/P1 signal and the + product repository contains two or three outcome-focused rules under + `## Code Review Rules` in the applicable `AGENTS.md`. Keep mechanical checks + in CI. ### Persistent-access checklist diff --git a/automation/templates/codex-task.md b/automation/templates/codex-task.md new file mode 100644 index 0000000..9cff500 --- /dev/null +++ b/automation/templates/codex-task.md @@ -0,0 +1,30 @@ +# Codex task + +Repository: getyak/REPOSITORY +Environment: getyak-REPOSITORY +Slack channel: #build +Source thread: https://getyak.slack.com/archives/CHANNEL_ID/pTIMESTAMP +Fingerprint: REPOSITORY:task:STABLE_ID +Outcome: Describe one observable repository outcome. +Evidence: https://github.com/getyak/REPOSITORY/issues/ISSUE_NUMBER + +## Constraints + +- Keep the change within the named repository and outcome. +- Preserve unrelated work and existing compatibility guarantees. + +## Acceptance + +- Name the exact tests, checks, artifact, or observable behavior that must pass. +- Return the pull request URL in the originating Slack thread. + +## Rollback / recovery + +- Revert the pull request or disable the new behavior through the documented, + repository-owned recovery path. + +## Human boundary + +- Do not deploy, rotate or expose secrets, change permissions or OAuth, + communicate externally, or perform another consequential action without an + explicit human message naming the action and target.