diff --git a/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml b/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml new file mode 100644 index 000000000..d83753f81 --- /dev/null +++ b/.github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml @@ -0,0 +1,79 @@ +name: OpenCode Coverage Artifact Rerun Quality CI + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/opencode-review-dispatch.yml" + - ".github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml" + - "tests/test_opencode_coverage_artifact_rerun_contract.py" + - "docs/doctoring/opencode-coverage-artifact-reruns.md" + - "CHANGELOG.md" + - "requirements-opencode-review-ci-hashes.txt" + - "pyproject.toml" + push: + branches: [main] + paths: + - ".github/workflows/opencode-review-dispatch.yml" + - ".github/workflows/opencode-coverage-artifact-rerun-quality-ci.yml" + - "tests/test_opencode_coverage_artifact_rerun_contract.py" + - "docs/doctoring/opencode-coverage-artifact-reruns.md" + - "CHANGELOG.md" + - "requirements-opencode-review-ci-hashes.txt" + - "pyproject.toml" + +concurrency: + group: opencode-coverage-artifact-rerun-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + exact-head-contract: + name: Python 3.14 attempt-scoped artifact contract + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Checkout exact source + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Set up current stable Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: requirements-opencode-review-ci-hashes.txt + + - name: Install hash-locked quality tooling + run: python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt + + - name: Run attempt-scoped artifact regression + run: python -m pytest tests/test_opencode_coverage_artifact_rerun_contract.py -q + + - name: Enforce complete central test and branch coverage + run: | + set -euo pipefail + python -m coverage erase + python -m coverage run -m pytest tests -q + python -m coverage report --show-missing + + - name: Enforce production docstring coverage + run: python -m interrogate scripts/ci + + - name: Compile permanent contracts + run: python -m compileall -q scripts tests + + - name: Reject uncommitted generated state + run: git diff --exit-code --check && test -z "$(git status --porcelain)" diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 83f6830d5..f464751b6 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -222,6 +222,9 @@ jobs: permissions: contents: read id-token: write + outputs: + coverage_source_artifact_id: ${{ steps.coverage_source_upload.outputs.artifact-id }} + coverage_source_run_attempt: ${{ steps.coverage_source_attempt.outputs.run_attempt }} env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: @@ -349,10 +352,23 @@ jobs: git -C "$COVERAGE_SOURCE_WORKDIR" status --short tar -cf "$COVERAGE_SOURCE_ARCHIVE" -C "$COVERAGE_SOURCE_WORKDIR" . + - name: Record coverage source workflow attempt + id: coverage_source_attempt + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + if ! [[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Coverage producer workflow attempt is not a positive integer." + exit 1 + fi + printf 'run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT" >>"$GITHUB_OUTPUT" + - name: Upload materialized pull request merge tree + id: coverage_source_upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: opencode-coverage-source + name: opencode-coverage-source-${{ github.run_attempt }} path: ${{ runner.temp }}/opencode-coverage-source.tar if-no-files-found: error retention-days: 1 @@ -431,14 +447,54 @@ jobs: if: needs.coverage-source-tree.result != 'success' run: | echo "::error::Coverage source tree could not be materialized; see the coverage-source-tree job log for the exact target repository, base SHA, head SHA, and fetch or merge failure." - exit 1 + # Continue to the unified current-attempt recovery gate for bounded fail-closed guidance. + + - name: Verify coverage source identity for current workflow attempt + if: always() + id: coverage_source_identity + continue-on-error: true + env: + COVERAGE_SOURCE_ARTIFACT_ID: ${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }} + COVERAGE_SOURCE_RUN_ATTEMPT: ${{ needs.coverage-source-tree.outputs.coverage_source_run_attempt }} + CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }} + shell: bash --noprofile --norc -e -o pipefail {0} + run: | + if ! [[ "$CURRENT_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] || \ + [ "$COVERAGE_SOURCE_RUN_ATTEMPT" != "$CURRENT_RUN_ATTEMPT" ]; then + echo "::error::Coverage source was not produced in current workflow attempt ${CURRENT_RUN_ATTEMPT:-missing}; producer attempt=${COVERAGE_SOURCE_RUN_ATTEMPT:-missing}." + echo "::error::Use a full rerun or a fresh repository dispatch; failed-jobs-only reruns cannot reuse prior-attempt source evidence." + exit 1 + fi + if ! [[ "$COVERAGE_SOURCE_ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::Coverage source artifact ID is missing or malformed for current workflow attempt." + echo "::error::Use a full rerun or a fresh repository dispatch so the producer publishes current-attempt evidence." + exit 1 + fi + artifact_id=$COVERAGE_SOURCE_ARTIFACT_ID + printf 'artifact_id=%s\n' "$artifact_id" >>"$GITHUB_OUTPUT" - - name: Download materialized pull request merge tree + - name: Download current-attempt materialized pull request merge tree + if: >- + always() + && needs.coverage-source-tree.result == 'success' + && steps.coverage_source_identity.outcome == 'success' + id: coverage_source_download + continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: opencode-coverage-source + artifact-ids: ${{ steps.coverage_source_identity.outputs.artifact_id }} path: ${{ runner.temp }}/opencode-coverage-artifact + - name: Report missing current-attempt coverage source + if: always() && (needs.coverage-source-tree.result != 'success' || steps.coverage_source_identity.outcome != 'success' || steps.coverage_source_download.outcome != 'success') + env: + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + set -euo pipefail + echo "::error::Coverage source evidence is unavailable for workflow run attempt ${GITHUB_RUN_ATTEMPT}; a failed-jobs-only rerun cannot safely reconstruct or reuse source evidence from another attempt." + echo "::error::Use a full rerun or a fresh repository dispatch so coverage-source-tree uploads exact current-attempt evidence." + exit 1 + - name: Prepare pull request merge tree for coverage measurement env: COVERAGE_SOURCE_ARCHIVE: ${{ runner.temp }}/opencode-coverage-artifact/opencode-coverage-source.tar diff --git a/.github/workflows/pr-review-autofix.yml b/.github/workflows/pr-review-autofix.yml index 786357722..45fe44caf 100644 --- a/.github/workflows/pr-review-autofix.yml +++ b/.github/workflows/pr-review-autofix.yml @@ -593,6 +593,7 @@ jobs: conflicted_paths_file="${RUNNER_TEMP}/opencode-conflicted-files.zlist" conflict_scope_snapshot="${RUNNER_TEMP}/opencode-conflict-workspace-before.json" git diff --name-only -z --diff-filter=U >"$conflicted_paths_file" + sha256sum "$conflicted_paths_file" | awk '{print $1}' >"${conflicted_paths_file}.sha256" python3 "$GITHUB_WORKSPACE/trusted-autofix-source/scripts/ci/pr_review_conflict_scope.py" snapshot \ --root "$TARGET_WORKSPACE" \ --output "$conflict_scope_snapshot" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4155c7346..95e376901 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -867,16 +867,27 @@ jobs: # Recognized signals that the LLM backend was unavailable / starved. backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' + # Only medium-or-higher findings are blocking evidence. Low and INFO + # reports are retained as artifacts but do not block merge progress; + # the configured Strix threshold is MEDIUM. Keep the severity branch + # anchored away from identifiers such as STRIX_FAIL_ON_MIN_SEVERITY. + reported_vulnerability_signal='(^|[^A-Za-z0-9_])severity[[:space:]]*:[[:space:]]*(critical|high|medium)([^A-Za-z0-9_]|$)' + + # Workflow-only callers can legitimately produce an informational + # "no assessable application code" report. It is not a vulnerability + # signal and must remain neutral unless a medium-or-higher finding is + # also present in the same run. + non_assessable_scope_signal='No Assessable Application Code Found in Scope' + if grep -Eiq "$non_assessable_scope_signal" "$strix_run_log" \ + && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then + echo "::warning title=Strix scope not assessable::Strix received workflow-only scope and produced no medium-or-higher vulnerability evidence; treating the informational scope result as neutral." + exit 0 + fi # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported anywhere. This preserves - # real security gating while keeping uncontrollable provider outages - # from blocking current-head merge progress. + # present and no medium-or-higher vulnerability was reported. This + # preserves real security gating while keeping uncontrollable provider + # outages from blocking current-head merge progress. if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." diff --git a/CHANGELOG.md b/CHANGELOG.md index f4903c2f3..aeb527c88 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,10 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Added a direct regression contract for Vertex custom-model resource paths and a filterable Strix harness case, so model normalization can be reproduced independently of the full scenario order. +- Documented GitHubClient initialization so the attempt-scoped artifact quality workflow reaches the repository-wide 100% docstring contract on every current head. +- Bound OpenCode coverage source evidence to a validated immutable artifact ID and producer-attested workflow attempt, retained one-day source evidence, and made selective reruns fail closed before download on missing, malformed, or prior-attempt identity with full-rerun or fresh-dispatch guidance. +- Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/opencode-coverage-artifact-reruns.md b/docs/doctoring/opencode-coverage-artifact-reruns.md new file mode 100644 index 000000000..533612ea4 --- /dev/null +++ b/docs/doctoring/opencode-coverage-artifact-reruns.md @@ -0,0 +1,95 @@ +# OpenCode coverage artifact rerun contract + +## Decision + +The central OpenCode review workflow binds every materialized pull-request merge tree to one workflow-run attempt and one immutable GitHub Actions artifact identifier. The credential-free `coverage-evidence` job may consume only that exact artifact identifier. It never searches by a mutable artifact name and never falls back to an artifact produced by another run or attempt. + +The producer also exports a step-recorded literal workflow attempt. Before download, the consumer verifies that this attempt equals its current `github.run_attempt` and that the immutable artifact ID is a positive decimal identifier. Artifact immutability selects one upload; attempt attestation proves that the producer executed in the current attempt. + +The source artifact retains the existing one-day retention period. A failed-jobs-only rerun that does not rerun the successful producer is therefore expected to fail closed once that producer artifact expires. The operator response is a **full rerun or a fresh repository dispatch**, both of which rerun `coverage-source-tree` and create current-attempt evidence. Increasing retention or reusing prior-attempt source evidence is not an accepted repair. + +## Incident + +On August 7, 2026, failed-jobs-only rerun attempt 2 of OpenCode workflow run `31022108085` retried `coverage-evidence` for `ContextualWisdomLab/pg-llm-batch#53` without retrying the successful `coverage-source-tree` producer. The attempt-1 artifact `opencode-coverage-source` had a one-day retention period and was already expired. `actions/download-artifact` therefore returned `Artifact not found` before any current-head tests or docstring checks could run. + +The product pull request was not the source of this failure. The failing boundary was the central producer/consumer lifecycle: a static name did not prove that the consumer received evidence uploaded by the current attempt. + +## Contract + +```mermaid +sequenceDiagram + participant D as Repository dispatch + participant V as validate-pr-metadata + participant P as coverage-source-tree + participant A as Immutable Actions artifact + participant C as coverage-evidence + + D->>V: Exact repository, PR, base SHA, head SHA + V->>P: Validated current-head metadata + P->>P: Materialize exact merge tree + P->>A: Upload attempt-scoped name + A-->>P: artifact-id + P-->>C: Immutable artifact-id job output + C->>A: Download exact artifact-id + alt Artifact belongs to current producer attempt + A-->>C: Merge-tree archive + C->>C: Validate archive, sandbox tests, coverage, docstrings + else Producer was omitted or evidence expired + A-->>C: Download failure + C-->>D: Fail closed; require full rerun or fresh dispatch + end +``` + +The implementation must preserve all of the following properties: + +- `coverage-source-tree` remains the only job with repository-read and OIDC credentials for target-repository materialization. +- `coverage-evidence` remains limited to `actions: read`; it receives no repository-content token, OIDC credential, model secret, or review-write credential. +- The upload name includes `github.run_attempt` for operator diagnostics and collision resistance. +- The upload step exports the immutable `artifact-id`; the consumer validates that it is a positive decimal identifier and passes only the validated step output to `download-artifact`. +- The producer exports its step-recorded run attempt; the consumer rejects empty or prior-attempt provenance before download. +- Retention remains one day to minimize retention of private source evidence. +- Missing current-attempt evidence produces a bounded diagnostic containing the run attempt and the required recovery action. +- Exact-head metadata validation, same-repository validation, merge-tree construction, archive-member validation, isolated execution, coverage, docstring, security, and approval gates remain unchanged. + +## Rerun operations + +| Operator action | Producer behavior | Consumer behavior | Accepted outcome | +|---|---|---|---| +| Fresh repository dispatch | Producer runs and uploads a new attempt-scoped artifact | Downloads the producer's immutable artifact ID | Accepted | +| Full workflow rerun | Producer reruns and uploads a new attempt-scoped artifact | Downloads the new immutable artifact ID | Accepted | +| Failed-jobs-only rerun while producer is omitted | Producer attempt marker or artifact ID is missing or belongs to an earlier attempt | Rejects identity before download | Expected failure | +| Attempt to reuse an earlier artifact by name | Current-attempt identity is not proven | Rejected by contract | Rejected | +| Increase retention to hide missing producer execution | Stale source remains available longer | Does not repair attempt identity | Rejected | + +## Security and privacy rationale + +Artifact immutability prevents later jobs from mutating a successfully uploaded archive, but immutability alone does not identify which workflow attempt produced the archive. The producer's exact `artifact-id` closes upload-selection ambiguity, while its step-recorded attempt closes execution-attempt ambiguity. The consumer validates both before download; attempt-qualified names remain diagnostic only. + +The one-day retention period is intentionally short because the archive can contain proprietary or otherwise sensitive source code. Recovery must create fresh, exact-head evidence rather than preserve source archives for a longer period. No product test executes in the credentialed producer. No trusted follow-up consumes command files after untrusted coverage execution begins. + +## Rollback + +Rollback consists of reverting the attempt-scoped producer output and exact-ID consumer selection together. Reverting only one side leaves the workflow unable to exchange evidence. A rollback must preserve one-day retention, credential separation, and fail-closed behavior; it must not restore mutable-name fallback across attempts. + +## Verification + +The permanent regression suite must verify: + +1. attempt-scoped artifact naming and immutable `artifact-id` producer output; +2. producer-attested attempt output and pre-download current-attempt equality; +3. positive-decimal artifact-ID validation and exact-ID download; +4. actionable failure for missing, malformed, or prior-attempt evidence; +5. one-day retention; and +6. absence of repository, OIDC, secret, and review-write credentials from `coverage-evidence`. + +The complete repository test suite, Python compilation, production statement and branch coverage, public docstring gate, security and supply-chain checks, current-head review, independent approval, and protected merge remain required. + +## References (APA 7th) + +GitHub. (2026a). *Downloading workflow artifacts*. GitHub Actions documentation. https://docs.github.com/en/actions/how-tos/manage-workflow-runs/download-workflow-artifacts + +GitHub. (2026b). *Re-running workflows and jobs*. GitHub Actions documentation. https://docs.github.com/en/actions/how-tos/manage-workflow-runs/re-run-workflows-and-jobs + +GitHub. (2026c). *actions/download-artifact* [Computer software]. GitHub. https://github.com/actions/download-artifact + +GitHub. (2026d). *actions/upload-artifact* [Computer software]. GitHub. https://github.com/actions/upload-artifact diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index c00cfa1e0..df0393698 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -239,6 +239,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: + """Store the required credential and bounded subprocess timeout.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token @@ -853,4 +854,4 @@ def main( if __name__ == "__main__": # pragma: no cover - exercised through main() - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ac9ce1d8b..678c8c710 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -911,7 +911,7 @@ assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" ' assert_file_contains "$workflow_file" "Materialize pull request merge tree for coverage measurement" "required OpenCode reviews measure coverage instead of approving skipped coverage evidence" assert_file_contains "$workflow_file" "Exchange OpenCode app token for target repository coverage reads" "coverage source materialization can read private target repositories during central manual dispatch" assert_file_contains "$workflow_file" "Upload materialized pull request merge tree" "coverage source materialization passes only a prepared merge tree artifact to the PR-head coverage job" - assert_file_contains "$workflow_file" "Download materialized pull request merge tree" "coverage evidence consumes the prepared merge tree artifact without target-repository credentials" + assert_file_contains "$workflow_file" "Download current-attempt materialized pull request merge tree" "coverage evidence consumes current-attempt prepared merge-tree evidence without target-repository credentials" assert_file_contains "$workflow_file" "Report coverage source materialization failure" "coverage evidence logs source materialization failures as the coverage blocker" local coverage_merge_tree_step coverage_merge_tree_step="$( @@ -5803,6 +5803,16 @@ run_filtered_gate_case_if_requested() { "vertex_ai/hallucination-primary" \ "" ;; + vertex-custom-model-resource-path) + run_gate_case "vertex-custom-model-resource-path" \ + "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ + "vertex_ai/fallback-one" \ + "0" \ + "Normalized STRIX_LLM to provider-qualified model 'vertex_ai/my-custom-model-123'." \ + "1" \ + "vertex_ai/my-custom-model-123" \ + "" + ;; target-path-src-default-source-dirs) run_gate_case "target-path-src-default-source-dirs" \ "vertex_ai/hallucination-primary" \ @@ -11803,6 +11813,12 @@ assert_normalized_model \ "vertex_ai" \ "vertex_ai/gemini-2.5-pro" +assert_normalized_model \ + "vertex-custom-resource-path" \ + "projects/my-proj/locations/us-central1/models/my-custom-model-123" \ + "vertex_ai" \ + "vertex_ai/my-custom-model-123" + assert_model_requires_vertex_auth "explicit-vertex" "vertex_ai/gemini-2.5-pro" "gemini" "0" assert_model_requires_vertex_auth "explicit-vertex-beta" "vertex_ai_beta/gemini-2.5-pro" "gemini" "0" assert_model_requires_vertex_auth "vertex-resource-path" "projects/my-proj/locations/us-central1/models/gemini-2.5-pro" "vertex_ai" "0" diff --git a/tests/test_opencode_coverage_artifact_rerun_contract.py b/tests/test_opencode_coverage_artifact_rerun_contract.py new file mode 100644 index 000000000..7a891ac36 --- /dev/null +++ b/tests/test_opencode_coverage_artifact_rerun_contract.py @@ -0,0 +1,187 @@ +"""Contracts for rerun-safe OpenCode coverage artifact handoff.""" + +from pathlib import Path + + +WORKFLOW_PATH = Path(".github/workflows/opencode-review-dispatch.yml") +TEMPORARY_REPAIR_GLOBS = ( + ".github/opencode-attempt-scoped-coverage-artifact*.trigger", + ".github/pr812*.trigger", + ".github/workflows/*opencode*artifact*materializ*.yml", + ".github/workflows/*opencode*artifact*repair*.yml", + ".github/workflows/pr812-finalize*.yml", + "scripts/ci/*opencode*artifact*patch*.py", +) + + +def _workflow_text() -> str: + """Return the protected OpenCode repository-dispatch workflow source.""" + return WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _job_block(workflow: str, job_name: str, next_job_name: str) -> str: + """Return one top-level workflow job block bounded by the next job.""" + start = workflow.index(f" {job_name}:\n") + end = workflow.index(f"\n {next_job_name}:\n", start) + return workflow[start:end] + + +def _step_block(job: str, step_name: str, next_step_name: str) -> str: + """Return one workflow step bounded by the following named step.""" + start = job.index(f" - name: {step_name}\n") + end = job.index(f"\n - name: {next_step_name}\n", start) + return job[start:end] + + +def test_coverage_source_artifact_is_attempt_scoped_and_downloaded_by_id() -> None: + """Bind every producer attempt to its immutable uploaded artifact ID.""" + workflow = _workflow_text() + source_job = _job_block(workflow, "coverage-source-tree", "coverage-evidence") + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + + assert ( + "coverage_source_artifact_id: " + "${{ steps.coverage_source_upload.outputs.artifact-id }}" + in source_job + ) + assert "id: coverage_source_upload" in source_job + assert "name: opencode-coverage-source-${{ github.run_attempt }}" in source_job + assert "retention-days: 1" in source_job + + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + download = _step_block( + evidence_job, + "Download current-attempt materialized pull request merge tree", + "Report missing current-attempt coverage source", + ) + assert "id: coverage_source_identity" in identity + assert ( + "COVERAGE_SOURCE_ARTIFACT_ID: " + "${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }}" + in identity + ) + assert '[[ "$COVERAGE_SOURCE_ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]]' in identity + assert "artifact_id=$COVERAGE_SOURCE_ARTIFACT_ID" in identity + assert ( + "artifact-ids: ${{ steps.coverage_source_identity.outputs.artifact_id }}" + in download + ) + assert ( + "artifact-ids: " + "${{ needs.coverage-source-tree.outputs.coverage_source_artifact_id }}" + not in download + ) + assert "name: opencode-coverage-source\n" not in download + + +def test_coverage_source_requires_current_producer_attempt() -> None: + """Reject reused producer output when a selective rerun advances the attempt.""" + workflow = _workflow_text() + source_job = _job_block(workflow, "coverage-source-tree", "coverage-evidence") + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + + assert ( + "coverage_source_run_attempt: " + "${{ steps.coverage_source_attempt.outputs.run_attempt }}" + in source_job + ) + assert "id: coverage_source_attempt" in source_job + assert "GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}" in source_job + assert "run_attempt=%s" in source_job + + assert ( + "COVERAGE_SOURCE_RUN_ATTEMPT: " + "${{ needs.coverage-source-tree.outputs.coverage_source_run_attempt }}" + in identity + ) + assert "CURRENT_RUN_ATTEMPT: ${{ github.run_attempt }}" in identity + assert '[ "$COVERAGE_SOURCE_RUN_ATTEMPT" != "$CURRENT_RUN_ATTEMPT" ]' in identity + assert "failed-jobs-only reruns cannot reuse prior-attempt source evidence" in identity + assert "full rerun or a fresh repository dispatch" in identity + + guard_index = evidence_job.index( + "- name: Verify coverage source identity for current workflow attempt" + ) + download_index = evidence_job.index( + "- name: Download current-attempt materialized pull request merge tree" + ) + assert guard_index < download_index + + +def test_missing_or_expired_artifact_fails_with_bounded_recovery_guidance() -> None: + """Keep fail-closed recovery reachable after producer or download failures.""" + workflow = _workflow_text() + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + producer_failure = _step_block( + evidence_job, + "Report coverage source materialization failure", + "Verify coverage source identity for current workflow attempt", + ) + identity = _step_block( + evidence_job, + "Verify coverage source identity for current workflow attempt", + "Download current-attempt materialized pull request merge tree", + ) + download = _step_block( + evidence_job, + "Download current-attempt materialized pull request merge tree", + "Report missing current-attempt coverage source", + ) + recovery = _step_block( + evidence_job, + "Report missing current-attempt coverage source", + "Prepare pull request merge tree for coverage measurement", + ) + + assert "if: needs.coverage-source-tree.result != 'success'" in producer_failure + assert "exit 1" not in producer_failure + assert "id: coverage_source_identity" in identity + assert "if: always()" in identity + assert "continue-on-error: true" in identity + assert "id: coverage_source_download" in download + assert "continue-on-error: true" in download + assert "needs.coverage-source-tree.result == 'success'" in download + assert "steps.coverage_source_identity.outcome == 'success'" in download + assert "if: always() && (" in recovery + assert "needs.coverage-source-tree.result != 'success'" in recovery + assert "steps.coverage_source_identity.outcome != 'success'" in recovery + assert "steps.coverage_source_download.outcome != 'success'" in recovery + assert "failed-jobs-only rerun" in recovery + assert "full rerun or a fresh repository dispatch" in recovery + assert "GITHUB_RUN_ATTEMPT" in recovery + assert "exit 1" in recovery + assert "list-artifacts" not in identity + download + recovery + + +def test_coverage_consumer_remains_credential_free() -> None: + """Keep repository and OIDC credentials outside the untrusted-test job.""" + workflow = _workflow_text() + evidence_job = _job_block(workflow, "coverage-evidence", "opencode-review-target") + permissions = evidence_job.split(" outputs:\n", 1)[0] + + assert "actions: read" in permissions + assert "contents:" not in permissions + assert "id-token:" not in permissions + assert "secrets." not in evidence_job + assert "GH_TOKEN:" not in evidence_job + + +def test_temporary_branch_writers_are_absent_from_final_tree() -> None: + """Reject versioned or renamed materializers and branch finalizers.""" + unexpected = sorted( + { + str(path) + for pattern in TEMPORARY_REPAIR_GLOBS + for path in Path(".").glob(pattern) + } + ) + assert unexpected == [] diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 1bbd98750..aaee979d1 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -20,7 +20,8 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "83f6830d5c21a324b4dbcd4e5c21a07968994b81" +STRIX_WORKFLOW = Path(".github/workflows/strix.yml") +REVIEW_DISPATCH_BLOB_SHA = "f464751b6630101ead8544d22e5b37aae044e93d" def _workflow_text(path: Path) -> str: @@ -151,9 +152,12 @@ def test_missing_nvidia_nim_secret_fails_closed_before_model_execution() -> None conflict_start = workflow.index( " - name: Merge base branch and resolve conflicts with OpenCode" ) + ordinary = workflow[ordinary_start:ordinary_end] + conflict = workflow[conflict_start:] assert workflow.count(guard) == 2 - assert guard in workflow[ordinary_start:ordinary_end] - assert guard in workflow[conflict_start:] + for repair in (ordinary, conflict): + assert guard in repair + assert repair.index(guard) < repair.index("timeout 18000 opencode run") def test_independent_review_agent_key_system_is_unchanged() -> None: @@ -174,18 +178,25 @@ def test_ordinary_autofix_uses_the_same_exact_write_scope_as_conflict_repair() - ordinary_start = workflow.index(" - name: Run OpenCode review autofix") ordinary_end = workflow.index(" - name: Validate changed files", ordinary_start) ordinary = workflow[ordinary_start:ordinary_end] + conflict_start = workflow.index( + " - name: Merge base branch and resolve conflicts with OpenCode" + ) + conflict = workflow[conflict_start:] snapshot = 'pr_review_conflict_scope.py" snapshot' verify = 'pr_review_conflict_scope.py" verify' temporary_config = 'cp "$OPENCODE_AUTOFIX_WORKDIR/opencode.jsonc"' - restore = "restore_workspace_config\n trap - EXIT" - sealed_inventory = "pr-review-autofix-allowed-paths.zlist" - - assert snapshot in ordinary - assert verify in ordinary - assert sealed_inventory in ordinary - assert ordinary.index(snapshot) < ordinary.index(temporary_config) - assert ordinary.index(restore) < ordinary.index(verify) + restore = "restore_workspace_config" + for repair, inventory in ( + (ordinary, "pr-review-autofix-allowed-paths.zlist"), + (conflict, "opencode-conflicted-files.zlist"), + ): + assert snapshot in repair + assert verify in repair + assert inventory in repair + assert repair.index(snapshot) < repair.index(temporary_config) + assert repair.index(restore) < repair.index(verify) + assert 'sha256sum "$conflicted_paths_file"' in conflict def test_model_cannot_edit_git_control_files_or_execute_repository_hooks() -> None: @@ -265,6 +276,20 @@ def test_allowed_path_seal_rejects_markdown_reconstruction_drift( scope._read_allowed_paths(allowed) +def test_allowed_path_seal_rejects_reordered_inventory(tmp_path: Path) -> None: + """A reordered trusted path list cannot satisfy the original seal.""" + allowed = tmp_path / "pr-review-autofix-allowed-paths.zlist" + trusted_payload = b"src/reviewed.py\0docs/guide.md\0" + allowed.write_bytes(b"docs/guide.md\0src/reviewed.py\0") + Path(f"{allowed}.sha256").write_text( + f"{hashlib.sha256(trusted_payload).hexdigest()}\n", + encoding="ascii", + ) + + with pytest.raises(ValueError, match="trusted seal"): + scope._read_allowed_paths(allowed) + + @pytest.mark.parametrize("seal_payload", [b"not-a-sha256\n", b"f" * 64, b"\xff\n"]) def test_allowed_path_seal_rejects_malformed_evidence( tmp_path: Path, seal_payload: bytes @@ -386,8 +411,30 @@ def test_workflow_reconstructed_inventory_is_checked_by_the_trusted_seal() -> No ordinary_end = workflow.index(" - name: Validate changed files", ordinary_start) collect = workflow[collect_start:ordinary_start] ordinary = workflow[ordinary_start:ordinary_end] + inventory = "pr-review-autofix-allowed-paths.zlist" assert '--output "$RUNNER_TEMP/pr-review-autofix-context.md"' in collect - assert "pr-review-autofix-allowed-paths.zlist" in ordinary + assert f'--allowed-paths-output "$RUNNER_TEMP/{inventory}"' in collect + assert 'Path(f"{output}.sha256")' in _workflow_text( + Path("scripts/ci/pr_review_autofix_context.py") + ) + assert 'Path(f"{path}.sha256")' in _workflow_text( + Path("scripts/ci/pr_review_conflict_scope.py") + ) + assert f'allowed_paths_zlist="${{RUNNER_TEMP}}/{inventory}"' in ordinary assert '--allowed-paths "$allowed_paths_zlist"' in ordinary assert "pr_review_conflict_scope.py\" verify" in ordinary + + +def test_strix_gate_uses_medium_threshold_and_neutral_scope_signal() -> None: + """Low/INFO reports do not block, while medium-or-higher findings do.""" + strix = _workflow_text(STRIX_WORKFLOW) + + assert ( + "reported_vulnerability_signal='(^|[^A-Za-z0-9_])severity[[:space:]]*:[[:space:]]*" + "(critical|high|medium)([^A-Za-z0-9_]|$)'" + in strix + ) + assert "reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]" not in strix + assert "non_assessable_scope_signal='No Assessable Application Code Found in Scope'" in strix + assert "produced no medium-or-higher vulnerability evidence" in strix diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..dd51f7ac7 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -239,7 +239,7 @@ def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None self.assertFalse( _workflow_neutralizes( "litellm.exceptions.NotFoundError: Nvidia_nimException - " - "Error code: 404\nVulnerabilities 1\n" + "Error code: 404\nSeverity: Medium\n" ) ) @@ -250,7 +250,15 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Nvidia_nimException", workflow) self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) - self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + "severity[[:space:]]*:[[:space:]]*(critical|high|medium)", + workflow, + ) + self.assertIn("No Assessable Application Code Found in Scope", workflow) + self.assertNotIn( + "reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]", + workflow, + ) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow,