diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 83f6830d5..9e4701265 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -3767,91 +3767,6 @@ jobs: "context": 200000, "output": 32000 } - }, - "hy3-free": { - "name": "Hy3 Free", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 190000, - "output": 64000 - } - }, - "minimax-m3-free": { - "name": "MiniMax-M3 Free", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 200000, - "output": 32000 - } - }, - "glm-5-free": { - "name": "GLM-5 Free", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 204800, - "output": 131072 - } - }, - "kimi-k2.5-free": { - "name": "Kimi K2.5 Free", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 262144, - "output": 262144 - } - }, - "qwen3.6-plus-free": { - "name": "Qwen3.6 Plus Free", - "tool_call": true, - "reasoning": true, - "options": { - "reasoningEffort": "high" - }, - "variants": { - "high": { - "reasoningEffort": "high" - } - }, - "limit": { - "context": 262144, - "output": 65536 - } } } }, @@ -4305,15 +4220,16 @@ jobs: # existing keyed fallback list: OpenCode Zen GPT-5.6 Terra, DeepSeek # V3, the direct GPT-5.6 Luna slot, and pinned PAID # OpenRouter coder models (free-tier candidates hit the shared - # free-models-per-day cap and hung for the full candidate timeout, - # so the OpenRouter slots use cheap paid models billed against the + # free-models-per-day cap and can otherwise hold the queue for hours, + # so free-tier candidates use a bounded five-minute timeout and the + # OpenRouter slots use cheap paid models billed against the # org's OpenRouter credits), then the full-size GPT-4.1 long-context # endpoint and provider-specific GPT/o3 fallbacks. # The direct-OpenAI slot runs GPT-5.6 Luna: the newest family's # cost-efficient tier, cheaper than the legacy gpt-5 it replaced # ($1/$6 vs $1.25/$10 per 1M tokens) so the org OpenAI budget # stretches further between top-ups. - OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" + OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. @@ -4347,7 +4263,7 @@ jobs: OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1" OPENCODE_NVIDIA_NIM_RUN_TIMEOUT_SECONDS: "180" OPENCODE_NVIDIA_NIM_TOTAL_BUDGET_SECONDS: "900" - OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "3600" + OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "300" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is # larger. Keep the exact runtime failure visible without spending a diff --git a/CHANGELOG.md b/CHANGELOG.md index bf30091dd..68acf1b53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Removed five stale OpenCode free-model aliases and bounded remaining free-tier failover to 300 seconds so a hung free candidate cannot consume the hour-class paid/NIM review budget; a timeout still only advances the pool and cannot approve. - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. - Allowed commas and ASCII parentheses in the bounded Strix changed-file path policy so legal tracked Packrat fixtures can receive exact-head security analysis, while rejecting raw `..` components before normalization and keeping controls, backslashes, whitespace ambiguity, and shell punctuation fail-closed. - Bound each review-agent invocation key to the wrapper's complete canonical payload, including the base branch and requesting actor; altered fields with a valid-format key now fail before durable-leader election or forwarding, and wrapper write permission is job-scoped. diff --git a/docs/doctoring/opencode-free-failover-budget.md b/docs/doctoring/opencode-free-failover-budget.md new file mode 100644 index 000000000..288da4377 --- /dev/null +++ b/docs/doctoring/opencode-free-failover-budget.md @@ -0,0 +1,23 @@ +# OpenCode free-tier failover budget + +## Incident and buyer impact + +A stale free alias (`ling-3.0-flash-free` and five ungoverned catalog names) +could occupy the central review pool for 3600 seconds and exhaust the step +without a verdict. Paid NVIDIA NIM and GitHub Models candidates never ran. +Review evidence then looked as if the lab had no LLM review. + +## Decision + +Keep the seven governed free candidates. Bound +`OPENCODE_FREE_RUN_TIMEOUT_SECONDS` to 300. Leave the 5400-second paid/large +review budget and the 180/900-second NVIDIA NIM caps unchanged. A free-tier +timeout only advances the pool; it cannot approve. This allocates remaining +test-time compute to the deep paid/NIM path rather than a dead free worker. +Speed is not the success metric; completing a real review is. + +## References + +National Institute of Standards and Technology. (2020). *Security and privacy +controls for information systems and organizations* (NIST Special Publication +800-53 Rev. 5). https://doi.org/10.6028/NIST.SP.800-53r5 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7343c06ac..fd2a11670 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -733,7 +733,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review preserves legitimate full-hour provider sessions" -assert_file_contains "$workflow_file" 'OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "3600"' "opencode free-tier failover timeout is hour-class (~3600s)" +assert_file_contains "$workflow_file" 'OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "300"' "opencode free-tier failover timeout is bounded to five minutes" assert_file_contains "$workflow_file" 'OPENCODE_NVIDIA_NIM_RUN_TIMEOUT_SECONDS: "180"' "opencode NVIDIA NIM candidates have a short per-candidate failover timeout" assert_file_contains "$workflow_file" 'OPENCODE_NVIDIA_NIM_TOTAL_BUDGET_SECONDS: "900"' "opencode NVIDIA NIM candidates share a bounded combined runtime budget" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OPENCODE_RUN_TIMEOUT_SECONDS:-3600' "opencode pool defaults primary run timeout to hour-class (~3600s) for large repos" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index daeaa37a2..1f7337868 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -106,11 +106,7 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "opencode-free/ling-3.0-flash-free " "opencode-free/big-pickle " "opencode-free/mimo-v2.5-free " - "opencode-free/hy3-free " - "opencode-free/minimax-m3-free " - "opencode-free/glm-5-free " - "opencode-free/kimi-k2.5-free " - "opencode-free/qwen3.6-plus-free ' || '' }}" + "' || '' }}" ) candidates_text = candidates_match.group(1) assert candidates_text.startswith(conditional_public_candidate) @@ -129,11 +125,6 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "opencode-free/ling-3.0-flash-free", "opencode-free/big-pickle", "opencode-free/mimo-v2.5-free", - "opencode-free/hy3-free", - "opencode-free/minimax-m3-free", - "opencode-free/glm-5-free", - "opencode-free/kimi-k2.5-free", - "opencode-free/qwen3.6-plus-free", *candidates_text.removeprefix(conditional_public_candidate).split(), ] candidate_pairs = [candidate.split("/", 1) for candidate in candidates] @@ -172,11 +163,6 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): ["opencode-free", "ling-3.0-flash-free"], ["opencode-free", "big-pickle"], ["opencode-free", "mimo-v2.5-free"], - ["opencode-free", "hy3-free"], - ["opencode-free", "minimax-m3-free"], - ["opencode-free", "glm-5-free"], - ["opencode-free", "kimi-k2.5-free"], - ["opencode-free", "qwen3.6-plus-free"], ["opencode", "gpt-5.6-terra"], ["github-models", "deepseek/deepseek-v3-0324"], ["openai", "gpt-5.6-luna"], @@ -255,11 +241,6 @@ def test_opencode_model_pool_sets_high_effort_for_capable_candidates(): "ling-3.0-flash-free", "big-pickle", "mimo-v2.5-free", - "hy3-free", - "minimax-m3-free", - "glm-5-free", - "kimi-k2.5-free", - "qwen3.6-plus-free", } assert set(paid_zen_models) == {"gpt-5.6-terra"} terra_model = paid_zen_models["gpt-5.6-terra"] @@ -1526,11 +1507,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): "opencode-free/ling-3.0-flash-free " "opencode-free/big-pickle " "opencode-free/mimo-v2.5-free " - "opencode-free/hy3-free " - "opencode-free/minimax-m3-free " - "opencode-free/glm-5-free " - "opencode-free/kimi-k2.5-free " - "opencode-free/qwen3.6-plus-free ' || ''" + "' || ''" ) in workflow assert ( "opencode/gpt-5.6-terra " @@ -1569,7 +1546,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "1"' in workflow assert 'OPENCODE_NVIDIA_NIM_RUN_TIMEOUT_SECONDS: "180"' in workflow assert 'OPENCODE_NVIDIA_NIM_TOTAL_BUDGET_SECONDS: "900"' in workflow - assert 'OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "3600"' in workflow + assert 'OPENCODE_FREE_RUN_TIMEOUT_SECONDS: "300"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "1"' in workflow assert 'OPENCODE_BACKOFF_MAX_SECONDS: "30"' in workflow