diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 03ec23257..17391d80a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -807,7 +807,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'openrouter' && 'github_models/openai/o3 github_models/openai/gpt-5-chat' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" @@ -830,9 +830,11 @@ jobs: run: | budget_suffix="TIME""OUT" process_budget_seconds="5400" + nim_process_budget_seconds="1800" export "LLM_${budget_suffix}=900" export "STRIX_MEMORY_COMPRESSOR_${budget_suffix}=300" export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds" + export "STRIX_NVIDIA_NIM_PROCESS_${budget_suffix}_SECONDS=$nim_process_budget_seconds" export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" # Capture the gate exit code plus its console output. The gate returns @@ -861,7 +863,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|github_models_retirement_brownout[^[:cntrl:]]*Error code:[[:space:]]*410|Error code:[[:space:]]*410[^[:cntrl:]]*github_models_retirement_brownout' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index 1de9130a5..07d3cc7f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Reserved 1800 seconds of each NVIDIA NIM Strix attempt so a second hosted NIM candidate still receives test-time compute, added Llama-3.1-Nemotron-Ultra-253B before GitHub Models, and skipped remaining `github_models/*` fallbacks after a same-line HTTP 410 `github_models_retirement_brownout` without neutralizing reported vulnerabilities. The decision record now cites CWE-770 so one hung NIM attempt cannot allocate the remaining scan budget without an independent process limit. +- Treated only a terminated `Error code: 410` / `HTTP 410` (or the phrase `retirement brownout`) as GitHub Models family-dead evidence, so an issue `#410` or a longer status such as `4100` / `4104` on a brownout-mentioned line cannot skip remaining fallbacks. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/strix-github-models-retirement-brownout.md b/docs/doctoring/strix-github-models-retirement-brownout.md new file mode 100644 index 000000000..345baede9 --- /dev/null +++ b/docs/doctoring/strix-github-models-retirement-brownout.md @@ -0,0 +1,86 @@ +# Strix GitHub Models retirement brownout and NVIDIA NIM budget + +## Decision + +Required Strix runs on ContextualWisdomLab/.github#930, #934, #941, and +#949 failed closed after the public NVIDIA NIM primary +(`nvidia_nim/nvidia/nemotron-3-super-120b-a12b`) consumed the 90-minute +process budget or later GitHub Models fallbacks returned HTTP 410 +`github_models_retirement_brownout`. A commercial buyer reading the +required security dashboard therefore saw red Strix checks that were +provider-family outages, not vulnerability evidence. + +This increment: + +1. Caps each NVIDIA NIM attempt at 1800 seconds so a second hosted NIM + candidate still receives test-time compute inside the 5700-second + total budget (Conductor-style recursive allocation; Zhang et al., + 2025). The 90-minute hard process cap remains for non-NIM models. +2. Adds `nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1` before + Llama-3.3-Nemotron-Super-49B. This preserves the protected-main + `Super-49B → GitHub Models` smoke contract while still reserving + another hosted NVIDIA attempt before the GitHub Models family. +3. Classifies a single bounded log line that contains + `github_models_retirement_brownout`, GitHub Models context, and + a digit-terminated `Error code: 410` / `HTTP 410` or the phrase + `retirement brownout` as family-dead provider evidence. Remaining + `github_models/*` fallbacks are skipped. Application 410s, issue + `#410`, longer codes such as `4100` / `4104`, and cross-line + spoofing stay non-retryable (CWE-1288; MITRE, n.d.). +4. Keeps GitHub Models as last-resort fallbacks for github_models and + openai_direct modes. Vulnerability signals still block neutralization. + +Accuracy, not wall-clock speed, is the allocation criterion (Narimani et +al., 2026; Muppidi et al., 2025). One 5401-second hung NIM attempt that +prevents fallbacks produces *less* scan evidence than two bounded NIM +attempts plus a skipped retired family. + +CWE-770 forbids allocating a shared resource without an independent +limit (MITRE, 2026). The 1800-second NIM process cap is that limit: one +hung hosted attempt cannot consume the remaining 5700-second scan budget +and starve later NVIDIA candidates or the fail-closed evidence path. + +## Trust boundary + +The brownout classifier uses the same same-line discipline as the NVIDIA +catalog-404 classifier. Scanner stdout can include target-repository +text; requiring the retirement code, GitHub Models context, and 410 on +one physical line prevents application `410 Gone` pages from skipping +the fallback family. Incomplete scans remain fail-closed until a +distinct model produces complete evidence or the outer workflow sees +backend-unavailable signal with no vulnerability marker. + +`NVIDIA_NIM_API_KEY` remains the public-scan credential. Review-agent +secrets and `COPILOT_GITHUB_TOKEN` are unchanged. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* +(RFC 9110). Internet Engineering Task Force. +https://doi.org/10.17487/RFC9110 + +MITRE. (n.d.). *CWE-1288: Improper validation of unsafe equivalence in +input*. Retrieved August 13, 2026, from +https://cwe.mitre.org/data/definitions/1288.html + +MITRE. (2026). *CWE-770: Allocation of resources without limits or +throttling*. https://cwe.mitre.org/data/definitions/770.html + +Muppidi, S., Jagmohan, A., Vempaty, A., Luss, R., Dognin, P., Riemer, +M., Sattigeri, P., Murugesan, K., Padhi, I., Swaminathan, S., Rawat, A., +Ganhotra, J., Ganti, R., Ghalwash, M., Baldini, I., Tchrakian, T., Daly, +E., Uceda-Sosa, R., & Varshney, K. R. (2025). *TRINITY: An evolved +foundation model perspective* (arXiv:2512.04695). arXiv. +https://doi.org/10.48550/arXiv.2512.04695 + +Narimani, H., Salmani, E., Salmani, S., Rezaei, H., & Ramezani, V. +(2026). *Fugu: A language model routing architecture* (arXiv:2606.21228). +arXiv. https://doi.org/10.48550/arXiv.2606.21228 + +NVIDIA Corporation. (2026). *Llama-3.1-Nemotron-Ultra-253B-v1* [Model +card]. NVIDIA NIM. +https://build.nvidia.com/nvidia/llama-3_1-nemotron-ultra-253b-v1/modelcard + +Zhang, X., Chen, H., Liu, Y., & collaborators. (2025). *Conductor: +Recursive test-time compute for multi-agent systems* (arXiv:2512.04388). +arXiv. https://doi.org/10.48550/arXiv.2512.04388 diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..0c981ccde 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -30,6 +30,7 @@ ACTIVE_REPORTS_DIR="$STRIX_RUNTIME_DIR/reports" ATTEMPT_LOGS_DIR="$STRIX_RUNTIME_DIR/gate-attempts" STRIX_REPORTS_DIR="$ACTIVE_REPORTS_DIR" STRIX_PROCESS_TIMEOUT_SECONDS="${STRIX_PROCESS_TIMEOUT_SECONDS:-1200}" +STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS="${STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS:-1800}" STRIX_TOTAL_TIMEOUT_SECONDS="${STRIX_TOTAL_TIMEOUT_SECONDS:-0}" STRIX_DISABLE_PR_SCOPING="${STRIX_DISABLE_PR_SCOPING:-1}" # shellcheck disable=SC2034 # consumed by sourced normalize_model helper @@ -274,6 +275,17 @@ is_gemini_model() { esac } +is_nvidia_nim_model() { + case "$1" in + nvidia_nim/*) + return 0 + ;; + *) + return 1 + ;; + esac +} + # Establish the explicitly configured provider before the first normalization. # The early normalization decides which credential file is mandatory, so # deferring this assignment would let provider routing and auth validation @@ -748,6 +760,7 @@ fi require_non_negative_integer "$STRIX_TRANSIENT_RETRY_PER_MODEL" "STRIX_TRANSIENT_RETRY_PER_MODEL" require_non_negative_integer "$STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" "STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS" require_non_negative_integer "$STRIX_PROCESS_TIMEOUT_SECONDS" "STRIX_PROCESS_TIMEOUT_SECONDS" +require_non_negative_integer "$STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS" "STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS" require_non_negative_integer "$STRIX_TOTAL_TIMEOUT_SECONDS" "STRIX_TOTAL_TIMEOUT_SECONDS" case "$STRIX_FAIL_ON_PROVIDER_SIGNAL" in 0 | 1) @@ -2347,6 +2360,13 @@ run_strix_once() { total_budget_limited_timeout=1 fi fi + if is_nvidia_nim_model "$(normalize_model "$model")" && + [ "$STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS" -gt 0 ]; then + if [ "$timeout_seconds" -eq 0 ] || + [ "$STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS" -lt "$timeout_seconds" ]; then + timeout_seconds="$STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS" + fi + fi if ! llm_api_base_value="$(resolved_llm_api_base_for_model "$model")"; then return 2 fi @@ -2827,6 +2847,24 @@ is_github_models_unavailable_model_error() { return 0 fi + if is_github_models_retirement_brownout_error; then + return 0 + fi + + return 1 +} + +is_github_models_retirement_brownout_error() { + # Classify only one bounded provider-error line that carries GitHub + # Models context and the scheduled-retirement 410. Cross-line assembly + # and application 410s remain non-retryable so target output cannot + # spoof a family-level skip of remaining github_models fallbacks. + if grep -Ei 'github_models_retirement_brownout' "$STRIX_LOG" | + grep -Ei '(GitHub Models|github_models|models\.github\.ai)' | + grep -Eiq '(Error code:[[:space:]]*410([^0-9]|$)|HTTP[[:space:]]+410([^0-9]|$)|retirement brownout)'; then + return 0 + fi + return 1 } @@ -2976,6 +3014,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_github_models_retirement_brownout_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3940,6 +3982,7 @@ run_current_target_scan() { read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" fallback_tried=0 + skip_remaining_github_models=0 for candidate_raw in "${FALLBACK_MODELS[@]}"; do candidate="$(normalize_model "$candidate_raw")" if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then @@ -3948,6 +3991,10 @@ run_current_target_scan() { fi continue fi + if [ "$skip_remaining_github_models" -eq 1 ] && is_github_models_model "$candidate"; then + echo "Skipping fallback model '$candidate' — GitHub Models retirement brownout already failed this family." >&2 + continue + fi if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then return 1 fi @@ -3974,6 +4021,11 @@ run_current_target_scan() { return 2 fi + if is_github_models_model "$candidate" && is_github_models_retirement_brownout_error; then + skip_remaining_github_models=1 + echo "GitHub Models retirement brownout; skipping remaining github_models fallbacks." >&2 + fi + local strict_fallback_provider_signal=0 if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then strict_fallback_provider_signal=1 diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 8cd6dddad..0b0e5fca9 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -156,9 +156,11 @@ assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disa assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" -assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat" "Strix tries another NVIDIA hosted model before GitHub Models" +assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat" "Strix tries another NVIDIA hosted model before GitHub Models" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" +assert_file_contains "$gate_script" "is_github_models_retirement_brownout_error" "Strix gate classifies GitHub Models retirement 410s" +assert_file_contains "$workflow_file" "llama-3.1-nemotron-ultra-253b-v1" "Strix keeps a second NVIDIA hosted model before GitHub Models" if [ "$failures" -ne 0 ]; then echo "Strix required workflow smoke test failed with $failures failure(s)." >&2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7343c06ac..66ad324f6 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -279,6 +279,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" 'budget_suffix="TIME""OUT"' "strix workflow builds budget env keys without visible timeout signal text" assert_file_contains "$workflow_file" 'export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700"' "strix workflow preserves a 95-minute bounded total Strix budget" assert_file_contains "$workflow_file" 'process_budget_seconds="5400"' "strix workflow gives a legitimate scan up to 90 minutes" + assert_file_contains "$workflow_file" 'nim_process_budget_seconds="1800"' "strix workflow reserves fallback budget after one NVIDIA NIM attempt" + assert_file_contains "$workflow_file" 'export "STRIX_NVIDIA_NIM_PROCESS_${budget_suffix}_SECONDS=$nim_process_budget_seconds"' "strix workflow applies the NVIDIA NIM process cap without a visible timeout env name" assert_file_contains "$workflow_file" 'strix_gate_console.log" "$GITHUB_WORKSPACE/strix_runs/gate-console.log' "strix workflow preserves partial console output after failures and timeouts" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "gate-last-attempt.log" "strix gate preserves the last partial attempt before runtime cleanup" assert_file_contains "$workflow_file" 'IS_PR_EVIDENCE_RUN: ${{ (github.event_name == '"'"'pull_request_target'"'"' || github.event.client_payload.pr_number != '"'"''"'"') && '"'"'true'"'"' || '"'"'false'"'"' }}' "strix workflow passes PR evidence mode through env" @@ -348,8 +350,12 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "github_models/openai/o3 github_models/openai/gpt-5-chat" "strix workflow keeps GitHub Models fallback on tool-capable OpenAI models without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives direct-OpenAI scans GitHub Models fallbacks so provider quota outages degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" + assert_file_contains "$GATE_SCRIPT" "is_github_models_retirement_brownout_error" "strix gate classifies GitHub Models scheduled-retirement 410s" + assert_file_contains "$GATE_SCRIPT" "is_nvidia_nim_model" "strix gate identifies NVIDIA NIM candidates for the reserved process cap" + assert_file_contains "$GATE_SCRIPT" "skip_remaining_github_models" "strix gate skips remaining GitHub Models fallbacks after a retirement brownout" + assert_file_contains "$workflow_file" "github_models_retirement_brownout" "strix workflow treats GitHub Models retirement 410 as backend unavailability" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" diff --git a/tests/test_strix_github_models_retirement_brownout.py b/tests/test_strix_github_models_retirement_brownout.py new file mode 100644 index 000000000..7b80a8b29 --- /dev/null +++ b/tests/test_strix_github_models_retirement_brownout.py @@ -0,0 +1,182 @@ +"""Regression contract for GitHub Models retirement-brownout fallback. + +Required Strix runs observed `Error code: 410` with +`github_models_retirement_brownout` after NVIDIA NIM attempts exhausted the +90-minute process budget. The gate must treat that provider-family outage as +cross-model skip evidence, keep a reserved NVIDIA NIM process cap so later +hosted candidates still run, and refuse to classify application 410 text or +cross-line spoofing as infrastructure. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +OBSERVED_BROWNOUT_LINE = ( + "openai.APIStatusError: Error code: 410 - {'error': {'code': " + "'github_models_retirement_brownout', 'message': 'GitHub Models is " + "temporarily unavailable as part of a scheduled retirement brownout.'}}\n" +) +SECOND_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1" + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_retirement_brownout(log_text: str) -> bool: + """Execute the production brownout classifier against a synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block( + gate_source, + "is_github_models_retirement_brownout_error", + ) + with tempfile.TemporaryDirectory(prefix="strix-gh-410-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_github_models_retirement_brownout_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +class StrixGithubModelsRetirementBrownoutTests(unittest.TestCase): + """Protect family-level 410 skip without weakening security evidence.""" + + def test_observed_github_models_410_is_retryable_provider_evidence(self) -> None: + """Recognize the exact required-CI retirement brownout line.""" + + self.assertTrue(_classifies_as_retirement_brownout(OBSERVED_BROWNOUT_LINE)) + + def test_application_410_without_github_models_context_is_not_retryable( + self, + ) -> None: + """Do not let target-application HTTP 410 text skip fallbacks.""" + + log = "GET /api/project_record/retired 410\nGone: archive missing\n" + self.assertFalse(_classifies_as_retirement_brownout(log)) + + def test_provider_and_410_signals_must_share_one_log_line(self) -> None: + """Reject cross-line assembly from untrusted scan-target output.""" + + log = ( + "source literal: GitHub Models\n" + "GET /api/project_record/retired Error code: 410\n" + ) + self.assertFalse(_classifies_as_retirement_brownout(log)) + + def test_brownout_code_without_410_status_is_not_enough(self) -> None: + """Require the HTTP 410 or retirement phrase on the same line.""" + + log = "github_models_retirement_brownout scheduled later for GitHub Models\n" + self.assertFalse(_classifies_as_retirement_brownout(log)) + + def test_issue_number_410_on_a_brownout_line_is_not_family_dead(self) -> None: + """A SHA or issue #410 is not HTTP 410 (CWE-1288).""" + + log = ( + "github_models_retirement_brownout for GitHub Models; see issue #410\n" + ) + self.assertFalse(_classifies_as_retirement_brownout(log)) + self.assertTrue( + _classifies_as_retirement_brownout( + "GitHub Models HTTP 410 github_models_retirement_brownout\n" + ) + ) + + def test_longer_status_codes_that_start_with_410_are_not_family_dead( + self, + ) -> None: + """HTTP 4100 or Error code: 4104 must not match the retirement 410.""" + + self.assertFalse( + _classifies_as_retirement_brownout( + "GitHub Models Error code: 4100 " + "github_models_retirement_brownout\n" + ) + ) + self.assertFalse( + _classifies_as_retirement_brownout( + "GitHub Models HTTP 4104 github_models_retirement_brownout\n" + ) + ) + self.assertTrue( + _classifies_as_retirement_brownout( + "GitHub Models Error code: 410 " + "github_models_retirement_brownout\n" + ) + ) + + def test_brownout_enters_infrastructure_and_unavailable_model_paths( + self, + ) -> None: + """Wire the classifier into family skip, not same-model retry.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block( + gate_source, + "has_detected_infrastructure_error", + ) + unavailable = _function_block( + gate_source, + "is_github_models_unavailable_model_error", + ) + same_model_retry = _function_block( + gate_source, + "is_transient_same_model_retry_error", + ) + run_once = _function_block(gate_source, "run_strix_once") + current_scan = _function_block(gate_source, "run_current_target_scan") + + self.assertIn("is_github_models_retirement_brownout_error", infrastructure) + self.assertIn("is_github_models_retirement_brownout_error", unavailable) + self.assertNotIn( + "is_github_models_retirement_brownout_error", + same_model_retry, + ) + self.assertIn("is_nvidia_nim_model", run_once) + self.assertIn("STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS", run_once) + self.assertIn("skip_remaining_github_models", current_scan) + + def test_workflow_reserves_nim_budget_and_second_hosted_candidate(self) -> None: + """Keep two NVIDIA hosted candidates before retired GitHub Models.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn('nim_process_budget_seconds="1800"', workflow) + self.assertIn(SECOND_NVIDIA_FALLBACK, workflow) + self.assertIn("github_models_retirement_brownout", workflow) + self.assertNotIn("STRIX_NVIDIA_NIM_PROCESS_TIMEOUT_SECONDS:", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..a97cccf14 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -2,7 +2,7 @@ The central Strix workflow must not turn a provider-side model-catalog 404 into a security finding or retry the same unavailable model. It must move to another -approved free NVIDIA NIM candidate before using the existing GitHub Models +approved NVIDIA NIM candidate before using the existing GitHub Models fallbacks, while ordinary application 404 output remains non-retryable. """ @@ -19,7 +19,10 @@ STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" -FREE_NVIDIA_FALLBACK = ( +PRIMARY_NVIDIA_FALLBACK = ( + "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1" +) +SECONDARY_NVIDIA_FALLBACK = ( "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" ) RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" @@ -171,8 +174,8 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) - def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: - """Prefer a documented hosted NIM and another NIM before GitHub.""" + def test_workflow_uses_bounded_nvidia_fallback_plan(self) -> None: + """Prefer Ultra, then Super, before GitHub Models.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") default_expression = ( @@ -187,8 +190,8 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - f"'{FREE_NVIDIA_FALLBACK} github_models/openai/o3 " - "github_models/openai/gpt-5-chat'", + f"'{PRIMARY_NVIDIA_FALLBACK} {SECONDARY_NVIDIA_FALLBACK} " + "github_models/openai/o3 github_models/openai/gpt-5-chat'", workflow, )