From f188ba3f761767df82f780f779f8d8bbbae3106b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 17 Aug 2026 07:27:13 +0000 Subject: [PATCH] fix(strix): classify ModelBehaviorError flake with zero findings Treat pydantic-ai or LiteLLM ModelBehaviorError plus Vulnerabilities 0 as backend unavailability. Vulnerabilities [1-9] stay fail-closed. A source-file mention without the SDK exception prefix is not infra. --- .github/workflows/strix.yml | 2 +- .../strix-modelbehaviorerror-classifier.md | 33 ++++ scripts/ci/strix_quick_gate.sh | 20 +++ scripts/ci/strix_required_workflow_smoke.sh | 3 + ...est_strix_modelbehaviorerror_classifier.py | 145 ++++++++++++++++++ 5 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 docs/doctoring/strix-modelbehaviorerror-classifier.md create mode 100644 tests/test_strix_modelbehaviorerror_classifier.py diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index f8c361b95..5e1846762 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|pydantic_ai\.exceptions\.ModelBehaviorError|litellm(\.exceptions)?\.[A-Za-z]+Error[^[:cntrl:]]*ModelBehaviorError' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md new file mode 100644 index 000000000..bbb00fda2 --- /dev/null +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -0,0 +1,33 @@ +# Strix ModelBehaviorError classifier + +## Decision + +A pydantic-ai or LiteLLM `ModelBehaviorError` with `Vulnerabilities 0` +is provider-model flake, not a target-application finding. The central +Strix workflow classifies that exact SDK exception as backend +unavailability and may skip the required check as neutral. + +`Vulnerabilities [1-9]` stays fail-closed. A source-file mention of +`ModelBehaviorError` without the trusted `pydantic_ai.exceptions` or +`litellm` exception prefix is not infrastructure. + +## Trust boundary + +The classifier accepts only one bounded log line that names the trusted +SDK exception. Cross-line assembly from repository text is rejected. +This matches the NVIDIA NIM same-line rule +(`docs/doctoring/strix-nvidia-nim-not-found-fallback.md`). + +## Verification contract + +1. `pydantic_ai.exceptions.ModelBehaviorError` plus `Vulnerabilities 0` + is neutralized. +2. The same exception plus `Vulnerabilities 1` remains blocking. +3. A source literal `ModelBehaviorError` without the SDK prefix is not + classified. +4. `reported_vulnerability_signal` still matches `Vulnerabilities [1-9]`. + +## References + +Pydantic. (2026). *pydantic-ai exceptions* [Software documentation]. +https://ai.pydantic.dev/api/exceptions/ diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..a3cfceebb 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2646,6 +2646,18 @@ is_llm_service_unavailable_error() { return 1 } +is_model_behavior_error() { + # Classify only one bounded SDK exception line that names + # ModelBehaviorError. A source-file mention without the trusted + # pydantic_ai or litellm exception prefix is not infrastructure. + # Vulnerabilities [1-9] stay fail-closed in the outer workflow. + if grep -Eiq '(pydantic_ai\.exceptions\.ModelBehaviorError|litellm(\.exceptions)?\.[A-Za-z]+Error[^[:cntrl:]]*ModelBehaviorError)' "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + is_nvidia_nim_not_found_error() { # Classify only one bounded LiteLLM provider-error line that also # carries NVIDIA NIM context and model-catalog not-found evidence. @@ -2976,6 +2988,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3826,6 +3842,10 @@ is_model_retryable_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 8cd6dddad..458787da5 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -159,6 +159,9 @@ assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 github_models/openai/o3 github_models/openai/gpt-5-chat" "Strix tries another NVIDIA hosted model before GitHub Models" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" +assert_file_contains "$workflow_file" "pydantic_ai\\.exceptions\\.ModelBehaviorError" "Strix workflow classifies ModelBehaviorError as backend unavailability" +assert_file_contains "$gate_script" "is_model_behavior_error" "Strix gate classifies ModelBehaviorError flakes" +assert_file_contains "$workflow_file" 'Vulnerabilities[[:space:]]+[1-9]' "Strix workflow stays fail-closed on Vulnerabilities [1-9]" if [ "$failures" -ne 0 ]; then echo "Strix required workflow smoke test failed with $failures failure(s)." >&2 diff --git a/tests/test_strix_modelbehaviorerror_classifier.py b/tests/test_strix_modelbehaviorerror_classifier.py new file mode 100644 index 000000000..543caae2b --- /dev/null +++ b/tests/test_strix_modelbehaviorerror_classifier.py @@ -0,0 +1,145 @@ +"""Regression contract for Strix ModelBehaviorError flake classification. + +A trusted pydantic-ai or LiteLLM ModelBehaviorError with Vulnerabilities 0 +is backend unavailability. Vulnerabilities [1-9] stay fail-closed. A +source-file mention without the SDK exception prefix is not infrastructure. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" + + +def _function_block(source: str, function_name: str) -> str: + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^\}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_model_behavior_error(log_text: str) -> bool: + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_behavior_error") + with tempfile.TemporaryDirectory(prefix="strix-mbe-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_model_behavior_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-workflow-mbe-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixModelBehaviorErrorClassifierTests(unittest.TestCase): + """Neutralize ModelBehaviorError flake without weakening Vulnerabilities [1-9].""" + + def test_pydantic_ai_model_behavior_error_with_zero_findings_is_infra(self) -> None: + log = ( + "pydantic_ai.exceptions.ModelBehaviorError: " + "unexpected tool call arguments\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + self.assertTrue(_workflow_neutralizes(log)) + + def test_litellm_wrapped_model_behavior_error_with_zero_findings_is_infra(self) -> None: + log = ( + "litellm.exceptions.APIError: ModelBehaviorError from provider\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + self.assertTrue(_workflow_neutralizes(log)) + + def test_model_behavior_error_with_vulnerabilities_stays_fail_closed(self) -> None: + log = ( + "pydantic_ai.exceptions.ModelBehaviorError: unexpected tool call\n" + "Vulnerabilities 1\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + self.assertFalse(_workflow_neutralizes(log)) + + def test_source_literal_model_behavior_error_is_not_infra(self) -> None: + log = "raise ModelBehaviorError('spoofed from target source')\nVulnerabilities 0\n" + self.assertFalse(_classifies_as_model_behavior_error(log)) + self.assertFalse(_workflow_neutralizes(log)) + + def test_gate_wires_classifier_into_infra_and_retry(self) -> None: + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block(gate_source, "has_detected_infrastructure_error") + retryable = _function_block(gate_source, "is_model_retryable_error") + self.assertIn("is_model_behavior_error", infrastructure) + self.assertIn("is_model_behavior_error", retryable) + + def test_reported_vulnerability_signal_still_matches_one_through_nine(self) -> None: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", vulnerability_pattern)