diff --git a/.sampo/changesets/gemini-error-http-status.md b/.sampo/changesets/gemini-error-http-status.md new file mode 100644 index 000000000..4cc01a508 --- /dev/null +++ b/.sampo/changesets/gemini-error-http-status.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Failed Gemini calls are now captured with the HTTP status of the error as `$ai_http_status` (for example 429 or 503) instead of 0. google-genai errors carry the status as `code` rather than `status_code`, which the OpenAI and Anthropic wrappers already read. diff --git a/posthog/ai/gemini/_shared.py b/posthog/ai/gemini/_shared.py index fc3635589..07cb84b9e 100644 --- a/posthog/ai/gemini/_shared.py +++ b/posthog/ai/gemini/_shared.py @@ -16,6 +16,7 @@ _capture_ai_event, capture_streaming_event, finalize_ai_content, + _get_http_status, merge_system_prompt, with_privacy_mode, ) @@ -250,9 +251,7 @@ def _capture_embedding_outcome( privacy_mode, finalize_ai_content(contents, self._ph_client), ), - "$ai_http_status": ( - getattr(error, "status_code", 0) if error is not None else 200 - ), + "$ai_http_status": (_get_http_status(error) if error is not None else 200), # Omitted when the provider never reported a count: absent means # unknown, 0 is a report of nothing. **({"$ai_input_tokens": input_tokens} if input_tokens is not None else {}), diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index c943bcc28..b27e8b384 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -453,6 +453,21 @@ def merge_system_prompt( return [] +def _get_http_status(exc: BaseException) -> int: + """HTTP status of an SDK exception, 0 when it has none. + + OpenAI and Anthropic errors carry `status_code`; google-genai's `APIError` + carries the status as `code`. + """ + status = getattr(exc, "status_code", None) + if isinstance(status, int): + return status + code = getattr(exc, "code", None) + if isinstance(code, int) and not isinstance(code, bool) and 100 <= code <= 599: + return code + return 0 + + def call_llm_and_track_usage( posthog_distinct_id: Optional[str], ph_client: PostHogClient, @@ -484,9 +499,7 @@ def call_llm_and_track_usage( response = call_method(**kwargs) except Exception as exc: error = exc - http_status = getattr( - exc, "status_code", 0 - ) # default to 0 becuase its likely an SDK error + http_status = _get_http_status(exc) # 0 because its likely an SDK error error_params = { "$ai_is_error": True, "$ai_error": exc.__str__(), @@ -658,9 +671,7 @@ async def call_llm_and_track_usage_async( raise except Exception as exc: error = exc - http_status = getattr( - exc, "status_code", 0 - ) # default to 0 because its likely an SDK error + http_status = _get_http_status(exc) # 0 because its likely an SDK error error_params = { "$ai_is_error": True, "$ai_error": exc.__str__(), diff --git a/posthog/test/ai/test_http_status.py b/posthog/test/ai/test_http_status.py new file mode 100644 index 000000000..1f79fa276 --- /dev/null +++ b/posthog/test/ai/test_http_status.py @@ -0,0 +1,124 @@ +"""Failed provider calls are recorded with the HTTP status of the SDK error.""" + +from unittest.mock import patch + +import pytest + +from posthog.ai.utils import ( + call_llm_and_track_usage, + call_llm_and_track_usage_async, + _get_http_status, +) + + +@pytest.fixture +def mock_client(): + with patch("posthog.client.Client") as mock_client: + mock_client.privacy_mode = False + yield mock_client + + +class _WithStatusCode(Exception): + status_code = 503 + + +class _WithCode(Exception): + def __init__(self, code): + super().__init__("boom") + self.code = code + + +def test_get_http_status(): + assert _get_http_status(_WithStatusCode()) == 503 + assert _get_http_status(_WithCode(429)) == 429 + # A `code` that is not an HTTP status is not one + assert _get_http_status(_WithCode(2)) == 0 + assert _get_http_status(_WithCode("RESOURCE_EXHAUSTED")) == 0 + assert _get_http_status(RuntimeError("no status")) == 0 + + +def _genai_errors(): + errors = pytest.importorskip("google.genai.errors") + return errors + + +def _gemini_error(errors, cls, code): + return cls(code, {"error": {"code": code, "message": "nope", "status": "ERROR"}}) + + +@pytest.mark.parametrize( + ("cls_name", "code"), [("ClientError", 429), ("ServerError", 503)] +) +def test_gemini_api_error_status_is_recorded(mock_client, cls_name, code): + errors = _genai_errors() + err = _gemini_error(errors, getattr(errors, cls_name), code) + + def boom(**kwargs): + raise err + + with pytest.raises(getattr(errors, cls_name)): + call_llm_and_track_usage( + "user", + mock_client, + "gemini", + None, + None, + False, + None, + "https://generativelanguage.googleapis.com", + boom, + model="gemini-3.8-flash", + contents="hi", + ) + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == code + + +@pytest.mark.asyncio +async def test_gemini_api_error_status_is_recorded_async(mock_client): + errors = _genai_errors() + err = _gemini_error(errors, errors.ClientError, 429) + + async def boom(**kwargs): + raise err + + with pytest.raises(errors.ClientError): + await call_llm_and_track_usage_async( + "user", + mock_client, + "gemini", + None, + None, + False, + None, + "https://generativelanguage.googleapis.com", + boom, + model="gemini-3.8-flash", + contents="hi", + ) + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_http_status"] == 429 + + +def test_status_code_errors_keep_their_status(mock_client): + def boom(**kwargs): + raise _WithStatusCode("overloaded") + + with pytest.raises(_WithStatusCode): + call_llm_and_track_usage( + "user", + mock_client, + "openai", + None, + None, + False, + None, + "https://x", + boom, + model="m", + ) + + assert mock_client.capture.call_args[1]["properties"]["$ai_http_status"] == 503