diff --git a/.sampo/changesets/openai-chat-refusal-output.md b/.sampo/changesets/openai-chat-refusal-output.md new file mode 100644 index 000000000..4d222d4d2 --- /dev/null +++ b/.sampo/changesets/openai-chat-refusal-output.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +A non-streamed OpenAI chat completion that the model refused is no longer captured with an empty `$ai_output_choices`: the refusal text is recorded as a `refusal` content block, as it already is for streamed responses. diff --git a/posthog/ai/openai/openai_converter.py b/posthog/ai/openai/openai_converter.py index 0f391fb9f..f4b2e3e5e 100644 --- a/posthog/ai/openai/openai_converter.py +++ b/posthog/ai/openai/openai_converter.py @@ -151,6 +151,11 @@ def format_openai_response(response: Any) -> List[FormattedMessage]: } ) + # A refused request carries its text in `refusal` and leaves `content` empty + refusal = getattr(choice.message, "refusal", None) + if refusal: + content.append({"type": "refusal", "refusal": refusal}) + if hasattr(choice.message, "tool_calls") and choice.message.tool_calls: for tool_call in choice.message.tool_calls: content.append( diff --git a/posthog/test/ai/openai/test_openai_converter.py b/posthog/test/ai/openai/test_openai_converter.py index e23ecac87..a6ee44e6b 100644 --- a/posthog/test/ai/openai/test_openai_converter.py +++ b/posthog/test/ai/openai/test_openai_converter.py @@ -82,6 +82,37 @@ def test_message_object_does_not_crash(): assert out[1]["content"] == "hello" +def test_chat_response_keeps_refusal(): + """A refused chat completion has no content, only message.refusal, and used to be captured as [].""" + from openai.types.chat import ChatCompletion + from openai.types.chat.chat_completion import Choice + + from posthog.ai.openai.openai_converter import format_openai_response + + response = ChatCompletion( + id="chatcmpl-1", + object="chat.completion", + created=0, + model="gpt-4o", + choices=[ + Choice( + index=0, + finish_reason="stop", + message=ChatCompletionMessage( + role="assistant", content=None, refusal="I can't help with that" + ), + ) + ], + ) + + assert format_openai_response(response) == [ + { + "role": "assistant", + "content": [{"type": "refusal", "refusal": "I can't help with that"}], + } + ] + + def test_responses_output_keeps_all_items(): from posthog.ai.openai.openai_converter import format_openai_response