Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/openai-chat-refusal-output.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

A non-streamed OpenAI chat completion that the model refused is no longer captured with an empty `$ai_output_choices`: the refusal text is recorded as a `refusal` content block, as it already is for streamed responses.
5 changes: 5 additions & 0 deletions posthog/ai/openai/openai_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,11 @@ def format_openai_response(response: Any) -> List[FormattedMessage]:
}
)

# A refused request carries its text in `refusal` and leaves `content` empty
refusal = getattr(choice.message, "refusal", None)
if refusal:
content.append({"type": "refusal", "refusal": refusal})

if hasattr(choice.message, "tool_calls") and choice.message.tool_calls:
for tool_call in choice.message.tool_calls:
content.append(
Expand Down
31 changes: 31 additions & 0 deletions posthog/test/ai/openai/test_openai_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,37 @@ def test_message_object_does_not_crash():
assert out[1]["content"] == "hello"


def test_chat_response_keeps_refusal():
"""A refused chat completion has no content, only message.refusal, and used to be captured as []."""
from openai.types.chat import ChatCompletion
from openai.types.chat.chat_completion import Choice

from posthog.ai.openai.openai_converter import format_openai_response

response = ChatCompletion(
id="chatcmpl-1",
object="chat.completion",
created=0,
model="gpt-4o",
choices=[
Choice(
index=0,
finish_reason="stop",
message=ChatCompletionMessage(
role="assistant", content=None, refusal="I can't help with that"
),
)
],
)

assert format_openai_response(response) == [
{
"role": "assistant",
"content": [{"type": "refusal", "refusal": "I can't help with that"}],
}
]


def test_responses_output_keeps_all_items():
from posthog.ai.openai.openai_converter import format_openai_response

Expand Down
Loading