diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b6f64e2..e6fa584d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ + + +## 3.20.0 – 2025-03-13 + +1. Add support for OpenAI Responses API. + ## 3.19.2 – 2025-03-11 1. Fix install requirements for analytics package diff --git a/mypy-baseline.txt b/mypy-baseline.txt index 9d17951f..be96ec92 100644 --- a/mypy-baseline.txt +++ b/mypy-baseline.txt @@ -35,21 +35,7 @@ posthog/sentry/posthog_integration.py:0: error: Statement is unreachable [unrea posthog/ai/utils.py:0: error: Need type annotation for "output" (hint: "output: list[] = ...") [var-annotated] posthog/ai/utils.py:0: error: Function "builtins.any" is not valid as a type [valid-type] posthog/ai/utils.py:0: note: Perhaps you meant "typing.Any" instead of "any"? -posthog/ai/utils.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] posthog/ai/utils.py:0: error: Function "builtins.any" is not valid as a type [valid-type] posthog/ai/utils.py:0: note: Perhaps you meant "typing.Any" instead of "any"? -posthog/ai/utils.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] sentry_django_example/sentry_django_example/settings.py:0: error: Need type annotation for "ALLOWED_HOSTS" (hint: "ALLOWED_HOSTS: list[] = ...") [var-annotated] -sentry_django_example/sentry_django_example/settings.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "None") [assignment] -posthog/ai/openai/openai_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/openai/openai_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/openai/openai_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/openai/openai.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/openai/openai.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/openai/openai.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic_async.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] -posthog/ai/anthropic/anthropic.py:0: error: Incompatible types in assignment (expression has type "UUID", variable has type "str | None") [assignment] +sentry_django_example/sentry_django_example/settings.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "None") [assignment] \ No newline at end of file diff --git a/posthog/ai/anthropic/anthropic.py b/posthog/ai/anthropic/anthropic.py index 276a4c03..b4fde16f 100644 --- a/posthog/ai/anthropic/anthropic.py +++ b/posthog/ai/anthropic/anthropic.py @@ -54,7 +54,7 @@ def create( **kwargs: Arguments passed to Anthropic's messages.create """ if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) if kwargs.get("stream", False): return self._create_streaming( @@ -89,7 +89,7 @@ def stream( **kwargs: Any, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) return self._create_streaming( posthog_distinct_id, @@ -167,7 +167,7 @@ def _capture_streaming_event( output: str, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) event_properties = { "$ai_provider": "anthropic", diff --git a/posthog/ai/anthropic/anthropic_async.py b/posthog/ai/anthropic/anthropic_async.py index 0b155ca5..5dd05764 100644 --- a/posthog/ai/anthropic/anthropic_async.py +++ b/posthog/ai/anthropic/anthropic_async.py @@ -54,7 +54,7 @@ async def create( **kwargs: Arguments passed to Anthropic's messages.create """ if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) if kwargs.get("stream", False): return await self._create_streaming( @@ -89,7 +89,7 @@ async def stream( **kwargs: Any, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) return await self._create_streaming( posthog_distinct_id, @@ -167,7 +167,7 @@ async def _capture_streaming_event( output: str, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) event_properties = { "$ai_provider": "anthropic", diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index 3d88b169..0c2be5fe 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -32,6 +32,167 @@ def __init__(self, posthog_client: PostHogClient, **kwargs): self.chat = WrappedChat(self) self.embeddings = WrappedEmbeddings(self) self.beta = WrappedBeta(self) + self.responses = WrappedResponses(self) + + +class WrappedResponses(openai.resources.responses.Responses): + _client: OpenAI + + def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ): + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + if kwargs.get("stream", False): + return self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + return call_llm_and_track_usage( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + super().create, + **kwargs, + ) + + def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + usage_stats: Dict[str, int] = {} + final_content = [] + response = super().create(**kwargs) + + def generator(): + nonlocal usage_stats + nonlocal final_content + + try: + for chunk in response: + if hasattr(chunk, "type") and chunk.type == "response.completed": + res = chunk.response + if res.output and len(res.output) > 0: + final_content.append(res.output[0]) + + if hasattr(chunk, "usage") and chunk.usage: + usage_stats = { + k: getattr(chunk.usage, k, 0) + for k in [ + "input_tokens", + "output_tokens", + "total_tokens", + ] + } + + # Add support for cached tokens + if hasattr(chunk.usage, "output_tokens_details") and hasattr( + chunk.usage.output_tokens_details, "reasoning_tokens" + ): + usage_stats["reasoning_tokens"] = chunk.usage.output_tokens_details.reasoning_tokens + + if hasattr(chunk.usage, "input_tokens_details") and hasattr( + chunk.usage.input_tokens_details, "cached_tokens" + ): + usage_stats["cache_read_input_tokens"] = chunk.usage.input_tokens_details.cached_tokens + + yield chunk + + finally: + end_time = time.time() + latency = end_time - start_time + output = final_content + self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + usage_stats, + latency, + output, + ) + + return generator() + + def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + usage_stats: Dict[str, int], + latency: float, + output: Any, + tool_calls: Optional[List[Dict[str, Any]]] = None, + ): + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + event_properties = { + "$ai_provider": "openai", + "$ai_model": kwargs.get("model"), + "$ai_model_parameters": get_model_params(kwargs), + "$ai_input": with_privacy_mode(self._client._ph_client, posthog_privacy_mode, kwargs.get("input")), + "$ai_output_choices": with_privacy_mode( + self._client._ph_client, + posthog_privacy_mode, + output, + ), + "$ai_http_status": 200, + "$ai_input_tokens": usage_stats.get("input_tokens", 0), + "$ai_output_tokens": usage_stats.get("output_tokens", 0), + "$ai_cache_read_input_tokens": usage_stats.get("cache_read_input_tokens", 0), + "$ai_reasoning_tokens": usage_stats.get("reasoning_tokens", 0), + "$ai_latency": latency, + "$ai_trace_id": posthog_trace_id, + "$ai_base_url": str(self._client.base_url), + **(posthog_properties or {}), + } + + if tool_calls: + event_properties["$ai_tools"] = with_privacy_mode( + self._client._ph_client, + posthog_privacy_mode, + tool_calls, + ) + + if posthog_distinct_id is None: + event_properties["$process_person_profile"] = False + + if hasattr(self._client._ph_client, "capture"): + self._client._ph_client.capture( + distinct_id=posthog_distinct_id or posthog_trace_id, + event="$ai_generation", + properties=event_properties, + groups=posthog_groups, + ) class WrappedChat(openai.resources.chat.Chat): @@ -55,7 +216,7 @@ def create( **kwargs: Any, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) if kwargs.get("stream", False): return self._create_streaming( @@ -121,6 +282,11 @@ def generator(): ): usage_stats["cache_read_input_tokens"] = chunk.usage.prompt_tokens_details.cached_tokens + if hasattr(chunk.usage, "output_tokens_details") and hasattr( + chunk.usage.output_tokens_details, "reasoning_tokens" + ): + usage_stats["reasoning_tokens"] = chunk.usage.output_tokens_details.reasoning_tokens + if hasattr(chunk, "choices") and chunk.choices and len(chunk.choices) > 0: if chunk.choices[0].delta and chunk.choices[0].delta.content: content = chunk.choices[0].delta.content @@ -171,11 +337,11 @@ def _capture_streaming_event( kwargs: Dict[str, Any], usage_stats: Dict[str, int], latency: float, - output: str, + output: Any, tool_calls: Optional[List[Dict[str, Any]]] = None, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) event_properties = { "$ai_provider": "openai", @@ -191,6 +357,7 @@ def _capture_streaming_event( "$ai_input_tokens": usage_stats.get("prompt_tokens", 0), "$ai_output_tokens": usage_stats.get("completion_tokens", 0), "$ai_cache_read_input_tokens": usage_stats.get("cache_read_input_tokens", 0), + "$ai_reasoning_tokens": usage_stats.get("reasoning_tokens", 0), "$ai_latency": latency, "$ai_trace_id": posthog_trace_id, "$ai_base_url": str(self._client.base_url), @@ -241,7 +408,7 @@ def create( The response from OpenAI's embeddings.create call. """ if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) start_time = time.time() response = super().create(**kwargs) diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 9c2ed984..e12cd825 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -31,6 +31,167 @@ def __init__(self, posthog_client: PostHogClient, **kwargs): self.chat = WrappedChat(self) self.embeddings = WrappedEmbeddings(self) self.beta = WrappedBeta(self) + self.responses = WrappedResponses(self) + + +class WrappedResponses(openai.resources.responses.Responses): + _client: AsyncOpenAI + + async def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ): + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + if kwargs.get("stream", False): + return await self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + return await call_llm_and_track_usage_async( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + super().create, + **kwargs, + ) + + async def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + usage_stats: Dict[str, int] = {} + final_content = [] + response = await super().create(**kwargs) + + async def async_generator(): + nonlocal usage_stats + nonlocal final_content + + try: + async for chunk in response: + if hasattr(chunk, "type") and chunk.type == "response.completed": + res = chunk.response + if res.output and len(res.output) > 0: + final_content.append(res.output[0]) + + if hasattr(chunk, "usage") and chunk.usage: + usage_stats = { + k: getattr(chunk.usage, k, 0) + for k in [ + "input_tokens", + "output_tokens", + "total_tokens", + ] + } + + # Add support for cached tokens + if hasattr(chunk.usage, "output_tokens_details") and hasattr( + chunk.usage.output_tokens_details, "reasoning_tokens" + ): + usage_stats["reasoning_tokens"] = chunk.usage.output_tokens_details.reasoning_tokens + + if hasattr(chunk.usage, "input_tokens_details") and hasattr( + chunk.usage.input_tokens_details, "cached_tokens" + ): + usage_stats["cache_read_input_tokens"] = chunk.usage.input_tokens_details.cached_tokens + + yield chunk + + finally: + end_time = time.time() + latency = end_time - start_time + output = final_content + await self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + usage_stats, + latency, + output, + ) + + return async_generator() + + async def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + usage_stats: Dict[str, int], + latency: float, + output: Any, + tool_calls: Optional[List[Dict[str, Any]]] = None, + ): + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + event_properties = { + "$ai_provider": "openai", + "$ai_model": kwargs.get("model"), + "$ai_model_parameters": get_model_params(kwargs), + "$ai_input": with_privacy_mode(self._client._ph_client, posthog_privacy_mode, kwargs.get("input")), + "$ai_output_choices": with_privacy_mode( + self._client._ph_client, + posthog_privacy_mode, + output, + ), + "$ai_http_status": 200, + "$ai_input_tokens": usage_stats.get("input_tokens", 0), + "$ai_output_tokens": usage_stats.get("output_tokens", 0), + "$ai_cache_read_input_tokens": usage_stats.get("cache_read_input_tokens", 0), + "$ai_reasoning_tokens": usage_stats.get("reasoning_tokens", 0), + "$ai_latency": latency, + "$ai_trace_id": posthog_trace_id, + "$ai_base_url": str(self._client.base_url), + **(posthog_properties or {}), + } + + if tool_calls: + event_properties["$ai_tools"] = with_privacy_mode( + self._client._ph_client, + posthog_privacy_mode, + tool_calls, + ) + + if posthog_distinct_id is None: + event_properties["$process_person_profile"] = False + + if hasattr(self._client._ph_client, "capture"): + await self._client._ph_client.capture( + distinct_id=posthog_distinct_id or posthog_trace_id, + event="$ai_generation", + properties=event_properties, + groups=posthog_groups, + ) class WrappedChat(openai.resources.chat.AsyncChat): @@ -54,7 +215,7 @@ async def create( **kwargs: Any, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) # If streaming, handle streaming specifically if kwargs.get("stream", False): @@ -169,11 +330,11 @@ async def _capture_streaming_event( kwargs: Dict[str, Any], usage_stats: Dict[str, int], latency: float, - output: str, + output: Any, tool_calls: Optional[List[Dict[str, Any]]] = None, ): if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) event_properties = { "$ai_provider": "openai", @@ -206,7 +367,7 @@ async def _capture_streaming_event( event_properties["$process_person_profile"] = False if hasattr(self._client._ph_client, "capture"): - self._client._ph_client.capture( + await self._client._ph_client.capture( distinct_id=posthog_distinct_id or posthog_trace_id, event="$ai_generation", properties=event_properties, @@ -241,7 +402,7 @@ async def create( The response from OpenAI's embeddings.create call. """ if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) start_time = time.time() response = await super().create(**kwargs) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 51217f80..ebee7cf4 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1,6 +1,6 @@ import time import uuid -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from httpx import URL @@ -39,20 +39,46 @@ def get_usage(response, provider: str) -> Dict[str, Any]: } elif provider == "openai": cached_tokens = 0 + input_tokens = 0 + output_tokens = 0 + reasoning_tokens = 0 + + # responses api + if hasattr(response.usage, "input_tokens"): + input_tokens = response.usage.input_tokens + if hasattr(response.usage, "output_tokens"): + output_tokens = response.usage.output_tokens + if hasattr(response.usage, "input_tokens_details") and hasattr( + response.usage.input_tokens_details, "cached_tokens" + ): + cached_tokens = response.usage.input_tokens_details.cached_tokens + if hasattr(response.usage, "output_tokens_details") and hasattr( + response.usage.output_tokens_details, "reasoning_tokens" + ): + reasoning_tokens = response.usage.output_tokens_details.reasoning_tokens + + # chat completions + if hasattr(response.usage, "prompt_tokens"): + input_tokens = response.usage.prompt_tokens + if hasattr(response.usage, "completion_tokens"): + output_tokens = response.usage.completion_tokens if hasattr(response.usage, "prompt_tokens_details") and hasattr( response.usage.prompt_tokens_details, "cached_tokens" ): cached_tokens = response.usage.prompt_tokens_details.cached_tokens + return { - "input_tokens": response.usage.prompt_tokens, - "output_tokens": response.usage.completion_tokens, + "input_tokens": input_tokens, + "output_tokens": output_tokens, "cache_read_input_tokens": cached_tokens, + "reasoning_tokens": reasoning_tokens, } return { "input_tokens": 0, "output_tokens": 0, "cache_read_input_tokens": 0, "cache_creation_input_tokens": 0, + "reasoning_tokens": 0, } @@ -85,14 +111,62 @@ def format_response_anthropic(response): def format_response_openai(response): output = [] - for choice in response.choices: - if choice.message.content: - output.append( - { - "content": choice.message.content, - "role": choice.message.role, - } - ) + if hasattr(response, "choices"): + for choice in response.choices: + # Handle Chat Completions response format + if hasattr(choice, "message") and choice.message and choice.message.content: + output.append( + { + "content": choice.message.content, + "role": choice.message.role, + } + ) + # Handle Responses API format + if hasattr(response, "output"): + for item in response.output: + if item.type == "message": + # Extract text content from the content list + if hasattr(item, "content") and isinstance(item.content, list): + for content_item in item.content: + if ( + hasattr(content_item, "type") + and content_item.type == "output_text" + and hasattr(content_item, "text") + ): + output.append( + { + "content": content_item.text, + "role": item.role, + } + ) + elif hasattr(content_item, "text"): + output.append( + { + "content": content_item.text, + "role": item.role, + } + ) + elif ( + hasattr(content_item, "type") + and content_item.type == "input_image" + and hasattr(content_item, "image_url") + ): + output.append( + { + "content": { + "type": "image", + "image": content_item.image_url, + }, + "role": item.role, + } + ) + else: + output.append( + { + "content": item.content, + "role": item.role, + } + ) return output @@ -101,23 +175,61 @@ def format_tool_calls(response, provider: str): if hasattr(response, "tools") and response.tools and len(response.tools) > 0: return response.tools elif provider == "openai": - if ( - hasattr(response, "choices") - and response.choices - and hasattr(response.choices[0].message, "tool_calls") - and response.choices[0].message.tool_calls - ): - return response.choices[0].message.tool_calls + # Handle both Chat Completions and Responses API + if hasattr(response, "choices") and response.choices: + # Check for tool_calls in message (Chat Completions format) + if ( + hasattr(response.choices[0], "message") + and hasattr(response.choices[0].message, "tool_calls") + and response.choices[0].message.tool_calls + ): + return response.choices[0].message.tool_calls + + # Check for tool_calls directly in response (Responses API format) + if hasattr(response.choices[0], "tool_calls") and response.choices[0].tool_calls: + return response.choices[0].tool_calls return None def merge_system_prompt(kwargs: Dict[str, Any], provider: str): - if provider != "anthropic": - return kwargs.get("messages") - messages = kwargs.get("messages") or [] - if kwargs.get("system") is None: - return messages - return [{"role": "system", "content": kwargs.get("system")}] + messages + messages: List[Dict[str, Any]] = [] + if provider == "anthropic": + messages = kwargs.get("messages") or [] + if kwargs.get("system") is None: + return messages + return [{"role": "system", "content": kwargs.get("system")}] + messages + + # For OpenAI, handle both Chat Completions and Responses API + if kwargs.get("messages") is not None: + messages = list(kwargs.get("messages", [])) + + if kwargs.get("input") is not None: + input_data = kwargs.get("input") + if isinstance(input_data, list): + messages.extend(input_data) + else: + messages.append({"role": "user", "content": input_data}) + + # Check if system prompt is provided as a separate parameter + if kwargs.get("system") is not None: + has_system = any(msg.get("role") == "system" for msg in messages) + if not has_system: + messages = [{"role": "system", "content": kwargs.get("system")}] + messages + + # For Responses API, add instructions to the system prompt if provided + if kwargs.get("instructions") is not None: + # Find the system message if it exists + system_idx = next((i for i, msg in enumerate(messages) if msg.get("role") == "system"), None) + + if system_idx is not None: + # Append instructions to existing system message + system_content = messages[system_idx].get("content", "") + messages[system_idx]["content"] = f"{system_content}\n\n{kwargs.get('instructions')}" + else: + # Create a new system message with instructions + messages = [{"role": "system", "content": kwargs.get("instructions")}] + messages + + return messages def call_llm_and_track_usage( @@ -157,7 +269,7 @@ def call_llm_and_track_usage( latency = end_time - start_time if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) if response and hasattr(response, "usage"): usage = get_usage(response, provider) @@ -192,9 +304,18 @@ def call_llm_and_track_usage( if usage.get("cache_creation_input_tokens") is not None and usage.get("cache_creation_input_tokens", 0) > 0: event_properties["$ai_cache_creation_input_tokens"] = usage.get("cache_creation_input_tokens", 0) + if usage.get("reasoning_tokens") is not None and usage.get("reasoning_tokens", 0) > 0: + event_properties["$ai_reasoning_tokens"] = usage.get("reasoning_tokens", 0) + if posthog_distinct_id is None: event_properties["$process_person_profile"] = False + # Process instructions for Responses API + if provider == "openai" and kwargs.get("instructions") is not None: + event_properties["$ai_instructions"] = with_privacy_mode( + ph_client, posthog_privacy_mode, kwargs.get("instructions") + ) + # send the event to posthog if hasattr(ph_client, "capture") and callable(ph_client.capture): ph_client.capture( @@ -243,7 +364,7 @@ async def call_llm_and_track_usage_async( latency = end_time - start_time if posthog_trace_id is None: - posthog_trace_id = uuid.uuid4() + posthog_trace_id = str(uuid.uuid4()) if response and hasattr(response, "usage"): usage = get_usage(response, provider) @@ -281,6 +402,12 @@ async def call_llm_and_track_usage_async( if posthog_distinct_id is None: event_properties["$process_person_profile"] = False + # Process instructions for Responses API + if provider == "openai" and kwargs.get("instructions") is not None: + event_properties["$ai_instructions"] = with_privacy_mode( + ph_client, posthog_privacy_mode, kwargs.get("instructions") + ) + # send the event to posthog if hasattr(ph_client, "capture") and callable(ph_client.capture): ph_client.capture( diff --git a/posthog/test/ai/openai/test_openai.py b/posthog/test/ai/openai/test_openai.py index 071d7af6..25737b9b 100644 --- a/posthog/test/ai/openai/test_openai.py +++ b/posthog/test/ai/openai/test_openai.py @@ -12,6 +12,7 @@ from openai.types.completion_usage import CompletionUsage from openai.types.create_embedding_response import CreateEmbeddingResponse, Usage from openai.types.embedding import Embedding +from openai.types.responses import Response, ResponseOutputMessage, ResponseOutputText, ResponseUsage from posthog.ai.openai import OpenAI @@ -48,6 +49,48 @@ def mock_openai_response(): ) +@pytest.fixture +def mock_openai_response_with_responses_api(): + return Response( + id="test", + model="gpt-4o-mini", + object="response", + created_at=1741476542, + status="completed", + error=None, + incomplete_details=None, + instructions=None, + max_output_tokens=None, + tools=[], + tool_choice="auto", + output=[ + ResponseOutputMessage( + id="msg_123", + type="message", + role="assistant", + status="completed", + content=[ + ResponseOutputText( + type="output_text", + text="Test response", + annotations=[], + ) + ], + ) + ], + parallel_tool_calls=True, + previous_response_id=None, + usage=ResponseUsage( + input_tokens=10, + output_tokens=10, + output_tokens_details={"reasoning_tokens": 15}, + total_tokens=20, + ), + user=None, + metadata={}, + ) + + @pytest.fixture def mock_embedding_response(): return CreateEmbeddingResponse( @@ -499,3 +542,33 @@ def test_streaming_with_tool_calls(mock_client): # Check token usage assert props["$ai_input_tokens"] == 20 assert props["$ai_output_tokens"] == 15 + + +# test responses api +def test_responses_api(mock_client, mock_openai_response_with_responses_api): + with patch("openai.resources.responses.Responses.create", return_value=mock_openai_response_with_responses_api): + client = OpenAI(api_key="test-key", posthog_client=mock_client) + response = client.responses.create( + model="gpt-4o-mini", + input="Hello", + posthog_distinct_id="test-id", + posthog_properties={"foo": "bar"}, + ) + assert response == mock_openai_response_with_responses_api + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + + assert call_args["distinct_id"] == "test-id" + assert call_args["event"] == "$ai_generation" + assert props["$ai_provider"] == "openai" + assert props["$ai_model"] == "gpt-4o-mini" + assert props["$ai_input"] == [{"role": "user", "content": "Hello"}] + assert props["$ai_output_choices"] == [{"role": "assistant", "content": "Test response"}] + assert props["$ai_input_tokens"] == 10 + assert props["$ai_output_tokens"] == 10 + assert props["$ai_reasoning_tokens"] == 15 + assert props["$ai_http_status"] == 200 + assert props["foo"] == "bar" + assert isinstance(props["$ai_latency"], float) diff --git a/posthog/version.py b/posthog/version.py index 7b5d56f5..adf8bb46 100644 --- a/posthog/version.py +++ b/posthog/version.py @@ -1,4 +1,4 @@ -VERSION = "3.19.1" +VERSION = "3.20.0" if __name__ == "__main__": print(VERSION, end="") # noqa: T201