From 79a37882dea31ea2a99290a3dd011f4a2f1d9eb2 Mon Sep 17 00:00:00 2001 From: David Newell Date: Wed, 26 Nov 2025 15:30:47 +0000 Subject: [PATCH 01/16] feat: llma / error tracking integration --- posthog/ai/langchain/callbacks.py | 12 +++- posthog/integrations/lang_chain.py | 89 ++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 posthog/integrations/lang_chain.py diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 68840a63..b76d11ff 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -22,8 +22,8 @@ try: # LangChain 1.0+ and modern 0.x with langchain-core - from langchain_core.callbacks.base import BaseCallbackHandler from langchain_core.agents import AgentAction, AgentFinish + from langchain_core.callbacks.base import BaseCallbackHandler except (ImportError, ModuleNotFoundError): # Fallback for older LangChain versions from langchain.callbacks.base import BaseCallbackHandler @@ -35,15 +35,15 @@ FunctionMessage, HumanMessage, SystemMessage, - ToolMessage, ToolCall, + ToolMessage, ) from langchain_core.outputs import ChatGeneration, LLMResult from pydantic import BaseModel from posthog import setup -from posthog.ai.utils import get_model_params, with_privacy_mode from posthog.ai.sanitization import sanitize_langchain +from posthog.ai.utils import get_model_params, with_privacy_mode from posthog.client import Client log = logging.getLogger("posthog") @@ -580,6 +580,12 @@ def _capture_generation( event_properties["$ai_http_status"] = _get_http_status(output) event_properties["$ai_error"] = _stringify_exception(output) event_properties["$ai_is_error"] = True + + if self._ph_client.enable_exception_autocapture: + exception_id = self._ph_client.capture_exception( + output, properties=event_properties + ) + event_properties["$exception_event_id"] = exception_id else: # Add usage usage = _parse_usage(output, run.provider, run.model) diff --git a/posthog/integrations/lang_chain.py b/posthog/integrations/lang_chain.py new file mode 100644 index 00000000..83063f87 --- /dev/null +++ b/posthog/integrations/lang_chain.py @@ -0,0 +1,89 @@ +try: + import langchain_core # noqa: F401 +except ImportError: + raise ModuleNotFoundError( + "Please install LangChain to use this feature: 'pip install langchain-core'" + ) + +from typing import Any, Optional +from uuid import UUID + +from posthog.client import Client + +try: + # LangChain 1.0+ and modern 0.x with langchain-core + from langchain_core.callbacks.base import BaseCallbackHandler +except (ImportError, ModuleNotFoundError): + # Fallback for older LangChain versions + from langchain.callbacks.base import BaseCallbackHandler + + +class PostHogCallback(BaseCallbackHandler): + raise_error: bool = True + + def __init__(self, client: Optional[Client] = None) -> None: + self.client = client + + def capture_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + **kwargs: Any, + ) -> None: + from posthog import capture_exception + + properties = { + "$langchain_run_id": str(run_id), + "$langchain_parent_run_id": str(parent_run_id) if parent_run_id else None, + "$langchain_tags": tags, + } + + capture_fn = self.client.capture_exception if self.client else capture_exception + capture_fn(error, properties=properties) + + def on_chain_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + **kwargs: Any, + ) -> None: + self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) + + def on_tool_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + **kwargs: Any, + ) -> None: + self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) + + def on_llm_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + **kwargs: Any, + ) -> None: + self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) + + def on_retriever_error( + self, + error: BaseException, + *, + run_id: UUID, + parent_run_id: Optional[UUID] = None, + tags: Optional[list[str]] = None, + **kwargs: Any, + ) -> None: + self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) From 2dba2a45fe2617cb9fd99e9f8842ca1f79f6d1e1 Mon Sep 17 00:00:00 2001 From: David Newell Date: Wed, 26 Nov 2025 15:56:07 +0000 Subject: [PATCH 02/16] capture all metadata in llm event --- posthog/ai/langchain/callbacks.py | 17 +++--- posthog/client.py | 32 +++++------ posthog/integrations/lang_chain.py | 89 ------------------------------ 3 files changed, 26 insertions(+), 112 deletions(-) delete mode 100644 posthog/integrations/lang_chain.py diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index b76d11ff..4adb80bc 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -576,6 +576,12 @@ def _capture_generation( if run.tools: event_properties["$ai_tools"] = run.tools + if self._properties: + event_properties.update(self._properties) + + if self._distinct_id is None: + event_properties["$process_person_profile"] = False + if isinstance(output, BaseException): event_properties["$ai_http_status"] = _get_http_status(output) event_properties["$ai_error"] = _stringify_exception(output) @@ -583,7 +589,10 @@ def _capture_generation( if self._ph_client.enable_exception_autocapture: exception_id = self._ph_client.capture_exception( - output, properties=event_properties + output, + distinct_id=self._distinct_id, + groups=self._groups, + properties=event_properties, ) event_properties["$exception_event_id"] = exception_id else: @@ -613,12 +622,6 @@ def _capture_generation( self._ph_client, self._privacy_mode, completions ) - if self._properties: - event_properties.update(self._properties) - - if self._distinct_id is None: - event_properties["$process_person_profile"] = False - self._ph_client.capture( distinct_id=self._distinct_id or trace_id, event="$ai_generation", diff --git a/posthog/client.py b/posthog/client.py index 3c4d8b89..bb11bf99 100644 --- a/posthog/client.py +++ b/posthog/client.py @@ -4,24 +4,33 @@ import sys from datetime import datetime, timedelta from typing import Any, Dict, Optional, Union -from typing_extensions import Unpack from uuid import uuid4 from dateutil.tz import tzutc from six import string_types +from typing_extensions import Unpack -from posthog.args import OptionalCaptureArgs, OptionalSetArgs, ID_TYPES, ExceptionArg +from posthog.args import ID_TYPES, ExceptionArg, OptionalCaptureArgs, OptionalSetArgs from posthog.consumer import Consumer +from posthog.contexts import ( + _get_current_context, + get_capture_exception_code_variables_context, + get_code_variables_ignore_patterns_context, + get_code_variables_mask_patterns_context, + get_context_distinct_id, + get_context_session_id, + new_context, +) from posthog.exception_capture import ExceptionCapture from posthog.exception_utils import ( + DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS, + DEFAULT_CODE_VARIABLES_MASK_PATTERNS, exc_info_from_error, + exception_is_already_captured, exceptions_from_error_tuple, handle_in_app, - exception_is_already_captured, mark_exception_as_captured, try_attach_code_variables_to_frames, - DEFAULT_CODE_VARIABLES_MASK_PATTERNS, - DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS, ) from posthog.feature_flags import ( InconclusiveMatchError, @@ -38,15 +47,6 @@ get, remote_config, ) -from posthog.contexts import ( - _get_current_context, - get_context_distinct_id, - get_context_session_id, - get_capture_exception_code_variables_context, - get_code_variables_mask_patterns_context, - get_code_variables_ignore_patterns_context, - new_context, -) from posthog.types import ( FeatureFlag, FeatureFlagResult, @@ -2016,9 +2016,9 @@ def _initialize_flag_cache(self, cache_url): return None try: - from urllib.parse import urlparse, parse_qs + from urllib.parse import parse_qs, urlparse except ImportError: - from urlparse import urlparse, parse_qs + from urlparse import parse_qs, urlparse try: parsed = urlparse(cache_url) diff --git a/posthog/integrations/lang_chain.py b/posthog/integrations/lang_chain.py deleted file mode 100644 index 83063f87..00000000 --- a/posthog/integrations/lang_chain.py +++ /dev/null @@ -1,89 +0,0 @@ -try: - import langchain_core # noqa: F401 -except ImportError: - raise ModuleNotFoundError( - "Please install LangChain to use this feature: 'pip install langchain-core'" - ) - -from typing import Any, Optional -from uuid import UUID - -from posthog.client import Client - -try: - # LangChain 1.0+ and modern 0.x with langchain-core - from langchain_core.callbacks.base import BaseCallbackHandler -except (ImportError, ModuleNotFoundError): - # Fallback for older LangChain versions - from langchain.callbacks.base import BaseCallbackHandler - - -class PostHogCallback(BaseCallbackHandler): - raise_error: bool = True - - def __init__(self, client: Optional[Client] = None) -> None: - self.client = client - - def capture_error( - self, - error: BaseException, - *, - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[list[str]] = None, - **kwargs: Any, - ) -> None: - from posthog import capture_exception - - properties = { - "$langchain_run_id": str(run_id), - "$langchain_parent_run_id": str(parent_run_id) if parent_run_id else None, - "$langchain_tags": tags, - } - - capture_fn = self.client.capture_exception if self.client else capture_exception - capture_fn(error, properties=properties) - - def on_chain_error( - self, - error: BaseException, - *, - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[list[str]] = None, - **kwargs: Any, - ) -> None: - self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) - - def on_tool_error( - self, - error: BaseException, - *, - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[list[str]] = None, - **kwargs: Any, - ) -> None: - self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) - - def on_llm_error( - self, - error: BaseException, - *, - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[list[str]] = None, - **kwargs: Any, - ) -> None: - self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) - - def on_retriever_error( - self, - error: BaseException, - *, - run_id: UUID, - parent_run_id: Optional[UUID] = None, - tags: Optional[list[str]] = None, - **kwargs: Any, - ) -> None: - self.capture_error(error, run_id=run_id, parent_run_id=parent_run_id, **kwargs) From 8f37893c2ac47b1c26c23c8326911bca72a18c12 Mon Sep 17 00:00:00 2001 From: David Newell Date: Wed, 26 Nov 2025 18:10:03 +0000 Subject: [PATCH 03/16] instrument with contexts --- posthog/ai/langchain/callbacks.py | 43 +++- posthog/ai/utils.py | 369 ++++++++++++++++-------------- 2 files changed, 230 insertions(+), 182 deletions(-) diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 4adb80bc..0bd7c5c3 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -506,6 +506,14 @@ def _capture_trace_or_span( if isinstance(outputs, BaseException): event_properties["$ai_error"] = _stringify_exception(outputs) event_properties["$ai_is_error"] = True + event_properties = _capture_exception_and_update_properties( + self._ph_client, + outputs, + self._distinct_id, + self._groups, + event_properties, + ) + elif outputs is not None: event_properties["$ai_output_state"] = with_privacy_mode( self._ph_client, self._privacy_mode, outputs @@ -587,14 +595,13 @@ def _capture_generation( event_properties["$ai_error"] = _stringify_exception(output) event_properties["$ai_is_error"] = True - if self._ph_client.enable_exception_autocapture: - exception_id = self._ph_client.capture_exception( - output, - distinct_id=self._distinct_id, - groups=self._groups, - properties=event_properties, - ) - event_properties["$exception_event_id"] = exception_id + event_properties = _capture_exception_and_update_properties( + self._ph_client, + output, + self._distinct_id, + self._groups, + event_properties, + ) else: # Add usage usage = _parse_usage(output, run.provider, run.model) @@ -870,6 +877,26 @@ def _parse_usage( return llm_usage +def _capture_exception_and_update_properties( + client: Client, + exception: BaseException, + distinct_id: Union[str, int, UUID], + groups: Optional[Dict[str, Any]], + event_properties: Dict[str, Any], +): + if client.enable_exception_autocapture: + exception_id = client.capture_exception( + exception, + distinct_id=distinct_id, + groups=groups, + properties=event_properties, + ) + + event_properties["$exception_event_id"] = exception_id + + return event_properties + + def _get_http_status(error: BaseException) -> int: # OpenAI: https://github.com/openai/openai-python/blob/main/src/openai/_exceptions.py # Anthropic: https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/_exceptions.py diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 559860cc..9056bda6 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -2,14 +2,15 @@ import uuid from typing import Any, Callable, Dict, List, Optional, cast -from posthog.client import Client as PostHogClient -from posthog.ai.types import FormattedMessage, StreamingEventData, TokenUsage +from posthog import identify_context, new_context, tag from posthog.ai.sanitization import ( - sanitize_openai, sanitize_anthropic, sanitize_gemini, sanitize_langchain, + sanitize_openai, ) +from posthog.ai.types import FormattedMessage, StreamingEventData, TokenUsage +from posthog.client import Client as PostHogClient def merge_usage_stats( @@ -256,94 +257,104 @@ def call_llm_and_track_usage( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - try: - response = call_method(**kwargs) - except Exception as exc: - error = exc - http_status = getattr( - exc, "status_code", 0 - ) # default to 0 becuase its likely an SDK error - error_params = { - "$ai_is_error": True, - "$ai_error": exc.__str__(), - } - finally: - end_time = time.time() - latency = end_time - start_time - - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - if response and ( - hasattr(response, "usage") - or (provider == "gemini" and hasattr(response, "usage_metadata")) - ): - usage = get_usage(response, provider) - - messages = merge_system_prompt(kwargs, provider) - sanitized_messages = sanitize_messages(messages, provider) - - event_properties = { - "$ai_provider": provider, - "$ai_model": kwargs.get("model"), - "$ai_model_parameters": get_model_params(kwargs), - "$ai_input": with_privacy_mode( - ph_client, posthog_privacy_mode, sanitized_messages - ), - "$ai_output_choices": with_privacy_mode( - ph_client, posthog_privacy_mode, format_response(response, provider) - ), - "$ai_http_status": http_status, - "$ai_input_tokens": usage.get("input_tokens", 0), - "$ai_output_tokens": usage.get("output_tokens", 0), - "$ai_latency": latency, - "$ai_trace_id": posthog_trace_id, - "$ai_base_url": str(base_url), - **(posthog_properties or {}), - **(error_params or {}), - } - - available_tool_calls = extract_available_tool_calls(provider, kwargs) - - if available_tool_calls: - event_properties["$ai_tools"] = available_tool_calls - - cache_read = usage.get("cache_read_input_tokens") - if cache_read is not None and cache_read > 0: - event_properties["$ai_cache_read_input_tokens"] = cache_read - - cache_creation = usage.get("cache_creation_input_tokens") - if cache_creation is not None and cache_creation > 0: - event_properties["$ai_cache_creation_input_tokens"] = cache_creation - - reasoning = usage.get("reasoning_tokens") - if reasoning is not None and reasoning > 0: - event_properties["$ai_reasoning_tokens"] = reasoning - - web_search_count = usage.get("web_search_count") - if web_search_count is not None and web_search_count > 0: - event_properties["$ai_web_search_count"] = web_search_count - - if posthog_distinct_id is None: - event_properties["$process_person_profile"] = False - - # Process instructions for Responses API - if provider == "openai" and kwargs.get("instructions") is not None: - event_properties["$ai_instructions"] = with_privacy_mode( - ph_client, posthog_privacy_mode, kwargs.get("instructions") + with new_context(client=ph_client): + if posthog_distinct_id: + identify_context(posthog_distinct_id) + + try: + response = call_method(**kwargs) + except Exception as exc: + error = exc + http_status = getattr( + exc, "status_code", 0 + ) # default to 0 becuase its likely an SDK error + error_params = { + "$ai_is_error": True, + "$ai_error": exc.__str__(), + } + finally: + end_time = time.time() + latency = end_time - start_time + + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + if response and ( + hasattr(response, "usage") + or (provider == "gemini" and hasattr(response, "usage_metadata")) + ): + usage = get_usage(response, provider) + + messages = merge_system_prompt(kwargs, provider) + sanitized_messages = sanitize_messages(messages, provider) + + tag("$ai_provider", provider) + tag("$ai_model", kwargs.get("model")) + tag("$ai_model_parameters", get_model_params(kwargs)) + tag( + "$ai_input", + with_privacy_mode(ph_client, posthog_privacy_mode, sanitized_messages), ) - - # send the event to posthog - if hasattr(ph_client, "capture") and callable(ph_client.capture): - ph_client.capture( - distinct_id=posthog_distinct_id or posthog_trace_id, - event="$ai_generation", - properties=event_properties, - groups=posthog_groups, + tag( + "$ai_output_choices", + with_privacy_mode( + ph_client, posthog_privacy_mode, format_response(response, provider) + ), ) + tag("$ai_http_status", http_status) + tag("$ai_input_tokens", usage.get("input_tokens", 0)) + tag("$ai_output_tokens", usage.get("output_tokens", 0)) + tag("$ai_latency", latency) + tag("$ai_trace_id", posthog_trace_id) + tag("$ai_base_url", str(base_url)) + + available_tool_calls = extract_available_tool_calls(provider, kwargs) + + if available_tool_calls: + tag("$ai_tools", available_tool_calls) + + cache_read = usage.get("cache_read_input_tokens") + if cache_read is not None and cache_read > 0: + tag("$ai_cache_read_input_tokens", cache_read) + + cache_creation = usage.get("cache_creation_input_tokens") + if cache_creation is not None and cache_creation > 0: + tag("$ai_cache_creation_input_tokens", cache_creation) + + reasoning = usage.get("reasoning_tokens") + if reasoning is not None and reasoning > 0: + tag("$ai_reasoning_tokens", reasoning) + + web_search_count = usage.get("web_search_count") + if web_search_count is not None and web_search_count > 0: + tag("$ai_web_search_count", web_search_count) + + if posthog_distinct_id is None: + tag("$process_person_profile", False) + + # Process instructions for Responses API + if provider == "openai" and kwargs.get("instructions") is not None: + tag( + "$ai_instructions", + with_privacy_mode( + ph_client, posthog_privacy_mode, kwargs.get("instructions") + ), + ) - if error: - raise error + # send the event to posthog + if hasattr(ph_client, "capture") and callable(ph_client.capture): + ph_client.capture( + distinct_id=posthog_distinct_id or posthog_trace_id, + event="$ai_generation", + properties={ + **(posthog_properties or {}), + **(error_params or {}), + }, + groups=posthog_groups, + ) + + if error: + raise error return response @@ -367,96 +378,106 @@ async def call_llm_and_track_usage_async( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - try: - response = await call_async_method(**kwargs) - except Exception as exc: - error = exc - http_status = getattr( - exc, "status_code", 0 - ) # default to 0 because its likely an SDK error - error_params = { - "$ai_is_error": True, - "$ai_error": exc.__str__(), - } - finally: - end_time = time.time() - latency = end_time - start_time - - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - if response and ( - hasattr(response, "usage") - or (provider == "gemini" and hasattr(response, "usage_metadata")) - ): - usage = get_usage(response, provider) - - messages = merge_system_prompt(kwargs, provider) - sanitized_messages = sanitize_messages(messages, provider) - - event_properties = { - "$ai_provider": provider, - "$ai_model": kwargs.get("model"), - "$ai_model_parameters": get_model_params(kwargs), - "$ai_input": with_privacy_mode( - ph_client, posthog_privacy_mode, sanitized_messages - ), - "$ai_output_choices": with_privacy_mode( - ph_client, posthog_privacy_mode, format_response(response, provider) - ), - "$ai_http_status": http_status, - "$ai_input_tokens": usage.get("input_tokens", 0), - "$ai_output_tokens": usage.get("output_tokens", 0), - "$ai_latency": latency, - "$ai_trace_id": posthog_trace_id, - "$ai_base_url": str(base_url), - **(posthog_properties or {}), - **(error_params or {}), - } - - available_tool_calls = extract_available_tool_calls(provider, kwargs) - - if available_tool_calls: - event_properties["$ai_tools"] = available_tool_calls - - cache_read = usage.get("cache_read_input_tokens") - if cache_read is not None and cache_read > 0: - event_properties["$ai_cache_read_input_tokens"] = cache_read - - cache_creation = usage.get("cache_creation_input_tokens") - if cache_creation is not None and cache_creation > 0: - event_properties["$ai_cache_creation_input_tokens"] = cache_creation - - reasoning = usage.get("reasoning_tokens") - if reasoning is not None and reasoning > 0: - event_properties["$ai_reasoning_tokens"] = reasoning - - web_search_count = usage.get("web_search_count") - if web_search_count is not None and web_search_count > 0: - event_properties["$ai_web_search_count"] = web_search_count - - if posthog_distinct_id is None: - event_properties["$process_person_profile"] = False - - # Process instructions for Responses API - if provider == "openai" and kwargs.get("instructions") is not None: - event_properties["$ai_instructions"] = with_privacy_mode( - ph_client, posthog_privacy_mode, kwargs.get("instructions") + with new_context(client=ph_client): + if posthog_distinct_id: + identify_context(posthog_distinct_id) + + try: + response = await call_async_method(**kwargs) + except Exception as exc: + error = exc + http_status = getattr( + exc, "status_code", 0 + ) # default to 0 because its likely an SDK error + error_params = { + "$ai_is_error": True, + "$ai_error": exc.__str__(), + } + finally: + end_time = time.time() + latency = end_time - start_time + + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + if response and ( + hasattr(response, "usage") + or (provider == "gemini" and hasattr(response, "usage_metadata")) + ): + usage = get_usage(response, provider) + + messages = merge_system_prompt(kwargs, provider) + sanitized_messages = sanitize_messages(messages, provider) + + tag("$ai_provider", provider) + tag("$ai_model", kwargs.get("model")) + tag("$ai_model_parameters", get_model_params(kwargs)) + tag( + "$ai_input", + with_privacy_mode(ph_client, posthog_privacy_mode, sanitized_messages), ) - - # send the event to posthog - if hasattr(ph_client, "capture") and callable(ph_client.capture): - ph_client.capture( - distinct_id=posthog_distinct_id or posthog_trace_id, - event="$ai_generation", - properties=event_properties, - groups=posthog_groups, + tag( + "$ai_output_choices", + with_privacy_mode( + ph_client, posthog_privacy_mode, format_response(response, provider) + ), ) + tag("$ai_http_status", http_status) + tag("$ai_input_tokens", usage.get("input_tokens", 0)) + tag("$ai_output_tokens", usage.get("output_tokens", 0)) + tag("$ai_latency", latency) + tag("$ai_trace_id", posthog_trace_id) + tag("$ai_base_url", str(base_url)) + + available_tool_calls = extract_available_tool_calls(provider, kwargs) + + if available_tool_calls: + tag("$ai_tools", available_tool_calls) + + cache_read = usage.get("cache_read_input_tokens") + if cache_read is not None and cache_read > 0: + tag("$ai_cache_read_input_tokens", cache_read) + + cache_creation = usage.get("cache_creation_input_tokens") + if cache_creation is not None and cache_creation > 0: + tag("$ai_cache_creation_input_tokens", cache_creation) + + reasoning = usage.get("reasoning_tokens") + if reasoning is not None and reasoning > 0: + tag("$ai_reasoning_tokens", reasoning) + + web_search_count = usage.get("web_search_count") + if web_search_count is not None and web_search_count > 0: + tag("$ai_web_search_count", web_search_count) + + if posthog_distinct_id is None: + tag("$process_person_profile", False) + + # Process instructions for Responses API + if provider == "openai" and kwargs.get("instructions") is not None: + tag( + "$ai_instructions", + with_privacy_mode( + ph_client, posthog_privacy_mode, kwargs.get("instructions") + ), + ) - if error: - raise error + # send the event to posthog + if hasattr(ph_client, "capture") and callable(ph_client.capture): + ph_client.capture( + distinct_id=posthog_distinct_id or posthog_trace_id, + event="$ai_generation", + properties={ + **(posthog_properties or {}), + **(error_params or {}), + }, + groups=posthog_groups, + ) - return response + if error: + raise error + + return response def sanitize_messages(data: Any, provider: str) -> Any: From 26ed6ab2e201fee86f643b3ba7eae0c70a4c10a5 Mon Sep 17 00:00:00 2001 From: David Newell Date: Fri, 28 Nov 2025 12:04:36 +0000 Subject: [PATCH 04/16] bump version --- CHANGELOG.md | 5 +++++ posthog/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f00a91e..799db2a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# 7.1.0 - 2025-11-28 + +Capture Langchain, OpenAI and Anthropic errors as exceptions (if exception autocapture is enabled) +Add reference to exception in LLMA trace and span events + # 7.0.2 - 2025-11-18 Add support for Python 3.14. diff --git a/posthog/version.py b/posthog/version.py index 639be2bc..75b03c92 100644 --- a/posthog/version.py +++ b/posthog/version.py @@ -1,4 +1,4 @@ -VERSION = "7.0.1" +VERSION = "7.1.0" if __name__ == "__main__": print(VERSION, end="") # noqa: T201 From 29ce5575d833aa1b7723c02a7f6e25e7f80a1d6b Mon Sep 17 00:00:00 2001 From: David Newell Date: Fri, 28 Nov 2025 12:11:24 +0000 Subject: [PATCH 05/16] indentation --- posthog/ai/langchain/callbacks.py | 3 ++- posthog/ai/utils.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0bd7c5c3..9aff4ea0 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -892,7 +892,8 @@ def _capture_exception_and_update_properties( properties=event_properties, ) - event_properties["$exception_event_id"] = exception_id + if exception_id: + event_properties["$exception_event_id"] = exception_id return event_properties diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 9056bda6..144e0fc8 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -353,8 +353,8 @@ def call_llm_and_track_usage( groups=posthog_groups, ) - if error: - raise error + if error: + raise error return response @@ -474,10 +474,10 @@ async def call_llm_and_track_usage_async( groups=posthog_groups, ) - if error: - raise error + if error: + raise error - return response + return response def sanitize_messages(data: Any, provider: str) -> Any: From 99c8f60bcf3a631cfbcb325d97219e4311b50189 Mon Sep 17 00:00:00 2001 From: David Newell Date: Fri, 28 Nov 2025 13:49:11 +0000 Subject: [PATCH 06/16] linting --- posthog/ai/langchain/callbacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 9aff4ea0..db493405 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -880,7 +880,7 @@ def _parse_usage( def _capture_exception_and_update_properties( client: Client, exception: BaseException, - distinct_id: Union[str, int, UUID], + distinct_id: Optional[Union[str, int, UUID]], groups: Optional[Dict[str, Any]], event_properties: Dict[str, Any], ): From 10daf14945fcde4c7411f8134c34e0c0c9d87435 Mon Sep 17 00:00:00 2001 From: David Newell Date: Fri, 28 Nov 2025 14:47:05 +0000 Subject: [PATCH 07/16] tests --- posthog/test/ai/test_system_prompts.py | 51 +++++++++++++++----------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/posthog/test/ai/test_system_prompts.py b/posthog/test/ai/test_system_prompts.py index 2f37ccc4..eb049c86 100644 --- a/posthog/test/ai/test_system_prompts.py +++ b/posthog/test/ai/test_system_prompts.py @@ -11,7 +11,10 @@ import time import unittest -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + +from posthog.client import Client +from posthog.test.test_utils import FAKE_TEST_API_KEY class TestSystemPromptCapture(unittest.TestCase): @@ -24,7 +27,8 @@ def setUp(self): self.test_response = "I'm doing well, thank you!" # Create mock PostHog client - self.client = MagicMock() + self.client = Client(FAKE_TEST_API_KEY) + self.client._enqueue = MagicMock() self.client.privacy_mode = False def _assert_system_prompt_captured(self, captured_input): @@ -53,10 +57,11 @@ def _assert_system_prompt_captured(self, captured_input): def test_openai_messages_array_system_prompt(self): """Test OpenAI with system prompt in messages array.""" try: - from posthog.ai.openai import OpenAI from openai.types.chat import ChatCompletion, ChatCompletionMessage from openai.types.chat.chat_completion import Choice from openai.types.completion_usage import CompletionUsage + + from posthog.ai.openai import OpenAI except ImportError: self.skipTest("OpenAI package not available") @@ -94,17 +99,18 @@ def test_openai_messages_array_system_prompt(self): model="gpt-4", messages=messages, posthog_distinct_id="test-user" ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) def test_openai_separate_system_parameter(self): """Test OpenAI with system prompt as separate parameter.""" try: - from posthog.ai.openai import OpenAI from openai.types.chat import ChatCompletion, ChatCompletionMessage from openai.types.chat.chat_completion import Choice from openai.types.completion_usage import CompletionUsage + + from posthog.ai.openai import OpenAI except ImportError: self.skipTest("OpenAI package not available") @@ -142,18 +148,21 @@ def test_openai_separate_system_parameter(self): posthog_distinct_id="test-user", ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) def test_openai_streaming_system_parameter(self): """Test OpenAI streaming with system parameter.""" try: - from posthog.ai.openai import OpenAI - from openai.types.chat.chat_completion_chunk import ChatCompletionChunk + from openai.types.chat.chat_completion_chunk import ( + ChatCompletionChunk, + ChoiceDelta, + ) from openai.types.chat.chat_completion_chunk import Choice as ChoiceChunk - from openai.types.chat.chat_completion_chunk import ChoiceDelta from openai.types.completion_usage import CompletionUsage + + from posthog.ai.openai import OpenAI except ImportError: self.skipTest("OpenAI package not available") @@ -206,8 +215,8 @@ def test_openai_streaming_system_parameter(self): list(response_generator) # Consume generator - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) # Anthropic Tests @@ -239,8 +248,8 @@ def test_anthropic_messages_array_system_prompt(self): posthog_distinct_id="test-user", ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) def test_anthropic_separate_system_parameter(self): @@ -269,8 +278,8 @@ def test_anthropic_separate_system_parameter(self): posthog_distinct_id="test-user", ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) # Gemini Tests @@ -310,8 +319,8 @@ def test_gemini_contents_array_system_prompt(self): posthog_distinct_id="test-user", ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) def test_gemini_system_instruction_parameter(self): @@ -349,6 +358,6 @@ def test_gemini_system_instruction_parameter(self): posthog_distinct_id="test-user", ) - self.assertEqual(len(self.client.capture.call_args_list), 1) - properties = self.client.capture.call_args_list[0][1]["properties"] + self.assertEqual(len(self.client._enqueue.call_args_list), 1) + properties = self.client._enqueue.call_args_list[0][0][0]["properties"] self._assert_system_prompt_captured(properties["$ai_input"]) From 6231bab1eacb5d7ca90d460cf05447f751a7581d Mon Sep 17 00:00:00 2001 From: David Newell Date: Fri, 28 Nov 2025 14:48:21 +0000 Subject: [PATCH 08/16] raise --- posthog/ai/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 144e0fc8..a7a8bdb6 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -353,8 +353,8 @@ def call_llm_and_track_usage( groups=posthog_groups, ) - if error: - raise error + if error: + raise error return response @@ -474,8 +474,8 @@ async def call_llm_and_track_usage_async( groups=posthog_groups, ) - if error: - raise error + if error: + raise error return response From 53356e42ece9de6229d37b71bd8e276ca0cab306 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Tue, 6 Jan 2026 14:16:10 +0000 Subject: [PATCH 09/16] test: add exception capture integration tests for langchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add 6 tests covering the new LLMA + error tracking integration: - capture_exception called on span/generation errors - $exception_event_id added to AI events - No capture when autocapture disabled - AI properties passed to exception event - Handles None return from capture_exception 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/test/ai/langchain/test_callbacks.py | 197 ++++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 1f7edba7..430cab68 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2441,3 +2441,200 @@ def test_billable_with_real_chain(mock_client): assert props["$ai_billable"] is True assert props["$ai_model"] == "fake-model" assert props["$ai_provider"] == "fake" + + +# Exception Capture Integration Tests + + +def test_exception_autocapture_on_span_error(): + """Test that capture_exception is called when a span errors and autocapture is enabled.""" + mock_client = MagicMock() + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = True + mock_client.capture_exception.return_value = "exception-uuid-123" + + def failing_span(_): + raise ValueError("test error") + + callbacks = [CallbackHandler(mock_client)] + chain = RunnableLambda(failing_span) + + try: + chain.invoke({}, config={"callbacks": callbacks}) + except ValueError: + pass + + # Verify capture_exception was called + assert mock_client.capture_exception.call_count == 1 + exception_call = mock_client.capture_exception.call_args + assert isinstance(exception_call[0][0], ValueError) + assert str(exception_call[0][0]) == "test error" + + +def test_exception_autocapture_adds_exception_id_to_span_event(): + """Test that $exception_event_id is added to the span event properties.""" + mock_client = MagicMock() + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = True + mock_client.capture_exception.return_value = "exception-uuid-456" + + def failing_span(_): + raise ValueError("test error") + + callbacks = [CallbackHandler(mock_client)] + chain = RunnableLambda(failing_span) + + try: + chain.invoke({}, config={"callbacks": callbacks}) + except ValueError: + pass + + # Find the span event (should have $ai_is_error=True) + span_calls = [ + call for call in mock_client.capture.call_args_list + if call[1].get("properties", {}).get("$ai_is_error") is True + ] + assert len(span_calls) >= 1 + + span_props = span_calls[0][1]["properties"] + assert span_props["$exception_event_id"] == "exception-uuid-456" + assert span_props["$ai_error"] == "ValueError: test error" + + +def test_exception_autocapture_disabled_does_not_capture(): + """Test that capture_exception is NOT called when autocapture is disabled.""" + mock_client = MagicMock() + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = False + + def failing_span(_): + raise ValueError("test error") + + callbacks = [CallbackHandler(mock_client)] + chain = RunnableLambda(failing_span) + + try: + chain.invoke({}, config={"callbacks": callbacks}) + except ValueError: + pass + + # Verify capture_exception was NOT called + assert mock_client.capture_exception.call_count == 0 + + # But the span event should still have error info + span_calls = [ + call for call in mock_client.capture.call_args_list + if call[1].get("properties", {}).get("$ai_is_error") is True + ] + assert len(span_calls) >= 1 + + span_props = span_calls[0][1]["properties"] + assert "$exception_event_id" not in span_props + assert span_props["$ai_error"] == "ValueError: test error" + + +def test_exception_autocapture_on_llm_generation_error(mock_client): + """Test that capture_exception is called when an LLM generation fails.""" + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = True + mock_client.capture_exception.return_value = "exception-uuid-789" + + callbacks = CallbackHandler(mock_client) + run_id = uuid.uuid4() + + # Simulate LLM start + callbacks.on_llm_start( + serialized={"kwargs": {"openai_api_base": "https://api.openai.com"}}, + prompts=["Hello"], + run_id=run_id, + ) + + # Simulate LLM error + error = Exception("API rate limit exceeded") + callbacks.on_llm_error(error, run_id=run_id) + + # Verify capture_exception was called + assert mock_client.capture_exception.call_count == 1 + exception_call = mock_client.capture_exception.call_args + assert exception_call[0][0] is error + + # Verify the generation event has $exception_event_id + generation_calls = [ + call for call in mock_client.capture.call_args_list + if call[1].get("event") == "$ai_generation" + ] + assert len(generation_calls) == 1 + + gen_props = generation_calls[0][1]["properties"] + assert gen_props["$exception_event_id"] == "exception-uuid-789" + assert gen_props["$ai_is_error"] is True + + +def test_exception_autocapture_passes_ai_properties_to_exception(): + """Test that AI properties are passed to the exception event.""" + mock_client = MagicMock() + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = True + mock_client.capture_exception.return_value = "exception-uuid-abc" + + callbacks = CallbackHandler( + mock_client, + distinct_id="user-123", + properties={"custom_prop": "custom_value"}, + ) + run_id = uuid.uuid4() + + # Simulate LLM start + callbacks.on_llm_start( + serialized={"kwargs": {"openai_api_base": "https://api.openai.com"}}, + prompts=["Hello"], + run_id=run_id, + ) + + # Simulate LLM error + error = Exception("API error") + callbacks.on_llm_error(error, run_id=run_id) + + # Verify capture_exception received the properties + exception_call = mock_client.capture_exception.call_args + props = exception_call[1]["properties"] + + # Should have AI-related properties + assert "$ai_trace_id" in props + assert "$ai_is_error" in props + assert props["$ai_is_error"] is True + + # Should have distinct_id passed through + assert exception_call[1]["distinct_id"] == "user-123" + + +def test_exception_autocapture_none_return_no_exception_id(): + """Test that when capture_exception returns None, no $exception_event_id is added.""" + mock_client = MagicMock() + mock_client.privacy_mode = False + mock_client.enable_exception_autocapture = True + mock_client.capture_exception.return_value = None # e.g., exception already captured + + def failing_span(_): + raise ValueError("test error") + + callbacks = [CallbackHandler(mock_client)] + chain = RunnableLambda(failing_span) + + try: + chain.invoke({}, config={"callbacks": callbacks}) + except ValueError: + pass + + # capture_exception was called but returned None + assert mock_client.capture_exception.call_count == 1 + + # Span event should NOT have $exception_event_id + span_calls = [ + call for call in mock_client.capture.call_args_list + if call[1].get("properties", {}).get("$ai_is_error") is True + ] + assert len(span_calls) >= 1 + + span_props = span_calls[0][1]["properties"] + assert "$exception_event_id" not in span_props From 9a103e6e484f28e61e7dcee7600d93f956314dcc Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:20:51 +0000 Subject: [PATCH 10/16] fix: pass context tags to capture() for test compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Export get_tags() from posthog module - Explicitly pass context tags to capture() in AI utils - Fix $ai_model fallback to extract from response.model - Fix ruff formatting in langchain test_callbacks.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/__init__.py | 16 ++++++++++++++++ posthog/ai/utils.py | 8 +++++--- posthog/test/ai/langchain/test_callbacks.py | 16 +++++++++++----- 3 files changed, 32 insertions(+), 8 deletions(-) diff --git a/posthog/__init__.py b/posthog/__init__.py index f2ca6ecb..8dbed815 100644 --- a/posthog/__init__.py +++ b/posthog/__init__.py @@ -29,6 +29,9 @@ from posthog.contexts import ( tag as inner_tag, ) +from posthog.contexts import ( + get_tags as inner_get_tags, +) from posthog.exception_utils import ( DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS, DEFAULT_CODE_VARIABLES_MASK_PATTERNS, @@ -190,6 +193,19 @@ def tag(name: str, value: Any): return inner_tag(name, value) +def get_tags() -> Dict[str, Any]: + """ + Get all tags from the current context. + + Returns: + Dict of all tags in the current context + + Category: + Contexts + """ + return inner_get_tags() + + """Settings.""" api_key = None # type: Optional[str] host = None # type: Optional[str] diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index a7a8bdb6..d70c65c4 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -2,7 +2,7 @@ import uuid from typing import Any, Callable, Dict, List, Optional, cast -from posthog import identify_context, new_context, tag +from posthog import get_tags, identify_context, new_context, tag from posthog.ai.sanitization import ( sanitize_anthropic, sanitize_gemini, @@ -289,7 +289,7 @@ def call_llm_and_track_usage( sanitized_messages = sanitize_messages(messages, provider) tag("$ai_provider", provider) - tag("$ai_model", kwargs.get("model")) + tag("$ai_model", kwargs.get("model") or getattr(response, "model", None)) tag("$ai_model_parameters", get_model_params(kwargs)) tag( "$ai_input", @@ -347,6 +347,7 @@ def call_llm_and_track_usage( distinct_id=posthog_distinct_id or posthog_trace_id, event="$ai_generation", properties={ + **get_tags(), **(posthog_properties or {}), **(error_params or {}), }, @@ -410,7 +411,7 @@ async def call_llm_and_track_usage_async( sanitized_messages = sanitize_messages(messages, provider) tag("$ai_provider", provider) - tag("$ai_model", kwargs.get("model")) + tag("$ai_model", kwargs.get("model") or getattr(response, "model", None)) tag("$ai_model_parameters", get_model_params(kwargs)) tag( "$ai_input", @@ -468,6 +469,7 @@ async def call_llm_and_track_usage_async( distinct_id=posthog_distinct_id or posthog_trace_id, event="$ai_generation", properties={ + **get_tags(), **(posthog_properties or {}), **(error_params or {}), }, diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 430cab68..a96e71ee 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2491,7 +2491,8 @@ def failing_span(_): # Find the span event (should have $ai_is_error=True) span_calls = [ - call for call in mock_client.capture.call_args_list + call + for call in mock_client.capture.call_args_list if call[1].get("properties", {}).get("$ai_is_error") is True ] assert len(span_calls) >= 1 @@ -2523,7 +2524,8 @@ def failing_span(_): # But the span event should still have error info span_calls = [ - call for call in mock_client.capture.call_args_list + call + for call in mock_client.capture.call_args_list if call[1].get("properties", {}).get("$ai_is_error") is True ] assert len(span_calls) >= 1 @@ -2560,7 +2562,8 @@ def test_exception_autocapture_on_llm_generation_error(mock_client): # Verify the generation event has $exception_event_id generation_calls = [ - call for call in mock_client.capture.call_args_list + call + for call in mock_client.capture.call_args_list if call[1].get("event") == "$ai_generation" ] assert len(generation_calls) == 1 @@ -2613,7 +2616,9 @@ def test_exception_autocapture_none_return_no_exception_id(): mock_client = MagicMock() mock_client.privacy_mode = False mock_client.enable_exception_autocapture = True - mock_client.capture_exception.return_value = None # e.g., exception already captured + mock_client.capture_exception.return_value = ( + None # e.g., exception already captured + ) def failing_span(_): raise ValueError("test error") @@ -2631,7 +2636,8 @@ def failing_span(_): # Span event should NOT have $exception_event_id span_calls = [ - call for call in mock_client.capture.call_args_list + call + for call in mock_client.capture.call_args_list if call[1].get("properties", {}).get("$ai_is_error") is True ] assert len(span_calls) >= 1 From dc231438362d84699899dbcaab90d1d52018731f Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:31:37 +0000 Subject: [PATCH 11/16] fix: disable auto-capture exceptions in LLM context MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new_context() defaults to capture_exceptions=True which would auto-capture any exception regardless of enable_exception_autocapture setting. This was inconsistent with LangChain callbacks which explicitly check the setting. Pass capture_exceptions=False to let exception handling be controlled explicitly by the enable_exception_autocapture setting. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/ai/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index d70c65c4..2da9700f 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -257,7 +257,7 @@ def call_llm_and_track_usage( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client): + with new_context(client=ph_client, capture_exceptions=False): if posthog_distinct_id: identify_context(posthog_distinct_id) @@ -379,7 +379,7 @@ async def call_llm_and_track_usage_async( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client): + with new_context(client=ph_client, capture_exceptions=False): if posthog_distinct_id: identify_context(posthog_distinct_id) From 9ed2220c0e5b9d562bfe3c2023afcfb7c99eec26 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:32:16 +0000 Subject: [PATCH 12/16] fix: isolate LLM context with fresh=True to avoid tag inheritance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use fresh=True to start with a clean context for each LLM call. This avoids inheriting $ai_* tags from parent contexts which could cause mismatched AI metadata due to the tag merge order bug in contexts.py (parent tags incorrectly override child tags). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/ai/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 2da9700f..fb669def 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -257,7 +257,7 @@ def call_llm_and_track_usage( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client, capture_exceptions=False): + with new_context(client=ph_client, capture_exceptions=False, fresh=True): if posthog_distinct_id: identify_context(posthog_distinct_id) @@ -379,7 +379,7 @@ async def call_llm_and_track_usage_async( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client, capture_exceptions=False): + with new_context(client=ph_client, capture_exceptions=False, fresh=True): if posthog_distinct_id: identify_context(posthog_distinct_id) From 582ccd74502f27b2ddda10fb82ea85510516f689 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:39:41 +0000 Subject: [PATCH 13/16] fix: correct tag merge order so child tags take precedence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The collect_tags() method had a bug where parent tags would overwrite child tags, despite the comment saying the opposite. This fix ensures child context tags properly override parent tags. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/contexts.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/posthog/contexts.py b/posthog/contexts.py index 1051b1e1..39f9bfde 100644 --- a/posthog/contexts.py +++ b/posthog/contexts.py @@ -62,14 +62,13 @@ def get_distinct_id(self) -> Optional[str]: return None def collect_tags(self) -> Dict[str, Any]: - tags = self.tags.copy() if self.parent and not self.fresh: # We want child tags to take precedence over parent tags, - # so we can't use a simple update here, instead collecting - # the parent tags and then updating with the child tags. - new_tags = self.parent.collect_tags() - tags.update(new_tags) - return tags + # so collect parent tags first, then update with child tags. + tags = self.parent.collect_tags() + tags.update(self.tags) + return tags + return self.tags.copy() def get_capture_exception_code_variables(self) -> Optional[bool]: if self.capture_exception_code_variables is not None: From 49ad9ea76bcc3e1a5eebf77933fdb70e015aedee Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:43:23 +0000 Subject: [PATCH 14/16] refactor: remove fresh=True now that tag merge order is fixed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the collect_tags() bug fixed, child tags properly override parent tags. LLM events can now inherit useful parent context tags (request_id, user info, etc.) while still having their $ai_* tags take precedence. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/ai/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index fb669def..2da9700f 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -257,7 +257,7 @@ def call_llm_and_track_usage( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client, capture_exceptions=False, fresh=True): + with new_context(client=ph_client, capture_exceptions=False): if posthog_distinct_id: identify_context(posthog_distinct_id) @@ -379,7 +379,7 @@ async def call_llm_and_track_usage_async( usage: TokenUsage = TokenUsage() error_params: Dict[str, Any] = {} - with new_context(client=ph_client, capture_exceptions=False, fresh=True): + with new_context(client=ph_client, capture_exceptions=False): if posthog_distinct_id: identify_context(posthog_distinct_id) From 36f306b6bb4ff9a245217af932cbf6d4c1ffbd16 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:46:30 +0000 Subject: [PATCH 15/16] test: add test for child tags overriding parent tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verifies that in non-fresh contexts, child tags properly override parent tags with the same key while still inheriting other parent tags. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/test/test_contexts.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/posthog/test/test_contexts.py b/posthog/test/test_contexts.py index 113e86c5..3ff6fddd 100644 --- a/posthog/test/test_contexts.py +++ b/posthog/test/test_contexts.py @@ -191,6 +191,32 @@ def test_context_inheritance_non_fresh_context(self): assert get_context_distinct_id() == "user123" assert get_context_session_id() == "session456" + def test_child_tags_override_parent_tags_in_non_fresh_context(self): + with new_context(fresh=True): + tag("shared_key", "parent_value") + tag("parent_only", "parent") + + with new_context(fresh=False): + # Child should inherit parent tags + assert get_tags()["parent_only"] == "parent" + + # Child sets same key - should override parent + tag("shared_key", "child_value") + tag("child_only", "child") + + tags = get_tags() + # Child value should win for shared key + assert tags["shared_key"] == "child_value" + # Both parent and child tags should be present + assert tags["parent_only"] == "parent" + assert tags["child_only"] == "child" + + # Parent context should be unchanged + parent_tags = get_tags() + assert parent_tags["shared_key"] == "parent_value" + assert parent_tags["parent_only"] == "parent" + assert "child_only" not in parent_tags + def test_scoped_decorator_with_context_ids(self): @scoped() def function_with_context(): From f3f8587e14744c4cab372b051a466328f7120571 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 7 Jan 2026 12:54:56 +0000 Subject: [PATCH 16/16] chore: add TODO for OpenAI/Anthropic/Gemini exception capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document that exception capture needs to be added for the direct SDK wrappers, similar to how it's implemented in LangChain callbacks. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- posthog/ai/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 2da9700f..de110146 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -272,6 +272,9 @@ def call_llm_and_track_usage( "$ai_is_error": True, "$ai_error": exc.__str__(), } + # TODO: Add exception capture for OpenAI/Anthropic/Gemini wrappers when + # enable_exception_autocapture is True, similar to LangChain callbacks. + # See _capture_exception_and_update_properties in langchain/callbacks.py finally: end_time = time.time() latency = end_time - start_time @@ -394,6 +397,9 @@ async def call_llm_and_track_usage_async( "$ai_is_error": True, "$ai_error": exc.__str__(), } + # TODO: Add exception capture for OpenAI/Anthropic/Gemini wrappers when + # enable_exception_autocapture is True, similar to LangChain callbacks. + # See _capture_exception_and_update_properties in langchain/callbacks.py finally: end_time = time.time() latency = end_time - start_time