From 3af161a53123888b9c304e4320b27be71dc1b579 Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Tue, 12 Aug 2025 14:51:02 +0800 Subject: [PATCH 1/2] feat: add langsmith and langfuse integration --- python/poml/api.py | 89 ++++++++++++++++++++++++---- python/poml/integration/langfuse.py | 16 +++++ python/poml/integration/langsmith.py | 16 +++++ 3 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 python/poml/integration/langfuse.py create mode 100644 python/poml/integration/langsmith.py diff --git a/python/poml/api.py b/python/poml/api.py index 89a63e49..43ff5e16 100644 --- a/python/poml/api.py +++ b/python/poml/api.py @@ -25,15 +25,20 @@ _weave_enabled: bool = False _agentops_enabled: bool = False _mlflow_enabled: bool = False +_langsmith_enabled: bool = False +_langfuse_enabled: bool = False _trace_log: List[Dict[str, Any]] = [] _trace_dir: Optional[Path] = None -Backend = Literal["local", "weave", "agentops", "mlflow"] +Backend = Literal["local", "weave", "agentops", "mlflow", "langsmith", "langfuse"] OutputFormat = Literal["raw", "dict", "openai_chat", "langchain", "pydantic"] def set_trace( - enabled: bool | List[Backend] | Backend = True, /, *, trace_dir: Optional[str | Path] = None + enabled: bool | List[Backend] | Backend = True, + /, + *, + trace_dir: Optional[str | Path] = None, ) -> Optional[Path]: """Enable or disable tracing of ``poml`` calls with optional backend integrations. @@ -41,7 +46,7 @@ def set_trace( enabled: Controls which tracing backends to enable. Can be: - True: Enable local tracing only (equivalent to ["local"]) - False: Disable all tracing (equivalent to []) - - str: Enable a single backend ("local", "weave", "agentops", "mlflow") + - str: Enable a single backend ("local", "weave", "agentops", "mlflow", "langsmith", "langfuse") - List[str]: Enable multiple backends. "local" is auto-enabled if any backends are specified. trace_dir: Optional directory for local trace files. If provided when local tracing is enabled, a subdirectory named by the current timestamp @@ -57,6 +62,8 @@ def set_trace( - "weave": Log to Weights & Biases Weave (requires local tracing) - "agentops": Log to AgentOps (requires local tracing) - "mlflow": Log to MLflow (requires local tracing) + - "langsmith": Log to LangSmith (requires local tracing) + - "langfuse": Log to Langfuse (requires local tracing) """ if enabled is True: @@ -67,7 +74,7 @@ def set_trace( if isinstance(enabled, str): enabled = [enabled] - global _trace_enabled, _trace_dir, _weave_enabled, _agentops_enabled, _mlflow_enabled + global _trace_enabled, _trace_dir, _weave_enabled, _agentops_enabled, _mlflow_enabled, _langsmith_enabled, _langfuse_enabled if enabled or "local" in enabled: # When enabled is non-empty, we always enable local tracing. _trace_enabled = True @@ -104,6 +111,16 @@ def set_trace( else: _mlflow_enabled = False + if "langsmith" in enabled: + _langsmith_enabled = True + else: + _langsmith_enabled = False + + if "langfuse" in enabled: + _langfuse_enabled = True + else: + _langfuse_enabled = False + return _trace_dir @@ -225,7 +242,9 @@ def _poml_response_to_openai_chat(messages: List[PomlMessage]) -> List[Dict[str, contents.append( { "type": "image_url", - "image_url": {"url": f"data:{content_part.type};base64,{content_part.base64}"}, + "image_url": { + "url": f"data:{content_part.type};base64,{content_part.base64}" + }, } ) else: @@ -242,7 +261,9 @@ def _poml_response_to_langchain(messages: List[PomlMessage]) -> List[Dict[str, A langchain_messages = [] for msg in messages: if isinstance(msg.content, str): - langchain_messages.append({"type": msg.speaker, "data": {"content": msg.content}}) + langchain_messages.append( + {"type": msg.speaker, "data": {"content": msg.content}} + ) elif isinstance(msg.content, list): content_parts = [] for content_part in msg.content: @@ -259,7 +280,9 @@ def _poml_response_to_langchain(messages: List[PomlMessage]) -> List[Dict[str, A ) else: raise ValueError(f"Unexpected content part: {content_part}") - langchain_messages.append({"type": msg.speaker, "data": {"content": content_parts}}) + langchain_messages.append( + {"type": msg.speaker, "data": {"content": content_parts}} + ) else: raise ValueError(f"Unexpected content type: {type(msg.content)}") return langchain_messages @@ -458,7 +481,9 @@ def poml( trace_prefix = _latest_trace_prefix() current_version = _current_trace_version() if trace_prefix is None or current_version is None: - raise RuntimeError("Weave tracing requires local tracing to be enabled.") + raise RuntimeError( + "Weave tracing requires local tracing to be enabled." + ) poml_content = _read_latest_traced_file(".poml") context_content = _read_latest_traced_file(".context.json") stylesheet_content = _read_latest_traced_file(".stylesheet.json") @@ -477,7 +502,9 @@ def poml( trace_prefix = _latest_trace_prefix() current_version = _current_trace_version() if trace_prefix is None or current_version is None: - raise RuntimeError("AgentOps tracing requires local tracing to be enabled.") + raise RuntimeError( + "AgentOps tracing requires local tracing to be enabled." + ) poml_content = _read_latest_traced_file(".poml") context_content = _read_latest_traced_file(".context.json") stylesheet_content = _read_latest_traced_file(".stylesheet.json") @@ -495,7 +522,9 @@ def poml( trace_prefix = _latest_trace_prefix() current_version = _current_trace_version() if trace_prefix is None or current_version is None: - raise RuntimeError("MLflow tracing requires local tracing to be enabled.") + raise RuntimeError( + "MLflow tracing requires local tracing to be enabled." + ) poml_content = _read_latest_traced_file(".poml") context_content = _read_latest_traced_file(".context.json") stylesheet_content = _read_latest_traced_file(".stylesheet.json") @@ -507,6 +536,46 @@ def poml( result, ) + if _langsmith_enabled: + from .integration import langsmith + + trace_prefix = _latest_trace_prefix() + current_version = _current_trace_version() + if trace_prefix is None or current_version is None: + raise RuntimeError( + "LangSmith tracing requires local tracing to be enabled." + ) + poml_content = _read_latest_traced_file(".poml") + context_content = _read_latest_traced_file(".context.json") + stylesheet_content = _read_latest_traced_file(".stylesheet.json") + langsmith.log_poml_call( + trace_prefix.name, + poml_content or str(markup), + json.loads(context_content) if context_content else None, + json.loads(stylesheet_content) if stylesheet_content else None, + result, + ) + + if _langfuse_enabled: + from .integration import langfuse + + trace_prefix = _latest_trace_prefix() + current_version = _current_trace_version() + if trace_prefix is None or current_version is None: + raise RuntimeError( + "Langfuse tracing requires local tracing to be enabled." + ) + poml_content = _read_latest_traced_file(".poml") + context_content = _read_latest_traced_file(".context.json") + stylesheet_content = _read_latest_traced_file(".stylesheet.json") + langfuse.log_poml_call( + trace_prefix.name, + poml_content or str(markup), + json.loads(context_content) if context_content else None, + json.loads(stylesheet_content) if stylesheet_content else None, + result, + ) + if trace_record is not None: trace_record["result"] = result return result diff --git a/python/poml/integration/langfuse.py b/python/poml/integration/langfuse.py new file mode 100644 index 00000000..64295b04 --- /dev/null +++ b/python/poml/integration/langfuse.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import Any +from langfuse.decorators import observe + + +def log_poml_call( + name: str, prompt: str, context: dict | None, stylesheet: dict | None, result: Any +) -> Any: + """Log the entire poml call to Langfuse.""" + + @observe(name=name) + def poml(prompt, context, stylesheet): + return result + + poml(prompt=prompt, context=context, stylesheet=stylesheet) diff --git a/python/poml/integration/langsmith.py b/python/poml/integration/langsmith.py new file mode 100644 index 00000000..1bb14d97 --- /dev/null +++ b/python/poml/integration/langsmith.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from typing import Any +from langsmith import traceable + + +def log_poml_call( + name: str, prompt: str, context: dict | None, stylesheet: dict | None, result: Any +) -> Any: + """Log the entire poml call to LangSmith.""" + + @traceable(name=name) + def poml(prompt, context, stylesheet): + return result + + poml(prompt, context, stylesheet) From 36ac438402fe419cf8287c5fc5a8fc50afda171f Mon Sep 17 00:00:00 2001 From: Yuge Zhang Date: Thu, 14 Aug 2025 18:53:22 +0800 Subject: [PATCH 2/2] . --- python/poml/api.py | 49 +++++-------------- python/poml/integration/langfuse.py | 10 +++- python/poml/integration/langsmith.py | 16 ------ .../tests/manual/example_langfuse_original.py | 22 +++++++++ python/tests/manual/example_langfuse_poml.py | 19 +++++++ 5 files changed, 61 insertions(+), 55 deletions(-) delete mode 100644 python/poml/integration/langsmith.py create mode 100644 python/tests/manual/example_langfuse_original.py create mode 100644 python/tests/manual/example_langfuse_poml.py diff --git a/python/poml/api.py b/python/poml/api.py index 99ce1a74..107beed4 100644 --- a/python/poml/api.py +++ b/python/poml/api.py @@ -25,12 +25,11 @@ _weave_enabled: bool = False _agentops_enabled: bool = False _mlflow_enabled: bool = False -_langsmith_enabled: bool = False _langfuse_enabled: bool = False _trace_log: List[Dict[str, Any]] = [] _trace_dir: Optional[Path] = None -Backend = Literal["local", "weave", "agentops", "mlflow", "langsmith", "langfuse"] +Backend = Literal["local", "weave", "agentops", "mlflow", "langfuse"] OutputFormat = Literal["raw", "dict", "openai_chat", "langchain", "pydantic"] @@ -46,7 +45,7 @@ def set_trace( enabled: Controls which tracing backends to enable. Can be: - True: Enable local tracing only (equivalent to ["local"]) - False: Disable all tracing (equivalent to []) - - str: Enable a single backend ("local", "weave", "agentops", "mlflow", "langsmith", "langfuse") + - str: Enable a single backend ("local", "weave", "agentops", "mlflow", "langfuse") - List[str]: Enable multiple backends. "local" is auto-enabled if any backends are specified. trace_dir: Optional directory for local trace files. If provided when local tracing is enabled, a subdirectory named by the current timestamp @@ -62,7 +61,6 @@ def set_trace( - "weave": Log to Weights & Biases Weave (requires local tracing) - "agentops": Log to AgentOps (requires local tracing) - "mlflow": Log to MLflow (requires local tracing) - - "langsmith": Log to LangSmith (requires local tracing) - "langfuse": Log to Langfuse (requires local tracing) """ @@ -111,11 +109,6 @@ def set_trace( else: _mlflow_enabled = False - if "langsmith" in enabled: - _langsmith_enabled = True - else: - _langsmith_enabled = False - if "langfuse" in enabled: _langfuse_enabled = True else: @@ -457,24 +450,24 @@ def poml( # Do nothing pass else: - result = json.loads(result) - if isinstance(result, dict) and "messages" in result: + result_dict = json.loads(result) + if isinstance(result_dict, dict) and "messages" in result_dict: # The new versions will always return a dict with "messages" key. - result = result["messages"] + result_dict = result_dict["messages"] if format != "dict": # Continue to validate the format. if chat: - pydantic_result = [PomlMessage(**item) for item in result] + pydantic_result = [PomlMessage(**item) for item in result_dict] else: # TODO: Make it a RichContent object - pydantic_result = [PomlMessage(speaker="human", content=result)] + pydantic_result = [PomlMessage(speaker="human", content=result_dict)] # type: ignore if format == "pydantic": - return pydantic_result + result = pydantic_result elif format == "openai_chat": - return _poml_response_to_openai_chat(pydantic_result) + result = _poml_response_to_openai_chat(pydantic_result) elif format == "langchain": - return _poml_response_to_langchain(pydantic_result) + result = _poml_response_to_langchain(pydantic_result) else: raise ValueError(f"Unknown output format: {format}") @@ -539,26 +532,6 @@ def poml( result, ) - if _langsmith_enabled: - from .integration import langsmith - - trace_prefix = _latest_trace_prefix() - current_version = _current_trace_version() - if trace_prefix is None or current_version is None: - raise RuntimeError( - "LangSmith tracing requires local tracing to be enabled." - ) - poml_content = _read_latest_traced_file(".poml") - context_content = _read_latest_traced_file(".context.json") - stylesheet_content = _read_latest_traced_file(".stylesheet.json") - langsmith.log_poml_call( - trace_prefix.name, - poml_content or str(markup), - json.loads(context_content) if context_content else None, - json.loads(stylesheet_content) if stylesheet_content else None, - result, - ) - if _langfuse_enabled: from .integration import langfuse @@ -580,7 +553,7 @@ def poml( ) if trace_record is not None: - trace_record["result"] = result + trace_record["result"] = result_dict return result finally: if temp_input_file: diff --git a/python/poml/integration/langfuse.py b/python/poml/integration/langfuse.py index 64295b04..37d9088b 100644 --- a/python/poml/integration/langfuse.py +++ b/python/poml/integration/langfuse.py @@ -1,16 +1,24 @@ from __future__ import annotations from typing import Any -from langfuse.decorators import observe +from langfuse import get_client, observe def log_poml_call( name: str, prompt: str, context: dict | None, stylesheet: dict | None, result: Any ) -> Any: """Log the entire poml call to Langfuse.""" + client = get_client() @observe(name=name) def poml(prompt, context, stylesheet): + client.update_current_generation(prompt=prompt_client) return result + prompt_client = client.create_prompt( + name=name, + type="text", + prompt=prompt + ) + poml(prompt=prompt, context=context, stylesheet=stylesheet) diff --git a/python/poml/integration/langsmith.py b/python/poml/integration/langsmith.py deleted file mode 100644 index 1bb14d97..00000000 --- a/python/poml/integration/langsmith.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from typing import Any -from langsmith import traceable - - -def log_poml_call( - name: str, prompt: str, context: dict | None, stylesheet: dict | None, result: Any -) -> Any: - """Log the entire poml call to LangSmith.""" - - @traceable(name=name) - def poml(prompt, context, stylesheet): - return result - - poml(prompt, context, stylesheet) diff --git a/python/tests/manual/example_langfuse_original.py b/python/tests/manual/example_langfuse_original.py new file mode 100644 index 00000000..fe993a62 --- /dev/null +++ b/python/tests/manual/example_langfuse_original.py @@ -0,0 +1,22 @@ +import os +from langfuse.openai import openai + +client = openai.OpenAI( + base_url=os.environ["OPENAI_API_BASE"], + api_key=os.environ["OPENAI_API_KEY"], +) + +completion = client.chat.completions.create( + name="test-chat", + model="gpt-4.1-mini", + messages=[ + { + "role": "system", + "content": "You are a very accurate calculator. You output only the result of the calculation.", + }, + {"role": "user", "content": "1 + 1 = "}, + ], + metadata={"someMetadataKey": "someValue"}, +) + +print(completion) diff --git a/python/tests/manual/example_langfuse_poml.py b/python/tests/manual/example_langfuse_poml.py new file mode 100644 index 00000000..0c957184 --- /dev/null +++ b/python/tests/manual/example_langfuse_poml.py @@ -0,0 +1,19 @@ +import os +import poml +from langfuse.openai import openai + +client = openai.OpenAI( + base_url=os.environ["OPENAI_API_BASE"], + api_key=os.environ["OPENAI_API_KEY"], +) + +poml.set_trace("langfuse", trace_dir="logs") + +messages = poml.poml("example_poml.poml", context={"code_path": "example_agentops_original.py"}, format="openai_chat") + +response = client.chat.completions.create( + model="gpt-4o-mini", + messages=messages, +) + +print(response)