diff --git a/src/cai/sdk/agents/models/openai_chatcompletions.py b/src/cai/sdk/agents/models/openai_chatcompletions.py index 8931edd6..ae2db145 100644 --- a/src/cai/sdk/agents/models/openai_chatcompletions.py +++ b/src/cai/sdk/agents/models/openai_chatcompletions.py @@ -517,6 +517,16 @@ def set_agent_name(self, name: str) -> None: def _non_null_or_not_given(self, value: Any) -> Any: return value if value is not None else NOT_GIVEN + def _warn_empty_response(self, content: str | None, has_tool_calls: bool, has_refusal: bool) -> None: + """Log warning for empty or sentinel responses (e.g., <|endoftext|>).""" + if has_tool_calls or has_refusal: + return + is_empty = not content or not str(content).strip() + is_sentinel = content and "<|endoftext|>" in str(content) + if is_empty or is_sentinel: + detail = "empty" if is_empty else f"sentinel ({str(content)[:50]})" + logger.warning(f"Model completed without output ({detail}). May indicate model-agent compatibility issue.") + async def get_response( self, system_instructions: str | None, @@ -1124,6 +1134,13 @@ async def get_response( if not hasattr(response, "cost"): response.cost = None + # Warn if response is empty or contains sentinel token + self._warn_empty_response( + getattr(response.choices[0].message, "content", None), + bool(getattr(response.choices[0].message, "tool_calls", None)), + bool(getattr(response.choices[0].message, "refusal", None)) + ) + return ModelResponse( output=items, usage=usage, @@ -2198,6 +2215,17 @@ async def stream_response( }, ) + # Warn if streamed response is empty or contains sentinel token + text_content = "" + if state.text_content_index_and_output: + text_out = state.text_content_index_and_output[1] + text_content = getattr(text_out, 'text', '') or getattr(text_out, 'content', '') + self._warn_empty_response( + text_content, + bool(state.function_calls), + bool(state.refusal_content_index_and_output) + ) + yield ResponseCompletedEvent( response=final_response, type="response.completed", diff --git a/tests/core/test_openai_chatcompletions.py b/tests/core/test_openai_chatcompletions.py index 7c8ab039..528feda5 100644 --- a/tests/core/test_openai_chatcompletions.py +++ b/tests/core/test_openai_chatcompletions.py @@ -360,4 +360,147 @@ async def patched_fetch_response(self, *args, **kwargs): ) # Counter should now be 2 (one increment per turn, not per item) - assert model.interaction_counter == 2 \ No newline at end of file + assert model.interaction_counter == 2 + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_empty_response_warning(monkeypatch, caplog) -> None: + """ + When the model returns a response with empty content (no text, no tool calls, no refusal), + a warning should be logged indicating potential model-agent compatibility issues. + """ + import logging + + # Create a response with empty content + msg = ChatCompletionMessage(role="assistant", content="") + choice = Choice(index=0, finish_reason="stop", message=msg) + chat = ChatCompletion( + id="resp-id", + created=0, + model="fake", + object="chat.completion", + choices=[choice], + usage=CompletionUsage(completion_tokens=0, prompt_tokens=5, total_tokens=5), + ) + + async def patched_fetch_response(self, *args, **kwargs): + return chat + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider(use_responses=False).get_model(cai_model) + + # Enable logging capture + with caplog.at_level(logging.WARNING, logger="openai.agents"): + resp: ModelResponse = await model.get_response( + system_instructions=None, + input="Test", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + ) + + # Response should still be returned (non-breaking) + assert isinstance(resp, ModelResponse) + + # Warning should have been logged + assert any("Model completed without output" in record.message for record in caplog.records) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_sentinel_response_warning(monkeypatch, caplog) -> None: + """ + When the model returns a response with sentinel token (<|endoftext|>), + a warning should be logged indicating potential model-agent compatibility issues. + """ + import logging + + # Create a response with sentinel token + msg = ChatCompletionMessage(role="assistant", content="<|endoftext|>") + choice = Choice(index=0, finish_reason="stop", message=msg) + chat = ChatCompletion( + id="resp-id", + created=0, + model="fake", + object="chat.completion", + choices=[choice], + usage=CompletionUsage(completion_tokens=1, prompt_tokens=5, total_tokens=6), + ) + + async def patched_fetch_response(self, *args, **kwargs): + return chat + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider(use_responses=False).get_model(cai_model) + + # Enable logging capture + with caplog.at_level(logging.WARNING, logger="openai.agents"): + resp: ModelResponse = await model.get_response( + system_instructions=None, + input="Test", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + ) + + # Response should still be returned (non-breaking) + assert isinstance(resp, ModelResponse) + + # Warning should have been logged with sentinel info + assert any("sentinel token" in record.message for record in caplog.records) + + +@pytest.mark.allow_call_model_methods +@pytest.mark.asyncio +async def test_no_warning_for_tool_calls(monkeypatch, caplog) -> None: + """ + When the model returns a response with tool calls (even without text content), + no warning should be logged since this is expected behavior. + """ + import logging + + # Create a response with tool calls but no text content + tool_call = ChatCompletionMessageToolCall( + id="call_123", + type="function", + function=Function(name="test_tool", arguments='{"arg": "value"}'), + ) + msg = ChatCompletionMessage(role="assistant", content=None, tool_calls=[tool_call]) + choice = Choice(index=0, finish_reason="tool_calls", message=msg) + chat = ChatCompletion( + id="resp-id", + created=0, + model="fake", + object="chat.completion", + choices=[choice], + usage=CompletionUsage(completion_tokens=10, prompt_tokens=5, total_tokens=15), + ) + + async def patched_fetch_response(self, *args, **kwargs): + return chat + + monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response) + model = OpenAIProvider(use_responses=False).get_model(cai_model) + + # Enable logging capture + with caplog.at_level(logging.WARNING, logger="openai.agents"): + resp: ModelResponse = await model.get_response( + system_instructions=None, + input="Test", + model_settings=ModelSettings(), + tools=[], + output_schema=None, + handoffs=[], + tracing=ModelTracing.DISABLED, + ) + + # Response should be returned + assert isinstance(resp, ModelResponse) + + # No warning should be logged for tool calls + assert not any("Model completed without output" in record.message for record in caplog.records) \ No newline at end of file