diff --git a/lightspeed-stack.yaml b/lightspeed-stack.yaml index 9ac7f63a1..95ebb7315 100644 --- a/lightspeed-stack.yaml +++ b/lightspeed-stack.yaml @@ -21,5 +21,11 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" +# Conversation cache for storing Q&A history +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" # Persistent across requests, can be deleted between test runs + authentication: module: "noop" diff --git a/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml b/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml index 1cb8fcf8c..b415c5889 100644 --- a/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml +++ b/tests/e2e/configuration/lightspeed-stack-auth-noop-token.yaml @@ -21,5 +21,12 @@ user_data_collection: transcripts_enabled: true transcripts_storage: "/tmp/data/transcripts" +# Conversation cache for storing Q&A history +conversation_cache: + type: "sqlite" + sqlite: + db_path: "/tmp/data/conversation-cache.db" + authentication: module: "noop-with-token" + diff --git a/tests/e2e/configuration/lightspeed-stack-no-cache.yaml b/tests/e2e/configuration/lightspeed-stack-no-cache.yaml new file mode 100644 index 000000000..334884fa9 --- /dev/null +++ b/tests/e2e/configuration/lightspeed-stack-no-cache.yaml @@ -0,0 +1,28 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Uses a remote llama-stack service + # The instance would have already been started with a llama-stack-run.yaml file + use_as_library_client: false + # Alternative for "as library use" + # use_as_library_client: true + # library_client_config_path: + url: http://llama-stack:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" + +# NO conversation_cache configured - for testing error handling + +authentication: + module: "noop-with-token" + diff --git a/tests/e2e/features/conversation_cache_v2.feature b/tests/e2e/features/conversation_cache_v2.feature new file mode 100644 index 000000000..5ef36cfd4 --- /dev/null +++ b/tests/e2e/features/conversation_cache_v2.feature @@ -0,0 +1,397 @@ +@Authorized +Feature: Conversation Cache V2 API tests + + Background: + Given The service is started locally + And The system is in default state + + + # ==================================================================== + # V2 Conversations List Endpoint Tests + # ==================================================================== + + # BUG: Test without no_tools to expose AttributeError with empty vector database + # TODO: Remove @skip when bug is fixed (empty vector DB causes 500 error) + @skip + Scenario: V2 conversations endpoint WITHOUT no_tools (known bug - empty vector DB) + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Search the documentation for Kubernetes deployment strategies", "model": "{MODEL}", "provider": "{PROVIDER}"} + """ + Then The status code of the response is 200 + And I store conversation details + And REST API service prefix is /v2 + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 200 + And The conversation with conversation_id from above is returned + + + Scenario: V2 conversations endpoint finds the correct conversation when it exists + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Explain the benefits of containerization in cloud environments", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And REST API service prefix is /v2 + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 200 + And The conversation with conversation_id from above is returned + And The conversation has topic_summary and last_message_timestamp + And The body of the response has the following schema + """ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "conversations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "conversation_id": { "type": "string" }, + "topic_summary": { "type": ["string", "null"] }, + "last_message_timestamp": { "type": "number" } + }, + "required": ["conversation_id", "topic_summary", "last_message_timestamp"] + } + } + }, + "required": ["conversations"] + } + """ + + + Scenario: V2 conversations endpoint fails when auth header is not present + Given REST API service prefix is /v2 + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "No Authorization header found" + } + } + """ + + + # ==================================================================== + # V2 Conversation GET by ID Endpoint Tests + # ==================================================================== + + Scenario: V2 conversations/{conversation_id} endpoint finds conversation with full metadata + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "What is Kubernetes?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And I use "query" to ask question with same conversation_id + """ + {"query": "How do I install it?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The conversation history contains 2 messages + And The conversation history has correct metadata + And The conversation uses model {MODEL} and provider {PROVIDER} + And The body of the response has the following schema + """ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "conversation_id": { "type": "string" }, + "chat_history": { + "type": "array", + "items": { + "type": "object", + "properties": { + "provider": { "type": "string" }, + "model": { "type": "string" }, + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { "type": "string" }, + "type": { "type": "string", "enum": ["user", "assistant"] } + } + } + }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" } + }, + "required": ["provider", "model", "messages", "started_at", "completed_at"] + } + } + } + } + """ + + + Scenario: V2 conversations/{conversation_id} endpoint fails when auth header is not present + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Explain the difference between SQL and NoSQL databases", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And I remove the auth header + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "No Authorization header found" + } + } + """ + + + Scenario: V2 conversations/{conversation_id} GET endpoint fails when conversation_id is malformed + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP GET method + Then The status code of the response is 400 + And The body of the response is the following + """ + { + "detail": { + "response": "Invalid conversation ID format", + "cause": "The conversation ID abcdef has invalid format." + } + } + """ + + + Scenario: V2 conversations/{conversation_id} GET endpoint fails when conversation does not exist + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains Conversation not found + + + Scenario: Check conversations/{conversation_id} works when llama-stack is down + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "What is OpenShift?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And The llama-stack connection is disrupted + And REST API service prefix is /v2 + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 200 + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 200 + And The conversation history contains 1 messages + And The conversation history has correct metadata + And The conversation uses model {MODEL} and provider {PROVIDER} + + + @NoCacheConfig + Scenario: Check conversations/{conversation_id} fails when cache not configured + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 500 + And The body of the response contains Conversation cache not configured + + + # ==================================================================== + # V2 Conversation DELETE Endpoint Tests + # ==================================================================== + + Scenario: V2 conversations DELETE endpoint removes the correct conversation + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "What are the advantages of using Terraform for infrastructure as code?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The body of the response, ignoring the "conversation_id" field, is the following + """ + {"success": true, "response": "Conversation deleted successfully"} + """ + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains Conversation not found + + + Scenario: V2 conversations/{conversation_id} DELETE endpoint fails when auth header is not present + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "How does load balancing work in microservices architecture?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And I remove the auth header + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "No Authorization header found" + } + } + """ + + + Scenario: V2 conversations/{conversation_id} DELETE endpoint fails when conversation_id is malformed + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP DELETE method + Then The status code of the response is 400 + And The body of the response contains Invalid conversation ID format + + + Scenario: V2 conversations DELETE endpoint fails when the conversation does not exist + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" using HTTP DELETE method + Then The status code of the response is 404 + And The body of the response contains Conversation not found + + + Scenario: V2 conversations DELETE endpoint works even when llama-stack is down + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Test resilience", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And The llama-stack connection is disrupted + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The body of the response, ignoring the "conversation_id" field, is the following + """ + {"success": true, "response": "Conversation deleted successfully"} + """ + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 404 + And The body of the response contains Conversation not found + + + # ==================================================================== + # V2 Conversation PUT (Update Topic Summary) Endpoint Tests + # ==================================================================== + + Scenario: V2 conversations PUT endpoint successfully updates topic summary + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "How do I deploy applications on Kubernetes?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above and topic_summary "Kubernetes Deployment Strategies" using HTTP PUT method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The body of the response, ignoring the "conversation_id" field, is the following + """ + {"success": true, "message": "Topic summary updated successfully"} + """ + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 200 + And The conversation with conversation_id from above is returned + And The conversation topic_summary is "Kubernetes Deployment Strategies" + + + Scenario: V2 conversations PUT endpoint fails when auth header is not present + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "What is continuous integration?", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And I remove the auth header + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above and topic_summary "CI/CD Pipeline" using HTTP PUT method + Then The status code of the response is 401 + And The body of the response is the following + """ + { + "detail": { + "response": "Missing or invalid credentials provided by client", + "cause": "No Authorization header found" + } + } + """ + + + Scenario: V2 conversations PUT endpoint fails when conversation_id is malformed + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "invalid-id" and topic_summary "Updated Summary" using HTTP PUT method + Then The status code of the response is 400 + And The body of the response is the following + """ + { + "detail": { + "response": "Invalid conversation ID format", + "cause": "The conversation ID invalid-id has invalid format." + } + } + """ + + + Scenario: V2 conversations PUT endpoint fails when conversation does not exist + Given REST API service prefix is /v2 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" and topic_summary "Updated Summary" using HTTP PUT method + Then The status code of the response is 404 + And The body of the response contains Conversation not found + + + Scenario: V2 conversations PUT endpoint fails with empty topic summary (422) + Given REST API service prefix is /v1 + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Explain GraphQL advantages over REST", "model": "{MODEL}", "provider": "{PROVIDER}", "no_tools": true} + """ + And The status code of the response is 200 + And I store conversation details + And REST API service prefix is /v2 + When I use REST API conversation endpoint with conversation_id from above and empty topic_summary using HTTP PUT method + Then The status code of the response is 422 + And The body of the response contains String should have at least 1 character diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index ec995c954..9d2fc8972 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -79,6 +79,15 @@ def before_scenario(context: Context, scenario: Scenario) -> None: context.scenario_config = ( "tests/e2e/configuration/lightspeed-stack-invalid-feedback-storage.yaml" ) + if "NoCacheConfig" in scenario.effective_tags: + context.scenario_config = ( + "tests/e2e/configuration/lightspeed-stack-no-cache.yaml" + ) + # Switch config and restart immediately + switch_config( + context.scenario_config + ) # Copies to default lightspeed-stack.yaml + restart_container("lightspeed-stack") def after_scenario(context: Context, scenario: Scenario) -> None: @@ -86,6 +95,9 @@ def after_scenario(context: Context, scenario: Scenario) -> None: if "InvalidFeedbackStorageConfig" in scenario.effective_tags: switch_config(context.feature_config) restart_container("lightspeed-stack") + if "NoCacheConfig" in scenario.effective_tags: + switch_config(context.feature_config) + restart_container("lightspeed-stack") # Restore Llama Stack connection if it was disrupted if hasattr(context, "llama_stack_was_running") and context.llama_stack_was_running: diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index d29f60751..aed1c4336 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -120,9 +120,16 @@ def request_endpoint(context: Context, endpoint: str, hostname: str, port: int) def check_status_code(context: Context, status: int) -> None: """Check the HTTP status code for latest response from tested service.""" assert context.response is not None, "Request needs to be performed first" - assert ( - context.response.status_code == status - ), f"Status code is {context.response.status_code}" + if context.response.status_code != status: + # Include response body in error message for debugging + try: + error_body = context.response.json() + except Exception: + error_body = context.response.text + assert False, ( + f"Status code is {context.response.status_code}, expected {status}. " + f"Response: {error_body}" + ) @then('Content type of response should be set to "{content_type}"') diff --git a/tests/e2e/features/steps/conversation.py b/tests/e2e/features/steps/conversation.py index 488861a4e..4fa20921d 100644 --- a/tests/e2e/features/steps/conversation.py +++ b/tests/e2e/features/steps/conversation.py @@ -92,6 +92,78 @@ def access_conversation_endpoint_delete_specific( context.response = requests.delete(url, headers=headers, timeout=DEFAULT_TIMEOUT) +@when( + 'I use REST API conversation endpoint with conversation_id from above and topic_summary "{topic_summary}" using HTTP PUT method' +) +def access_conversation_endpoint_put(context: Context, topic_summary: str) -> None: + """Send PUT HTTP request to tested service for conversation/{conversation_id} with topic_summary.""" + assert hasattr(context, "response_data"), "response_data not found in context" + assert context.response_data.get("conversation_id"), "conversation id not stored" + + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{context.response_data['conversation_id']}".replace( + "//", "/" + ) + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + context.response = None + + if topic_summary == "": + topic_summary = "" + + payload = {"topic_summary": topic_summary} + + context.response = requests.put( + url, json=payload, headers=headers, timeout=DEFAULT_TIMEOUT + ) + + +@step( + 'I use REST API conversation endpoint with conversation_id "{conversation_id}" and topic_summary "{topic_summary}" using HTTP PUT method' +) +def access_conversation_endpoint_put_specific( + context: Context, conversation_id: str, topic_summary: str +) -> None: + """Send PUT HTTP request to tested service for conversation/{conversation_id} with topic_summary.""" + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{conversation_id}".replace("//", "/") + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + context.response = None + + payload = {"topic_summary": topic_summary} + + context.response = requests.put( + url, json=payload, headers=headers, timeout=DEFAULT_TIMEOUT + ) + + +@when( + "I use REST API conversation endpoint with conversation_id from above and empty topic_summary using HTTP PUT method" +) +def access_conversation_endpoint_put_empty(context: Context) -> None: + """Send PUT HTTP request with empty topic_summary to test validation.""" + assert hasattr(context, "response_data"), "response_data not found in context" + assert context.response_data.get("conversation_id"), "conversation id not stored" + + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{context.response_data['conversation_id']}".replace( + "//", "/" + ) + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + context.response = None + + payload = {"topic_summary": ""} + + context.response = requests.put( + url, json=payload, headers=headers, timeout=DEFAULT_TIMEOUT + ) + + @then("The conversation with conversation_id from above is returned") def check_returned_conversation_id(context: Context) -> None: """Check the conversation id in response.""" @@ -107,10 +179,44 @@ def check_returned_conversation_id(context: Context) -> None: assert found_conversation is not None, "conversation not found" +@then("The conversation has topic_summary and last_message_timestamp") +def check_conversation_metadata_not_empty(context: Context) -> None: + """Check that conversation has non-empty metadata fields.""" + found_conversation = context.found_conversation + + assert found_conversation is not None, "conversation not found in context" + + assert ( + "last_message_timestamp" in found_conversation + ), "last_message_timestamp field missing" + timestamp = found_conversation["last_message_timestamp"] + assert isinstance( + timestamp, (int, float) + ), f"last_message_timestamp should be a number, got {type(timestamp)}" + assert timestamp > 0, f"last_message_timestamp should be positive, got {timestamp}" + + assert "topic_summary" in found_conversation, "topic_summary field missing" + topic_summary = found_conversation["topic_summary"] + assert topic_summary is not None, "topic_summary should not be None" + + +@then('The conversation topic_summary is "{expected_summary}"') +def check_conversation_topic_summary(context: Context, expected_summary: str) -> None: + """Check that the conversation has the expected topic summary.""" + found_conversation = context.found_conversation + + assert found_conversation is not None, "conversation not found in context" + assert "topic_summary" in found_conversation, "topic_summary field missing" + + actual_summary = found_conversation["topic_summary"] + assert ( + actual_summary == expected_summary + ), f"Expected topic_summary '{expected_summary}', but got '{actual_summary}'" + + @then("The conversation details are following") def check_returned_conversation_content(context: Context) -> None: """Check the conversation content in response.""" - # Replace {MODEL} and {PROVIDER} placeholders with actual values json_str = replace_placeholders(context, context.text or "{}") expected_data = json.loads(json_str) @@ -156,3 +262,82 @@ def check_found_conversation_content(context: Context) -> None: def check_deleted_conversation(context: Context) -> None: """Check whether the deleted conversation is gone.""" assert context.response is not None + + +@then("The conversation history contains {count:d} messages") +def check_conversation_message_count(context: Context, count: int) -> None: + """Check that the conversation history has expected number of messages.""" + response_json = context.response.json() + + assert "chat_history" in response_json, "chat_history not found in response" + actual_count = len(response_json["chat_history"]) + + assert actual_count == count, ( + f"Expected {count} messages in conversation history, " + f"but found {actual_count}" + ) + + +@then("The conversation history has correct metadata") +def check_conversation_metadata(context: Context) -> None: + """Check that conversation history has correct model and provider info.""" + response_json = context.response.json() + + assert "chat_history" in response_json, "chat_history not found in response" + chat_history = response_json["chat_history"] + + assert len(chat_history) > 0, "chat_history is empty" + + for idx, turn in enumerate(chat_history): + assert "provider" in turn, f"Turn {idx} missing 'provider'" + assert "model" in turn, f"Turn {idx} missing 'model'" + assert "messages" in turn, f"Turn {idx} missing 'messages'" + assert "started_at" in turn, f"Turn {idx} missing 'started_at'" + assert "completed_at" in turn, f"Turn {idx} missing 'completed_at'" + + assert turn["provider"], f"Turn {idx} has empty provider" + assert turn["model"], f"Turn {idx} has empty model" + + messages = turn["messages"] + assert ( + len(messages) == 2 + ), f"Turn {idx} should have 2 messages (user + assistant)" + + user_msg = messages[0] + assert user_msg["type"] == "user", f"Turn {idx} first message should be user" + assert "content" in user_msg, f"Turn {idx} user message missing content" + + assistant_msg = messages[1] + assert ( + assistant_msg["type"] == "assistant" + ), f"Turn {idx} second message should be assistant" + assert ( + "content" in assistant_msg + ), f"Turn {idx} assistant message missing content" + + +@then("The conversation uses model {model} and provider {provider}") +def check_conversation_model_provider( + context: Context, model: str, provider: str +) -> None: + """Check that conversation used specific model and provider.""" + response_json = context.response.json() + + assert "chat_history" in response_json, "chat_history not found in response" + chat_history = response_json["chat_history"] + + assert len(chat_history) > 0, "chat_history is empty" + + expected_model = replace_placeholders(context, model) + expected_provider = replace_placeholders(context, provider) + + for idx, turn in enumerate(chat_history): + actual_model = turn.get("model") + actual_provider = turn.get("provider") + + assert ( + actual_model == expected_model + ), f"Turn {idx} expected model '{expected_model}', got '{actual_model}'" + assert ( + actual_provider == expected_provider + ), f"Turn {idx} expected provider '{expected_provider}', got '{actual_provider}'"