From 558f22eec599b47ba30192e26cda4654e6c49cb3 Mon Sep 17 00:00:00 2001 From: Radovan Fuchs Date: Mon, 6 Oct 2025 11:02:22 +0200 Subject: [PATCH] added tests for v1/conversations endpoints --- tests/e2e/features/conversations.feature | 227 +++++++++++++++++------ tests/e2e/features/feedback.feature | 10 +- tests/e2e/features/steps/common_http.py | 4 +- tests/e2e/features/steps/conversation.py | 152 ++++++++++++++- 4 files changed, 325 insertions(+), 68 deletions(-) diff --git a/tests/e2e/features/conversations.feature b/tests/e2e/features/conversations.feature index 82e59bc95..e7aac4a34 100644 --- a/tests/e2e/features/conversations.feature +++ b/tests/e2e/features/conversations.feature @@ -1,54 +1,175 @@ -# Feature: conversations endpoint API tests -#TODO: fix test - -# Background: -# Given The service is started locally -# And REST API service hostname is localhost -# And REST API service port is 8080 -# And REST API service prefix is /v1 - - -# Scenario: Check if conversations endpoint finds the correct conversation when it exists -# Given The system is in default state -# When I access REST API endpoint "conversations" using HTTP GET method -# Then The status code of the response is 200 -# And the proper conversation is returned - -# Scenario: Check if conversations endpoint does not finds the conversation when it does not exists -# Given The system is in default state -# When I access REST API endpoint "conversations" using HTTP GET method -# Then The status code of the response is 404 - -# Scenario: Check if conversations endpoint fails when conversation id is not provided -# Given The system is in default state -# When I access REST API endpoint "conversations" using HTTP GET method -# Then The status code of the response is 422 - -# Scenario: Check if conversations endpoint fails when service is unavailable -# Given The system is in default state -# And the service is stopped -# When I access REST API endpoint "conversations" using HTTP GET method -# Then The status code of the response is 503 - -# Scenario: Check if conversations/delete endpoint finds the correct conversation when it exists -# Given The system is in default state -# When I access REST API endpoint "conversations/delete" using HTTP GET method -# Then The status code of the response is 200 -# And the deleted conversation is not found - -# Scenario: Check if conversations/delete endpoint does not finds the conversation when it does not exists -# Given The system is in default state -# When I access REST API endpoint "conversations/delete" using HTTP GET method -# Then The status code of the response is 404 - -# Scenario: Check if conversations/delete endpoint fails when conversation id is not provided -# Given The system is in default state -# When I access REST API endpoint "conversations/delete" using HTTP GET method -# Then The status code of the response is 422 - -# Scenario: Check if conversations/delete endpoint fails when service is unavailable -# Given The system is in default state -# And the service is stopped -# When I access REST API endpoint "conversations/delete" using HTTP GET method -# Then The status code of the response is 503 +@Authorized +Feature: conversations endpoint API tests + Background: + Given The service is started locally + And REST API service hostname is localhost + And REST API service port is 8080 + And REST API service prefix is /v1 + + + Scenario: Check if conversations endpoint finds the correct conversation when it exists + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 200 + And The conversation with conversation_id from above is returned + And The conversation details are following + """ + {"last_used_model": "gpt-4-turbo", "last_used_provider": "openai", "message_count": 1} + """ + + Scenario: Check if conversations endpoint fails when the auth header is not present + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + And I remove the auth header + When I access REST API endpoint "conversations" using HTTP GET method + Then The status code of the response is 400 + And The body of the response is the following + """ + { + "detail": "No Authorization header found" + } + """ + + Scenario: Check if conversations/{conversation_id} endpoint finds the correct conversation when it exists + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The body of the response has following messages + """ + {"content": "Say hello", "type": "user", "content_response": "Hello", "type_response": "assistant"} + """ + And The body of the response has the following schema + """ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "conversation_id": { "type": "string" }, + "chat_history": { + "type": "array", + "items": { + "type": "object", + "properties": { + "messages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "content": { "type": "string" }, + "type": { "type": "string", "enum": ["user", "assistant"] } + } + } + }, + "started_at": { "type": "string", "format": "date-time" }, + "completed_at": { "type": "string", "format": "date-time" } + } + } + } + } + } + """ + + Scenario: Check if conversations/{conversation_id} endpoint fails when the auth header is not present + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + And I remove the auth header + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 400 + And The body of the response is the following + """ + { + "detail": "No Authorization header found" + } + """ + + Scenario: Check if conversations/{conversation_id} GET endpoint fails when conversation_id is malformed + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP GET method + Then The status code of the response is 422 + + Scenario: Check if conversations/{conversation_id} GET endpoint fails when llama-stack is unavailable + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + And The llama-stack connection is disrupted + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 503 + And The body of the response contains Unable to connect to Llama Stack + + Scenario: Check if conversations DELETE endpoint removes the correct conversation + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + When I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method + Then The status code of the response is 200 + And The returned conversation details have expected conversation_id + And The body of the response, ignoring the "conversation_id" field, is the following + """ + {"success": true, "response": "Conversation deleted successfully"} + """ + And I use REST API conversation endpoint with conversation_id from above using HTTP GET method + And The status code of the response is 404 + + Scenario: Check if conversations/{conversation_id} DELETE endpoint fails when conversation_id is malformed + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + When I use REST API conversation endpoint with conversation_id "abcdef" using HTTP DELETE method + Then The status code of the response is 422 + + Scenario: Check if conversations DELETE endpoint fails when the conversation does not exist + Given The system is in default state + When I use REST API conversation endpoint with conversation_id "12345678-abcd-0000-0123-456789abcdef" using HTTP DELETE method + Then The status code of the response is 404 + + Scenario: Check if conversations/{conversation_id} DELETE endpoint fails when llama-stack is unavailable + Given The system is in default state + And I set the Authorization header to Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Ikpva + And I use "query" to ask question with authorization header + """ + {"query": "Say hello", "model": "gpt-4-turbo", "provider": "openai"} + """ + And The status code of the response is 200 + And I store conversation details + And The llama-stack connection is disrupted + When I use REST API conversation endpoint with conversation_id from above using HTTP GET method + Then The status code of the response is 503 + And The body of the response contains Unable to connect to Llama Stack \ No newline at end of file diff --git a/tests/e2e/features/feedback.feature b/tests/e2e/features/feedback.feature index dce0d0514..5e8edfd34 100644 --- a/tests/e2e/features/feedback.feature +++ b/tests/e2e/features/feedback.feature @@ -13,7 +13,7 @@ Feature: feedback endpoint API tests Given The system is in default state When The feedback is enabled Then The status code of the response is 200 - And And the body of the response has the following structure + And the body of the response has the following structure """ { "status": @@ -27,7 +27,7 @@ Feature: feedback endpoint API tests Given The system is in default state When The feedback is disabled Then The status code of the response is 200 - And And the body of the response has the following structure + And the body of the response has the following structure """ { "status": @@ -46,7 +46,7 @@ Feature: feedback endpoint API tests } """ Then The status code of the response is 422 - And And the body of the response has the following structure + And the body of the response has the following structure """ { "detail": [ @@ -123,7 +123,7 @@ Feature: feedback endpoint API tests } """ Then The status code of the response is 422 - And And the body of the response has the following structure + And the body of the response has the following structure """ { "detail": [ @@ -211,7 +211,7 @@ Feature: feedback endpoint API tests } """ Then The status code of the response is 422 - And And the body of the response has the following structure + And the body of the response has the following structure """ { "detail": [{ diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 33ed48cbb..78490b7cf 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -115,7 +115,7 @@ def request_endpoint(context: Context, endpoint: str, hostname: str, port: int) ) -@then("The status code of the response is {status:d}") +@step("The status code of the response is {status:d}") def check_status_code(context: Context, status: int) -> None: """Check the HTTP status code for latest response from tested service.""" assert context.response is not None, "Request needs to be performed first" @@ -363,7 +363,7 @@ def check_for_null_attribute(context: Context, attribute: str) -> None: ), f"Attribute {attribute} should be null, but it contains {value}" -@then("And the body of the response has the following structure") +@then("the body of the response has the following structure") def check_response_partially(context: Context) -> None: """Validate that the response body matches the expected JSON structure. diff --git a/tests/e2e/features/steps/conversation.py b/tests/e2e/features/steps/conversation.py index 36c39d481..a4d94f7e4 100644 --- a/tests/e2e/features/steps/conversation.py +++ b/tests/e2e/features/steps/conversation.py @@ -1,18 +1,154 @@ """Implementation of common test steps.""" -from behave import then # pyright: ignore[reportAttributeAccessIssue] +import json +from behave import step, when, then # pyright: ignore[reportAttributeAccessIssue] from behave.runner import Context +import requests +# default timeout for HTTP operations +DEFAULT_TIMEOUT = 10 -@then("The proper conversation is returned") -def check_returned_conversation(context: Context) -> None: - """Check the conversation in response.""" - # TODO: add step implementation - assert context.response is not None + +@step( + "I use REST API conversation endpoint with conversation_id from above using HTTP GET method" +) +def access_conversation_endpoint_get(context: Context) -> None: + """Send GET HTTP request to tested service for conversation/{conversation_id}.""" + assert ( + context.response_data["conversation_id"] is not None + ), "conversation id not stored" + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{context.response_data['conversation_id']}".replace( + "//", "/" + ) + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + # initial value + context.response = None + + # perform REST API call + context.response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT) + + +@step( + 'I use REST API conversation endpoint with conversation_id "{conversation_id}" using HTTP GET method' +) +def access_conversation_endpoint_get_specific( + context: Context, conversation_id: str +) -> None: + """Send GET HTTP request to tested service for conversation/{conversation_id}.""" + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{conversation_id}".replace("//", "/") + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + # initial value + context.response = None + + # perform REST API call + context.response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT) + + +@when( + "I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method" +) +def access_conversation_endpoint_delete(context: Context) -> None: + """Send GET HTTP request to tested service for conversation/{conversation_id}.""" + assert ( + context.response_data["conversation_id"] is not None + ), "conversation id not stored" + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{context.response_data['conversation_id']}".replace( + "//", "/" + ) + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + # initial value + context.response = None + + # perform REST API call + context.response = requests.delete(url, headers=headers, timeout=DEFAULT_TIMEOUT) + + +@step( + 'I use REST API conversation endpoint with conversation_id "{conversation_id}" using HTTP DELETE method' +) +def access_conversation_endpoint_delete_specific( + context: Context, conversation_id: str +) -> None: + """Send GET HTTP request to tested service for conversation/{conversation_id}.""" + endpoint = "conversations" + base = f"http://{context.hostname}:{context.port}" + path = f"{context.api_prefix}/{endpoint}/{conversation_id}".replace("//", "/") + url = base + path + headers = context.auth_headers if hasattr(context, "auth_headers") else {} + # initial value + context.response = None + + # perform REST API call + context.response = requests.delete(url, headers=headers, timeout=DEFAULT_TIMEOUT) + + +@then("The conversation with conversation_id from above is returned") +def check_returned_conversation_id(context: Context) -> None: + """Check the conversation id in response.""" + response_json = context.response.json() + found_conversation = None + for conversation in response_json["conversations"]: + if conversation["conversation_id"] == context.response_data["conversation_id"]: + found_conversation = conversation + break + + context.found_conversation = found_conversation + + assert found_conversation is not None, "conversation not found" + + +@then("The conversation details are following") +def check_returned_conversation_content(context: Context) -> None: + """Check the conversation content in response.""" + expected_data = json.loads(context.text) + found_conversation = context.found_conversation + + assert ( + found_conversation["last_used_model"] == expected_data["last_used_model"] + ), f"last_used_model mismatch, was {found_conversation["last_used_model"]}" + assert ( + found_conversation["last_used_provider"] == expected_data["last_used_provider"] + ), f"last_used_provider mismatch, was {found_conversation["last_used_provider"]}" + assert ( + found_conversation["message_count"] == expected_data["message_count"] + ), f"message count mismatch, was {found_conversation["message_count"]}" + + +@then("The returned conversation details have expected conversation_id") +def check_found_conversation_id(context: Context) -> None: + """Check whether the conversation details have expected conversation_id.""" + response_json = context.response.json() + + assert ( + response_json["conversation_id"] == context.response_data["conversation_id"] + ), "found wrong conversation" + + +@then("The body of the response has following messages") +def check_found_conversation_content(context: Context) -> None: + """Check whether the conversation details have expected data.""" + expected_data = json.loads(context.text) + response_json = context.response.json() + chat_messages = response_json["chat_history"][0]["messages"] + + assert chat_messages[0]["content"] == expected_data["content"] + assert chat_messages[0]["type"] == expected_data["type"] + assert ( + expected_data["content_response"] in chat_messages[1]["content"] + ), f"expected substring not in response, has {chat_messages[1]["content"]}" + assert chat_messages[1]["type"] == expected_data["type_response"] -@then("the deleted conversation is not found") +@then("The conversation with details and conversation_id from above is not found") def check_deleted_conversation(context: Context) -> None: """Check whether the deleted conversation is gone.""" - # TODO: add step implementation assert context.response is not None