From f35ad2e8a4fc89c39007d70f14d8c0a956605b11 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Wed, 5 Mar 2025 14:19:36 +0700 Subject: [PATCH] test: add e2e message --- .../api/message/test_api_create_message.py | 139 ++++++++++++++ .../api/message/test_api_delete_message.py | 91 +++++++++ .../api/message/test_api_get_list_message.py | 181 ++++++++++++++++++ .../api/message/test_api_get_message.py | 167 ++++++++++++++++ .../runner/cortex-llamacpp-e2e-nightly.py | 4 + engine/e2e-test/runner/main.py | 4 + 6 files changed, 586 insertions(+) create mode 100644 engine/e2e-test/api/message/test_api_create_message.py create mode 100644 engine/e2e-test/api/message/test_api_delete_message.py create mode 100644 engine/e2e-test/api/message/test_api_get_list_message.py create mode 100644 engine/e2e-test/api/message/test_api_get_message.py diff --git a/engine/e2e-test/api/message/test_api_create_message.py b/engine/e2e-test/api/message/test_api_create_message.py new file mode 100644 index 000000000..44cf31f39 --- /dev/null +++ b/engine/e2e-test/api/message/test_api_create_message.py @@ -0,0 +1,139 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiCreateMessage: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_create_message_successfully(self): + title = "New Thread" + + data = { + "metadata": { + "title": title + } + } + + post_thread_url = f"http://localhost:3928/v1/threads" + response = requests.post( + post_thread_url, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_create_message_successfully") + assert_equal(response.status_code,200) + + thread_id = json_data["id"] + + post_message_data = { + "role": "user", + "content": "Hello, world!" + } + + post_message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages" + new_message_response = requests.post(post_message_url, json=post_message_data) + json_data_new_message = new_message_response.json() + log_response(json_data_new_message, "test_api_create_message_successfully") + assert_equal(new_message_response.status_code,200) + + schema = { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the message" + }, + "object": { + "type": "string", + "description": "Type of object, always 'thread.message'" + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp of when the message was created" + }, + "completed_at": { + "type": "integer", + "description": "Unix timestamp of when the message was completed" + }, + "thread_id": { + "type": "string", + "description": "ID of the thread this message belongs to" + }, + "role": { + "type": "string", + "description": "Role of the message sender", + "enum": [ + "user", + "assistant" + ] + }, + "status": { + "type": "string", + "description": "Status of the message", + "enum": [ + "completed" + ] + }, + "content": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of content", + "enum": [ + "text" + ] + }, + "text": { + "type": "object", + "properties": { + "value": { + "type": "string", + "description": "The message text" + }, + "annotations": { + "type": "array", + "description": "Array of annotations for the text" + } + } + } + } + } + }, + "metadata": { + "type": "object", + "description": "Additional metadata for the message" + } + }, + "required": [ + "id", + "object", + "created_at", + "completed_at", + "thread_id", + "role", + "status", + "content" + ] + } + + # Validate response schema + jsonschema.validate(instance=json_data_new_message, schema=schema) + + assert_equal(json_data_new_message["content"][0]['text']["value"], "Hello, world!") \ No newline at end of file diff --git a/engine/e2e-test/api/message/test_api_delete_message.py b/engine/e2e-test/api/message/test_api_delete_message.py new file mode 100644 index 000000000..8e99ab1e0 --- /dev/null +++ b/engine/e2e-test/api/message/test_api_delete_message.py @@ -0,0 +1,91 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiDeleteMessage: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_delete_message_successfully(self): + title = "New Thread" + + data = { + "metadata": { + "title": title + } + } + # Create new thread + post_thread_url = f"http://localhost:3928/v1/threads" + response = requests.post( + post_thread_url, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_delete_message_successfully") + assert_equal(response.status_code,200) + + thread_id = json_data["id"] + + post_message_data = { + "role": "user", + "content": "Hello, world!" + } + + # Create new message + message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages" + new_message_response = requests.post(message_url, json=post_message_data) + json_data_new_message = new_message_response.json() + log_response(json_data_new_message, "test_api_delete_message_successfully") + assert_equal(new_message_response.status_code,200) + + message_id = json_data_new_message["id"] + + # Delete message with id + del_message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages/{message_id}" + del_message_response = requests.delete(del_message_url) + json_data_del_message = del_message_response.json() + log_response(json_data_del_message, "test_api_delete_message_successfully") + assert_equal(del_message_response.status_code,200) + + schema = { + "type": "object", + "properties": { + "deleted": { + "type": "boolean", + "description": "Indicates if the message was successfully deleted" + }, + "id": { + "type": "string", + "description": "ID of the deleted message" + }, + "object": { + "type": "string", + "description": "Type of object, always 'thread.message.deleted'" + } + }, + "required": [ + "deleted", + "id", + "object" + ] + } + + # Validate response schema + jsonschema.validate(instance=json_data_del_message, schema=schema) + + assert_equal(json_data_del_message["deleted"], True) + assert_equal(json_data_del_message["id"], message_id) + assert_equal(json_data_del_message["object"], "thread.message.deleted") \ No newline at end of file diff --git a/engine/e2e-test/api/message/test_api_get_list_message.py b/engine/e2e-test/api/message/test_api_get_list_message.py new file mode 100644 index 000000000..5a9cd40b4 --- /dev/null +++ b/engine/e2e-test/api/message/test_api_get_list_message.py @@ -0,0 +1,181 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiGetListMessage: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_list_message_successfully(self): + title = "New Thread" + + data = { + "metadata": { + "title": title + } + } + # Create new thread + post_thread_url = f"http://localhost:3928/v1/threads" + response = requests.post( + post_thread_url, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_get_list_message_successfully") + assert_equal(response.status_code,200) + + thread_id = json_data["id"] + + post_message_data = { + "role": "user", + "content": "Hello, world!" + } + + # Create new message + message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages" + new_message_response = requests.post(message_url, json=post_message_data) + json_data_new_message = new_message_response.json() + log_response(json_data_new_message, "test_api_get_list_message_successfully") + assert_equal(new_message_response.status_code,200) + + # Get list message + list_message_response = requests.get(message_url) + json_data_list_message = list_message_response.json() + log_response(json_data_list_message, "test_api_get_list_message_successfully") + assert_equal(list_message_response.status_code,200) + + schema = { + "type": "object", + "properties": { + "object": { + "type": "string", + "description": "Type of the list response, always 'list'" + }, + "data": { + "type": "array", + "description": "Array of message objects", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the message" + }, + "object": { + "type": "string", + "description": "Type of object, always 'thread.message'" + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp of when the message was created" + }, + "thread_id": { + "type": "string", + "description": "ID of the thread this message belongs to" + }, + "role": { + "type": "string", + "description": "Role of the message sender", + "enum": [ + "assistant", + "user" + ] + }, + "status": { + "type": "string", + "description": "Status of the message", + "enum": [ + "completed" + ] + }, + "content": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of content", + "enum": [ + "text" + ] + }, + "text": { + "type": "object", + "properties": { + "value": { + "type": "string", + "description": "The message text" + }, + "annotations": { + "type": "array", + "description": "Array of annotations for the text" + } + } + } + } + } + }, + "metadata": { + "type": "object", + "description": "Additional metadata for the message" + }, + "attachments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file_id": { + "type": "string", + "description": "ID of the attached file" + }, + "tools": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of tool used" + } + } + } + } + } + } + } + }, + "required": [ + "id", + "object", + "created_at", + "thread_id", + "role", + "content" + ] + } + } + }, + "required": [ + "object", + "data" + ] + } + + # Validate response schema + jsonschema.validate(instance=json_data_list_message, schema=schema) + + assert_equal(json_data_list_message["data"][0]["content"][0]['text']["value"], "Hello, world!") \ No newline at end of file diff --git a/engine/e2e-test/api/message/test_api_get_message.py b/engine/e2e-test/api/message/test_api_get_message.py new file mode 100644 index 000000000..73dbca9d0 --- /dev/null +++ b/engine/e2e-test/api/message/test_api_get_message.py @@ -0,0 +1,167 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiGetMessage: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_message_successfully(self): + title = "New Thread" + + data = { + "metadata": { + "title": title + } + } + # Create new thread + post_thread_url = f"http://localhost:3928/v1/threads" + response = requests.post( + post_thread_url, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_get_message_successfully") + assert_equal(response.status_code,200) + + thread_id = json_data["id"] + + post_message_data = { + "role": "user", + "content": "Hello, world!" + } + + # Create new message + message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages" + new_message_response = requests.post(message_url, json=post_message_data) + json_data_new_message = new_message_response.json() + log_response(json_data_new_message, "test_api_get_message_successfully") + assert_equal(new_message_response.status_code,200) + + message_id = json_data_new_message["id"] + + # Get message with id + get_message_url = f"http://localhost:3928/v1/threads/{thread_id}/messages/{message_id}" + get_message_response = requests.get(get_message_url) + json_data_message = get_message_response.json() + log_response(json_data_message, "test_api_get_message_successfully") + assert_equal(get_message_response.status_code,200) + + schema = { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "Unique identifier for the message" + }, + "object": { + "type": "string", + "description": "Type of object, always 'thread.message'" + }, + "created_at": { + "type": "integer", + "description": "Unix timestamp of when the message was created" + }, + "thread_id": { + "type": "string", + "description": "ID of the thread this message belongs to" + }, + "role": { + "type": "string", + "description": "Role of the message sender", + "enum": [ + "assistant", + "user" + ] + }, + "status": { + "type": "string", + "description": "Status of the message", + "enum": [ + "completed" + ] + }, + "content": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of content", + "enum": [ + "text" + ] + }, + "text": { + "type": "object", + "properties": { + "value": { + "type": "string", + "description": "The message text" + }, + "annotations": { + "type": "array", + "description": "Array of annotations for the text" + } + } + } + } + } + }, + "metadata": { + "type": "object", + "description": "Additional metadata for the message" + }, + "attachments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "file_id": { + "type": "string", + "description": "ID of the attached file" + }, + "tools": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Type of tool used" + } + } + } + } + } + } + } + }, + "required": [ + "id", + "object", + "created_at", + "thread_id", + "role", + "content" + ] + } + + # Validate response schema + jsonschema.validate(instance=json_data_message, schema=schema) + + assert_equal(json_data_message["content"][0]['text']["value"], "Hello, world!") \ No newline at end of file diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index 2bf3af09b..4fec5dc78 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -24,6 +24,10 @@ from test_api_post_default_engine import TestApiSetDefaultEngine from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport +from api.message.test_api_get_message import TestApiGetMessage +from api.message.test_api_get_list_message import TestApiGetListMessage +from api.message.test_api_create_message import TestApiCreateMessage +from api.message.test_api_delete_message import TestApiDeleteMessage ### from cli.engines.test_cli_engine_get import TestCliEngineGet diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index 009f4d718..73dd3ad8b 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -24,6 +24,10 @@ from test_api_post_default_engine import TestApiSetDefaultEngine from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport +from api.message.test_api_get_message import TestApiGetMessage +from api.message.test_api_get_list_message import TestApiGetListMessage +from api.message.test_api_create_message import TestApiCreateMessage +from api.message.test_api_delete_message import TestApiDeleteMessage ### from cli.engines.test_cli_engine_get import TestCliEngineGet