diff --git a/engine/e2e-test/api/assistants/test_api_create_assistant.py b/engine/e2e-test/api/assistants/test_api_create_assistant.py new file mode 100644 index 000000000..5168eda04 --- /dev/null +++ b/engine/e2e-test/api/assistants/test_api_create_assistant.py @@ -0,0 +1,165 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiCreateAssistant: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_create_assistant_successfully(self): + headers = { + "Content-Type": "application/json", + } + + data = { + "description": "", + "instructions": "", + "metadata": { + "ANY_ADDITIONAL_PROPERTY": "anything" + }, + "model": "tinyllama:1b", + "name": "test_assistant", + "response_format": "auto", + "temperature": 1, + "tool_resources": { + "code_interpreter": {}, + "file_search": {} + }, + "tools": [ + { + "type": "code_interpreter" + } + ], + "top_p": 1 + } + + post_assistant_url = "http://localhost:3928/v1/assistants" + res=requests.get(post_assistant_url) + log_response(res.text, "test_api_create_assistant_successfully") + + + response = requests.post( + post_assistant_url, headers=headers, json=data + ) + log_response(response.text, "test_api_create_assistant_successfully") + json_data = response.json() + assert_equal(response.status_code,200) + + schema = { + "properties": { + "created_at": { + "description": "Unix timestamp (in seconds) of when the assistant was created.", + "type": "integer" + }, + "description": { + "description": "The description of the assistant.", + "type": "string" + }, + "id": { + "description": "The unique identifier of the assistant.", + "type": "string" + }, + "instructions": { + "description": "Instructions for the assistant's behavior.", + "type": "string" + }, + "metadata": { + "additionalProperties": True, + "description": "Set of key-value pairs that can be attached to the assistant.", + "type": "object" + }, + "model": { + "description": "The model identifier used by the assistant.", + "type": "string" + }, + "name": { + "description": "The name of the assistant.", + "type": "string" + }, + "object": { + "description": "The object type, which is always 'assistant'.", + "enum": [ + "assistant" + ], + "type": "string" + }, + "response_format": { + "oneOf": [ + { + "enum": [ + "auto" + ], + "type": "string" + }, + { + "type": "object" + } + ] + }, + "temperature": { + "description": "Temperature parameter for response generation.", + "format": "float", + "type": "number" + }, + "tool_resources": { + "description": "Resources used by the assistant's tools.", + "properties": { + "code_interpreter": { + "type": "object" + }, + "file_search": { + "type": "object" + } + }, + "type": "object" + }, + "tools": { + "description": "A list of tools enabled on the assistant.", + "items": { + "properties": { + "type": { + "enum": [ + "code_interpreter", + "file_search", + "function" + ], + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "top_p": { + "description": "Top p parameter for response generation.", + "format": "float", + "type": "number" + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ], + "type": "object" + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + \ No newline at end of file diff --git a/engine/e2e-test/api/assistants/test_api_delete_assistant.py b/engine/e2e-test/api/assistants/test_api_delete_assistant.py new file mode 100644 index 000000000..e45529a4d --- /dev/null +++ b/engine/e2e-test/api/assistants/test_api_delete_assistant.py @@ -0,0 +1,95 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiDeleteAssistant: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_delete_assistant_successfully(self): + headers = { + "Content-Type": "application/json", + } + + data = { + "description": "", + "instructions": "", + "metadata": { + "ANY_ADDITIONAL_PROPERTY": "anything" + }, + "model": "tinyllama:1b", + "name": "test_assistant", + "response_format": "auto", + "temperature": 1, + "tool_resources": { + "code_interpreter": {}, + "file_search": {} + }, + "tools": [ + { + "type": "code_interpreter" + } + ], + "top_p": 1 + } + + assistant_url = "http://localhost:3928/v1/assistants" + response = requests.post( + assistant_url, headers=headers, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_delete_assistant_successfully") + assert_equal(response.status_code,200) + + assistant_id=json_data["id"] + + # Get list assistant + assistantid_url=f"http://localhost:3928/v1/assistants/{assistant_id}" + response_del_assistant = requests.delete( + assistantid_url + ) + json_data_del_assistant = response_del_assistant.json() + log_response(json_data_del_assistant, "test_api_delete_assistant_successfully") + assert_equal(response_del_assistant.status_code,200) + + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "object": { + "type": "string", + "enum": ["assistant.deleted"] + } + }, + "required": ["deleted", "id", "object"] + } + + + # Validate response schema + jsonschema.validate(instance=json_data_del_assistant, schema=schema) + + # Assert content + assert_equal(json_data_del_assistant["deleted"], True) + assert_equal(json_data_del_assistant["id"], assistant_id) + assert_equal(json_data_del_assistant["object"], "assistant.deleted") + \ No newline at end of file diff --git a/engine/e2e-test/api/assistants/test_api_get_assistant.py b/engine/e2e-test/api/assistants/test_api_get_assistant.py new file mode 100644 index 000000000..d90d77a58 --- /dev/null +++ b/engine/e2e-test/api/assistants/test_api_get_assistant.py @@ -0,0 +1,112 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiGetAssistant: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_assistant_successfully(self): + headers = { + "Content-Type": "application/json", + } + + data = { + "description": "", + "instructions": "", + "metadata": { + "ANY_ADDITIONAL_PROPERTY": "anything" + }, + "model": "tinyllama:1b", + "name": "test_assistant", + "response_format": "auto", + "temperature": 1, + "tool_resources": { + "code_interpreter": {}, + "file_search": {} + }, + "tools": [ + { + "type": "code_interpreter" + } + ], + "top_p": 1 + } + + assistant_url = "http://localhost:3928/v1/assistants" + response = requests.post( + assistant_url, headers=headers, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_get_assistant_successfully") + assert_equal(response.status_code,200) + + assistant_id=json_data["id"] + + # Get list assistant + headers = { + "OpenAI-Beta": "assistants=v2" + } + assistantid_url=f"http://localhost:3928/v1/assistants/{assistant_id}" + response_assistant = requests.get( + assistantid_url, headers= headers + ) + json_data_assistant = response_assistant.json() + log_response(json_data_assistant, "test_api_get_assistant_successfully") + assert_equal(response_assistant.status_code,200) + + schema = { + "properties": { + "created_at": { + "description": "Unix timestamp (in seconds) of when the assistant was created.", + "type": "integer" + }, + "id": { + "description": "The unique identifier of the assistant.", + "type": "string" + }, + "metadata": { + "additionalProperties": True, + "description": "Set of key-value pairs attached to the assistant.", + "type": "object" + }, + "model": { + "description": "The model identifier used by the assistant.", + "type": "string" + }, + "object": { + "description": "The object type, which is always 'assistant'.", + "enum": [ + "assistant" + ], + "type": "string" + } + }, + "required": [ + "id", + "object", + "created_at", + "model", + "metadata" + ], + "type": "object" + } + + # Validate response schema + jsonschema.validate(instance=json_data_assistant, schema=schema) + assert_equal(json_data_assistant["id"], assistant_id) + \ No newline at end of file diff --git a/engine/e2e-test/api/assistants/test_api_get_list_assistant.py b/engine/e2e-test/api/assistants/test_api_get_list_assistant.py new file mode 100644 index 000000000..4115bbe8c --- /dev/null +++ b/engine/e2e-test/api/assistants/test_api_get_list_assistant.py @@ -0,0 +1,167 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiGetListAssistant: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_list_assistant_successfully(self): + headers = { + "Content-Type": "application/json", + } + + data = { + "description": "", + "instructions": "", + "metadata": { + "ANY_ADDITIONAL_PROPERTY": "anything" + }, + "model": "tinyllama:1b", + "name": "test_assistant", + "response_format": "auto", + "temperature": 1, + "tool_resources": { + "code_interpreter": {}, + "file_search": {} + }, + "tools": [ + { + "type": "code_interpreter" + } + ], + "top_p": 1 + } + + assistant_url = "http://localhost:3928/v1/assistants" + response = requests.post( + assistant_url, headers=headers, json=data + ) + json_data = response.json() + log_response(json_data, "test_api_get_list_assistant_successfully") + assert_equal(response.status_code,200) + + # Get list assistant + response_list_assistant = requests.get( + assistant_url + ) + json_data_list_assistant = response_list_assistant.json() + log_response(json_data_list_assistant, "test_api_get_list_assistant_successfully") + assert_equal(response_list_assistant.status_code,200) + + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "object": { + "type": "string", + "enum": ["list"] + }, + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "created_at": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "id": { + "type": "string" + }, + "instructions": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "model": { + "type": "string" + }, + "name": { + "type": "string" + }, + "object": { + "type": "string", + "enum": ["assistant"] + }, + "temperature": { + "type": "number" + }, + "tool_resources": { + "type": "object", + "properties": { + "file_search": { + "type": "object", + "properties": { + "vector_store_ids": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["vector_store_ids"] + } + }, + "required": ["file_search"] + }, + "tools": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["code_interpreter"] + } + }, + "required": ["type"] + } + }, + "top_p": { + "type": "number" + } + }, + "required": [ + "created_at", + "description", + "id", + "instructions", + "metadata", + "model", + "name", + "object", + "temperature", + "tool_resources", + "tools", + "top_p" + ] + } + } + }, + "required": ["object", "data"] + } + + # Validate response schema + jsonschema.validate(instance=json_data_list_assistant, schema=schema) + \ No newline at end of file diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index e78baf951..dcd3bccc9 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -36,6 +36,10 @@ from api.thread.test_api_delete_thread import TestApiDeleteThread from api.thread.test_api_get_thread import TestApiGetThread from api.thread.test_api_get_list_thread import TestApiGetListThread +from api.assistants.test_api_create_assistant import TestApiCreateAssistant +from api.assistants.test_api_get_list_assistant import TestApiGetListAssistant +from api.assistants.test_api_get_assistant import TestApiGetAssistant +from api.assistants.test_api_delete_assistant import TestApiDeleteAssistant ### from cli.engines.test_cli_engine_get import TestCliEngineGet diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index 3327625d2..fa8c97632 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -36,6 +36,11 @@ from api.thread.test_api_delete_thread import TestApiDeleteThread from api.thread.test_api_get_thread import TestApiGetThread from api.thread.test_api_get_list_thread import TestApiGetListThread +from api.assistants.test_api_create_assistant import TestApiCreateAssistant +from api.assistants.test_api_get_list_assistant import TestApiGetListAssistant +from api.assistants.test_api_get_assistant import TestApiGetAssistant +from api.assistants.test_api_delete_assistant import TestApiDeleteAssistant + ### from cli.engines.test_cli_engine_get import TestCliEngineGet