From 6518c54e2f0721c6724160ef4e90dcc14496cd3d Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sat, 15 Feb 2025 23:35:29 +0700 Subject: [PATCH 01/13] test: try 1 --- .github/workflows/cortex-cpp-quality-gate.yml | 118 +++++----- engine/e2e-test/main-harry.py | 6 + engine/e2e-test/requirements.txt | 3 +- engine/e2e-test/test_api_engine.py | 212 +++++++++++++----- 4 files changed, 220 insertions(+), 119 deletions(-) create mode 100644 engine/e2e-test/main-harry.py diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 8a76e4669..ec8e1eae2 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -1,12 +1,12 @@ name: CI Quality Gate Cortex CPP on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - paths: ["engine/**", ".github/workflows/cortex-cpp-quality-gate.yml"] + # pull_request: + # types: [opened, synchronize, reopened, ready_for_review] + # paths: ["engine/**", ".github/workflows/cortex-cpp-quality-gate.yml"] workflow_dispatch: - schedule: - - cron: '0 22 * * *' + # schedule: + # - cron: '0 22 * * *' env: LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf @@ -131,7 +131,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/main-harry rm build/cortex-nightly rm build/cortex-beta env: @@ -145,7 +145,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/main-harry rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -159,7 +159,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/main-harry.py rm build/cortex-nightly rm build/cortex-beta env: @@ -173,7 +173,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/main-harry.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -218,58 +218,58 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}" AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" - build-docker-and-test: - runs-on: ubuntu-24-04-docker - steps: - - name: Getting the repo - uses: actions/checkout@v3 - with: - submodules: 'recursive' + # build-docker-and-test: + # runs-on: ubuntu-24-04-docker + # steps: + # - name: Getting the repo + # uses: actions/checkout@v3 + # with: + # submodules: 'recursive' - - name: Run Docker - if: github.event_name != 'schedule' - run: | - docker build \ - --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ - --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ - --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ - --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ - -t menloltd/cortex:test -f docker/Dockerfile.cache . - docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test - sleep 20 + # - name: Run Docker + # if: github.event_name != 'schedule' + # run: | + # docker build \ + # --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ + # --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ + # --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ + # --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ + # -t menloltd/cortex:test -f docker/Dockerfile.cache . + # docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test + # sleep 20 - - name: Run Docker - if: github.event_name == 'schedule' - run: | - latest_prerelease=$(curl -s https://api.github.com/repos/cortexcpp/cortex.cpp/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1) - echo "cortex.llamacpp latest release: $latest_prerelease" - docker build \ - --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ - --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ - --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ - --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ - --build-arg CORTEX_CPP_VERSION="${latest_prerelease}" \ - -t menloltd/cortex:test -f docker/Dockerfile.cache . - docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test - sleep 20 + # - name: Run Docker + # if: github.event_name == 'schedule' + # run: | + # latest_prerelease=$(curl -s https://api.github.com/repos/cortexcpp/cortex.cpp/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1) + # echo "cortex.llamacpp latest release: $latest_prerelease" + # docker build \ + # --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ + # --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ + # --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ + # --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ + # --build-arg CORTEX_CPP_VERSION="${latest_prerelease}" \ + # -t menloltd/cortex:test -f docker/Dockerfile.cache . + # docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test + # sleep 20 - - name: use python - uses: actions/setup-python@v5 - with: - python-version: "3.10" + # - name: use python + # uses: actions/setup-python@v5 + # with: + # python-version: "3.10" - - name: Run e2e tests - run: | - cd engine - python -m pip install --upgrade pip - python -m pip install -r e2e-test/requirements.txt - pytest e2e-test/test_api_docker.py + # - name: Run e2e tests + # run: | + # cd engine + # python -m pip install --upgrade pip + # python -m pip install -r e2e-test/requirements.txt + # pytest e2e-test/test_api_docker.py - - name: Run Docker - continue-on-error: true - if: always() - run: | - docker logs cortex - docker stop cortex - docker rm cortex - echo "y\n" | docker system prune -af + # - name: Run Docker + # continue-on-error: true + # if: always() + # run: | + # docker logs cortex + # docker stop cortex + # docker rm cortex + # echo "y\n" | docker system prune -af diff --git a/engine/e2e-test/main-harry.py b/engine/e2e-test/main-harry.py new file mode 100644 index 000000000..a7dcf0d36 --- /dev/null +++ b/engine/e2e-test/main-harry.py @@ -0,0 +1,6 @@ +import pytest +import sys +from test_api_engine import TestApiEngine + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/requirements.txt b/engine/e2e-test/requirements.txt index 05b47e0b0..d9c436a11 100644 --- a/engine/e2e-test/requirements.txt +++ b/engine/e2e-test/requirements.txt @@ -2,4 +2,5 @@ websockets pytest pytest-asyncio requests -pyyaml \ No newline at end of file +pyyaml +jsonschema \ No newline at end of file diff --git a/engine/e2e-test/test_api_engine.py b/engine/e2e-test/test_api_engine.py index 57b47b879..d2d50afe6 100644 --- a/engine/e2e-test/test_api_engine.py +++ b/engine/e2e-test/test_api_engine.py @@ -6,6 +6,11 @@ stop_server, wait_for_websocket_download_success_event, ) +import json +import jsonschema + +# logging.basicConfig(level=logging.INFO, force=True) # Ensure logs show +# logger = logging.getLogger(__name__) class TestApiEngine: @@ -21,80 +26,169 @@ def setup_and_teardown(self): # Teardown stop_server() + + + # # engines install + # def test_engines_install_llamacpp_specific_version_and_variant(self): + # data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} + # response = requests.post( + # "http://localhost:3928/v1/engines/llama-cpp/install", json=data + # ) + # assert response.status_code == 200 + # with open("response-install.json", "w") as file: + # json.dump(response.json(), file, indent=4) + # engines get def test_engines_get_llamacpp_should_be_successful(self): - response = requests.get("http://localhost:3928/engines/llama-cpp") - assert response.status_code == 200 - - # engines install - def test_engines_install_llamacpp_specific_version_and_variant(self): + engine= "llama-cpp" + name= "linux-amd64-avx-cuda-11-7" + version= "v0.1.35-27.10.24" + + # data = {"version": version, "variant": name} data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} + post_url = f"http://localhost:3928/v1/engines/{engine}/install" response = requests.post( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data + post_url, json=data ) assert response.status_code == 200 + + get_url = f"http://localhost:3928/v1/engines/{engine}" + response = requests.get(get_url) + count = 0 + while len(response.json()) == 0: + time.sleep(1) + response = requests.get(get_url) + count += 1 - def test_engines_install_llamacpp_specific_version_and_null_variant(self): - data = {"version": "v0.1.35-27.10.24"} - response = requests.post( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) - assert response.status_code == 200 - - # engines uninstall - @pytest.mark.asyncio - async def test_engines_install_uninstall_llamacpp_should_be_successful(self): - response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") + json_data = response.json() + with open("e2e-test/response.json", "w") as file: + json.dump(json_data, file, indent=4) + # file.write(f"\nCount: {count}\n") assert response.status_code == 200 - await wait_for_websocket_download_success_event(timeout=None) - time.sleep(30) - response = requests.delete("http://localhost:3928/v1/engines/llama-cpp/install") - assert response.status_code == 200 + schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "engine": {"type": "string"}, + "name": {"type": "string"}, + "version": {"type": "string"} + }, + "required": ["engine", "name", "version"] + } + } - @pytest.mark.asyncio - async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self): - # install first - data = {"variant": "mac-arm64"} - install_response = requests.post( - "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data - ) - await wait_for_websocket_download_success_event(timeout=120) - assert install_response.status_code == 200 + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) - data = {"version": "v0.1.35"} - response = requests.delete( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) - assert response.status_code == 400 - assert response.json()["message"] == "No variant provided" - - @pytest.mark.asyncio - async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self): - # install first - data = {"variant": "mac-arm64"} - install_response = requests.post( - "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data + assert json_data[0]["engine"] == engine + assert json_data[0]["version"] == version + assert json_data[0]["name"] == name + + delete_url = f"http://localhost:3928/v1/engines/{engine}/install" + delete_response = requests.delete( + delete_url, json=data ) - await wait_for_websocket_download_success_event(timeout=120) - assert install_response.status_code == 200 + with open("e2e-test/response_engine_uninstall.json", "w") as file: + json.dump(delete_response.json(), file, indent=4) + assert delete_response.status_code ==200 + assert delete_response.json()["message"] == "Engine llama-cpp uninstalled successfully!" - response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install") - assert response.status_code == 200 + get_url = f"http://localhost:3928/v1/engines/{engine}" + get_response = requests.get(get_url) + assert len(get_response.json()) == 0 - def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( - self, - ): - data = {"variant": "mac-arm64", "version": "v0.1.35"} - # install first - install_response = requests.post( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) - assert install_response.status_code == 200 - response = requests.delete( - "http://localhost:3928/v1/engines/llama-cpp/install", json=data - ) + def test_engines_get_llamacpp_release_list(self): + engine= "llama-cpp" + + get_url = f"http://localhost:3928/v1/engines/{engine}/releases" + response = requests.get(get_url) assert response.status_code == 200 + json_data = response.json() + with open("e2e-test/response_engine_release.json", "w") as file: + json.dump(json_data, file, indent=4) + + schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "draft": {"type": "boolean"}, + "name": {"type": "string"}, + "prerelease": {"type": "boolean"}, + "published_at": {"type": "string", "format": "date-time"}, + "url": {"type": "string", "format": "uri"} + }, + "required": ["draft", "name", "prerelease", "published_at", "url"] + } + } + + jsonschema.validate(instance=json_data, schema=schema) + + # def test_engines_install_llamacpp_specific_version_and_null_variant(self): + # data = {"version": "v0.1.35-27.10.24"} + # response = requests.post( + # "http://localhost:3928/v1/engines/llama-cpp/install", json=data + # ) + # assert response.status_code == 200 + + # # engines uninstall + # @pytest.mark.asyncio + # async def test_engines_install_uninstall_llamacpp_should_be_successful(self): + # response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") + # assert response.status_code == 200 + # await wait_for_websocket_download_success_event(timeout=None) + # time.sleep(30) + + # response = requests.delete("http://localhost:3928/v1/engines/llama-cpp/install") + # assert response.status_code == 200 + + # @pytest.mark.asyncio + # async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self): + # # install first + # data = {"variant": "mac-arm64"} + # install_response = requests.post( + # "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data + # ) + # await wait_for_websocket_download_success_event(timeout=120) + # assert install_response.status_code == 200 + + # data = {"version": "v0.1.35"} + # response = requests.delete( + # "http://localhost:3928/v1/engines/llama-cpp/install", json=data + # ) + # assert response.status_code == 400 + # assert response.json()["message"] == "No variant provided" + + # @pytest.mark.asyncio + # async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self): + # # install first + # data = {"variant": "mac-arm64"} + # install_response = requests.post( + # "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data + # ) + # await wait_for_websocket_download_success_event(timeout=120) + # assert install_response.status_code == 200 + + # response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install") + # assert response.status_code == 200 + + # def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( + # self, + # ): + # data = {"variant": "mac-arm64", "version": "v0.1.35"} + # # install first + # install_response = requests.post( + # "http://localhost:3928/v1/engines/llama-cpp/install", json=data + # ) + # assert install_response.status_code == 200 + + # response = requests.delete( + # "http://localhost:3928/v1/engines/llama-cpp/install", json=data + # ) + # assert response.status_code == 200 + \ No newline at end of file From 66a5cc7fe12d787edf7f984e3ac564e48bec82cc Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sat, 15 Feb 2025 23:54:15 +0700 Subject: [PATCH 02/13] test: try 2 --- .github/workflows/cortex-cpp-quality-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index ec8e1eae2..235eb4119 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -131,7 +131,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry + python e2e-test/main-harry.py rm build/cortex-nightly rm build/cortex-beta env: @@ -145,7 +145,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry + python e2e-test/main-harry.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: From 24c3f2d42a92d3bd43f99e6bfb20d08ff282765e Mon Sep 17 00:00:00 2001 From: Harry Le Date: Thu, 20 Feb 2025 13:07:40 +0700 Subject: [PATCH 03/13] test: reorganize folder --- .../e2e-test/api/engines/test_api_engine.py | 100 +++++++++ .../test_api_engine_install_nightly.py | 2 +- .../{ => api/engines}/test_api_engine_list.py | 2 +- .../engines}/test_api_engine_update.py | 2 +- .../test_api_cortexso_hub_llamacpp_engine.py | 2 +- .../{ => api/model}/test_api_docker.py | 2 +- .../{ => api/model}/test_api_model.py | 2 +- .../{ => api/model}/test_api_model_import.py | 2 +- .../{ => cli/common}/test_cli_server_start.py | 4 +- .../{ => cli/common}/test_cortex_update.py | 2 +- .../common}/test_create_log_folder.py | 4 +- .../{ => cli/engines}/test_cli_engine_get.py | 2 +- .../engines}/test_cli_engine_install.py | 2 +- .../test_cli_engine_install_nightly.py | 2 +- .../{ => cli/engines}/test_cli_engine_list.py | 2 +- .../engines}/test_cli_engine_uninstall.py | 2 +- .../{ => cli/model}/test_cli_model.py | 2 +- .../{ => cli/model}/test_cli_model_import.py | 4 +- ..._cli_model_pull_cortexso_with_selection.py | 2 +- .../test_cli_model_pull_from_cortexso.py | 2 +- ..._cli_model_pull_hugging_face_repository.py | 2 +- .../e2e-test/cortex-llamacpp-e2e-nightly.py | 22 -- engine/e2e-test/main-harry.py | 6 - engine/e2e-test/main.py | 22 -- .../runner/cortex-llamacpp-e2e-nightly.py | 36 ++++ engine/e2e-test/runner/main.py | 36 ++++ engine/e2e-test/test_api_engine.py | 194 ------------------ engine/e2e-test/utils/logger.py | 15 ++ engine/e2e-test/{ => utils}/test_runner.py | 0 29 files changed, 210 insertions(+), 267 deletions(-) create mode 100644 engine/e2e-test/api/engines/test_api_engine.py rename engine/e2e-test/{ => api/engines}/test_api_engine_install_nightly.py (93%) rename engine/e2e-test/{ => api/engines}/test_api_engine_list.py (91%) rename engine/e2e-test/{ => api/engines}/test_api_engine_update.py (97%) rename engine/e2e-test/{ => api/hub}/test_api_cortexso_hub_llamacpp_engine.py (99%) rename engine/e2e-test/{ => api/model}/test_api_docker.py (97%) rename engine/e2e-test/{ => api/model}/test_api_model.py (99%) rename engine/e2e-test/{ => api/model}/test_api_model_import.py (97%) rename engine/e2e-test/{ => cli/common}/test_cli_server_start.py (84%) rename engine/e2e-test/{ => cli/common}/test_cortex_update.py (94%) rename engine/e2e-test/{ => cli/common}/test_create_log_folder.py (90%) rename engine/e2e-test/{ => cli/engines}/test_cli_engine_get.py (97%) rename engine/e2e-test/{ => cli/engines}/test_cli_engine_install.py (98%) rename engine/e2e-test/{ => cli/engines}/test_cli_engine_install_nightly.py (97%) rename engine/e2e-test/{ => cli/engines}/test_cli_engine_list.py (95%) rename engine/e2e-test/{ => cli/engines}/test_cli_engine_uninstall.py (96%) rename engine/e2e-test/{ => cli/model}/test_cli_model.py (98%) rename engine/e2e-test/{ => cli/model}/test_cli_model_import.py (89%) rename engine/e2e-test/{ => cli/model}/test_cli_model_pull_cortexso_with_selection.py (94%) rename engine/e2e-test/{ => cli/model}/test_cli_model_pull_from_cortexso.py (95%) rename engine/e2e-test/{ => cli/model}/test_cli_model_pull_hugging_face_repository.py (96%) delete mode 100644 engine/e2e-test/cortex-llamacpp-e2e-nightly.py delete mode 100644 engine/e2e-test/main-harry.py delete mode 100644 engine/e2e-test/main.py create mode 100644 engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py create mode 100644 engine/e2e-test/runner/main.py delete mode 100644 engine/e2e-test/test_api_engine.py create mode 100644 engine/e2e-test/utils/logger.py rename engine/e2e-test/{ => utils}/test_runner.py (100%) diff --git a/engine/e2e-test/api/engines/test_api_engine.py b/engine/e2e-test/api/engines/test_api_engine.py new file mode 100644 index 000000000..aa491caf7 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_engine.py @@ -0,0 +1,100 @@ +import pytest +import requests +import time +from utils.test_runner import ( + start_server, + stop_server, + wait_for_websocket_download_success_event, +) + +class TestApiEngine: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + # engines get + def test_engines_get_llamacpp_should_be_successful(self): + response = requests.get("http://localhost:3928/engines/llama-cpp") + assert response.status_code == 200 + + # engines install + def test_engines_install_llamacpp_specific_version_and_variant(self): + data = {"version": "v0.1.40-b4354", "variant": "linux-amd64-avx-cuda-11-7"} + response = requests.post( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 200 + + def test_engines_install_llamacpp_specific_version_and_null_variant(self): + data = {"version": "v0.1.40-b4354"} + response = requests.post( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 200 + + # engines uninstall + @pytest.mark.asyncio + async def test_engines_install_uninstall_llamacpp_should_be_successful(self): + response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") + assert response.status_code == 200 + await wait_for_websocket_download_success_event(timeout=None) + time.sleep(30) + + response = requests.delete("http://localhost:3928/v1/engines/llama-cpp/install") + assert response.status_code == 200 + + @pytest.mark.asyncio + async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self): + # install first + data = {"variant": "mac-arm64"} + install_response = requests.post( + "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data + ) + await wait_for_websocket_download_success_event(timeout=120) + assert install_response.status_code == 200 + + data = {"version": "v0.1.35"} + response = requests.delete( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 400 + assert response.json()["message"] == "No variant provided" + + @pytest.mark.asyncio + async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self): + # install first + data = {"variant": "mac-arm64"} + install_response = requests.post( + "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data + ) + await wait_for_websocket_download_success_event(timeout=120) + assert install_response.status_code == 200 + + response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install") + assert response.status_code == 200 + + def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( + self, + ): + data = {"variant": "mac-arm64", "version": "v0.1.35"} + # install first + install_response = requests.post( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert install_response.status_code == 200 + + response = requests.delete( + "http://localhost:3928/v1/engines/llama-cpp/install", json=data + ) + assert response.status_code == 200 + + \ No newline at end of file diff --git a/engine/e2e-test/test_api_engine_install_nightly.py b/engine/e2e-test/api/engines/test_api_engine_install_nightly.py similarity index 93% rename from engine/e2e-test/test_api_engine_install_nightly.py rename to engine/e2e-test/api/engines/test_api_engine_install_nightly.py index de4914c28..34fda2d18 100644 --- a/engine/e2e-test/test_api_engine_install_nightly.py +++ b/engine/e2e-test/api/engines/test_api_engine_install_nightly.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import start_server, stop_server, get_latest_pre_release_tag +from utils.test_runner import start_server, stop_server, get_latest_pre_release_tag latest_pre_release_tag = get_latest_pre_release_tag("janhq", "cortex.llamacpp") diff --git a/engine/e2e-test/test_api_engine_list.py b/engine/e2e-test/api/engines/test_api_engine_list.py similarity index 91% rename from engine/e2e-test/test_api_engine_list.py rename to engine/e2e-test/api/engines/test_api_engine_list.py index f149f1450..10346c988 100644 --- a/engine/e2e-test/test_api_engine_list.py +++ b/engine/e2e-test/api/engines/test_api_engine_list.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import start_server, stop_server +from utils.test_runner import start_server, stop_server class TestApiEngineList: diff --git a/engine/e2e-test/test_api_engine_update.py b/engine/e2e-test/api/engines/test_api_engine_update.py similarity index 97% rename from engine/e2e-test/test_api_engine_update.py rename to engine/e2e-test/api/engines/test_api_engine_update.py index 23939f038..bcc371406 100644 --- a/engine/e2e-test/test_api_engine_update.py +++ b/engine/e2e-test/api/engines/test_api_engine_update.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import ( +from utils.test_runner import ( start_server, stop_server, wait_for_websocket_download_success_event, diff --git a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py b/engine/e2e-test/api/hub/test_api_cortexso_hub_llamacpp_engine.py similarity index 99% rename from engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py rename to engine/e2e-test/api/hub/test_api_cortexso_hub_llamacpp_engine.py index 9aecd3654..7a3c2e232 100644 --- a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py +++ b/engine/e2e-test/api/hub/test_api_cortexso_hub_llamacpp_engine.py @@ -4,7 +4,7 @@ import yaml from pathlib import Path -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_api_docker.py b/engine/e2e-test/api/model/test_api_docker.py similarity index 97% rename from engine/e2e-test/test_api_docker.py rename to engine/e2e-test/api/model/test_api_docker.py index b46b1f782..7276f2078 100644 --- a/engine/e2e-test/test_api_docker.py +++ b/engine/e2e-test/api/model/test_api_docker.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import wait_for_websocket_download_success_event +from utils.test_runner import wait_for_websocket_download_success_event repo_branches = ["tinyllama:1b-gguf"] diff --git a/engine/e2e-test/test_api_model.py b/engine/e2e-test/api/model/test_api_model.py similarity index 99% rename from engine/e2e-test/test_api_model.py rename to engine/e2e-test/api/model/test_api_model.py index 8f2e4b07a..dd3a0fc65 100644 --- a/engine/e2e-test/test_api_model.py +++ b/engine/e2e-test/api/model/test_api_model.py @@ -1,7 +1,7 @@ import pytest import requests import time -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_api_model_import.py b/engine/e2e-test/api/model/test_api_model_import.py similarity index 97% rename from engine/e2e-test/test_api_model_import.py rename to engine/e2e-test/api/model/test_api_model_import.py index 7efbd52da..46a22e59e 100644 --- a/engine/e2e-test/test_api_model_import.py +++ b/engine/e2e-test/api/model/test_api_model_import.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import start_server, stop_server +from utils.test_runner import start_server, stop_server class TestApiModelImport: @pytest.fixture(autouse=True) diff --git a/engine/e2e-test/test_cli_server_start.py b/engine/e2e-test/cli/common/test_cli_server_start.py similarity index 84% rename from engine/e2e-test/test_cli_server_start.py rename to engine/e2e-test/cli/common/test_cli_server_start.py index abe236cd0..549191458 100644 --- a/engine/e2e-test/test_cli_server_start.py +++ b/engine/e2e-test/cli/common/test_cli_server_start.py @@ -1,8 +1,8 @@ import platform import os import pytest, requests -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCliServerStart: diff --git a/engine/e2e-test/test_cortex_update.py b/engine/e2e-test/cli/common/test_cortex_update.py similarity index 94% rename from engine/e2e-test/test_cortex_update.py rename to engine/e2e-test/cli/common/test_cortex_update.py index 8f6f8d7f8..7bcbdba5c 100644 --- a/engine/e2e-test/test_cortex_update.py +++ b/engine/e2e-test/cli/common/test_cortex_update.py @@ -2,7 +2,7 @@ import tempfile import pytest -from test_runner import run +from utils.test_runner import run class TestCortexUpdate: diff --git a/engine/e2e-test/test_create_log_folder.py b/engine/e2e-test/cli/common/test_create_log_folder.py similarity index 90% rename from engine/e2e-test/test_create_log_folder.py rename to engine/e2e-test/cli/common/test_create_log_folder.py index 5dbbd521c..7ad6ee452 100644 --- a/engine/e2e-test/test_create_log_folder.py +++ b/engine/e2e-test/cli/common/test_create_log_folder.py @@ -2,8 +2,8 @@ import os from pathlib import Path import pytest, requests, shutil -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCreateLogFolder: diff --git a/engine/e2e-test/test_cli_engine_get.py b/engine/e2e-test/cli/engines/test_cli_engine_get.py similarity index 97% rename from engine/e2e-test/test_cli_engine_get.py rename to engine/e2e-test/cli/engines/test_cli_engine_get.py index c26bedfae..b17ea7cc8 100644 --- a/engine/e2e-test/test_cli_engine_get.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_get.py @@ -1,7 +1,7 @@ import platform import pytest -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineGet: diff --git a/engine/e2e-test/test_cli_engine_install.py b/engine/e2e-test/cli/engines/test_cli_engine_install.py similarity index 98% rename from engine/e2e-test/test_cli_engine_install.py rename to engine/e2e-test/cli/engines/test_cli_engine_install.py index a998f3183..3ffe659ea 100644 --- a/engine/e2e-test/test_cli_engine_install.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_install.py @@ -3,7 +3,7 @@ import pytest import requests -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineInstall: diff --git a/engine/e2e-test/test_cli_engine_install_nightly.py b/engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py similarity index 97% rename from engine/e2e-test/test_cli_engine_install_nightly.py rename to engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py index 8c66c284c..e4e46ecfb 100644 --- a/engine/e2e-test/test_cli_engine_install_nightly.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_install_nightly.py @@ -3,7 +3,7 @@ import pytest import requests -from test_runner import run, start_server, stop_server, get_latest_pre_release_tag +from utils.test_runner import run, start_server, stop_server, get_latest_pre_release_tag latest_pre_release_tag = get_latest_pre_release_tag("janhq", "cortex.llamacpp") diff --git a/engine/e2e-test/test_cli_engine_list.py b/engine/e2e-test/cli/engines/test_cli_engine_list.py similarity index 95% rename from engine/e2e-test/test_cli_engine_list.py rename to engine/e2e-test/cli/engines/test_cli_engine_list.py index 6a79bb449..b2d59e855 100644 --- a/engine/e2e-test/test_cli_engine_list.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_list.py @@ -1,7 +1,7 @@ import platform import pytest -from test_runner import run, start_server, stop_server +from utils.test_runner import run, start_server, stop_server class TestCliEngineList: diff --git a/engine/e2e-test/test_cli_engine_uninstall.py b/engine/e2e-test/cli/engines/test_cli_engine_uninstall.py similarity index 96% rename from engine/e2e-test/test_cli_engine_uninstall.py rename to engine/e2e-test/cli/engines/test_cli_engine_uninstall.py index fcc5f5c73..a33a2d5cc 100644 --- a/engine/e2e-test/test_cli_engine_uninstall.py +++ b/engine/e2e-test/cli/engines/test_cli_engine_uninstall.py @@ -1,6 +1,6 @@ import pytest import requests -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_cli_model.py b/engine/e2e-test/cli/model/test_cli_model.py similarity index 98% rename from engine/e2e-test/test_cli_model.py rename to engine/e2e-test/cli/model/test_cli_model.py index f6aad4ae9..63261c214 100644 --- a/engine/e2e-test/test_cli_model.py +++ b/engine/e2e-test/cli/model/test_cli_model.py @@ -2,7 +2,7 @@ import requests import os from pathlib import Path -from test_runner import ( +from utils.test_runner import ( run, start_server, stop_server, diff --git a/engine/e2e-test/test_cli_model_import.py b/engine/e2e-test/cli/model/test_cli_model_import.py similarity index 89% rename from engine/e2e-test/test_cli_model_import.py rename to engine/e2e-test/cli/model/test_cli_model_import.py index cf94d1a2a..e2c5010a4 100644 --- a/engine/e2e-test/test_cli_model_import.py +++ b/engine/e2e-test/cli/model/test_cli_model_import.py @@ -1,6 +1,6 @@ import pytest -from test_runner import run -from test_runner import start_server, stop_server +from utils.test_runner import run +from utils.test_runner import start_server, stop_server class TestCliModelImport: diff --git a/engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py b/engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py similarity index 94% rename from engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py rename to engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py index 8c3de8d98..d6a78843b 100644 --- a/engine/e2e-test/test_cli_model_pull_cortexso_with_selection.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_cortexso_with_selection.py @@ -1,4 +1,4 @@ -from test_runner import popen +from utils.test_runner import popen import os from pathlib import Path diff --git a/engine/e2e-test/test_cli_model_pull_from_cortexso.py b/engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py similarity index 95% rename from engine/e2e-test/test_cli_model_pull_from_cortexso.py rename to engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py index 1791e39a6..2a66ae19e 100644 --- a/engine/e2e-test/test_cli_model_pull_from_cortexso.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_from_cortexso.py @@ -1,5 +1,5 @@ import pytest -from test_runner import run +from utils.test_runner import run class TestCliModelPullCortexso: diff --git a/engine/e2e-test/test_cli_model_pull_hugging_face_repository.py b/engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py similarity index 96% rename from engine/e2e-test/test_cli_model_pull_hugging_face_repository.py rename to engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py index 996ac086c..49678ee7a 100644 --- a/engine/e2e-test/test_cli_model_pull_hugging_face_repository.py +++ b/engine/e2e-test/cli/model/test_cli_model_pull_hugging_face_repository.py @@ -1,5 +1,5 @@ import pytest -from test_runner import popen +from utils.test_runner import popen class TestCliModelPullHuggingFaceRepository: diff --git a/engine/e2e-test/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/cortex-llamacpp-e2e-nightly.py deleted file mode 100644 index 0511277f3..000000000 --- a/engine/e2e-test/cortex-llamacpp-e2e-nightly.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import sys - -### e2e tests are expensive, have to keep engines tests in order -from test_api_engine_list import TestApiEngineList -from test_api_engine_install_nightly import TestApiEngineInstall -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport - -### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install_nightly import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder -from test_cli_model_import import TestCliModelImport - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/main-harry.py b/engine/e2e-test/main-harry.py deleted file mode 100644 index a7dcf0d36..000000000 --- a/engine/e2e-test/main-harry.py +++ /dev/null @@ -1,6 +0,0 @@ -import pytest -import sys -from test_api_engine import TestApiEngine - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/main.py b/engine/e2e-test/main.py deleted file mode 100644 index e874ab3a0..000000000 --- a/engine/e2e-test/main.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest -import sys - -### e2e tests are expensive, have to keep engines tests in order -from test_api_engine_list import TestApiEngineList -from test_api_engine import TestApiEngine -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport - -### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder -from test_cli_model_import import TestCliModelImport - -if __name__ == "__main__": - sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py new file mode 100644 index 000000000..193d26d36 --- /dev/null +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -0,0 +1,36 @@ +import pytest +import sys +import os + +# Add the project root to sys.path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +# Add all necessary paths +sys.path.append(PROJECT_ROOT) +sys.path.append(os.path.join(PROJECT_ROOT, "api/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/hub")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) + +### e2e tests are expensive, have to keep engines tests in order +from api.engines.test_api_engine_list import TestApiEngineList +from api.engines.test_api_engine_install_nightly import TestApiEngineInstall +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport + +### +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install_nightly import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py new file mode 100644 index 000000000..7a96f1164 --- /dev/null +++ b/engine/e2e-test/runner/main.py @@ -0,0 +1,36 @@ +import pytest +import sys +import os + +# Add the project root to sys.path +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) + + +# Add all necessary paths +sys.path.append(PROJECT_ROOT) +sys.path.append(os.path.join(PROJECT_ROOT, "api/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/hub")) +sys.path.append(os.path.join(PROJECT_ROOT, "api/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/engines")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/model")) +sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) + +### e2e tests are expensive, have to keep engines tests in order +from api.engines.test_api_engine_list import TestApiEngineList +from api.engines.test_api_engine import TestApiEngine +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport + +### +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/test_api_engine.py b/engine/e2e-test/test_api_engine.py deleted file mode 100644 index d2d50afe6..000000000 --- a/engine/e2e-test/test_api_engine.py +++ /dev/null @@ -1,194 +0,0 @@ -import pytest -import requests -import time -from test_runner import ( - start_server, - stop_server, - wait_for_websocket_download_success_event, -) -import json -import jsonschema - -# logging.basicConfig(level=logging.INFO, force=True) # Ensure logs show -# logger = logging.getLogger(__name__) - -class TestApiEngine: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - - - # # engines install - # def test_engines_install_llamacpp_specific_version_and_variant(self): - # data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} - # response = requests.post( - # "http://localhost:3928/v1/engines/llama-cpp/install", json=data - # ) - # assert response.status_code == 200 - # with open("response-install.json", "w") as file: - # json.dump(response.json(), file, indent=4) - - # engines get - def test_engines_get_llamacpp_should_be_successful(self): - engine= "llama-cpp" - name= "linux-amd64-avx-cuda-11-7" - version= "v0.1.35-27.10.24" - - # data = {"version": version, "variant": name} - data = {"version": "v0.1.35-27.10.24", "variant": "linux-amd64-avx-cuda-11-7"} - post_url = f"http://localhost:3928/v1/engines/{engine}/install" - response = requests.post( - post_url, json=data - ) - assert response.status_code == 200 - - get_url = f"http://localhost:3928/v1/engines/{engine}" - response = requests.get(get_url) - count = 0 - while len(response.json()) == 0: - time.sleep(1) - response = requests.get(get_url) - count += 1 - - json_data = response.json() - with open("e2e-test/response.json", "w") as file: - json.dump(json_data, file, indent=4) - # file.write(f"\nCount: {count}\n") - assert response.status_code == 200 - - schema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "engine": {"type": "string"}, - "name": {"type": "string"}, - "version": {"type": "string"} - }, - "required": ["engine", "name", "version"] - } - } - - # Validate response schema - jsonschema.validate(instance=json_data, schema=schema) - - assert json_data[0]["engine"] == engine - assert json_data[0]["version"] == version - assert json_data[0]["name"] == name - - delete_url = f"http://localhost:3928/v1/engines/{engine}/install" - delete_response = requests.delete( - delete_url, json=data - ) - with open("e2e-test/response_engine_uninstall.json", "w") as file: - json.dump(delete_response.json(), file, indent=4) - assert delete_response.status_code ==200 - assert delete_response.json()["message"] == "Engine llama-cpp uninstalled successfully!" - - get_url = f"http://localhost:3928/v1/engines/{engine}" - get_response = requests.get(get_url) - assert len(get_response.json()) == 0 - - - def test_engines_get_llamacpp_release_list(self): - engine= "llama-cpp" - - get_url = f"http://localhost:3928/v1/engines/{engine}/releases" - response = requests.get(get_url) - assert response.status_code == 200 - - json_data = response.json() - with open("e2e-test/response_engine_release.json", "w") as file: - json.dump(json_data, file, indent=4) - - schema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "draft": {"type": "boolean"}, - "name": {"type": "string"}, - "prerelease": {"type": "boolean"}, - "published_at": {"type": "string", "format": "date-time"}, - "url": {"type": "string", "format": "uri"} - }, - "required": ["draft", "name", "prerelease", "published_at", "url"] - } - } - - jsonschema.validate(instance=json_data, schema=schema) - - # def test_engines_install_llamacpp_specific_version_and_null_variant(self): - # data = {"version": "v0.1.35-27.10.24"} - # response = requests.post( - # "http://localhost:3928/v1/engines/llama-cpp/install", json=data - # ) - # assert response.status_code == 200 - - # # engines uninstall - # @pytest.mark.asyncio - # async def test_engines_install_uninstall_llamacpp_should_be_successful(self): - # response = requests.post("http://localhost:3928/v1/engines/llama-cpp/install") - # assert response.status_code == 200 - # await wait_for_websocket_download_success_event(timeout=None) - # time.sleep(30) - - # response = requests.delete("http://localhost:3928/v1/engines/llama-cpp/install") - # assert response.status_code == 200 - - # @pytest.mark.asyncio - # async def test_engines_install_uninstall_llamacpp_with_only_version_should_be_failed(self): - # # install first - # data = {"variant": "mac-arm64"} - # install_response = requests.post( - # "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data - # ) - # await wait_for_websocket_download_success_event(timeout=120) - # assert install_response.status_code == 200 - - # data = {"version": "v0.1.35"} - # response = requests.delete( - # "http://localhost:3928/v1/engines/llama-cpp/install", json=data - # ) - # assert response.status_code == 400 - # assert response.json()["message"] == "No variant provided" - - # @pytest.mark.asyncio - # async def test_engines_install_uninstall_llamacpp_with_variant_should_be_successful(self): - # # install first - # data = {"variant": "mac-arm64"} - # install_response = requests.post( - # "http://127.0.0.1:3928/v1/engines/llama-cpp/install", json=data - # ) - # await wait_for_websocket_download_success_event(timeout=120) - # assert install_response.status_code == 200 - - # response = requests.delete("http://127.0.0.1:3928/v1/engines/llama-cpp/install") - # assert response.status_code == 200 - - # def test_engines_install_uninstall_llamacpp_with_specific_variant_and_version_should_be_successful( - # self, - # ): - # data = {"variant": "mac-arm64", "version": "v0.1.35"} - # # install first - # install_response = requests.post( - # "http://localhost:3928/v1/engines/llama-cpp/install", json=data - # ) - # assert install_response.status_code == 200 - - # response = requests.delete( - # "http://localhost:3928/v1/engines/llama-cpp/install", json=data - # ) - # assert response.status_code == 200 - - \ No newline at end of file diff --git a/engine/e2e-test/utils/logger.py b/engine/e2e-test/utils/logger.py new file mode 100644 index 000000000..578a4f2cc --- /dev/null +++ b/engine/e2e-test/utils/logger.py @@ -0,0 +1,15 @@ +import json +import os + +def log_response(data, test_name): + """Log the data to a file named after the test.""" + log_dir="e2e-test/logs" + os.makedirs(log_dir, exist_ok=True) # Ensure log directory exists + file_path = os.path.join(log_dir, f"{test_name}.txt") # Log file per test + + try: + with open(file_path, "a", encoding="utf-8") as file: + json.dump(data, file, indent=4) + file.write("\n") # Ensure a new line between entries + except Exception as e: + print(f"Error logging response: {e}") diff --git a/engine/e2e-test/test_runner.py b/engine/e2e-test/utils/test_runner.py similarity index 100% rename from engine/e2e-test/test_runner.py rename to engine/e2e-test/utils/test_runner.py From 08dd5dbb601b6584700b42b7f6777f2fc6d2a4b5 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Thu, 20 Feb 2025 13:41:03 +0700 Subject: [PATCH 04/13] test: revert wfl --- .github/workflows/cortex-cpp-quality-gate.yml | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 235eb4119..4e741040a 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -1,12 +1,12 @@ name: CI Quality Gate Cortex CPP on: - # pull_request: - # types: [opened, synchronize, reopened, ready_for_review] - # paths: ["engine/**", ".github/workflows/cortex-cpp-quality-gate.yml"] + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: ["engine/**", ".github/workflows/cortex-cpp-quality-gate.yml"] workflow_dispatch: - # schedule: - # - cron: '0 22 * * *' + schedule: + - cron: '0 22 * * *' env: LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf @@ -218,58 +218,58 @@ jobs: AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}" AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}" - # build-docker-and-test: - # runs-on: ubuntu-24-04-docker - # steps: - # - name: Getting the repo - # uses: actions/checkout@v3 - # with: - # submodules: 'recursive' + build-docker-and-test: + runs-on: ubuntu-24-04-docker + steps: + - name: Getting the repo + uses: actions/checkout@v3 + with: + submodules: 'recursive' - # - name: Run Docker - # if: github.event_name != 'schedule' - # run: | - # docker build \ - # --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ - # --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ - # --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ - # --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ - # -t menloltd/cortex:test -f docker/Dockerfile.cache . - # docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test - # sleep 20 + - name: Run Docker + if: github.event_name != 'schedule' + run: | + docker build \ + --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ + --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ + --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ + --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ + -t menloltd/cortex:test -f docker/Dockerfile.cache . + docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test + sleep 20 - # - name: Run Docker - # if: github.event_name == 'schedule' - # run: | - # latest_prerelease=$(curl -s https://api.github.com/repos/cortexcpp/cortex.cpp/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1) - # echo "cortex.llamacpp latest release: $latest_prerelease" - # docker build \ - # --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ - # --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ - # --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ - # --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ - # --build-arg CORTEX_CPP_VERSION="${latest_prerelease}" \ - # -t menloltd/cortex:test -f docker/Dockerfile.cache . - # docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test - # sleep 20 + - name: Run Docker + if: github.event_name == 'schedule' + run: | + latest_prerelease=$(curl -s https://api.github.com/repos/cortexcpp/cortex.cpp/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1) + echo "cortex.llamacpp latest release: $latest_prerelease" + docker build \ + --build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \ + --build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \ + --build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \ + --build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \ + --build-arg CORTEX_CPP_VERSION="${latest_prerelease}" \ + -t menloltd/cortex:test -f docker/Dockerfile.cache . + docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test + sleep 20 - # - name: use python - # uses: actions/setup-python@v5 - # with: - # python-version: "3.10" + - name: use python + uses: actions/setup-python@v5 + with: + python-version: "3.10" - # - name: Run e2e tests - # run: | - # cd engine - # python -m pip install --upgrade pip - # python -m pip install -r e2e-test/requirements.txt - # pytest e2e-test/test_api_docker.py + - name: Run e2e tests + run: | + cd engine + python -m pip install --upgrade pip + python -m pip install -r e2e-test/requirements.txt + pytest e2e-test/test_api_docker.py - # - name: Run Docker - # continue-on-error: true - # if: always() - # run: | - # docker logs cortex - # docker stop cortex - # docker rm cortex - # echo "y\n" | docker system prune -af + - name: Run Docker + continue-on-error: true + if: always() + run: | + docker logs cortex + docker stop cortex + docker rm cortex + echo "y\n" | docker system prune -af From f656c68214b582d5733cb420bd0b136a73f282d5 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Thu, 20 Feb 2025 13:43:56 +0700 Subject: [PATCH 05/13] test: update log folder --- engine/e2e-test/utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/e2e-test/utils/logger.py b/engine/e2e-test/utils/logger.py index 578a4f2cc..3417f07e2 100644 --- a/engine/e2e-test/utils/logger.py +++ b/engine/e2e-test/utils/logger.py @@ -3,7 +3,7 @@ def log_response(data, test_name): """Log the data to a file named after the test.""" - log_dir="e2e-test/logs" + log_dir="logs" os.makedirs(log_dir, exist_ok=True) # Ensure log directory exists file_path = os.path.join(log_dir, f"{test_name}.txt") # Log file per test From 6e58dec740494e90e7d0cbdf5c7c526e252e7618 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Thu, 20 Feb 2025 14:27:00 +0700 Subject: [PATCH 06/13] test: move docker test to root --- engine/e2e-test/{api/model => }/test_api_docker.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename engine/e2e-test/{api/model => }/test_api_docker.py (100%) diff --git a/engine/e2e-test/api/model/test_api_docker.py b/engine/e2e-test/test_api_docker.py similarity index 100% rename from engine/e2e-test/api/model/test_api_docker.py rename to engine/e2e-test/test_api_docker.py From 4249234dbcb3e30d9fa52e199d6448e73f86215f Mon Sep 17 00:00:00 2001 From: Harry Le Date: Fri, 21 Feb 2025 10:24:51 +0700 Subject: [PATCH 07/13] test: revert change wfl --- .github/workflows/cortex-cpp-quality-gate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index d98631ef7..87be2cf47 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -168,7 +168,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry.py + python e2e-test/main.py rm build/cortex-nightly rm build/cortex-beta env: @@ -182,7 +182,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry.py + python e2e-test/main.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -196,7 +196,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry.py + python e2e-test/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly rm build/cortex-beta env: @@ -210,7 +210,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main-harry.py + python e2e-test/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: From 1b5d382e397dc0b1ba6fc55165dc03c2390e48a8 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Fri, 21 Feb 2025 10:30:50 +0700 Subject: [PATCH 08/13] test: update path in wfl --- .github/workflows/cortex-cpp-quality-gate.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cortex-cpp-quality-gate.yml b/.github/workflows/cortex-cpp-quality-gate.yml index 87be2cf47..68d4d9c09 100644 --- a/.github/workflows/cortex-cpp-quality-gate.yml +++ b/.github/workflows/cortex-cpp-quality-gate.yml @@ -168,7 +168,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly rm build/cortex-beta env: @@ -182,7 +182,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -196,7 +196,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly rm build/cortex-beta env: @@ -210,7 +210,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -443,7 +443,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly rm build/cortex-beta env: @@ -457,7 +457,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py + python e2e-test/runner/main.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: @@ -471,7 +471,7 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly rm build/cortex-beta env: @@ -485,7 +485,7 @@ jobs: cp build/cortex.exe build/cortex-beta.exe python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/cortex-llamacpp-e2e-nightly.py + python e2e-test/runner/cortex-llamacpp-e2e-nightly.py rm build/cortex-nightly.exe rm build/cortex-beta.exe env: From 4fe771aa43c88478e12cb50a5b5ae3528d9dc1c3 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Fri, 21 Feb 2025 21:41:24 +0700 Subject: [PATCH 09/13] test: update path, add-update utils --- .../runner/cortex-llamacpp-e2e-nightly.py | 26 +++++++++---------- engine/e2e-test/runner/main.py | 26 +++++++++---------- .../test_api_cortexso_hub_llamacpp_engine.py | 0 engine/e2e-test/utils/assertion.py | 3 +++ engine/e2e-test/utils/logger.py | 2 +- 5 files changed, 30 insertions(+), 27 deletions(-) rename engine/e2e-test/{api/hub => }/test_api_cortexso_hub_llamacpp_engine.py (100%) create mode 100644 engine/e2e-test/utils/assertion.py diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index 193d26d36..711f9e44c 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -16,21 +16,21 @@ sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) ### e2e tests are expensive, have to keep engines tests in order -from api.engines.test_api_engine_list import TestApiEngineList -from api.engines.test_api_engine_install_nightly import TestApiEngineInstall -from api.model.test_api_model import TestApiModel -from api.model.test_api_model_import import TestApiModelImport +from test_api_get_list_engine import TestApiEngineList +from test_api_engine_install_nightly import TestApiEngineInstall +from test_api_model import TestApiModel +from test_api_model_import import TestApiModelImport ### -from cli.engines.test_cli_engine_get import TestCliEngineGet -from cli.engines.test_cli_engine_install_nightly import TestCliEngineInstall -from cli.engines.test_cli_engine_list import TestCliEngineList -from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall -from cli.model.test_cli_model import TestCliModel -from cli.model.test_cli_model_import import TestCliModelImport -from cli.common.test_cli_server_start import TestCliServerStart -from cli.common.test_cortex_update import TestCortexUpdate -from cli.common.test_create_log_folder import TestCreateLogFolder +from test_cli_engine_get import TestCliEngineGet +from test_cli_engine_install_nightly import TestCliEngineInstall +from test_cli_engine_list import TestCliEngineList +from test_cli_engine_uninstall import TestCliEngineUninstall +from test_cli_model import TestCliModel +from test_cli_model_import import TestCliModelImport +from test_cli_server_start import TestCliServerStart +from test_cortex_update import TestCortexUpdate +from test_create_log_folder import TestCreateLogFolder if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index 7a96f1164..2d7dcc1cb 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -16,21 +16,21 @@ sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) ### e2e tests are expensive, have to keep engines tests in order -from api.engines.test_api_engine_list import TestApiEngineList -from api.engines.test_api_engine import TestApiEngine -from api.model.test_api_model import TestApiModel -from api.model.test_api_model_import import TestApiModelImport +from test_api_get_list_engine import TestApiEngineList +from test_api_engine import TestApiEngine +from test_api_model import TestApiModel +from test_api_model_import import TestApiModelImport ### -from cli.engines.test_cli_engine_get import TestCliEngineGet -from cli.engines.test_cli_engine_install import TestCliEngineInstall -from cli.engines.test_cli_engine_list import TestCliEngineList -from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall -from cli.model.test_cli_model import TestCliModel -from cli.model.test_cli_model_import import TestCliModelImport -from cli.common.test_cli_server_start import TestCliServerStart -from cli.common.test_cortex_update import TestCortexUpdate -from cli.common.test_create_log_folder import TestCreateLogFolder +from test_cli_engine_get import TestCliEngineGet +from test_cli_engine_install import TestCliEngineInstall +from test_cli_engine_list import TestCliEngineList +from test_cli_engine_uninstall import TestCliEngineUninstall +from test_cli_model import TestCliModel +from test_cli_model_import import TestCliModelImport +from test_cli_server_start import TestCliServerStart +from test_cortex_update import TestCortexUpdate +from test_create_log_folder import TestCreateLogFolder if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/api/hub/test_api_cortexso_hub_llamacpp_engine.py b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py similarity index 100% rename from engine/e2e-test/api/hub/test_api_cortexso_hub_llamacpp_engine.py rename to engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py diff --git a/engine/e2e-test/utils/assertion.py b/engine/e2e-test/utils/assertion.py new file mode 100644 index 000000000..067335732 --- /dev/null +++ b/engine/e2e-test/utils/assertion.py @@ -0,0 +1,3 @@ +def assert_equal(actual, expected): + """Custom assertion to compare actual and expected values.""" + assert actual == expected, f"Assertion failed: Expected {expected}, but got {actual}" diff --git a/engine/e2e-test/utils/logger.py b/engine/e2e-test/utils/logger.py index 3417f07e2..578a4f2cc 100644 --- a/engine/e2e-test/utils/logger.py +++ b/engine/e2e-test/utils/logger.py @@ -3,7 +3,7 @@ def log_response(data, test_name): """Log the data to a file named after the test.""" - log_dir="logs" + log_dir="e2e-test/logs" os.makedirs(log_dir, exist_ok=True) # Ensure log directory exists file_path = os.path.join(log_dir, f"{test_name}.txt") # Log file per test From d3b05f66888058479d0c6bfa89b327d39a60d25c Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sat, 22 Feb 2025 20:12:09 +0700 Subject: [PATCH 10/13] test: add test --- .../api/engines/test_api_engine_list.py | 25 ------ .../engines/test_api_get_default_engine.py | 85 +++++++++++++++++++ .../api/engines/test_api_get_list_engine.py | 73 ++++++++++++++++ 3 files changed, 158 insertions(+), 25 deletions(-) delete mode 100644 engine/e2e-test/api/engines/test_api_engine_list.py create mode 100644 engine/e2e-test/api/engines/test_api_get_default_engine.py create mode 100644 engine/e2e-test/api/engines/test_api_get_list_engine.py diff --git a/engine/e2e-test/api/engines/test_api_engine_list.py b/engine/e2e-test/api/engines/test_api_engine_list.py deleted file mode 100644 index 10346c988..000000000 --- a/engine/e2e-test/api/engines/test_api_engine_list.py +++ /dev/null @@ -1,25 +0,0 @@ -import pytest -import requests -from utils.test_runner import start_server, stop_server - - -class TestApiEngineList: - - @pytest.fixture(autouse=True) - def setup_and_teardown(self): - # Setup - # Not sure why but on macOS amd, the first start server timeouts with CI - start_server() - stop_server() - success = start_server() - if not success: - raise Exception("Failed to start server") - - yield - - # Teardown - stop_server() - - def test_engines_list_api_run_successfully(self): - response = requests.get("http://localhost:3928/engines") - assert response.status_code == 200 \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_default_engine.py b/engine/e2e-test/api/engines/test_api_get_default_engine.py new file mode 100644 index 000000000..600f2c38f --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_default_engine.py @@ -0,0 +1,85 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiDefaultEngine: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_default_engine_successfully(self): + # Data test + engine= "llama-cpp" + name= "linux-amd64-avx-cuda-11-7" + version= "v0.1.35-27.10.24" + + data = {"version": version, "variant": name} + post_install_url = f"http://localhost:3928/v1/engines/{engine}/install" + response = requests.post( + post_install_url, json=data + ) + assert_equal(response.status_code,200) + + get_list_url = f"http://localhost:3928/v1/engines/{engine}" + get_default_url = f"http://localhost:3928/v1/engines/{engine}/default" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_list_url) + + response_default_engine = requests.get(get_default_url) + json_data = response_default_engine.json() + + log_response(json_data, "test_api_get_default_engine_successfully") + assert_equal(response_default_engine.status_code, 200) + + schema = { + "type": "object", + "properties": { + "engine": {"type": "string"}, + "variant": {"type": "string"}, + "version": {"type": "string"} + }, + "required": ["engine", "variant", "version"] + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + assert_equal(json_data["engine"], engine) + assert_equal(json_data["version"], version) + assert_equal(json_data["variant"], name) + + def test_api_get_default_engine_successfully(self): + # Data test + engine= "invalid" + + get_default_url = f"http://localhost:3928/v1/engines/{engine}/default" + + response_default_engine = requests.get(get_default_url) + json_data_get_default = response_default_engine.json() + + log_response(json_data_get_default, "test_api_get_default_engine_successfully") + assert_equal(response_default_engine.status_code, 400) + + assert_equal(json_data_get_default["message"], f"Engine {engine} is not supported yet!") \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_list_engine.py b/engine/e2e-test/api/engines/test_api_get_list_engine.py new file mode 100644 index 000000000..9a1552de6 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_list_engine.py @@ -0,0 +1,73 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal + + +class TestApiEngineList: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_list_engines_successfully(self): + # Data test + engine= "llama-cpp" + name= "linux-amd64-avx-cuda-11-7" + version= "v0.1.35-27.10.24" + + data = {"version": version, "variant": name} + post_install_url = f"http://localhost:3928/v1/engines/{engine}/install" + response = requests.post( + post_install_url, json=data + ) + assert_equal(response.status_code,200) + + get_list_url = f"http://localhost:3928/v1/engines/{engine}" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + return response + + response_get_list = get_request(get_list_url) + json_data = response_get_list.json() + + log_response(json_data, "test_api_get_list_engines_successfully") + assert_equal(response_get_list.status_code, 200) + + schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "engine": {"type": "string"}, + "name": {"type": "string"}, + "version": {"type": "string"} + }, + "required": ["engine", "name", "version"] + } + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + assert_equal(len(json_data), 1) + assert_equal(json_data[0]["engine"], engine) + assert_equal(json_data[0]["version"], version) + assert_equal(json_data[0]["name"], name) \ No newline at end of file From 8a20ce739d042d0d849bf65c1ca89ad418e853fa Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sat, 22 Feb 2025 20:29:33 +0700 Subject: [PATCH 11/13] test: update path --- .../runner/cortex-llamacpp-e2e-nightly.py | 26 +++++++++---------- engine/e2e-test/runner/main.py | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index 711f9e44c..f4aca303d 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -16,21 +16,21 @@ sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) ### e2e tests are expensive, have to keep engines tests in order -from test_api_get_list_engine import TestApiEngineList -from test_api_engine_install_nightly import TestApiEngineInstall -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport +from api.engines.test_api_get_list_engine import TestApiEngineList +from api.engines.test_api_engine_install_nightly import TestApiEngineInstall +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport ### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install_nightly import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_model_import import TestCliModelImport -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install_nightly import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index 2d7dcc1cb..a8c32bbac 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -16,21 +16,21 @@ sys.path.append(os.path.join(PROJECT_ROOT, "cli/common")) ### e2e tests are expensive, have to keep engines tests in order -from test_api_get_list_engine import TestApiEngineList -from test_api_engine import TestApiEngine -from test_api_model import TestApiModel -from test_api_model_import import TestApiModelImport +from api.engines.test_api_get_list_engine import TestApiEngineList +from api.engines.test_api_engine import TestApiEngine +from api.model.test_api_model import TestApiModel +from api.model.test_api_model_import import TestApiModelImport ### -from test_cli_engine_get import TestCliEngineGet -from test_cli_engine_install import TestCliEngineInstall -from test_cli_engine_list import TestCliEngineList -from test_cli_engine_uninstall import TestCliEngineUninstall -from test_cli_model import TestCliModel -from test_cli_model_import import TestCliModelImport -from test_cli_server_start import TestCliServerStart -from test_cortex_update import TestCortexUpdate -from test_create_log_folder import TestCreateLogFolder +from cli.engines.test_cli_engine_get import TestCliEngineGet +from cli.engines.test_cli_engine_install import TestCliEngineInstall +from cli.engines.test_cli_engine_list import TestCliEngineList +from cli.engines.test_cli_engine_uninstall import TestCliEngineUninstall +from cli.model.test_cli_model import TestCliModel +from cli.model.test_cli_model_import import TestCliModelImport +from cli.common.test_cli_server_start import TestCliServerStart +from cli.common.test_cortex_update import TestCortexUpdate +from cli.common.test_create_log_folder import TestCreateLogFolder if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) From 43a1b2090575511ccb1301b1947a6175a97e0811 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sat, 22 Feb 2025 20:42:48 +0700 Subject: [PATCH 12/13] test: add tenacity to package --- engine/e2e-test/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/e2e-test/requirements.txt b/engine/e2e-test/requirements.txt index d9c436a11..6991b03d3 100644 --- a/engine/e2e-test/requirements.txt +++ b/engine/e2e-test/requirements.txt @@ -3,4 +3,5 @@ pytest pytest-asyncio requests pyyaml -jsonschema \ No newline at end of file +jsonschema +tenacity \ No newline at end of file From 53e87d02a76a012c25cb9a8e8d5842e86ce6b080 Mon Sep 17 00:00:00 2001 From: Harry Le Date: Sun, 23 Feb 2025 08:17:28 +0700 Subject: [PATCH 13/13] test: add more --- .../engines/test_api_get_default_engine.py | 9 +-- .../engines/test_api_get_engine_release.py | 76 +++++++++++++++++++ .../test_api_get_engine_release_latest.py | 73 ++++++++++++++++++ .../runner/cortex-llamacpp-e2e-nightly.py | 3 + engine/e2e-test/runner/main.py | 3 + engine/e2e-test/utils/assertion.py | 6 +- 6 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 engine/e2e-test/api/engines/test_api_get_engine_release.py create mode 100644 engine/e2e-test/api/engines/test_api_get_engine_release_latest.py diff --git a/engine/e2e-test/api/engines/test_api_get_default_engine.py b/engine/e2e-test/api/engines/test_api_get_default_engine.py index 600f2c38f..0320c1cc1 100644 --- a/engine/e2e-test/api/engines/test_api_get_default_engine.py +++ b/engine/e2e-test/api/engines/test_api_get_default_engine.py @@ -33,6 +33,7 @@ def test_api_get_default_engine_successfully(self): post_install_url, json=data ) assert_equal(response.status_code,200) + log_response(response.json(), "test_api_get_default_engine_successfully") get_list_url = f"http://localhost:3928/v1/engines/{engine}" get_default_url = f"http://localhost:3928/v1/engines/{engine}/default" @@ -66,11 +67,7 @@ def get_request(url): # Validate response schema jsonschema.validate(instance=json_data, schema=schema) - assert_equal(json_data["engine"], engine) - assert_equal(json_data["version"], version) - assert_equal(json_data["variant"], name) - - def test_api_get_default_engine_successfully(self): + def test_api_get_default_engine_failed_invalid_engine(self): # Data test engine= "invalid" @@ -79,7 +76,7 @@ def test_api_get_default_engine_successfully(self): response_default_engine = requests.get(get_default_url) json_data_get_default = response_default_engine.json() - log_response(json_data_get_default, "test_api_get_default_engine_successfully") + log_response(json_data_get_default, "test_api_get_default_engine_failed_invalid_engine") assert_equal(response_default_engine.status_code, 400) assert_equal(json_data_get_default["message"], f"Engine {engine} is not supported yet!") \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_engine_release.py b/engine/e2e-test/api/engines/test_api_get_engine_release.py new file mode 100644 index 000000000..1e68de121 --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_engine_release.py @@ -0,0 +1,76 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal, assert_contains + + +class TestApiEngineRelease: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_engine_release_successfully(self): + # Data test + engine= "llama-cpp" + get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_release_url) + + response_engine_release = requests.get(get_release_url) + json_data = response_engine_release.json() + + log_response(json_data, "test_api_get_engine_release_successfully") + assert_equal(response_engine_release.status_code, 200) + + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "array", + "items": { + "type": "object", + "properties": { + "draft": { "type": "boolean" }, + "name": { "type": "string" }, + "prerelease": { "type": "boolean" }, + "published_at": { "type": "string", "format": "date-time" }, + "url": { "type": "string", "format": "uri" } + }, + "required": ["draft", "name", "prerelease", "published_at", "url"] + } + } + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) + + def test_api_ge_engine_release_failed_invalid_engine(self): + # Data test + engine= "invalid" + + get_default_url = f"http://localhost:3928/v1/engines/{engine}/releases" + + response_default_engine = requests.get(get_default_url) + json_data_get_default = response_default_engine.json() + + log_response(json_data_get_default, "test_api_ge_engine_release_failed_invalid_engine") + assert_equal(response_default_engine.status_code, 400) + + assert_contains(json_data_get_default["message"], "Not Found") \ No newline at end of file diff --git a/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py b/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py new file mode 100644 index 000000000..be65141ea --- /dev/null +++ b/engine/e2e-test/api/engines/test_api_get_engine_release_latest.py @@ -0,0 +1,73 @@ +import pytest +import requests +from utils.test_runner import start_server, stop_server +import jsonschema +from tenacity import retry, wait_exponential, stop_after_attempt +from utils.logger import log_response +from utils.assertion import assert_equal, assert_contains + + +class TestApiEngineReleaseLatest: + + @pytest.fixture(autouse=True) + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + + yield + + # Teardown + stop_server() + + def test_api_get_engine_release_latest_successfully(self): + # Data test + engine= "llama-cpp" + get_release_url = f"http://localhost:3928/v1/engines/{engine}/releases/latest" + + @retry( + wait=wait_exponential(multiplier=2, min=2, max=30), + stop=stop_after_attempt(5) + ) + def get_request(url): + response = requests.get(url) + assert len(response.json()) > 0 + + get_request(get_release_url) + + response_engine_release = requests.get(get_release_url) + json_data = response_engine_release.json() + + log_response(json_data, "test_api_get_engine_release_latest_successfully") + assert_equal(response_engine_release.status_code, 200) + + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "array", + "items": { + "type": "object", + "properties": { + "created_at": { + "type": "string", + "format": "date-time" + }, + "download_count": { + "type": "integer", + "minimum": 0 + }, + "name": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + } + }, + "required": ["created_at", "download_count", "name", "size"] + } + } + + + # Validate response schema + jsonschema.validate(instance=json_data, schema=schema) \ No newline at end of file diff --git a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py index f4aca303d..61b153267 100644 --- a/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py +++ b/engine/e2e-test/runner/cortex-llamacpp-e2e-nightly.py @@ -18,6 +18,9 @@ ### e2e tests are expensive, have to keep engines tests in order from api.engines.test_api_get_list_engine import TestApiEngineList from api.engines.test_api_engine_install_nightly import TestApiEngineInstall +from api.engines.test_api_get_default_engine import TestApiDefaultEngine +from api.engines.test_api_get_engine_release import TestApiEngineRelease +from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport diff --git a/engine/e2e-test/runner/main.py b/engine/e2e-test/runner/main.py index a8c32bbac..9b2a0316c 100644 --- a/engine/e2e-test/runner/main.py +++ b/engine/e2e-test/runner/main.py @@ -18,6 +18,9 @@ ### e2e tests are expensive, have to keep engines tests in order from api.engines.test_api_get_list_engine import TestApiEngineList from api.engines.test_api_engine import TestApiEngine +from api.engines.test_api_get_default_engine import TestApiDefaultEngine +from api.engines.test_api_get_engine_release import TestApiEngineRelease +from api.engines.test_api_get_engine_release_latest import TestApiEngineReleaseLatest from api.model.test_api_model import TestApiModel from api.model.test_api_model_import import TestApiModelImport diff --git a/engine/e2e-test/utils/assertion.py b/engine/e2e-test/utils/assertion.py index 067335732..63443981f 100644 --- a/engine/e2e-test/utils/assertion.py +++ b/engine/e2e-test/utils/assertion.py @@ -1,3 +1,7 @@ def assert_equal(actual, expected): """Custom assertion to compare actual and expected values.""" - assert actual == expected, f"Assertion failed: Expected {expected}, but got {actual}" + assert actual == expected, f"Assertion failed: Expected '{expected}', but got '{actual}'" + +def assert_contains(main_string, sub_string): + """Custom assertion to compare actual and expected values.""" + assert sub_string in main_string, f"Assertion failed: Expected '{main_string}' has '{sub_string}'"