From 2da5f29f28a445d4af657a4c51c9d9b4225dadac Mon Sep 17 00:00:00 2001 From: Hien To Date: Wed, 30 Oct 2024 16:57:35 +0700 Subject: [PATCH 1/4] feat: e2e testing cortexso model hub --- .../test_api_cortexso_hub_llamacpp_engine.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py diff --git a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py new file mode 100644 index 000000000..a6410b6b1 --- /dev/null +++ b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py @@ -0,0 +1,65 @@ +import pytest +from test_runner import start_server, stop_server + +import requests +import os + +def get_repos_in_collection(collection_id, token): + # API endpoint to get list of repos in the collection + url = f"https://huggingface.co/api/collections/{collection_id}" + headers = {"Authorization": f"Bearer {token}"} + response = requests.get(url, headers=headers) + + # Check response and retrieve repo IDs if successful + if response.status_code == 200: + return [repo['id'] for repo in response.json()["items"]] + else: + print("Error fetching repos:", response.status_code, response.json()) + return [] + +def get_repo_branches(repo_id, token): + # API endpoint to get list of branches for each repo + url = f"https://huggingface.co/api/models/{repo_id}/refs" + headers = {"Authorization": f"Bearer {token}"} + response = requests.get(url, headers=headers) + + # Check response and get the gguf branch + if response.status_code == 200: + branches = response.json()["branches"] + return [branch['name'] for branch in branches if branch['name'] == 'gguf'] + else: + print(f"Error fetching branches for {repo_id}:", response.status_code, response.json()) + return [] + +def get_all_repos_and_default_branches_gguf(collection_id, token): + # Get list of repos from the collection + repos = get_repos_in_collection(collection_id, token) + combined_list = [] + + # Iterate over each repo and fetch branches + for repo_id in repos: + branches = get_repo_branches(repo_id, token) + for branch in branches: + combined_list.append(f"{repo_id.split('/')[1]}:{branch}") + + return combined_list + +collection_id = "cortexso/local-models-6683a6e29e8f3018845b16db" +token = os.getenv("HF_TOKEN") +if not token: + raise ValueError("HF_TOKEN environment variable not set") + +# Call the function and print the results +repo_branches = get_all_repos_and_default_branches_gguf(collection_id, token) + +class TestCortexsoModels: + + def setup_and_teardown(self): + # Setup + success = start_server() + if not success: + raise Exception("Failed to start server") + yield + # Teardown + stop_server() + From 9436ec54f5f383b4e29b63b4df2d2bc9fdca288d Mon Sep 17 00:00:00 2001 From: Hien To Date: Thu, 31 Oct 2024 13:35:54 +0700 Subject: [PATCH 2/4] chore: schedule to run models test weekly --- .github/workflows/test-cortexso-model-hub.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-cortexso-model-hub.yml b/.github/workflows/test-cortexso-model-hub.yml index 523086570..320369235 100644 --- a/.github/workflows/test-cortexso-model-hub.yml +++ b/.github/workflows/test-cortexso-model-hub.yml @@ -2,7 +2,7 @@ name: Test cortexso Model Hub on: schedule: - - cron: "0 16 * * *" # every day at 23:00 UTC+7 + - cron: "0 16 * * 5" # every Friday at 23:00 UTC+7 workflow_dispatch: jobs: @@ -74,7 +74,6 @@ jobs: cp build/cortex build/cortex-beta python -m pip install --upgrade pip python -m pip install -r e2e-test/requirements.txt - python e2e-test/main.py pytest e2e-test/test_api_cortexso_hub_llamacpp_engine.py rm build/cortex-nightly rm build/cortex-beta From 16e2bfd950a838afa4d9100b887d64e622f9443a Mon Sep 17 00:00:00 2001 From: Hien To Date: Thu, 31 Oct 2024 14:46:24 +0700 Subject: [PATCH 3/4] chore: resolve warning pytest --- engine/e2e-test/pytest.ini | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 engine/e2e-test/pytest.ini diff --git a/engine/e2e-test/pytest.ini b/engine/e2e-test/pytest.ini new file mode 100644 index 000000000..0102b0a97 --- /dev/null +++ b/engine/e2e-test/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_default_fixture_loop_scope = function From 9cbdea01f3af86310716e091f798921b6f65f476 Mon Sep 17 00:00:00 2001 From: Hien To Date: Thu, 31 Oct 2024 15:46:47 +0700 Subject: [PATCH 4/4] chore: use default branch cortexso hub --- engine/e2e-test/requirements.txt | 1 + .../test_api_cortexso_hub_llamacpp_engine.py | 30 ++++++++++--------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/engine/e2e-test/requirements.txt b/engine/e2e-test/requirements.txt index f0eabb974..05b47e0b0 100644 --- a/engine/e2e-test/requirements.txt +++ b/engine/e2e-test/requirements.txt @@ -2,3 +2,4 @@ websockets pytest pytest-asyncio requests +pyyaml \ No newline at end of file diff --git a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py index ec785d565..e13c4827a 100644 --- a/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py +++ b/engine/e2e-test/test_api_cortexso_hub_llamacpp_engine.py @@ -1,6 +1,7 @@ import pytest import requests import os +import yaml from pathlib import Path from test_runner import ( @@ -28,35 +29,36 @@ def get_repos_in_collection(collection_id, token): print("Error fetching repos:", response.status_code, response.json()) return [] -def get_repo_branches(repo_id, token): - # API endpoint to get list of branches for each repo - url = f"https://huggingface.co/api/models/{repo_id}/refs" +def get_repo_default_branch(repo_id, token): + # Direct link to metadata.yaml on the main branch + url = f"https://huggingface.co/{repo_id}/resolve/main/metadata.yml" headers = {"Authorization": f"Bearer {token}"} response = requests.get(url, headers=headers) - # Check response and get the gguf branch + # Check response and retrieve the 'default' field value if response.status_code == 200: - branches = response.json()["branches"] - return [branch['name'] for branch in branches if branch['name'] == 'gguf'] + # Read YAML content from response text + metadata = yaml.safe_load(response.text) + return metadata.get("default") else: - print(f"Error fetching branches for {repo_id}:", response.status_code, response.json()) - return [] + print(f"Error fetching metadata for {repo_id}:", response.status_code, response.json()) + return None -def get_all_repos_and_default_branches_gguf(collection_id, token): +def get_all_repos_and_default_branches_from_metadata(collection_id, token): # Get list of repos from the collection repos = get_repos_in_collection(collection_id, token) combined_list = [] - # Iterate over each repo and fetch branches + # Iterate over each repo and fetch the default branch from metadata for repo_id in repos: - branches = get_repo_branches(repo_id, token) - for branch in branches: - combined_list.append(f"{repo_id.split('/')[1]}:{branch}") + default_branch = get_repo_default_branch(repo_id, token) + if default_branch and "gguf" in default_branch: + combined_list.append(f"{repo_id.split('/')[1]}:{default_branch}") return combined_list #Call the function and print the results -repo_branches = get_all_repos_and_default_branches_gguf(collection_id, token) +repo_branches = get_all_repos_and_default_branches_from_metadata(collection_id, token) class TestCortexsoModels: