From 26c2b5ea29f3444fc05c309078df28796f833fe0 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Thu, 27 Jul 2023 13:08:33 -0700 Subject: [PATCH 001/165] Started image upload implementation Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 61 ++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 3cf3edc1..2ae6d518 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,7 +29,21 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn - +from pydantic import BaseModel + +class ParamInput(BaseModel): + asset_name: str + asset_function: str + asset_class: str + active_version: str + gen_description: str + license_type: str + license_description: str + container_registry_name: str + container_service_provider: str + vcpus: str + ram: str + gpus: str class ModelFactory: """A static class for creating and exploring Model Objects. @@ -172,3 +186,48 @@ def get_first_k_assets( error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" logging.error(error_message) return [] + + @classmethod + def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, description: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use ParamInput here for input type checking. + # Use Ibrahim's endpoint here and return output. + create_url = f"{config.BACKEND_URL}/sdk/models/register" + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + payload = { + "name": name, + "hostingMachine": hosting_machine, + "alwaysOn": always_on, + "version": version, + "description": description + } + payload = json.dumps(payload) + response = _request_with_retry("post", create_url, headers=headers, data=payload) + return response + + @classmethod + def asset_repo_login(cls, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use Ibrahim's endpoint here and return output. + # TODO + create_url = f"{config.BACKEND_URL}/sdk/ecr/login" # TODO Add Ibrahim's repo login endpoint here + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + params = {"Authorization": team_api_key} + response = _request_with_retry("post", create_url, headers=headers, params=params) + return response + + @classmethod + def list_image_repo_tags(cls, team_id: Text, repo_name: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use Ibrahim's endpoint here and return output. + # TODO + create_url = f"{config.BACKEND_URL}/TODO" # TODO Add Ibrahim's repo tag endpoint here + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", create_url, headers=headers) + return response \ No newline at end of file From 1d17a54ae7e4b2ac164034a9360ecc87554813c7 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Wed, 9 Aug 2023 11:45:30 -0700 Subject: [PATCH 002/165] Adding create_asset Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 137 ++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 39 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 2ae6d518..44de7fd6 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,21 +29,6 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn -from pydantic import BaseModel - -class ParamInput(BaseModel): - asset_name: str - asset_function: str - asset_class: str - active_version: str - gen_description: str - license_type: str - license_description: str - container_registry_name: str - container_service_provider: str - vcpus: str - ram: str - gpus: str class ModelFactory: """A static class for creating and exploring Model Objects. @@ -188,46 +173,120 @@ def get_first_k_assets( return [] @classmethod - def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, description: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use ParamInput here for input type checking. - # Use Ibrahim's endpoint here and return output. + def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: + """Lists available hosting machines for model. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each hosting machine. + """ + machines_url = f"{config.BACKEND_URL}/sdk/hosting-machines" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", machines_url, headers=headers) + return response + + @classmethod + def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, + description: Text, function: Text, api_key: Optional[Text] = None) -> Dict: + """Creates an image repository for this model and registers it in the + platform backend. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model funciton obtained via #TODO add function endpoint + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ create_url = f"{config.BACKEND_URL}/sdk/models/register" - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} payload = { "name": name, "hostingMachine": hosting_machine, "alwaysOn": always_on, "version": version, - "description": description + "description": description, + "function": function } payload = json.dumps(payload) response = _request_with_retry("post", create_url, headers=headers, data=payload) return response @classmethod - def asset_repo_login(cls, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use Ibrahim's endpoint here and return output. - # TODO - create_url = f"{config.BACKEND_URL}/sdk/ecr/login" # TODO Add Ibrahim's repo login endpoint here - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: + """Return login credentials for the image repository that corresponds with + the given API_KEY. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + login_url = f"{config.BACKEND_URL}/sdk/ecr/login" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", login_url, headers=headers) + return response + + @classmethod + def onboard_model(cls, model_id: Text, image_tag: Optional[Text], api_key: Optional[Text] = None) -> Dict: + """Onboard a model after its image has been pushed to ECR. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + Dict: Backend response + """ + onboard_url = f"{config.BACKEND_URL}/sdk/inventory/models/{model_id}/onboarding" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - params = {"Authorization": team_api_key} - response = _request_with_retry("post", create_url, headers=headers, params=params) + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + payload = { + "image": image_tag + } + payload = json.dumps(payload) + response = _request_with_retry("post", onboard_url, headers=headers, data=payload) return response + + @classmethod + def is_onboarded(cls): + pass @classmethod - def list_image_repo_tags(cls, team_id: Text, repo_name: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use Ibrahim's endpoint here and return output. - # TODO - create_url = f"{config.BACKEND_URL}/TODO" # TODO Add Ibrahim's repo tag endpoint here - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Dict: + """List the contents of the image repository corresponding to API_KEY. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + list_url = f"{config.BACKEND_URL}/sdk/models/{model_id}/images" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - response = _request_with_retry("post", create_url, headers=headers) + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", list_url, headers=headers) return response \ No newline at end of file From 882ba6d6a66a2529fb484d67460c62830621b554 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 12:03:22 -0700 Subject: [PATCH 003/165] Adding CLI Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 3 +++ setup.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 44de7fd6..cca08098 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,6 +29,7 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn +import click class ModelFactory: """A static class for creating and exploring Model Objects. @@ -172,6 +173,8 @@ def get_first_k_assets( logging.error(error_message) return [] + @click.command() + @click.option("--team-api-key", default=None, help="Team API key") @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. diff --git a/setup.py b/setup.py index dc103302..049c2dbb 100644 --- a/setup.py +++ b/setup.py @@ -98,4 +98,9 @@ "Documentation": "https://github.com/aixplain/pipelines/tree/main/docs", "Source": "https://github.com/aixplain/aiXplain", }, + entry_points = { + 'console_scripts': [ + 'hosted-machines = aixplain.factories.model_factory:list_host_machines' + ] + }, ) From e8095d449342f0699ba2d737731cbfb53992084d Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 12:08:29 -0700 Subject: [PATCH 004/165] classmethod last Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index cca08098..f4ab970f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -173,9 +173,9 @@ def get_first_k_assets( logging.error(error_message) return [] + @classmethod @click.command() @click.option("--team-api-key", default=None, help="Team API key") - @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 04ec84c43c39aeee40d8875d37ac84175317955a Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 13:17:04 -0700 Subject: [PATCH 005/165] Corrected entry point path Signed-off-by: mikelam-us --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 049c2dbb..353f96b8 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ }, entry_points = { 'console_scripts': [ - 'hosted-machines = aixplain.factories.model_factory:list_host_machines' + 'hosted-machines = aixplain.factories.model_factory.ModelFactory:list_host_machines' ] }, ) From 1534ff40083068caaff1f1eccce1fb7eb1cba202 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 13:19:27 -0700 Subject: [PATCH 006/165] Working on entry point path Signed-off-by: mikelam-us --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 353f96b8..f96e8c30 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ }, entry_points = { 'console_scripts': [ - 'hosted-machines = aixplain.factories.model_factory.ModelFactory:list_host_machines' + 'hosted-machines = aixplain.factories.model_factory:ModelFactory.list_host_machines' ] }, ) From 924b93dd7cfd53b6f1412d719a35cb6a2efcad6c Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Tue, 15 Aug 2023 09:27:53 -0700 Subject: [PATCH 007/165] No CLI for now Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 30 ++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index f4ab970f..a14bd65e 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -174,8 +174,8 @@ def get_first_k_assets( return [] @classmethod - @click.command() - @click.option("--team-api-key", default=None, help="Team API key") + # @click.command() + # @click.option("--team-api-key", default=None, help="Team API key") def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. @@ -249,7 +249,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: return response @classmethod - def onboard_model(cls, model_id: Text, image_tag: Optional[Text], api_key: Optional[Text] = None) -> Dict: + def onboard_model(cls, model_id: Text, image_tag: Optional[Text] = None, api_key: Optional[Text] = None) -> Dict: """Onboard a model after its image has been pushed to ECR. Args: @@ -272,8 +272,28 @@ def onboard_model(cls, model_id: Text, image_tag: Optional[Text], api_key: Optio return response @classmethod - def is_onboarded(cls): - pass + def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Optional[Text] = None): + """Check whether a model has been onboarded. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + Dict: Backend response + """ + is_onboarded_url = f"{config.BACKEND_URL}/webhook/models/onboarding" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + payload = { + "id": model_id, + "host": host, + "version": version + } + payload = json.dumps(payload) + response = _request_with_retry("post", is_onboarded_url, headers=headers, data=payload) + return response @classmethod def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Dict: From 55d9b7d3fc76e040c9f628adda0fbe89daaa2861 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Tue, 15 Aug 2023 09:43:25 -0700 Subject: [PATCH 008/165] Fixed list host post -> get Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index a14bd65e..1d53d612 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -191,7 +191,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} - response = _request_with_retry("post", machines_url, headers=headers) + response = _request_with_retry("get", machines_url, headers=headers) return response @classmethod From 5af593588a8dc1d9390f2f3ab783713d5c485de1 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Tue, 15 Aug 2023 10:05:05 -0700 Subject: [PATCH 009/165] Removed slash from URL Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 1d53d612..3053e1da 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -186,7 +186,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each hosting machine. """ - machines_url = f"{config.BACKEND_URL}/sdk/hosting-machines" + machines_url = f"{config.BACKEND_URL}sdk/hosting-machines" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -212,7 +212,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v Returns: Dict: Backend response """ - create_url = f"{config.BACKEND_URL}/sdk/models/register" + create_url = f"{config.BACKEND_URL}sdk/models/register" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -240,7 +240,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Returns: Dict: Backend response """ - login_url = f"{config.BACKEND_URL}/sdk/ecr/login" + login_url = f"{config.BACKEND_URL}sdk/ecr/login" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -259,7 +259,7 @@ def onboard_model(cls, model_id: Text, image_tag: Optional[Text] = None, api_key Returns: Dict: Backend response """ - onboard_url = f"{config.BACKEND_URL}/sdk/inventory/models/{model_id}/onboarding" + onboard_url = f"{config.BACKEND_URL}sdk/inventory/models/{model_id}/onboarding" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -281,7 +281,7 @@ def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Option Returns: Dict: Backend response """ - is_onboarded_url = f"{config.BACKEND_URL}/webhook/models/onboarding" + is_onboarded_url = f"{config.BACKEND_URL}webhook/models/onboarding" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -306,7 +306,7 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Returns: Dict: Backend response """ - list_url = f"{config.BACKEND_URL}/sdk/models/{model_id}/images" + list_url = f"{config.BACKEND_URL}sdk/models/{model_id}/images" if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: From 367d27c54705c1458f1097131f8c4510ae4d607f Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Tue, 15 Aug 2023 11:07:15 -0700 Subject: [PATCH 010/165] Added is async Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 3053e1da..0862bd90 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -196,7 +196,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: @classmethod def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, - description: Text, function: Text, api_key: Optional[Text] = None) -> Dict: + description: Text, function: Text, is_async: bool, api_key: Optional[Text] = None) -> Dict: """Creates an image repository for this model and registers it in the platform backend. @@ -223,7 +223,8 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v "alwaysOn": always_on, "version": version, "description": description, - "function": function + "function": function, + "isAsync": is_async } payload = json.dumps(payload) response = _request_with_retry("post", create_url, headers=headers, data=payload) From 65dee4d34594f3b12b52d7dfc3f72be467a61f43 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:20:12 -0700 Subject: [PATCH 011/165] Adding first functional tests Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 6 ++++-- tests/image_upload_test.py | 26 ++++++++++++++++++++++++ tests/mock_responses/login_response.json | 5 +++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 tests/image_upload_test.py create mode 100644 tests/mock_responses/login_response.json diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 0862bd90..b0166a6b 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -207,6 +207,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v version (Text): Model version description (Text): Model description function (Text): Model funciton obtained via #TODO add function endpoint + is_async (bool): Whether the model is asynchronous or not api_key (Text, optional): Team API key. Defaults to None. Returns: @@ -250,7 +251,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: return response @classmethod - def onboard_model(cls, model_id: Text, image_tag: Optional[Text] = None, api_key: Optional[Text] = None) -> Dict: + def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: """Onboard a model after its image has been pushed to ECR. Args: @@ -266,7 +267,8 @@ def onboard_model(cls, model_id: Text, image_tag: Optional[Text] = None, api_key else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} payload = { - "image": image_tag + "image": image_tag, + "sha": image_hash } payload = json.dumps(payload) response = _request_with_retry("post", onboard_url, headers=headers, data=payload) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py new file mode 100644 index 00000000..08d7990d --- /dev/null +++ b/tests/image_upload_test.py @@ -0,0 +1,26 @@ +__author__ = "michaellam" + +import json +import pytest +import requests_mock +from pathlib import Path +from aixplain.utils import config + +from aixplain.factories.model_factory import ModelFactory +from aixplain.factories.dataset_factory import DatasetFactory +from aixplain.factories.metric_factory import MetricFactory + +AUTH_FIXED_HEADER = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} +API_FIXED_HEADER = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} + + +def test_login(): + url = f"{config.BACKEND_URL}/sdk/ecr/login" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/login_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) + creds_dict = creds.to_dict() + mock_dict = mock_json.to_dict() + assert creds_dict == mock_dict \ No newline at end of file diff --git a/tests/mock_responses/login_response.json b/tests/mock_responses/login_response.json new file mode 100644 index 00000000..ae755aca --- /dev/null +++ b/tests/mock_responses/login_response.json @@ -0,0 +1,5 @@ +{ + "username": "AWS", + "password": "eyJwYXlsb2FkIjoiNlNkQmp0WkRWbDRtQmxYMk5ZWHh0dFJQRTJFeWpScDFVczI1RTl2WUJMRmN5SVU1TE1wd3hiK2FaZWtnbjZidy9Ea3UxQ1FpcnIwRURwNXZNMTJuZXBJVzhITjNkVEtmeFNCa1RwcTNESSt2ZnVtSm5MVXM3KzlPNEo3cmRySDE5NjdnazYyb0NIRVV1WmZvOUFuUm5CeHUyU2ZmZWFndFlYVyt0dDVXeWtMQjRCRlNFaGJUelNtSnllSW9pNTlkNFNYdGtXY3pDT1RZQ281MUVlVEI0L1c4NGZMVVZQRVF6VThmdmtYRVl1TDNEUWFzc3F3dUxxcHp2bWtrSCtNOHNrdFp6bHZubXlxMnFGYkR0aElhamNXNW1Ud1BkVjJMN2w0ZFJVSTlTQ3Y1SlExbnlZZ01obUxHeDRDRG5KYmh0NndzeEtWcVpxbmMzMDR6WXZnQlZTcWFEY2VvWXV0SFEwSTVSQU1DaUtNd09SZHF0Skt1Y3FxRVBwTkxPaDhlcUFScmd4bkVCYnhQZm4zZ0M5L0x2bHBiZ1I5UFRIWGlqZlFWczNnUW5vTzFmd0R2d1dudTRsMjJDWjdSUTN4WlRNL29NdFNtZ2RScmplclpqNWo0RVMycTdQTEFXOU9UcUtieDRpZklMRUVucTIxbDBXaFNtc0xlR2g4Rm9GZkpOSGJ5L2wzUklTY2hjUzBYUUdYMXJ0cFhFOTc3bUVtdzY0WDdYT3h5UGlnZytzNWowMjhFY0VqSzV6R01sNzdDYUprcVVyZjVUUWZraTU4VURCMTNXWDlvVDVGQVUvcU9DY3F0SlQ5TlBZTnFXQ0xhamdFdk93TXFsQndkVzhKTEhwMTkwZ3psNE1nN0YwRDIvTFpScWRDVVh2SXRBSFJJUmROa1U3RDI1Y3VoL0xjSjlhZUQ2MnJiVDA1R2FIWkV5Z0d5MmxnRWlmekUvbWhPSGNUclBOSnlPTGhHaFc2L0F5dCt1MDRxNEdqMzVFQk1GSHZ0a0lLUEQ5MU04NTVKZnVMV3F3d09QR1NlZnNGRXlRNExxRGZtMkNueVpqd3NuNWRFSlR5VUZhTUMyODMwbCtBV3lZMFBQQ3l6eTFJK0FoMHV0VkJvMlBabkFPZVk1c3hOL05uOFhlbmRMbTA0Mm1wTENWOCtHd3lzYnVFM1BHRDdNV3pDaVVicm0rbXdBLzk0c3hTODlTNkJpVWhnUHp5RC84TWhyVUNNL1FTRGNFY3ZUTjVFc0N0UDM1cUdUT28yOWdxc3VzdWRLZHdEQkhWMlpkaGNNR0xQMElWNEZKN01CQVZSMnd4OTRiZXpDMm4xU3V5TGRGVVBQYVFKa2wwWmw2M3E4MU5FRjdMSzQ0M0FJbzlpV3FuazltbFBYRVo1OHdVUERnMUpZbWw4b3BCYVprazJtM2dvYk5HdEFWUHY1dDlXZitXY2Q2MDN3WnJ1TlhwUTNPSlk2WWI4ZXBMNlZpN1ErTkpaa2Z0NWl5M1FQRFpUUFZjSCs0c1VjZ0E2dmFMSUY2aEZCUncwWitRS0pvK0VZUWtFK0RTQXhMaldFYkt5ZzBSN1V3UHg0VThENjQ4My9mMlV2cU5jSFRORHNkbXRKcjlXcUwxNHRoc1BqQTNqZ3Bqc0pydDJJWTA1bEdNOWJJbGpmbUtGWFdsemppQ2ptSUNsSm14SUxIdzgvSTlKb3JYb2NmNXpoSHVzbCswUkdKc1NMTHAyOWc9PSIsImRhdGFrZXkiOiJBUUVCQUhod20wWWFJU0plUnRKbTVuMUc2dXFlZWtYdW9YWFBlNVVGY2U5UnE4LzE0d0FBQUg0d2ZBWUpLb1pJaHZjTkFRY0dvRzh3YlFJQkFEQm9CZ2txaGtpRzl3MEJCd0V3SGdZSllJWklBV1VEQkFFdU1CRUVERGFyODZkalUxNVFHNCtZaEFJQkVJQTdvY0xIeWFpUHViY2VTQ0g5djB6THd2UFZGbHU0WmJqZ09JSGkrdmxiNEpCVTBlNyt5VmpnT3BpcWVmQlkxbFBGWktKalgvMEIwMkJDcU1nPSIsInZlcnNpb24iOiIyIiwidHlwZSI6IkRBVEFfS0VZIiwiZXhwaXJhdGlvbiI6MTY5MjYxNDYwMX0=", + "registry": "https://535945872701.dkr.ecr.us-east-1.amazonaws.com" +} \ No newline at end of file From 6c51d81668da5e69a5ba973e02f1c413234c88fd Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:28:07 -0700 Subject: [PATCH 012/165] printing urls Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 1 + tests/image_upload_test.py | 1 + 2 files changed, 2 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index b0166a6b..e584ed21 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -243,6 +243,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Dict: Backend response """ login_url = f"{config.BACKEND_URL}sdk/ecr/login" + print(login_url) if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 08d7990d..8a18ff28 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -16,6 +16,7 @@ def test_login(): url = f"{config.BACKEND_URL}/sdk/ecr/login" + print(url) with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/login_response.json")) as f: mock_json = json.load(f) From c25bbe9a1369ef53329b7ad08a538c86b977e960 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:29:03 -0700 Subject: [PATCH 013/165] Corrected test URL Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 8a18ff28..4dbb9c4e 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -15,7 +15,7 @@ def test_login(): - url = f"{config.BACKEND_URL}/sdk/ecr/login" + url = f"{config.BACKEND_URL}sdk/ecr/login" print(url) with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/login_response.json")) as f: From 64663e892d38f76e63f5e4d7a84deb8cdf1c83d9 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:30:50 -0700 Subject: [PATCH 014/165] Changing url to post Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 4dbb9c4e..1fcf8a5a 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -20,7 +20,7 @@ def test_login(): with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/login_response.json")) as f: mock_json = json.load(f) - mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) creds_dict = creds.to_dict() mock_dict = mock_json.to_dict() From 46805739d8af76894fa403c893c1f6de8cb30a36 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:32:05 -0700 Subject: [PATCH 015/165] pulling response correctly Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 1fcf8a5a..fcaab825 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -22,6 +22,6 @@ def test_login(): mock_json = json.load(f) mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) - creds_dict = creds.to_dict() + creds_dict = creds.text.to_dict() mock_dict = mock_json.to_dict() assert creds_dict == mock_dict \ No newline at end of file From 7ed30998904dd83c2e1ba8d30446e5cd36c25d4a Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:33:18 -0700 Subject: [PATCH 016/165] pulling response correctly Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index fcaab825..ef2b0f32 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -22,6 +22,6 @@ def test_login(): mock_json = json.load(f) mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) - creds_dict = creds.text.to_dict() + creds_dict = json.loads(creds) mock_dict = mock_json.to_dict() assert creds_dict == mock_dict \ No newline at end of file From 6e7fced210fc8d53e9688e2c35b6c44da2238d66 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:34:41 -0700 Subject: [PATCH 017/165] to json Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index ef2b0f32..95acc1cb 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -22,6 +22,6 @@ def test_login(): mock_json = json.load(f) mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) - creds_dict = json.loads(creds) + creds_dict = creds.json() mock_dict = mock_json.to_dict() assert creds_dict == mock_dict \ No newline at end of file From de770d94806b6c70b3f072f0cf144e99ad84e1e2 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 16:35:10 -0700 Subject: [PATCH 018/165] to json Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 95acc1cb..86775166 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -23,5 +23,5 @@ def test_login(): mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) creds_dict = creds.json() - mock_dict = mock_json.to_dict() + mock_dict = mock_json assert creds_dict == mock_dict \ No newline at end of file From d8cb963803e2fa59d768598a1eb4081ed939af53 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:01:45 -0700 Subject: [PATCH 019/165] Create_asset_repo_test Signed-off-by: mikelam-us --- tests/image_upload_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 86775166..f6bbc9e3 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -16,12 +16,19 @@ def test_login(): url = f"{config.BACKEND_URL}sdk/ecr/login" - print(url) with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/login_response.json")) as f: mock_json = json.load(f) mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) - creds_dict = creds.json() - mock_dict = mock_json - assert creds_dict == mock_dict \ No newline at end of file + assert creds == mock_json + +def test_create_asset_repo(): + url = f"{config.BACKEND_URL}sdk/models/register" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: + mock_json = json.load(f) + mock.post(url, headers=API_FIXED_HEADER, json=mock_json) + model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", True, "mock_version", + "mock_description", "mock_function", False, config.TEAM_API_KEY) + assert model_id == mock_json \ No newline at end of file From 05b3747fe89f6447d3c9ca04a32b309581dc3095 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:02:13 -0700 Subject: [PATCH 020/165] create_asset_repo response Signed-off-by: mikelam-us --- tests/mock_responses/create_asset_repo_response.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/mock_responses/create_asset_repo_response.json diff --git a/tests/mock_responses/create_asset_repo_response.json b/tests/mock_responses/create_asset_repo_response.json new file mode 100644 index 00000000..b9606471 --- /dev/null +++ b/tests/mock_responses/create_asset_repo_response.json @@ -0,0 +1,3 @@ +{ + "modelId": "mockId" +} \ No newline at end of file From c1be93acde7c013a7fc607560230fa2af93fe8a1 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:04:09 -0700 Subject: [PATCH 021/165] dict conversion error Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index e584ed21..8f7ab418 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -229,7 +229,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v } payload = json.dumps(payload) response = _request_with_retry("post", create_url, headers=headers, data=payload) - return response + return response.json() @classmethod def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: @@ -243,13 +243,12 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Dict: Backend response """ login_url = f"{config.BACKEND_URL}sdk/ecr/login" - print(login_url) if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("post", login_url, headers=headers) - return response + return response.json() @classmethod def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: @@ -297,7 +296,7 @@ def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Option } payload = json.dumps(payload) response = _request_with_retry("post", is_onboarded_url, headers=headers, data=payload) - return response + return response.json() @classmethod def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Dict: From 4cd6454e7c6cd394d20bdf196c74627fa5885345 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:08:56 -0700 Subject: [PATCH 022/165] Added list_host_machines test Signed-off-by: mikelam-us --- tests/image_upload_test.py | 11 ++++++++++- .../list_host_machines_response.json | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/mock_responses/list_host_machines_response.json diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index f6bbc9e3..0e940116 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -31,4 +31,13 @@ def test_create_asset_repo(): mock.post(url, headers=API_FIXED_HEADER, json=mock_json) model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", True, "mock_version", "mock_description", "mock_function", False, config.TEAM_API_KEY) - assert model_id == mock_json \ No newline at end of file + assert model_id == mock_json + +def list_host_machines(): + url = f"{config.BACKEND_URL}sdk/hosting-machines" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_host_machines_response.json")) as f: + mock_json = json.load(f) + mock.post(url, headers=API_FIXED_HEADER, json=mock_json) + machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) + assert machines == mock_json \ No newline at end of file diff --git a/tests/mock_responses/list_host_machines_response.json b/tests/mock_responses/list_host_machines_response.json new file mode 100644 index 00000000..35bcb7e4 --- /dev/null +++ b/tests/mock_responses/list_host_machines_response.json @@ -0,0 +1,18 @@ +[ + { + "id": "64dce914adc92335dc35beb5", + "code": "aix-2c-8g-od", + "type": "on-demand", + "cores": 2, + "memory": 8, + "hourlyCost": 0.12 + }, + { + "id": "64dceafdadc92335dc35beb6", + "code": "aix-2c-8g", + "type": "always-on", + "cores": 2, + "memory": 8, + "hourlyCost": 0.096 + } +] \ No newline at end of file From ba97b6cf7ea992d3de0a54729d93fc35d5a25b7c Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:09:54 -0700 Subject: [PATCH 023/165] corrected test function naem Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 0e940116..4ca4d4b5 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -33,7 +33,7 @@ def test_create_asset_repo(): "mock_description", "mock_function", False, config.TEAM_API_KEY) assert model_id == mock_json -def list_host_machines(): +def test_list_host_machines(): url = f"{config.BACKEND_URL}sdk/hosting-machines" with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/list_host_machines_response.json")) as f: From 0e0c9b1ba9c92c288b230bc277ed21b5f1a7accd Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:11:50 -0700 Subject: [PATCH 024/165] post -> get Signed-off-by: mikelam-us --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 4ca4d4b5..1f54e40e 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -38,6 +38,6 @@ def test_list_host_machines(): with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/list_host_machines_response.json")) as f: mock_json = json.load(f) - mock.post(url, headers=API_FIXED_HEADER, json=mock_json) + mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) assert machines == mock_json \ No newline at end of file From d64d9ca03fb4dc9a8ec0a0ee03916d091b8f9e6f Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:12:50 -0700 Subject: [PATCH 025/165] dict conversion Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 8f7ab418..fa13dbc5 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -192,7 +192,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - return response + return response.json() @classmethod def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, From dcc0d7b7ded17679d923462301b05992a02ae1c1 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 20:18:34 -0700 Subject: [PATCH 026/165] Added image_repo_tag test Signed-off-by: mikelam-us --- tests/image_upload_test.py | 12 +++++++++++- .../list_image_repo_tags_response.json | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/mock_responses/list_image_repo_tags_response.json diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 1f54e40e..dd3c7c1a 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -40,4 +40,14 @@ def test_list_host_machines(): mock_json = json.load(f) mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) - assert machines == mock_json \ No newline at end of file + assert machines == mock_json + +def test_list_image_repo_tags(): + model_id = "mock_id" + url = f"{config.BACKEND_URL}sdk/models/{model_id}/images" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_image_repo_tags_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + tags = ModelFactory.list_image_repo_tags(config.TEAM_API_KEY) + assert tags == mock_json \ No newline at end of file diff --git a/tests/mock_responses/list_image_repo_tags_response.json b/tests/mock_responses/list_image_repo_tags_response.json new file mode 100644 index 00000000..f69b90dc --- /dev/null +++ b/tests/mock_responses/list_image_repo_tags_response.json @@ -0,0 +1 @@ +["tag1", "tag2", "tag3", "tag4"] \ No newline at end of file From fc3ca6eeb742d3bdc73b04cec9e2a2008d23a423 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 20:50:22 -0700 Subject: [PATCH 027/165] Finished list tags test Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- tests/image_upload_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index fa13dbc5..fdd32d79 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -315,4 +315,4 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("post", list_url, headers=headers) - return response \ No newline at end of file + return response.json() \ No newline at end of file diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index dd3c7c1a..f679fa52 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -49,5 +49,5 @@ def test_list_image_repo_tags(): with open(Path("tests/mock_responses/list_image_repo_tags_response.json")) as f: mock_json = json.load(f) mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) - tags = ModelFactory.list_image_repo_tags(config.TEAM_API_KEY) + tags = ModelFactory.list_image_repo_tags(model_id, config.TEAM_API_KEY) assert tags == mock_json \ No newline at end of file From 72271990b2dfdd1650436113156818380a73b98e Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 20:51:24 -0700 Subject: [PATCH 028/165] Debugging list tags test Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index fdd32d79..d6dd1f96 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -314,5 +314,5 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} - response = _request_with_retry("post", list_url, headers=headers) + response = _request_with_retry("get", list_url, headers=headers) return response.json() \ No newline at end of file From 017f4dfcb9f432773013f4b300f4caddb6c74ae4 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 21:02:38 -0700 Subject: [PATCH 029/165] Added is_onboarded Signed-off-by: mikelam-us --- tests/image_upload_test.py | 14 +++++++++++++- tests/mock_responses/is_onboarded_response.json | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 tests/mock_responses/is_onboarded_response.json diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index f679fa52..4ba26191 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -50,4 +50,16 @@ def test_list_image_repo_tags(): mock_json = json.load(f) mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) tags = ModelFactory.list_image_repo_tags(model_id, config.TEAM_API_KEY) - assert tags == mock_json \ No newline at end of file + assert tags == mock_json + +def test_is_onboarded(): + model_id = "mock_id" + host = "mock_host" + version = "mock_version" + url = f"{config.BACKEND_URL}webhook/models/onboarding" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/is_onboarded_response.json")) as f: + mock_json = json.load(f) + mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) + is_onboarded_response = ModelFactory.is_onboarded(model_id, host, version, config.TEAM_API_KEY) + assert is_onboarded_response == mock_json \ No newline at end of file diff --git a/tests/mock_responses/is_onboarded_response.json b/tests/mock_responses/is_onboarded_response.json new file mode 100644 index 00000000..a959a4dc --- /dev/null +++ b/tests/mock_responses/is_onboarded_response.json @@ -0,0 +1,3 @@ +{ + "isOnboarded": true +} \ No newline at end of file From bf208f60cbcfbe7d8b6a01274b8c18a0f4465fa2 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 21 Aug 2023 21:01:00 -0700 Subject: [PATCH 030/165] Added list_functions function + test Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 19 + tests/image_upload_test.py | 9 + .../list_functions_response.json | 2880 +++++++++++++++++ 3 files changed, 2908 insertions(+) create mode 100644 tests/mock_responses/list_functions_response.json diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index d6dd1f96..0a3548f0 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -194,6 +194,25 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: response = _request_with_retry("get", machines_url, headers=headers) return response.json() + @classmethod + def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: + """Lists supported model functions on platform. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each supported function. + """ + functions_url = f"{config.BACKEND_URL}sdk/functions" + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("get", functions_url, headers=headers) + return response.json() + @classmethod def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, description: Text, function: Text, is_async: bool, api_key: Optional[Text] = None) -> Dict: diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 4ba26191..4bcd5cc1 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -42,6 +42,15 @@ def test_list_host_machines(): machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) assert machines == mock_json +def test_get_functions(): + url = f"{config.BACKEND_URL}sdk/functions" + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_functions_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + functions = ModelFactory.list_functions(config.TEAM_API_KEY) + assert functions == mock_json + def test_list_image_repo_tags(): model_id = "mock_id" url = f"{config.BACKEND_URL}sdk/models/{model_id}/images" diff --git a/tests/mock_responses/list_functions_response.json b/tests/mock_responses/list_functions_response.json new file mode 100644 index 00000000..0512e79d --- /dev/null +++ b/tests/mock_responses/list_functions_response.json @@ -0,0 +1,2880 @@ +{ + "total": 55, + "pageTotal": 55, + "items": [ + { + "id": "language-identification", + "name": "Language Identification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "ocr", + "name": "OCR", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "featuretypes", + "name": "Feature Types", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + } + ] + }, + { + "id": "image-label-detection", + "name": "Image Label Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "video-forced-alignment", + "name": "Video Forced Alignment", + "output": [ + { + "name": "Text", + "code": "text", + "dataType": "text", + "defaultValue": [] + }, + { + "name": "Video", + "code": "video", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "offensive-language-identification", + "name": "Offensive Language Identification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-forced-alignment", + "name": "Audio Forced Alignment", + "output": [ + { + "name": "Text", + "code": "text", + "dataType": "text", + "defaultValue": [] + }, + { + "name": "Audio", + "code": "audio", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-generation", + "name": "Video Generation", + "output": [ + { + "name": "Video", + "code": "data", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "split-on-silence", + "name": "Split On Silence", + "output": [ + { + "name": "Segments", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-audio-generation-metric", + "name": "Referenceless Audio Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-text-generation-metric-default", + "name": "Referenceless Text Generation Metric Default", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speaker-diarization-video", + "name": "Speaker Diarization Video", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-generation-metric", + "name": "Audio Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "text-reconstruction", + "name": "Text Reconstruction", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Segments", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speech-classification", + "name": "Speech Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "metric-aggregation", + "name": "Metric Aggregation", + "output": [ + { + "name": "Aggregates", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "text", + "name": "Score Aggregation Data", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + } + ] + }, + { + "id": "dialect-detection", + "name": "Dialect Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "fill-text-mask", + "name": "Fill Text Mask", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "json", + "multipleValues": true + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-content-moderation", + "name": "Video Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "image-compression", + "name": "Image Compression", + "output": [ + { + "name": "image", + "code": "image", + "dataType": "image", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "apl_qfactor", + "name": "apl_qfactor", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "80", + "label": "80" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "extract-audio-from-video", + "name": "Extract Audio From Video", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio" + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video" + } + ] + }, + { + "id": "speech-non-speech-classification", + "name": "Speech or Non-Speech Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-text-generation-metric", + "name": "Referenceless Text Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "text-generation-metric-default", + "name": "Text Generation Metric Default", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speech-embedding", + "name": "Speech Embedding", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-reconstruction", + "name": "Audio Reconstruction", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Segments", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + } + ] + }, + { + "id": "voice-cloning", + "name": "Voice Cloning", + "output": [ + { + "name": "Target Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "type", + "name": "Type", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-generation", + "name": "Text Generation", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "session_id", + "name": "session_id", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + }, + { + "code": "option_index", + "name": "option_index", + "required": false, + "isFixed": false, + "dataType": "integer", + "dataSubType": "integer" + }, + { + "code": "next_state", + "name": "next_state", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-transcript-improvement", + "name": "Audio Transcript Improvement", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "ASR Supplier", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "is_medical", + "name": "Is Medical", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "diacritization", + "name": "Diacritization", + "output": [ + { + "name": "Target Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "emotion-detection", + "name": "Emotion Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-summarization", + "name": "Text summarization", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "entity-linking", + "name": "Entity Linking", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "domain", + "name": "Domain", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-spam-detection", + "name": "Text Spam Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-generation-metric", + "name": "Text Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "split-on-linebreak", + "name": "Split On Linebreak", + "output": [ + { + "name": "text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "voice-activity-detection", + "name": "Voice Activity Detection", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "onset", + "name": "Onset", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "offset", + "name": "Offset", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "min_duration_on", + "name": "Min Duration On", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "1", + "label": "1" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "min_duration_off", + "name": "Min Duration Off", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "sentiment-analysis", + "name": "Sentiment Analysis", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-classification", + "name": "Text Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "speech-synthesis", + "name": "Speech Synthesis", + "output": [ + { + "name": "Target Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "type", + "name": "Type", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-label-detection", + "name": "Video Label Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "asr-quality-estimation", + "name": "ASR Quality Estimation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "json", + "multipleValues": true + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-content-moderation", + "name": "Text Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "image-content-moderation", + "name": "Image Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "audio-transcript-analysis", + "name": "Audio Transcript Analysis", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "ASR Supplier", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-denormalization", + "name": "Text Denormalization", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "lowercase_latin", + "name": "To Lower Case", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0", + "label": "No" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "remove_accents", + "name": "Remove Accents", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "1", + "label": "Yes" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "remove_punctuation", + "name": "Remove Punctuation", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0", + "label": "No" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + } + ] + }, + { + "id": "search", + "name": "Search", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speaker-diarization-audio", + "name": "Speaker Diarization Audio", + "output": [ + { + "name": "Segments", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "named-entity-recognition", + "name": "Named Entity Recognition", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "domain", + "name": "Domain", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "topic-classification", + "name": "Topic Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-to-image-generation", + "name": "Text To Image Generation", + "output": [ + { + "name": "Generated Image", + "code": "image", + "dataType": "image", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text Prompt", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-normalization", + "name": "Text Normalization", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "settings", + "name": "Settings", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "remove urls", + "label": "remove urls" + }, + { + "value": "remove emails", + "label": "remove emails" + }, + { + "value": "remove phone numbers", + "label": "remove phone numbers" + }, + { + "value": "remove emojis", + "label": "remove emojis" + }, + { + "value": "remove html tags", + "label": "remove html tags" + }, + { + "value": "normalize quotes", + "label": "normalize quotes" + }, + { + "value": "lowercase text", + "label": "lowercase text" + }, + { + "value": "remove default arabic diacritics", + "label": "remove default arabic diacritics" + }, + { + "value": "remove full arabic diacritics", + "label": "remove full arabic diacritics" + }, + { + "value": "normalize default arabic", + "label": "normalize default arabic" + }, + { + "value": "normalize full arabic", + "label": "normalize full arabic" + }, + { + "value": "remove arabic superfluous", + "label": "remove arabic superfluous" + }, + { + "value": "remove kashida dagger", + "label": "remove kashida dagger" + }, + { + "value": "normalize spoken text", + "label": "normalize spoken text" + }, + { + "value": "denormalize spoken text", + "label": "denormalize spoken text" + }, + { + "value": "tokenize text", + "label": "tokenize text" + } + ], + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + } + ] + }, + { + "id": "subtitling-translation", + "name": "Subtitling Translation", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "target_supplier", + "name": "Machine Translation Supplier", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "modernmt", + "label": "ModernMT" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "google", + "label": "Google" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguages", + "name": "Target Languages", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "en", + "label": "English" + }, + { + "value": "de", + "label": "German" + }, + { + "value": "nl", + "label": "Dutch" + }, + { + "value": "fr", + "label": "French" + }, + { + "value": "el", + "label": "Greek" + }, + { + "value": "it", + "label": "Italian" + }, + { + "value": "pl", + "label": "Polish" + }, + { + "value": "pt", + "label": "Portuguese" + }, + { + "value": "ru", + "label": "Russian" + }, + { + "value": "es", + "label": "Spanish" + }, + { + "value": "zh", + "label": "Chinese" + }, + { + "value": "sl", + "label": "Slovenian" + }, + { + "value": "ar", + "label": "Arabic" + }, + { + "value": "ja", + "label": "Japanese" + }, + { + "value": "ko", + "label": "Korean" + }, + { + "value": "tr", + "label": "Turkish" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": true + } + ] + }, + { + "id": "translation", + "name": "Translation", + "output": [ + { + "name": "Target Text", + "code": "data", + "dataType": "text", + "defaultValue": true + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguage", + "name": "Target Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script_in", + "name": "Script In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script_out", + "name": "Script Out", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_out", + "name": "Dialect Out", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "context", + "name": "Context", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "subtitling", + "name": "Subtitling", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "Speech Recognition Supplier", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "aws", + "label": "AWS" + } + ], + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "google", + "label": "Google" + }, + { + "value": "deepgram", + "label": "Deepgram" + }, + { + "value": "revai", + "label": "Rev.AI" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "openai", + "label": "OpenAI" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "target_supplier", + "name": "Machine Translation Supplier", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "modernmt", + "label": "ModernMT" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "google", + "label": "Google" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguages", + "name": "Target Languages", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "en", + "label": "English" + }, + { + "value": "de", + "label": "German" + }, + { + "value": "nl", + "label": "Dutch" + }, + { + "value": "fr", + "label": "French" + }, + { + "value": "el", + "label": "Greek" + }, + { + "value": "it", + "label": "Italian" + }, + { + "value": "pl", + "label": "Polish" + }, + { + "value": "pt", + "label": "Portuguese" + }, + { + "value": "ru", + "label": "Russian" + }, + { + "value": "es", + "label": "Spanish" + }, + { + "value": "zh", + "label": "Chinese" + }, + { + "value": "sl", + "label": "Slovenian" + }, + { + "value": "ar", + "label": "Arabic" + }, + { + "value": "ja", + "label": "Japanese" + }, + { + "value": "ko", + "label": "Korean" + }, + { + "value": "tr", + "label": "Turkish" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": true + } + ] + }, + { + "id": "speech-recognition", + "name": "Speech Recognition", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + } + ] +} \ No newline at end of file From e4d4d2cbfe9ce9694029220452cedbebbad0f650 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Fri, 25 Aug 2023 08:46:56 -0700 Subject: [PATCH 031/165] e2e image upload testing Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 0a3548f0..c40632f0 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -225,7 +225,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v always_on (bool): Whether the model should always be on version (Text): Model version description (Text): Model description - function (Text): Model funciton obtained via #TODO add function endpoint + function (Text): Model function obtained via #TODO add function endpoint is_async (bool): Whether the model is asynchronous or not api_key (Text, optional): Team API key. Defaults to None. From 26546c8608a08b029caafee9cf2f40a5ef6652d6 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Wed, 30 Aug 2023 10:42:12 -0700 Subject: [PATCH 032/165] Updated image upload test Signed-off-by: mikelam-us --- tests/image_upload_test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 4bcd5cc1..284907be 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -1,14 +1,11 @@ __author__ = "michaellam" import json -import pytest import requests_mock from pathlib import Path from aixplain.utils import config from aixplain.factories.model_factory import ModelFactory -from aixplain.factories.dataset_factory import DatasetFactory -from aixplain.factories.metric_factory import MetricFactory AUTH_FIXED_HEADER = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} API_FIXED_HEADER = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} From 823ece20f3ad49af73e19c9348237da10f0cda57 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 5 Sep 2023 10:57:11 -0700 Subject: [PATCH 033/165] Create_asset_repo updated to include source language Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c40632f0..178e1d00 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -214,8 +214,9 @@ def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: return response.json() @classmethod - def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, - description: Text, function: Text, is_async: bool, api_key: Optional[Text] = None) -> Dict: + def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, is_async: bool, + source_language: Text, api_key: Optional[Text] = None) -> Dict: """Creates an image repository for this model and registers it in the platform backend. @@ -237,6 +238,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + always_on = False payload = { "name": name, "hostingMachine": hosting_machine, @@ -244,7 +246,8 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, v "version": version, "description": description, "function": function, - "isAsync": is_async + "isAsync": is_async, + "sourceLanguage": source_language } payload = json.dumps(payload) response = _request_with_retry("post", create_url, headers=headers, data=payload) From fe5d56bbe794225c433dd38ccdd24518a167c366 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 5 Sep 2023 21:14:31 -0700 Subject: [PATCH 034/165] Added comments to model_factory Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 178e1d00..2e6756ed 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -226,8 +226,9 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, always_on (bool): Whether the model should always be on version (Text): Model version description (Text): Model description - function (Text): Model function obtained via #TODO add function endpoint + function (Text): Model function obtained via LIST_HOST_MACHINES is_async (bool): Whether the model is asynchronous or not + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. api_key (Text, optional): Team API key. Defaults to None. Returns: From 7920f04960956ec743122ae0fde53d7f96f3db69 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 6 Sep 2023 10:35:52 -0700 Subject: [PATCH 035/165] Added logging, minor change to URLs Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 2e6756ed..f399e813 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -186,7 +186,8 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each hosting machine. """ - machines_url = f"{config.BACKEND_URL}sdk/hosting-machines" + machines_url = f"{config.BACKEND_URL}/sdk/hosting-machines" + logging.debug(f"URL: {machines_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -205,7 +206,8 @@ def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each supported function. """ - functions_url = f"{config.BACKEND_URL}sdk/functions" + functions_url = f"{config.BACKEND_URL}/sdk/functions" + logging.debug(f"URL: {functions_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -234,7 +236,8 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, Returns: Dict: Backend response """ - create_url = f"{config.BACKEND_URL}sdk/models/register" + create_url = f"{config.BACKEND_URL}/sdk/models/register" + logging.debug(f"URL: {create_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -251,6 +254,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, "sourceLanguage": source_language } payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) return response.json() @@ -265,7 +269,8 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Returns: Dict: Backend response """ - login_url = f"{config.BACKEND_URL}sdk/ecr/login" + login_url = f"{config.BACKEND_URL}/sdk/ecr/login" + logging.debug(f"URL: {login_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -284,7 +289,8 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke Returns: Dict: Backend response """ - onboard_url = f"{config.BACKEND_URL}sdk/inventory/models/{model_id}/onboarding" + onboard_url = f"{config.BACKEND_URL}/sdk/inventory/models/{model_id}/onboarding" + logging.debug(f"URL: {onboard_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -294,6 +300,7 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke "sha": image_hash } payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", onboard_url, headers=headers, data=payload) return response @@ -307,7 +314,8 @@ def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Option Returns: Dict: Backend response """ - is_onboarded_url = f"{config.BACKEND_URL}webhook/models/onboarding" + is_onboarded_url = f"{config.BACKEND_URL}/webhook/models/onboarding" + logging.debug(f"URL: {is_onboarded_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: @@ -317,6 +325,7 @@ def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Option "host": host, "version": version } + logging.debug(f"Body: {str(payload)}") payload = json.dumps(payload) response = _request_with_retry("post", is_onboarded_url, headers=headers, data=payload) return response.json() @@ -332,7 +341,8 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Returns: Dict: Backend response """ - list_url = f"{config.BACKEND_URL}sdk/models/{model_id}/images" + list_url = f"{config.BACKEND_URL}/sdk/models/{model_id}/images" + logging.debug(f"URL: {list_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: From 436d05fde8d51c23a5e1f17d56a142d0d43ca185 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 7 Sep 2023 11:48:35 -0700 Subject: [PATCH 036/165] Corrected URL generation Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 37 +++++------------------------ 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index f399e813..391507d9 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -186,7 +186,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each hosting machine. """ - machines_url = f"{config.BACKEND_URL}/sdk/hosting-machines" + machines_url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") logging.debug(f"URL: {machines_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -206,7 +206,7 @@ def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each supported function. """ - functions_url = f"{config.BACKEND_URL}/sdk/functions" + functions_url = urljoin(config.BACKEND_URL, f"sdk/functions") logging.debug(f"URL: {functions_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -236,7 +236,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, Returns: Dict: Backend response """ - create_url = f"{config.BACKEND_URL}/sdk/models/register" + create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") logging.debug(f"URL: {create_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -269,7 +269,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Returns: Dict: Backend response """ - login_url = f"{config.BACKEND_URL}/sdk/ecr/login" + login_url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") logging.debug(f"URL: {login_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -289,7 +289,7 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke Returns: Dict: Backend response """ - onboard_url = f"{config.BACKEND_URL}/sdk/inventory/models/{model_id}/onboarding" + onboard_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}/onboarding") logging.debug(f"URL: {onboard_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -304,31 +304,6 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke response = _request_with_retry("post", onboard_url, headers=headers, data=payload) return response - @classmethod - def is_onboarded(cls, model_id: Text, host: Text, version: Text, api_key: Optional[Text] = None): - """Check whether a model has been onboarded. - - Args: - model_id (Text): Model ID obtained from CREATE_ASSET_REPO. - api_key (Text, optional): Team API key. Defaults to None. - Returns: - Dict: Backend response - """ - is_onboarded_url = f"{config.BACKEND_URL}/webhook/models/onboarding" - logging.debug(f"URL: {is_onboarded_url}") - if api_key: - headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} - else: - headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} - payload = { - "id": model_id, - "host": host, - "version": version - } - logging.debug(f"Body: {str(payload)}") - payload = json.dumps(payload) - response = _request_with_retry("post", is_onboarded_url, headers=headers, data=payload) - return response.json() @classmethod def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Dict: @@ -341,7 +316,7 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Returns: Dict: Backend response """ - list_url = f"{config.BACKEND_URL}/sdk/models/{model_id}/images" + list_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/images") logging.debug(f"URL: {list_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} From aa45ae970368cd78b1140c520f969934482c76e7 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 09:42:03 -0700 Subject: [PATCH 037/165] Removed ID field from host machines Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 391507d9..fe42ecdf 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,7 +193,9 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - return response.json() + response_dict = response.json() + del response_dict["id"] + return response_dict @classmethod def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: From ad5044c7c92d0a40db26957315156614fc5ac827 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 09:50:57 -0700 Subject: [PATCH 038/165] deleting key Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index fe42ecdf..570df98f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,7 +193,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - response_dict = response.json() + response_dict = dict(response.json()) del response_dict["id"] return response_dict From b93073b8793a920574374895027208e4643edea8 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 09:52:48 -0700 Subject: [PATCH 039/165] Corrected json -> dict conversion Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 570df98f..97141000 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,7 +193,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - response_dict = dict(response.json()) + response_dict = json.loads(response.text) del response_dict["id"] return response_dict From b2251d0e12bcabc1f4742648b7a05d4c7897876e Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 09:58:36 -0700 Subject: [PATCH 040/165] Corrected json -> dict conversion again Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 97141000..03636763 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -194,7 +194,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) response_dict = json.loads(response.text) - del response_dict["id"] + response_dict.pop("id") return response_dict @classmethod From 11e8d5ff6562e29e93756f64cde882be2d3102a4 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 09:59:38 -0700 Subject: [PATCH 041/165] Corrected json -> dict conversion again with dict typing Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 03636763..ee98a3d7 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,7 +193,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - response_dict = json.loads(response.text) + response_dict = dict(json.loads(response.text)) response_dict.pop("id") return response_dict From c42fa762c36006ad593f6d3e3d51581b93a64f6f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 10:01:23 -0700 Subject: [PATCH 042/165] Adding debugging : Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index ee98a3d7..c3cbac5a 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,6 +193,8 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) + print(response) + print(json.loads(response.text)) response_dict = dict(json.loads(response.text)) response_dict.pop("id") return response_dict From 30a6f929c68c75eba823cca723416c0985b390e2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 10:03:01 -0700 Subject: [PATCH 043/165] Corrected for list Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c3cbac5a..25232497 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -195,9 +195,10 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: response = _request_with_retry("get", machines_url, headers=headers) print(response) print(json.loads(response.text)) - response_dict = dict(json.loads(response.text)) - response_dict.pop("id") - return response_dict + response_dicts = json.loads(response.text) + for dictionary in response_dicts: + del dictionary["id"] + return response_dicts @classmethod def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: From ac9ffc740be003e377b5f9b8550266a8432e7f85 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 10:03:57 -0700 Subject: [PATCH 044/165] removed print statements Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 25232497..82651bf5 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -193,8 +193,6 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", machines_url, headers=headers) - print(response) - print(json.loads(response.text)) response_dicts = json.loads(response.text) for dictionary in response_dicts: del dictionary["id"] From 56534bde20fb9f4eb95828d999ca941c30a44a72 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 10:39:44 -0700 Subject: [PATCH 045/165] Added 'verbose' option for list_functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 82651bf5..7315da6b 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -199,7 +199,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: return response_dicts @classmethod - def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: + def list_functions(cls, verbose: bool = False, api_key: Optional[Text] = None) -> List[Dict]: """Lists supported model functions on platform. Args: @@ -216,7 +216,14 @@ def list_functions(cls, api_key: Optional[Text] = None) -> List[Dict]: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", functions_url, headers=headers) - return response.json() + response_dict = json.loads(response.text) + if verbose: + return response_dict + function_list = response_dict["items"] + for function_dict in function_list: + del function_dict["output"] + del function_dict["params"] + return response_dict @classmethod def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, From 53c5ab33bfd536dcdf712ed9923e4a71d7d5af81 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 12 Sep 2023 11:19:31 -0700 Subject: [PATCH 046/165] Function name instead of ID Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 7315da6b..b9c6f08c 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -223,6 +223,7 @@ def list_functions(cls, verbose: bool = False, api_key: Optional[Text] = None) - for function_dict in function_list: del function_dict["output"] del function_dict["params"] + del function_dict["id"] return response_dict @classmethod @@ -238,14 +239,22 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, always_on (bool): Whether the model should always be on version (Text): Model version description (Text): Model description - function (Text): Model function obtained via LIST_HOST_MACHINES - is_async (bool): Whether the model is asynchronous or not + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. api_key (Text, optional): Team API key. Defaults to None. Returns: Dict: Backend response """ + # Reconcile function mame to be function ID in the backend + function_list = cls.list_functions(True, cls.api_key)["items"] + function_id = None + for function_dict in function_list: + if function_dict["name"] == function: + function_id = function_dict["id"] + if function_id is None: + raise Exception("Invalid function name") create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") logging.debug(f"URL: {create_url}") if api_key: @@ -253,13 +262,14 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} always_on = False + is_async = False # Hard-coded to False for first release payload = { "name": name, "hostingMachine": hosting_machine, "alwaysOn": always_on, "version": version, "description": description, - "function": function, + "function": function_id, "isAsync": is_async, "sourceLanguage": source_language } From 48d375c08e665b0e15a80dd92c117415520ac326 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 13 Sep 2023 09:27:28 -0700 Subject: [PATCH 047/165] First try at setting environment variables during login Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index b9c6f08c..fe66b034 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,6 +29,7 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn +import os import click class ModelFactory: @@ -279,11 +280,12 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, return response.json() @classmethod - def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: + def asset_repo_login(cls, set_env: bool = False, api_key: Optional[Text] = None) -> Dict: """Return login credentials for the image repository that corresponds with the given API_KEY. Args: + set_env (bool, optional): If True, sets the login variables as environment variables api_key (Text, optional): Team API key. Defaults to None. Returns: @@ -296,7 +298,12 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("post", login_url, headers=headers) - return response.json() + response_dict = json.loads(response.text) + if set_env: + os.environ["USERNAME"] = response_dict["username"] + os.environ["PASSWORD"] = response_dict["password"] + os.environ["REGISTRY"] = response_dict["registry"] + return response_dict @classmethod def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: From 7280d39f5559145170dc7aabcaf0a3227ae4fe3a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 13 Sep 2023 15:53:06 -0700 Subject: [PATCH 048/165] Removed 'is_onboarded' from test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 284907be..ef5b76fd 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -56,16 +56,4 @@ def test_list_image_repo_tags(): mock_json = json.load(f) mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) tags = ModelFactory.list_image_repo_tags(model_id, config.TEAM_API_KEY) - assert tags == mock_json - -def test_is_onboarded(): - model_id = "mock_id" - host = "mock_host" - version = "mock_version" - url = f"{config.BACKEND_URL}webhook/models/onboarding" - with requests_mock.Mocker() as mock: - with open(Path("tests/mock_responses/is_onboarded_response.json")) as f: - mock_json = json.load(f) - mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) - is_onboarded_response = ModelFactory.is_onboarded(model_id, host, version, config.TEAM_API_KEY) - assert is_onboarded_response == mock_json \ No newline at end of file + assert tags == mock_json \ No newline at end of file From de9b29d091e556bd658e903e54e78c64f97f8e4d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 13 Sep 2023 16:05:36 -0700 Subject: [PATCH 049/165] Corrected tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index ef5b76fd..d11f02b7 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -4,6 +4,7 @@ import requests_mock from pathlib import Path from aixplain.utils import config +from urllib.parse import urljoin from aixplain.factories.model_factory import ModelFactory @@ -12,7 +13,7 @@ def test_login(): - url = f"{config.BACKEND_URL}sdk/ecr/login" + url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/login_response.json")) as f: mock_json = json.load(f) @@ -21,7 +22,7 @@ def test_login(): assert creds == mock_json def test_create_asset_repo(): - url = f"{config.BACKEND_URL}sdk/models/register" + url = urljoin(config.BACKEND_URL, f"sdk/models/register") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: mock_json = json.load(f) @@ -31,7 +32,7 @@ def test_create_asset_repo(): assert model_id == mock_json def test_list_host_machines(): - url = f"{config.BACKEND_URL}sdk/hosting-machines" + url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/list_host_machines_response.json")) as f: mock_json = json.load(f) @@ -40,7 +41,7 @@ def test_list_host_machines(): assert machines == mock_json def test_get_functions(): - url = f"{config.BACKEND_URL}sdk/functions" + url = urljoin(config.BACKEND_URL, f"sdk/functions") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/list_functions_response.json")) as f: mock_json = json.load(f) @@ -50,7 +51,7 @@ def test_get_functions(): def test_list_image_repo_tags(): model_id = "mock_id" - url = f"{config.BACKEND_URL}sdk/models/{model_id}/images" + url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/images") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/list_image_repo_tags_response.json")) as f: mock_json = json.load(f) From 728774efde26ffad02b6b88958aa4a4d213740b1 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 11:13:02 -0700 Subject: [PATCH 050/165] Testing CLI Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 32 +++++++++++++++++++++++++++++ aixplain/factories/model_factory.py | 6 +++--- setup.py | 2 +- 3 files changed, 36 insertions(+), 4 deletions(-) create mode 100644 aixplain/cli_groups.py diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py new file mode 100644 index 00000000..36cac08e --- /dev/null +++ b/aixplain/cli_groups.py @@ -0,0 +1,32 @@ +import click +from aixplain.factories.model_factory import ModelFactory + +@click.group('cli') +def cli(): + pass + +@click.group('list') +def list(): + pass + +@click.group('get') +def get(): + pass + +@click.group('create') +def create(): + pass + +@click.group('onboard') +def onboard(): + pass + +cli.add_command(list) +cli.add_command(get) +cli.add_command(create) +cli.add_command(onboard) + +list.add_command(ModelFactory.list_host_machines) + +def run_cli(): + cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index fe66b034..9b5dede2 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -30,6 +30,7 @@ from urllib.parse import urljoin from warnings import warn import os +from aixplain.cli_groups import list, get, create, onboard import click class ModelFactory: @@ -173,10 +174,9 @@ def get_first_k_assets( error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" logging.error(error_message) return [] - + + @list.command("hosts") @classmethod - # @click.command() - # @click.option("--team-api-key", default=None, help="Team API key") def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. diff --git a/setup.py b/setup.py index f96e8c30..1bd9800f 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ }, entry_points = { 'console_scripts': [ - 'hosted-machines = aixplain.factories.model_factory:ModelFactory.list_host_machines' + 'aixplain = aixplain.cli_groups:run_cli' ] }, ) From cdf560d242f47f79dd6808a47883fecc4350cf5c Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 11:29:18 -0700 Subject: [PATCH 051/165] Removing circular import Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 9b5dede2..864cb419 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -30,7 +30,6 @@ from urllib.parse import urljoin from warnings import warn import os -from aixplain.cli_groups import list, get, create, onboard import click class ModelFactory: @@ -175,7 +174,7 @@ def get_first_k_assets( logging.error(error_message) return [] - @list.command("hosts") + @click.command("hosts") @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 5a7f7af4f32dad31329cdc77bf5bd2d476f0b72a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 11:55:04 -0700 Subject: [PATCH 052/165] Trying swapping classmethod decorator Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 864cb419..cdfdafd3 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -173,9 +173,9 @@ def get_first_k_assets( error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" logging.error(error_message) return [] - + + @classmethod @click.command("hosts") - @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From a84696102fcf76732b08570e6a3400f462eee7aa Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 12:02:53 -0700 Subject: [PATCH 053/165] Experimenting with calls Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index cdfdafd3..ba1bebeb 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -173,9 +173,9 @@ def get_first_k_assets( error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" logging.error(error_message) return [] - - @classmethod + @click.command("hosts") + @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 3d445f8cd1c84cc086657da8435cf252a597124d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 13:08:51 -0700 Subject: [PATCH 054/165] static method Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index ba1bebeb..7d4826ac 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -175,8 +175,8 @@ def get_first_k_assets( return [] @click.command("hosts") - @classmethod - def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: + @staticmethod + def list_host_machines(api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. Args: From 28f50f400b26f61841f7fe5cc26337f55ad75777 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 14 Sep 2023 13:16:54 -0700 Subject: [PATCH 055/165] random stuff Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 36cac08e..cac0e889 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -26,7 +26,7 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -list.add_command(ModelFactory.list_host_machines) +list.add_command(lambda: ModelFactory.list_host_machines()) def run_cli(): cli() \ No newline at end of file From a1a70af61b9d97de75be1bb2c873167b14983167 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 07:54:05 -0700 Subject: [PATCH 056/165] non-classmethod Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 +- aixplain/factories/model_factory.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index cac0e889..36cac08e 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -26,7 +26,7 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -list.add_command(lambda: ModelFactory.list_host_machines()) +list.add_command(ModelFactory.list_host_machines) def run_cli(): cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 7d4826ac..d535f0c1 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -175,7 +175,6 @@ def get_first_k_assets( return [] @click.command("hosts") - @staticmethod def list_host_machines(api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 2ac1b2d12ccbc32b5066398fcd2d7e01cf645939 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 07:59:38 -0700 Subject: [PATCH 057/165] Experimenting with non-classmethod Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index d535f0c1..d0d3a91d 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -175,7 +175,7 @@ def get_first_k_assets( return [] @click.command("hosts") - def list_host_machines(api_key: Optional[Text] = None) -> List[Dict]: + def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. Args: From debe3b184d527a4ac871e0dac25e6337c3a71761 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 08:01:43 -0700 Subject: [PATCH 058/165] Experimenting with non-classmethod Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 36cac08e..2626013b 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -26,7 +26,9 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -list.add_command(ModelFactory.list_host_machines) +def mock(): + return ModelFactory.list_host_machines(ModelFactory) +list.add_command(mock) def run_cli(): cli() \ No newline at end of file From d7ddc9fc88cc8716182052c7dab2b538a26a4128 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:03:20 -0700 Subject: [PATCH 059/165] testing out non-classmethod Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 11 +++++------ aixplain/factories/model_factory.py | 6 +++++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 2626013b..6ffa98ce 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,13 +1,14 @@ import click -from aixplain.factories.model_factory import ModelFactory +from aixplain.factories.model_factory import ModelFactory, test_func @click.group('cli') def cli(): pass @click.group('list') -def list(): - pass +@click.pass_context +def list(ctx): + ctx.obj = ModelFactory() @click.group('get') def get(): @@ -26,9 +27,7 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -def mock(): - return ModelFactory.list_host_machines(ModelFactory) -list.add_command(mock) +list.add_command(test_func) def run_cli(): cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index d0d3a91d..5711296e 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -348,4 +348,8 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", list_url, headers=headers) - return response.json() \ No newline at end of file + return response.json() + +@click.command("test") +def test_func(): + return "trolololololol" \ No newline at end of file From a1111682c251fca3839cfed878af68ecdcdaaeb5 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:05:00 -0700 Subject: [PATCH 060/165] print Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 5711296e..5e2cd655 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -352,4 +352,5 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> @click.command("test") def test_func(): + print("test1") return "trolololololol" \ No newline at end of file From 23146636876d4520e9922dddc688cbc940e0c3d2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:16:38 -0700 Subject: [PATCH 061/165] Get another error Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 1 + aixplain/factories/model_factory.py | 1 + 2 files changed, 2 insertions(+) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 6ffa98ce..e0b4719f 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -28,6 +28,7 @@ def onboard(): cli.add_command(onboard) list.add_command(test_func) +list.add_command(ModelFactory.list_host_machines) def run_cli(): cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 5e2cd655..b2bebfb5 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -195,6 +195,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: response_dicts = json.loads(response.text) for dictionary in response_dicts: del dictionary["id"] + click.echo(response_dicts) return response_dicts @classmethod From b15a15a752abd40efd746819b2150109bf3b068f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:17:34 -0700 Subject: [PATCH 062/165] Added classmethod decorator Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index b2bebfb5..32bd997a 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -175,6 +175,7 @@ def get_first_k_assets( return [] @click.command("hosts") + @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 53b3204d11301c5b3d9236e9fa2acc12006e93b1 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:20:28 -0700 Subject: [PATCH 063/165] Adding __get__ Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index e0b4719f..19b4067f 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -28,7 +28,7 @@ def onboard(): cli.add_command(onboard) list.add_command(test_func) -list.add_command(ModelFactory.list_host_machines) +list.add_command(ModelFactory.list_host_machines__get__(ModelFactory)) def run_cli(): cli() \ No newline at end of file From 9b72db29fc15988b9a34667b151d26a3312b5d69 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:21:09 -0700 Subject: [PATCH 064/165] Corrected syntax error Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 19b4067f..f391f1c8 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -28,7 +28,7 @@ def onboard(): cli.add_command(onboard) list.add_command(test_func) -list.add_command(ModelFactory.list_host_machines__get__(ModelFactory)) +list.add_command(ModelFactory.list_host_machines.__get__(ModelFactory)) def run_cli(): cli() \ No newline at end of file From 87e124c22e7c82bbfcfd34991adeedcba6e836b0 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:22:23 -0700 Subject: [PATCH 065/165] Added debugger Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index f391f1c8..e42a989c 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -31,4 +31,6 @@ def onboard(): list.add_command(ModelFactory.list_host_machines.__get__(ModelFactory)) def run_cli(): + import ipdb + ipdb.set_trace() cli() \ No newline at end of file From 8670959e01634dadb141e17ca31dd5b969e9f3e8 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:27:15 -0700 Subject: [PATCH 066/165] Adding separate cli file Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 8 +++----- aixplain/factories/model_factory_cli.py | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 aixplain/factories/model_factory_cli.py diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index e42a989c..46a3c1a0 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,5 +1,5 @@ import click -from aixplain.factories.model_factory import ModelFactory, test_func +from aixplain.factories.model_factory_cli import list_host_machines @click.group('cli') def cli(): @@ -27,10 +27,8 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -list.add_command(test_func) -list.add_command(ModelFactory.list_host_machines.__get__(ModelFactory)) +# list.add_command(test_func) +list.add_command(list_host_machines) def run_cli(): - import ipdb - ipdb.set_trace() cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py new file mode 100644 index 00000000..af16aac4 --- /dev/null +++ b/aixplain/factories/model_factory_cli.py @@ -0,0 +1,8 @@ +from aixplain.factories.model_factory import ModelFactory +import click + +@click.command("hosts") +def list_host_machines(): + ret_val = ModelFactory.list_host_machines() + click.echo(ret_val) + return ret_val \ No newline at end of file From 7688b11c646f2758260d5e718ab84ccdc2bbec36 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:42:32 -0700 Subject: [PATCH 067/165] Removed unused line Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 46a3c1a0..b4c9f11b 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,5 +1,6 @@ import click from aixplain.factories.model_factory_cli import list_host_machines +from aixplain.factories.model_factory import ModelFactory @click.group('cli') def cli(): @@ -8,7 +9,7 @@ def cli(): @click.group('list') @click.pass_context def list(ctx): - ctx.obj = ModelFactory() + pass @click.group('get') def get(): From c4b719aedb0e2fe721bf5b13e6d2a7c3b881ac91 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 10:51:12 -0700 Subject: [PATCH 068/165] Removed unnecessary decorator Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 32bd997a..a3c816f5 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -174,7 +174,6 @@ def get_first_k_assets( logging.error(error_message) return [] - @click.command("hosts") @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From f7e64c00391530d968e6bf20ceb72d1dd2dd39fb Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 11:47:01 -0700 Subject: [PATCH 069/165] Adding arguments Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 -- aixplain/factories/model_factory.py | 1 - aixplain/factories/model_factory_cli.py | 11 ++++++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index b4c9f11b..54f92e58 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,6 +1,5 @@ import click from aixplain.factories.model_factory_cli import list_host_machines -from aixplain.factories.model_factory import ModelFactory @click.group('cli') def cli(): @@ -28,7 +27,6 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) -# list.add_command(test_func) list.add_command(list_host_machines) def run_cli(): diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index a3c816f5..7b8a643a 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -195,7 +195,6 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: response_dicts = json.loads(response.text) for dictionary in response_dicts: del dictionary["id"] - click.echo(response_dicts) return response_dicts @classmethod diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py index af16aac4..c9ee0b2b 100644 --- a/aixplain/factories/model_factory_cli.py +++ b/aixplain/factories/model_factory_cli.py @@ -2,7 +2,12 @@ import click @click.command("hosts") -def list_host_machines(): - ret_val = ModelFactory.list_host_machines() +@click.argument('--api-key', default=None, help='Number of greetings.') +def list_host_machines(api_key): + ret_val = ModelFactory.list_host_machines(api_key) click.echo(ret_val) - return ret_val \ No newline at end of file + return ret_val + +@click.command("functions") +def list_functions(verbose): + pass \ No newline at end of file From a994a40bfe9a683d583b07e08bcbafc4f87a722a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 11:49:56 -0700 Subject: [PATCH 070/165] argument->option Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py index c9ee0b2b..82fa06c7 100644 --- a/aixplain/factories/model_factory_cli.py +++ b/aixplain/factories/model_factory_cli.py @@ -2,7 +2,7 @@ import click @click.command("hosts") -@click.argument('--api-key', default=None, help='Number of greetings.') +@click.option('--api-key', default=None, help='Number of greetings.') def list_host_machines(api_key): ret_val = ModelFactory.list_host_machines(api_key) click.echo(ret_val) From 0025c2529f3c599027a2b67382e68e563caad881 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 11:57:34 -0700 Subject: [PATCH 071/165] Added list_functions cli Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory_cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py index 82fa06c7..b5fd4cf0 100644 --- a/aixplain/factories/model_factory_cli.py +++ b/aixplain/factories/model_factory_cli.py @@ -2,12 +2,15 @@ import click @click.command("hosts") -@click.option('--api-key', default=None, help='Number of greetings.') +@click.option("--api-key", default=None) def list_host_machines(api_key): ret_val = ModelFactory.list_host_machines(api_key) click.echo(ret_val) return ret_val @click.command("functions") -def list_functions(verbose): - pass \ No newline at end of file +@click.option('--verbose', default=False) +def list_functions(verbose, api_key): + ret_val = ModelFactory.list_functions(verbose, api_key) + click.echo(ret_val) + return ret_val \ No newline at end of file From 129dc2945fa31b64dca3bce835050fe49e8aa1a0 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 11:59:29 -0700 Subject: [PATCH 072/165] Added list_functions cli Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 54f92e58..dfd31f1f 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,5 +1,5 @@ import click -from aixplain.factories.model_factory_cli import list_host_machines +from aixplain.factories.model_factory_cli import list_host_machines, list_functions @click.group('cli') def cli(): @@ -28,6 +28,7 @@ def onboard(): cli.add_command(onboard) list.add_command(list_host_machines) +list.add_command(list_functions) def run_cli(): cli() \ No newline at end of file From 0fb76308b5cd77b24e382a4a3646c7b63d824096 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Fri, 15 Sep 2023 12:00:45 -0700 Subject: [PATCH 073/165] Fixed function decorator Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory_cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py index b5fd4cf0..ce8d551d 100644 --- a/aixplain/factories/model_factory_cli.py +++ b/aixplain/factories/model_factory_cli.py @@ -10,6 +10,7 @@ def list_host_machines(api_key): @click.command("functions") @click.option('--verbose', default=False) +@click.option("--api-key", default=None) def list_functions(verbose, api_key): ret_val = ModelFactory.list_functions(verbose, api_key) click.echo(ret_val) From 27d2a4952271dc6c85fb3c168c9e604072851f7d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 08:21:55 -0700 Subject: [PATCH 074/165] Finished CLI draft Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 27 ++++++++++++++++++++++- aixplain/factories/model_factory.py | 29 ++++++++++--------------- aixplain/factories/model_factory_cli.py | 17 --------------- 3 files changed, 38 insertions(+), 35 deletions(-) delete mode 100644 aixplain/factories/model_factory_cli.py diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index dfd31f1f..4847eece 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -1,5 +1,27 @@ +__author__ = "aiXplain" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Michael Lam +Date: September 18th 2023 +Description: + CLI Runner +""" import click -from aixplain.factories.model_factory_cli import list_host_machines, list_functions +from aixplain.factories.cli.model_factory_cli import list_host_machines, list_functions, create_asset_repo, asset_repo_login, onboard_model @click.group('cli') def cli(): @@ -29,6 +51,9 @@ def onboard(): list.add_command(list_host_machines) list.add_command(list_functions) +create.add_command(create_asset_repo) +get.add_command(asset_repo_login) +onboard.add_command(onboard_model) def run_cli(): cli() \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 7b8a643a..fb13b693 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,8 +29,6 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn -import os -import click class ModelFactory: """A static class for creating and exploring Model Objects. @@ -198,10 +196,13 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: return response_dicts @classmethod - def list_functions(cls, verbose: bool = False, api_key: Optional[Text] = None) -> List[Dict]: + def list_functions(cls, verbose: Optional[bool] = False, + api_key: Optional[Text] = None) -> List[Dict]: """Lists supported model functions on platform. Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. api_key (Text, optional): Team API key. Defaults to None. Returns: @@ -225,10 +226,14 @@ def list_functions(cls, verbose: bool = False, api_key: Optional[Text] = None) - del function_dict["id"] return response_dict + # Will add "always_on" and "is_async" when we support them. + # def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + # description: Text, function: Text, is_async: bool, + # source_language: Text, api_key: Optional[Text] = None) -> Dict: @classmethod def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, - description: Text, function: Text, is_async: bool, - source_language: Text, api_key: Optional[Text] = None) -> Dict: + description: Text, function: Text, source_language: Text, + api_key: Optional[Text] = None) -> Dict: """Creates an image repository for this model and registers it in the platform backend. @@ -278,12 +283,11 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, return response.json() @classmethod - def asset_repo_login(cls, set_env: bool = False, api_key: Optional[Text] = None) -> Dict: + def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: """Return login credentials for the image repository that corresponds with the given API_KEY. Args: - set_env (bool, optional): If True, sets the login variables as environment variables api_key (Text, optional): Team API key. Defaults to None. Returns: @@ -297,10 +301,6 @@ def asset_repo_login(cls, set_env: bool = False, api_key: Optional[Text] = None) headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("post", login_url, headers=headers) response_dict = json.loads(response.text) - if set_env: - os.environ["USERNAME"] = response_dict["username"] - os.environ["PASSWORD"] = response_dict["password"] - os.environ["REGISTRY"] = response_dict["registry"] return response_dict @classmethod @@ -348,9 +348,4 @@ def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> else: headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} response = _request_with_retry("get", list_url, headers=headers) - return response.json() - -@click.command("test") -def test_func(): - print("test1") - return "trolololololol" \ No newline at end of file + return response.json() \ No newline at end of file diff --git a/aixplain/factories/model_factory_cli.py b/aixplain/factories/model_factory_cli.py deleted file mode 100644 index ce8d551d..00000000 --- a/aixplain/factories/model_factory_cli.py +++ /dev/null @@ -1,17 +0,0 @@ -from aixplain.factories.model_factory import ModelFactory -import click - -@click.command("hosts") -@click.option("--api-key", default=None) -def list_host_machines(api_key): - ret_val = ModelFactory.list_host_machines(api_key) - click.echo(ret_val) - return ret_val - -@click.command("functions") -@click.option('--verbose', default=False) -@click.option("--api-key", default=None) -def list_functions(verbose, api_key): - ret_val = ModelFactory.list_functions(verbose, api_key) - click.echo(ret_val) - return ret_val \ No newline at end of file From cdd222cfe9709ecef3e2e1057b0fc9b4cb84b8b1 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 10:48:31 -0700 Subject: [PATCH 075/165] Removed unnecessary context Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index 4847eece..ba9297c0 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -28,8 +28,7 @@ def cli(): pass @click.group('list') -@click.pass_context -def list(ctx): +def list(): pass @click.group('get') From 34810e82e9b02c0e2e0e05f1deaf94e8dd2f2798 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 10:51:20 -0700 Subject: [PATCH 076/165] Experimenting Signed-off-by: mikelam-us-aixplain --- aixplain/cli_groups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index ba9297c0..d61a6caa 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -48,9 +48,9 @@ def onboard(): cli.add_command(create) cli.add_command(onboard) +create.add_command(create_asset_repo) list.add_command(list_host_machines) list.add_command(list_functions) -create.add_command(create_asset_repo) get.add_command(asset_repo_login) onboard.add_command(onboard_model) From 091a8a38078b86f71c8c7c3ba2356784c0c58696 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 10:52:27 -0700 Subject: [PATCH 077/165] Refactored directories Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 124 ++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 aixplain/factories/cli/model_factory_cli.py diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py new file mode 100644 index 00000000..c0c2ed76 --- /dev/null +++ b/aixplain/factories/cli/model_factory_cli.py @@ -0,0 +1,124 @@ +__author__ = "aiXplain" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Michael Lam +Date: September 18th 2023 +Description: + Model Factory CLI +""" + +from aixplain.factories.model_factory import ModelFactory +from typing import Dict, List, Optional, Text +import click + +@click.command("hosts") +@click.option("--api-key", default=None) +def list_host_machines(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_HOST_MACHINES function in + ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.list_host_machines(api_key) + click.echo(ret_val) + +@click.command("functions") +@click.option("--verbose", default=False) +@click.option("--api-key", default=None) +def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_FUNCTIONS function in ModelFactory. + + Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + None + """ + ret_val = ModelFactory.list_functions(verbose, api_key) + click.echo(ret_val) + +@click.command("image-repo") +@click.option("--name") +@click.option("--hosting-machine") +@click.option("--version") +@click.option("--description") +@click.option("--function") +@click.option("--source-language", default="en") +@click.option("--api-key", default=None) +def create_asset_repo(name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, + source_language: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the CREATE_ASSET_REPO function in ModelFactory. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, + description, function, + source_language, api_key) + click.echo(ret_val) + +@click.command("image-repo-login") +@click.option("--api-key", default=None) +def asset_repo_login(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ASSET_REPO_LOGIN function in ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.asset_repo_login(api_key) + click.echo(ret_val) + +@click.command("model") +@click.option("--model-id") +@click.option("--image-tag") +@click.option("--image-hash") +@click.option("--api-key", default=None) +def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ONBOARD_MODEL function in ModelFactory. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) + click.echo(ret_val) \ No newline at end of file From 8879c2eb4ef023ce06fe16f0e821cf6a5bdd9509 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 11:56:26 -0700 Subject: [PATCH 078/165] Adding debug print statements to create_asset_repo Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index fb13b693..6df1a12d 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -267,6 +267,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} always_on = False is_async = False # Hard-coded to False for first release + print(function_id) payload = { "name": name, "hostingMachine": hosting_machine, @@ -277,6 +278,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, "isAsync": is_async, "sourceLanguage": source_language } + print(payload) payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) From 7f4d2b22c318a8a969135829547ca315a5a3ae56 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:06:12 -0700 Subject: [PATCH 079/165] debugging weird function Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 6df1a12d..287dcfb8 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -253,6 +253,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, """ # Reconcile function mame to be function ID in the backend function_list = cls.list_functions(True, cls.api_key)["items"] + print(function_list) function_id = None for function_dict in function_list: if function_dict["name"] == function: From b76542f7119aa4519e3ffe44bb73cc31842746c4 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:07:34 -0700 Subject: [PATCH 080/165] id -> code Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 287dcfb8..c52cf978 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -257,7 +257,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, function_id = None for function_dict in function_list: if function_dict["name"] == function: - function_id = function_dict["id"] + function_id = function_dict["code"] if function_id is None: raise Exception("Invalid function name") create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") From d24a942145366193b73bd0342d6e420259f4be87 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:10:50 -0700 Subject: [PATCH 081/165] Removed debugging statements Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c52cf978..cab36edb 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -253,7 +253,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, """ # Reconcile function mame to be function ID in the backend function_list = cls.list_functions(True, cls.api_key)["items"] - print(function_list) function_id = None for function_dict in function_list: if function_dict["name"] == function: @@ -268,7 +267,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} always_on = False is_async = False # Hard-coded to False for first release - print(function_id) payload = { "name": name, "hostingMachine": hosting_machine, @@ -279,7 +277,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, "isAsync": is_async, "sourceLanguage": source_language } - print(payload) payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) From 1cb72e18f29e22a61b917b661e920962392b5784 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:11:46 -0700 Subject: [PATCH 082/165] Removed unused import Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index cab36edb..da3f5250 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -24,7 +24,6 @@ import json import logging from aixplain.modules.model import Model -from aixplain.utils.config import MODELS_RUN_URL from aixplain.utils import config from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin From ac56f69a6ccd26bc2b37a59e10e90275f5b6ae1d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 07:57:14 -0700 Subject: [PATCH 083/165] Correcting rudimentary tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 4 ++-- tests/mock_responses/is_onboarded_response.json | 3 --- tests/mock_responses/list_host_machines_response.json | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 tests/mock_responses/is_onboarded_response.json diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index d11f02b7..ccf93361 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -27,8 +27,8 @@ def test_create_asset_repo(): with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: mock_json = json.load(f) mock.post(url, headers=API_FIXED_HEADER, json=mock_json) - model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", True, "mock_version", - "mock_description", "mock_function", False, config.TEAM_API_KEY) + model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", + "mock_description", "mock_function", "en", config.TEAM_API_KEY) assert model_id == mock_json def test_list_host_machines(): diff --git a/tests/mock_responses/is_onboarded_response.json b/tests/mock_responses/is_onboarded_response.json deleted file mode 100644 index a959a4dc..00000000 --- a/tests/mock_responses/is_onboarded_response.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "isOnboarded": true -} \ No newline at end of file diff --git a/tests/mock_responses/list_host_machines_response.json b/tests/mock_responses/list_host_machines_response.json index 35bcb7e4..0f2f1e97 100644 --- a/tests/mock_responses/list_host_machines_response.json +++ b/tests/mock_responses/list_host_machines_response.json @@ -1,6 +1,5 @@ [ { - "id": "64dce914adc92335dc35beb5", "code": "aix-2c-8g-od", "type": "on-demand", "cores": 2, @@ -8,7 +7,6 @@ "hourlyCost": 0.12 }, { - "id": "64dceafdadc92335dc35beb6", "code": "aix-2c-8g", "type": "always-on", "cores": 2, From 3141d0e5957de2f580825a2ce42ce77caa010f43 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 08:09:05 -0700 Subject: [PATCH 084/165] Debugging image upload test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 5 ++++- tests/mock_responses/list_host_machines_response.json | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index ccf93361..fe968ab0 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -38,7 +38,10 @@ def test_list_host_machines(): mock_json = json.load(f) mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) - assert machines == mock_json + machine_dict = json.loads(machines) + mock_json_dict = json.loads(mock_json) + for key in machine_dict.keys(): + assert machine_dict[key] == mock_json_dict[key] def test_get_functions(): url = urljoin(config.BACKEND_URL, f"sdk/functions") diff --git a/tests/mock_responses/list_host_machines_response.json b/tests/mock_responses/list_host_machines_response.json index 0f2f1e97..35bcb7e4 100644 --- a/tests/mock_responses/list_host_machines_response.json +++ b/tests/mock_responses/list_host_machines_response.json @@ -1,5 +1,6 @@ [ { + "id": "64dce914adc92335dc35beb5", "code": "aix-2c-8g-od", "type": "on-demand", "cores": 2, @@ -7,6 +8,7 @@ "hourlyCost": 0.12 }, { + "id": "64dceafdadc92335dc35beb6", "code": "aix-2c-8g", "type": "always-on", "cores": 2, From ae18a5b9c9a4e62b34a0d0d36f73922a327335cd Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 08:18:33 -0700 Subject: [PATCH 085/165] Working on list_host_machines test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index fe968ab0..c987da34 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -38,10 +38,17 @@ def test_list_host_machines(): mock_json = json.load(f) mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) - machine_dict = json.loads(machines) - mock_json_dict = json.loads(mock_json) - for key in machine_dict.keys(): - assert machine_dict[key] == mock_json_dict[key] + machine_dict_list = [] + for machine in machines: + machine_dict_list.append(json.loads(machine)) + mock_json_list = [] + for mock_json in mock_json_list: + mock_json_list.append(json.loads(mock_json)) + for i in range(len(machine_dict_list)): + machine_dict = machine_dict_list[i] + mock_json_dict = mock_json_list[i] + for key in machine_dict.keys(): + assert machine_dict[key] == mock_json_dict[key] def test_get_functions(): url = urljoin(config.BACKEND_URL, f"sdk/functions") From 532bc3b5ba11c3a3f2ed4e8fd5dbe26c0b0947e3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 08:23:17 -0700 Subject: [PATCH 086/165] Working on list_host_machines test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index c987da34..62178984 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -38,15 +38,17 @@ def test_list_host_machines(): mock_json = json.load(f) mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) + ''' machine_dict_list = [] for machine in machines: machine_dict_list.append(json.loads(machine)) mock_json_list = [] for mock_json in mock_json_list: mock_json_list.append(json.loads(mock_json)) - for i in range(len(machine_dict_list)): - machine_dict = machine_dict_list[i] - mock_json_dict = mock_json_list[i] + ''' + for i in range(len(machines)): + machine_dict = machines[i] + mock_json_dict = mock_json[i] for key in machine_dict.keys(): assert machine_dict[key] == mock_json_dict[key] From 99b9df74266f277afa27a5e008a299748c976e8a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 08:26:00 -0700 Subject: [PATCH 087/165] Debugging create_asset_repo test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 62178984..2b6162db 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -27,6 +27,9 @@ def test_create_asset_repo(): with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: mock_json = json.load(f) mock.post(url, headers=API_FIXED_HEADER, json=mock_json) + with open(Path("tests/mock_responses/list_functions_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", "mock_description", "mock_function", "en", config.TEAM_API_KEY) assert model_id == mock_json @@ -38,14 +41,6 @@ def test_list_host_machines(): mock_json = json.load(f) mock.get(url, headers=API_FIXED_HEADER, json=mock_json) machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) - ''' - machine_dict_list = [] - for machine in machines: - machine_dict_list.append(json.loads(machine)) - mock_json_list = [] - for mock_json in mock_json_list: - mock_json_list.append(json.loads(mock_json)) - ''' for i in range(len(machines)): machine_dict = machines[i] mock_json_dict = mock_json[i] From 9f5eac4fea13033e892119d9c52eb7aa1b18f6ee Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 09:23:22 -0700 Subject: [PATCH 088/165] Debugging image upload test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 2b6162db..a1c1387e 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -22,14 +22,15 @@ def test_login(): assert creds == mock_json def test_create_asset_repo(): - url = urljoin(config.BACKEND_URL, f"sdk/models/register") + url_register = urljoin(config.BACKEND_URL, f"sdk/models/register") + url_function = urljoin(config.BACKEND_URL, f"sdk/functions") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: mock_json = json.load(f) - mock.post(url, headers=API_FIXED_HEADER, json=mock_json) + mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json) with open(Path("tests/mock_responses/list_functions_response.json")) as f: mock_json = json.load(f) - mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json) model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", "mock_description", "mock_function", "en", config.TEAM_API_KEY) assert model_id == mock_json From ed2888c5a5bb0a895d2a22db7abe489a81c521e3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 09:28:02 -0700 Subject: [PATCH 089/165] Corrected function name in test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index a1c1387e..2716b828 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -32,7 +32,7 @@ def test_create_asset_repo(): mock_json = json.load(f) mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json) model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", - "mock_description", "mock_function", "en", config.TEAM_API_KEY) + "mock_description", "Speech Recognition", "en", config.TEAM_API_KEY) assert model_id == mock_json def test_list_host_machines(): From cb5331c94d490aa7fe82576caf2f9ee205a630a3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 09:30:38 -0700 Subject: [PATCH 090/165] Corrected ID Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index da3f5250..2fa3e5f6 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -255,7 +255,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, function_id = None for function_dict in function_list: if function_dict["name"] == function: - function_id = function_dict["code"] + function_id = function_dict["id"] if function_id is None: raise Exception("Invalid function name") create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") From 9e9b3eb424aea6bb181380ce84c08e28f5df22e8 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 09:40:21 -0700 Subject: [PATCH 091/165] Corrected test errors Signed-off-by: mikelam-us-aixplain --- tests/image_upload_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 2716b828..23446477 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -26,14 +26,14 @@ def test_create_asset_repo(): url_function = urljoin(config.BACKEND_URL, f"sdk/functions") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: - mock_json = json.load(f) - mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json) + mock_json_register = json.load(f) + mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json_register) with open(Path("tests/mock_responses/list_functions_response.json")) as f: - mock_json = json.load(f) - mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json) + mock_json_functions = json.load(f) + mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json_functions) model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", "mock_description", "Speech Recognition", "en", config.TEAM_API_KEY) - assert model_id == mock_json + assert model_id == mock_json_register def test_list_host_machines(): url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") From d467a37bcd300b62960efec83a8952f2a4e67392 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 10:25:19 -0700 Subject: [PATCH 092/165] Removing 'results' from list_function Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 2fa3e5f6..f2705102 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -218,6 +218,7 @@ def list_functions(cls, verbose: Optional[bool] = False, response_dict = json.loads(response.text) if verbose: return response_dict + del response_dict["results"] function_list = response_dict["items"] for function_dict in function_list: del function_dict["output"] From 08e54370fd4a405b6b690e8b5208caa574830f91 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 20 Sep 2023 07:39:16 -0700 Subject: [PATCH 093/165] 'mame' -> 'name' Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index f2705102..dd4ee68f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -251,7 +251,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, Returns: Dict: Backend response """ - # Reconcile function mame to be function ID in the backend + # Reconcile function name to be function ID in the backend function_list = cls.list_functions(True, cls.api_key)["items"] function_id = None for function_dict in function_list: From 44658eadd619eee0a4e3373d36b81820d26cbe7c Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 20 Sep 2023 12:04:46 -0700 Subject: [PATCH 094/165] Adding pretty-print for list_functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index c0c2ed76..801238e6 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -24,6 +24,7 @@ from aixplain.factories.model_factory import ModelFactory from typing import Dict, List, Optional, Text import click +import yaml @click.command("hosts") @click.option("--api-key", default=None) @@ -54,7 +55,8 @@ def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.list_functions(verbose, api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("image-repo") @click.option("--name") From bc14db8d822f588962b79dbfedb7b5d2fa4f5e55 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 20 Sep 2023 12:08:11 -0700 Subject: [PATCH 095/165] Added pretty-print for all functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index 801238e6..bcd41fbe 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -39,7 +39,8 @@ def list_host_machines(api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.list_host_machines(api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("functions") @click.option("--verbose", default=False) @@ -89,7 +90,8 @@ def create_asset_repo(name: Text, hosting_machine: Text, version: Text, ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, description, function, source_language, api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("image-repo-login") @click.option("--api-key", default=None) @@ -103,7 +105,8 @@ def asset_repo_login(api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.asset_repo_login(api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("model") @click.option("--model-id") @@ -123,4 +126,5 @@ def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, None """ ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) - click.echo(ret_val) \ No newline at end of file + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) \ No newline at end of file From 645e22f80109fdc9f7e2cce222725ab6c2b68d23 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 07:45:55 -0700 Subject: [PATCH 096/165] Added functional test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/image_upload_functional_test.py diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py new file mode 100644 index 00000000..12d29798 --- /dev/null +++ b/tests/image_upload_functional_test.py @@ -0,0 +1,57 @@ +__author__ = "michaellam" +from pathlib import Path +import json +import requests + +from aixplain.factories.model_factory import ModelFactory + +def test_login(): + response = ModelFactory.asset_repo_login() + assert response["username"] == "AWS" + assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + +def test_create_asset_repo(): + with open(Path("tests/mock_requests/create_asset_request.json")) as f: + mock_register_payload = json.load(f) + name = mock_register_payload["name"] + host_machine = mock_register_payload["hostingMachine"] + version = mock_register_payload["version"] + description = mock_register_payload["description"] + function = mock_register_payload["function"] + source_language = mock_register_payload["source_language"] + response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + response_dict = dict(response) + assert "id" in response_dict.keys() + assert "repositoryName" in response_dict.keys() + +def test_list_host_machines(): + response = ModelFactory.list_host_machines() + for hosting_machine_dict in response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() + +def test_get_functions(): + # Verbose + response = ModelFactory.list_functions(True) + items = response["items"] + for item in items: + assert "output" in item.keys() + assert "params" in item.keys() + assert "id" in item.keys() + assert "name" in item.keys() + + # Non-verbose + response = ModelFactory.list_functions() # Not verbose by default + for function in response: + assert "output" not in function.keys() + assert "params" not in function.keys() + assert "id" not in function.keys() + assert "name" in function.keys() + +def list_image_repo_tags(): + response = ModelFactory.list_image_repo_tags() + assert "Image tags" in response.keys() + assert "nextToken" in response.keys() \ No newline at end of file From 002f48db6afac07a2350305b5a4ca7e5048f34fd Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 07:57:45 -0700 Subject: [PATCH 097/165] Added test cleanup. Testing tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 26 ++++++++++++++++++- tests/test_requests/create_asset_request.json | 8 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/test_requests/create_asset_request.json diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 12d29798..923635a5 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -2,6 +2,10 @@ from pathlib import Path import json import requests +import logging +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin +from aixplain.utils import config from aixplain.factories.model_factory import ModelFactory @@ -9,6 +13,7 @@ def test_login(): response = ModelFactory.asset_repo_login() assert response["username"] == "AWS" assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in response.keys() def test_create_asset_repo(): with open(Path("tests/mock_requests/create_asset_request.json")) as f: @@ -24,6 +29,9 @@ def test_create_asset_repo(): assert "id" in response_dict.keys() assert "repositoryName" in response_dict.keys() + # Test cleanup + delete_asset(requests["id"], config.TEAM_API_KEY) + def test_list_host_machines(): response = ModelFactory.list_host_machines() for hosting_machine_dict in response: @@ -54,4 +62,20 @@ def test_get_functions(): def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() assert "Image tags" in response.keys() - assert "nextToken" in response.keys() \ No newline at end of file + assert "nextToken" in response.keys() + +def delete_asset(model_id, api_key): + """List the contents of the image repository corresponding to API_KEY. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + response = _request_with_retry("delete", delete_url, headers=headers) + return response.json() \ No newline at end of file diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json new file mode 100644 index 00000000..bb8cc9a9 --- /dev/null +++ b/tests/test_requests/create_asset_request.json @@ -0,0 +1,8 @@ +{ + "name": "mock_name", + "hostingMachine": "aix-2c-8g-od", + "version": "mock_version", + "description": "mock_description", + "function": "speech-recognition", + "sourceLanguage": "en" +} \ No newline at end of file From c7ba3cd9245748389baf5fe6988e25c0fc1bace7 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:01:26 -0700 Subject: [PATCH 098/165] Adding debugging print statements Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 923635a5..48e35b68 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -54,6 +54,7 @@ def test_get_functions(): # Non-verbose response = ModelFactory.list_functions() # Not verbose by default for function in response: + print(function) assert "output" not in function.keys() assert "params" not in function.keys() assert "id" not in function.keys() From c8003fd3e45edfbfcec9beb9d825c3009608b098 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:09:23 -0700 Subject: [PATCH 099/165] Corrected some tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 48e35b68..d8791efb 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -16,7 +16,7 @@ def test_login(): assert "password" in response.keys() def test_create_asset_repo(): - with open(Path("tests/mock_requests/create_asset_request.json")) as f: + with open(Path("tests/test_requests/create_asset_request.json")) as f: mock_register_payload = json.load(f) name = mock_register_payload["name"] host_machine = mock_register_payload["hostingMachine"] @@ -53,12 +53,13 @@ def test_get_functions(): # Non-verbose response = ModelFactory.list_functions() # Not verbose by default - for function in response: + items = response["items"] + for item in items: print(function) - assert "output" not in function.keys() - assert "params" not in function.keys() - assert "id" not in function.keys() - assert "name" in function.keys() + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() From b8d10305cbdf9ddb34397e2627aed27903d9c9db Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:10:41 -0700 Subject: [PATCH 100/165] Corrected some tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index d8791efb..912bd201 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -23,7 +23,7 @@ def test_create_asset_repo(): version = mock_register_payload["version"] description = mock_register_payload["description"] function = mock_register_payload["function"] - source_language = mock_register_payload["source_language"] + source_language = mock_register_payload["sourceLanguage"] response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) response_dict = dict(response) assert "id" in response_dict.keys() @@ -55,7 +55,6 @@ def test_get_functions(): response = ModelFactory.list_functions() # Not verbose by default items = response["items"] for item in items: - print(function) assert "output" not in item.keys() assert "params" not in item.keys() assert "id" not in item.keys() From feca2b8784f2069ccf484aad9eee4c0b648450e7 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:11:44 -0700 Subject: [PATCH 101/165] Corrected json Signed-off-by: mikelam-us-aixplain --- tests/test_requests/create_asset_request.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json index bb8cc9a9..4683e526 100644 --- a/tests/test_requests/create_asset_request.json +++ b/tests/test_requests/create_asset_request.json @@ -3,6 +3,6 @@ "hostingMachine": "aix-2c-8g-od", "version": "mock_version", "description": "mock_description", - "function": "speech-recognition", + "function": "Speech Recognition", "sourceLanguage": "en" } \ No newline at end of file From a5992f02efad44011a7a30027cbdfe3b6f293cdb Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:14:08 -0700 Subject: [PATCH 102/165] corrected module error Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 912bd201..ae5d254a 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -30,7 +30,7 @@ def test_create_asset_repo(): assert "repositoryName" in response_dict.keys() # Test cleanup - delete_asset(requests["id"], config.TEAM_API_KEY) + delete_asset(response["id"], config.TEAM_API_KEY) def test_list_host_machines(): response = ModelFactory.list_host_machines() From 19de919ead87ca33cb5fc54e777235d25a876a99 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:34:04 -0700 Subject: [PATCH 103/165] delete has no response Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index ae5d254a..a2c20945 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -79,4 +79,5 @@ def delete_asset(model_id, api_key): logging.debug(f"URL: {delete_url}") headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} response = _request_with_retry("delete", delete_url, headers=headers) - return response.json() \ No newline at end of file + print(f"Response {response}") + # return response.json() \ No newline at end of file From 8ab3b27677107025d0dbee24f82cf8219053efc5 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 14:57:23 -0700 Subject: [PATCH 104/165] Added delete_service_account Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index a2c20945..d3432b4f 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -1,7 +1,6 @@ __author__ = "michaellam" from pathlib import Path import json -import requests import logging from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin @@ -15,6 +14,9 @@ def test_login(): assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" assert "password" in response.keys() + # Test cleanup + delete_service_account(config.TEAM_API_KEY) + def test_create_asset_repo(): with open(Path("tests/test_requests/create_asset_request.json")) as f: mock_register_payload = json.load(f) @@ -66,18 +68,13 @@ def list_image_repo_tags(): assert "nextToken" in response.keys() def delete_asset(model_id, api_key): - """List the contents of the image repository corresponding to API_KEY. - - Args: - model_id (Text): Model ID obtained from CREATE_ASSET_REPO. - api_key (Text, optional): Team API key. Defaults to None. - - Returns: - Dict: Backend response - """ delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") logging.debug(f"URL: {delete_url}") headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - response = _request_with_retry("delete", delete_url, headers=headers) - print(f"Response {response}") - # return response.json() \ No newline at end of file + _ = _request_with_retry("delete", delete_url, headers=headers) + +def delete_service_account(api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("delete", delete_url, headers=headers) \ No newline at end of file From 798941c9ec81be25ab8d3c32cf24fd01523b3433 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:50:20 -0700 Subject: [PATCH 105/165] Added e2e test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 47 +++++++++++++++++++++++++++ tests/image_upload_functional_test.py | 19 ++--------- tests/test_utils.py | 16 +++++++++ 3 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 tests/image_upload_e2e_test.py create mode 100644 tests/test_utils.py diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py new file mode 100644 index 00000000..fea3ef75 --- /dev/null +++ b/tests/image_upload_e2e_test.py @@ -0,0 +1,47 @@ +__author__ = "michaellam" + +from pathlib import Path +import json +from aixplain.factories.model_factory import ModelFactory +from tests.test_utils import delete_asset, delete_service_account +from aixplain.utils import config +import docker + +def test_create_and_upload_model(): + # List the host machines + ModelFactory.list_host_machines() + + # List the functions + ModelFactory.list_functions() + + # Register the model, and create an image repository for it. + with open(Path("tests/test_requests/create_asset_request.json")) as f: + register_payload = json.load(f) + name = register_payload["name"] + host_machine = register_payload["hostingMachine"] + version = register_payload["version"] + description = register_payload["description"] + function = register_payload["function"] + source_language = register_payload["sourceLanguage"] + register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + model_id = register_response["id"] + repo_name = register_response["repositoryName"] + + # Log into the image repository. + login_response = ModelFactory.asset_repo_login() + username = login_response["username"] + password = login_response["password"] + registry = login_response["registry"] + + # Push an image to ECR + docker_client = docker.from_env(version='1.41') + docker_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + + # Send an email to finalize onboarding process + ModelFactory.onboard_model(model_id, "latest", "fake_hash") + + # Clean up + delete_service_account(config.TEAM_API_KEY) + delete_asset(model_id, config.TEAM_API_KEY) \ No newline at end of file diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index d3432b4f..ceee8b35 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -1,11 +1,8 @@ __author__ = "michaellam" from pathlib import Path import json -import logging -from aixplain.utils.file_utils import _request_with_retry -from urllib.parse import urljoin from aixplain.utils import config - +from tests.test_utils import delete_asset, delete_service_account from aixplain.factories.model_factory import ModelFactory def test_login(): @@ -65,16 +62,4 @@ def test_get_functions(): def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() assert "Image tags" in response.keys() - assert "nextToken" in response.keys() - -def delete_asset(model_id, api_key): - delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") - logging.debug(f"URL: {delete_url}") - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - _ = _request_with_retry("delete", delete_url, headers=headers) - -def delete_service_account(api_key): - delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") - logging.debug(f"URL: {delete_url}") - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - _ = _request_with_retry("delete", delete_url, headers=headers) \ No newline at end of file + assert "nextToken" in response.keys() \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..dde5b543 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,16 @@ +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin +import logging +from aixplain.utils import config + +def delete_asset(model_id, api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("delete", delete_url, headers=headers) + +def delete_service_account(api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("post", delete_url, headers=headers) \ No newline at end of file From 4dcb2da7175aac358dfdf84a1d5ee68d183da9e0 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:57:25 -0700 Subject: [PATCH 106/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index fea3ef75..a9e9397f 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -34,8 +34,9 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR + low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') docker_client = docker.from_env(version='1.41') - docker_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) From cd89990cd671aec4e7d1b17e848adb55d2a0ca26 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:58:12 -0700 Subject: [PATCH 107/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index a9e9397f..b1bc5ca9 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -37,7 +37,7 @@ def test_create_and_upload_model(): low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process From 2b97adf95ca130e941fbf04f33d6b3c11a8da745 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:59:37 -0700 Subject: [PATCH 108/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index b1bc5ca9..913dba06 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -35,10 +35,10 @@ def test_create_and_upload_model(): # Push an image to ECR low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - docker_client = docker.from_env(version='1.41') + # docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From ad7b005d1c6856efb72ba4b869b79124673e7b3f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:04:35 -0700 Subject: [PATCH 109/165] Added assert statements Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 913dba06..6e25bee7 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -9,10 +9,22 @@ def test_create_and_upload_model(): # List the host machines - ModelFactory.list_host_machines() + host_response = ModelFactory.list_host_machines() + for hosting_machine_dict in host_response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() # List the functions - ModelFactory.list_functions() + response = ModelFactory.list_functions() + items = response["items"] + for item in items: + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() # Register the model, and create an image repository for it. with open(Path("tests/test_requests/create_asset_request.json")) as f: @@ -24,18 +36,24 @@ def test_create_and_upload_model(): function = register_payload["function"] source_language = register_payload["sourceLanguage"] register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + assert "id" in register_response.keys() + assert "repositoryName" in register_response.keys() model_id = register_response["id"] repo_name = register_response["repositoryName"] # Log into the image repository. login_response = ModelFactory.asset_repo_login() + + assert login_response["username"] == "AWS" + assert login_response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in login_response.keys() + username = login_response["username"] password = login_response["password"] registry = login_response["registry"] # Push an image to ECR low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - # docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) From f6f2bda0b3d57874569411c1b5cb58ca32c21f1a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:05:57 -0700 Subject: [PATCH 110/165] What if I don't add images? Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 6e25bee7..13d1d856 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -53,10 +53,10 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR - low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + # low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') + # low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + # low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + # low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From 1847e98c492d9ac04164b7d21484ca262d8d34f4 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:30:30 -0700 Subject: [PATCH 111/165] delete corrected Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 13d1d856..6e25bee7 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -53,10 +53,10 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR - # low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - # low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - # low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - # low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') + low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From 957f3b4251b233bb868a713cf17521a7f9301925 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:34:15 -0700 Subject: [PATCH 112/165] Added line for onboarding Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index dd4ee68f..2e4f8177 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -327,6 +327,7 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", onboard_url, headers=headers, data=payload) + print("Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed.") return response From 306c438ebb7a9406ca4ed742f84bd3698f858629 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 27 Sep 2023 07:48:02 -0700 Subject: [PATCH 113/165] Removed list_image_tags because it won't be ready by this release Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 2e4f8177..ff115f6f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -328,25 +328,4 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", onboard_url, headers=headers, data=payload) print("Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed.") - return response - - - @classmethod - def list_image_repo_tags(cls, model_id: Text, api_key: Optional[Text] = None) -> Dict: - """List the contents of the image repository corresponding to API_KEY. - - Args: - model_id (Text): Model ID obtained from CREATE_ASSET_REPO. - api_key (Text, optional): Team API key. Defaults to None. - - Returns: - Dict: Backend response - """ - list_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/images") - logging.debug(f"URL: {list_url}") - if api_key: - headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} - else: - headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} - response = _request_with_retry("get", list_url, headers=headers) - return response.json() \ No newline at end of file + return response \ No newline at end of file From 2d939769912f471b490f0b78c53b0d706a4ee786 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 27 Sep 2023 07:57:33 -0700 Subject: [PATCH 114/165] Added help for list_hosts Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index bcd41fbe..7a67902e 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -27,7 +27,7 @@ import yaml @click.command("hosts") -@click.option("--api-key", default=None) +@click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment") def list_host_machines(api_key: Optional[Text] = None) -> None: """CLI wrapper function for the LIST_HOST_MACHINES function in ModelFactory. From 78dde002f6640a653ce6e75ebe1fa7c918db6d1e Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 27 Sep 2023 08:05:22 -0700 Subject: [PATCH 115/165] Added help messages to all the functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 37 ++++++++++++--------- aixplain/factories/metric_factory.py | 1 - 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index 7a67902e..7b5bab3e 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -27,7 +27,8 @@ import yaml @click.command("hosts") -@click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment") def list_host_machines(api_key: Optional[Text] = None) -> None: """CLI wrapper function for the LIST_HOST_MACHINES function in ModelFactory. @@ -43,8 +44,10 @@ def list_host_machines(api_key: Optional[Text] = None) -> None: click.echo(ret_val_yaml) @click.command("functions") -@click.option("--verbose", default=False) -@click.option("--api-key", default=None) +@click.option("--verbose", default=False, + help="List all function details, False by default.") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: """CLI wrapper function for the LIST_FUNCTIONS function in ModelFactory. @@ -60,13 +63,16 @@ def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: click.echo(ret_val_yaml) @click.command("image-repo") -@click.option("--name") -@click.option("--hosting-machine") -@click.option("--version") -@click.option("--description") -@click.option("--function") -@click.option("--source-language", default="en") -@click.option("--api-key", default=None) +@click.option("--name", help="Model name.") +@click.option("--hosting-machine", + help="Hosting machine code obtained from LIST_HOSTS.") +@click.option("--version", help="Model version.") +@click.option("--description", help="Description of model.") +@click.option("--function", help="Function name obtained from LIST_FUNCTIONS.") +@click.option("--source-language", default="en", + help="Model source language in 2-character 639-1 code or 3-character 639-3 code.") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") def create_asset_repo(name: Text, hosting_machine: Text, version: Text, description: Text, function: Text, source_language: Text, @@ -94,7 +100,8 @@ def create_asset_repo(name: Text, hosting_machine: Text, version: Text, click.echo(ret_val_yaml) @click.command("image-repo-login") -@click.option("--api-key", default=None) +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") def asset_repo_login(api_key: Optional[Text] = None) -> None: """CLI wrapper function for the ASSET_REPO_LOGIN function in ModelFactory. @@ -109,10 +116,10 @@ def asset_repo_login(api_key: Optional[Text] = None) -> None: click.echo(ret_val_yaml) @click.command("model") -@click.option("--model-id") -@click.option("--image-tag") -@click.option("--image-hash") -@click.option("--api-key", default=None) +@click.option("--model-id", help="Model ID from CREATE_IMAGE_REPO.") +@click.option("--image-tag", help="The tag of the image that you would like hosted.") +@click.option("--image-hash", help="The hash of the image you would like onboarded.") +@click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> None: """CLI wrapper function for the ONBOARD_MODEL function in ModelFactory. diff --git a/aixplain/factories/metric_factory.py b/aixplain/factories/metric_factory.py index f9866888..4fda368d 100644 --- a/aixplain/factories/metric_factory.py +++ b/aixplain/factories/metric_factory.py @@ -22,7 +22,6 @@ """ import logging -import os from typing import List from aixplain.modules.metric import Metric from aixplain.utils import config From a5e106f8dc481d5a62d7127fb7cbb58a25c9978a Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Date: Mon, 2 Oct 2023 08:17:52 -0700 Subject: [PATCH 116/165] M 5079254326 container upload cli (#70) * Testing multi-word CLI Signed-off-by: mikelam-us * changing back to single-workd command Signed-off-by: mikelam-us * Playing around with script Signed-off-by: mikelam-us * Started CLI Signed-off-by: mikelam-us-aixplain * Updated user documentation Signed-off-by: mikelam-us-aixplain * Rough draft of user docs Signed-off-by: mikelam-us-aixplain * Adding model hosting doc Signed-off-by: mikelam-us-aixplain * Updated user doc link Signed-off-by: mikelam-us-aixplain * Added Dockerfile example Signed-off-by: mikelam-us-aixplain * Added docker specifics to documentation Signed-off-by: mikelam-us-aixplain * Updated documentation Signed-off-by: mikelam-us-aixplain * Described each parameter and specified whether they were optional Signed-off-by: mikelam-us-aixplain * Updated 'verbose' documentation Signed-off-by: mikelam-us-aixplain * Updated documentation and removed extraneous lines from model_factory Signed-off-by: mikelam-us-aixplain * Corrected link Signed-off-by: mikelam-us-aixplain * Added 'help' for documentation Signed-off-by: mikelam-us-aixplain * Synced model_factory over branches Signed-off-by: mikelam-us-aixplain * logging.info for onboard and moved user doc Signed-off-by: mikelam-us-aixplain * Further changes to documentation Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 3 +- docs/user/user_doc.md | 88 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index ff115f6f..30ec90e4 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -327,5 +327,6 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", onboard_url, headers=headers, data=payload) - print("Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed.") + message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." + logging.info(message) return response \ No newline at end of file diff --git a/docs/user/user_doc.md b/docs/user/user_doc.md index ae443fe7..673b3751 100644 --- a/docs/user/user_doc.md +++ b/docs/user/user_doc.md @@ -53,6 +53,94 @@ poll_url = start_response["url"] poll_response = model.poll(poll_url) ``` +### Uploading Models +In addition to exploring and running models, the aiXplain SDK allows you to upload your own models to the aiXplain platform. This requires a working model image in line with the template specified [here](https://github.com/aixplain/model-interfaces/blob/main/docs/user/model_setup.md). [These](https://github.com/aixplain/model-interfaces/tree/main) are the interfaces with which you will be working. You will also be required to have an aiXplain account as well as a TEAM_API_KEY which should be set either as an environment variable or passed into each of the following functions. + +First, choose a hosting machine appropriate for your model. Note down the host machines "code". You can list the available hosting machines' specifications by running the following: +```console +$ aixplain list hosts [--api-key ] +- code: aix-2c-8g-od + cores: 2 + hourlyCost: 0.12 + memory: 8 + type: on-demand +- code: aix-2c-8g + cores: 2 + hourlyCost: 0.096 + memory: 8 + type: always-on + ... +``` +Note: For any of the CLI commands, running `aixplain [verb] [resource] --help` will display a description of each argument that should be passed into that command. + +The `api-key` parameter is optional and is only used if the environment variable isn't set or you would like to override the existing environment variable. + +Find a supported function type that best describes your model's purpose. Note down the function's ID. +```console +$ aixplain list functions [--verbose ] [--api-key ] +filteredFrom: 55 +items: +- name: Language Identification +- name: OCR +- name: Image Label Detection +- name: Video Forced Alignment +- name: Offensive Language Identification +- name: Audio Forced Alignment +- name: Video Generation +- name: Split On Silence +- name: Referenceless Audio Generation Metric +- name: Audio Generation Metric +- name: Speaker Diarization Video +- name: Referenceless Text Generation Metric Default +... +``` +`verbose` is optional and is set to False by default, meaning only the function names are listed. Setting this to True will additionally list the function ID, output, and params. Again, `api-key` is optional. + +Once you have chosen a suitable host machine and function, register your model and create an image repository: + +```console +$ aixplain image-create repo --name --hosting-machine --version --description --function --source-language [--api-key ] +{ + "repoName": , + "modelId": +} +``` +`name` is your model's name. `hosting-machine` should include the code of the hosting machine you would like to use. The `version` field should be set to your model's version number. `description` should hold a short summary of your model's purpose. Specify the function name most closely describe your model's purpose in the `function` field. Finally, `source-language` should contain your model's source language. + +This returns a model ID and a repository name. Next, obtain login credentials for the newly created repository: + +```console +$ aixplain get image-repo-login [--api-key ] +{ + "username": , + "password": , + "registry": +} +``` + +These credentials are valid for 12 hours, after which you much again log in for a fresh set of valid credentials. If you are using Docker, you can use these credentials to log in with the following: +```console +docker login --username $USERNAME --password $PASSWORD +``` + +Next, tag your image to match the registry and repository name given in the previous steps. If you are using Docker, this would look like the following: +```console +$ docker tag {$REGISTRY}/{$REPO_NAME}: +``` + +Push the newly tagged image to the corresponding repository: +```console +$ docker push {$REGISTRY}/{$REPO_NAME}: +``` + +Once this is done, onboard the model: +```console +$ aixplain onboard model --model-id --image-tag --image-hash [--api-key ] +``` +`model-id` should be the model ID returned by the image-create-repo function used earlier. `image-tag` should be set to whatever string you used to tag your model image. + +This will send an email to an aiXplain associate to finalize the onboarding process. + ## Pipelines [Design](https://aixplain.com/platform/studio/) is aiXplain’s no-code AI pipeline builder tool that accelerates AI development by providing a seamless experience to build complex AI systems and deploy them within minutes. You can visit our platform and design your own custom pipeline [here](https://platform.aixplain.com/studio). From f191f62bfcc1be55412bb6c8d466f07ba8a479df Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 4 Oct 2023 10:51:22 -0700 Subject: [PATCH 117/165] Added yaml requirement Signed-off-by: mikelam-us-aixplain --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1bd9800f..3e08bd50 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ ) sys.exit(1) -requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0"] +requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0", "PyYAML>=6.0.1"] test_requirements = ["python-dotenv~=0.20.0", "pytest>=6.1.0"] about = {} From 77a3206249372283bc2c6d92d3e3adf3c5bdc817 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 4 Oct 2023 11:01:29 -0700 Subject: [PATCH 118/165] Added click requirement Signed-off-by: mikelam-us-aixplain --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3e08bd50..0ca5ec2a 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ ) sys.exit(1) -requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0", "PyYAML>=6.0.1"] +requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0", "PyYAML>=6.0.1", "click>=8.1.7"] test_requirements = ["python-dotenv~=0.20.0", "pytest>=6.1.0"] about = {} From 55e27e70614ff2bd7f212cacde68000e083bfb3f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 4 Oct 2023 14:09:42 -0700 Subject: [PATCH 119/165] Moving those over to pyproject.toml Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 41 +++++++++++++++++++++++++++++++++ setup.py => setup_deprecated.py | 0 2 files changed, 41 insertions(+) create mode 100644 pyproject.toml rename setup.py => setup_deprecated.py (100%) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0cec4816 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +# aixplain/pyproject.toml + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["."] +include = ["aixplain"] +namespaces = true + +[project] +name = "aiXplain" +version = "1.0.3" +description = "aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain.com/) platform. With aixplain, developers can quickly and easily discover aiXplain's ever-expanding catalog of 35,000+ ready-to-use AI models and utilize them, benchmark AI systems by choosing models, datasets and metrics, and design their own custom pipelines and run them." +readme = "README.md" +license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } +dependencies = [ + "requests>=2.1.0", + "tqdm>=4.1.0", + "pandas>=1.2.1", + "python-dotenv>=1.0.0", + "validators>=0.20.0", + "filetype>=1.2.0" +] +requires-python = ">=3.5, <4" + +[project.optional-dependencies] +model-builder = [ + "model_interfaces @ git+https://{GITHUB_TOKEN}@github.com/aixplain/aixplain-models-internal.git@M-4944778984-namespace-change" +] +test = [ + "python-dotenv~=0.20.0", + "pytest>=6.1.0" +] + +[project.scripts] +aixplain = "aixplain.cli_groups:run_cli" + +[project.urls] +Homepage = "https://github.com/aixplain/aiXplain" \ No newline at end of file diff --git a/setup.py b/setup_deprecated.py similarity index 100% rename from setup.py rename to setup_deprecated.py From 815c762cc9750b62ea8400022e1292b232cbdaa3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 08:46:56 -0700 Subject: [PATCH 120/165] Updating pyproject.toml to replace setup.py Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0cec4816..639b7783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,11 @@ namespaces = true [project] name = "aiXplain" version = "1.0.3" -description = "aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain.com/) platform. With aixplain, developers can quickly and easily discover aiXplain's ever-expanding catalog of 35,000+ ready-to-use AI models and utilize them, benchmark AI systems by choosing models, datasets and metrics, and design their own custom pipelines and run them." +description = "aiXplain SDK adds AI functions to software." +author = "aiXplain" +author_email = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" +url = "https://github.com/aixplain/pipelines/tree/main/docs" + readme = "README.md" license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } dependencies = [ @@ -24,6 +28,30 @@ dependencies = [ "filetype>=1.2.0" ] requires-python = ">=3.5, <4" +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries", +] +license = "http://www.apache.org/licenses/LICENSE-2.0" + [project.optional-dependencies] model-builder = [ From 5d95cf7802e2fc0213143b725a64537f044dae25 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 09:23:13 -0700 Subject: [PATCH 121/165] Fully converted to pyproject.toml. Now testing Signed-off-by: mikelam-us-aixplain --- aixplain/__version__.py | 21 -------- pyproject.toml | 41 ++++++++-------- setup_deprecated.py | 106 ---------------------------------------- 3 files changed, 21 insertions(+), 147 deletions(-) delete mode 100644 aixplain/__version__.py delete mode 100644 setup_deprecated.py diff --git a/aixplain/__version__.py b/aixplain/__version__.py deleted file mode 100644 index 47a432e7..00000000 --- a/aixplain/__version__.py +++ /dev/null @@ -1,21 +0,0 @@ -__title__ = "aixplain" -__description__ = "aiXplain SDK adds AI functions to software." -__url__ = "https://github.com/aixplain/pipelines/tree/main/docs" -__version__ = "0.2.1" -__author__ = "aiXplain" -__author_email__ = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" -__license__ = "http://www.apache.org/licenses/LICENSE-2.0" -__copyright__ = """ -Copyright 2023 The aiXplain SDK authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/pyproject.toml b/pyproject.toml index 639b7783..6bc3472b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,21 +13,14 @@ namespaces = true name = "aiXplain" version = "1.0.3" description = "aiXplain SDK adds AI functions to software." -author = "aiXplain" -author_email = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" -url = "https://github.com/aixplain/pipelines/tree/main/docs" - readme = "README.md" +requires-python = ">=3.5, <4" license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } -dependencies = [ - "requests>=2.1.0", - "tqdm>=4.1.0", - "pandas>=1.2.1", - "python-dotenv>=1.0.0", - "validators>=0.20.0", - "filetype>=1.2.0" +authors = [ + {email = "thiago.ferreira@aixplain.com"}, + {email = "krishna.durai@aixplain.com"}, + {email = "lucas.pavanelli@aixplain.com"} ] -requires-python = ">=3.5, <4" classifiers = [ "Development Status :: 2 - Pre-Alpha", "Environment :: Web Environment", @@ -50,8 +43,22 @@ classifiers = [ "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries", ] -license = "http://www.apache.org/licenses/LICENSE-2.0" +dependencies = [ + "requests>=2.1.0", + "tqdm>=4.1.0", + "pandas>=1.2.1", + "python-dotenv>=1.0.0", + "validators>=0.20.0", + "filetype>=1.2.0" +] +license = {file = "http://www.apache.org/licenses/LICENSE-2.0"} +[project.urls] +Homepage = "https://github.com/aixplain/aiXplain" +Documentation = "https://github.com/aixplain/pipelines/tree/main/docs" + +[project.scripts] +aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ @@ -60,10 +67,4 @@ model-builder = [ test = [ "python-dotenv~=0.20.0", "pytest>=6.1.0" -] - -[project.scripts] -aixplain = "aixplain.cli_groups:run_cli" - -[project.urls] -Homepage = "https://github.com/aixplain/aiXplain" \ No newline at end of file +] \ No newline at end of file diff --git a/setup_deprecated.py b/setup_deprecated.py deleted file mode 100644 index 0ca5ec2a..00000000 --- a/setup_deprecated.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python -""" -Copyright 2022 The aixplain authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: Duraikrishna Selvaraju, Thiago Castro Ferreira and Lucas Pavanelli -Date: May 9th 2022 -""" - -import os -import sys - -from setuptools import setup, find_packages - -CURRENT_PYTHON = sys.version_info[:2] -REQUIRED_PYTHON = (3, 5) - -if CURRENT_PYTHON < REQUIRED_PYTHON: - sys.stderr.write( - """ -========================== -Unsupported Python version -========================== -This version of Requests requires at least Python {}.{}, but -you're trying to install it on Python {}.{}. To resolve this, -consider upgrading to a supported Python version. -""".format( - *(REQUIRED_PYTHON + CURRENT_PYTHON) - ) - ) - sys.exit(1) - -requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0", "PyYAML>=6.0.1", "click>=8.1.7"] -test_requirements = ["python-dotenv~=0.20.0", "pytest>=6.1.0"] - -about = {} -here = os.path.abspath(os.path.dirname(__file__)) - -with open(os.path.join(here, "aixplain", "__version__.py"), "r") as f: - exec(f.read(), about) - -with open("README.md", "r") as f: - readme = f.read() - -setup( - name=about["__title__"], - version=about["__version__"], - description=about["__description__"], - long_description=readme, - long_description_content_type="text/markdown", - author=about["__author__"], - author_email=about["__author_email__"], - url=about["__url__"], - packages=find_packages(exclude=["test"]), - package_dir={"aixplain": "aixplain"}, - package_data={"": ["LICENSE"]}, - include_package_data=True, - python_requires=">=3.5, <4", - install_requires=requires, - license=about["__license__"], - zip_safe=False, - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries", - ], - tests_require=test_requirements, - extras_require={}, - project_urls={ - "Documentation": "https://github.com/aixplain/pipelines/tree/main/docs", - "Source": "https://github.com/aixplain/aiXplain", - }, - entry_points = { - 'console_scripts': [ - 'aixplain = aixplain.cli_groups:run_cli' - ] - }, -) From 2a684891528ae2f2bf0172c2527220edecd541da Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 09:31:27 -0700 Subject: [PATCH 122/165] Removed extra license Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6bc3472b..682e6c75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,6 @@ dependencies = [ "validators>=0.20.0", "filetype>=1.2.0" ] -license = {file = "http://www.apache.org/licenses/LICENSE-2.0"} [project.urls] Homepage = "https://github.com/aixplain/aiXplain" From b0ff85899791d17963f6c85d8a9d536fd09d16af Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 10:52:36 -0700 Subject: [PATCH 123/165] Merged namespace-change into this branch Signed-off-by: mikelam-us-aixplain --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index efeb039f..b03d7c0a 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,16 @@ aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain ## Getting Started ### Installation -To install simply, +To install the base package, simply, ```bash pip install aixplain ``` +To install aiXplain with additional model building support: +```bash +pip install aixplain[model-builder] +``` + ### API Key Setup Before you can use the aixplain SDK, you'll need to obtain an API key from our platform. For details refer this [Team API Key Guide](docs/user/api_setup.md). @@ -60,4 +65,4 @@ Raise issues for support in this repository. Pull requests are welcome! ## Note -The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. +The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. \ No newline at end of file From b37ed7666cf4cf6ce08f28bd16b8038b640c91fa Mon Sep 17 00:00:00 2001 From: MAlyafeai18 Date: Thu, 5 Oct 2023 19:37:05 +0000 Subject: [PATCH 124/165] Working on extra dependency Signed-off-by: MAlyafeai18 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 682e6c75..1c1f9020 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,9 +61,9 @@ aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ - "model_interfaces @ git+https://{GITHUB_TOKEN}@github.com/aixplain/aixplain-models-internal.git@M-4944778984-namespace-change" + "model_interfaces~=0.0.1rc2" ] test = [ "python-dotenv~=0.20.0", "pytest>=6.1.0" -] \ No newline at end of file +] From f5d8de101357fce4ffd4ca3a83abafd112522bae Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Thu, 27 Jul 2023 13:08:33 -0700 Subject: [PATCH 125/165] Started image upload implementation Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 59 +++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c3a9a6f1..2d7cd7f2 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -29,7 +29,21 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn +from pydantic import BaseModel +class ParamInput(BaseModel): + asset_name: str + asset_function: str + asset_class: str + active_version: str + gen_description: str + license_type: str + license_description: str + container_registry_name: str + container_service_provider: str + vcpus: str + ram: str + gpus: str class ModelFactory: """A static class for creating and exploring Model Objects. @@ -216,3 +230,48 @@ def list( error_message = f"Listing Models: Error in Listing Models : {e}" logging.error(error_message, exc_info=True) raise Exception(error_message) + + @classmethod + def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, description: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use ParamInput here for input type checking. + # Use Ibrahim's endpoint here and return output. + create_url = f"{config.BACKEND_URL}/sdk/models/register" + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + payload = { + "name": name, + "hostingMachine": hosting_machine, + "alwaysOn": always_on, + "version": version, + "description": description + } + payload = json.dumps(payload) + response = _request_with_retry("post", create_url, headers=headers, data=payload) + return response + + @classmethod + def asset_repo_login(cls, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use Ibrahim's endpoint here and return output. + # TODO + create_url = f"{config.BACKEND_URL}/sdk/ecr/login" # TODO Add Ibrahim's repo login endpoint here + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + params = {"Authorization": team_api_key} + response = _request_with_retry("post", create_url, headers=headers, params=params) + return response + + @classmethod + def list_image_repo_tags(cls, team_id: Text, repo_name: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: + # Use Ibrahim's endpoint here and return output. + # TODO + create_url = f"{config.BACKEND_URL}/TODO" # TODO Add Ibrahim's repo tag endpoint here + if cls.aixplain_key != "": + headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", create_url, headers=headers) + return response From e077df908dc9262b3cf6207f482eeec49ac060e3 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Wed, 9 Aug 2023 11:45:30 -0700 Subject: [PATCH 126/165] Adding create_asset Signed-off-by: mikelam-us --- aixplain/factories/model_factory.py | 319 ++++++++++++++++------------ 1 file changed, 187 insertions(+), 132 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 2d7cd7f2..1cb0b7c3 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -20,30 +20,14 @@ Description: Model Factory Class """ -from typing import Dict, List, Optional, Text, Union +from typing import Dict, List, Optional, Text import json import logging from aixplain.modules.model import Model -from aixplain.enums import Function, Language, Supplier from aixplain.utils import config from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn -from pydantic import BaseModel - -class ParamInput(BaseModel): - asset_name: str - asset_function: str - asset_class: str - active_version: str - gen_description: str - license_type: str - license_description: str - container_registry_name: str - container_service_provider: str - vcpus: str - ram: str - gpus: str class ModelFactory: """A static class for creating and exploring Model Objects. @@ -69,22 +53,7 @@ def _create_model_from_response(cls, response: Dict) -> Model: """ if "api_key" not in response: response["api_key"] = cls.api_key - - parameters = {} - if "params" in response: - for param in response["params"]: - if "language" in param["name"]: - parameters[param["name"]] = [w["value"] for w in param["values"]] - - return Model( - response["id"], - response["name"], - supplier=response["supplier"]["id"], - api_key=response["api_key"], - pricing=response["pricing"], - function=Function(response["function"]["id"]), - parameters=parameters, - ) + return Model(response["id"], response["name"], supplier=response["supplier"]["id"], api_key=response["api_key"]) @classmethod def get(cls, model_id: Text, api_key: Optional[Text] = None) -> Model: @@ -134,144 +103,230 @@ def create_asset_from_id(cls, model_id: Text) -> Model: return cls.get(model_id) @classmethod - def _get_assets_from_page( - cls, - query, - page_number: int, - page_size: int, - function: Function, - suppliers: Union[Supplier, List[Supplier]], - source_languages: Union[Language, List[Language]], - target_languages: Union[Language, List[Language]], - is_finetunable: bool = None, + def get_assets_from_page( + cls, page_number: int, task: Text, input_language: Optional[Text] = None, output_language: Optional[Text] = None ) -> List[Model]: + """Get the list of models from a given page. Additional task and language filters can be also be provided + + Args: + page_number (int): Page from which models are to be listed + task (Text): Task of listed model + input_language (Text, optional): Input language of listed model. Defaults to None. + output_language (Text, optional): Output langugage of listed model. Defaults to None. + + Returns: + List[Model]: List of models based on given filters + """ try: - url = urljoin(cls.backend_url, f"sdk/models/paginate") - filter_params = {"q": query, "pageNumber": page_number, "pageSize": page_size} - if is_finetunable is not None: - filter_params["isFineTunable"] = is_finetunable - if function is not None: - filter_params["functions"] = [function.value] - if suppliers is not None: - if isinstance(suppliers, Supplier) is True: - suppliers = [suppliers] - filter_params["suppliers"] = [supplier.value for supplier in suppliers] - lang_filter_params = [] - if source_languages is not None: - if isinstance(source_languages, Language): - source_languages = [source_languages] - if function == Function.TRANSLATION: - lang_filter_params.append({"code": "sourcelanguage", "value": source_languages[0].value["language"]}) + url = urljoin(cls.backend_url, f"sdk/models/?pageNumber={page_number}&function={task}") + filter_params = [] + if input_language is not None: + if task == "translation": + code = "sourcelanguage" else: - lang_filter_params.append({"code": "language", "value": source_languages[0].value["language"]}) - if source_languages[0].value["dialect"] != "": - lang_filter_params.append({"code": "dialect", "value": source_languages[0].value["dialect"]}) - if target_languages is not None: - if isinstance(target_languages, Language): - target_languages = [target_languages] - if function == Function.TRANSLATION: + code = "language" + filter_params.append({"code": code, "value": input_language}) + if output_language is not None: + if task == "translation": code = "targetlanguage" - lang_filter_params.append({"code": code, "value": target_languages[0].value["language"]}) - if len(lang_filter_params) != 0: - filter_params["ioFilter"] = lang_filter_params + filter_params.append({"code": code, "value": output_language}) if cls.aixplain_key != "": headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} else: headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - - r = _request_with_retry("post", url, headers=headers, json=filter_params) + r = _request_with_retry("get", url, headers=headers, params={"ioFilter": json.dumps(filter_params)}) resp = r.json() - logging.info(f"Listing Models: Status of getting Models on Page {page_number}: {r.status_code}") + logging.info(f"Listing Models: Status of getting Models on Page {page_number} for {task}: {resp}") all_models = resp["items"] model_list = [cls._create_model_from_response(model_info_json) for model_info_json in all_models] - return model_list, resp["total"] + return model_list except Exception as e: - error_message = f"Listing Models: Error in getting Models on Page {page_number}: {e}" - logging.error(error_message, exc_info=True) + error_message = f"Listing Models: Error in getting Models on Page {page_number} for {task}: {e}" + logging.error(error_message) return [] @classmethod - def list( - cls, - query: Optional[Text] = "", - function: Optional[Function] = None, - suppliers: Optional[Union[Supplier, List[Supplier]]] = None, - source_languages: Optional[Union[Language, List[Language]]] = None, - target_languages: Optional[Union[Language, List[Language]]] = None, - is_finetunable: Optional[bool] = None, - page_number: int = 0, - page_size: int = 20, + def get_first_k_assets( + cls, k: int, task: Text, input_language: Optional[Text] = None, output_language: Optional[Text] = None ) -> List[Model]: """Gets the first k given models based on the provided task and language filters Args: - function (Optional[Function], optional): function filter. Defaults to None. - source_languages (Optional[Union[Language, List[Language]]], optional): language filter of input data. Defaults to None. - target_languages (Optional[Union[Language, List[Language]]], optional): language filter of output data. Defaults to None. - is_finetunable (Optional[bool], optional): can be finetuned or not. Defaults to None. - page_number (int, optional): page number. Defaults to 0. - page_size (int, optional): page size. Defaults to 20. + k (int): Number of models to get + task (Text): Task of listed model + input_language (Text, optional): Input language of listed model. Defaults to None. + output_language (Text, optional): Output language of listed model. Defaults to None. Returns: List[Model]: List of models based on given filters """ - print(f"Function: {function}") try: - models, total = cls._get_assets_from_page( - query, page_number, page_size, function, suppliers, source_languages, target_languages, is_finetunable - ) - return { - "results": models, - "page_total": min(page_size, len(models)), - "page_number": page_number, - "total": total, - } + model_list = [] + assert k > 0 + for page_number in range(k // 10 + 1): + model_list += cls.get_assets_from_page(page_number, task, input_language, output_language) + return model_list[0:k] except Exception as e: - error_message = f"Listing Models: Error in Listing Models : {e}" - logging.error(error_message, exc_info=True) - raise Exception(error_message) + error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" + logging.error(error_message) + return [] + + @classmethod + def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: + """Lists available hosting machines for model. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each hosting machine. + """ + machines_url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") + logging.debug(f"URL: {machines_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("get", machines_url, headers=headers) + response_dicts = json.loads(response.text) + for dictionary in response_dicts: + del dictionary["id"] + return response_dicts @classmethod - def create_asset_repo(cls, name: Text, hosting_machine: Text, always_on: bool, version: Text, description: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use ParamInput here for input type checking. - # Use Ibrahim's endpoint here and return output. - create_url = f"{config.BACKEND_URL}/sdk/models/register" - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + def list_functions(cls, verbose: Optional[bool] = False, + api_key: Optional[Text] = None) -> List[Dict]: + """Lists supported model functions on platform. + + Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each supported function. + """ + functions_url = urljoin(config.BACKEND_URL, f"sdk/functions") + logging.debug(f"URL: {functions_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("get", functions_url, headers=headers) + response_dict = json.loads(response.text) + if verbose: + return response_dict + del response_dict["results"] + function_list = response_dict["items"] + for function_dict in function_list: + del function_dict["output"] + del function_dict["params"] + del function_dict["id"] + return response_dict + + # Will add "always_on" and "is_async" when we support them. + # def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + # description: Text, function: Text, is_async: bool, + # source_language: Text, api_key: Optional[Text] = None) -> Dict: + @classmethod + def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, source_language: Text, + api_key: Optional[Text] = None) -> Dict: + """Creates an image repository for this model and registers it in the + platform backend. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + # Reconcile function name to be function ID in the backend + function_list = cls.list_functions(True, cls.api_key)["items"] + function_id = None + for function_dict in function_list: + if function_dict["name"] == function: + function_id = function_dict["id"] + if function_id is None: + raise Exception("Invalid function name") + create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") + logging.debug(f"URL: {create_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + always_on = False + is_async = False # Hard-coded to False for first release payload = { "name": name, "hostingMachine": hosting_machine, "alwaysOn": always_on, "version": version, - "description": description + "description": description, + "function": function_id, + "isAsync": is_async, + "sourceLanguage": source_language } payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) - return response + return response.json() @classmethod - def asset_repo_login(cls, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use Ibrahim's endpoint here and return output. - # TODO - create_url = f"{config.BACKEND_URL}/sdk/ecr/login" # TODO Add Ibrahim's repo login endpoint here - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} - else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - params = {"Authorization": team_api_key} - response = _request_with_retry("post", create_url, headers=headers, params=params) - return response + def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: + """Return login credentials for the image repository that corresponds with + the given API_KEY. + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + login_url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") + logging.debug(f"URL: {login_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", login_url, headers=headers) + response_dict = json.loads(response.text) + return response_dict + @classmethod - def list_image_repo_tags(cls, team_id: Text, repo_name: Text, team_api_key: Text=config.TEAM_API_KEY) -> Dict: - # Use Ibrahim's endpoint here and return output. - # TODO - create_url = f"{config.BACKEND_URL}/TODO" # TODO Add Ibrahim's repo tag endpoint here - if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: + """Onboard a model after its image has been pushed to ECR. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + Dict: Backend response + """ + onboard_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}/onboarding") + logging.debug(f"URL: {onboard_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: - headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - response = _request_with_retry("post", create_url, headers=headers) + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + payload = { + "image": image_tag, + "sha": image_hash + } + payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") + response = _request_with_retry("post", onboard_url, headers=headers, data=payload) + message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." + logging.info(message) return response From 812027171361e39c7aec52a4a1810fec418f35bf Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 12:03:22 -0700 Subject: [PATCH 127/165] Adding CLI Signed-off-by: mikelam-us --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index b8a7f591..3f4e4483 100644 --- a/setup.py +++ b/setup.py @@ -98,4 +98,9 @@ "Documentation": "https://github.com/aixplain/aiXplain/blob/main/docs/user/user_doc.md", "Source": "https://github.com/aixplain/aiXplain", }, + entry_points = { + 'console_scripts': [ + 'hosted-machines = aixplain.factories.model_factory:list_host_machines' + ] + }, ) From 134e9ee91d67814e6c19045c9c9cd444bea40920 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 13:17:04 -0700 Subject: [PATCH 128/165] Corrected entry point path Signed-off-by: mikelam-us --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f4e4483..7cc49133 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ }, entry_points = { 'console_scripts': [ - 'hosted-machines = aixplain.factories.model_factory:list_host_machines' + 'hosted-machines = aixplain.factories.model_factory.ModelFactory:list_host_machines' ] }, ) From 6813288c072745e64bc1a3acbc241bae06480f27 Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Mon, 14 Aug 2023 13:19:27 -0700 Subject: [PATCH 129/165] Working on entry point path Signed-off-by: mikelam-us --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7cc49133..1c8f7c43 100644 --- a/setup.py +++ b/setup.py @@ -100,7 +100,7 @@ }, entry_points = { 'console_scripts': [ - 'hosted-machines = aixplain.factories.model_factory.ModelFactory:list_host_machines' + 'hosted-machines = aixplain.factories.model_factory:ModelFactory.list_host_machines' ] }, ) From 8e24629ad074dab5785b5db729b7c00d75a81a1e Mon Sep 17 00:00:00 2001 From: mikelam-us Date: Sun, 20 Aug 2023 17:02:13 -0700 Subject: [PATCH 130/165] create_asset_repo response Signed-off-by: mikelam-us --- tests/mock_responses/create_asset_repo_response.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/mock_responses/create_asset_repo_response.json diff --git a/tests/mock_responses/create_asset_repo_response.json b/tests/mock_responses/create_asset_repo_response.json new file mode 100644 index 00000000..b9606471 --- /dev/null +++ b/tests/mock_responses/create_asset_repo_response.json @@ -0,0 +1,3 @@ +{ + "modelId": "mockId" +} \ No newline at end of file From 4ebd34c700862d60951b0c17078aa962282fc591 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 10:52:27 -0700 Subject: [PATCH 131/165] Refactored directories Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 124 ++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 aixplain/factories/cli/model_factory_cli.py diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py new file mode 100644 index 00000000..c0c2ed76 --- /dev/null +++ b/aixplain/factories/cli/model_factory_cli.py @@ -0,0 +1,124 @@ +__author__ = "aiXplain" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Michael Lam +Date: September 18th 2023 +Description: + Model Factory CLI +""" + +from aixplain.factories.model_factory import ModelFactory +from typing import Dict, List, Optional, Text +import click + +@click.command("hosts") +@click.option("--api-key", default=None) +def list_host_machines(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_HOST_MACHINES function in + ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.list_host_machines(api_key) + click.echo(ret_val) + +@click.command("functions") +@click.option("--verbose", default=False) +@click.option("--api-key", default=None) +def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_FUNCTIONS function in ModelFactory. + + Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + None + """ + ret_val = ModelFactory.list_functions(verbose, api_key) + click.echo(ret_val) + +@click.command("image-repo") +@click.option("--name") +@click.option("--hosting-machine") +@click.option("--version") +@click.option("--description") +@click.option("--function") +@click.option("--source-language", default="en") +@click.option("--api-key", default=None) +def create_asset_repo(name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, + source_language: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the CREATE_ASSET_REPO function in ModelFactory. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, + description, function, + source_language, api_key) + click.echo(ret_val) + +@click.command("image-repo-login") +@click.option("--api-key", default=None) +def asset_repo_login(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ASSET_REPO_LOGIN function in ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.asset_repo_login(api_key) + click.echo(ret_val) + +@click.command("model") +@click.option("--model-id") +@click.option("--image-tag") +@click.option("--image-hash") +@click.option("--api-key", default=None) +def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ONBOARD_MODEL function in ModelFactory. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) + click.echo(ret_val) \ No newline at end of file From 41a630b2e88914455bd032f9d0e0e102e296224e Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 11:56:26 -0700 Subject: [PATCH 132/165] Adding debug print statements to create_asset_repo Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 1cb0b7c3..67e8204f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -267,6 +267,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} always_on = False is_async = False # Hard-coded to False for first release + print(function_id) payload = { "name": name, "hostingMachine": hosting_machine, @@ -277,6 +278,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, "isAsync": is_async, "sourceLanguage": source_language } + print(payload) payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) From 55bb614b4abe236386a40b899ef76cee9cf2b3ca Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:06:12 -0700 Subject: [PATCH 133/165] debugging weird function Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 67e8204f..96797368 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -253,6 +253,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, """ # Reconcile function name to be function ID in the backend function_list = cls.list_functions(True, cls.api_key)["items"] + print(function_list) function_id = None for function_dict in function_list: if function_dict["name"] == function: From cad9b48dcd10e7f59db56dedd832830ca18c00ca Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:07:34 -0700 Subject: [PATCH 134/165] id -> code Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 96797368..c5263da1 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -257,7 +257,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, function_id = None for function_dict in function_list: if function_dict["name"] == function: - function_id = function_dict["id"] + function_id = function_dict["code"] if function_id is None: raise Exception("Invalid function name") create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") From 5528beffeead0367ce27a36cd9a3a6b0588ed363 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 18 Sep 2023 12:10:50 -0700 Subject: [PATCH 135/165] Removed debugging statements Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c5263da1..42affc32 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -253,7 +253,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, """ # Reconcile function name to be function ID in the backend function_list = cls.list_functions(True, cls.api_key)["items"] - print(function_list) function_id = None for function_dict in function_list: if function_dict["name"] == function: @@ -268,7 +267,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} always_on = False is_async = False # Hard-coded to False for first release - print(function_id) payload = { "name": name, "hostingMachine": hosting_machine, @@ -279,7 +277,6 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, "isAsync": is_async, "sourceLanguage": source_language } - print(payload) payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") response = _request_with_retry("post", create_url, headers=headers, data=payload) From 6cb67c34f019c5816fd75343b047590cad13ec63 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 19 Sep 2023 09:30:38 -0700 Subject: [PATCH 136/165] Corrected ID Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 42affc32..1cb0b7c3 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -256,7 +256,7 @@ def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, function_id = None for function_dict in function_list: if function_dict["name"] == function: - function_id = function_dict["code"] + function_id = function_dict["id"] if function_id is None: raise Exception("Invalid function name") create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") From 6316237a71523f96f28bae89f391aa7f941856c8 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 20 Sep 2023 12:04:46 -0700 Subject: [PATCH 137/165] Adding pretty-print for list_functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index c0c2ed76..801238e6 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -24,6 +24,7 @@ from aixplain.factories.model_factory import ModelFactory from typing import Dict, List, Optional, Text import click +import yaml @click.command("hosts") @click.option("--api-key", default=None) @@ -54,7 +55,8 @@ def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.list_functions(verbose, api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("image-repo") @click.option("--name") From ecf3da8f39943bdcc206b7ca735b2e2420c433bb Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 20 Sep 2023 12:08:11 -0700 Subject: [PATCH 138/165] Added pretty-print for all functions Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index 801238e6..bcd41fbe 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -39,7 +39,8 @@ def list_host_machines(api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.list_host_machines(api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("functions") @click.option("--verbose", default=False) @@ -89,7 +90,8 @@ def create_asset_repo(name: Text, hosting_machine: Text, version: Text, ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, description, function, source_language, api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("image-repo-login") @click.option("--api-key", default=None) @@ -103,7 +105,8 @@ def asset_repo_login(api_key: Optional[Text] = None) -> None: None """ ret_val = ModelFactory.asset_repo_login(api_key) - click.echo(ret_val) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) @click.command("model") @click.option("--model-id") @@ -123,4 +126,5 @@ def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, None """ ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) - click.echo(ret_val) \ No newline at end of file + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) \ No newline at end of file From 423354842599ade6762c92049ba3fe53b3dcb911 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 07:45:55 -0700 Subject: [PATCH 139/165] Added functional test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 tests/image_upload_functional_test.py diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py new file mode 100644 index 00000000..12d29798 --- /dev/null +++ b/tests/image_upload_functional_test.py @@ -0,0 +1,57 @@ +__author__ = "michaellam" +from pathlib import Path +import json +import requests + +from aixplain.factories.model_factory import ModelFactory + +def test_login(): + response = ModelFactory.asset_repo_login() + assert response["username"] == "AWS" + assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + +def test_create_asset_repo(): + with open(Path("tests/mock_requests/create_asset_request.json")) as f: + mock_register_payload = json.load(f) + name = mock_register_payload["name"] + host_machine = mock_register_payload["hostingMachine"] + version = mock_register_payload["version"] + description = mock_register_payload["description"] + function = mock_register_payload["function"] + source_language = mock_register_payload["source_language"] + response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + response_dict = dict(response) + assert "id" in response_dict.keys() + assert "repositoryName" in response_dict.keys() + +def test_list_host_machines(): + response = ModelFactory.list_host_machines() + for hosting_machine_dict in response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() + +def test_get_functions(): + # Verbose + response = ModelFactory.list_functions(True) + items = response["items"] + for item in items: + assert "output" in item.keys() + assert "params" in item.keys() + assert "id" in item.keys() + assert "name" in item.keys() + + # Non-verbose + response = ModelFactory.list_functions() # Not verbose by default + for function in response: + assert "output" not in function.keys() + assert "params" not in function.keys() + assert "id" not in function.keys() + assert "name" in function.keys() + +def list_image_repo_tags(): + response = ModelFactory.list_image_repo_tags() + assert "Image tags" in response.keys() + assert "nextToken" in response.keys() \ No newline at end of file From 8b62ecf22735aee7e243d48e8ac7e8c0fcfddf0d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 07:57:45 -0700 Subject: [PATCH 140/165] Added test cleanup. Testing tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 26 ++++++++++++++++++- tests/test_requests/create_asset_request.json | 8 ++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/test_requests/create_asset_request.json diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 12d29798..923635a5 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -2,6 +2,10 @@ from pathlib import Path import json import requests +import logging +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin +from aixplain.utils import config from aixplain.factories.model_factory import ModelFactory @@ -9,6 +13,7 @@ def test_login(): response = ModelFactory.asset_repo_login() assert response["username"] == "AWS" assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in response.keys() def test_create_asset_repo(): with open(Path("tests/mock_requests/create_asset_request.json")) as f: @@ -24,6 +29,9 @@ def test_create_asset_repo(): assert "id" in response_dict.keys() assert "repositoryName" in response_dict.keys() + # Test cleanup + delete_asset(requests["id"], config.TEAM_API_KEY) + def test_list_host_machines(): response = ModelFactory.list_host_machines() for hosting_machine_dict in response: @@ -54,4 +62,20 @@ def test_get_functions(): def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() assert "Image tags" in response.keys() - assert "nextToken" in response.keys() \ No newline at end of file + assert "nextToken" in response.keys() + +def delete_asset(model_id, api_key): + """List the contents of the image repository corresponding to API_KEY. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + response = _request_with_retry("delete", delete_url, headers=headers) + return response.json() \ No newline at end of file diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json new file mode 100644 index 00000000..bb8cc9a9 --- /dev/null +++ b/tests/test_requests/create_asset_request.json @@ -0,0 +1,8 @@ +{ + "name": "mock_name", + "hostingMachine": "aix-2c-8g-od", + "version": "mock_version", + "description": "mock_description", + "function": "speech-recognition", + "sourceLanguage": "en" +} \ No newline at end of file From 09d349071439014d7f499701fb0c441ebfa0afd2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:01:26 -0700 Subject: [PATCH 141/165] Adding debugging print statements Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 923635a5..48e35b68 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -54,6 +54,7 @@ def test_get_functions(): # Non-verbose response = ModelFactory.list_functions() # Not verbose by default for function in response: + print(function) assert "output" not in function.keys() assert "params" not in function.keys() assert "id" not in function.keys() From 1b7dbc7a5b1ff2b1d20741ef7e88b4a46a5568fc Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:09:23 -0700 Subject: [PATCH 142/165] Corrected some tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 48e35b68..d8791efb 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -16,7 +16,7 @@ def test_login(): assert "password" in response.keys() def test_create_asset_repo(): - with open(Path("tests/mock_requests/create_asset_request.json")) as f: + with open(Path("tests/test_requests/create_asset_request.json")) as f: mock_register_payload = json.load(f) name = mock_register_payload["name"] host_machine = mock_register_payload["hostingMachine"] @@ -53,12 +53,13 @@ def test_get_functions(): # Non-verbose response = ModelFactory.list_functions() # Not verbose by default - for function in response: + items = response["items"] + for item in items: print(function) - assert "output" not in function.keys() - assert "params" not in function.keys() - assert "id" not in function.keys() - assert "name" in function.keys() + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() From 521d38bdc01f0e5ae2c0afec47e03ee08e330d81 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:10:41 -0700 Subject: [PATCH 143/165] Corrected some tests Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index d8791efb..912bd201 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -23,7 +23,7 @@ def test_create_asset_repo(): version = mock_register_payload["version"] description = mock_register_payload["description"] function = mock_register_payload["function"] - source_language = mock_register_payload["source_language"] + source_language = mock_register_payload["sourceLanguage"] response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) response_dict = dict(response) assert "id" in response_dict.keys() @@ -55,7 +55,6 @@ def test_get_functions(): response = ModelFactory.list_functions() # Not verbose by default items = response["items"] for item in items: - print(function) assert "output" not in item.keys() assert "params" not in item.keys() assert "id" not in item.keys() From 249b87f4b98295f2d43d3efbb022e852e3c4edb1 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:11:44 -0700 Subject: [PATCH 144/165] Corrected json Signed-off-by: mikelam-us-aixplain --- tests/test_requests/create_asset_request.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json index bb8cc9a9..4683e526 100644 --- a/tests/test_requests/create_asset_request.json +++ b/tests/test_requests/create_asset_request.json @@ -3,6 +3,6 @@ "hostingMachine": "aix-2c-8g-od", "version": "mock_version", "description": "mock_description", - "function": "speech-recognition", + "function": "Speech Recognition", "sourceLanguage": "en" } \ No newline at end of file From bcfde7e4e971cdcc9767a7577689e5b3998f6cf2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:14:08 -0700 Subject: [PATCH 145/165] corrected module error Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index 912bd201..ae5d254a 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -30,7 +30,7 @@ def test_create_asset_repo(): assert "repositoryName" in response_dict.keys() # Test cleanup - delete_asset(requests["id"], config.TEAM_API_KEY) + delete_asset(response["id"], config.TEAM_API_KEY) def test_list_host_machines(): response = ModelFactory.list_host_machines() From fd7517b581537091e7b5a644e9356ee9dc69eaf2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 08:34:04 -0700 Subject: [PATCH 146/165] delete has no response Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index ae5d254a..a2c20945 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -79,4 +79,5 @@ def delete_asset(model_id, api_key): logging.debug(f"URL: {delete_url}") headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} response = _request_with_retry("delete", delete_url, headers=headers) - return response.json() \ No newline at end of file + print(f"Response {response}") + # return response.json() \ No newline at end of file From a686f29270d05820381c025101e9331434b2bef3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 14:57:23 -0700 Subject: [PATCH 147/165] Added delete_service_account Signed-off-by: mikelam-us-aixplain --- tests/image_upload_functional_test.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index a2c20945..d3432b4f 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -1,7 +1,6 @@ __author__ = "michaellam" from pathlib import Path import json -import requests import logging from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin @@ -15,6 +14,9 @@ def test_login(): assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" assert "password" in response.keys() + # Test cleanup + delete_service_account(config.TEAM_API_KEY) + def test_create_asset_repo(): with open(Path("tests/test_requests/create_asset_request.json")) as f: mock_register_payload = json.load(f) @@ -66,18 +68,13 @@ def list_image_repo_tags(): assert "nextToken" in response.keys() def delete_asset(model_id, api_key): - """List the contents of the image repository corresponding to API_KEY. - - Args: - model_id (Text): Model ID obtained from CREATE_ASSET_REPO. - api_key (Text, optional): Team API key. Defaults to None. - - Returns: - Dict: Backend response - """ delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") logging.debug(f"URL: {delete_url}") headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - response = _request_with_retry("delete", delete_url, headers=headers) - print(f"Response {response}") - # return response.json() \ No newline at end of file + _ = _request_with_retry("delete", delete_url, headers=headers) + +def delete_service_account(api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("delete", delete_url, headers=headers) \ No newline at end of file From 93b46379147f633e6c32f22dca895ddeeae66c88 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:50:20 -0700 Subject: [PATCH 148/165] Added e2e test Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 47 +++++++++++++++++++++++++++ tests/image_upload_functional_test.py | 19 ++--------- tests/test_utils.py | 16 +++++++++ 3 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 tests/image_upload_e2e_test.py create mode 100644 tests/test_utils.py diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py new file mode 100644 index 00000000..fea3ef75 --- /dev/null +++ b/tests/image_upload_e2e_test.py @@ -0,0 +1,47 @@ +__author__ = "michaellam" + +from pathlib import Path +import json +from aixplain.factories.model_factory import ModelFactory +from tests.test_utils import delete_asset, delete_service_account +from aixplain.utils import config +import docker + +def test_create_and_upload_model(): + # List the host machines + ModelFactory.list_host_machines() + + # List the functions + ModelFactory.list_functions() + + # Register the model, and create an image repository for it. + with open(Path("tests/test_requests/create_asset_request.json")) as f: + register_payload = json.load(f) + name = register_payload["name"] + host_machine = register_payload["hostingMachine"] + version = register_payload["version"] + description = register_payload["description"] + function = register_payload["function"] + source_language = register_payload["sourceLanguage"] + register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + model_id = register_response["id"] + repo_name = register_response["repositoryName"] + + # Log into the image repository. + login_response = ModelFactory.asset_repo_login() + username = login_response["username"] + password = login_response["password"] + registry = login_response["registry"] + + # Push an image to ECR + docker_client = docker.from_env(version='1.41') + docker_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + + # Send an email to finalize onboarding process + ModelFactory.onboard_model(model_id, "latest", "fake_hash") + + # Clean up + delete_service_account(config.TEAM_API_KEY) + delete_asset(model_id, config.TEAM_API_KEY) \ No newline at end of file diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index d3432b4f..ceee8b35 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -1,11 +1,8 @@ __author__ = "michaellam" from pathlib import Path import json -import logging -from aixplain.utils.file_utils import _request_with_retry -from urllib.parse import urljoin from aixplain.utils import config - +from tests.test_utils import delete_asset, delete_service_account from aixplain.factories.model_factory import ModelFactory def test_login(): @@ -65,16 +62,4 @@ def test_get_functions(): def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() assert "Image tags" in response.keys() - assert "nextToken" in response.keys() - -def delete_asset(model_id, api_key): - delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") - logging.debug(f"URL: {delete_url}") - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - _ = _request_with_retry("delete", delete_url, headers=headers) - -def delete_service_account(api_key): - delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") - logging.debug(f"URL: {delete_url}") - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - _ = _request_with_retry("delete", delete_url, headers=headers) \ No newline at end of file + assert "nextToken" in response.keys() \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..dde5b543 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,16 @@ +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin +import logging +from aixplain.utils import config + +def delete_asset(model_id, api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("delete", delete_url, headers=headers) + +def delete_service_account(api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("post", delete_url, headers=headers) \ No newline at end of file From 807fa9ddc1e1574f23042ece7c3d6080925b8a05 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:57:25 -0700 Subject: [PATCH 149/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index fea3ef75..a9e9397f 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -34,8 +34,9 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR + low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') docker_client = docker.from_env(version='1.41') - docker_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) From 314dd017d1417fc81076ecb3a1dc321c04fd9d0d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:58:12 -0700 Subject: [PATCH 150/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index a9e9397f..b1bc5ca9 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -37,7 +37,7 @@ def test_create_and_upload_model(): low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - docker_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process From 19bb215998c6ff8056fb66ad4ed5016102390f75 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 15:59:37 -0700 Subject: [PATCH 151/165] low-level client Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index b1bc5ca9..913dba06 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -35,10 +35,10 @@ def test_create_and_upload_model(): # Push an image to ECR low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - docker_client = docker.from_env(version='1.41') + # docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - docker_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From 59c8b3151691843f001da2d09f22bcec037d807f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:04:35 -0700 Subject: [PATCH 152/165] Added assert statements Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 913dba06..6e25bee7 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -9,10 +9,22 @@ def test_create_and_upload_model(): # List the host machines - ModelFactory.list_host_machines() + host_response = ModelFactory.list_host_machines() + for hosting_machine_dict in host_response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() # List the functions - ModelFactory.list_functions() + response = ModelFactory.list_functions() + items = response["items"] + for item in items: + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() # Register the model, and create an image repository for it. with open(Path("tests/test_requests/create_asset_request.json")) as f: @@ -24,18 +36,24 @@ def test_create_and_upload_model(): function = register_payload["function"] source_language = register_payload["sourceLanguage"] register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + assert "id" in register_response.keys() + assert "repositoryName" in register_response.keys() model_id = register_response["id"] repo_name = register_response["repositoryName"] # Log into the image repository. login_response = ModelFactory.asset_repo_login() + + assert login_response["username"] == "AWS" + assert login_response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in login_response.keys() + username = login_response["username"] password = login_response["password"] registry = login_response["registry"] # Push an image to ECR low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - # docker_client = docker.from_env(version='1.41') low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) From 767dd52c803ccf425de9f8dca72783b91df37d72 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:05:57 -0700 Subject: [PATCH 153/165] What if I don't add images? Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 6e25bee7..13d1d856 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -53,10 +53,10 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR - low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + # low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') + # low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + # low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + # low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From 9b1706bb121c0577adbe103a8a3608a3003c4461 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 26 Sep 2023 16:30:30 -0700 Subject: [PATCH 154/165] delete corrected Signed-off-by: mikelam-us-aixplain --- tests/image_upload_e2e_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py index 13d1d856..6e25bee7 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/image_upload_e2e_test.py @@ -53,10 +53,10 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR - # low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') - # low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - # low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") - # low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') + low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) # Send an email to finalize onboarding process ModelFactory.onboard_model(model_id, "latest", "fake_hash") From b39970ef7b4d62e31a6795156f7589dad92068bf Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 27 Sep 2023 07:57:33 -0700 Subject: [PATCH 155/165] Added help for list_hosts Signed-off-by: mikelam-us-aixplain --- aixplain/factories/cli/model_factory_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index bcd41fbe..7a67902e 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -27,7 +27,7 @@ import yaml @click.command("hosts") -@click.option("--api-key", default=None) +@click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment") def list_host_machines(api_key: Optional[Text] = None) -> None: """CLI wrapper function for the LIST_HOST_MACHINES function in ModelFactory. From ee3f1e44d8a96d2b8d8a3de9772c24c75f6dfeea Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Wed, 4 Oct 2023 14:09:42 -0700 Subject: [PATCH 156/165] Moving those over to pyproject.toml Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 41 +++++++++++++++++++++++++++++++++ setup.py => setup_deprecated.py | 0 2 files changed, 41 insertions(+) create mode 100644 pyproject.toml rename setup.py => setup_deprecated.py (100%) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0cec4816 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +# aixplain/pyproject.toml + +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["."] +include = ["aixplain"] +namespaces = true + +[project] +name = "aiXplain" +version = "1.0.3" +description = "aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain.com/) platform. With aixplain, developers can quickly and easily discover aiXplain's ever-expanding catalog of 35,000+ ready-to-use AI models and utilize them, benchmark AI systems by choosing models, datasets and metrics, and design their own custom pipelines and run them." +readme = "README.md" +license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } +dependencies = [ + "requests>=2.1.0", + "tqdm>=4.1.0", + "pandas>=1.2.1", + "python-dotenv>=1.0.0", + "validators>=0.20.0", + "filetype>=1.2.0" +] +requires-python = ">=3.5, <4" + +[project.optional-dependencies] +model-builder = [ + "model_interfaces @ git+https://{GITHUB_TOKEN}@github.com/aixplain/aixplain-models-internal.git@M-4944778984-namespace-change" +] +test = [ + "python-dotenv~=0.20.0", + "pytest>=6.1.0" +] + +[project.scripts] +aixplain = "aixplain.cli_groups:run_cli" + +[project.urls] +Homepage = "https://github.com/aixplain/aiXplain" \ No newline at end of file diff --git a/setup.py b/setup_deprecated.py similarity index 100% rename from setup.py rename to setup_deprecated.py From 63009e1e7fcc6559ec7e530b3fd88d3a01919c6c Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 08:46:56 -0700 Subject: [PATCH 157/165] Updating pyproject.toml to replace setup.py Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0cec4816..639b7783 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,11 @@ namespaces = true [project] name = "aiXplain" version = "1.0.3" -description = "aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain.com/) platform. With aixplain, developers can quickly and easily discover aiXplain's ever-expanding catalog of 35,000+ ready-to-use AI models and utilize them, benchmark AI systems by choosing models, datasets and metrics, and design their own custom pipelines and run them." +description = "aiXplain SDK adds AI functions to software." +author = "aiXplain" +author_email = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" +url = "https://github.com/aixplain/pipelines/tree/main/docs" + readme = "README.md" license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } dependencies = [ @@ -24,6 +28,30 @@ dependencies = [ "filetype>=1.2.0" ] requires-python = ">=3.5, <4" +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries", +] +license = "http://www.apache.org/licenses/LICENSE-2.0" + [project.optional-dependencies] model-builder = [ From 1663dd7c6d79c83fcfa430f037773d5dc6099c0d Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 09:31:27 -0700 Subject: [PATCH 158/165] Removed extra license Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 639b7783..4b2bef7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,3 @@ -# aixplain/pyproject.toml - [build-system] requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" @@ -13,21 +11,14 @@ namespaces = true name = "aiXplain" version = "1.0.3" description = "aiXplain SDK adds AI functions to software." -author = "aiXplain" -author_email = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" -url = "https://github.com/aixplain/pipelines/tree/main/docs" - readme = "README.md" +requires-python = ">=3.5, <4" license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } -dependencies = [ - "requests>=2.1.0", - "tqdm>=4.1.0", - "pandas>=1.2.1", - "python-dotenv>=1.0.0", - "validators>=0.20.0", - "filetype>=1.2.0" +authors = [ + {email = "thiago.ferreira@aixplain.com"}, + {email = "krishna.durai@aixplain.com"}, + {email = "lucas.pavanelli@aixplain.com"} ] -requires-python = ">=3.5, <4" classifiers = [ "Development Status :: 2 - Pre-Alpha", "Environment :: Web Environment", @@ -50,8 +41,21 @@ classifiers = [ "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries", ] -license = "http://www.apache.org/licenses/LICENSE-2.0" +dependencies = [ + "requests>=2.1.0", + "tqdm>=4.1.0", + "pandas>=1.2.1", + "python-dotenv>=1.0.0", + "validators>=0.20.0", + "filetype>=1.2.0" +] + +[project.urls] +Homepage = "https://github.com/aixplain/aiXplain" +Documentation = "https://github.com/aixplain/pipelines/tree/main/docs" +[project.scripts] +aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ @@ -61,9 +65,3 @@ test = [ "python-dotenv~=0.20.0", "pytest>=6.1.0" ] - -[project.scripts] -aixplain = "aixplain.cli_groups:run_cli" - -[project.urls] -Homepage = "https://github.com/aixplain/aiXplain" \ No newline at end of file From fe118e9ba5ed25bee36154ab2f793456b2b910c3 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Thu, 5 Oct 2023 10:52:36 -0700 Subject: [PATCH 159/165] Merged namespace-change into this branch Signed-off-by: mikelam-us-aixplain --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b32b84ae..24bbba9b 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,16 @@ aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain ## Getting Started ### Installation -To install simply, +To install the base package, simply, ```bash pip install aixplain ``` +To install aiXplain with additional model building support: +```bash +pip install aixplain[model-builder] +``` + ### API Key Setup Before you can use the aixplain SDK, you'll need to obtain an API key from our platform. For details refer this [Team API Key Guide](docs/user/api_setup.md). @@ -63,4 +68,4 @@ Raise issues for support in this repository. Pull requests are welcome! ## Note -The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. +The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. \ No newline at end of file From 20643e0a5e7e3ea14c15bd40eec3b44e394ee4e8 Mon Sep 17 00:00:00 2001 From: MAlyafeai18 Date: Thu, 5 Oct 2023 19:37:05 +0000 Subject: [PATCH 160/165] Working on extra dependency Signed-off-by: MAlyafeai18 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4b2bef7d..ec9ecc21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ - "model_interfaces @ git+https://{GITHUB_TOKEN}@github.com/aixplain/aixplain-models-internal.git@M-4944778984-namespace-change" + "model_interfaces~=0.0.1rc2" ] test = [ "python-dotenv~=0.20.0", From 08bd66035d66bd65e8021d8e583574895576934a Mon Sep 17 00:00:00 2001 From: MAlyafeai18 Date: Sun, 8 Oct 2023 22:55:05 +0000 Subject: [PATCH 161/165] Updated dependencies Signed-off-by: MAlyafeai18 --- pyproject.toml | 16 +++++++--------- tests/image_upload_functional_test.py | 5 ++++- tests/image_upload_test.py | 4 +++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d458531e..e7964461 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,4 @@ -<<<<<<< HEAD -======= # aixplain/pyproject.toml - ->>>>>>> b37ed7666cf4cf6ce08f28bd16b8038b640c91fa [build-system] requires = ["setuptools", "setuptools-scm"] build-backend = "setuptools.build_meta" @@ -14,7 +10,7 @@ namespaces = true [project] name = "aiXplain" -version = "1.0.3" +version = "0.2.4rc0" description = "aiXplain SDK adds AI functions to software." readme = "README.md" requires-python = ">=3.5, <4" @@ -52,7 +48,9 @@ dependencies = [ "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", - "filetype>=1.2.0" + "filetype>=1.2.0", + "click>=8.1.7", + "PyYAML>=6.0.1" ] [project.urls] @@ -64,9 +62,9 @@ aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ - "model_interfaces~=0.0.1rc2" + "model_interfaces>=0.0.1rc2" ] test = [ - "python-dotenv~=0.20.0", - "pytest>=6.1.0" + "pytest>=6.1.0", + "docker>=6.1.3" ] diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py index ceee8b35..0d6aa219 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/image_upload_functional_test.py @@ -4,6 +4,7 @@ from aixplain.utils import config from tests.test_utils import delete_asset, delete_service_account from aixplain.factories.model_factory import ModelFactory +import pytest def test_login(): response = ModelFactory.asset_repo_login() @@ -24,6 +25,7 @@ def test_create_asset_repo(): function = mock_register_payload["function"] source_language = mock_register_payload["sourceLanguage"] response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + print(response) response_dict = dict(response) assert "id" in response_dict.keys() assert "repositoryName" in response_dict.keys() @@ -59,7 +61,8 @@ def test_get_functions(): assert "id" not in item.keys() assert "name" in item.keys() +@pytest.mark.skip(reason="Not included in first release") def list_image_repo_tags(): response = ModelFactory.list_image_repo_tags() assert "Image tags" in response.keys() - assert "nextToken" in response.keys() \ No newline at end of file + assert "nextToken" in response.keys() diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py index 23446477..bb120533 100644 --- a/tests/image_upload_test.py +++ b/tests/image_upload_test.py @@ -5,6 +5,7 @@ from pathlib import Path from aixplain.utils import config from urllib.parse import urljoin +import pytest from aixplain.factories.model_factory import ModelFactory @@ -57,6 +58,7 @@ def test_get_functions(): functions = ModelFactory.list_functions(config.TEAM_API_KEY) assert functions == mock_json +@pytest.mark.skip(reason="Not currently supported.") def test_list_image_repo_tags(): model_id = "mock_id" url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/images") @@ -65,4 +67,4 @@ def test_list_image_repo_tags(): mock_json = json.load(f) mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) tags = ModelFactory.list_image_repo_tags(model_id, config.TEAM_API_KEY) - assert tags == mock_json \ No newline at end of file + assert tags == mock_json From 30c7c4d207151525624f19355cd25eb4ed1a7c93 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Sun, 8 Oct 2023 16:12:44 -0700 Subject: [PATCH 162/165] Fixed incorrect merge with main in model factory Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 136 +++++++++++++++++++--------- 1 file changed, 91 insertions(+), 45 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 1cb0b7c3..b73b3b80 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -20,15 +20,17 @@ Description: Model Factory Class """ -from typing import Dict, List, Optional, Text +from typing import Dict, List, Optional, Text, Union import json import logging from aixplain.modules.model import Model +from aixplain.enums import Function, Language, Supplier from aixplain.utils import config from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn + class ModelFactory: """A static class for creating and exploring Model Objects. @@ -53,7 +55,22 @@ def _create_model_from_response(cls, response: Dict) -> Model: """ if "api_key" not in response: response["api_key"] = cls.api_key - return Model(response["id"], response["name"], supplier=response["supplier"]["id"], api_key=response["api_key"]) + + parameters = {} + if "params" in response: + for param in response["params"]: + if "language" in param["name"]: + parameters[param["name"]] = [w["value"] for w in param["values"]] + + return Model( + response["id"], + response["name"], + supplier=response["supplier"]["id"], + api_key=response["api_key"], + pricing=response["pricing"], + function=Function(response["function"]["id"]), + parameters=parameters, + ) @classmethod def get(cls, model_id: Text, api_key: Optional[Text] = None) -> Model: @@ -103,74 +120,103 @@ def create_asset_from_id(cls, model_id: Text) -> Model: return cls.get(model_id) @classmethod - def get_assets_from_page( - cls, page_number: int, task: Text, input_language: Optional[Text] = None, output_language: Optional[Text] = None + def _get_assets_from_page( + cls, + query, + page_number: int, + page_size: int, + function: Function, + suppliers: Union[Supplier, List[Supplier]], + source_languages: Union[Language, List[Language]], + target_languages: Union[Language, List[Language]], + is_finetunable: bool = None, ) -> List[Model]: - """Get the list of models from a given page. Additional task and language filters can be also be provided - - Args: - page_number (int): Page from which models are to be listed - task (Text): Task of listed model - input_language (Text, optional): Input language of listed model. Defaults to None. - output_language (Text, optional): Output langugage of listed model. Defaults to None. - - Returns: - List[Model]: List of models based on given filters - """ try: - url = urljoin(cls.backend_url, f"sdk/models/?pageNumber={page_number}&function={task}") - filter_params = [] - if input_language is not None: - if task == "translation": - code = "sourcelanguage" + url = urljoin(cls.backend_url, f"sdk/models/paginate") + filter_params = {"q": query, "pageNumber": page_number, "pageSize": page_size} + if is_finetunable is not None: + filter_params["isFineTunable"] = is_finetunable + if function is not None: + filter_params["functions"] = [function.value] + if suppliers is not None: + if isinstance(suppliers, Supplier) is True: + suppliers = [suppliers] + filter_params["suppliers"] = [supplier.value for supplier in suppliers] + lang_filter_params = [] + if source_languages is not None: + if isinstance(source_languages, Language): + source_languages = [source_languages] + if function == Function.TRANSLATION: + lang_filter_params.append({"code": "sourcelanguage", "value": source_languages[0].value["language"]}) else: - code = "language" - filter_params.append({"code": code, "value": input_language}) - if output_language is not None: - if task == "translation": + lang_filter_params.append({"code": "language", "value": source_languages[0].value["language"]}) + if source_languages[0].value["dialect"] != "": + lang_filter_params.append({"code": "dialect", "value": source_languages[0].value["dialect"]}) + if target_languages is not None: + if isinstance(target_languages, Language): + target_languages = [target_languages] + if function == Function.TRANSLATION: code = "targetlanguage" - filter_params.append({"code": code, "value": output_language}) + lang_filter_params.append({"code": code, "value": target_languages[0].value["language"]}) + if len(lang_filter_params) != 0: + filter_params["ioFilter"] = lang_filter_params if cls.aixplain_key != "": headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} else: headers = {"Authorization": f"Token {cls.api_key}", "Content-Type": "application/json"} - r = _request_with_retry("get", url, headers=headers, params={"ioFilter": json.dumps(filter_params)}) + + r = _request_with_retry("post", url, headers=headers, json=filter_params) resp = r.json() - logging.info(f"Listing Models: Status of getting Models on Page {page_number} for {task}: {resp}") + logging.info(f"Listing Models: Status of getting Models on Page {page_number}: {r.status_code}") all_models = resp["items"] model_list = [cls._create_model_from_response(model_info_json) for model_info_json in all_models] - return model_list + return model_list, resp["total"] except Exception as e: - error_message = f"Listing Models: Error in getting Models on Page {page_number} for {task}: {e}" - logging.error(error_message) + error_message = f"Listing Models: Error in getting Models on Page {page_number}: {e}" + logging.error(error_message, exc_info=True) return [] @classmethod - def get_first_k_assets( - cls, k: int, task: Text, input_language: Optional[Text] = None, output_language: Optional[Text] = None + def list( + cls, + query: Optional[Text] = "", + function: Optional[Function] = None, + suppliers: Optional[Union[Supplier, List[Supplier]]] = None, + source_languages: Optional[Union[Language, List[Language]]] = None, + target_languages: Optional[Union[Language, List[Language]]] = None, + is_finetunable: Optional[bool] = None, + page_number: int = 0, + page_size: int = 20, ) -> List[Model]: """Gets the first k given models based on the provided task and language filters Args: - k (int): Number of models to get - task (Text): Task of listed model - input_language (Text, optional): Input language of listed model. Defaults to None. - output_language (Text, optional): Output language of listed model. Defaults to None. + function (Optional[Function], optional): function filter. Defaults to None. + source_languages (Optional[Union[Language, List[Language]]], optional): language filter of input data. Defaults to None. + target_languages (Optional[Union[Language, List[Language]]], optional): language filter of output data. Defaults to None. + is_finetunable (Optional[bool], optional): can be finetuned or not. Defaults to None. + page_number (int, optional): page number. Defaults to 0. + page_size (int, optional): page size. Defaults to 20. Returns: List[Model]: List of models based on given filters """ + print(f"Function: {function}") try: - model_list = [] - assert k > 0 - for page_number in range(k // 10 + 1): - model_list += cls.get_assets_from_page(page_number, task, input_language, output_language) - return model_list[0:k] + models, total = cls._get_assets_from_page( + query, page_number, page_size, function, suppliers, source_languages, target_languages, is_finetunable + ) + return { + "results": models, + "page_total": min(page_size, len(models)), + "page_number": page_number, + "total": total, + } except Exception as e: - error_message = f"Listing Models: Error in getting {k} Models for {task} : {e}" - logging.error(error_message) - return [] - + error_message = f"Listing Models: Error in Listing Models : {e}" + logging.error(error_message, exc_info=True) + raise Exception(error_message) + @classmethod def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: """Lists available hosting machines for model. From 5f7dbfe5f6351231eebd5f054aa70a57c3e389c3 Mon Sep 17 00:00:00 2001 From: MAlyafeai18 Date: Mon, 9 Oct 2023 03:10:57 +0000 Subject: [PATCH 163/165] Added requests_mock Signed-off-by: MAlyafeai18 --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7964461..ebe79bf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,5 +66,6 @@ model-builder = [ ] test = [ "pytest>=6.1.0", - "docker>=6.1.3" + "docker>=6.1.3", + "requests-mock>=1.11.0" ] From fb6df039e016460218026231e131b2080eeca79f Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Mon, 9 Oct 2023 11:54:16 -0700 Subject: [PATCH 164/165] Updated URLs Signed-off-by: mikelam-us-aixplain --- aixplain/factories/model_factory.py | 2 +- tests/test_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index b73b3b80..c6eadf21 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -360,7 +360,7 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke Returns: Dict: Backend response """ - onboard_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}/onboarding") + onboard_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/onboarding") logging.debug(f"URL: {onboard_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} diff --git a/tests/test_utils.py b/tests/test_utils.py index dde5b543..e7a41e16 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -4,7 +4,7 @@ from aixplain.utils import config def delete_asset(model_id, api_key): - delete_url = urljoin(config.BACKEND_URL, f"sdk/inventory/models/{model_id}") + delete_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}") logging.debug(f"URL: {delete_url}") headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} _ = _request_with_retry("delete", delete_url, headers=headers) From 6bebd2c17d0c47b565f3e55e350021f48f9691f2 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain Date: Tue, 10 Oct 2023 16:52:55 +0000 Subject: [PATCH 165/165] Passing all tests Signed-off-by: mikelam-us-aixplain --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ebe79bf4..500cd671 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ aixplain = "aixplain.cli_groups:run_cli" [project.optional-dependencies] model-builder = [ - "model_interfaces>=0.0.1rc2" + "model-interfaces~=0.0.1rc2" ] test = [ "pytest>=6.1.0",