diff --git a/README.md b/README.md index b32b84ae..24bbba9b 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,16 @@ aixplain is a software development kit (SDK) for the [aiXplain](https://aixplain ## Getting Started ### Installation -To install simply, +To install the base package, simply, ```bash pip install aixplain ``` +To install aiXplain with additional model building support: +```bash +pip install aixplain[model-builder] +``` + ### API Key Setup Before you can use the aixplain SDK, you'll need to obtain an API key from our platform. For details refer this [Team API Key Guide](docs/user/api_setup.md). @@ -63,4 +68,4 @@ Raise issues for support in this repository. Pull requests are welcome! ## Note -The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. +The **aiXtend** python package was renamed to **aiXplain** from the release v0.1.1. \ No newline at end of file diff --git a/aixplain/__version__.py b/aixplain/__version__.py deleted file mode 100644 index 386cd0ad..00000000 --- a/aixplain/__version__.py +++ /dev/null @@ -1,21 +0,0 @@ -__title__ = "aixplain" -__description__ = "aiXplain SDK adds AI functions to software." -__url__ = "https://github.com/aixplain/pipelines/tree/main/docs" -__version__ = "0.2.3" -__author__ = "aiXplain" -__author_email__ = "thiago.ferreira@aixplain.com, krishna.durai@aixplain.com, lucas.pavanelli@aixplain.com" -__license__ = "http://www.apache.org/licenses/LICENSE-2.0" -__copyright__ = """ -Copyright 2023 The aiXplain SDK authors -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py new file mode 100644 index 00000000..d61a6caa --- /dev/null +++ b/aixplain/cli_groups.py @@ -0,0 +1,58 @@ +__author__ = "aiXplain" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Michael Lam +Date: September 18th 2023 +Description: + CLI Runner +""" +import click +from aixplain.factories.cli.model_factory_cli import list_host_machines, list_functions, create_asset_repo, asset_repo_login, onboard_model + +@click.group('cli') +def cli(): + pass + +@click.group('list') +def list(): + pass + +@click.group('get') +def get(): + pass + +@click.group('create') +def create(): + pass + +@click.group('onboard') +def onboard(): + pass + +cli.add_command(list) +cli.add_command(get) +cli.add_command(create) +cli.add_command(onboard) + +create.add_command(create_asset_repo) +list.add_command(list_host_machines) +list.add_command(list_functions) +get.add_command(asset_repo_login) +onboard.add_command(onboard_model) + +def run_cli(): + cli() \ No newline at end of file diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py new file mode 100644 index 00000000..69f48d4d --- /dev/null +++ b/aixplain/factories/cli/model_factory_cli.py @@ -0,0 +1,137 @@ +__author__ = "aiXplain" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Michael Lam +Date: September 18th 2023 +Description: + Model Factory CLI +""" + +from aixplain.factories.model_factory import ModelFactory +from typing import Dict, List, Optional, Text +import click +import yaml + +@click.command("hosts") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment") +def list_host_machines(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_HOST_MACHINES function in + ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.list_host_machines(api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) + +@click.command("functions") +@click.option("--verbose", default=False, + help="List all function details, False by default.") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") +def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_FUNCTIONS function in ModelFactory. + + Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + None + """ + ret_val = ModelFactory.list_functions(verbose, api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) + +@click.command("image-repo") +@click.option("--name", help="Model name.") +@click.option("--hosting-machine", + help="Hosting machine code obtained from LIST_HOSTS.") +@click.option("--version", help="Model version.") +@click.option("--description", help="Description of model.") +@click.option("--function", help="Function name obtained from LIST_FUNCTIONS.") +@click.option("--source-language", default="en", + help="Model source language in 2-character 639-1 code or 3-character 639-3 code.") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") +def create_asset_repo(name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, + source_language: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the CREATE_ASSET_REPO function in ModelFactory. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, + description, function, + source_language, api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) + +@click.command("image-repo-login") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") +def asset_repo_login(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ASSET_REPO_LOGIN function in ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.asset_repo_login(api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) + +@click.command("model") +@click.option("--model-id", help="Model ID from CREATE_IMAGE_REPO.") +@click.option("--image-tag", help="The tag of the image that you would like hosted.") +@click.option("--image-hash", help="The hash of the image you would like onboarded.") +@click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") +def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, + api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the ONBOARD_MODEL function in ModelFactory. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + None + """ + ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c3a9a6f1..c6eadf21 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -216,3 +216,163 @@ def list( error_message = f"Listing Models: Error in Listing Models : {e}" logging.error(error_message, exc_info=True) raise Exception(error_message) + + @classmethod + def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: + """Lists available hosting machines for model. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each hosting machine. + """ + machines_url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") + logging.debug(f"URL: {machines_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("get", machines_url, headers=headers) + response_dicts = json.loads(response.text) + for dictionary in response_dicts: + del dictionary["id"] + return response_dicts + + @classmethod + def list_functions(cls, verbose: Optional[bool] = False, + api_key: Optional[Text] = None) -> List[Dict]: + """Lists supported model functions on platform. + + Args: + verbose (Boolean, optional): Set to True if a detailed response + is desired; is otherwise False by default. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[Dict]: List of dictionaries containing information about + each supported function. + """ + functions_url = urljoin(config.BACKEND_URL, f"sdk/functions") + logging.debug(f"URL: {functions_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("get", functions_url, headers=headers) + response_dict = json.loads(response.text) + if verbose: + return response_dict + del response_dict["results"] + function_list = response_dict["items"] + for function_dict in function_list: + del function_dict["output"] + del function_dict["params"] + del function_dict["id"] + return response_dict + + # Will add "always_on" and "is_async" when we support them. + # def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + # description: Text, function: Text, is_async: bool, + # source_language: Text, api_key: Optional[Text] = None) -> Dict: + @classmethod + def create_asset_repo(cls, name: Text, hosting_machine: Text, version: Text, + description: Text, function: Text, source_language: Text, + api_key: Optional[Text] = None) -> Dict: + """Creates an image repository for this model and registers it in the + platform backend. + + Args: + name (Text): Model name + hosting_machine (Text): Hosting machine ID obtained via list_host_machines + always_on (bool): Whether the model should always be on + version (Text): Model version + description (Text): Model description + function (Text): Model function name obtained via LIST_HOST_MACHINES + is_async (bool): Whether the model is asynchronous or not (False in first release) + source_language (Text): 2-character 639-1 code or 3-character 639-3 language code. + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + # Reconcile function name to be function ID in the backend + function_list = cls.list_functions(True, cls.api_key)["items"] + function_id = None + for function_dict in function_list: + if function_dict["name"] == function: + function_id = function_dict["id"] + if function_id is None: + raise Exception("Invalid function name") + create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") + logging.debug(f"URL: {create_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + always_on = False + is_async = False # Hard-coded to False for first release + payload = { + "name": name, + "hostingMachine": hosting_machine, + "alwaysOn": always_on, + "version": version, + "description": description, + "function": function_id, + "isAsync": is_async, + "sourceLanguage": source_language + } + payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") + response = _request_with_retry("post", create_url, headers=headers, data=payload) + return response.json() + + @classmethod + def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: + """Return login credentials for the image repository that corresponds with + the given API_KEY. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + Dict: Backend response + """ + login_url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") + logging.debug(f"URL: {login_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + response = _request_with_retry("post", login_url, headers=headers) + response_dict = json.loads(response.text) + return response_dict + + @classmethod + def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: + """Onboard a model after its image has been pushed to ECR. + + Args: + model_id (Text): Model ID obtained from CREATE_ASSET_REPO. + image_tag (Text): Image tag to be onboarded. + api_key (Text, optional): Team API key. Defaults to None. + Returns: + Dict: Backend response + """ + onboard_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/onboarding") + logging.debug(f"URL: {onboard_url}") + if api_key: + headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + else: + headers = {"x-api-key": f"{cls.api_key}", "Content-Type": "application/json"} + payload = { + "image": image_tag, + "sha": image_hash + } + payload = json.dumps(payload) + logging.debug(f"Body: {str(payload)}") + response = _request_with_retry("post", onboard_url, headers=headers, data=payload) + message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." + logging.info(message) + return response diff --git a/docs/user/user_doc.md b/docs/user/user_doc.md index 0e7a6e55..6ab3ee92 100644 --- a/docs/user/user_doc.md +++ b/docs/user/user_doc.md @@ -57,6 +57,94 @@ poll_url = start_response["url"] poll_response = model.poll(poll_url) ``` +### Uploading Models +In addition to exploring and running models, the aiXplain SDK allows you to upload your own models to the aiXplain platform. This requires a working model image in line with the template specified [here](https://github.com/aixplain/model-interfaces/blob/main/docs/user/model_setup.md). [These](https://github.com/aixplain/model-interfaces/tree/main) are the interfaces with which you will be working. You will also be required to have an aiXplain account as well as a TEAM_API_KEY which should be set either as an environment variable or passed into each of the following functions. + +First, choose a hosting machine appropriate for your model. Note down the host machines "code". You can list the available hosting machines' specifications by running the following: +```console +$ aixplain list hosts [--api-key ] +- code: aix-2c-8g-od + cores: 2 + hourlyCost: 0.12 + memory: 8 + type: on-demand +- code: aix-2c-8g + cores: 2 + hourlyCost: 0.096 + memory: 8 + type: always-on + ... +``` +Note: For any of the CLI commands, running `aixplain [verb] [resource] --help` will display a description of each argument that should be passed into that command. + +The `api-key` parameter is optional and is only used if the environment variable isn't set or you would like to override the existing environment variable. + +Find a supported function type that best describes your model's purpose. Note down the function's ID. +```console +$ aixplain list functions [--verbose ] [--api-key ] +filteredFrom: 55 +items: +- name: Language Identification +- name: OCR +- name: Image Label Detection +- name: Video Forced Alignment +- name: Offensive Language Identification +- name: Audio Forced Alignment +- name: Video Generation +- name: Split On Silence +- name: Referenceless Audio Generation Metric +- name: Audio Generation Metric +- name: Speaker Diarization Video +- name: Referenceless Text Generation Metric Default +... +``` +`verbose` is optional and is set to False by default, meaning only the function names are listed. Setting this to True will additionally list the function ID, output, and params. Again, `api-key` is optional. + +Once you have chosen a suitable host machine and function, register your model and create an image repository: + +```console +$ aixplain image-create repo --name --hosting-machine --version --description --function --source-language [--api-key ] +{ + "repoName": , + "modelId": +} +``` +`name` is your model's name. `hosting-machine` should include the code of the hosting machine you would like to use. The `version` field should be set to your model's version number. `description` should hold a short summary of your model's purpose. Specify the function name most closely describe your model's purpose in the `function` field. Finally, `source-language` should contain your model's source language. + +This returns a model ID and a repository name. Next, obtain login credentials for the newly created repository: + +```console +$ aixplain get image-repo-login [--api-key ] +{ + "username": , + "password": , + "registry": +} +``` + +These credentials are valid for 12 hours, after which you much again log in for a fresh set of valid credentials. If you are using Docker, you can use these credentials to log in with the following: +```console +docker login --username $USERNAME --password $PASSWORD +``` + +Next, tag your image to match the registry and repository name given in the previous steps. If you are using Docker, this would look like the following: +```console +$ docker tag {$REGISTRY}/{$REPO_NAME}: +``` + +Push the newly tagged image to the corresponding repository: +```console +$ docker push {$REGISTRY}/{$REPO_NAME}: +``` + +Once this is done, onboard the model: +```console +$ aixplain onboard model --model-id --image-tag --image-hash [--api-key ] +``` +`model-id` should be the model ID returned by the image-create-repo function used earlier. `image-tag` should be set to whatever string you used to tag your model image. + +This will send an email to an aiXplain associate to finalize the onboarding process. + ## Pipelines [Design](https://aixplain.com/platform/studio/) is aiXplain’s no-code AI pipeline builder tool that accelerates AI development by providing a seamless experience to build complex AI systems and deploy them within minutes. You can visit our platform and design your own custom pipeline [here](https://platform.aixplain.com/studio). diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..500cd671 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +# aixplain/pyproject.toml +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["."] +include = ["aixplain"] +namespaces = true + +[project] +name = "aiXplain" +version = "0.2.4rc0" +description = "aiXplain SDK adds AI functions to software." +readme = "README.md" +requires-python = ">=3.5, <4" +license = { text = "Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0" } +authors = [ + {email = "thiago.ferreira@aixplain.com"}, + {email = "krishna.durai@aixplain.com"}, + {email = "lucas.pavanelli@aixplain.com"} +] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Topic :: Internet :: WWW/HTTP", + "Topic :: Software Development :: Libraries", +] +dependencies = [ + "requests>=2.1.0", + "tqdm>=4.1.0", + "pandas>=1.2.1", + "python-dotenv>=1.0.0", + "validators>=0.20.0", + "filetype>=1.2.0", + "click>=8.1.7", + "PyYAML>=6.0.1" +] + +[project.urls] +Homepage = "https://github.com/aixplain/aiXplain" +Documentation = "https://github.com/aixplain/pipelines/tree/main/docs" + +[project.scripts] +aixplain = "aixplain.cli_groups:run_cli" + +[project.optional-dependencies] +model-builder = [ + "model-interfaces~=0.0.1rc2" +] +test = [ + "pytest>=6.1.0", + "docker>=6.1.3", + "requests-mock>=1.11.0" +] diff --git a/setup.py b/setup.py deleted file mode 100644 index b8a7f591..00000000 --- a/setup.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python -""" -Copyright 2022 The aixplain authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Author: Duraikrishna Selvaraju, Thiago Castro Ferreira and Lucas Pavanelli -Date: May 9th 2022 -""" - -import os -import sys - -from setuptools import setup, find_packages - -CURRENT_PYTHON = sys.version_info[:2] -REQUIRED_PYTHON = (3, 5) - -if CURRENT_PYTHON < REQUIRED_PYTHON: - sys.stderr.write( - """ -========================== -Unsupported Python version -========================== -This version of Requests requires at least Python {}.{}, but -you're trying to install it on Python {}.{}. To resolve this, -consider upgrading to a supported Python version. -""".format( - *(REQUIRED_PYTHON + CURRENT_PYTHON) - ) - ) - sys.exit(1) - -requires = ["requests>=2.1.0", "tqdm>=4.1.0", "pandas>=1.2.1", "python-dotenv>=1.0.0", "validators>=0.20.0", "filetype>=1.2.0"] -test_requirements = ["python-dotenv~=0.20.0", "coverage>=7.3.2", "pytest>=6.1.0", "requests-mock>=1.11.0"] - -about = {} -here = os.path.abspath(os.path.dirname(__file__)) - -with open(os.path.join(here, "aixplain", "__version__.py"), "r") as f: - exec(f.read(), about) - -with open("README.md", "r") as f: - readme = f.read() - -setup( - name=about["__title__"], - version=about["__version__"], - description=about["__description__"], - long_description=readme, - long_description_content_type="text/markdown", - author=about["__author__"], - author_email=about["__author_email__"], - url=about["__url__"], - packages=find_packages(exclude=["test"]), - package_dir={"aixplain": "aixplain"}, - package_data={"": ["LICENSE"]}, - include_package_data=True, - python_requires=">=3.5, <4", - install_requires=requires, - license=about["__license__"], - zip_safe=False, - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Environment :: Web Environment", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", - "Topic :: Internet :: WWW/HTTP", - "Topic :: Software Development :: Libraries", - ], - tests_require=test_requirements, - extras_require={}, - project_urls={ - "Documentation": "https://github.com/aixplain/aiXplain/blob/main/docs/user/user_doc.md", - "Source": "https://github.com/aixplain/aiXplain", - }, -) diff --git a/tests/image_upload_e2e_test.py b/tests/image_upload_e2e_test.py new file mode 100644 index 00000000..6e25bee7 --- /dev/null +++ b/tests/image_upload_e2e_test.py @@ -0,0 +1,66 @@ +__author__ = "michaellam" + +from pathlib import Path +import json +from aixplain.factories.model_factory import ModelFactory +from tests.test_utils import delete_asset, delete_service_account +from aixplain.utils import config +import docker + +def test_create_and_upload_model(): + # List the host machines + host_response = ModelFactory.list_host_machines() + for hosting_machine_dict in host_response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() + + # List the functions + response = ModelFactory.list_functions() + items = response["items"] + for item in items: + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() + + # Register the model, and create an image repository for it. + with open(Path("tests/test_requests/create_asset_request.json")) as f: + register_payload = json.load(f) + name = register_payload["name"] + host_machine = register_payload["hostingMachine"] + version = register_payload["version"] + description = register_payload["description"] + function = register_payload["function"] + source_language = register_payload["sourceLanguage"] + register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + assert "id" in register_response.keys() + assert "repositoryName" in register_response.keys() + model_id = register_response["id"] + repo_name = register_response["repositoryName"] + + # Log into the image repository. + login_response = ModelFactory.asset_repo_login() + + assert login_response["username"] == "AWS" + assert login_response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in login_response.keys() + + username = login_response["username"] + password = login_response["password"] + registry = login_response["registry"] + + # Push an image to ECR + low_level_client = docker.APIClient(base_url='unix://var/run/docker.sock') + low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") + low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") + low_level_client.push(f"{registry}/{repo_name}", auth_config={"username":username, "password":password}) + + # Send an email to finalize onboarding process + ModelFactory.onboard_model(model_id, "latest", "fake_hash") + + # Clean up + delete_service_account(config.TEAM_API_KEY) + delete_asset(model_id, config.TEAM_API_KEY) \ No newline at end of file diff --git a/tests/image_upload_functional_test.py b/tests/image_upload_functional_test.py new file mode 100644 index 00000000..0d6aa219 --- /dev/null +++ b/tests/image_upload_functional_test.py @@ -0,0 +1,68 @@ +__author__ = "michaellam" +from pathlib import Path +import json +from aixplain.utils import config +from tests.test_utils import delete_asset, delete_service_account +from aixplain.factories.model_factory import ModelFactory +import pytest + +def test_login(): + response = ModelFactory.asset_repo_login() + assert response["username"] == "AWS" + assert response["registry"] == "535945872701.dkr.ecr.us-east-1.amazonaws.com" + assert "password" in response.keys() + + # Test cleanup + delete_service_account(config.TEAM_API_KEY) + +def test_create_asset_repo(): + with open(Path("tests/test_requests/create_asset_request.json")) as f: + mock_register_payload = json.load(f) + name = mock_register_payload["name"] + host_machine = mock_register_payload["hostingMachine"] + version = mock_register_payload["version"] + description = mock_register_payload["description"] + function = mock_register_payload["function"] + source_language = mock_register_payload["sourceLanguage"] + response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + print(response) + response_dict = dict(response) + assert "id" in response_dict.keys() + assert "repositoryName" in response_dict.keys() + + # Test cleanup + delete_asset(response["id"], config.TEAM_API_KEY) + +def test_list_host_machines(): + response = ModelFactory.list_host_machines() + for hosting_machine_dict in response: + assert "code" in hosting_machine_dict.keys() + assert "type" in hosting_machine_dict.keys() + assert "cores" in hosting_machine_dict.keys() + assert "memory" in hosting_machine_dict.keys() + assert "hourlyCost" in hosting_machine_dict.keys() + +def test_get_functions(): + # Verbose + response = ModelFactory.list_functions(True) + items = response["items"] + for item in items: + assert "output" in item.keys() + assert "params" in item.keys() + assert "id" in item.keys() + assert "name" in item.keys() + + # Non-verbose + response = ModelFactory.list_functions() # Not verbose by default + items = response["items"] + for item in items: + assert "output" not in item.keys() + assert "params" not in item.keys() + assert "id" not in item.keys() + assert "name" in item.keys() + +@pytest.mark.skip(reason="Not included in first release") +def list_image_repo_tags(): + response = ModelFactory.list_image_repo_tags() + assert "Image tags" in response.keys() + assert "nextToken" in response.keys() diff --git a/tests/image_upload_test.py b/tests/image_upload_test.py new file mode 100644 index 00000000..bb120533 --- /dev/null +++ b/tests/image_upload_test.py @@ -0,0 +1,70 @@ +__author__ = "michaellam" + +import json +import requests_mock +from pathlib import Path +from aixplain.utils import config +from urllib.parse import urljoin +import pytest + +from aixplain.factories.model_factory import ModelFactory + +AUTH_FIXED_HEADER = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} +API_FIXED_HEADER = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} + + +def test_login(): + url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/login_response.json")) as f: + mock_json = json.load(f) + mock.post(url, headers=AUTH_FIXED_HEADER, json=mock_json) + creds = ModelFactory.asset_repo_login(config.TEAM_API_KEY) + assert creds == mock_json + +def test_create_asset_repo(): + url_register = urljoin(config.BACKEND_URL, f"sdk/models/register") + url_function = urljoin(config.BACKEND_URL, f"sdk/functions") + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: + mock_json_register = json.load(f) + mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json_register) + with open(Path("tests/mock_responses/list_functions_response.json")) as f: + mock_json_functions = json.load(f) + mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json_functions) + model_id = ModelFactory.create_asset_repo("mock_name", "mock_machines", "mock_version", + "mock_description", "Speech Recognition", "en", config.TEAM_API_KEY) + assert model_id == mock_json_register + +def test_list_host_machines(): + url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_host_machines_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=API_FIXED_HEADER, json=mock_json) + machines = ModelFactory.list_host_machines(config.TEAM_API_KEY) + for i in range(len(machines)): + machine_dict = machines[i] + mock_json_dict = mock_json[i] + for key in machine_dict.keys(): + assert machine_dict[key] == mock_json_dict[key] + +def test_get_functions(): + url = urljoin(config.BACKEND_URL, f"sdk/functions") + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_functions_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + functions = ModelFactory.list_functions(config.TEAM_API_KEY) + assert functions == mock_json + +@pytest.mark.skip(reason="Not currently supported.") +def test_list_image_repo_tags(): + model_id = "mock_id" + url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}/images") + with requests_mock.Mocker() as mock: + with open(Path("tests/mock_responses/list_image_repo_tags_response.json")) as f: + mock_json = json.load(f) + mock.get(url, headers=AUTH_FIXED_HEADER, json=mock_json) + tags = ModelFactory.list_image_repo_tags(model_id, config.TEAM_API_KEY) + assert tags == mock_json diff --git a/tests/mock_responses/create_asset_repo_response.json b/tests/mock_responses/create_asset_repo_response.json new file mode 100644 index 00000000..b9606471 --- /dev/null +++ b/tests/mock_responses/create_asset_repo_response.json @@ -0,0 +1,3 @@ +{ + "modelId": "mockId" +} \ No newline at end of file diff --git a/tests/mock_responses/list_functions_response.json b/tests/mock_responses/list_functions_response.json new file mode 100644 index 00000000..0512e79d --- /dev/null +++ b/tests/mock_responses/list_functions_response.json @@ -0,0 +1,2880 @@ +{ + "total": 55, + "pageTotal": 55, + "items": [ + { + "id": "language-identification", + "name": "Language Identification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "ocr", + "name": "OCR", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "featuretypes", + "name": "Feature Types", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + } + ] + }, + { + "id": "image-label-detection", + "name": "Image Label Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "video-forced-alignment", + "name": "Video Forced Alignment", + "output": [ + { + "name": "Text", + "code": "text", + "dataType": "text", + "defaultValue": [] + }, + { + "name": "Video", + "code": "video", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "offensive-language-identification", + "name": "Offensive Language Identification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-forced-alignment", + "name": "Audio Forced Alignment", + "output": [ + { + "name": "Text", + "code": "text", + "dataType": "text", + "defaultValue": [] + }, + { + "name": "Audio", + "code": "audio", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-generation", + "name": "Video Generation", + "output": [ + { + "name": "Video", + "code": "data", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "split-on-silence", + "name": "Split On Silence", + "output": [ + { + "name": "Segments", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-audio-generation-metric", + "name": "Referenceless Audio Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-text-generation-metric-default", + "name": "Referenceless Text Generation Metric Default", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speaker-diarization-video", + "name": "Speaker Diarization Video", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "video", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-generation-metric", + "name": "Audio Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "text-reconstruction", + "name": "Text Reconstruction", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Segments", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speech-classification", + "name": "Speech Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "metric-aggregation", + "name": "Metric Aggregation", + "output": [ + { + "name": "Aggregates", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "text", + "name": "Score Aggregation Data", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + } + ] + }, + { + "id": "dialect-detection", + "name": "Dialect Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "fill-text-mask", + "name": "Fill Text Mask", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "json", + "multipleValues": true + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-content-moderation", + "name": "Video Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "image-compression", + "name": "Image Compression", + "output": [ + { + "name": "image", + "code": "image", + "dataType": "image", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "apl_qfactor", + "name": "apl_qfactor", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "80", + "label": "80" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "extract-audio-from-video", + "name": "Extract Audio From Video", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio" + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video" + } + ] + }, + { + "id": "speech-non-speech-classification", + "name": "Speech or Non-Speech Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "referenceless-text-generation-metric", + "name": "Referenceless Text Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "text-generation-metric-default", + "name": "Text Generation Metric Default", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speech-embedding", + "name": "Speech Embedding", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-reconstruction", + "name": "Audio Reconstruction", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Segments", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + } + ] + }, + { + "id": "voice-cloning", + "name": "Voice Cloning", + "output": [ + { + "name": "Target Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "type", + "name": "Type", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-generation", + "name": "Text Generation", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "session_id", + "name": "session_id", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + }, + { + "code": "option_index", + "name": "option_index", + "required": false, + "isFixed": false, + "dataType": "integer", + "dataSubType": "integer" + }, + { + "code": "next_state", + "name": "next_state", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text" + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "audio-transcript-improvement", + "name": "Audio Transcript Improvement", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "ASR Supplier", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "is_medical", + "name": "Is Medical", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "diacritization", + "name": "Diacritization", + "output": [ + { + "name": "Target Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "emotion-detection", + "name": "Emotion Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-summarization", + "name": "Text summarization", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "entity-linking", + "name": "Entity Linking", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "domain", + "name": "Domain", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-spam-detection", + "name": "Text Spam Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-generation-metric", + "name": "Text Generation Metric", + "output": [ + { + "name": "Score", + "code": "data", + "dataType": "text" + } + ], + "params": [ + { + "code": "hypotheses", + "name": "Hypotheses", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "references", + "name": "References", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "sources", + "name": "Sources", + "required": false, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + }, + { + "code": "score_identifier", + "name": "Score Identifier", + "required": true, + "isFixed": true, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "split-on-linebreak", + "name": "Split On Linebreak", + "output": [ + { + "name": "text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "voice-activity-detection", + "name": "Voice Activity Detection", + "output": [ + { + "name": "Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "onset", + "name": "Onset", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "offset", + "name": "Offset", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "min_duration_on", + "name": "Min Duration On", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "1", + "label": "1" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + }, + { + "code": "min_duration_off", + "name": "Min Duration Off", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "sentiment-analysis", + "name": "Sentiment Analysis", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-classification", + "name": "Text Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "speech-synthesis", + "name": "Speech Synthesis", + "output": [ + { + "name": "Target Audio", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": false, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "type", + "name": "Type", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "video-label-detection", + "name": "Video Label Detection", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "video", + "name": "Video", + "required": true, + "isFixed": false, + "dataType": "video", + "dataSubType": "video", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "asr-quality-estimation", + "name": "ASR Quality Estimation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "json", + "multipleValues": true + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-content-moderation", + "name": "Text Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "image-content-moderation", + "name": "Image Content Moderation", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "image", + "name": "Image", + "required": true, + "isFixed": false, + "dataType": "image", + "dataSubType": "image", + "multipleValues": false + }, + { + "code": "min_confidence", + "name": "Min Confidence", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0.5", + "label": "0.5" + } + ], + "dataType": "text", + "dataSubType": "number", + "multipleValues": false + } + ] + }, + { + "id": "audio-transcript-analysis", + "name": "Audio Transcript Analysis", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "ASR Supplier", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-denormalization", + "name": "Text Denormalization", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "lowercase_latin", + "name": "To Lower Case", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0", + "label": "No" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "remove_accents", + "name": "Remove Accents", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "1", + "label": "Yes" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + }, + { + "code": "remove_punctuation", + "name": "Remove Punctuation", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "0", + "label": "No" + } + ], + "dataType": "text", + "dataSubType": "boolean", + "multipleValues": false + } + ] + }, + { + "id": "search", + "name": "Search", + "output": [ + { + "name": "Text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + } + ] + }, + { + "id": "speaker-diarization-audio", + "name": "Speaker Diarization Audio", + "output": [ + { + "name": "Segments", + "code": "data", + "dataType": "audio", + "defaultValue": [] + } + ], + "params": [ + { + "code": "audio", + "name": "Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "named-entity-recognition", + "name": "Named Entity Recognition", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "domain", + "name": "Domain", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "topic-classification", + "name": "Topic Classification", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-to-image-generation", + "name": "Text To Image Generation", + "output": [ + { + "name": "Generated Image", + "code": "image", + "dataType": "image", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text Prompt", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "text-normalization", + "name": "Text Normalization", + "output": [ + { + "name": "Label", + "code": "data", + "dataType": "label", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "language", + "name": "Language", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "settings", + "name": "Settings", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "remove urls", + "label": "remove urls" + }, + { + "value": "remove emails", + "label": "remove emails" + }, + { + "value": "remove phone numbers", + "label": "remove phone numbers" + }, + { + "value": "remove emojis", + "label": "remove emojis" + }, + { + "value": "remove html tags", + "label": "remove html tags" + }, + { + "value": "normalize quotes", + "label": "normalize quotes" + }, + { + "value": "lowercase text", + "label": "lowercase text" + }, + { + "value": "remove default arabic diacritics", + "label": "remove default arabic diacritics" + }, + { + "value": "remove full arabic diacritics", + "label": "remove full arabic diacritics" + }, + { + "value": "normalize default arabic", + "label": "normalize default arabic" + }, + { + "value": "normalize full arabic", + "label": "normalize full arabic" + }, + { + "value": "remove arabic superfluous", + "label": "remove arabic superfluous" + }, + { + "value": "remove kashida dagger", + "label": "remove kashida dagger" + }, + { + "value": "normalize spoken text", + "label": "normalize spoken text" + }, + { + "value": "denormalize spoken text", + "label": "denormalize spoken text" + }, + { + "value": "tokenize text", + "label": "tokenize text" + } + ], + "dataType": "text", + "dataSubType": "text", + "multipleValues": true + } + ] + }, + { + "id": "subtitling-translation", + "name": "Subtitling Translation", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "target_supplier", + "name": "Machine Translation Supplier", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "modernmt", + "label": "ModernMT" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "google", + "label": "Google" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguages", + "name": "Target Languages", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "en", + "label": "English" + }, + { + "value": "de", + "label": "German" + }, + { + "value": "nl", + "label": "Dutch" + }, + { + "value": "fr", + "label": "French" + }, + { + "value": "el", + "label": "Greek" + }, + { + "value": "it", + "label": "Italian" + }, + { + "value": "pl", + "label": "Polish" + }, + { + "value": "pt", + "label": "Portuguese" + }, + { + "value": "ru", + "label": "Russian" + }, + { + "value": "es", + "label": "Spanish" + }, + { + "value": "zh", + "label": "Chinese" + }, + { + "value": "sl", + "label": "Slovenian" + }, + { + "value": "ar", + "label": "Arabic" + }, + { + "value": "ja", + "label": "Japanese" + }, + { + "value": "ko", + "label": "Korean" + }, + { + "value": "tr", + "label": "Turkish" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": true + } + ] + }, + { + "id": "translation", + "name": "Translation", + "output": [ + { + "name": "Target Text", + "code": "data", + "dataType": "text", + "defaultValue": true + } + ], + "params": [ + { + "code": "text", + "name": "Source Text", + "required": true, + "isFixed": false, + "dataType": "text", + "dataSubType": "text", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguage", + "name": "Target Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script_in", + "name": "Script In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "script_out", + "name": "Script Out", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_out", + "name": "Dialect Out", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "context", + "name": "Context", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + }, + { + "id": "subtitling", + "name": "Subtitling", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "sourcelanguage", + "name": "Source Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect_in", + "name": "Dialect In", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_supplier", + "name": "Speech Recognition Supplier", + "required": false, + "isFixed": false, + "defaultValues": [ + { + "value": "aws", + "label": "AWS" + } + ], + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "google", + "label": "Google" + }, + { + "value": "deepgram", + "label": "Deepgram" + }, + { + "value": "revai", + "label": "Rev.AI" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "openai", + "label": "OpenAI" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "target_supplier", + "name": "Machine Translation Supplier", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "aws", + "label": "AWS" + }, + { + "value": "azure", + "label": "Azure" + }, + { + "value": "modernmt", + "label": "ModernMT" + }, + { + "value": "apptek", + "label": "AppTek" + }, + { + "value": "google", + "label": "Google" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "targetlanguages", + "name": "Target Languages", + "required": false, + "isFixed": false, + "availableOptions": [ + { + "value": "en", + "label": "English" + }, + { + "value": "de", + "label": "German" + }, + { + "value": "nl", + "label": "Dutch" + }, + { + "value": "fr", + "label": "French" + }, + { + "value": "el", + "label": "Greek" + }, + { + "value": "it", + "label": "Italian" + }, + { + "value": "pl", + "label": "Polish" + }, + { + "value": "pt", + "label": "Portuguese" + }, + { + "value": "ru", + "label": "Russian" + }, + { + "value": "es", + "label": "Spanish" + }, + { + "value": "zh", + "label": "Chinese" + }, + { + "value": "sl", + "label": "Slovenian" + }, + { + "value": "ar", + "label": "Arabic" + }, + { + "value": "ja", + "label": "Japanese" + }, + { + "value": "ko", + "label": "Korean" + }, + { + "value": "tr", + "label": "Turkish" + } + ], + "dataType": "label", + "dataSubType": "label", + "multipleValues": true + } + ] + }, + { + "id": "speech-recognition", + "name": "Speech Recognition", + "output": [ + { + "name": "Target text", + "code": "data", + "dataType": "text", + "defaultValue": [] + } + ], + "params": [ + { + "code": "language", + "name": "Language", + "required": true, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "dialect", + "name": "Dialect", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "voice", + "name": "Voice", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + }, + { + "code": "source_audio", + "name": "Source Audio", + "required": true, + "isFixed": false, + "dataType": "audio", + "dataSubType": "audio", + "multipleValues": false + }, + { + "code": "script", + "name": "Script", + "required": false, + "isFixed": true, + "dataType": "label", + "dataSubType": "label", + "multipleValues": false + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/mock_responses/list_host_machines_response.json b/tests/mock_responses/list_host_machines_response.json new file mode 100644 index 00000000..35bcb7e4 --- /dev/null +++ b/tests/mock_responses/list_host_machines_response.json @@ -0,0 +1,18 @@ +[ + { + "id": "64dce914adc92335dc35beb5", + "code": "aix-2c-8g-od", + "type": "on-demand", + "cores": 2, + "memory": 8, + "hourlyCost": 0.12 + }, + { + "id": "64dceafdadc92335dc35beb6", + "code": "aix-2c-8g", + "type": "always-on", + "cores": 2, + "memory": 8, + "hourlyCost": 0.096 + } +] \ No newline at end of file diff --git a/tests/mock_responses/list_image_repo_tags_response.json b/tests/mock_responses/list_image_repo_tags_response.json new file mode 100644 index 00000000..f69b90dc --- /dev/null +++ b/tests/mock_responses/list_image_repo_tags_response.json @@ -0,0 +1 @@ +["tag1", "tag2", "tag3", "tag4"] \ No newline at end of file diff --git a/tests/mock_responses/login_response.json b/tests/mock_responses/login_response.json new file mode 100644 index 00000000..ae755aca --- /dev/null +++ b/tests/mock_responses/login_response.json @@ -0,0 +1,5 @@ +{ + "username": "AWS", + "password": "eyJwYXlsb2FkIjoiNlNkQmp0WkRWbDRtQmxYMk5ZWHh0dFJQRTJFeWpScDFVczI1RTl2WUJMRmN5SVU1TE1wd3hiK2FaZWtnbjZidy9Ea3UxQ1FpcnIwRURwNXZNMTJuZXBJVzhITjNkVEtmeFNCa1RwcTNESSt2ZnVtSm5MVXM3KzlPNEo3cmRySDE5NjdnazYyb0NIRVV1WmZvOUFuUm5CeHUyU2ZmZWFndFlYVyt0dDVXeWtMQjRCRlNFaGJUelNtSnllSW9pNTlkNFNYdGtXY3pDT1RZQ281MUVlVEI0L1c4NGZMVVZQRVF6VThmdmtYRVl1TDNEUWFzc3F3dUxxcHp2bWtrSCtNOHNrdFp6bHZubXlxMnFGYkR0aElhamNXNW1Ud1BkVjJMN2w0ZFJVSTlTQ3Y1SlExbnlZZ01obUxHeDRDRG5KYmh0NndzeEtWcVpxbmMzMDR6WXZnQlZTcWFEY2VvWXV0SFEwSTVSQU1DaUtNd09SZHF0Skt1Y3FxRVBwTkxPaDhlcUFScmd4bkVCYnhQZm4zZ0M5L0x2bHBiZ1I5UFRIWGlqZlFWczNnUW5vTzFmd0R2d1dudTRsMjJDWjdSUTN4WlRNL29NdFNtZ2RScmplclpqNWo0RVMycTdQTEFXOU9UcUtieDRpZklMRUVucTIxbDBXaFNtc0xlR2g4Rm9GZkpOSGJ5L2wzUklTY2hjUzBYUUdYMXJ0cFhFOTc3bUVtdzY0WDdYT3h5UGlnZytzNWowMjhFY0VqSzV6R01sNzdDYUprcVVyZjVUUWZraTU4VURCMTNXWDlvVDVGQVUvcU9DY3F0SlQ5TlBZTnFXQ0xhamdFdk93TXFsQndkVzhKTEhwMTkwZ3psNE1nN0YwRDIvTFpScWRDVVh2SXRBSFJJUmROa1U3RDI1Y3VoL0xjSjlhZUQ2MnJiVDA1R2FIWkV5Z0d5MmxnRWlmekUvbWhPSGNUclBOSnlPTGhHaFc2L0F5dCt1MDRxNEdqMzVFQk1GSHZ0a0lLUEQ5MU04NTVKZnVMV3F3d09QR1NlZnNGRXlRNExxRGZtMkNueVpqd3NuNWRFSlR5VUZhTUMyODMwbCtBV3lZMFBQQ3l6eTFJK0FoMHV0VkJvMlBabkFPZVk1c3hOL05uOFhlbmRMbTA0Mm1wTENWOCtHd3lzYnVFM1BHRDdNV3pDaVVicm0rbXdBLzk0c3hTODlTNkJpVWhnUHp5RC84TWhyVUNNL1FTRGNFY3ZUTjVFc0N0UDM1cUdUT28yOWdxc3VzdWRLZHdEQkhWMlpkaGNNR0xQMElWNEZKN01CQVZSMnd4OTRiZXpDMm4xU3V5TGRGVVBQYVFKa2wwWmw2M3E4MU5FRjdMSzQ0M0FJbzlpV3FuazltbFBYRVo1OHdVUERnMUpZbWw4b3BCYVprazJtM2dvYk5HdEFWUHY1dDlXZitXY2Q2MDN3WnJ1TlhwUTNPSlk2WWI4ZXBMNlZpN1ErTkpaa2Z0NWl5M1FQRFpUUFZjSCs0c1VjZ0E2dmFMSUY2aEZCUncwWitRS0pvK0VZUWtFK0RTQXhMaldFYkt5ZzBSN1V3UHg0VThENjQ4My9mMlV2cU5jSFRORHNkbXRKcjlXcUwxNHRoc1BqQTNqZ3Bqc0pydDJJWTA1bEdNOWJJbGpmbUtGWFdsemppQ2ptSUNsSm14SUxIdzgvSTlKb3JYb2NmNXpoSHVzbCswUkdKc1NMTHAyOWc9PSIsImRhdGFrZXkiOiJBUUVCQUhod20wWWFJU0plUnRKbTVuMUc2dXFlZWtYdW9YWFBlNVVGY2U5UnE4LzE0d0FBQUg0d2ZBWUpLb1pJaHZjTkFRY0dvRzh3YlFJQkFEQm9CZ2txaGtpRzl3MEJCd0V3SGdZSllJWklBV1VEQkFFdU1CRUVERGFyODZkalUxNVFHNCtZaEFJQkVJQTdvY0xIeWFpUHViY2VTQ0g5djB6THd2UFZGbHU0WmJqZ09JSGkrdmxiNEpCVTBlNyt5VmpnT3BpcWVmQlkxbFBGWktKalgvMEIwMkJDcU1nPSIsInZlcnNpb24iOiIyIiwidHlwZSI6IkRBVEFfS0VZIiwiZXhwaXJhdGlvbiI6MTY5MjYxNDYwMX0=", + "registry": "https://535945872701.dkr.ecr.us-east-1.amazonaws.com" +} \ No newline at end of file diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json new file mode 100644 index 00000000..4683e526 --- /dev/null +++ b/tests/test_requests/create_asset_request.json @@ -0,0 +1,8 @@ +{ + "name": "mock_name", + "hostingMachine": "aix-2c-8g-od", + "version": "mock_version", + "description": "mock_description", + "function": "Speech Recognition", + "sourceLanguage": "en" +} \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 00000000..e7a41e16 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,16 @@ +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin +import logging +from aixplain.utils import config + +def delete_asset(model_id, api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/models/{model_id}") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("delete", delete_url, headers=headers) + +def delete_service_account(api_key): + delete_url = urljoin(config.BACKEND_URL, f"sdk/ecr/logout") + logging.debug(f"URL: {delete_url}") + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + _ = _request_with_retry("post", delete_url, headers=headers) \ No newline at end of file