From 23b2898f9e1e5a6c56448ee68b0f42184bb709e1 Mon Sep 17 00:00:00 2001 From: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Date: Thu, 2 May 2024 14:03:50 -0300 Subject: [PATCH 01/25] Update Finetuner search metadata functional tests (#172) --- tests/functional/finetune/data/finetune_test_end2end.json | 8 ++++---- tests/functional/finetune/finetune_functional_test.py | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/functional/finetune/data/finetune_test_end2end.json b/tests/functional/finetune/data/finetune_test_end2end.json index ead1fd88..80768de9 100644 --- a/tests/functional/finetune/data/finetune_test_end2end.json +++ b/tests/functional/finetune/data/finetune_test_end2end.json @@ -10,17 +10,17 @@ { "model_name": "aiR", "model_id": "6499cc946eb5633de15d82a1", - "dataset_name": "Test search dataset metadata", + "dataset_name": "Test search dataset", "inference_data": "Hello!", "required_dev": false, - "search_metadata": true + "search_metadata": false }, { "model_name": "vectara", "model_id": "655e20f46eb563062a1aa301", - "dataset_name": "Test search dataset metadata", + "dataset_name": "Test search dataset", "inference_data": "Hello!", "required_dev": false, - "search_metadata": true + "search_metadata": false } ] \ No newline at end of file diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index 94693f05..ffa9ad5a 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -83,6 +83,8 @@ def test_end2end(run_input_map): time.sleep(5) end = time.time() assert finetune_model.check_finetune_status().model_status.value == "onboarded" + time.sleep(30) + print(f"Model dict: {finetune_model.__dict__}") result = finetune_model.run(run_input_map["inference_data"]) print(f"Result: {result}") assert result is not None From 208a0814ff2adbb562f923993759f265bd57c0f2 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Thu, 2 May 2024 14:09:12 -0300 Subject: [PATCH 02/25] Downgrade dataclasses-json for compatibility (#170) Co-authored-by: Thiago Castro Ferreira --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ad67878..112c8f9a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ namespaces = true [project] name = "aiXplain" -version = "0.2.5rc" +version = "0.2.12" description = "aiXplain SDK adds AI functions to software." readme = "README.md" requires-python = ">=3.5, <4" @@ -51,7 +51,7 @@ dependencies = [ "filetype>=1.2.0", "click>=7.1.2,<8.0.0", "PyYAML>=6.0.1", - "dataclasses-json==0.6.1" + "dataclasses-json>=0.5.2" ] [project.urls] From a837e1a5777ee12de2a1bddb991b4c47510247f1 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Fri, 10 May 2024 11:33:49 -0300 Subject: [PATCH 03/25] Fix model cost parameters (#179) Co-authored-by: Thiago Castro Ferreira --- aixplain/factories/model_factory.py | 22 +++++++++---------- aixplain/modules/asset.py | 3 ++- aixplain/modules/metric.py | 15 ++++++++----- aixplain/modules/model.py | 12 ++++++---- .../general_assets/asset_functional_test.py | 4 ++-- 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index cd7de970..9ed3138f 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -65,7 +65,7 @@ def _create_model_from_response(cls, response: Dict) -> Model: response["name"], supplier=response["supplier"], api_key=response["api_key"], - pricing=response["pricing"], + cost=response["pricing"], function=Function(response["function"]["id"]), parameters=parameters, is_subscribed=True if "subscription" in response else False, @@ -404,9 +404,11 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." logging.info(message) return response - + @classmethod - def deploy_huggingface_model(cls, name: Text, hf_repo_id: Text, hf_token: Optional[Text] = "", api_key: Optional[Text] = None) -> Dict: + def deploy_huggingface_model( + cls, name: Text, hf_repo_id: Text, hf_token: Optional[Text] = "", api_key: Optional[Text] = None + ) -> Dict: """Onboards and deploys a Hugging Face large language model. Args: @@ -433,20 +435,16 @@ def deploy_huggingface_model(cls, name: Text, hf_repo_id: Text, hf_token: Option "sourceLanguage": "en", }, "source": "huggingface", - "onboardingParams": { - "hf_model_name": model_name, - "hf_supplier": supplier, - "hf_token": hf_token - } + "onboardingParams": {"hf_model_name": model_name, "hf_supplier": supplier, "hf_token": hf_token}, } response = _request_with_retry("post", deploy_url, headers=headers, json=body) logging.debug(response.text) response_dicts = json.loads(response.text) return response_dicts - + @classmethod def get_huggingface_model_status(cls, model_id: Text, api_key: Optional[Text] = None): - """Gets the on-boarding status of a Hugging Face model with ID MODEL_ID. + """Gets the on-boarding status of a Hugging Face model with ID MODEL_ID. Args: model_id (Text): The model's ID as returned by DEPLOY_HUGGINGFACE_MODEL @@ -466,6 +464,6 @@ def get_huggingface_model_status(cls, model_id: Text, api_key: Optional[Text] = "status": response_dicts["status"], "name": response_dicts["name"], "id": response_dicts["id"], - "pricing": response_dicts["pricing"] + "pricing": response_dicts["pricing"], } - return ret_dict \ No newline at end of file + return ret_dict diff --git a/aixplain/modules/asset.py b/aixplain/modules/asset.py index 34fea4e4..52b79912 100644 --- a/aixplain/modules/asset.py +++ b/aixplain/modules/asset.py @@ -36,7 +36,7 @@ def __init__( version: Text = "1.0", license: Optional[License] = None, privacy: Privacy = Privacy.PRIVATE, - cost: float = 0, + cost: Optional[Union[Dict, float]] = None, ) -> None: """Create an Asset with the necessary information @@ -46,6 +46,7 @@ def __init__( description (Text): Description of the Asset supplier (Union[Dict, Text, Supplier, int], optional): supplier of the asset. Defaults to "aiXplain". version (Optional[Text], optional): asset version. Defaults to "1.0". + cost (Optional[Union[Dict, float]], optional): asset price. Defaults to None. """ self.id = id self.name = name diff --git a/aixplain/modules/metric.py b/aixplain/modules/metric.py index 04a0bdd7..d591772b 100644 --- a/aixplain/modules/metric.py +++ b/aixplain/modules/metric.py @@ -61,12 +61,10 @@ def __init__( supplier (Text): author of the Metric is_reference_required (bool): does the metric use reference is_source_required (bool): does the metric use source - cost (float): cost of the metric + cost (float): price of the metric normalization_options(list, []) **additional_info: Any additional Metric info to be saved """ - - super().__init__(id, name, description="", supplier=supplier, version="1.0", cost=cost) self.is_source_required = is_source_required self.is_reference_required = is_reference_required @@ -76,7 +74,7 @@ def __init__( def __repr__(self) -> str: return f"" - + def add_normalization_options(self, normalization_options: List[str]): """Add a given set of normalization options to be used while benchmarking @@ -85,7 +83,12 @@ def add_normalization_options(self, normalization_options: List[str]): """ self.normalization_options.append(normalization_options) - def run(self, hypothesis: Optional[Union[str, List[str]]]=None, source: Optional[Union[str, List[str]]]=None, reference: Optional[Union[str, List[str]]]=None): + def run( + self, + hypothesis: Optional[Union[str, List[str]]] = None, + source: Optional[Union[str, List[str]]] = None, + reference: Optional[Union[str, List[str]]] = None, + ): """Run the metric to calculate the scores. Args: @@ -94,6 +97,7 @@ def run(self, hypothesis: Optional[Union[str, List[str]]]=None, source: Optional reference (Optional[Union[str, List[str]]], optional): Can give a single reference or a list of references for metric calculation. Defaults to None. """ from aixplain.factories.model_factory import ModelFactory + model = ModelFactory.get(self.id) payload = { "function": self.function, @@ -115,4 +119,3 @@ def run(self, hypothesis: Optional[Union[str, List[str]]]=None, source: Optional reference = [[ref] for ref in reference] payload["references"] = reference return model.run(payload) - diff --git a/aixplain/modules/model.py b/aixplain/modules/model.py index fc3a82cd..983737c7 100644 --- a/aixplain/modules/model.py +++ b/aixplain/modules/model.py @@ -48,6 +48,7 @@ class Model(Asset): function (Text, optional): model AI function. Defaults to None. url (str): URL to run the model. backend_url (str): URL of the backend. + pricing (Dict, optional): model price. Defaults to None. **additional_info: Any additional Model info to be saved """ @@ -61,6 +62,7 @@ def __init__( version: Optional[Text] = None, function: Optional[Text] = None, is_subscribed: bool = False, + cost: Optional[Dict] = None, **additional_info, ) -> None: """Model Init @@ -74,9 +76,10 @@ def __init__( version (Text, optional): version of the model. Defaults to "1.0". function (Text, optional): model AI function. Defaults to None. is_subscribed (bool, optional): Is the user subscribed. Defaults to False. + cost (Dict, optional): model price. Defaults to None. **additional_info: Any additional Model info to be saved """ - super().__init__(id, name, description, supplier, version) + super().__init__(id, name, description, supplier, version, cost=cost) self.api_key = api_key self.additional_info = additional_info self.url = config.MODELS_RUN_URL @@ -264,6 +267,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): """ from aixplain.enums.asset_status import AssetStatus from aixplain.modules.finetune.status import FinetuneStatus + headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} resp = None try: @@ -274,7 +278,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): finetune_status = AssetStatus(resp["finetuneStatus"]) model_status = AssetStatus(resp["modelStatus"]) logs = sorted(resp["logs"], key=lambda x: float(x["epoch"])) - + target_epoch = None if after_epoch is not None: logs = [log for log in logs if float(log["epoch"]) > after_epoch] @@ -282,7 +286,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): target_epoch = float(logs[0]["epoch"]) elif len(logs) > 0: target_epoch = float(logs[-1]["epoch"]) - + if target_epoch is not None: log = None for log_ in logs: @@ -294,7 +298,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): log["trainLoss"] = log_["trainLoss"] if log_["evalLoss"] is not None: log["evalLoss"] = log_["evalLoss"] - + status = FinetuneStatus( status=finetune_status, model_status=model_status, diff --git a/tests/functional/general_assets/asset_functional_test.py b/tests/functional/general_assets/asset_functional_test.py index 6a9dceda..93a3b297 100644 --- a/tests/functional/general_assets/asset_functional_test.py +++ b/tests/functional/general_assets/asset_functional_test.py @@ -82,8 +82,8 @@ def test_model_sort(): prev_model = models[idx - 1] model = models[idx] - prev_model_price = prev_model.additional_info["pricing"]["price"] - model_price = model.additional_info["pricing"]["price"] + prev_model_price = prev_model.cost["price"] + model_price = model.cost["price"] assert prev_model_price >= model_price From 754f478cf94f0a0af4660242d44312cededf335e Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Wed, 15 May 2024 19:46:58 -0300 Subject: [PATCH 04/25] Treat label URLs (#176) Co-authored-by: Thiago Castro Ferreira --- aixplain/processes/data_onboarding/process_media_files.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/aixplain/processes/data_onboarding/process_media_files.py b/aixplain/processes/data_onboarding/process_media_files.py index 8b333d72..62fd369a 100644 --- a/aixplain/processes/data_onboarding/process_media_files.py +++ b/aixplain/processes/data_onboarding/process_media_files.py @@ -5,6 +5,7 @@ import pandas as pd import shutil import tarfile +import validators from aixplain.enums.data_subtype import DataSubtype from aixplain.enums.data_type import DataType @@ -115,6 +116,13 @@ def run(metadata: MetaData, paths: List, folder: Path, batch_size: int = 100) -> shutil.copy2(media_path, new_path) batch.append(fname) else: + if metadata.storage_type == StorageType.TEXT and ( + str(media_path).startswith("s3://") + or str(media_path).startswith("http://") + or str(media_path).startswith("https://") + or validators.url(media_path) + ): + media_path = "DONOTDOWNLOAD" + str(media_path) batch.append(media_path) # crop intervals can not be used with interval data types From f1c9935c53afd7185243a455d9026e9088a46b59 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:48:28 -0300 Subject: [PATCH 05/25] Add new metric test (#181) * Add new metric test * Enable testing new pipeline executor --------- Co-authored-by: Thiago Castro Ferreira --- tests/functional/pipelines/run_test.py | 139 +++++++++++++++++++------ 1 file changed, 108 insertions(+), 31 deletions(-) diff --git a/tests/functional/pipelines/run_test.py b/tests/functional/pipelines/run_test.py index e4389587..e8bc4d9c 100644 --- a/tests/functional/pipelines/run_test.py +++ b/tests/functional/pipelines/run_test.py @@ -18,6 +18,7 @@ import pytest import os +import requests from aixplain.factories import DatasetFactory, PipelineFactory @@ -38,61 +39,110 @@ def test_get_pipeline(): assert hypothesis_pipeline.id == reference_pipeline.id -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_single_str(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_single_str(batchmode: bool, version: str): pipeline = PipelineFactory.list(query="SingleNodePipeline")["results"][0] - response = pipeline.run(data="Translate this thing", **{"batchmode": batchmode}) + response = pipeline.run(data="Translate this thing", **{"batchmode": batchmode, "version": version}) assert response["status"] == "SUCCESS" -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_single_local_file(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_single_local_file(batchmode: bool, version: str): pipeline = PipelineFactory.list(query="SingleNodePipeline")["results"][0] fname = "translate_this.txt" with open(fname, "w") as f: f.write("Translate this thing") - response = pipeline.run(data=fname, **{"batchmode": batchmode}) + response = pipeline.run(data=fname, **{"batchmode": batchmode, "version": version}) os.remove(fname) assert response["status"] == "SUCCESS" -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_with_url(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_with_url(batchmode: bool, version: str): pipeline = PipelineFactory.list(query="SingleNodePipeline")["results"][0] response = pipeline.run( data="https://aixplain-platform-assets.s3.amazonaws.com/data/dev/64c81163f8bdcac7443c2dad/data/f8.txt", - **{"batchmode": batchmode} + **{"batchmode": batchmode, "version": version} ) assert response["status"] == "SUCCESS" -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_with_dataset(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_with_dataset(batchmode: bool, version: str): dataset = DatasetFactory.list(query="for_functional_tests")["results"][0] data_asset_id = dataset.id data_id = dataset.source_data["en"].id pipeline = PipelineFactory.list(query="SingleNodePipeline")["results"][0] - response = pipeline.run(data=data_id, data_asset=data_asset_id, **{"batchmode": batchmode}) + response = pipeline.run(data=data_id, data_asset=data_asset_id, **{"batchmode": batchmode, "version": version}) assert response["status"] == "SUCCESS" -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_multipipe_with_strings(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_multipipe_with_strings(batchmode: bool, version: str): pipeline = PipelineFactory.list(query="MultiInputPipeline")["results"][0] response = pipeline.run( - data={"Input": "Translate this thing.", "Reference": "Traduza esta coisa."}, **{"batchmode": batchmode} + data={"Input": "Translate this thing.", "Reference": "Traduza esta coisa."}, + **{"batchmode": batchmode, "version": version} ) assert response["status"] == "SUCCESS" -@pytest.mark.parametrize("batchmode", [True, False]) -def test_run_multipipe_with_datasets(batchmode: bool): +@pytest.mark.parametrize( + "batchmode,version", + [ + (True, "2.0"), + (True, "3.0"), + (False, "2.0"), + (False, "3.0"), + ], +) +def test_run_multipipe_with_datasets(batchmode: bool, version: str): pipeline = PipelineFactory.list(query="MultiInputPipeline")["results"][0] dataset = DatasetFactory.list(query="for_functional_tests")["results"][0] @@ -104,27 +154,50 @@ def test_run_multipipe_with_datasets(batchmode: bool): response = pipeline.run( data={"Input": input_id, "Reference": reference_id}, data_asset={"Input": data_asset_id, "Reference": data_asset_id}, - **{"batchmode": batchmode} + **{"batchmode": batchmode, "version": version} ) assert response["status"] == "SUCCESS" -def test_run_segment_reconstruct(): +@pytest.mark.parametrize("version", ["2.0", "3.0"]) +def test_run_segment_reconstruct(version: str): pipeline = PipelineFactory.list(query="Segmentation/Reconstruction Functional Test - DO NOT DELETE")["results"][0] - response = pipeline.run("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav") + response = pipeline.run("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", **{"version": version}) assert response["status"] == "SUCCESS" output = response["data"][0] assert output["label"] == "Output 1" -def test_run_metric(): +@pytest.mark.parametrize("version", ["2.0", "3.0"]) +def test_run_translation_metric(version: str): + dataset = DatasetFactory.list(query="for_functional_tests")["results"][0] + data_asset_id = dataset.id + + reference_id = dataset.target_data["pt"][0].id + + pipeline = PipelineFactory.list(query="Translation Metric Functional Test - DO NOT DELETE")["results"][0] + response = pipeline.run( + data={"TextInput": reference_id, "ReferenceInput": reference_id}, + data_asset={"TextInput": data_asset_id, "ReferenceInput": data_asset_id}, + **{"version": version} + ) + + assert response["status"] == "SUCCESS" + data = response["data"][0]["segments"][0]["response"] + data = requests.get(data).text + assert float(data) == 100.0 + + +@pytest.mark.parametrize("version", ["2.0", "3.0"]) +def test_run_metric(version: str): pipeline = PipelineFactory.list(query="ASR Metric Functional Test - DO NOT DELETE")["results"][0] response = pipeline.run( { "AudioInput": "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", "ReferenceInput": "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.txt", - } + }, + **{"version": version} ) assert response["status"] == "SUCCESS" @@ -134,15 +207,17 @@ def test_run_metric(): @pytest.mark.parametrize( - "input_data,output_data", + "input_data,output_data,version", [ - ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", "AudioOutput"), - ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.txt", "TextOutput"), + ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", "AudioOutput", "2.0"), + ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.txt", "TextOutput", "2.0"), + ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", "AudioOutput", "3.0"), + ("https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.txt", "TextOutput", "3.0"), ], ) -def test_run_router(input_data: str, output_data: str): +def test_run_router(input_data: str, output_data: str, version: str): pipeline = PipelineFactory.list(query="Router Test - DO NOT DELETE")["results"][0] - response = pipeline.run(input_data) + response = pipeline.run(input_data, **{"version": version}) assert response["status"] == "SUCCESS" assert response["data"][0]["label"] == output_data @@ -151,13 +226,15 @@ def test_run_router(input_data: str, output_data: str): @pytest.mark.parametrize( "input_data,output_data", [ - ("I love it.", "PositiveOutput"), - ("I hate it.", "NegativeOutput"), + ("I love it.", "PositiveOutput", "2.0"), + ("I hate it.", "NegativeOutput", "2.0"), + ("I love it.", "PositiveOutput", "3.0"), + ("I hate it.", "NegativeOutput", "3.0"), ], ) -def test_run_decision(input_data: str, output_data: str): +def test_run_decision(input_data: str, output_data: str, version: str): pipeline = PipelineFactory.list(query="Decision Test - DO NOT DELETE")["results"][0] - response = pipeline.run(input_data) + response = pipeline.run(input_data, **{"version": version}) assert response["status"] == "SUCCESS" assert response["data"][0]["label"] == output_data From a48ccfd6be9ceaba774cbf50cc0b07f5d87299aa Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Wed, 5 Jun 2024 10:01:47 -0300 Subject: [PATCH 06/25] LLMModel class and parameters (#184) * LLMModel class and parameters * Change in the documentation * Changing LLMModel for LLM * Remove frequency penalty --------- Co-authored-by: Thiago Castro Ferreira --- .pre-commit-config.yaml | 9 +- aixplain/factories/model_factory.py | 24 +- aixplain/modules/__init__.py | 1 + .../modules/{model.py => model/__init__.py} | 8 +- aixplain/modules/model/llm_model.py | 227 ++++++++++++++++++ docs/user/user_doc.md | 18 ++ .../general_assets/asset_functional_test.py | 10 +- tests/functional/model/run_model_test.py | 22 ++ 8 files changed, 304 insertions(+), 15 deletions(-) rename aixplain/modules/{model.py => model/__init__.py} (98%) create mode 100644 aixplain/modules/model/llm_model.py create mode 100644 tests/functional/model/run_model_test.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1395dfa6..a79973ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,4 +15,11 @@ repos: - id: black language_version: python3 args: # arguments to configure black - - --line-length=128 \ No newline at end of file + - --line-length=128 + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.0.0 # Use the latest version + hooks: + - id: flake8 + args: # arguments to configure black + - --ignore=E402,E501 \ No newline at end of file diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 9ed3138f..221fd94d 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -24,6 +24,7 @@ import json import logging from aixplain.modules.model import Model +from aixplain.modules.model.llm_model import LLM from aixplain.enums import Function, Language, OwnershipType, Supplier, SortBy, SortOrder from aixplain.utils import config from aixplain.utils.file_utils import _request_with_retry @@ -60,13 +61,18 @@ def _create_model_from_response(cls, response: Dict) -> Model: if "language" in param["name"]: parameters[param["name"]] = [w["value"] for w in param["values"]] - return Model( + function = Function(response["function"]["id"]) + ModelClass = Model + if function == Function.TEXT_GENERATION: + ModelClass = LLM + + return ModelClass( response["id"], response["name"], supplier=response["supplier"], api_key=response["api_key"], cost=response["pricing"], - function=Function(response["function"]["id"]), + function=function, parameters=parameters, is_subscribed=True if "subscription" in response else False, version=response["version"]["id"], @@ -100,7 +106,7 @@ def get(cls, model_id: Text, api_key: Optional[Text] = None) -> Model: model = cls._create_model_from_response(resp) logging.info(f"Model Creation: Model {model_id} instantiated.") return model - except Exception as e: + except Exception: if resp is not None and "statusCode" in resp: status_code = resp["statusCode"] message = resp["message"] @@ -135,7 +141,7 @@ def _get_assets_from_page( sort_order: SortOrder = SortOrder.ASCENDING, ) -> List[Model]: try: - url = urljoin(cls.backend_url, f"sdk/models/paginate") + url = urljoin(cls.backend_url, "sdk/models/paginate") filter_params = {"q": query, "pageNumber": page_number, "pageSize": page_size} if is_finetunable is not None: filter_params["isFineTunable"] = is_finetunable @@ -253,7 +259,7 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: List[Dict]: List of dictionaries containing information about each hosting machine. """ - machines_url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") + machines_url = urljoin(config.BACKEND_URL, "sdk/hosting-machines") logging.debug(f"URL: {machines_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -278,7 +284,7 @@ def list_functions(cls, verbose: Optional[bool] = False, api_key: Optional[Text] List[Dict]: List of dictionaries containing information about each supported function. """ - functions_url = urljoin(config.BACKEND_URL, f"sdk/functions") + functions_url = urljoin(config.BACKEND_URL, "sdk/functions") logging.debug(f"URL: {functions_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -336,7 +342,7 @@ def create_asset_repo( function_id = function_dict["id"] if function_id is None: raise Exception("Invalid function name") - create_url = urljoin(config.BACKEND_URL, f"sdk/models/register") + create_url = urljoin(config.BACKEND_URL, "sdk/models/register") logging.debug(f"URL: {create_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -370,7 +376,7 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: Returns: Dict: Backend response """ - login_url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") + login_url = urljoin(config.BACKEND_URL, "sdk/ecr/login") logging.debug(f"URL: {login_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} @@ -420,7 +426,7 @@ def deploy_huggingface_model( Dict: Backend response """ supplier, model_name = hf_repo_id.split("/") - deploy_url = urljoin(config.BACKEND_URL, f"sdk/model-onboarding/onboard") + deploy_url = urljoin(config.BACKEND_URL, "sdk/model-onboarding/onboard") if api_key: headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} else: diff --git a/aixplain/modules/__init__.py b/aixplain/modules/__init__.py index bb9e696b..488c8c2f 100644 --- a/aixplain/modules/__init__.py +++ b/aixplain/modules/__init__.py @@ -27,6 +27,7 @@ from .metadata import MetaData from .metric import Metric from .model import Model +from .model.llm_model import LLM from .pipeline import Pipeline from .finetune import Finetune, FinetuneCost from .finetune.status import FinetuneStatus diff --git a/aixplain/modules/model.py b/aixplain/modules/model/__init__.py similarity index 98% rename from aixplain/modules/model.py rename to aixplain/modules/model/__init__.py index 983737c7..12c96977 100644 --- a/aixplain/modules/model.py +++ b/aixplain/modules/model/__init__.py @@ -45,7 +45,7 @@ class Model(Asset): url (Text, optional): endpoint of the model. Defaults to config.MODELS_RUN_URL. supplier (Union[Dict, Text, Supplier, int], optional): supplier of the asset. Defaults to "aiXplain". version (Text, optional): version of the model. Defaults to "1.0". - function (Text, optional): model AI function. Defaults to None. + function (Function, optional): model AI function. Defaults to None. url (str): URL to run the model. backend_url (str): URL of the backend. pricing (Dict, optional): model price. Defaults to None. @@ -60,7 +60,7 @@ def __init__( api_key: Optional[Text] = None, supplier: Union[Dict, Text, Supplier, int] = "aiXplain", version: Optional[Text] = None, - function: Optional[Text] = None, + function: Optional[Function] = None, is_subscribed: bool = False, cost: Optional[Dict] = None, **additional_info, @@ -102,7 +102,7 @@ def __repr__(self): except Exception: return f"" - def __polling(self, poll_url: Text, name: Text = "model_process", wait_time: float = 0.5, timeout: float = 300) -> Dict: + def sync_poll(self, poll_url: Text, name: Text = "model_process", wait_time: float = 0.5, timeout: float = 300) -> Dict: """Keeps polling the platform to check whether an asynchronous call is done. Args: @@ -198,7 +198,7 @@ def run( return response poll_url = response["url"] end = time.time() - response = self.__polling(poll_url, name=name, timeout=timeout, wait_time=wait_time) + response = self.sync_poll(poll_url, name=name, timeout=timeout, wait_time=wait_time) return response except Exception as e: msg = f"Error in request for {name} - {traceback.format_exc()}" diff --git a/aixplain/modules/model/llm_model.py b/aixplain/modules/model/llm_model.py new file mode 100644 index 00000000..349ea595 --- /dev/null +++ b/aixplain/modules/model/llm_model.py @@ -0,0 +1,227 @@ +__author__ = "lucaspavanelli" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Thiago Castro Ferreira, Shreyas Sharma and Lucas Pavanelli +Date: June 4th 2024 +Description: + Large Language Model Class +""" +import time +import json +import logging +import traceback +from aixplain.factories.file_factory import FileFactory +from aixplain.enums import Function, Supplier +from aixplain.modules.model import Model +from aixplain.utils import config +from aixplain.utils.file_utils import _request_with_retry +from typing import Union, Optional, List, Text, Dict + + +class LLM(Model): + """Ready-to-use LLM model. This model can be run in both synchronous and asynchronous manner. + + Attributes: + id (Text): ID of the Model + name (Text): Name of the Model + description (Text, optional): description of the model. Defaults to "". + api_key (Text, optional): API key of the Model. Defaults to None. + url (Text, optional): endpoint of the model. Defaults to config.MODELS_RUN_URL. + supplier (Union[Dict, Text, Supplier, int], optional): supplier of the asset. Defaults to "aiXplain". + version (Text, optional): version of the model. Defaults to "1.0". + function (Text, optional): model AI function. Defaults to None. + url (str): URL to run the model. + backend_url (str): URL of the backend. + pricing (Dict, optional): model price. Defaults to None. + **additional_info: Any additional Model info to be saved + """ + + def __init__( + self, + id: Text, + name: Text, + description: Text = "", + api_key: Optional[Text] = None, + supplier: Union[Dict, Text, Supplier, int] = "aiXplain", + version: Optional[Text] = None, + function: Optional[Function] = None, + is_subscribed: bool = False, + cost: Optional[Dict] = None, + **additional_info, + ) -> None: + """LLM Init + + Args: + id (Text): ID of the Model + name (Text): Name of the Model + description (Text, optional): description of the model. Defaults to "". + api_key (Text, optional): API key of the Model. Defaults to None. + supplier (Union[Dict, Text, Supplier, int], optional): supplier of the asset. Defaults to "aiXplain". + version (Text, optional): version of the model. Defaults to "1.0". + function (Function, optional): model AI function. Defaults to None. + is_subscribed (bool, optional): Is the user subscribed. Defaults to False. + cost (Dict, optional): model price. Defaults to None. + **additional_info: Any additional Model info to be saved + """ + assert function == Function.TEXT_GENERATION, "LLM only supports large language models (i.e. text generation function)" + super().__init__( + id=id, + name=name, + description=description, + supplier=supplier, + version=version, + cost=cost, + function=function, + is_subscribed=is_subscribed, + api_key=api_key, + **additional_info, + ) + self.url = config.MODELS_RUN_URL + self.backend_url = config.BACKEND_URL + + def run( + self, + data: Text, + context: Optional[Text] = None, + prompt: Optional[Text] = None, + history: Optional[List[Dict]] = None, + temperature: float = 0.001, + max_tokens: int = 128, + top_p: float = 1.0, + name: Text = "model_process", + timeout: float = 300, + parameters: Dict = {}, + wait_time: float = 0.5, + ) -> Dict: + """Synchronously running a Large Language Model (LLM) model. + + Args: + data (Union[Text, Dict]): Text to LLM or last user utterance of a conversation. + context (Optional[Text], optional): System message. Defaults to None. + prompt (Optional[Text], optional): Prompt Message which comes on the left side of the last utterance. Defaults to None. + history (Optional[List[Dict]], optional): Conversation history in OpenAI format ([{ "role": "assistant", "content": "Hello, world!"}]). Defaults to None. + temperature (float, optional): LLM temperature. Defaults to 0.001. + max_tokens (int, optional): Maximum Generation Tokens. Defaults to 128. + top_p (float, optional): Top P. Defaults to 1.0. + name (Text, optional): ID given to a call. Defaults to "model_process". + timeout (float, optional): total polling time. Defaults to 300. + parameters (Dict, optional): optional parameters to the model. Defaults to "{}". + wait_time (float, optional): wait time in seconds between polling calls. Defaults to 0.5. + + Returns: + Dict: parsed output from model + """ + start = time.time() + try: + response = self.run_async( + data, + name=name, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + context=context, + prompt=prompt, + history=history, + parameters=parameters, + ) + if response["status"] == "FAILED": + end = time.time() + response["elapsed_time"] = end - start + return response + poll_url = response["url"] + end = time.time() + response = self.sync_poll(poll_url, name=name, timeout=timeout, wait_time=wait_time) + return response + except Exception as e: + msg = f"Error in request for {name} - {traceback.format_exc()}" + logging.error(f"LLM Run: Error in running for {name}: {e}") + end = time.time() + return {"status": "FAILED", "error": msg, "elapsed_time": end - start} + + def run_async( + self, + data: Text, + context: Optional[Text] = None, + prompt: Optional[Text] = None, + history: Optional[List[Dict]] = None, + temperature: float = 0.001, + max_tokens: int = 128, + top_p: float = 1.0, + name: Text = "model_process", + parameters: Dict = {}, + ) -> Dict: + """Runs asynchronously a model call. + + Args: + data (Union[Text, Dict]): Text to LLM or last user utterance of a conversation. + context (Optional[Text], optional): System message. Defaults to None. + prompt (Optional[Text], optional): Prompt Message which comes on the left side of the last utterance. Defaults to None. + history (Optional[List[Dict]], optional): Conversation history in OpenAI format ([{ "role": "assistant", "content": "Hello, world!"}]). Defaults to None. + temperature (float, optional): LLM temperature. Defaults to 0.001. + max_tokens (int, optional): Maximum Generation Tokens. Defaults to 128. + top_p (float, optional): Top P. Defaults to 1.0. + name (Text, optional): ID given to a call. Defaults to "model_process". + parameters (Dict, optional): optional parameters to the model. Defaults to "{}". + + Returns: + dict: polling URL in response + """ + headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + + data = FileFactory.to_link(data) + if isinstance(data, dict): + payload = data + else: + try: + payload = json.loads(data) + if isinstance(payload, dict) is False: + if isinstance(payload, int) is True or isinstance(payload, float) is True: + payload = str(payload) + payload = {"data": payload} + except Exception: + payload = {"data": data} + parameters.update( + { + "context": context, + "prompt": prompt, + "history": history, + "temperature": temperature, + "max_tokens": max_tokens, + "top_p": top_p, + } + ) + payload.update(parameters) + payload = json.dumps(payload) + + call_url = f"{self.url}/{self.id}" + r = _request_with_retry("post", call_url, headers=headers, data=payload) + logging.info(f"Model Run Async: Start service for {name} - {self.url} - {payload} - {headers}") + + resp = None + try: + resp = r.json() + logging.info(f"Result of request for {name} - {r.status_code} - {resp}") + + poll_url = resp["data"] + response = {"status": "IN_PROGRESS", "url": poll_url} + except Exception: + response = {"status": "FAILED"} + msg = f"Error in request for {name} - {traceback.format_exc()}" + logging.error(f"Model Run Async: Error in running for {name}: {resp}") + if resp is not None: + response["error"] = msg + return response diff --git a/docs/user/user_doc.md b/docs/user/user_doc.md index 400ad0d2..a7fa2178 100644 --- a/docs/user/user_doc.md +++ b/docs/user/user_doc.md @@ -57,6 +57,24 @@ poll_url = start_response["url"] ## Poll to see current job status poll_response = model.poll(poll_url) ``` + +You may also set special parameters for Large Language Models in the platform. + +```python +from aixplain.factories import ModelFactory +from aixplain.enums import Function +model = ModelFactory.list(query="GPT-4o", function=Function.TEXT_GENERATION)["results"][0] +response = model.run( + data="What is my name?", # last utterance + context="Always assist with care, respect, and truth. Respond with utmost utility yet securely. Avoid harmful, unethical, prejudiced, or negative content. Ensure replies promote fairness and positivity.", # system prompt + history=[ + { "role": "user", "content": "Hello! My name is James." }, + { "role": "assistant", "content": "Hello!" } + ], # conversation history, + temperature=0.7 +) +``` + ### Deploying Hugging Face Large Language Models You can deploy your very own Hugging Face large language models on our platform using the aiXplain SDK: ```console diff --git a/tests/functional/general_assets/asset_functional_test.py b/tests/functional/general_assets/asset_functional_test.py index 93a3b297..d35a4d9a 100644 --- a/tests/functional/general_assets/asset_functional_test.py +++ b/tests/functional/general_assets/asset_functional_test.py @@ -3,6 +3,7 @@ load_dotenv() from aixplain.factories import ModelFactory, DatasetFactory, MetricFactory, PipelineFactory +from aixplain.modules import LLM from pathlib import Path from aixplain.enums import Function, Language, OwnershipType, Supplier, SortBy, SortOrder @@ -90,7 +91,7 @@ def test_model_sort(): def test_model_ownership(): models = ModelFactory.list(ownership=OwnershipType.SUBSCRIBED)["results"] for model in models: - assert model.is_subscribed == True + assert model.is_subscribed is True def test_model_query(): @@ -101,6 +102,13 @@ def test_model_query(): def test_model_deletion(): + """Test that a model cannot be deleted.""" model = ModelFactory.get("640b517694bf816d35a59125") with pytest.raises(Exception): model.delete() + + +def test_llm_instantiation(): + """Test that the LLM model is correctly instantiated.""" + models = ModelFactory.list(function=Function.TEXT_GENERATION)["results"] + assert isinstance(models[0], LLM) diff --git a/tests/functional/model/run_model_test.py b/tests/functional/model/run_model_test.py new file mode 100644 index 00000000..79979357 --- /dev/null +++ b/tests/functional/model/run_model_test.py @@ -0,0 +1,22 @@ +__author__ = "thiagocastroferreira" + +import pytest + +from aixplain.enums import Function +from aixplain.factories import ModelFactory +from aixplain.modules import LLM + + +@pytest.mark.parametrize("llm_model", ["Groq Llama 3 70B", "Chat GPT 3.5", "GPT-4o", "GPT 4 (32k)"]) +def test_llm_run(llm_model): + """Testing LLMs with history context""" + model = ModelFactory.list(query=llm_model, function=Function.TEXT_GENERATION)["results"][0] + + assert isinstance(model, LLM) + + response = model.run( + data="What is my name?", + history=[{"role": "user", "content": "Hello! My name is Thiago."}, {"role": "assistant", "content": "Hello!"}], + ) + assert response["status"] == "SUCCESS" + assert "thiago" in response["data"].lower() From c7f59ce74c033b46e2f2f190dff19254f10ec9e6 Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Date: Wed, 5 Jun 2024 11:34:33 -0700 Subject: [PATCH 07/25] Gpus (#185) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Release. (#141) * Merge dev to test (#107) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Development to Test (#109) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> * Merge to test (#111) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#118) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Merge to test (#124) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#126) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#129) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge to test (#135) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) * Enabling dataset ID and model ID as parameters for finetuner creation (#131) Co-authored-by: Thiago Castro Ferreira * Fix supplier representation of a model (#132) * Fix supplier representation of a model * Fixing parameter typing --------- Co-authored-by: Thiago Castro Ferreira * Fixing indentation in documentation sample code (#134) Co-authored-by: Thiago Castro Ferreira --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#137) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) * Enabling dataset ID and model ID as parameters for finetuner creation (#131) Co-authored-by: Thiago Castro Ferreira * Fix supplier representation of a model (#132) * Fix supplier representation of a model * Fixing parameter typing --------- Co-authored-by: Thiago Castro Ferreira * Fixing indentation in documentation sample code (#134) Co-authored-by: Thiago Castro Ferreira * Update FineTune unit and functional tests (#136) --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira * Merge to prod. (#152) * Merge dev to test (#107) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Development to Test (#109) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> * Merge to test (#111) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#118) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Merge to test (#124) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#126) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#129) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira * Merge to test (#135) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) * Enabling dataset ID and model ID as parameters for finetuner creation (#131) Co-authored-by: Thiago Castro Ferreira * Fix supplier representation of a model (#132) * Fix supplier representation of a model * Fixing parameter typing --------- Co-authored-by: Thiago Castro Ferreira * Fixing indentation in documentation sample code (#134) Co-authored-by: Thiago Castro Ferreira --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira * Merge dev to test (#137) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) * Enabling dataset ID and model ID as parameters for finetuner creation (#131) Co-authored-by: Thiago Castro Ferreira * Fix supplier representation of a model (#132) * Fix supplier representation of a model * Fixing parameter typing --------- Co-authored-by: Thiago Castro Ferreira * Fixing indentation in documentation sample code (#134) Co-authored-by: Thiago Castro Ferreira * Update FineTune unit and functional tests (#136) --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira * Merge to test (#142) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#117) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Do not download textual URLs (#120) * Do not download textual URLs * Treat as string --------- Co-authored-by: Thiago Castro Ferreira * Enable api key parameter in data asset creation (#122) Co-authored-by: Thiago Castro Ferreira * Update Finetuner hyperparameters (#125) * Update Finetuner hyperparameters * Change hyperparameters error message * Add new LLMs finetuner models (mistral and solar) (#128) * Enabling dataset ID and model ID as parameters for finetuner creation (#131) Co-authored-by: Thiago Castro Ferreira * Fix supplier representation of a model (#132) * Fix supplier representation of a model * Fixing parameter typing --------- Co-authored-by: Thiago Castro Ferreira * Fixing indentation in documentation sample code (#134) Co-authored-by: Thiago Castro Ferreira * Update FineTune unit and functional tests (#136) * Click fix (#140) * Merge to prod (#119) * Merge dev to test (#107) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Development to Test (#109) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> * Merge to test (#111) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) --------- Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Merge dev to test (#113) * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Update Finetuner functional tests (#112) --------- Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> * Hf deployment test (#114) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Hf deployment test (#118) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Create bounds for FineTune hyperparameters (#103) * Test bound to hyperparameters * Update finetune llm hyperparameters * Remove option to use PEFT, always on use now * Fixing pipeline general asset test (#106) * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain * Update Finetuner functional tests (#112) * Hf deployment test (#115) * Started adding Hugging Face deployment to aiXplain SDK Signed-off-by: mikelam-us-aixplain * Added model status function to SDK Signed-off-by: mikelam-us-aixplain * Updating Signed-off-by: mikelam-us-aixplain * Updated CLI Signed-off-by: mikelam-us * Adding CLI Signed-off-by: mikelam-us-aixplain * Corrected request error Signed-off-by: mikelam-us-aixplain * Clearing out unnecessary information in return Signed-off-by: mikelam-us-aixplain * Adding status Signed-off-by: mikelam-us-aixplain * Simplifying status Signed-off-by: mikelam-us-aixplain * Adding tests and correcting tokens Signed-off-by: mikelam-us-aixplain * Added bad repo ID test Signed-off-by: mikelam-us-aixplain * Finished rough draft of tests Signed-off-by: mikelam-us-aixplain * Adding tests Signed-off-by: mikelam-us-aixplain * Fixing hf token Signed-off-by: mikelam-us-aixplain * Adding hf token Signed-off-by: mikelam-us-aixplain * Correcting first test Signed-off-by: mikelam-us-aixplain * Testing Signed-off-by: mikelam-us-aixplain * Adding config Signed-off-by: mikelam-us-aixplain * Added user doc Signed-off-by: mikelam-us-aixplain * Added gated model test Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us * Adding HF token Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.githu… * Updated onboarding docs (#155) * Updated onboarding docs Signed-off-by: mikelam-us-aixplain * under -> undergoing Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain * Added GPU endpoint Signed-off-by: mikelam-us-aixplain * Added host machine Signed-off-by: mikelam-us-aixplain * Adding CLI Signed-off-by: mikelam-us-aixplain * Updating create_asset_repo Signed-off-by: Michael Lam * Added CLI for creating asset repo Signed-off-by: mikelam-us-aixplain * Updating self-onboarding documentation Signed-off-by: Michael Lam * Updating user doc Signed-off-by: Michael Lam * cli change Signed-off-by: Michael Lam * Minor naming/documentation corrections Signed-off-by: mikelam-us-aixplain * Updated flag Signed-off-by: mikelam-us-aixplain * Updated verbose flag Signed-off-by: mikelam-us-aixplain * Hugging Face deployer is baaaacckkgit add aixplain/factories/model_factory.py Signed-off-by: mikelam-us-aixplain * Added revision argument to Hugging Face deployment Signed-off-by: mikelam-us-aixplain * Updated documentation Signed-off-by: mikelam-us-aixplain --------- Signed-off-by: mikelam-us-aixplain Signed-off-by: mikelam-us Signed-off-by: Michael Lam Co-authored-by: ikxplain <88332269+ikxplain@users.noreply.github.com> Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira Co-authored-by: Shreyas Sharma <85180538+shreyasXplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira --- aixplain/cli_groups.py | 5 +- aixplain/factories/cli/model_factory_cli.py | 52 +++++++--- aixplain/factories/model_factory.py | 93 ++++++++++++------ aixplain/modules/metric.py | 1 - aixplain/modules/model/__init__.py | 4 - docs/user/user_doc.md | 94 ++++++++++--------- .../data/finetune_test_cost_estimation.json | 2 +- .../finetune/data/finetune_test_end2end.json | 4 +- .../finetune/finetune_functional_test.py | 2 +- 9 files changed, 157 insertions(+), 100 deletions(-) diff --git a/aixplain/cli_groups.py b/aixplain/cli_groups.py index c5f05826..ea5e28be 100644 --- a/aixplain/cli_groups.py +++ b/aixplain/cli_groups.py @@ -21,7 +21,7 @@ CLI Runner """ import click -from aixplain.factories.cli.model_factory_cli import list_host_machines, list_functions, create_asset_repo, asset_repo_login, onboard_model, deploy_huggingface_model, get_huggingface_model_status +from aixplain.factories.cli.model_factory_cli import list_host_machines, list_functions, create_asset_repo, asset_repo_login, onboard_model, deploy_huggingface_model, get_huggingface_model_status, list_gpus @click.group('cli') def cli(): @@ -51,6 +51,7 @@ def onboard(): create.add_command(create_asset_repo) list.add_command(list_host_machines) list.add_command(list_functions) +list.add_command(list_gpus) get.add_command(asset_repo_login) get.add_command(get_huggingface_model_status) onboard.add_command(onboard_model) @@ -58,4 +59,4 @@ def onboard(): def run_cli(): - cli() \ No newline at end of file + cli() diff --git a/aixplain/factories/cli/model_factory_cli.py b/aixplain/factories/cli/model_factory_cli.py index 264fadd9..b83d61cc 100644 --- a/aixplain/factories/cli/model_factory_cli.py +++ b/aixplain/factories/cli/model_factory_cli.py @@ -44,7 +44,7 @@ def list_host_machines(api_key: Optional[Text] = None) -> None: click.echo(ret_val_yaml) @click.command("functions") -@click.option("--verbose", default=False, +@click.option("--verbose", is_flag=True, help="List all function details, False by default.") @click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") @@ -62,21 +62,37 @@ def list_functions(verbose: bool, api_key: Optional[Text] = None) -> None: ret_val_yaml = yaml.dump(ret_val) click.echo(ret_val_yaml) +@click.command("gpus") +@click.option("--api-key", default=None, + help="TEAM_API_KEY if not already set in environment.") +def list_gpus(api_key: Optional[Text] = None) -> None: + """CLI wrapper function for the LIST_GPUS function in ModelFactory. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + Returns: + None + """ + ret_val = ModelFactory.list_gpus(api_key) + ret_val_yaml = yaml.dump(ret_val) + click.echo(ret_val_yaml) + @click.command("image-repo") @click.option("--name", help="Model name.") -@click.option("--hosting-machine", - help="Hosting machine code obtained from LIST_HOSTS.") -@click.option("--version", help="Model version.") @click.option("--description", help="Description of model.") @click.option("--function", help="Function name obtained from LIST_FUNCTIONS.") @click.option("--source-language", default="en", help="Model source language in 2-character 639-1 code or 3-character 639-3 code.") +@click.option("--input-modality", help="Input type (text, video, image, etc.)") +@click.option("--output-modality", help="Output type (text, video, image, etc.)") +@click.option("--documentation-url", default="", help="Link to model documentation.") @click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") -def create_asset_repo(name: Text, hosting_machine: Text, version: Text, - description: Text, function: Text, - source_language: Text, - api_key: Optional[Text] = None) -> None: +def create_asset_repo(name: Text, description: Text, function: Text, + source_language: Text, input_modality: Text, + output_modality: Text, + documentation_url: Optional[Text] = "", + api_key: Optional[Text] = None) -> None: """CLI wrapper function for the CREATE_ASSET_REPO function in ModelFactory. Args: @@ -93,9 +109,10 @@ def create_asset_repo(name: Text, hosting_machine: Text, version: Text, Returns: None """ - ret_val = ModelFactory.create_asset_repo(name, hosting_machine, version, - description, function, - source_language, api_key) + ret_val = ModelFactory.create_asset_repo(name, description, function, + source_language, input_modality, + output_modality, documentation_url, + api_key) ret_val_yaml = yaml.dump(ret_val) click.echo(ret_val_yaml) @@ -119,8 +136,10 @@ def asset_repo_login(api_key: Optional[Text] = None) -> None: @click.option("--model-id", help="Model ID from CREATE_IMAGE_REPO.") @click.option("--image-tag", help="The tag of the image that you would like hosted.") @click.option("--image-hash", help="The hash of the image you would like onboarded.") +@click.option("--host-machine", default="", help="The machine on which to host the model.") @click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, + host_machine: Optional[Text] = "", api_key: Optional[Text] = None) -> None: """CLI wrapper function for the ONBOARD_MODEL function in ModelFactory. @@ -132,17 +151,20 @@ def onboard_model(model_id: Text, image_tag: Text, image_hash: Text, Returns: None """ - ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, api_key) + ret_val = ModelFactory.onboard_model(model_id, image_tag, image_hash, + host_machine, api_key) ret_val_yaml = yaml.dump(ret_val) click.echo(ret_val_yaml) @click.command("hf-model") @click.option("--name", help="User-defined name for Hugging Face model.") @click.option("--hf-repo-id", help="Repository ID from Hugging Face in {supplier}/{model name} form.") -@click.option("--hf-token", help="Hugging Face token used to authenticate to this model.") +@click.option("--revision", default="", help="Commit hash of repository.") +@click.option("--hf-token", default=None, help="Hugging Face token used to authenticate to this model.") @click.option("--api-key", default=None, help="TEAM_API_KEY if not already set in environment.") def deploy_huggingface_model(name: Text, hf_repo_id: Text, hf_token: Optional[Text] = None, + revision: Optional[Text] = None, api_key: Optional[Text] = None) -> None: """CLI wrapper function for the DEPLOY_HUGGINGFACE_MODEL function in ModelFactory. @@ -153,7 +175,7 @@ def deploy_huggingface_model(name: Text, hf_repo_id: Text, Returns: None """ - ret_val = ModelFactory.deploy_huggingface_model(name, hf_repo_id, hf_token, api_key) + ret_val = ModelFactory.deploy_huggingface_model(name, hf_repo_id, revision, hf_token, api_key) ret_val_yaml = yaml.dump(ret_val) click.echo(ret_val_yaml) @@ -172,4 +194,4 @@ def get_huggingface_model_status(model_id: Text, api_key: Optional[Text] = None) """ ret_val = ModelFactory.get_huggingface_model_status(model_id, api_key) ret_val_yaml = yaml.dump(ret_val) - click.echo(ret_val_yaml) \ No newline at end of file + click.echo(ret_val_yaml) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 221fd94d..0fb845f1 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -270,6 +270,25 @@ def list_host_machines(cls, api_key: Optional[Text] = None) -> List[Dict]: for dictionary in response_dicts: del dictionary["id"] return response_dicts + + @classmethod + def list_gpus(cls, api_key: Optional[Text] = None) -> List[List[Text]]: + """List GPU names on which you can host your language model. + + Args: + api_key (Text, optional): Team API key. Defaults to None. + + Returns: + List[List[Text]]: List of all available GPUs and their prices. + """ + gpu_url = urljoin(config.BACKEND_URL, "sdk/model-onboarding/gpus") + if api_key: + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + else: + headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + response = _request_with_retry("get", gpu_url, headers=headers) + response_list = json.loads(response.text) + return response_list @classmethod def list_functions(cls, verbose: Optional[bool] = False, api_key: Optional[Text] = None) -> List[Dict]: @@ -310,12 +329,13 @@ def list_functions(cls, verbose: Optional[bool] = False, api_key: Optional[Text] def create_asset_repo( cls, name: Text, - hosting_machine: Text, - version: Text, description: Text, function: Text, source_language: Text, - api_key: Optional[Text] = None, + input_modality: Text, + output_modality: Text, + documentation_url: Optional[Text] = "", + api_key: Optional[Text] = None ) -> Dict: """Creates an image repository for this model and registers it in the platform backend. @@ -342,27 +362,36 @@ def create_asset_repo( function_id = function_dict["id"] if function_id is None: raise Exception("Invalid function name") - create_url = urljoin(config.BACKEND_URL, "sdk/models/register") + create_url = urljoin(config.BACKEND_URL, f"sdk/models/onboard") logging.debug(f"URL: {create_url}") if api_key: headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} - always_on = False - is_async = False # Hard-coded to False for first release + payload = { - "name": name, - "hostingMachine": hosting_machine, - "alwaysOn": always_on, - "version": version, - "description": description, - "function": function_id, - "isAsync": is_async, - "sourceLanguage": source_language, + "model": { + "name": name, + "description": description, + "connectionType": [ + "synchronous" + ], + "function": function_id, + "modalities": [ + f"{input_modality}-{output_modality}" + ], + "documentationUrl": documentation_url, + "sourceLanguage": source_language + }, + "source": "aixplain-ecr", + "onboardingParams": { + } } - payload = json.dumps(payload) logging.debug(f"Body: {str(payload)}") - response = _request_with_retry("post", create_url, headers=headers, data=payload) + response = _request_with_retry("post", create_url, headers=headers, json=payload) + + assert response.status_code == 201 + return response.json() @classmethod @@ -379,20 +408,23 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: login_url = urljoin(config.BACKEND_URL, "sdk/ecr/login") logging.debug(f"URL: {login_url}") if api_key: - headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} else: - headers = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} + headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} response = _request_with_retry("post", login_url, headers=headers) + print(f"Response: {response}") response_dict = json.loads(response.text) return response_dict @classmethod - def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_key: Optional[Text] = None) -> Dict: + def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, host_machine: Optional[Text] = "", api_key: Optional[Text] = None) -> Dict: """Onboard a model after its image has been pushed to ECR. Args: model_id (Text): Model ID obtained from CREATE_ASSET_REPO. image_tag (Text): Image tag to be onboarded. + image_hash (Text): Image digest. + host_machine (Text, optional): Machine on which to host model. api_key (Text, optional): Team API key. Defaults to None. Returns: Dict: Backend response @@ -403,18 +435,18 @@ def onboard_model(cls, model_id: Text, image_tag: Text, image_hash: Text, api_ke headers = {"x-api-key": f"{api_key}", "Content-Type": "application/json"} else: headers = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} - payload = {"image": image_tag, "sha": image_hash} - payload = json.dumps(payload) + payload = {"image": image_tag, "sha": image_hash, "hostMachine": host_machine} logging.debug(f"Body: {str(payload)}") - response = _request_with_retry("post", onboard_url, headers=headers, data=payload) - message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." - logging.info(message) + response = _request_with_retry("post", onboard_url, headers=headers, json=payload) + if response.status_code == 201: + message = "Your onboarding request has been submitted to an aiXplain specialist for finalization. We will notify you when the process is completed." + logging.info(message) + else: + message = "An error has occurred. Please make sure your model_id is valid and your host_machine, if set, is a valid option from the LIST_GPUS function." return response @classmethod - def deploy_huggingface_model( - cls, name: Text, hf_repo_id: Text, hf_token: Optional[Text] = "", api_key: Optional[Text] = None - ) -> Dict: + def deploy_huggingface_model(cls, name: Text, hf_repo_id: Text, revision: Optional[Text] = "", hf_token: Optional[Text] = "", api_key: Optional[Text] = None) -> Dict: """Onboards and deploys a Hugging Face large language model. Args: @@ -441,7 +473,12 @@ def deploy_huggingface_model( "sourceLanguage": "en", }, "source": "huggingface", - "onboardingParams": {"hf_model_name": model_name, "hf_supplier": supplier, "hf_token": hf_token}, + "onboardingParams": { + "hf_supplier": supplier, + "hf_model_name": model_name, + "hf_token": hf_token, + "revision": revision + } } response = _request_with_retry("post", deploy_url, headers=headers, json=body) logging.debug(response.text) diff --git a/aixplain/modules/metric.py b/aixplain/modules/metric.py index d591772b..16bf4541 100644 --- a/aixplain/modules/metric.py +++ b/aixplain/modules/metric.py @@ -97,7 +97,6 @@ def run( reference (Optional[Union[str, List[str]]], optional): Can give a single reference or a list of references for metric calculation. Defaults to None. """ from aixplain.factories.model_factory import ModelFactory - model = ModelFactory.get(self.id) payload = { "function": self.function, diff --git a/aixplain/modules/model/__init__.py b/aixplain/modules/model/__init__.py index 12c96977..06811332 100644 --- a/aixplain/modules/model/__init__.py +++ b/aixplain/modules/model/__init__.py @@ -267,7 +267,6 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): """ from aixplain.enums.asset_status import AssetStatus from aixplain.modules.finetune.status import FinetuneStatus - headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} resp = None try: @@ -278,7 +277,6 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): finetune_status = AssetStatus(resp["finetuneStatus"]) model_status = AssetStatus(resp["modelStatus"]) logs = sorted(resp["logs"], key=lambda x: float(x["epoch"])) - target_epoch = None if after_epoch is not None: logs = [log for log in logs if float(log["epoch"]) > after_epoch] @@ -286,7 +284,6 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): target_epoch = float(logs[0]["epoch"]) elif len(logs) > 0: target_epoch = float(logs[-1]["epoch"]) - if target_epoch is not None: log = None for log_ in logs: @@ -298,7 +295,6 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): log["trainLoss"] = log_["trainLoss"] if log_["evalLoss"] is not None: log["evalLoss"] = log_["evalLoss"] - status = FinetuneStatus( status=finetune_status, model_status=model_status, diff --git a/docs/user/user_doc.md b/docs/user/user_doc.md index a7fa2178..4466e121 100644 --- a/docs/user/user_doc.md +++ b/docs/user/user_doc.md @@ -76,75 +76,66 @@ response = model.run( ``` ### Deploying Hugging Face Large Language Models + You can deploy your very own Hugging Face large language models on our platform using the aiXplain SDK: ```console -$ aixplain onboard hf-model --name --hf-repo-id --hf-token [--api-key ] +$ aixplain onboard hf-model --name --hf-repo-id --revision --hf-token [--api-key ] ``` This command will return your model's ID. The on-boarding process will take 5 to 15 minutes, during which you can check the on-boarding status by running the following: ```console $ aixplain get hf-model-status --model-id [--api-key ] ``` -Once the on-boarding process has completed, you can use this newly-deployed large language model just like any other model on our platform. Note that our platform currently only supports language models up 7 billion parameters in size (~30 GB), so any attempts to deploy larger models will result in an error message. +Once the on-boarding process has completed, you can use this newly-deployed large language model just like any other private model on our platform. Note that our platform currently only supports language models up 7 billion parameters in size (~30 GB), so any attempts to deploy larger models will result in an error message. ### Uploading Models +## Uploading Models In addition to exploring and running models, the aiXplain SDK allows you to upload your own models to the aiXplain platform. This requires a working model image in line with the template specified [here](https://github.com/aixplain/model-interfaces/blob/main/docs/user/model_setup.md). [These](https://github.com/aixplain/model-interfaces/tree/main) are the interfaces with which you will be working. You will also be required to have an aiXplain account as well as a TEAM_API_KEY which should be set either as an environment variable or passed into each of the following functions. -First, choose a hosting machine appropriate for your model. Note down the host machines "code". You can list the available hosting machines' specifications by running the following: -```console -$ aixplain list hosts [--api-key ] -- code: aix-2c-8g-od - cores: 2 - hourlyCost: 0.12 - memory: 8 - type: on-demand -- code: aix-2c-8g - cores: 2 - hourlyCost: 0.096 - memory: 8 - type: always-on - ... -``` Note: For any of the CLI commands, running `aixplain [verb] [resource] --help` will display a description of each argument that should be passed into that command. The `api-key` parameter is optional and is only used if the environment variable isn't set or you would like to override the existing environment variable. Find a supported function type that best describes your model's purpose. Note down the function's ID. ```console -$ aixplain list functions [--verbose ] [--api-key ] -filteredFrom: 55 +aixplain list functions [--verbose] [--api-key ] +filteredFrom: 63 items: -- name: Language Identification -- name: OCR -- name: Image Label Detection -- name: Video Forced Alignment -- name: Offensive Language Identification -- name: Audio Forced Alignment -- name: Video Generation -- name: Split On Silence -- name: Referenceless Audio Generation Metric -- name: Audio Generation Metric -- name: Speaker Diarization Video -- name: Referenceless Text Generation Metric Default +- modalities: + - text-number + name: Object Detection +- modalities: + - text-label + name: Language Identification +- modalities: + - image-text + - document-text + name: OCR +- modalities: + - image-label + name: Image Label Detection +- modalities: + - image-text + name: Image Captioning ... ``` -`verbose` is optional and is set to False by default, meaning only the function names are listed. Setting this to True will additionally list the function ID, output, and params. Again, `api-key` is optional. +`verbose` is optional and is set to False by default. Again, `api-key` is optional. Once you have chosen a suitable host machine and function, register your model and create an image repository: ```console -$ aixplain create image-repo --name --hosting-machine --version --description --function --source-language [--api-key ] +aixplain create image-repo --name --description --function --source-language --input-modality --output-modality --documentation-url [--api-key ] { "repoName": , "modelId": } ``` -`name` is your model's name. `hosting-machine` should include the code of the hosting machine you would like to use. The `version` field should be set to your model's version number. `description` should hold a short summary of your model's purpose. Specify the function name most closely describe your model's purpose in the `function` field. Finally, `source-language` should contain your model's source language. +`name` is your model's name. `description` should hold a short summary of your model's purpose. Specify the function name most closely describe your model's purpose in the `function` field. Finally, `source-language` should contain your model's source language. This returns a model ID and a repository name. Next, obtain login credentials for the newly created repository: ```console -$ aixplain get image-repo-login [--api-key ] +aixplain get image-repo-login [--api-key ] { "username": , "password": , @@ -152,34 +143,45 @@ $ aixplain get image-repo-login [--api-key ] } ``` -These credentials are valid for 12 hours, after which you much again log in for a fresh set of valid credentials. If you are using Docker, you can use these credentials to log in with the following: +These credentials are valid for 12 hours, after which you must again log in for a fresh set of valid credentials. If you are using Docker, you can use these credentials to log in with the following: ```console docker login --username $USERNAME --password $PASSWORD 535945872701.dkr.ecr.us-east-1.amazonaws.com ``` You must first build your image using the following: ```console -$ docker build . -t 535945872701.dkr.ecr.us-east-1.amazonaws.com/: +docker build . -t $REGISTRY/$REPO_NAME: ``` -where the `` is that returned by `aixplain create image-repo` and `` is some sort of descriptor (usually version number) for your specific model. +where `` is some sort of descriptor (usually a version tag like v0.0.1) for your specific model. -Next, tag your image to match the registry and repository name given in the previous steps. If you are using Docker, this would look like the following: +Push the newly tagged image to the corresponding repository: ```console -$ docker tag {$REGISTRY}/{$REPO_NAME}: +$ docker push $REGISTRY/$REPO_NAME: ``` -Push the newly tagged image to the corresponding repository: + +Once this is done, onboard the model: ```console -$ docker push {$REGISTRY}/{$REPO_NAME}: +$ aixplain onboard model --model-id --image-tag --image-hash --host-machine [--api-key ] ``` +`model-id` should be the model ID returned by the image-create-repo function used earlier. `image-tag` should be set to whatever string you used to tag your model image. The image sha256 hash can be obtained by running `docker images --digests`. Choose the hash corresponding to the image you would like onboarded. `host-machine` should contain the machine code on which to host the model. A list of all the models can be obtained via `aixplain list gpus` as follow: -Once this is done, onboard the model: +Note down the host machines "code": ```console -$ aixplain onboard model --model-id --image-tag --image-hash [--api-key ] +aixplain list gpus [--api-key ] +- - nvidia-t4-1 + - 'Price: 0.752' + - 'Units: $/hr' +- - nvidia-a10g-1 + - 'Price: 1.006' + - 'Units: $/hr' +- - nvidia-a10g-4 + - 'Price: 5.672' + - 'Units: $/hr' + ... ``` -`model-id` should be the model ID returned by the image-create-repo function used earlier. `image-tag` should be set to whatever string you used to tag your model image. The image sha256 hash can be obtained by running `docker images --digests`. Choose the hash corresponding to the image you would like onboarded. -This will send an email to an aiXplain associate to finalize the onboarding process. +This will send an email to an aiXplain associate to finalize the onboarding process. ## Pipelines [Design](https://aixplain.com/platform/studio/) is aiXplain’s no-code AI pipeline builder tool that accelerates AI development by providing a seamless experience to build complex AI systems and deploy them within minutes. You can visit our platform and design your own custom pipeline [here](https://platform.aixplain.com/studio). diff --git a/tests/functional/finetune/data/finetune_test_cost_estimation.json b/tests/functional/finetune/data/finetune_test_cost_estimation.json index 80f4d331..44707255 100644 --- a/tests/functional/finetune/data/finetune_test_cost_estimation.json +++ b/tests/functional/finetune/data/finetune_test_cost_estimation.json @@ -9,4 +9,4 @@ {"model_name": "MPT 7b storywriter", "model_id": "6551a870bf42e6037ab109db", "dataset_name": "Test text generation dataset"}, {"model_name": "BloomZ 7b", "model_id": "6551ab17bf42e6037ab109e0", "dataset_name": "Test text generation dataset"}, {"model_name": "BloomZ 7b MT", "model_id": "656e80147ca71e334752d5a3", "dataset_name": "Test text generation dataset"} -] \ No newline at end of file +] diff --git a/tests/functional/finetune/data/finetune_test_end2end.json b/tests/functional/finetune/data/finetune_test_end2end.json index 80768de9..f744f0e6 100644 --- a/tests/functional/finetune/data/finetune_test_end2end.json +++ b/tests/functional/finetune/data/finetune_test_end2end.json @@ -10,7 +10,7 @@ { "model_name": "aiR", "model_id": "6499cc946eb5633de15d82a1", - "dataset_name": "Test search dataset", + "dataset_name": "Test search dataset metadata", "inference_data": "Hello!", "required_dev": false, "search_metadata": false @@ -23,4 +23,4 @@ "required_dev": false, "search_metadata": false } -] \ No newline at end of file +] diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index ffa9ad5a..7b45613c 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -130,4 +130,4 @@ def test_prompt_validator(validate_prompt_input_map): finetune = FinetuneFactory.create( str(uuid.uuid4()), dataset_list, model, prompt_template=validate_prompt_input_map["prompt_template"] ) - assert exc_info.type is AssertionError \ No newline at end of file + assert exc_info.type is AssertionError From 16eb2e116f190bc8380312487327007570a0f4d1 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Fri, 7 Jun 2024 13:03:06 -0300 Subject: [PATCH 08/25] Create and get Pipelines with api key as input parameter (#187) Co-authored-by: Thiago Castro Ferreira --- aixplain/factories/pipeline_factory.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/aixplain/factories/pipeline_factory.py b/aixplain/factories/pipeline_factory.py index 404a5556..4ebdc439 100644 --- a/aixplain/factories/pipeline_factory.py +++ b/aixplain/factories/pipeline_factory.py @@ -73,7 +73,9 @@ def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: resp = None try: url = urljoin(cls.backend_url, f"sdk/pipelines/{pipeline_id}") - if cls.aixplain_key != "": + if api_key is not None: + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + elif cls.aixplain_key != "": headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} else: headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} @@ -86,7 +88,7 @@ def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: resp["api_key"] = api_key pipeline = cls.__from_response(resp) return pipeline - except Exception as e: + except Exception: status_code = 400 if resp is not None and "statusCode" in resp: status_code = resp["statusCode"] @@ -172,7 +174,7 @@ def list( else: headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} - assert 0 < page_size <= 100, f"Pipeline List Error: Page size must be greater than 0 and not exceed 100." + assert 0 < page_size <= 100, "Pipeline List Error: Page size must be greater than 0 and not exceed 100." payload = { "pageSize": page_size, "pageNumber": page_number, @@ -223,13 +225,16 @@ def list( return {"results": pipelines, "page_total": page_total, "page_number": page_number, "total": total} @classmethod - def create(cls, name: Text, pipeline: Union[Text, Dict], status: Text = "draft") -> Pipeline: + def create( + cls, name: Text, pipeline: Union[Text, Dict], status: Text = "draft", api_key: Optional[Text] = None + ) -> Pipeline: """Pipeline Creation Args: name (Text): Pipeline Name pipeline (Union[Text, Dict]): Pipeline as a Python dictionary or in a JSON file status (Text, optional): Status of the pipeline. Currently only draft pipelines can be saved. Defaults to "draft". + api_key (Optional[Text], optional): _description_. Defaults to None. Raises: Exception: Currently just the creation of draft pipelines are supported @@ -250,11 +255,12 @@ def create(cls, name: Text, pipeline: Union[Text, Dict], status: Text = "draft") # prepare payload payload = {"name": name, "status": "draft", "architecture": pipeline} url = urljoin(cls.backend_url, "sdk/pipelines") - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + api_key = api_key if api_key is not None else config.TEAM_API_KEY + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} logging.info(f"Start service for POST Create Pipeline - {url} - {headers} - {json.dumps(payload)}") r = _request_with_retry("post", url, headers=headers, json=payload) response = r.json() - return Pipeline(response["id"], name, config.TEAM_API_KEY) + return Pipeline(response["id"], name, api_key) except Exception as e: raise Exception(e) From 04246b152898640adaa975bef7969a8eab5853d6 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:15:50 -0300 Subject: [PATCH 09/25] M 6769474660 save pipelines (#191) * Saving pipelines as asset * Pipeline delete service * Function type AI to lower case --------- Co-authored-by: Thiago Castro Ferreira --- aixplain/factories/pipeline_factory.py | 15 ++++----- aixplain/modules/pipeline.py | 39 +++++++++++++++++------ tests/functional/pipelines/create_test.py | 7 ++-- tests/unit/pipeline_test.py | 3 +- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/aixplain/factories/pipeline_factory.py b/aixplain/factories/pipeline_factory.py index 4ebdc439..fac42283 100644 --- a/aixplain/factories/pipeline_factory.py +++ b/aixplain/factories/pipeline_factory.py @@ -225,16 +225,13 @@ def list( return {"results": pipelines, "page_total": page_total, "page_number": page_number, "total": total} @classmethod - def create( - cls, name: Text, pipeline: Union[Text, Dict], status: Text = "draft", api_key: Optional[Text] = None - ) -> Pipeline: - """Pipeline Creation + def create(cls, name: Text, pipeline: Union[Text, Dict], api_key: Optional[Text] = None) -> Pipeline: + """Draft Pipeline Creation Args: name (Text): Pipeline Name pipeline (Union[Text, Dict]): Pipeline as a Python dictionary or in a JSON file - status (Text, optional): Status of the pipeline. Currently only draft pipelines can be saved. Defaults to "draft". - api_key (Optional[Text], optional): _description_. Defaults to None. + api_key (Optional[Text], optional): Team API Key to create the Pipeline. Defaults to None. Raises: Exception: Currently just the creation of draft pipelines are supported @@ -243,15 +240,17 @@ def create( Pipeline: instance of the new pipeline """ try: - assert status == "draft", "Pipeline Creation Error: Currently just the creation of draft pipelines are supported." if isinstance(pipeline, str) is True: _, ext = os.path.splitext(pipeline) assert ( os.path.exists(pipeline) and ext == ".json" - ), "Pipeline Creation Error: Make sure the pipeline to be save is in a JSON file." + ), "Pipeline Creation Error: Make sure the pipeline to be saved is in a JSON file." with open(pipeline) as f: pipeline = json.load(f) + for i, node in enumerate(pipeline["nodes"]): + if "functionType" in node and node["functionType"] == "AI": + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() # prepare payload payload = {"name": name, "status": "draft", "architecture": pipeline} url = urljoin(cls.backend_url, "sdk/pipelines") diff --git a/aixplain/modules/pipeline.py b/aixplain/modules/pipeline.py index 3de49756..b079e2a3 100644 --- a/aixplain/modules/pipeline.py +++ b/aixplain/modules/pipeline.py @@ -101,12 +101,12 @@ def __polling( time.sleep(wait_time) if wait_time < 60: wait_time *= 1.1 - except Exception as e: + except Exception: logging.error(f"Polling for Pipeline: polling for {name} : Continue") if response_body and response_body["status"] == "SUCCESS": try: logging.debug(f"Polling for Pipeline: Final status of polling for {name} : SUCCESS - {response_body}") - except Exception as e: + except Exception: logging.error(f"Polling for Pipeline: Final status of polling for {name} : ERROR - {response_body}") else: logging.error( @@ -130,7 +130,7 @@ def poll(self, poll_url: Text, name: Text = "pipeline_process") -> Dict: try: resp = r.json() logging.info(f"Single Poll for Pipeline: Status of polling for {name} : {resp}") - except Exception as e: + except Exception: resp = {"status": "FAILED"} return resp @@ -206,7 +206,7 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ if isinstance(payload, int) is True or isinstance(payload, float) is True: payload = str(payload) payload = {"data": payload} - except Exception as e: + except Exception: payload = {"data": data} else: payload = {} @@ -251,7 +251,7 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ if target_row.id == data[node_label]: data_found = True break - if data_found == True: + if data_found is True: break except Exception: data_asset_found = False @@ -303,17 +303,18 @@ def run_async( poll_url = resp["url"] response = {"status": "IN_PROGRESS", "url": poll_url} - except Exception as e: + except Exception: response = {"status": "FAILED"} if resp is not None: response["error"] = resp return response - def update(self, pipeline: Union[Text, Dict]): + def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False): """Update Pipeline Args: pipeline (Union[Text, Dict]): Pipeline as a Python dictionary or in a JSON file + save_as_asset (bool, optional): Save as asset (True) or draft (False). Defaults to False. Raises: Exception: Make sure the pipeline to be save is in a JSON file. @@ -323,12 +324,18 @@ def update(self, pipeline: Union[Text, Dict]): _, ext = os.path.splitext(pipeline) assert ( os.path.exists(pipeline) and ext == ".json" - ), "Pipeline Update Error: Make sure the pipeline to be save is in a JSON file." + ), "Pipeline Update Error: Make sure the pipeline to be saved is in a JSON file." with open(pipeline) as f: pipeline = json.load(f) + for i, node in enumerate(pipeline["nodes"]): + if "functionType" in node and node["functionType"] == "AI": + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() # prepare payload - payload = {"name": self.name, "status": "draft", "architecture": pipeline} + status = "draft" + if save_as_asset is True: + status = "onboarded" + payload = {"name": self.name, "status": status, "architecture": pipeline} url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} logging.info(f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}") @@ -337,3 +344,17 @@ def update(self, pipeline: Union[Text, Dict]): logging.info(f"Pipeline {response['id']} Updated.") except Exception as e: raise Exception(e) + + def delete(self) -> None: + """Delete Dataset service""" + try: + url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") + headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + logging.info(f"Start service for DELETE Pipeline - {url} - {headers}") + r = _request_with_retry("delete", url, headers=headers) + if r.status_code != 200: + raise Exception() + except Exception: + message = "Pipeline Deletion Error: Make sure the pipeline exists and you are the owner." + logging.error(message) + raise Exception(f"{message}") diff --git a/tests/functional/pipelines/create_test.py b/tests/functional/pipelines/create_test.py index f2c1a9c9..6431bd41 100644 --- a/tests/functional/pipelines/create_test.py +++ b/tests/functional/pipelines/create_test.py @@ -30,6 +30,7 @@ def test_create_pipeline_from_json(): assert isinstance(pipeline, Pipeline) assert pipeline.id != "" + pipeline.delete() def test_create_pipeline_from_string(): @@ -42,6 +43,7 @@ def test_create_pipeline_from_string(): assert isinstance(pipeline, Pipeline) assert pipeline.id != "" + pipeline.delete() def test_update_pipeline(): @@ -52,13 +54,14 @@ def test_update_pipeline(): pipeline_name = str(uuid4()) pipeline = PipelineFactory.create(name=pipeline_name, pipeline=pipeline_dict) - pipeline.update(pipeline=pipeline_json) + pipeline.update(pipeline=pipeline_json, save_as_asset=True) assert isinstance(pipeline, Pipeline) assert pipeline.id != "" + pipeline.delete() def test_create_pipeline_wrong_path(): pipeline_name = str(uuid4()) with pytest.raises(Exception): - pipeline = PipelineFactory.create(name=pipeline_name, pipeline="/") + PipelineFactory.create(name=pipeline_name, pipeline="/") diff --git a/tests/unit/pipeline_test.py b/tests/unit/pipeline_test.py index 68a399aa..e983a298 100644 --- a/tests/unit/pipeline_test.py +++ b/tests/unit/pipeline_test.py @@ -24,7 +24,6 @@ from aixplain.factories import PipelineFactory from aixplain.modules import Pipeline from urllib.parse import urljoin -import pytest def test_create_pipeline(): @@ -34,6 +33,6 @@ def test_create_pipeline(): ref_response = {"id": "12345"} mock.post(url, headers=headers, json=ref_response) ref_pipeline = Pipeline(id="12345", name="Pipeline Test", api_key=config.TEAM_API_KEY) - hyp_pipeline = PipelineFactory.create(pipeline={}, name="Pipeline Test") + hyp_pipeline = PipelineFactory.create(pipeline={"nodes": []}, name="Pipeline Test") assert hyp_pipeline.id == ref_pipeline.id assert hyp_pipeline.name == ref_pipeline.name From 73021a79c4670d1cfdd1436199af5748285fc6a1 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 18 Jun 2024 11:40:44 -0300 Subject: [PATCH 10/25] M 6769474660 save pipelines (#192) * Saving pipelines as asset * Pipeline delete service * Function type AI to lower case * API Key as a parameter --------- Co-authored-by: Thiago Castro Ferreira --- aixplain/modules/pipeline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/aixplain/modules/pipeline.py b/aixplain/modules/pipeline.py index b079e2a3..ed131018 100644 --- a/aixplain/modules/pipeline.py +++ b/aixplain/modules/pipeline.py @@ -309,12 +309,13 @@ def run_async( response["error"] = resp return response - def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False): + def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False, api_key: Optional[Text] = None): """Update Pipeline Args: pipeline (Union[Text, Dict]): Pipeline as a Python dictionary or in a JSON file save_as_asset (bool, optional): Save as asset (True) or draft (False). Defaults to False. + api_key (Optional[Text], optional): Team API Key to create the Pipeline. Defaults to None. Raises: Exception: Make sure the pipeline to be save is in a JSON file. @@ -337,7 +338,8 @@ def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False): status = "onboarded" payload = {"name": self.name, "status": status, "architecture": pipeline} url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + api_key = api_key if api_key is not None else config.TEAM_API_KEY + headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} logging.info(f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}") r = _request_with_retry("put", url, headers=headers, json=payload) response = r.json() From 474602b7b94126b442c83441e2545b7925c79702 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Wed, 26 Jun 2024 08:50:04 -0300 Subject: [PATCH 11/25] Solving bug when LLM parameters are set on data (#196) Co-authored-by: Thiago Castro Ferreira --- aixplain/modules/model/llm_model.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/aixplain/modules/model/llm_model.py b/aixplain/modules/model/llm_model.py index 349ea595..14b9c7f4 100644 --- a/aixplain/modules/model/llm_model.py +++ b/aixplain/modules/model/llm_model.py @@ -196,12 +196,12 @@ def run_async( payload = {"data": data} parameters.update( { - "context": context, - "prompt": prompt, - "history": history, - "temperature": temperature, - "max_tokens": max_tokens, - "top_p": top_p, + "context": payload["context"] if "context" in payload else context, + "prompt": payload["prompt"] if "prompt" in payload else prompt, + "history": payload["history"] if "history" in payload else history, + "temperature": payload["temperature"] if "temperature" in payload else temperature, + "max_tokens": payload["max_tokens"] if "max_tokens" in payload else max_tokens, + "top_p": payload["top_p"] if "top_p" in payload else top_p, } ) payload.update(parameters) From 3695686a5e8c34ebdc10301acad1a46f957905d6 Mon Sep 17 00:00:00 2001 From: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Date: Wed, 3 Jul 2024 05:52:17 -0300 Subject: [PATCH 12/25] Fix pipeline functional test (#200) --- tests/functional/pipelines/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/pipelines/run_test.py b/tests/functional/pipelines/run_test.py index e8bc4d9c..25fadaf4 100644 --- a/tests/functional/pipelines/run_test.py +++ b/tests/functional/pipelines/run_test.py @@ -224,7 +224,7 @@ def test_run_router(input_data: str, output_data: str, version: str): @pytest.mark.parametrize( - "input_data,output_data", + "input_data,output_data,version", [ ("I love it.", "PositiveOutput", "2.0"), ("I hate it.", "NegativeOutput", "2.0"), From 90140617466393d7a7baf75e350ab4f098bbfe47 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Sat, 13 Jul 2024 23:21:52 +0200 Subject: [PATCH 13/25] M 6656407247 agentification (#197) * Agent CRUD * Fixes in the structure * Delete agent method * Add input/output to PipelineFactory and use api_key from parameter (#182) * Enabling pipeline tools * M 6875703542 agentification deployment (#195) * First changes for agent integration with backend * Official creation and deletion services * Running agent method --------- Co-authored-by: Thiago Castro Ferreira * Fix bug when supplier and tools are not given * Add agents functional tests (#204) --------- Co-authored-by: Thiago Castro Ferreira Co-authored-by: Lucas Pavanelli <86805709+lucas-aixplain@users.noreply.github.com> Co-authored-by: Thiago Castro Ferreira Co-authored-by: Lucas Pavanelli --- aixplain/factories/__init__.py | 1 + aixplain/factories/agent_factory/__init__.py | 166 +++++++++++++++ aixplain/factories/agent_factory/utils.py | 48 +++++ aixplain/factories/pipeline_factory.py | 9 +- aixplain/modules/__init__.py | 2 + aixplain/modules/agent/__init__.py | 195 ++++++++++++++++++ aixplain/modules/agent/tool.py | 59 ++++++ aixplain/modules/agent/tool/__init__.py | 53 +++++ aixplain/modules/agent/tool/model_tool.py | 60 ++++++ aixplain/modules/agent/tool/pipeline_tool.py | 52 +++++ aixplain/modules/asset.py | 8 +- aixplain/modules/finetune/__init__.py | 5 +- aixplain/modules/metric.py | 3 - aixplain/modules/model/__init__.py | 17 +- pyproject.toml | 2 +- .../functional/agent/agent_functional_test.py | 75 +++++++ .../agent/data/agent_test_end2end.json | 14 ++ 17 files changed, 752 insertions(+), 17 deletions(-) create mode 100644 aixplain/factories/agent_factory/__init__.py create mode 100644 aixplain/factories/agent_factory/utils.py create mode 100644 aixplain/modules/agent/__init__.py create mode 100644 aixplain/modules/agent/tool.py create mode 100644 aixplain/modules/agent/tool/__init__.py create mode 100644 aixplain/modules/agent/tool/model_tool.py create mode 100644 aixplain/modules/agent/tool/pipeline_tool.py create mode 100644 tests/functional/agent/agent_functional_test.py create mode 100644 tests/functional/agent/data/agent_test_end2end.json diff --git a/aixplain/factories/__init__.py b/aixplain/factories/__init__.py index 36147c6e..7b876899 100644 --- a/aixplain/factories/__init__.py +++ b/aixplain/factories/__init__.py @@ -20,6 +20,7 @@ limitations under the License. """ from .asset_factory import AssetFactory +from .agent_factory import AgentFactory from .benchmark_factory import BenchmarkFactory from .corpus_factory import CorpusFactory from .data_factory import DataFactory diff --git a/aixplain/factories/agent_factory/__init__.py b/aixplain/factories/agent_factory/__init__.py new file mode 100644 index 00000000..36380a76 --- /dev/null +++ b/aixplain/factories/agent_factory/__init__.py @@ -0,0 +1,166 @@ +__author__ = "lucaspavanelli" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Thiago Castro Ferreira and Lucas Pavanelli +Date: May 16th 2024 +Description: + Agent Factory Class +""" + +import json +import logging + +from aixplain.enums.supplier import Supplier +from aixplain.modules.agent import Agent, Tool +from aixplain.modules.agent.tool.model_tool import ModelTool +from aixplain.modules.agent.tool.pipeline_tool import PipelineTool +from aixplain.utils import config +from typing import Dict, List, Optional, Text, Union + +from aixplain.factories.agent_factory.utils import build_agent +from aixplain.utils.file_utils import _request_with_retry +from urllib.parse import urljoin + + +class AgentFactory: + @classmethod + def create( + cls, + name: Text, + llm_id: Text, + tools: List[Tool] = [], + description: Text = "", + api_key: Text = config.TEAM_API_KEY, + supplier: Union[Dict, Text, Supplier, int] = "aiXplain", + version: Optional[Text] = None, + ) -> Agent: + """Create a new agent in the platform.""" + try: + agent = None + url = urljoin(config.BACKEND_URL, "sdk/agents") + headers = {"x-api-key": api_key} + + if isinstance(supplier, dict): + supplier = supplier["code"] + elif isinstance(supplier, Supplier): + supplier = supplier.value["code"] + + tool_payload = [] + for tool in tools: + if isinstance(tool, ModelTool): + tool_payload.append( + { + "function": tool.function.value, + "type": "model", + "description": tool.description, + "supplier": tool.supplier.value["code"] if tool.supplier else None, + "version": tool.version if tool.version else None, + } + ) + elif isinstance(tool, PipelineTool): + tool_payload.append( + { + "assetId": tool.pipeline, + "description": tool.description, + "type": "pipeline", + } + ) + else: + raise Exception("Agent Creation Error: Tool type not supported.") + + payload = { + "name": name, + "assets": tool_payload, + "description": description, + "supplier": supplier, + "version": version, + } + if llm_id is not None: + payload["llmId"] = llm_id + + logging.info(f"Start service for POST Create Agent - {url} - {headers} - {json.dumps(payload)}") + r = _request_with_retry("post", url, headers=headers, json=payload) + if 200 <= r.status_code < 300: + response = r.json() + agent = build_agent(payload=response, api_key=api_key) + else: + error = r.json() + error_msg = "Agent Onboarding Error: Please contant the administrators." + if "message" in error: + msg = error["message"] + if error["message"] == "err.name_already_exists": + msg = "Agent name already exists." + elif error["message"] == "err.asset_is_not_available": + msg = "Some the tools are not available." + error_msg = f"Agent Onboarding Error (HTTP {r.status_code}): {msg}" + logging.exception(error_msg) + raise Exception(error_msg) + except Exception as e: + raise Exception(e) + return agent + + @classmethod + def list(cls) -> Dict: + """List all agents available in the platform.""" + url = urljoin(config.BACKEND_URL, "sdk/agents") + headers = {"x-api-key": config.TEAM_API_KEY, "Content-Type": "application/json"} + + payload = {} + logging.info(f"Start service for GET List Agents - {url} - {headers} - {json.dumps(payload)}") + try: + r = _request_with_retry("get", url, headers=headers) + resp = r.json() + + if 200 <= r.status_code < 300: + agents, page_total, total = [], 0, 0 + results = resp + page_total = len(results) + total = len(results) + logging.info(f"Response for GET List Agents - Page Total: {page_total} / Total: {total}") + for agent in results: + agents.append(build_agent(agent)) + return {"results": agents, "page_total": page_total, "page_number": 0, "total": total} + else: + error_msg = "Agent Listing Error: Please contant the administrators." + if "message" in resp: + msg = resp["message"] + error_msg = f"Agent Listing Error (HTTP {r.status_code}): {msg}" + logging.exception(error_msg) + raise Exception(error_msg) + except Exception as e: + raise Exception(e) + + @classmethod + def get(cls, agent_id: Text, api_key: Optional[Text] = None) -> Agent: + """Get agent by id.""" + url = urljoin(config.BACKEND_URL, f"sdk/agents/{agent_id}") + if config.AIXPLAIN_API_KEY != "": + headers = {"x-aixplain-key": f"{config.AIXPLAIN_API_KEY}", "Content-Type": "application/json"} + else: + api_key = api_key if api_key is not None else config.TEAM_API_KEY + headers = {"x-api-key": api_key, "Content-Type": "application/json"} + logging.info(f"Start service for GET Agent - {url} - {headers}") + r = _request_with_retry("get", url, headers=headers) + resp = r.json() + if 200 <= r.status_code < 300: + return build_agent(resp) + else: + msg = "Please contant the administrators." + if "message" in resp: + msg = resp["message"] + error_msg = f"Agent Get Error (HTTP {r.status_code}): {msg}" + raise Exception(error_msg) diff --git a/aixplain/factories/agent_factory/utils.py b/aixplain/factories/agent_factory/utils.py new file mode 100644 index 00000000..6363a08e --- /dev/null +++ b/aixplain/factories/agent_factory/utils.py @@ -0,0 +1,48 @@ +__author__ = "thiagocastroferreira" + +import aixplain.utils.config as config +from aixplain.enums import Function, Supplier +from aixplain.enums.asset_status import AssetStatus +from aixplain.modules.agent import Agent, ModelTool, PipelineTool +from typing import Dict, Text +from urllib.parse import urljoin + + +def build_agent(payload: Dict, api_key: Text = config.TEAM_API_KEY) -> Agent: + """Instantiate a new agent in the platform.""" + tools = payload["assets"] + for i, tool in enumerate(tools): + if tool["type"] == "model": + for supplier in Supplier: + if tool["supplier"] is not None and tool["supplier"].lower() in [ + supplier.value["code"].lower(), + supplier.value["name"].lower(), + ]: + tool["supplier"] = supplier + break + + tool = ModelTool( + function=Function(tool["function"]), + supplier=tool["supplier"], + version=tool["version"], + ) + elif tool["type"] == "pipeline": + tool = PipelineTool(description=tool["description"], pipeline=tool["assetId"]) + else: + raise Exception("Agent Creation Error: Tool type not supported.") + tools[i] = tool + + agent = Agent( + id=payload["id"], + name=payload["name"] if "name" in payload else "", + tools=tools, + description=payload["description"] if "description" in payload else "", + supplier=payload["teamId"] if "teamId" in payload else None, + version=payload["version"] if "version" in payload else None, + cost=payload["cost"] if "cost" in payload else None, + llm_id=payload["llmId"] if "llmId" in payload else "6646261c6eb563165658bbb1", + api_key=api_key, + status=AssetStatus(payload["status"]), + ) + agent.url = urljoin(config.BACKEND_URL, f"sdk/agents/{agent.id}/run") + return agent diff --git a/aixplain/factories/pipeline_factory.py b/aixplain/factories/pipeline_factory.py index fac42283..61bcb214 100644 --- a/aixplain/factories/pipeline_factory.py +++ b/aixplain/factories/pipeline_factory.py @@ -45,6 +45,11 @@ class PipelineFactory: aixplain_key = config.AIXPLAIN_API_KEY backend_url = config.BACKEND_URL + @classmethod + def __get_typed_nodes(cls, response: Dict, type: str) -> List[Dict]: + # read "nodes" field from response and return the nodes that are marked by "type": type + return [node for node in response["nodes"] if node["type"].lower() == type.lower()] + @classmethod def __from_response(cls, response: Dict) -> Pipeline: """Converts response Json to 'Pipeline' object @@ -57,7 +62,9 @@ def __from_response(cls, response: Dict) -> Pipeline: """ if "api_key" not in response: response["api_key"] = config.TEAM_API_KEY - return Pipeline(response["id"], response["name"], response["api_key"]) + input = cls.__get_typed_nodes(response, "input") + output = cls.__get_typed_nodes(response, "output") + return Pipeline(response["id"], response["name"], response["api_key"], input=input, output=output) @classmethod def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: diff --git a/aixplain/modules/__init__.py b/aixplain/modules/__init__.py index 488c8c2f..c7246dac 100644 --- a/aixplain/modules/__init__.py +++ b/aixplain/modules/__init__.py @@ -33,3 +33,5 @@ from .finetune.status import FinetuneStatus from .benchmark import Benchmark from .benchmark_job import BenchmarkJob +from .agent import Agent +from .agent.tool import Tool diff --git a/aixplain/modules/agent/__init__.py b/aixplain/modules/agent/__init__.py new file mode 100644 index 00000000..2f244d56 --- /dev/null +++ b/aixplain/modules/agent/__init__.py @@ -0,0 +1,195 @@ +__author__ = "aiXplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Lucas Pavanelli and Thiago Castro Ferreira +Date: May 16th 2024 +Description: + Agentification Class +""" +import json +import logging +import time +import traceback + +from aixplain.utils.file_utils import _request_with_retry +from aixplain.enums.supplier import Supplier +from aixplain.enums.asset_status import AssetStatus +from aixplain.modules.model import Model +from aixplain.modules.agent.tool import Tool +from aixplain.modules.agent.tool.model_tool import ModelTool +from aixplain.modules.agent.tool.pipeline_tool import PipelineTool +from typing import Dict, List, Text, Optional, Union +from urllib.parse import urljoin + +from aixplain.utils import config + + +class Agent(Model): + """Advanced AI system capable of performing tasks by leveraging specialized software tools and resources from aiXplain marketplace. + + Attributes: + id (Text): ID of the Agent + name (Text): Name of the Agent + tools (List[Tool]): List of tools that the Agent uses. + description (Text, optional): description of the Agent. Defaults to "". + llm_id (Text, optional): large language model. Defaults to GPT-4o (6646261c6eb563165658bbb1). + supplier (Text): Supplier of the Agent. + version (Text): Version of the Agent. + backend_url (str): URL of the backend. + api_key (str): The TEAM API key used for authentication. + cost (Dict, optional): model price. Defaults to None. + """ + + def __init__( + self, + id: Text, + name: Text, + tools: List[Tool] = [], + description: Text = "", + llm_id: Text = "6646261c6eb563165658bbb1", + api_key: Optional[Text] = config.TEAM_API_KEY, + supplier: Union[Dict, Text, Supplier, int] = "aiXplain", + version: Optional[Text] = None, + cost: Optional[Dict] = None, + status: AssetStatus = AssetStatus.ONBOARDING, + **additional_info, + ) -> None: + """Create a FineTune with the necessary information. + + Args: + id (Text): ID of the Agent + name (Text): Name of the Agent + tools (List[Tool]): List of tools that the Agent uses. + description (Text, optional): description of the Agent. Defaults to "". + llm_id (Text, optional): large language model. Defaults to GPT-4o (6646261c6eb563165658bbb1). + supplier (Text): Supplier of the Agent. + version (Text): Version of the Agent. + backend_url (str): URL of the backend. + api_key (str): The TEAM API key used for authentication. + cost (Dict, optional): model price. Defaults to None. + """ + # assert len(tools) > 0, "At least one tool must be provided." + super().__init__(id, name, description, api_key, supplier, version, cost=cost) + self.additional_info = additional_info + self.tools = tools + self.llm_id = llm_id + if isinstance(status, str): + try: + status = AssetStatus(status) + except Exception: + status = AssetStatus.ONBOARDING + self.status = status + + def run( + self, + query: Text, + session_id: Optional[Text] = None, + history: Optional[List[Dict]] = None, + name: Text = "model_process", + timeout: float = 300, + parameters: Dict = {}, + wait_time: float = 0.5, + ) -> Dict: + """Runs an agent call. + + Args: + query (Text): query to be processed by the agent. + session_id (Optional[Text], optional): conversation Session ID. Defaults to None. + history (Optional[List[Dict]], optional): chat history (in case session ID is None). Defaults to None. + name (Text, optional): ID given to a call. Defaults to "model_process". + timeout (float, optional): total polling time. Defaults to 300. + parameters (Dict, optional): optional parameters to the model. Defaults to "{}". + wait_time (float, optional): wait time in seconds between polling calls. Defaults to 0.5. + + Returns: + Dict: parsed output from model + """ + start = time.time() + try: + response = self.run_async(query=query, session_id=session_id, history=history, name=name, parameters=parameters) + if response["status"] == "FAILED": + end = time.time() + response["elapsed_time"] = end - start + return response + poll_url = response["url"] + end = time.time() + response = self.sync_poll(poll_url, name=name, timeout=timeout, wait_time=wait_time) + return response + except Exception as e: + msg = f"Error in request for {name} - {traceback.format_exc()}" + logging.error(f"Model Run: Error in running for {name}: {e}") + end = time.time() + return {"status": "FAILED", "error": msg, "elapsed_time": end - start} + + def run_async( + self, + query: Text, + session_id: Optional[Text] = None, + history: Optional[List[Dict]] = None, + name: Text = "model_process", + parameters: Dict = {}, + ) -> Dict: + """Runs asynchronously an agent call. + + Args: + query (Text): query to be processed by the agent. + session_id (Optional[Text], optional): conversation Session ID. Defaults to None. + history (Optional[List[Dict]], optional): chat history (in case session ID is None). Defaults to None. + name (Text, optional): ID given to a call. Defaults to "model_process". + parameters (Dict, optional): optional parameters to the model. Defaults to "{}". + + Returns: + dict: polling URL in response + """ + headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + from aixplain.factories.file_factory import FileFactory + + payload = {"id": self.id, "query": FileFactory.to_link(query), "sessionId": session_id, "history": history} + payload.update(parameters) + payload = json.dumps(payload) + + r = _request_with_retry("post", self.url, headers=headers, data=payload) + logging.info(f"Model Run Async: Start service for {name} - {self.url} - {payload} - {headers}") + + resp = None + try: + resp = r.json() + logging.info(f"Result of request for {name} - {r.status_code} - {resp}") + + poll_url = resp["data"] + response = {"status": "IN_PROGRESS", "url": poll_url} + except Exception: + response = {"status": "FAILED"} + msg = f"Error in request for {name} - {traceback.format_exc()}" + logging.error(f"Model Run Async: Error in running for {name}: {resp}") + if resp is not None: + response["error"] = msg + return response + + def delete(self) -> None: + """Delete Corpus service""" + try: + url = urljoin(config.BACKEND_URL, f"sdk/agents/{self.id}") + headers = {"x-api-key": config.TEAM_API_KEY, "Content-Type": "application/json"} + logging.debug(f"Start service for DELETE Agent - {url} - {headers}") + r = _request_with_retry("delete", url, headers=headers) + if r.status_code != 200: + raise Exception() + except Exception: + message = f"Agent Deletion Error (HTTP {r.status_code}): Make sure the agent exists and you are the owner." + logging.error(message) + raise Exception(f"{message}") diff --git a/aixplain/modules/agent/tool.py b/aixplain/modules/agent/tool.py new file mode 100644 index 00000000..6651afe7 --- /dev/null +++ b/aixplain/modules/agent/tool.py @@ -0,0 +1,59 @@ +__author__ = "aiXplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Lucas Pavanelli and Thiago Castro Ferreira +Date: May 16th 2024 +Description: + Agentification Class +""" +from typing import Text, Optional + +from aixplain.enums.function import Function +from aixplain.enums.supplier import Supplier + + +class Tool: + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Attributes: + name (Text): name of the tool + description (Text): descriptiion of the tool + function (Function): task that the tool performs + supplier (Optional[Union[Dict, Text, Supplier, int]], optional): Preferred supplier to perform the task. Defaults to None. + """ + + def __init__( + self, + name: Text, + description: Text, + function: Function, + supplier: Optional[Supplier] = None, + **additional_info, + ) -> None: + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Args: + name (Text): name of the tool + description (Text): descriptiion of the tool + function (Function): task that the tool performs + supplier (Optional[Union[Dict, Text, Supplier, int]], optional): Preferred supplier to perform the task. Defaults to None. + """ + self.name = name + self.description = description + self.function = function + self.supplier = supplier + self.additional_info = additional_info diff --git a/aixplain/modules/agent/tool/__init__.py b/aixplain/modules/agent/tool/__init__.py new file mode 100644 index 00000000..2a22511a --- /dev/null +++ b/aixplain/modules/agent/tool/__init__.py @@ -0,0 +1,53 @@ +__author__ = "aiXplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Lucas Pavanelli and Thiago Castro Ferreira +Date: May 16th 2024 +Description: + Agentification Class +""" +from abc import ABC +from typing import Optional, Text + + +class Tool(ABC): + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Attributes: + name (Text): name of the tool + description (Text): descriptiion of the tool + version (Text): version of the tool + """ + + def __init__( + self, + name: Text, + description: Text, + version: Optional[Text] = None, + **additional_info, + ) -> None: + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Args: + name (Text): name of the tool + description (Text): descriptiion of the tool + version (Text): version of the tool + """ + self.name = name + self.description = description + self.version = version + self.additional_info = additional_info diff --git a/aixplain/modules/agent/tool/model_tool.py b/aixplain/modules/agent/tool/model_tool.py new file mode 100644 index 00000000..69bf28d5 --- /dev/null +++ b/aixplain/modules/agent/tool/model_tool.py @@ -0,0 +1,60 @@ +__author__ = "aiXplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Lucas Pavanelli and Thiago Castro Ferreira +Date: May 16th 2024 +Description: + Agentification Class +""" +from typing import Optional + +from aixplain.enums.function import Function +from aixplain.enums.supplier import Supplier +from aixplain.modules.agent.tool import Tool + + +class ModelTool(Tool): + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Attributes: + function (Function): task that the tool performs + supplier (Optional[Union[Dict, Text, Supplier, int]], optional): Preferred supplier to perform the task. Defaults to None. + """ + + def __init__( + self, + function: Function, + supplier: Optional[Supplier] = None, + **additional_info, + ) -> None: + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Args: + function (Function): task that the tool performs + supplier (Optional[Union[Dict, Text, Supplier, int]], optional): Preferred supplier to perform the task. Defaults to None. + """ + super().__init__("", "", **additional_info) + if isinstance(function, str): + function = Function(function) + self.function = function + + try: + if isinstance(supplier, dict): + supplier = Supplier(supplier) + except Exception: + supplier = None + self.supplier = supplier diff --git a/aixplain/modules/agent/tool/pipeline_tool.py b/aixplain/modules/agent/tool/pipeline_tool.py new file mode 100644 index 00000000..a517b198 --- /dev/null +++ b/aixplain/modules/agent/tool/pipeline_tool.py @@ -0,0 +1,52 @@ +__author__ = "aiXplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: Lucas Pavanelli and Thiago Castro Ferreira +Date: May 16th 2024 +Description: + Agentification Class +""" +from typing import Text, Union + +from aixplain.modules.agent.tool import Tool +from aixplain.modules.pipeline import Pipeline + + +class PipelineTool(Tool): + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Attributes: + description (Text): descriptiion of the tool + pipeline (Union[Text, Pipeline]): pipeline + """ + + def __init__( + self, + description: Text, + pipeline: Union[Text, Pipeline], + **additional_info, + ) -> None: + """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. + + Args: + description (Text): description of the tool + pipeline (Union[Text, Pipeline]): pipeline + """ + super().__init__("", description, **additional_info) + if isinstance(pipeline, Pipeline): + pipeline = pipeline.id + self.pipeline = pipeline diff --git a/aixplain/modules/asset.py b/aixplain/modules/asset.py index 52b79912..c453415d 100644 --- a/aixplain/modules/asset.py +++ b/aixplain/modules/asset.py @@ -57,7 +57,13 @@ def __init__( elif isinstance(supplier, Dict) is True: self.supplier = Supplier(supplier) else: - self.supplier = supplier + self.supplier = None + for supplier_ in Supplier: + if supplier.lower() in [supplier_.value["code"].lower(), supplier_.value["name"].lower()]: + self.supplier = supplier_ + break + if self.supplier is None: + self.supplier = supplier except Exception: self.supplier = str(supplier) self.version = version diff --git a/aixplain/modules/finetune/__init__.py b/aixplain/modules/finetune/__init__.py index e1b63941..fe2cb15c 100644 --- a/aixplain/modules/finetune/__init__.py +++ b/aixplain/modules/finetune/__init__.py @@ -26,7 +26,6 @@ from urllib.parse import urljoin from aixplain.modules.finetune.cost import FinetuneCost from aixplain.modules.finetune.hyperparameters import Hyperparameters -from aixplain.factories.model_factory import ModelFactory from aixplain.modules.asset import Asset from aixplain.modules.dataset import Dataset from aixplain.modules.model import Model @@ -110,7 +109,7 @@ def start(self) -> Model: """ payload = {} try: - url = urljoin(self.backend_url, f"sdk/finetune") + url = urljoin(self.backend_url, "sdk/finetune") headers = {"Authorization": f"Token {self.api_key}", "Content-Type": "application/json"} payload = { "name": self.name, @@ -134,6 +133,8 @@ def start(self) -> Model: r = _request_with_retry("post", url, headers=headers, json=payload) resp = r.json() logging.info(f"Response for POST Start FineTune - Name: {self.name} / Status {resp}") + from aixplain.factories.model_factory import ModelFactory + return ModelFactory().get(resp["id"]) except Exception: message = "" diff --git a/aixplain/modules/metric.py b/aixplain/modules/metric.py index d591772b..86c08a08 100644 --- a/aixplain/modules/metric.py +++ b/aixplain/modules/metric.py @@ -24,9 +24,6 @@ from typing import Optional, Text, List, Union from aixplain.modules.asset import Asset -from aixplain.utils.file_utils import _request_with_retry -from aixplain.factories.model_factory import ModelFactory - class Metric(Asset): """Represents a metric to be computed on one or more peices of data. It is usually linked to a machine learning task. diff --git a/aixplain/modules/model/__init__.py b/aixplain/modules/model/__init__.py index 285cbe55..4be40225 100644 --- a/aixplain/modules/model/__init__.py +++ b/aixplain/modules/model/__init__.py @@ -24,9 +24,7 @@ import json import logging import traceback -from typing import List -from aixplain.factories.file_factory import FileFactory -from aixplain.enums import Function, Supplier +from aixplain.enums import Supplier, Function from aixplain.modules.asset import Asset from aixplain.utils import config from urllib.parse import urljoin @@ -57,7 +55,7 @@ def __init__( id: Text, name: Text, description: Text = "", - api_key: Optional[Text] = None, + api_key: Text = config.TEAM_API_KEY, supplier: Union[Dict, Text, Supplier, int] = "aiXplain", version: Optional[Text] = None, function: Optional[Function] = None, @@ -163,7 +161,7 @@ def poll(self, poll_url: Text, name: Text = "model_process") -> Dict: resp["status"] = "FAILED" else: resp["status"] = "IN_PROGRESS" - logging.info(f"Single Poll for Model: Status of polling for {name}: {resp}") + logging.debug(f"Single Poll for Model: Status of polling for {name}: {resp}") except Exception as e: resp = {"status": "FAILED"} logging.error(f"Single Poll for Model: Error of polling for {name}: {e}") @@ -218,6 +216,7 @@ def run_async(self, data: Union[Text, Dict], name: Text = "model_process", param dict: polling URL in response """ headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + from aixplain.factories.file_factory import FileFactory data = FileFactory.to_link(data) if isinstance(data, dict): @@ -229,7 +228,7 @@ def run_async(self, data: Union[Text, Dict], name: Text = "model_process", param if isinstance(payload, int) is True or isinstance(payload, float) is True: payload = str(payload) payload = {"data": payload} - except Exception as e: + except Exception: payload = {"data": data} payload.update(parameters) payload = json.dumps(payload) @@ -245,7 +244,7 @@ def run_async(self, data: Union[Text, Dict], name: Text = "model_process", param poll_url = resp["data"] response = {"status": "IN_PROGRESS", "url": poll_url} - except Exception as e: + except Exception: response = {"status": "FAILED"} msg = f"Error in request for {name} - {traceback.format_exc()}" logging.error(f"Model Run Async: Error in running for {name}: {resp}") @@ -267,7 +266,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): """ from aixplain.enums.asset_status import AssetStatus from aixplain.modules.finetune.status import FinetuneStatus - + headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} resp = None try: @@ -311,7 +310,7 @@ def check_finetune_status(self, after_epoch: Optional[int] = None): logging.info(f"Response for GET Check FineTune status Model - Id {self.id} / Status {status.status.value}.") return status - except Exception as e: + except Exception: message = "" if resp is not None and "statusCode" in resp: status_code = resp["statusCode"] diff --git a/pyproject.toml b/pyproject.toml index 112c8f9a..73980717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ namespaces = true [project] name = "aiXplain" -version = "0.2.12" +version = "0.2.13rc2" description = "aiXplain SDK adds AI functions to software." readme = "README.md" requires-python = ">=3.5, <4" diff --git a/tests/functional/agent/agent_functional_test.py b/tests/functional/agent/agent_functional_test.py new file mode 100644 index 00000000..f58dcb63 --- /dev/null +++ b/tests/functional/agent/agent_functional_test.py @@ -0,0 +1,75 @@ +__author__ = "lucaspavanelli" + +""" +Copyright 2022 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import json +from dotenv import load_dotenv + +load_dotenv() +from aixplain.factories import AgentFactory +from aixplain.modules.agent import ModelTool, PipelineTool +from aixplain.enums.supplier import Supplier + +import pytest + +RUN_FILE = "tests/functional/agent/data/agent_test_end2end.json" + + +def read_data(data_path): + return json.load(open(data_path, "r")) + + +@pytest.fixture(scope="module", params=read_data(RUN_FILE)) +def run_input_map(request): + return request.param + + +def test_end2end(run_input_map): + tools = [] + if "model_tools" in run_input_map: + for tool in run_input_map["model_tools"]: + for supplier in Supplier: + if tool["supplier"] is not None and tool["supplier"].lower() in [ + supplier.value["code"].lower(), + supplier.value["name"].lower(), + ]: + tool["supplier"] = supplier + break + tools.append(ModelTool(function=tool["function"], supplier=tool["supplier"])) + if "pipeline_tools" in run_input_map: + for tool in run_input_map["pipeline_tools"]: + tools.append(PipelineTool(description=tool["description"], pipeline=tool["pipeline_id"])) + print(f"Creating agent with tools: {tools}") + agent = AgentFactory.create(name=run_input_map["agent_name"], llm_id=run_input_map["llm_id"], tools=tools) + print(f"Agent created: {agent.__dict__}") + print("Running agent") + response = agent.run(query=run_input_map["query"]) + print(f"Agent response: {response}") + assert response is not None + assert response["completed"] is True + assert response["status"].lower() == "success" + assert "data" in response + assert response["data"]["session_id"] is not None + assert response["data"]["output"] is not None + print("Deleting agent") + agent.delete() + + +def test_list_agents(): + agents = AgentFactory.list() + assert "results" in agents + agents_result = agents["results"] + assert type(agents_result) is list diff --git a/tests/functional/agent/data/agent_test_end2end.json b/tests/functional/agent/data/agent_test_end2end.json new file mode 100644 index 00000000..147928fe --- /dev/null +++ b/tests/functional/agent/data/agent_test_end2end.json @@ -0,0 +1,14 @@ +[ + { + "agent_name": "[TEST] Translation agent", + "llm_id": "6626a3a8c8f1d089790cf5a2", + "llm_name": "Groq Llama 3 70B", + "query": "Who is the president of Brazil right now? Translate to pt", + "model_tools": [ + { + "function": "translation", + "supplier": "AWS" + } + ] + } +] From e9091c228585b0aa7a155b1630d7bac5f0dc8582 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 30 Jul 2024 21:34:32 +0200 Subject: [PATCH 14/25] Fixing circular import in the SDK (#211) Co-authored-by: Thiago Castro Ferreira --- aixplain/modules/model/llm_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aixplain/modules/model/llm_model.py b/aixplain/modules/model/llm_model.py index 14b9c7f4..5c5c4140 100644 --- a/aixplain/modules/model/llm_model.py +++ b/aixplain/modules/model/llm_model.py @@ -24,7 +24,6 @@ import json import logging import traceback -from aixplain.factories.file_factory import FileFactory from aixplain.enums import Function, Supplier from aixplain.modules.model import Model from aixplain.utils import config @@ -182,6 +181,8 @@ def run_async( """ headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + from aixplain.factories.file_factory import FileFactory + data = FileFactory.to_link(data) if isinstance(data, dict): payload = data From f4378153221ae5f5cc247f1667b1f709b880215e Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Fri, 2 Aug 2024 11:00:19 -0300 Subject: [PATCH 15/25] create model/pipeline tools from AgentFactory (#214) Co-authored-by: Thiago Castro Ferreira --- aixplain/factories/agent_factory/__init__.py | 23 +++++++++++++++++++ .../functional/agent/agent_functional_test.py | 5 ++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/aixplain/factories/agent_factory/__init__.py b/aixplain/factories/agent_factory/__init__.py index 36380a76..7507eef4 100644 --- a/aixplain/factories/agent_factory/__init__.py +++ b/aixplain/factories/agent_factory/__init__.py @@ -24,10 +24,12 @@ import json import logging +from aixplain.enums.function import Function from aixplain.enums.supplier import Supplier from aixplain.modules.agent import Agent, Tool from aixplain.modules.agent.tool.model_tool import ModelTool from aixplain.modules.agent.tool.pipeline_tool import PipelineTool +from aixplain.modules.pipeline import Pipeline from aixplain.utils import config from typing import Dict, List, Optional, Text, Union @@ -113,6 +115,27 @@ def create( raise Exception(e) return agent + @classmethod + def create_model_tool(cls, function: Union[Function, Text], supplier: Optional[Union[Supplier, Text]] = None) -> ModelTool: + """Create a new model tool.""" + if isinstance(function, str): + function = Function(function) + + if supplier is not None: + if isinstance(supplier, str): + for supplier_ in Supplier: + if supplier.lower() in [supplier.value["code"].lower(), supplier.value["name"].lower()]: + supplier = supplier_ + break + if isinstance(supplier, str): + supplier = None + return ModelTool(function=function, supplier=supplier) + + @classmethod + def create_pipeline_tool(cls, description: Text, pipeline: Union[Pipeline, Text]) -> PipelineTool: + """Create a new pipeline tool.""" + return PipelineTool(description=description, pipeline=pipeline) + @classmethod def list(cls) -> Dict: """List all agents available in the platform.""" diff --git a/tests/functional/agent/agent_functional_test.py b/tests/functional/agent/agent_functional_test.py index f58dcb63..766ba386 100644 --- a/tests/functional/agent/agent_functional_test.py +++ b/tests/functional/agent/agent_functional_test.py @@ -20,7 +20,6 @@ load_dotenv() from aixplain.factories import AgentFactory -from aixplain.modules.agent import ModelTool, PipelineTool from aixplain.enums.supplier import Supplier import pytest @@ -48,10 +47,10 @@ def test_end2end(run_input_map): ]: tool["supplier"] = supplier break - tools.append(ModelTool(function=tool["function"], supplier=tool["supplier"])) + tools.append(AgentFactory.create_model_tool(function=tool["function"], supplier=tool["supplier"])) if "pipeline_tools" in run_input_map: for tool in run_input_map["pipeline_tools"]: - tools.append(PipelineTool(description=tool["description"], pipeline=tool["pipeline_id"])) + tools.append(AgentFactory.create_pipeline_tool(pipeline=tool["pipeline_id"], description=tool["description"])) print(f"Creating agent with tools: {tools}") agent = AgentFactory.create(name=run_input_map["agent_name"], llm_id=run_input_map["llm_id"], tools=tools) print(f"Agent created: {agent.__dict__}") From 03009c6bc474c4cee416423d9b96ffc2f539bdea Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Wed, 7 Aug 2024 11:36:23 -0300 Subject: [PATCH 16/25] Set model ID as a parameter (#216) --- aixplain/factories/agent_factory/__init__.py | 15 +++++++--- aixplain/factories/agent_factory/utils.py | 3 +- aixplain/modules/agent/tool/model_tool.py | 28 +++++++++++++++---- .../functional/agent/agent_functional_test.py | 2 +- .../agent/data/agent_test_end2end.json | 5 ++++ 5 files changed, 41 insertions(+), 12 deletions(-) diff --git a/aixplain/factories/agent_factory/__init__.py b/aixplain/factories/agent_factory/__init__.py index 7507eef4..6076eef6 100644 --- a/aixplain/factories/agent_factory/__init__.py +++ b/aixplain/factories/agent_factory/__init__.py @@ -29,6 +29,7 @@ from aixplain.modules.agent import Agent, Tool from aixplain.modules.agent.tool.model_tool import ModelTool from aixplain.modules.agent.tool.pipeline_tool import PipelineTool +from aixplain.modules.model import Model from aixplain.modules.pipeline import Pipeline from aixplain.utils import config from typing import Dict, List, Optional, Text, Union @@ -66,11 +67,12 @@ def create( if isinstance(tool, ModelTool): tool_payload.append( { - "function": tool.function.value, + "function": tool.function.value if tool.function is not None else None, "type": "model", "description": tool.description, "supplier": tool.supplier.value["code"] if tool.supplier else None, "version": tool.version if tool.version else None, + "assetId": tool.model, } ) elif isinstance(tool, PipelineTool): @@ -116,9 +118,14 @@ def create( return agent @classmethod - def create_model_tool(cls, function: Union[Function, Text], supplier: Optional[Union[Supplier, Text]] = None) -> ModelTool: + def create_model_tool( + cls, + model: Optional[Union[Model, Text]] = None, + function: Optional[Union[Function, Text]] = None, + supplier: Optional[Union[Supplier, Text]] = None, + ) -> ModelTool: """Create a new model tool.""" - if isinstance(function, str): + if function is not None and isinstance(function, str): function = Function(function) if supplier is not None: @@ -129,7 +136,7 @@ def create_model_tool(cls, function: Union[Function, Text], supplier: Optional[U break if isinstance(supplier, str): supplier = None - return ModelTool(function=function, supplier=supplier) + return ModelTool(function=function, supplier=supplier, model=model) @classmethod def create_pipeline_tool(cls, description: Text, pipeline: Union[Pipeline, Text]) -> PipelineTool: diff --git a/aixplain/factories/agent_factory/utils.py b/aixplain/factories/agent_factory/utils.py index 6363a08e..4b314ef7 100644 --- a/aixplain/factories/agent_factory/utils.py +++ b/aixplain/factories/agent_factory/utils.py @@ -22,9 +22,10 @@ def build_agent(payload: Dict, api_key: Text = config.TEAM_API_KEY) -> Agent: break tool = ModelTool( - function=Function(tool["function"]), + function=Function(tool["function"]) if tool["function"] is not None else None, supplier=tool["supplier"], version=tool["version"], + model=tool["assetId"], ) elif tool["type"] == "pipeline": tool = PipelineTool(description=tool["description"], pipeline=tool["assetId"]) diff --git a/aixplain/modules/agent/tool/model_tool.py b/aixplain/modules/agent/tool/model_tool.py index 69bf28d5..a5acab30 100644 --- a/aixplain/modules/agent/tool/model_tool.py +++ b/aixplain/modules/agent/tool/model_tool.py @@ -20,11 +20,13 @@ Description: Agentification Class """ -from typing import Optional +from typing import Optional, Union, Text from aixplain.enums.function import Function from aixplain.enums.supplier import Supplier +from aixplain.factories.model_factory import ModelFactory from aixplain.modules.agent.tool import Tool +from aixplain.modules.model import Model class ModelTool(Tool): @@ -37,19 +39,25 @@ class ModelTool(Tool): def __init__( self, - function: Function, + function: Optional[Function] = None, supplier: Optional[Supplier] = None, + model: Optional[Union[Text, Model]] = None, **additional_info, ) -> None: """Specialized software or resource designed to assist the AI in executing specific tasks or functions based on user commands. Args: - function (Function): task that the tool performs - supplier (Optional[Union[Dict, Text, Supplier, int]], optional): Preferred supplier to perform the task. Defaults to None. + function (Optional[Function], optional): task that the tool performs. Defaults to None. + supplier (Optional[Supplier], optional): Preferred supplier to perform the task. Defaults to None.. Defaults to None. + model (Optional[Union[Text, Model]], optional): Model function. Defaults to None. """ + assert ( + function is not None or model is not None + ), "Agent Creation Error: Either function or model must be provided when instantiating a tool." super().__init__("", "", **additional_info) - if isinstance(function, str): - function = Function(function) + if function is not None: + if isinstance(function, str): + function = Function(function) self.function = function try: @@ -57,4 +65,12 @@ def __init__( supplier = Supplier(supplier) except Exception: supplier = None + + if model is not None: + if isinstance(model, Text) is True: + model = ModelFactory.get(model) + if isinstance(model.supplier, Supplier): + supplier = model.supplier + model = model.id self.supplier = supplier + self.model = model diff --git a/tests/functional/agent/agent_functional_test.py b/tests/functional/agent/agent_functional_test.py index 766ba386..427f62e5 100644 --- a/tests/functional/agent/agent_functional_test.py +++ b/tests/functional/agent/agent_functional_test.py @@ -47,7 +47,7 @@ def test_end2end(run_input_map): ]: tool["supplier"] = supplier break - tools.append(AgentFactory.create_model_tool(function=tool["function"], supplier=tool["supplier"])) + tools.append(AgentFactory.create_model_tool(**tool)) if "pipeline_tools" in run_input_map: for tool in run_input_map["pipeline_tools"]: tools.append(AgentFactory.create_pipeline_tool(pipeline=tool["pipeline_id"], description=tool["description"])) diff --git a/tests/functional/agent/data/agent_test_end2end.json b/tests/functional/agent/data/agent_test_end2end.json index 147928fe..94bfc94b 100644 --- a/tests/functional/agent/data/agent_test_end2end.json +++ b/tests/functional/agent/data/agent_test_end2end.json @@ -8,6 +8,11 @@ { "function": "translation", "supplier": "AWS" + }, + { + "model": "60ddefca8d38c51c58860108", + "function": null, + "supplier": null } ] } From 02f7482466e94422e070acbda901cd426709cf82 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Wed, 7 Aug 2024 19:13:42 -0300 Subject: [PATCH 17/25] Content inputs to be processed according to the query. (#215) * Content inputs to be processed according to the query. * Add data and query parameters on running agent --- aixplain/modules/agent/__init__.py | 53 ++++++++++++++++--- .../functional/agent/agent_functional_test.py | 2 +- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/aixplain/modules/agent/__init__.py b/aixplain/modules/agent/__init__.py index 2f244d56..8a5cd120 100644 --- a/aixplain/modules/agent/__init__.py +++ b/aixplain/modules/agent/__init__.py @@ -28,6 +28,7 @@ from aixplain.utils.file_utils import _request_with_retry from aixplain.enums.supplier import Supplier from aixplain.enums.asset_status import AssetStatus +from aixplain.enums.storage_type import StorageType from aixplain.modules.model import Model from aixplain.modules.agent.tool import Tool from aixplain.modules.agent.tool.model_tool import ModelTool @@ -96,31 +97,43 @@ def __init__( def run( self, - query: Text, + data: Optional[Union[Dict, Text]] = None, + query: Optional[Text] = None, session_id: Optional[Text] = None, history: Optional[List[Dict]] = None, name: Text = "model_process", timeout: float = 300, parameters: Dict = {}, wait_time: float = 0.5, + content: List[Text] = [], ) -> Dict: """Runs an agent call. Args: - query (Text): query to be processed by the agent. + data (Optional[Union[Dict, Text]], optional): data to be processed by the agent. Defaults to None. + query (Optional[Text], optional): query to be processed by the agent. Defaults to None. session_id (Optional[Text], optional): conversation Session ID. Defaults to None. history (Optional[List[Dict]], optional): chat history (in case session ID is None). Defaults to None. name (Text, optional): ID given to a call. Defaults to "model_process". timeout (float, optional): total polling time. Defaults to 300. parameters (Dict, optional): optional parameters to the model. Defaults to "{}". wait_time (float, optional): wait time in seconds between polling calls. Defaults to 0.5. + content (List[Text], optional): Content inputs to be processed according to the query. Defaults to []. Returns: Dict: parsed output from model """ start = time.time() try: - response = self.run_async(query=query, session_id=session_id, history=history, name=name, parameters=parameters) + response = self.run_async( + data=data, + query=query, + session_id=session_id, + history=history, + name=name, + parameters=parameters, + content=content, + ) if response["status"] == "FAILED": end = time.time() response["elapsed_time"] = end - start @@ -137,27 +150,55 @@ def run( def run_async( self, - query: Text, + data: Optional[Union[Dict, Text]] = None, + query: Optional[Text] = None, session_id: Optional[Text] = None, history: Optional[List[Dict]] = None, name: Text = "model_process", parameters: Dict = {}, + content: List[Text] = [], ) -> Dict: """Runs asynchronously an agent call. Args: - query (Text): query to be processed by the agent. + data (Optional[Union[Dict, Text]], optional): data to be processed by the agent. Defaults to None. + query (Optional[Text], optional): query to be processed by the agent. Defaults to None. session_id (Optional[Text], optional): conversation Session ID. Defaults to None. history (Optional[List[Dict]], optional): chat history (in case session ID is None). Defaults to None. name (Text, optional): ID given to a call. Defaults to "model_process". parameters (Dict, optional): optional parameters to the model. Defaults to "{}". + content (List[Text], optional): Content inputs to be processed according to the query. Defaults to []. Returns: dict: polling URL in response """ - headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} from aixplain.factories.file_factory import FileFactory + assert data is not None or query is not None, "Either 'data' or 'query' must be provided." + if data is not None: + if isinstance(data, dict): + assert "query" in data and data["query"] is not None, "When providing a dictionary, 'query' must be provided." + query = data.get("query") + if session_id is None: + session_id = data.get("session_id") + if history is None: + history = data.get("history") + if len(content) == 0: + content = data.get("content", []) + else: + query = data + + # process content inputs + content = list(set(content)) + if len(content) > 0: + assert FileFactory.check_storage_type(query) == StorageType.TEXT, "When providing 'content', query must be text." + assert len(content) <= 3, "The maximum number of content inputs is 3." + for input_link in content: + input_link = FileFactory.to_link(input_link) + query += f"\n{input_link}" + + headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + payload = {"id": self.id, "query": FileFactory.to_link(query), "sessionId": session_id, "history": history} payload.update(parameters) payload = json.dumps(payload) diff --git a/tests/functional/agent/agent_functional_test.py b/tests/functional/agent/agent_functional_test.py index 427f62e5..cefd34c3 100644 --- a/tests/functional/agent/agent_functional_test.py +++ b/tests/functional/agent/agent_functional_test.py @@ -55,7 +55,7 @@ def test_end2end(run_input_map): agent = AgentFactory.create(name=run_input_map["agent_name"], llm_id=run_input_map["llm_id"], tools=tools) print(f"Agent created: {agent.__dict__}") print("Running agent") - response = agent.run(query=run_input_map["query"]) + response = agent.run(data=run_input_map["query"]) print(f"Agent response: {response}") assert response is not None assert response["completed"] is True From 4947959d65c4ddcf94410fbcbb73449bdf0a08a9 Mon Sep 17 00:00:00 2001 From: kadirpekel Date: Sat, 10 Aug 2024 02:12:23 +0300 Subject: [PATCH 18/25] ENG-1: programmatic api introduced (#219) * ENG-1: programmatic api introduced * pipeline.py in .gitignore * jinja dep * fixed data type refs * migrated to designer structure * updated readme * fixed gitignore * enabled node number and label * auto populate params by spec * naming review * designer functional tests * Pipeline getter * unit tests restored * pipeline naming fixed * minor improvements * Fixture on designer functional tests * Fixing input node and link param mapping * Script Nodes and filling missing /o values * Decision getter * README update * minor * moved all designer samples as functional tests --------- Co-authored-by: kadir pekel Co-authored-by: Thiago Castro Ferreira --- .gitignore | 3 + aixplain/enums/data_type.py | 4 +- aixplain/enums/function.py | 1 + aixplain/factories/file_factory.py | 6 +- .../__init__.py} | 117 +- aixplain/factories/pipeline_factory/utils.py | 100 + aixplain/factories/script_factory.py | 31 + aixplain/modules/pipeline/__init__.py | 3 + .../{pipeline.py => pipeline/asset.py} | 195 +- aixplain/modules/pipeline/default.py | 17 + aixplain/modules/pipeline/designer/README.md | 194 + .../modules/pipeline/designer/__init__.py | 66 + aixplain/modules/pipeline/designer/base.py | 375 ++ aixplain/modules/pipeline/designer/enums.py | 43 + aixplain/modules/pipeline/designer/mixins.py | 85 + aixplain/modules/pipeline/designer/nodes.py | 464 ++ .../modules/pipeline/designer/pipeline.py | 328 ++ aixplain/modules/pipeline/generate.py | 227 + aixplain/modules/pipeline/pipeline.py | 4548 +++++++++++++++++ pyproject.toml | 3 +- tests/functional/pipelines/data/script.py | 51 + tests/functional/pipelines/designer_test.py | 248 + tests/unit/designer_test.py | 707 +++ 23 files changed, 7746 insertions(+), 70 deletions(-) rename aixplain/factories/{pipeline_factory.py => pipeline_factory/__init__.py} (77%) create mode 100644 aixplain/factories/pipeline_factory/utils.py create mode 100644 aixplain/factories/script_factory.py create mode 100644 aixplain/modules/pipeline/__init__.py rename aixplain/modules/{pipeline.py => pipeline/asset.py} (68%) create mode 100644 aixplain/modules/pipeline/default.py create mode 100644 aixplain/modules/pipeline/designer/README.md create mode 100644 aixplain/modules/pipeline/designer/__init__.py create mode 100644 aixplain/modules/pipeline/designer/base.py create mode 100644 aixplain/modules/pipeline/designer/enums.py create mode 100644 aixplain/modules/pipeline/designer/mixins.py create mode 100644 aixplain/modules/pipeline/designer/nodes.py create mode 100644 aixplain/modules/pipeline/designer/pipeline.py create mode 100644 aixplain/modules/pipeline/generate.py create mode 100644 aixplain/modules/pipeline/pipeline.py create mode 100644 tests/functional/pipelines/data/script.py create mode 100644 tests/functional/pipelines/designer_test.py create mode 100644 tests/unit/designer_test.py diff --git a/.gitignore b/.gitignore index b6e47617..843c6556 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ dmypy.json # Pyre type checker .pyre/ + +# Vscode +.vscode diff --git a/aixplain/enums/data_type.py b/aixplain/enums/data_type.py index fa79d070..11432bcf 100644 --- a/aixplain/enums/data_type.py +++ b/aixplain/enums/data_type.py @@ -24,7 +24,7 @@ from enum import Enum -class DataType(Enum): +class DataType(str, Enum): AUDIO = "audio" FLOAT = "float" IMAGE = "image" @@ -33,6 +33,8 @@ class DataType(Enum): TENSOR = "tensor" TEXT = "text" VIDEO = "video" + EMBEDDING = "embedding" + NUMBER = "number" def __str__(self): return self._value_ diff --git a/aixplain/enums/function.py b/aixplain/enums/function.py index bfab8427..9a6f47d4 100644 --- a/aixplain/enums/function.py +++ b/aixplain/enums/function.py @@ -55,6 +55,7 @@ def load_functions(): if input_data_object["required"] is True }, "output": {output_data_object["dataType"] for output_data_object in function["output"]}, + "spec": function } for function in resp["items"] } diff --git a/aixplain/factories/file_factory.py b/aixplain/factories/file_factory.py index e7d7c4da..adbebcd3 100644 --- a/aixplain/factories/file_factory.py +++ b/aixplain/factories/file_factory.py @@ -104,7 +104,7 @@ def check_storage_type(cls, input_link: Any) -> StorageType: return StorageType.TEXT @classmethod - def to_link(cls, data: Union[Text, Dict]) -> Union[Text, Dict]: + def to_link(cls, data: Union[Text, Dict], **kwargs) -> Union[Text, Dict]: """If user input data is a local file, upload to aiXplain platform Args: @@ -117,10 +117,10 @@ def to_link(cls, data: Union[Text, Dict]) -> Union[Text, Dict]: for key in data: if isinstance(data[key], str): if cls.check_storage_type(data[key]) == StorageType.FILE: - data[key] = cls.upload(local_path=data[key]) + data[key] = cls.upload(local_path=data[key], **kwargs) elif isinstance(data, str): if cls.check_storage_type(data) == StorageType.FILE: - data = cls.upload(local_path=data) + data = cls.upload(local_path=data, **kwargs) return data @classmethod diff --git a/aixplain/factories/pipeline_factory.py b/aixplain/factories/pipeline_factory/__init__.py similarity index 77% rename from aixplain/factories/pipeline_factory.py rename to aixplain/factories/pipeline_factory/__init__.py index 61bcb214..051c63fb 100644 --- a/aixplain/factories/pipeline_factory.py +++ b/aixplain/factories/pipeline_factory/__init__.py @@ -21,9 +21,10 @@ Pipeline Factory Class """ import json -import logging import os +import logging from typing import Dict, List, Optional, Text, Union +from aixplain.factories.pipeline_factory.utils import build_from_response from aixplain.enums.data_type import DataType from aixplain.enums.function import Function from aixplain.enums.supplier import Supplier @@ -45,27 +46,6 @@ class PipelineFactory: aixplain_key = config.AIXPLAIN_API_KEY backend_url = config.BACKEND_URL - @classmethod - def __get_typed_nodes(cls, response: Dict, type: str) -> List[Dict]: - # read "nodes" field from response and return the nodes that are marked by "type": type - return [node for node in response["nodes"] if node["type"].lower() == type.lower()] - - @classmethod - def __from_response(cls, response: Dict) -> Pipeline: - """Converts response Json to 'Pipeline' object - - Args: - response (Dict): Json from API - - Returns: - Pipeline: Coverted 'Pipeline' object - """ - if "api_key" not in response: - response["api_key"] = config.TEAM_API_KEY - input = cls.__get_typed_nodes(response, "input") - output = cls.__get_typed_nodes(response, "output") - return Pipeline(response["id"], response["name"], response["api_key"], input=input, output=output) - @classmethod def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: """Create a 'Pipeline' object from pipeline id @@ -81,11 +61,20 @@ def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: try: url = urljoin(cls.backend_url, f"sdk/pipelines/{pipeline_id}") if api_key is not None: - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Token {api_key}", + "Content-Type": "application/json", + } elif cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + headers = { + "x-aixplain-key": f"{cls.aixplain_key}", + "Content-Type": "application/json", + } else: - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Token {config.TEAM_API_KEY}", + "Content-Type": "application/json", + } logging.info(f"Start service for GET Pipeline - {url} - {headers}") r = _request_with_retry("get", url, headers=headers) resp = r.json() @@ -93,16 +82,17 @@ def get(cls, pipeline_id: Text, api_key: Optional[Text] = None) -> Pipeline: resp["api_key"] = config.TEAM_API_KEY if api_key is not None: resp["api_key"] = api_key - pipeline = cls.__from_response(resp) + pipeline = build_from_response(resp, load_architecture=True) return pipeline - except Exception: + except Exception as e: + logging.exception(e) status_code = 400 if resp is not None and "statusCode" in resp: status_code = resp["statusCode"] message = resp["message"] message = f"Pipeline Creation: Status {status_code} - {message}" else: - message = "Pipeline Creation: Unspecified Error" + message = f"Pipeline Creation: Unspecified Error {e}" logging.error(message) raise Exception(f"Status {status_code}: {message}") @@ -127,14 +117,20 @@ def get_assets_from_page(cls, page_number: int) -> List[Pipeline]: try: url = urljoin(cls.backend_url, f"sdk/pipelines/?pageNumber={page_number}") if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + headers = { + "x-aixplain-key": f"{cls.aixplain_key}", + "Content-Type": "application/json", + } else: - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Token {config.TEAM_API_KEY}", + "Content-Type": "application/json", + } r = _request_with_retry("get", url, headers=headers) resp = r.json() logging.info(f"Listing Pipelines: Status of getting Pipelines on Page {page_number}: {resp}") all_pipelines = resp["items"] - pipeline_list = [cls.__from_response(pipeline_info_json) for pipeline_info_json in all_pipelines] + pipeline_list = [build_from_response(pipeline_info_json) for pipeline_info_json in all_pipelines] return pipeline_list except Exception as e: error_message = f"Listing Pipelines: Error in getting Pipelines on Page {page_number}: {e}" @@ -177,9 +173,15 @@ def list( url = urljoin(cls.backend_url, "sdk/pipelines/paginate") if cls.aixplain_key != "": - headers = {"x-aixplain-key": f"{cls.aixplain_key}", "Content-Type": "application/json"} + headers = { + "x-aixplain-key": f"{cls.aixplain_key}", + "Content-Type": "application/json", + } else: - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Token {config.TEAM_API_KEY}", + "Content-Type": "application/json", + } assert 0 < page_size <= 100, "Pipeline List Error: Page size must be greater than 0 and not exceed 100." payload = { @@ -228,11 +230,43 @@ def list( total = resp["total"] logging.info(f"Response for POST List Pipeline - Page Total: {page_total} / Total: {total}") for pipeline in results: - pipelines.append(cls.__from_response(pipeline)) - return {"results": pipelines, "page_total": page_total, "page_number": page_number, "total": total} + pipelines.append(build_from_response(pipeline)) + return { + "results": pipelines, + "page_total": page_total, + "page_number": page_number, + "total": total, + } @classmethod - def create(cls, name: Text, pipeline: Union[Text, Dict], api_key: Optional[Text] = None) -> Pipeline: + def init(cls, name: Text, api_key: Optional[Text] = None) -> Pipeline: + """Initialize a new Pipeline + + Args: + name (Text): Pipeline Name + api_key (Optional[Text], optional): Team API Key to create the Pipeline. Defaults to None. + + Returns: + Pipeline: instance of the new pipeline + """ + if api_key is None: + api_key = config.TEAM_API_KEY + return Pipeline( + id="", + name=name, + api_key=api_key, + nodes=[], + links=[], + instance=None, + ) + + @classmethod + def create( + cls, + name: Text, + pipeline: Union[Text, Dict], + api_key: Optional[Text] = None, + ) -> Pipeline: """Draft Pipeline Creation Args: @@ -259,10 +293,17 @@ def create(cls, name: Text, pipeline: Union[Text, Dict], api_key: Optional[Text] if "functionType" in node and node["functionType"] == "AI": pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() # prepare payload - payload = {"name": name, "status": "draft", "architecture": pipeline} + payload = { + "name": name, + "status": "draft", + "architecture": pipeline, + } url = urljoin(cls.backend_url, "sdk/pipelines") api_key = api_key if api_key is not None else config.TEAM_API_KEY - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} + headers = { + "Authorization": f"Token {api_key}", + "Content-Type": "application/json", + } logging.info(f"Start service for POST Create Pipeline - {url} - {headers} - {json.dumps(payload)}") r = _request_with_retry("post", url, headers=headers, json=payload) response = r.json() diff --git a/aixplain/factories/pipeline_factory/utils.py b/aixplain/factories/pipeline_factory/utils.py new file mode 100644 index 00000000..465e5e7f --- /dev/null +++ b/aixplain/factories/pipeline_factory/utils.py @@ -0,0 +1,100 @@ +__author__ = "aixplain" +import logging + +from aixplain.enums import DataType +import aixplain.utils.config as config +from aixplain.modules.pipeline import Pipeline +from aixplain.modules.pipeline.designer import ( + Input, + Output, + AssetNode, + Decision, + Router, + Route, + Script, + Link, +) +from typing import Dict + + +def build_from_response(response: Dict, load_architecture: bool = False) -> Pipeline: + """Converts response Json to 'Pipeline' object + + Args: + response (Dict): Json from API + load_architecture (bool, optional): If True, the architecture will be loaded. Defaults to False. + + Returns: + Pipeline: Coverted 'Pipeline' object + """ + if "api_key" not in response: + response["api_key"] = config.TEAM_API_KEY + + # instantiating pipeline generic info + pipeline = Pipeline(response["id"], response["name"], response["api_key"]) + if load_architecture is True: + try: + # instantiating nodes + for node_json in response["nodes"]: + print(node_json) + if node_json["type"].lower() == "input": + node = Input( + data=node_json["data"] if "data" in node_json else None, + data_types=[DataType(dt) for dt in node_json["dataType"]], + ) + elif node_json["type"].lower() == "asset": + node = AssetNode(asset_id=node_json["assetId"]) + elif node_json["type"].lower() == "segmentor": + raise NotImplementedError() + elif node_json["type"].lower() == "reconstructor": + raise NotImplementedError() + elif node_json["type"].lower() == "decision": + node = Decision(routes=[Route(**route) for route in node_json["routes"]]) + elif node_json["type"].lower() == "router": + node = Router(routes=[Route(**route) for route in node_json["routes"]]) + elif node_json["type"].lower() == "script": + node = Script(fileId=node_json["fileId"]) + elif node_json["type"].lower() == "output": + node = Output() + + if "inputValues" in node_json: + [ + node.inputs.create_param( + data_type=DataType(input_param["dataType"]) if "dataType" in input_param else None, + code=input_param["code"], + value=input_param["value"] if "value" in input_param else None, + is_required=input_param["isRequired"] if "isRequired" in input_param else False, + ) + for input_param in node_json["inputValues"] + if input_param["code"] not in node.inputs + ] + if "outputValues" in node_json: + [ + node.outputs.create_param( + data_type=DataType(output_param["dataType"]) if "dataType" in output_param else None, + code=output_param["code"], + value=output_param["value"] if "value" in output_param else None, + is_required=output_param["isRequired"] if "isRequired" in output_param else False, + ) + for output_param in node_json["outputValues"] + if output_param["code"] not in node.outputs + ] + node.number = node_json["number"] + node.label = node_json["label"] + pipeline.add_node(node) + + # instantiating links + for link_json in response["links"]: + for param_mapping in link_json["paramMapping"]: + link = Link( + from_node=pipeline.get_node(link_json["from"]), + to_node=pipeline.get_node(link_json["to"]), + from_param=param_mapping["from"], + to_param=param_mapping["to"], + ) + pipeline.add_link(link) + except Exception as e: + logging.warning("Error loading pipeline architecture:, error: %s", e) + pipeline.nodes = [] + pipeline.links = [] + return pipeline diff --git a/aixplain/factories/script_factory.py b/aixplain/factories/script_factory.py new file mode 100644 index 00000000..35789561 --- /dev/null +++ b/aixplain/factories/script_factory.py @@ -0,0 +1,31 @@ +import os +import json +from typing import Tuple + +import requests + +from aixplain.utils import config + + +class ScriptFactory: + + @classmethod + def upload_script(cls, script_path: str) -> Tuple[str, str]: + try: + url = f"{config.BACKEND_URL}/sdk/pipelines/script" + headers = {"Authorization": f"Token {config.TEAM_API_KEY}"} + r = requests.post( + url, headers=headers, files={"file": open(script_path, "rb")} + ) + if 200 <= r.status_code < 300: + response = r.json() + else: + raise Exception() + except Exception: + response = {"fileId": ""} + + # get metadata info + fname = os.path.splitext(os.path.basename(script_path))[0] + file_size_kb = int(os.path.getsize(script_path) / 1024) + metadata = json.dumps({"name": fname, "size": file_size_kb}) + return response["fileId"], metadata diff --git a/aixplain/modules/pipeline/__init__.py b/aixplain/modules/pipeline/__init__.py new file mode 100644 index 00000000..d2381c3a --- /dev/null +++ b/aixplain/modules/pipeline/__init__.py @@ -0,0 +1,3 @@ +from .pipeline import Pipeline + +__all__ = ["Pipeline"] diff --git a/aixplain/modules/pipeline.py b/aixplain/modules/pipeline/asset.py similarity index 68% rename from aixplain/modules/pipeline.py rename to aixplain/modules/pipeline/asset.py index ed131018..ce168036 100644 --- a/aixplain/modules/pipeline.py +++ b/aixplain/modules/pipeline/asset.py @@ -24,6 +24,7 @@ import time import json import os +import uuid import logging from aixplain.modules.asset import Asset from aixplain.utils import config @@ -66,13 +67,20 @@ def __init__( version (Text, optional): version of the pipeline. Defaults to "1.0". **additional_info: Any additional Pipeline info to be saved """ + if not name: + raise ValueError("Pipeline name is required") + super().__init__(id, name, "", supplier, version) self.api_key = api_key self.url = f"{url}/assets/pipeline/execution/run" self.additional_info = additional_info def __polling( - self, poll_url: Text, name: Text = "pipeline_process", wait_time: float = 1.0, timeout: float = 20000.0 + self, + poll_url: Text, + name: Text = "pipeline_process", + wait_time: float = 1.0, + timeout: float = 20000.0, ) -> Dict: """Keeps polling the platform to check whether an asynchronous call is done. @@ -93,7 +101,9 @@ def __polling( while not completed and (end - start) < timeout: try: response_body = self.poll(poll_url, name=name) - logging.debug(f"Polling for Pipeline: Status of polling for {name} : {response_body}") + logging.debug( + f"Polling for Pipeline: Status of polling for {name} : {response_body}" + ) completed = response_body["completed"] end = time.time() @@ -102,12 +112,18 @@ def __polling( if wait_time < 60: wait_time *= 1.1 except Exception: - logging.error(f"Polling for Pipeline: polling for {name} : Continue") + logging.error( + f"Polling for Pipeline: polling for {name} : Continue" + ) if response_body and response_body["status"] == "SUCCESS": try: - logging.debug(f"Polling for Pipeline: Final status of polling for {name} : SUCCESS - {response_body}") + logging.debug( + f"Polling for Pipeline: Final status of polling for {name} : SUCCESS - {response_body}" + ) except Exception: - logging.error(f"Polling for Pipeline: Final status of polling for {name} : ERROR - {response_body}") + logging.error( + f"Polling for Pipeline: Final status of polling for {name} : ERROR - {response_body}" + ) else: logging.error( f"Polling for Pipeline: Final status of polling for {name} : No response in {timeout} seconds - {response_body}" @@ -125,11 +141,16 @@ def poll(self, poll_url: Text, name: Text = "pipeline_process") -> Dict: Dict: response obtained by polling call """ - headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + headers = { + "x-api-key": self.api_key, + "Content-Type": "application/json", + } r = _request_with_retry("get", poll_url, headers=headers) try: resp = r.json() - logging.info(f"Single Poll for Pipeline: Status of polling for {name} : {resp}") + logging.info( + f"Single Poll for Pipeline: Status of polling for {name} : {resp}" + ) except Exception: resp = {"status": "FAILED"} return resp @@ -158,23 +179,35 @@ def run( """ start = time.time() try: - response = self.run_async(data, data_asset=data_asset, name=name, **kwargs) + response = self.run_async( + data, data_asset=data_asset, name=name, **kwargs + ) if response["status"] == "FAILED": end = time.time() response["elapsed_time"] = end - start return response poll_url = response["url"] end = time.time() - response = self.__polling(poll_url, name=name, timeout=timeout, wait_time=wait_time) + response = self.__polling( + poll_url, name=name, timeout=timeout, wait_time=wait_time + ) return response except Exception as e: error_message = f"Error in request for {name}: {str(e)}" logging.error(error_message) logging.exception(error_message) end = time.time() - return {"status": "FAILED", "error": error_message, "elapsed_time": end - start} + return { + "status": "FAILED", + "error": error_message, + "elapsed_time": end - start, + } - def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[Text, Dict]] = None) -> Dict: + def __prepare_payload( + self, + data: Union[Text, Dict], + data_asset: Optional[Union[Text, Dict]] = None, + ) -> Dict: """Prepare pipeline execution payload, validating the input data Args: @@ -184,7 +217,11 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ Returns: Dict: pipeline execution payload """ - from aixplain.factories import CorpusFactory, DatasetFactory, FileFactory + from aixplain.factories import ( + CorpusFactory, + DatasetFactory, + FileFactory, + ) # if an input data asset is provided, just handle the data if data_asset is None: @@ -203,7 +240,10 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ try: payload = json.loads(data) if isinstance(payload, dict) is False: - if isinstance(payload, int) is True or isinstance(payload, float) is True: + if ( + isinstance(payload, int) is True + or isinstance(payload, float) is True + ): payload = str(payload) payload = {"data": payload} except Exception: @@ -233,15 +273,33 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ try: dasset = CorpusFactory.get(str(data_asset[node_label])) asset_payload["dataAsset"]["corpus_id"] = dasset.id - if len([d for d in dasset.data if d.id == data[node_label]]) > 0: + if ( + len( + [ + d + for d in dasset.data + if d.id == data[node_label] + ] + ) + > 0 + ): data_found = True except Exception: try: - dasset = DatasetFactory.get(str(data_asset[node_label])) + dasset = DatasetFactory.get( + str(data_asset[node_label]) + ) asset_payload["dataAsset"]["dataset_id"] = dasset.id if ( - len([dfield for dfield in dasset.source_data if dasset.source_data[dfield].id == data[node_label]]) + len( + [ + dfield + for dfield in dasset.source_data + if dasset.source_data[dfield].id + == data[node_label] + ] + ) > 0 ): data_found = True @@ -274,7 +332,11 @@ def __prepare_payload(self, data: Union[Text, Dict], data_asset: Optional[Union[ return payload def run_async( - self, data: Union[Text, Dict], data_asset: Optional[Union[Text, Dict]] = None, name: Text = "pipeline_process", **kwargs + self, + data: Union[Text, Dict], + data_asset: Optional[Union[Text, Dict]] = None, + name: Text = "pipeline_process", + **kwargs ) -> Dict: """Runs asynchronously a pipeline call. @@ -287,19 +349,26 @@ def run_async( Returns: Dict: polling URL in response """ - headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} + headers = { + "x-api-key": self.api_key, + "Content-Type": "application/json", + } payload = self.__prepare_payload(data=data, data_asset=data_asset) payload.update(kwargs) payload = json.dumps(payload) call_url = f"{self.url}/{self.id}" logging.info(f"Start service for {name} - {call_url} - {payload}") - r = _request_with_retry("post", call_url, headers=headers, data=payload) + r = _request_with_retry( + "post", call_url, headers=headers, data=payload + ) resp = None try: resp = r.json() - logging.info(f"Result of request for {name} - {r.status_code} - {resp}") + logging.info( + f"Result of request for {name} - {r.status_code} - {resp}" + ) poll_url = resp["url"] response = {"status": "IN_PROGRESS", "url": poll_url} @@ -309,7 +378,12 @@ def run_async( response["error"] = resp return response - def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False, api_key: Optional[Text] = None): + def update( + self, + pipeline: Union[Text, Dict], + save_as_asset: bool = False, + api_key: Optional[Text] = None, + ): """Update Pipeline Args: @@ -331,16 +405,27 @@ def update(self, pipeline: Union[Text, Dict], save_as_asset: bool = False, api_k for i, node in enumerate(pipeline["nodes"]): if "functionType" in node and node["functionType"] == "AI": - pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][ + i + ]["functionType"].lower() # prepare payload status = "draft" if save_as_asset is True: status = "onboarded" - payload = {"name": self.name, "status": status, "architecture": pipeline} + payload = { + "name": self.name, + "status": status, + "architecture": pipeline, + } url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") api_key = api_key if api_key is not None else config.TEAM_API_KEY - headers = {"Authorization": f"Token {api_key}", "Content-Type": "application/json"} - logging.info(f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}") + headers = { + "Authorization": f"Token {api_key}", + "Content-Type": "application/json", + } + logging.info( + f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}" + ) r = _request_with_retry("put", url, headers=headers, json=payload) response = r.json() logging.info(f"Pipeline {response['id']} Updated.") @@ -351,8 +436,13 @@ def delete(self) -> None: """Delete Dataset service""" try: url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} - logging.info(f"Start service for DELETE Pipeline - {url} - {headers}") + headers = { + "Authorization": f"Token {config.TEAM_API_KEY}", + "Content-Type": "application/json", + } + logging.info( + f"Start service for DELETE Pipeline - {url} - {headers}" + ) r = _request_with_retry("delete", url, headers=headers) if r.status_code != 200: raise Exception() @@ -360,3 +450,54 @@ def delete(self) -> None: message = "Pipeline Deletion Error: Make sure the pipeline exists and you are the owner." logging.error(message) raise Exception(f"{message}") + + def save( + self, save_as_asset: bool = False, api_key: Optional[Text] = None + ): + """Save Pipeline + + Args: + save_as_asset (bool, optional): Save as asset (True) or draft (False). Defaults to False. + api_key (Optional[Text], optional): Team API Key to create the Pipeline. Defaults to None. + + Raises: + Exception: Make sure the pipeline to be save is in a JSON file. + """ + try: + pipeline = self.to_dict() + + for i, node in enumerate(pipeline["nodes"]): + if "functionType" in node and node["functionType"] == "AI": + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][ + i + ]["functionType"].lower() + # prepare payload + status = "draft" + if save_as_asset is True: + status = "onboarded" + payload = { + "name": self.name, + "status": status, + "architecture": pipeline, + } + + if self.id != "": + method = "put" + url = urljoin(config.BACKEND_URL, f"sdk/pipelines/{self.id}") + else: + method = "post" + url = urljoin(config.BACKEND_URL, "sdk/pipelines") + api_key = api_key if api_key is not None else config.TEAM_API_KEY + headers = { + "Authorization": f"Token {api_key}", + "Content-Type": "application/json", + } + logging.info( + f"Start service for Save Pipeline - {url} - {headers} - {json.dumps(payload)}" + ) + r = _request_with_retry(method, url, headers=headers, json=payload) + response = r.json() + self.id = response["id"] + logging.info(f"Pipeline {response['id']} Saved.") + except Exception as e: + raise Exception(e) diff --git a/aixplain/modules/pipeline/default.py b/aixplain/modules/pipeline/default.py new file mode 100644 index 00000000..b0499159 --- /dev/null +++ b/aixplain/modules/pipeline/default.py @@ -0,0 +1,17 @@ +from .asset import Pipeline as PipelineAsset +from .designer import DesignerPipeline + + +class DefaultPipeline(PipelineAsset, DesignerPipeline): + + def __init__(self, *args, **kwargs): + PipelineAsset.__init__(self, *args, **kwargs) + DesignerPipeline.__init__(self) + + def save(self, *args, **kwargs): + self.auto_infer() + self.validate() + super().save(*args, **kwargs) + + def to_dict(self) -> dict: + return self.serialize() diff --git a/aixplain/modules/pipeline/designer/README.md b/aixplain/modules/pipeline/designer/README.md new file mode 100644 index 00000000..a1806868 --- /dev/null +++ b/aixplain/modules/pipeline/designer/README.md @@ -0,0 +1,194 @@ +# Aixplan SDK User Guide + +## Introduction + +Aixplan SDK provides a programmatic API to create pipelines for building solutions on the Aixplain platform. + +## Minimal Example + +Here's a quick example to get you started: + +```python +from aixplain.factories.pipeline_factory import PipelineFactory + +TRANSLATION_ASSET_ID = 'your-translation-asset-id' + +pipeline = PipelineFactory.init('Translation Pipeline') +input_node = pipeline.input() +translation_node = pipeline.translation(assetId=TRANSLATION_ASSET_ID) + +input_node.link(translation_node, 'input', 'text') + +output_node = translation_node.use_output('data') + +pipeline.save() +outputs = pipeline.run('This is example text to translate') + +print(outputs) +``` + +## Instantiating Nodes + +To create a pipeline and instantiate nodes, use the following code: + +```python +from aixplain.factories.pipeline_factory import PipelineFactory +from aixplain.modules.pipeline.designer import Input + +pipeline = PipelineFactory.init("My Pipeline") +input_node = Input(*args, **kwargs) +input_node.attach(pipeline) +``` + +Alternatively, add nodes to the pipeline using `add_node`: + +```python +input_node = pipeline.add_node(Input(*args, **kwargs)) +``` + +You can also pass the pipeline to the node constructor: + +```python +input_node = Input(*args, pipeline=pipeline, **kwargs) +``` + +Or directly instantiate the node within the pipeline: + +```python +input_node = pipeline.input(*args, **kwargs) +``` + +## Adding Output Nodes + +Each pipeline should have at least one input, asset, and output node. Add output nodes like any other node: + +```python +translation_node = pipeline.translation(assetId=TRANSLATION_ASSET_ID) +output_node = pipeline.output(*args, **kwargs) +translation_node.link(output_node, 'data', 'output') +``` + +For nodes implementing the `Outputable` mixin, use the shortcut syntax: + +```python +output_node = translation_node.use_output('parameter_name_we_are_interested_in') +``` + +## Asset Nodes and Automatic Population + +Asset nodes are used to run models and should have an asset ID. Once instantiated, an asset node contains all model information and parameters which is populated automatically by interacting with the Aixplain platform. + +```python +translation_node = pipeline.translation(assetId=TRANSLATION_ASSET_ID) +print(translation_node.inputs) +print(translation_node.outputs) +``` + +## Handling Parameters + +Parameters are accessed via the `inputs` and `outputs` attributes of the node, behaving as proxy objects to the parameters. + +```python +print(translation_node.inputs.text) +print(translation_node.outputs.data) +``` + +Add parameters to a node using `create_param` on corresponding `inputs` or `outputs` attribute: + +```python +translation_node.inputs.create_param('source_language', DataType.TEXT) +translation_node.outputs.create_param('source_audio', DataType.AUDIO) +``` + +Alternatively, instantiate parameters directly using `InputParam` or `OutputParam` classes: + +```python +from aixplain.modules.pipeline.designer import InputParam, OutputParam + +source_language = InputParam( + code='source_language', + dataType=DataType.TEXT, + is_required=True, + node=translation_node +) +``` + +Or add parameters explicitly: + +```python +source_audio = OutputParam(dataType=DataType.AUDIO, code='source_audio') +translation_node.outputs.add_param(source_audio) +``` + +In case of need, any parameter value can be set directly without requiring node linking: + +```python +translation_node.inputs.text = 'This is example text to translate' +translation_node.inputs.source_language = 'en' +``` + +This will implicity set the `value` attribute of the parameter object. + +## Linking Nodes + +Link nodes to pass data between them using the `link` method. This method links the output of one node to the input of another on specified parameters. + +Consider the following nodes: + +```python +input_node = pipeline.input() +translation_node = pipeline.translation(assetId=TRANSLATION_ASSET_ID) +``` + +Link nodes together: + +```python +input_node.link(translation_node, 'input', 'text') +``` + +Specify parameters explicitly: + +```python +input_node.link(translation_node, from_param='input', to_param='text') +``` + +Or use parameter instances: + +```python +input_node.link(translation_node, from_param=input_node.outputs.input, to_param=translation_node.inputs.text) +``` + +You can also link parameters directly if you find it more convenient: + +```python +input_node.outputs.input.link(translation_node.inputs.text) +``` + +## Validating the Pipeline + +Use the `validate` method to ensure the pipeline is valid and ready to run. This method raises an exception if the pipeline has issues. + +```python +pipeline.validate() +``` + +This method will check the following: + * Contains at least one input, asset, and output node + * All input nodes are linked in, output nodes are linked out, and rest are linked in and out + * All links pointing to the correct nodes and corresponding params. + * All required params are either set or linked + * All linked params have the same data type + +Otherwise raises `ValueError` with a cause if the pipeline is not valid. + +## Save and Run the Pipeline + +Save the pipeline before running it. The `save` method implicitly calls the `validate` method. Use the `run` method to execute the pipeline with input data. + +```python +pipeline.save() # Raises an exception if there are semantic issues +outputs = pipeline.run('This is example text to translate') +print(outputs) +``` + +This guide covers the basic usage of the programmatic api of Aixplan SDK for creating and running pipelines. For more advanced features, refer to the code itself. \ No newline at end of file diff --git a/aixplain/modules/pipeline/designer/__init__.py b/aixplain/modules/pipeline/designer/__init__.py new file mode 100644 index 00000000..0bb56542 --- /dev/null +++ b/aixplain/modules/pipeline/designer/__init__.py @@ -0,0 +1,66 @@ +from .nodes import ( + AssetNode, + Decision, + Script, + Input, + Output, + Route, + Router, + BaseReconstructor, + BaseSegmentor, +) +from .pipeline import DesignerPipeline +from .base import ( + Node, + Link, + Param, + ParamProxy, + InputParam, + OutputParam, + Inputs, + Outputs, + TI, + TO, +) +from .enums import ( + ParamType, + RouteType, + Operation, + NodeType, + AssetType, + FunctionType, +) +from .mixins import LinkableMixin, OutputableMixin, RoutableMixin + + +__all__ = [ + "DesignerPipeline", + "AssetNode", + "Decision", + "Script", + "Input", + "Output", + "Route", + "Router", + "BaseReconstructor", + "BaseSegmentor", + "Node", + "Link", + "Param", + "ParamType", + "InputParam", + "OutputParam", + "RouteType", + "Operation", + "NodeType", + "AssetType", + "FunctionType", + "LinkableMixin", + "OutputableMixin", + "RoutableMixin", + "Inputs", + "Outputs", + "ParamProxy", + "TI", + "TO", +] diff --git a/aixplain/modules/pipeline/designer/base.py b/aixplain/modules/pipeline/designer/base.py new file mode 100644 index 00000000..8bea73d6 --- /dev/null +++ b/aixplain/modules/pipeline/designer/base.py @@ -0,0 +1,375 @@ +from typing import ( + List, + Union, + TYPE_CHECKING, + Generic, + TypeVar, + Type, + Optional, + Iterator, +) + +from aixplain.enums import DataType +from .enums import NodeType, ParamType + + +if TYPE_CHECKING: + from .pipeline import DesignerPipeline + +TI = TypeVar("TI", bound="Inputs") +TO = TypeVar("TO", bound="Outputs") + + +class Serializable: + def serialize(self) -> dict: + raise NotImplementedError() + + +class Param(Serializable): + """ + Param class, this class will be used to create the parameters of the node. + """ + + code: str + param_type: ParamType + data_type: Optional[DataType] = None + value: Optional[str] = None + node: Optional["Node"] = None + link_: Optional["Link"] = None + + def __init__( + self, + code: str, + data_type: Optional[DataType] = None, + value: Optional[str] = None, + node: Optional["Node"] = None, + param_type: Optional[ParamType] = None, + ): + self.code = code + self.data_type = data_type + self.value = value + + # is subclasses do not set the param type, set it to None + self.param_type = getattr(self, "param_type", param_type) + + if node: + self.attach_to(node) + + def attach_to(self, node: "Node") -> "Param": + """ + Attach the param to the node. + :param node: the node + :return: the param + """ + assert not self.node, "Param already attached to a node" + assert self.param_type, "Param type not set" + if self.param_type == ParamType.INPUT: + node.inputs.add_param(self) + elif self.param_type == ParamType.OUTPUT: + node.outputs.add_param(self) + else: + raise ValueError("Invalid param type") + self.node = node + return self + + def link(self, to_param: "Param") -> "Param": + """ + Link the output of the param to the input of another param. + :param to_param: the input param + :return: the param + """ + assert self.node, "Param not attached to a node" + assert to_param.param_type == ParamType.INPUT, "Invalid param type" + assert self in self.node.outputs, "Param not registered as output" + return to_param.back_link(self) + + def back_link(self, from_param: "Param") -> "Param": + """ + Link the input of the param to the output of another param. + :param from_param: the output param + :return: the param + """ + assert self.node, "Param not attached to a node" + assert from_param.param_type == ParamType.OUTPUT, "Invalid param type" + assert self.code in self.node.inputs, "Param not registered as input" + link = from_param.node.link(self.node, from_param, self) + self.link_ = link + from_param.link_ = link + return link + + def serialize(self) -> dict: + return { + "code": self.code, + "dataType": self.data_type, + "value": self.value, + } + + +class InputParam(Param): + + param_type: ParamType = ParamType.INPUT + is_required: bool = True + + def __init__(self, *args, is_required: bool = True, **kwargs): + super().__init__(*args, **kwargs) + self.is_required = is_required + + +class OutputParam(Param): + + param_type: ParamType = ParamType.OUTPUT + + +class Link(Serializable): + """ + Link class, this class will be used to link the output of the node to the + input of another node. + """ + + from_node: "Node" + to_node: "Node" + from_param: str + to_param: str + + pipeline: Optional["DesignerPipeline"] = None + + def __init__( + self, + from_node: "Node", + to_node: "Node", + from_param: Union[Param, str], + to_param: Union[Param, str], + pipeline: "DesignerPipeline" = None, + ): + + assert from_param in from_node.outputs, "Invalid from param" + assert to_param in to_node.inputs, "Invalid to param" + + if isinstance(from_param, Param): + from_param = from_param.code + if isinstance(to_param, Param): + to_param = to_param.code + + self.from_node = from_node + self.to_node = to_node + self.from_param = from_param + self.to_param = to_param + + if pipeline: + self.attach_to(pipeline) + + # self.validate() + self.auto_infer() + + def auto_infer(self): + from_param = self.from_node.outputs[self.from_param] + to_param = self.to_node.inputs[self.to_param] + + # if one of the data types is missing, infer the other one + data_type = from_param.data_type or to_param.data_type + from_param.data_type = data_type + to_param.data_type = data_type + + def infer_data_type(node): + from .nodes import Input, Output + + if isinstance(node, Input) or isinstance(node, Output): + if data_type and data_type not in node.data_types: + node.data_types.append(data_type) + + infer_data_type(self.from_node) + infer_data_type(self.to_node) + + def validate(self): + from_param = self.from_node.outputs[self.from_param] + to_param = self.to_node.inputs[self.to_param] + + # Should we check for data type mismatch? + if from_param.data_type and to_param.data_type: + if from_param.data_type != to_param.data_type: + raise ValueError( + f"Data type mismatch between {from_param.data_type} and {to_param.data_type}" + ) # noqa + + def attach_to(self, pipeline: "DesignerPipeline"): + """ + Attach the link to the pipeline. + :param pipeline: the pipeline + """ + assert not self.pipeline, "Link already attached to a pipeline" + if not self.from_node.pipeline or self.from_node not in pipeline.nodes: + self.from_node.attach_to(pipeline) + if not self.to_node.pipeline or self.to_node not in pipeline.nodes: + self.to_node.attach_to(pipeline) + + self.pipeline = pipeline + self.pipeline.links.append(self) + return self + + def serialize(self) -> dict: + assert self.from_node.number is not None, "From node number not set" + assert self.to_node.number is not None, "To node number not set" + return { + "from": self.from_node.number, + "to": self.to_node.number, + "paramMapping": [ + { + "from": self.from_param, + "to": self.to_param, + } + ], + } + + +class ParamProxy(Serializable): + + node: "Node" + + def __init__(self, node: "Node", *args, **kwargs): + super().__init__(*args, **kwargs) + self.node = node + self._params = [] + + def add_param(self, param: Param) -> None: + # check if param already registered + if param in self: + raise ValueError( + f"Parameter with code '{param.code}' already exists." + ) + self._params.append(param) + # also set attribute on the node dynamically if there's no + # any attribute with the same name + if not hasattr(self, param.code): + setattr(self, param.code, param) + + def _create_param( + self, code: str, data_type: DataType = None, value: any = None + ) -> Param: + raise NotImplementedError() + + def create_param( + self, + code: str, + data_type: DataType = None, + value: any = None, + is_required: bool = False, + ) -> Param: + param = self._create_param(code, data_type, value) + param.is_required = is_required + self.add_param(param) + param.node = self.node + return param + + def __getitem__(self, code: str) -> Param: + for param in self._params: + if param.code == code: + return param + raise KeyError(f"Parameter with code '{code}' not found.") + + def __setitem__(self, code: str, value: str) -> None: + # set param value on set item to avoid setting it manually + self[code].value = value + + def __setattr__(self, name: str, value: any) -> None: + # set param value on attribute assignment to avoid setting it manually + if isinstance(value, str) and hasattr(self, name): + self[name].value = value + else: + super().__setattr__(name, value) + + def __contains__(self, param: Union[str, Param]) -> bool: + code = param if isinstance(param, str) else param.code + return any(param.code == code for param in self._params) + + def __iter__(self) -> Iterator[Param]: + return iter(self._params) + + def __len__(self) -> int: + return len(self._params) + + def serialize(self) -> List[dict]: + return [param.serialize() for param in self._params] + + +class Inputs(ParamProxy): + def _create_param( + self, + code: str, + data_type: DataType = None, + value: any = None, + is_required: bool = False, + ) -> InputParam: + return InputParam( + code=code, + data_type=data_type, + value=value, + is_required=is_required, + ) + + +class Outputs(ParamProxy): + def _create_param( + self, code: str, data_type: DataType = None, value: any = None + ) -> OutputParam: + return OutputParam(code=code, data_type=data_type, value=value) + + +class Node(Generic[TI, TO], Serializable): + """ + Node class is the base class for all the nodes in the pipeline. This class + will be used to create the nodes and link them together. + """ + + number: Optional[int] = None + label: Optional[str] = None + type: Optional[NodeType] = None + + inputs: Optional[TI] = None + outputs: Optional[TO] = None + inputs_class: Optional[Type[TI]] = Inputs + outputs_class: Optional[Type[TO]] = Outputs + pipeline: Optional["DesignerPipeline"] = None + + def __init__( + self, + pipeline: "DesignerPipeline" = None, + number: Optional[int] = None, + label: Optional[str] = None, + ): + self.inputs = self.inputs_class(node=self) + self.outputs = self.outputs_class(node=self) + self.number = number + self.label = label + + if pipeline: + self.attach_to(pipeline) + + def attach_to(self, pipeline: "DesignerPipeline"): + """ + Attach the node to the pipeline. + :param pipeline: the pipeline + """ + assert not self.pipeline, "Node already attached to a pipeline" + assert ( + self not in pipeline.nodes + ), "Node already attached to a pipeline" + assert self.type, "Node type not set" + + self.pipeline = pipeline + if self.number is None: + self.number = len(pipeline.nodes) + if self.label is None: + self.label = f"{self.type.value}(ID={self.number})" + + assert not pipeline.get_node(self.number), "Node number already exists" + pipeline.nodes.append(self) + return self + + def serialize(self) -> dict: + return { + "number": self.number, + "label": self.label, + "type": self.type.value, + "inputValues": self.inputs.serialize(), + "outputValues": self.outputs.serialize(), + } diff --git a/aixplain/modules/pipeline/designer/enums.py b/aixplain/modules/pipeline/designer/enums.py new file mode 100644 index 00000000..4c044dba --- /dev/null +++ b/aixplain/modules/pipeline/designer/enums.py @@ -0,0 +1,43 @@ +from enum import Enum + + +class RouteType(str, Enum): + CHECK_TYPE = "checkType" + CHECK_VALUE = "checkValue" + + +class Operation(str, Enum): + GREATER_THAN = "greaterThan" + GREATER_THAN_OR_EQUAL = "greaterThanOrEqual" + LESS_THAN = "lessThan" + LESS_THAN_OR_EQUAL = "lessThanOrEqual" + EQUAL = "equal" + DIFFERENT = "different" + CONTAIN = "contain" + NOT_CONTAIN = "notContain" + + +class NodeType(str, Enum): + ASSET = "ASSET" + INPUT = "INPUT" + OUTPUT = "OUTPUT" + SCRIPT = "SCRIPT" + SEGMENTOR = "SEGMENT" + RECONSTRUCTOR = "RECONSTRUCT" + ROUTER = "ROUTER" + DECISION = "DECISION" + + +class AssetType(str, Enum): + MODEL = "MODEL" + + +class FunctionType(str, Enum): + AI = "AI" + SEGMENTOR = "SEGMENTOR" + RECONSTRUCTOR = "RECONSTRUCTOR" + + +class ParamType: + INPUT = "INPUT" + OUTPUT = "OUTPUT" diff --git a/aixplain/modules/pipeline/designer/mixins.py b/aixplain/modules/pipeline/designer/mixins.py new file mode 100644 index 00000000..e5aad3c4 --- /dev/null +++ b/aixplain/modules/pipeline/designer/mixins.py @@ -0,0 +1,85 @@ +from typing import Union +from .base import Node, Link, Param + + +class LinkableMixin: + """ + Linkable mixin class, this class will be used to link the output of the + node to the input of another node. + + This class will be used to link the output of the node to the input of + another node. + """ + + def link( + self, + to_node: Node, + from_param: Union[str, Param], + to_param: Union[str, Param], + ) -> Link: + """ + Link the output of the node to the input of another node. This method + will link the output of the node to the input of another node. + + :param to_node: the node to link to the output + :param from_param: the output parameter or the code of the output + parameter + :param to_param: the input parameter or the code of the input parameter + :return: the link + """ + return Link( + pipeline=self.pipeline, + from_node=self, + to_node=to_node, + from_param=from_param, + to_param=to_param, + ) + + +class RoutableMixin: + """ + Routable mixin class, this class will be used to route the input data to + different nodes based on the input data type. + """ + + def route(self, *params: Param) -> Node: + """ + Route the input data to different nodes based on the input data type. + This method will automatically link the input data to the output data + of the node. + + :param params: the output parameters + :return: the router node + """ + assert self.pipeline, "Node not attached to a pipeline" + + router = self.pipeline.router( + [(param.data_type, param.node) for param in params] + ) + self.outputs.input.link(router.inputs.input) + for param in params: + router.outputs.input.link(param) + return router + + +class OutputableMixin: + """ + Outputable mixin class, this class will be used to link the output of the + node to the output node of the pipeline. + """ + + def use_output(self, param: Union[str, Param]) -> Node: + """ + Use the output of the node as the output of the pipeline. + This method will automatically link the output of the node to the + output node of the pipeline. + + :param param: the output parameter or the code of the output parameter + :return: the output node + """ + assert self.pipeline, "Node not attached to a pipeline" + output = self.pipeline.output() + if isinstance(param, str): + param = self.outputs[param] + param.link(output.inputs.output) + return output diff --git a/aixplain/modules/pipeline/designer/nodes.py b/aixplain/modules/pipeline/designer/nodes.py new file mode 100644 index 00000000..22152239 --- /dev/null +++ b/aixplain/modules/pipeline/designer/nodes.py @@ -0,0 +1,464 @@ +from typing import List, Union, Type, TYPE_CHECKING, Optional + +from aixplain.modules import Model +from aixplain.enums import DataType + +from .enums import ( + NodeType, + FunctionType, + RouteType, + Operation, + AssetType, +) +from .base import ( + Node, + Link, + Param, + InputParam, + OutputParam, + TI, + TO, + Inputs, + Outputs, + Serializable, +) +from .mixins import LinkableMixin, OutputableMixin, RoutableMixin + +if TYPE_CHECKING: + from .pipeline import DesignerPipeline + + +class AssetNode(Node[TI, TO], LinkableMixin, OutputableMixin): + """ + Asset node class, this node will be used to fetch the asset from the + aixplain platform and use it in the pipeline. + + `assetId` is required and will be used to fetch the asset from the + aixplain platform. + + Input and output parameters will be automatically added based on the + asset function spec. + """ + + asset_id: Union[Model, str] = None + function: str = None + supplier: str = None + version: str = None + assetType: AssetType = AssetType.MODEL + functionType: FunctionType = FunctionType.AI + + type: NodeType = NodeType.ASSET + + def __init__( + self, + asset_id: Union[Model, str] = None, + supplier: str = None, + version: str = None, + pipeline: "DesignerPipeline" = None, + ): + super().__init__(pipeline=pipeline) + self.asset_id = asset_id + self.supplier = supplier + self.version = version + + if self.asset_id: + self.populate_asset() + + def populate_asset(self): + from aixplain.factories.model_factory import ModelFactory + + if isinstance(self.asset_id, str): + self.asset = ModelFactory.get(self.asset_id) + elif isinstance(self.asset_id, Model): + self.asset = self.asset_id + self.asset_id = self.asset_id.id + else: + raise ValueError("assetId should be a string or an Asset instance") + + try: + self.supplier = self.asset.supplier.value["code"] + except Exception: + self.supplier = str(self.asset.supplier) + + self.version = self.asset.version + + if self.function: + if self.asset.function.value != self.function: + raise ValueError( + f"Function {self.function} is not supported by asset {self.asset_id}" + ) # noqa + else: + self.function = self.asset.function.value + self._auto_populate_params() + + self._auto_set_params() + + def _auto_populate_params(self): + from aixplain.enums.function import FunctionInputOutput + + spec = FunctionInputOutput[self.asset.function.value]["spec"] + for item in spec["params"]: + self.inputs.create_param( + code=item["code"], + data_type=item["dataType"], + is_required=item["required"], + ) + + for item in spec["output"]: + self.outputs.create_param( + code=item["code"], + data_type=item["dataType"], + ) + + def _auto_set_params(self): + for k, v in self.asset.additional_info["parameters"].items(): + if isinstance(v, list): + self.inputs[k] = v[0] + else: + self.inputs[k] = v + + def serialize(self) -> dict: + obj = super().serialize() + obj["function"] = self.function + obj["assetId"] = self.asset_id + obj["supplier"] = self.supplier + obj["version"] = self.version + obj["assetType"] = self.assetType + obj["functionType"] = self.functionType + obj["type"] = self.type + return obj + + +class InputInputs(Inputs): + pass + + +class InputOutputs(Outputs): + input: OutputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.input = self.create_param("input") + + +class Input(Node[InputInputs, InputOutputs], LinkableMixin, RoutableMixin): + """ + Input node class, this node will be used to input the data to the + pipeline. + + Input nodes has only one output parameter called `input`. + + `data` is a special convenient parameter that will be uploaded to the + aixplain platform and the link will be passed as the input to the node. + """ + + data_types: Optional[List[DataType]] = None + data: Optional[str] = None + type: NodeType = NodeType.INPUT + inputs_class: Type[TI] = InputInputs + outputs_class: Type[TO] = InputOutputs + + def __init__( + self, + data: Optional[str] = None, + data_types: Optional[List[DataType]] = None, + pipeline: "DesignerPipeline" = None, + ): + from aixplain.factories.file_factory import FileFactory + + super().__init__(pipeline=pipeline) + self.data_types = data_types or [] + self.data = data + + if self.data: + self.data = FileFactory.to_link(self.data, is_temp=True) + + def serialize(self) -> dict: + obj = super().serialize() + obj["data"] = self.data + obj["dataType"] = self.data_types + return obj + + +class OutputInputs(Inputs): + output: InputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.output = self.create_param("output") + + +class OutputOutputs(Outputs): + pass + + +class Output(Node[OutputInputs, OutputOutputs]): + """ + Output node class, this node will be used to output the result of the + pipeline. + + Output nodes has only one input parameter called `output`. + """ + + data_types: Optional[List[DataType]] = None + type: NodeType = NodeType.OUTPUT + inputs_class: Type[TI] = OutputInputs + outputs_class: Type[TO] = OutputOutputs + + def __init__( + self, + data_types: Optional[List[DataType]] = None, + pipeline: "DesignerPipeline" = None, + ): + super().__init__(pipeline=pipeline) + self.data_types = data_types or [] + + def serialize(self) -> dict: + obj = super().serialize() + obj["dataType"] = self.data_types + return obj + + +class Script(Node[TI, TO], LinkableMixin, OutputableMixin): + """ + Script node class, this node will be used to run a script on the input + data. + + `script_path` is a special convenient parameter that will be uploaded to + the aixplain platform and the link will be passed as the input to the node. + """ + + fileId: Optional[str] = None + script_path: Optional[str] = None + type: NodeType = NodeType.SCRIPT + + def __init__( + self, + pipeline: "DesignerPipeline" = None, + script_path: Optional[str] = None, + fileId: Optional[str] = None, + ): + from aixplain.factories.script_factory import ScriptFactory + + super().__init__(pipeline=pipeline) + + assert script_path or fileId, "script_path or fileId is required" + + if not fileId: + self.fileId = ScriptFactory.upload_script(script_path) + else: + self.fileId = fileId + + def serialize(self) -> dict: + obj = super().serialize() + obj["fileId"] = self.fileId + return obj + + +class Route(Serializable): + """ + Route class, this class will be used to route the input data to different + nodes based on the input data type. + """ + + value: DataType + path: List[Union[Node, int]] + operation: Operation + type: RouteType + + def __init__( + self, + value: DataType, + path: List[Union[Node, int]], + operation: Operation, + type: RouteType, + ): + """ + Post init method to convert the nodes to node numbers if they are + nodes. + """ + self.value = value + self.path = path + self.operation = operation + self.type = type + + if not self.path: + raise ValueError("Path is not valid, should be a list of nodes") + + # convert nodes to node numbers if they are nodes + self.path = [ + node.number if isinstance(node, Node) else node + for node in self.path + ] + + def serialize(self) -> dict: + return { + "value": self.value, + "path": self.path, + "operation": self.operation, + "type": self.type, + } + + +class RouterInputs(Inputs): + input: InputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.input = self.create_param("input") + + +class RouterOutputs(Outputs): + input: OutputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.input = self.create_param("input") + + +class Router(Node[RouterInputs, RouterOutputs], LinkableMixin): + """ + Router node class, this node will be used to route the input data to + different nodes based on the input data type. + """ + + routes: List[Route] + type: NodeType = NodeType.ROUTER + inputs_class: Type[TI] = RouterInputs + outputs_class: Type[TO] = RouterOutputs + + def __init__( + self, routes: List[Route], pipeline: "DesignerPipeline" = None + ): + super().__init__(pipeline=pipeline) + self.routes = routes + + def serialize(self) -> dict: + obj = super().serialize() + obj["routes"] = [route.serialize() for route in self.routes] + return obj + + +class DecisionInputs(Inputs): + comparison: InputParam = None + passthrough: InputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.comparison = self.create_param("comparison") + self.passthrough = self.create_param("passthrough") + + +class DecisionOutputs(Outputs): + input: OutputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.input = self.create_param("input") + + +class Decision(Node[DecisionInputs, DecisionOutputs], LinkableMixin): + """ + Decision node class, this node will be used to make decisions based on + the input data. + """ + + routes: List[Route] + type: NodeType = NodeType.DECISION + inputs_class: Type[TI] = DecisionInputs + outputs_class: Type[TO] = DecisionOutputs + + def __init__( + self, routes: List[Route], pipeline: "DesignerPipeline" = None + ): + super().__init__(pipeline=pipeline) + self.routes = routes + + def link( + self, + to_node: Node, + from_param: Union[str, Param], + to_param: Union[str, Param], + ) -> Link: + link = super().link(to_node, from_param, to_param) + self.outputs.input.data_type = self.inputs.passthrough.data_type + return link + + def serialize(self) -> dict: + obj = super().serialize() + obj["routes"] = [route.serialize() for route in self.routes] + return obj + + +class BaseSegmentor(AssetNode[TI, TO]): + """ + Segmentor node class, this node will be used to segment the input data + into smaller fragments for much easier and efficient processing. + """ + + type: NodeType = NodeType.SEGMENTOR + functionType: FunctionType = FunctionType.SEGMENTOR + + +class SegmentorInputs(Inputs): + pass + + +class SegmentorOutputs(Outputs): + audio: OutputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.audio = self.create_param("audio") + + +class BareSegmentor(BaseSegmentor[SegmentorInputs, SegmentorOutputs]): + """ + Segmentor node class, this node will be used to segment the input data + into smaller fragments for much easier and efficient processing. + """ + + type: NodeType = NodeType.SEGMENTOR + functionType: FunctionType = FunctionType.SEGMENTOR + inputs_class: Type[TI] = SegmentorInputs + outputs_class: Type[TO] = SegmentorOutputs + + +class BaseReconstructor(AssetNode[TI, TO]): + """ + Reconstructor node class, this node will be used to reconstruct the + output of the segmented lines of execution. + """ + + type: NodeType = NodeType.RECONSTRUCTOR + functionType: FunctionType = FunctionType.RECONSTRUCTOR + + +class ReconstructorInputs(Inputs): + data: InputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.data = self.create_param("data") + + +class ReconstructorOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node: Node): + super().__init__(node) + self.data = self.create_param("data") + + +class BareReconstructor( + BaseReconstructor[ReconstructorInputs, ReconstructorOutputs] +): + """ + Reconstructor node class, this node will be used to reconstruct the + output of the segmented lines of execution. + """ + + type: NodeType = NodeType.RECONSTRUCTOR + functionType: FunctionType = FunctionType.RECONSTRUCTOR + inputs_class: Type[TI] = ReconstructorInputs + outputs_class: Type[TO] = ReconstructorOutputs diff --git a/aixplain/modules/pipeline/designer/pipeline.py b/aixplain/modules/pipeline/designer/pipeline.py new file mode 100644 index 00000000..5304d202 --- /dev/null +++ b/aixplain/modules/pipeline/designer/pipeline.py @@ -0,0 +1,328 @@ +from typing import List, Type, Tuple, TypeVar + +from aixplain.enums import DataType + +from .base import Serializable, Node, Link +from .nodes import ( + AssetNode, + Decision, + Script, + Input, + Output, + Router, + Route, + BareReconstructor, + BareSegmentor, +) +from .enums import NodeType, RouteType, Operation + + +T = TypeVar("T", bound="AssetNode") + + +class DesignerPipeline(Serializable): + nodes: List[Node] = None + links: List[Link] = None + instance: any = None + + def __init__(self): + self.nodes = [] + self.links = [] + + def add_node(self, node: Node): + """ + Add a node to the current pipeline. + + This method will take care of setting the pipeline instance to the + node and setting the node number if it's not set. + + :param node: the node + :return: the node + """ + return node.attach_to(self) + + def add_nodes(self, *nodes: Node) -> List[Node]: + """ + Add multiple nodes to the current pipeline. + + :param nodes: the nodes + :return: the nodes + """ + return [self.add_node(node) for node in nodes] + + def add_link(self, link: Link) -> Link: + """ + Add a link to the current pipeline. + :param link: the link + :return: the link + """ + return link.attach_to(self) + + def serialize(self) -> dict: + """ + Serialize the pipeline to a dictionary. This method will serialize the + pipeline to a dictionary. + + :return: the pipeline as a dictionary + """ + return { + "nodes": [node.serialize() for node in self.nodes], + "links": [link.serialize() for link in self.links], + } + + def validate_nodes(self): + """ + Validate the linkage of the pipeline. This method will validate the + linkage of the pipeline by applying the following checks: + - All input nodes are linked out + - All output nodes are linked in + - All other nodes are linked in and out + + :raises ValueError: if the pipeline is not valid + """ + link_from_map = {link.from_node.number: link for link in self.links} + link_to_map = {link.to_node.number: link for link in self.links} + contains_input = False + contains_output = False + contains_asset = False + for node in self.nodes: + # validate every input node is linked out + if node.type == NodeType.INPUT: + contains_input = True + if node.number not in link_from_map: + raise ValueError(f"Input node {node.label} not linked out") + # validate every output node is linked in + elif node.type == NodeType.OUTPUT: + contains_output = True + if node.number not in link_to_map: + raise ValueError(f"Output node {node.label} not linked in") + # validate rest of the nodes are linked in and out + else: + if isinstance(node, AssetNode): + contains_asset = True + if node.number not in link_from_map: + raise ValueError(f"Node {node.label} not linked in") + if node.number not in link_to_map: + raise ValueError(f"Node {node.label} not linked out") + + if not contains_input or not contains_output or not contains_asset: + raise ValueError( + "Pipeline must contain at least one input, output and asset node" # noqa + ) + + def is_param_linked(self, node, param): + """ + Check if the param is linked to another node. This method will check + if the param is linked to another node. + :param node: the node + :param param: the param + :return: True if the param is linked, False otherwise + """ + for link in self.links: + if ( + link.to_node.number == node.number + and param.code == link.to_param + ): + return True + + return False + + def is_param_set(self, node, param): + """ + Check if the param is set. This method will check if the param is set + or linked to another node. + :param node: the node + :param param: the param + :return: True if the param is set, False otherwise + """ + return param.value or self.is_param_linked(node, param) + + def validate_params(self): + """ + This method will check if all required params are either set or linked + + :raises ValueError: if the pipeline is not valid + """ + for node in self.nodes: + for param in node.inputs: + if param.is_required and not self.is_param_set(node, param): + raise ValueError( + f"Param {param.code} of node {node.label} is required" + ) + + def validate(self): + """ + Validate the pipeline. This method will validate the pipeline by + series of checks: + - Validate all nodes are linked correctly + - Validate all required params are set or linked + + Any other validation checks can be added here. + + :raises ValueError: if the pipeline is not valid + """ + self.validate_nodes() + self.validate_params() + + def get_link(self, from_node: int, to_node: int) -> Link: + """ + Get the link between two nodes. This method will return the link + between two nodes. + + :param from_node: the from node number + :param to_node: the to node number + :return: the link + """ + return next( + ( + link + for link in self.links + if link.from_node == from_node and link.to_node == to_node + ), + None, + ) + + def get_node(self, node_number: int) -> Node: + """ + Get the node by its number. This method will return the node with the + given number. + + :param node_number: the node number + :return: the node + """ + return next( + (node for node in self.nodes if node.number == node_number), None + ) + + def auto_infer(self): + """ + Automatically infer the data types of the nodes in the pipeline. + This method will automatically infer the data types of the nodes in the + pipeline by traversing the pipeline and setting the data types of the + nodes based on the data types of the connected nodes. + """ + for link in self.links: + from_node = self.get_node(link.from_node) + to_node = self.get_node(link.to_node) + if not from_node or not to_node: + continue # will be handled by the validation + for param in link.param_mapping: + from_param = from_node.outputs[param.from_param] + to_param = to_node.inputs[param.to_param] + if not from_param or not to_param: + continue # will be handled by the validation + # if one of the data types is missing, infer the other one + dataType = from_param.data_type or to_param.data_type + from_param.data_type = dataType + to_param.data_type = dataType + + def infer_data_type(node): + from .nodes import Input, Output + + if isinstance(node, Input) or isinstance(node, Output): + if dataType and dataType not in node.data_types: + node.data_types.append(dataType) + + infer_data_type(self) + infer_data_type(to_node) + + def asset( + self, asset_id: str, *args, asset_class: Type[T] = AssetNode, **kwargs + ) -> T: + """ + Shortcut to create an asset node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return asset_class(asset_id, *args, pipeline=self, **kwargs) + + def decision(self, *args, **kwargs) -> Decision: + """ + Shortcut to create an decision node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return Decision(*args, pipeline=self, **kwargs) + + def script(self, *args, **kwargs) -> Script: + """ + Shortcut to create an script node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return Script(*args, pipeline=self, **kwargs) + + def input(self, *args, **kwargs) -> Input: + """ + Shortcut to create an input node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return Input(*args, pipeline=self, **kwargs) + + def output(self, *args, **kwargs) -> Output: + """ + Shortcut to create an output node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return Output(*args, pipeline=self, **kwargs) + + def router(self, routes: Tuple[DataType, Node], *args, **kwargs) -> Router: + """ + Shortcut to create an decision node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. The routes will be handled specially and will be + converted to Route instances in a convenient way. + + :param routes: the routes + :param kwargs: keyword arguments + :return: the node + """ + kwargs["routes"] = [ + Route( + value=route[0], + path=[route[1]], + type=RouteType.CHECK_TYPE, + operation=Operation.EQUAL, + ) + for route in routes + ] + return Router(*args, pipeline=self, **kwargs) + + def bare_reconstructor(self, *args, **kwargs) -> BareReconstructor: + """ + Shortcut to create an reconstructor node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return BareReconstructor(*args, pipeline=self, **kwargs) + + def bare_segmentor(self, *args, **kwargs) -> BareSegmentor: + """ + Shortcut to create an segmentor node for the current pipeline. + All params will be passed as keyword arguments to the node + constructor. + + :param kwargs: keyword arguments + :return: the node + """ + return BareSegmentor(*args, pipeline=self, **kwargs) diff --git a/aixplain/modules/pipeline/generate.py b/aixplain/modules/pipeline/generate.py new file mode 100644 index 00000000..c71e8ae6 --- /dev/null +++ b/aixplain/modules/pipeline/generate.py @@ -0,0 +1,227 @@ +import pathlib + +import requests +from urllib.parse import urljoin +from jinja2 import Environment, BaseLoader + +from aixplain.utils import config + +SEGMENTOR_FUNCTIONS = [ + "split-on-linebreak", + "speaker-diarization-audio", + "voice-activity-detection", +] + +RECONSTRUCTOR_FUNCTIONS = ["text-reconstruction", "audio-reconstruction"] + +MODULE_NAME = "pipeline" +TEMPLATE = """# This is an auto generated module. PLEASE DO NOT EDIT + + +from typing import Union, Type +from aixplain.enums import DataType + +from .designer import ( + InputParam, + OutputParam, + Inputs, + Outputs, + TI, + TO, + AssetNode, + BaseReconstructor, + BaseSegmentor, +) +from .default import DefaultPipeline +from aixplain.modules import asset + +{% for spec in specs %} + +class {{ spec.class_name }}Inputs(Inputs): +{% for input in spec.inputs %} + {{ input.name }}: InputParam = None +{% endfor %} + + def __init__(self, node=None): + super().__init__(node=node) +{% for input in spec.inputs %} + self.{{ input.name }} = self.create_param(code="{{ input.name }}", data_type=DataType.{{ input.data_type | upper }}, is_required={{ input.is_required }}) +{% endfor %} + + +class {{ spec.class_name }}Outputs(Outputs): +{% for output in spec.outputs %} + {{ output.name }}: OutputParam = None +{% endfor %} +{% if spec.is_segmentor %} + audio: OutputParam = None +{% endif %} + + def __init__(self, node=None): + super().__init__(node=node) +{% for output in spec.outputs %} + self.{{ output.name }} = self.create_param(code="{{ output.name }}", data_type=DataType.{{ output.data_type | upper }}) +{% endfor %} +{% if spec.is_segmentor %} + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO) +{% endif %} + + +class {{ spec.class_name }}({{spec.base_class}}[{{ spec.class_name }}Inputs, {{ spec.class_name }}Outputs]): + \"\"\" + {{ spec.description | wordwrap }} + + InputType: {{ spec.input_type }} + OutputType: {{ spec.output_type }} + \"\"\" + function: str = "{{ spec.id }}" + input_type: str = DataType.{{ spec.input_type | upper }} + output_type: str = DataType.{{ spec.output_type | upper }} + + inputs_class: Type[TI] = {{ spec.class_name }}Inputs + outputs_class: Type[TO] = {{ spec.class_name }}Outputs + +{% endfor %} + + +class Pipeline(DefaultPipeline): + +{% for spec in specs %} + def {{ spec.function_name }}(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> {{ spec.class_name }}: + \"\"\" + {{ spec.description | wordwrap }} + \"\"\" + return {{ spec.class_name }}(*args, asset_id=asset_id, pipeline=self, **kwargs) + +{% endfor %} +""" + + +def fetch_functions(): + """ + Fetch functions from the backend + """ + api_key = config.TEAM_API_KEY + aixplain_key = config.AIXPLAIN_API_KEY + backend_url = config.BACKEND_URL + + url = urljoin(backend_url, "sdk/functions") + headers = { + "Content-Type": "application/json", + } + + if aixplain_key: + headers["x-aixplain-key"] = aixplain_key + else: + headers["x-api-key"] = api_key + + r = requests.get(url, headers=headers) + try: + r.raise_for_status() + except requests.exceptions.HTTPError as e: + print("Functions could not be loaded, see error below") + raise e + + resp = r.json() + return resp["items"] + + +def populate_data_types(functions: list): + """ + Populate the data types + """ + data_types = set() + for function in functions: + for param in function["params"]: + data_types.add(param["dataType"]) + for output in function["output"]: + data_types.add(output["dataType"]) + return data_types + + +def populate_specs(functions: list): + """ + Populate the function class specs + """ + function_class_specs = [] + for function in functions: + # slugify function name by trimming some special chars and + # transforming it to snake case + function_name = ( + function["id"] + .replace("-", "_") + .replace("(", "_") + .replace(")", "_") + ) + base_class = "AssetNode" + is_segmentor = function["id"] in SEGMENTOR_FUNCTIONS + is_reconstructor = function["id"] in RECONSTRUCTOR_FUNCTIONS + if is_segmentor: + base_class = "BaseSegmentor" + elif is_reconstructor: + base_class = "BaseReconstructor" + + spec = { + "id": function["id"], + "is_segmentor": function["id"] in SEGMENTOR_FUNCTIONS, + "is_reconstructor": function["id"] in RECONSTRUCTOR_FUNCTIONS, + "function_name": function_name, + "base_class": base_class, + "class_name": "".join( + [w.title() for w in function_name.split("_")] + ), + "description": function["metaData"]["description"], + "input_type": function["metaData"]["InputType"], + "output_type": function["metaData"]["OutputType"], + "inputs": [ + { + "name": param["code"], + "data_type": param["dataType"], + "is_required": param["required"], + "is_list": param.get("multipleValues", False), + "default": param.get("defaultValues"), + "is_fixed": param.get("isFixed", False), + } + for param in function["params"] + ], + "outputs": [ + { + "name": output["code"], + "data_type": output["dataType"], + "default": output.get("defaultValue"), + } + for output in function["output"] + ], + } + + function_class_specs.append(spec) + + return function_class_specs + + +if __name__ == "__main__": + print("Fetching function specs") + + functions = fetch_functions() + data_types = populate_data_types(functions) + specs = populate_specs(functions) + + print( + f"Populating module with {len(data_types)} data types and {len(specs)} specs" + ) + env = Environment( + loader=BaseLoader(), + trim_blocks=True, + lstrip_blocks=True, + ) + template = env.from_string(TEMPLATE) + output = template.render(data_types=data_types, specs=specs) + + current_dir = pathlib.Path(__file__).parent + file_path = current_dir / f"{MODULE_NAME}.py" + + print(f"Writing module to file: {file_path}") + with open(file_path, "w") as f: + f.write(output) + + print("Module generated successfully") diff --git a/aixplain/modules/pipeline/pipeline.py b/aixplain/modules/pipeline/pipeline.py new file mode 100644 index 00000000..36bc643d --- /dev/null +++ b/aixplain/modules/pipeline/pipeline.py @@ -0,0 +1,4548 @@ +# This is an auto generated module. PLEASE DO NOT EDIT + + +from typing import Union, Type +from aixplain.enums import DataType + +from .designer import ( + InputParam, + OutputParam, + Inputs, + Outputs, + TI, + TO, + AssetNode, + BaseReconstructor, + BaseSegmentor, +) +from .default import DefaultPipeline +from aixplain.modules import asset + + +class ObjectDetectionInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + + +class ObjectDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class ObjectDetection(AssetNode[ObjectDetectionInputs, ObjectDetectionOutputs]): + """ + Object Detection is a computer vision technology that identifies and locates +objects within an image, typically by drawing bounding boxes around the +detected objects and classifying them into predefined categories. + + InputType: video + OutputType: text + """ + function: str = "object-detection" + input_type: str = DataType.VIDEO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ObjectDetectionInputs + outputs_class: Type[TO] = ObjectDetectionOutputs + + +class LanguageIdentificationInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class LanguageIdentificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class LanguageIdentification(AssetNode[LanguageIdentificationInputs, LanguageIdentificationOutputs]): + """ + Language Identification is the process of automatically determining the +language in which a given piece of text is written. + + InputType: text + OutputType: text + """ + function: str = "language-identification" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = LanguageIdentificationInputs + outputs_class: Type[TO] = LanguageIdentificationOutputs + + +class OcrInputs(Inputs): + image: InputParam = None + featuretypes: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + self.featuretypes = self.create_param(code="featuretypes", data_type=DataType.TEXT, is_required=True) + + +class OcrOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class Ocr(AssetNode[OcrInputs, OcrOutputs]): + """ + OCR, or Optical Character Recognition, is a technology that converts different +types of documents, such as scanned paper documents, PDFs, or images captured +by a digital camera, into editable and searchable data by recognizing and +extracting text from the images. + + InputType: image + OutputType: text + """ + function: str = "ocr" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = OcrInputs + outputs_class: Type[TO] = OcrOutputs + + +class ScriptExecutionInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class ScriptExecutionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ScriptExecution(AssetNode[ScriptExecutionInputs, ScriptExecutionOutputs]): + """ + Script Execution refers to the process of running a set of programmed +instructions or code within a computing environment, enabling the automated +performance of tasks, calculations, or operations as defined by the script. + + InputType: text + OutputType: text + """ + function: str = "script-execution" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ScriptExecutionInputs + outputs_class: Type[TO] = ScriptExecutionOutputs + + +class ImageLabelDetectionInputs(Inputs): + image: InputParam = None + min_confidence: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + self.min_confidence = self.create_param(code="min_confidence", data_type=DataType.TEXT, is_required=False) + + +class ImageLabelDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class ImageLabelDetection(AssetNode[ImageLabelDetectionInputs, ImageLabelDetectionOutputs]): + """ + Image Label Detection is a function that automatically identifies and assigns +descriptive tags or labels to objects, scenes, or elements within an image, +enabling easier categorization, search, and analysis of visual content. + + InputType: image + OutputType: label + """ + function: str = "image-label-detection" + input_type: str = DataType.IMAGE + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = ImageLabelDetectionInputs + outputs_class: Type[TO] = ImageLabelDetectionOutputs + + +class ImageCaptioningInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + + +class ImageCaptioningOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ImageCaptioning(AssetNode[ImageCaptioningInputs, ImageCaptioningOutputs]): + """ + Image Captioning is a process that involves generating a textual description of +an image, typically using machine learning models to analyze the visual content +and produce coherent and contextually relevant sentences that describe the +objects, actions, and scenes depicted in the image. + + InputType: image + OutputType: text + """ + function: str = "image-captioning" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ImageCaptioningInputs + outputs_class: Type[TO] = ImageCaptioningOutputs + + +class AudioLanguageIdentificationInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + + +class AudioLanguageIdentificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AudioLanguageIdentification(AssetNode[AudioLanguageIdentificationInputs, AudioLanguageIdentificationOutputs]): + """ + Audio Language Identification is a process that involves analyzing an audio +recording to determine the language being spoken. + + InputType: audio + OutputType: label + """ + function: str = "audio-language-identification" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AudioLanguageIdentificationInputs + outputs_class: Type[TO] = AudioLanguageIdentificationOutputs + + +class AsrAgeClassificationInputs(Inputs): + source_audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + + +class AsrAgeClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AsrAgeClassification(AssetNode[AsrAgeClassificationInputs, AsrAgeClassificationOutputs]): + """ + The ASR Age Classification function is designed to analyze audio recordings of +speech to determine the speaker's age group by leveraging automatic speech +recognition (ASR) technology and machine learning algorithms. + + InputType: audio + OutputType: label + """ + function: str = "asr-age-classification" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AsrAgeClassificationInputs + outputs_class: Type[TO] = AsrAgeClassificationOutputs + + +class BenchmarkScoringMtInputs(Inputs): + input: InputParam = None + text: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.input = self.create_param(code="input", data_type=DataType.TEXT, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class BenchmarkScoringMtOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class BenchmarkScoringMt(AssetNode[BenchmarkScoringMtInputs, BenchmarkScoringMtOutputs]): + """ + Benchmark Scoring MT is a function designed to evaluate and score machine +translation systems by comparing their output against a set of predefined +benchmarks, thereby assessing their accuracy and performance. + + InputType: text + OutputType: label + """ + function: str = "benchmark-scoring-mt" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = BenchmarkScoringMtInputs + outputs_class: Type[TO] = BenchmarkScoringMtOutputs + + +class AsrGenderClassificationInputs(Inputs): + source_audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + + +class AsrGenderClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AsrGenderClassification(AssetNode[AsrGenderClassificationInputs, AsrGenderClassificationOutputs]): + """ + The ASR Gender Classification function analyzes audio recordings to determine +and classify the speaker's gender based on their voice characteristics. + + InputType: audio + OutputType: label + """ + function: str = "asr-gender-classification" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AsrGenderClassificationInputs + outputs_class: Type[TO] = AsrGenderClassificationOutputs + + +class BaseModelInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class BaseModelOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class BaseModel(AssetNode[BaseModelInputs, BaseModelOutputs]): + """ + The Base-Model function serves as a foundational framework designed to provide +essential features and capabilities upon which more specialized or advanced +models can be built and customized. + + InputType: text + OutputType: text + """ + function: str = "base-model" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = BaseModelInputs + outputs_class: Type[TO] = BaseModelOutputs + + +class LanguageIdentificationAudioInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + + +class LanguageIdentificationAudioOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class LanguageIdentificationAudio(AssetNode[LanguageIdentificationAudioInputs, LanguageIdentificationAudioOutputs]): + """ + The Language Identification Audio function analyzes audio input to determine +and identify the language being spoken. + + InputType: audio + OutputType: label + """ + function: str = "language-identification-audio" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = LanguageIdentificationAudioInputs + outputs_class: Type[TO] = LanguageIdentificationAudioOutputs + + +class LoglikelihoodInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class LoglikelihoodOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.NUMBER) + + +class Loglikelihood(AssetNode[LoglikelihoodInputs, LoglikelihoodOutputs]): + """ + The Log Likelihood function measures the probability of observing the given +data under a specific statistical model by taking the natural logarithm of the +likelihood function, thereby transforming the product of probabilities into a +sum, which simplifies the process of optimization and parameter estimation. + + InputType: text + OutputType: number + """ + function: str = "loglikelihood" + input_type: str = DataType.TEXT + output_type: str = DataType.NUMBER + + inputs_class: Type[TI] = LoglikelihoodInputs + outputs_class: Type[TO] = LoglikelihoodOutputs + + +class VideoEmbeddingInputs(Inputs): + language: InputParam = None + video: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=False) + + +class VideoEmbeddingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.EMBEDDING) + + +class VideoEmbedding(AssetNode[VideoEmbeddingInputs, VideoEmbeddingOutputs]): + """ + Video Embedding is a process that transforms video content into a fixed- +dimensional vector representation, capturing essential features and patterns to +facilitate tasks such as retrieval, classification, and recommendation. + + InputType: video + OutputType: embedding + """ + function: str = "video-embedding" + input_type: str = DataType.VIDEO + output_type: str = DataType.EMBEDDING + + inputs_class: Type[TI] = VideoEmbeddingInputs + outputs_class: Type[TO] = VideoEmbeddingOutputs + + +class TextSegmenationInputs(Inputs): + text: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + + +class TextSegmenationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextSegmenation(AssetNode[TextSegmenationInputs, TextSegmenationOutputs]): + """ + Text Segmentation is the process of dividing a continuous text into meaningful +units, such as words, sentences, or topics, to facilitate easier analysis and +understanding. + + InputType: text + OutputType: text + """ + function: str = "text-segmenation" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextSegmenationInputs + outputs_class: Type[TO] = TextSegmenationOutputs + + +class ImageEmbeddingInputs(Inputs): + language: InputParam = None + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class ImageEmbeddingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ImageEmbedding(AssetNode[ImageEmbeddingInputs, ImageEmbeddingOutputs]): + """ + Image Embedding is a process that transforms an image into a fixed-dimensional +vector representation, capturing its essential features and enabling efficient +comparison, retrieval, and analysis in various machine learning and computer +vision tasks. + + InputType: image + OutputType: text + """ + function: str = "image-embedding" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ImageEmbeddingInputs + outputs_class: Type[TO] = ImageEmbeddingOutputs + + +class ImageManipulationInputs(Inputs): + image: InputParam = None + targetimage: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + self.targetimage = self.create_param(code="targetimage", data_type=DataType.IMAGE, is_required=True) + + +class ImageManipulationOutputs(Outputs): + image: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE) + + +class ImageManipulation(AssetNode[ImageManipulationInputs, ImageManipulationOutputs]): + """ + Image Manipulation refers to the process of altering or enhancing digital +images using various techniques and tools to achieve desired visual effects, +correct imperfections, or transform the image's appearance. + + InputType: image + OutputType: image + """ + function: str = "image-manipulation" + input_type: str = DataType.IMAGE + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = ImageManipulationInputs + outputs_class: Type[TO] = ImageManipulationOutputs + + +class ImageToVideoGenerationInputs(Inputs): + language: InputParam = None + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class ImageToVideoGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.VIDEO) + + +class ImageToVideoGeneration(AssetNode[ImageToVideoGenerationInputs, ImageToVideoGenerationOutputs]): + """ + The Image To Video Generation function transforms a series of static images +into a cohesive, dynamic video sequence, often incorporating transitions, +effects, and synchronization with audio to create a visually engaging +narrative. + + InputType: image + OutputType: video + """ + function: str = "image-to-video-generation" + input_type: str = DataType.IMAGE + output_type: str = DataType.VIDEO + + inputs_class: Type[TI] = ImageToVideoGenerationInputs + outputs_class: Type[TO] = ImageToVideoGenerationOutputs + + +class AudioForcedAlignmentInputs(Inputs): + audio: InputParam = None + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class AudioForcedAlignmentOutputs(Outputs): + text: OutputParam = None + audio: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO) + + +class AudioForcedAlignment(AssetNode[AudioForcedAlignmentInputs, AudioForcedAlignmentOutputs]): + """ + Audio Forced Alignment is a process that synchronizes a given audio recording +with its corresponding transcript by precisely aligning each spoken word or +phoneme to its exact timing within the audio. + + InputType: audio + OutputType: audio + """ + function: str = "audio-forced-alignment" + input_type: str = DataType.AUDIO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = AudioForcedAlignmentInputs + outputs_class: Type[TO] = AudioForcedAlignmentOutputs + + +class BenchmarkScoringAsrInputs(Inputs): + input: InputParam = None + text: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.input = self.create_param(code="input", data_type=DataType.AUDIO, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class BenchmarkScoringAsrOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class BenchmarkScoringAsr(AssetNode[BenchmarkScoringAsrInputs, BenchmarkScoringAsrOutputs]): + """ + Benchmark Scoring ASR is a function that evaluates and compares the performance +of automatic speech recognition systems by analyzing their accuracy, speed, and +other relevant metrics against a standardized set of benchmarks. + + InputType: audio + OutputType: label + """ + function: str = "benchmark-scoring-asr" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = BenchmarkScoringAsrInputs + outputs_class: Type[TO] = BenchmarkScoringAsrOutputs + + +class VisualQuestionAnsweringInputs(Inputs): + text: InputParam = None + language: InputParam = None + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class VisualQuestionAnsweringOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class VisualQuestionAnswering(AssetNode[VisualQuestionAnsweringInputs, VisualQuestionAnsweringOutputs]): + """ + Visual Question Answering (VQA) is a task in artificial intelligence that +involves analyzing an image and providing accurate, contextually relevant +answers to questions posed about the visual content of that image. + + InputType: image + OutputType: video + """ + function: str = "visual-question-answering" + input_type: str = DataType.IMAGE + output_type: str = DataType.VIDEO + + inputs_class: Type[TI] = VisualQuestionAnsweringInputs + outputs_class: Type[TO] = VisualQuestionAnsweringOutputs + + +class DocumentImageParsingInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class DocumentImageParsingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class DocumentImageParsing(AssetNode[DocumentImageParsingInputs, DocumentImageParsingOutputs]): + """ + Document Image Parsing is the process of analyzing and converting scanned or +photographed images of documents into structured, machine-readable formats by +identifying and extracting text, layout, and other relevant information. + + InputType: image + OutputType: text + """ + function: str = "document-image-parsing" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = DocumentImageParsingInputs + outputs_class: Type[TO] = DocumentImageParsingOutputs + + +class DocumentInformationExtractionInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class DocumentInformationExtractionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class DocumentInformationExtraction(AssetNode[DocumentInformationExtractionInputs, DocumentInformationExtractionOutputs]): + """ + Document Information Extraction is the process of automatically identifying, +extracting, and structuring relevant data from unstructured or semi-structured +documents, such as invoices, receipts, contracts, and forms, to facilitate +easier data management and analysis. + + InputType: image + OutputType: text + """ + function: str = "document-information-extraction" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = DocumentInformationExtractionInputs + outputs_class: Type[TO] = DocumentInformationExtractionOutputs + + +class DepthEstimationInputs(Inputs): + language: InputParam = None + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class DepthEstimationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class DepthEstimation(AssetNode[DepthEstimationInputs, DepthEstimationOutputs]): + """ + Depth estimation is a computational process that determines the distance of +objects from a viewpoint, typically using visual data from cameras or sensors +to create a three-dimensional understanding of a scene. + + InputType: image + OutputType: text + """ + function: str = "depth-estimation" + input_type: str = DataType.IMAGE + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = DepthEstimationInputs + outputs_class: Type[TO] = DepthEstimationOutputs + + +class VideoGenerationInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class VideoGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.VIDEO) + + +class VideoGeneration(AssetNode[VideoGenerationInputs, VideoGenerationOutputs]): + """ + Video Generation is the process of creating video content through automated or +semi-automated means, often utilizing algorithms, artificial intelligence, or +software tools to produce visual and audio elements that can range from simple +animations to complex, realistic scenes. + + InputType: text + OutputType: video + """ + function: str = "video-generation" + input_type: str = DataType.TEXT + output_type: str = DataType.VIDEO + + inputs_class: Type[TI] = VideoGenerationInputs + outputs_class: Type[TO] = VideoGenerationOutputs + + +class ReferencelessAudioGenerationMetricInputs(Inputs): + hypotheses: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.AUDIO, is_required=True) + self.sources = self.create_param(code="sources", data_type=DataType.AUDIO, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class ReferencelessAudioGenerationMetricOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ReferencelessAudioGenerationMetric(AssetNode[ReferencelessAudioGenerationMetricInputs, ReferencelessAudioGenerationMetricOutputs]): + """ + The Referenceless Audio Generation Metric is a tool designed to evaluate the +quality of generated audio content without the need for a reference or original +audio sample for comparison. + + InputType: text + OutputType: text + """ + function: str = "referenceless-audio-generation-metric" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ReferencelessAudioGenerationMetricInputs + outputs_class: Type[TO] = ReferencelessAudioGenerationMetricOutputs + + +class MultiClassImageClassificationInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class MultiClassImageClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class MultiClassImageClassification(AssetNode[MultiClassImageClassificationInputs, MultiClassImageClassificationOutputs]): + """ + Multi Class Image Classification is a machine learning task where an algorithm +is trained to categorize images into one of several predefined classes or +categories based on their visual content. + + InputType: image + OutputType: label + """ + function: str = "multi-class-image-classification" + input_type: str = DataType.IMAGE + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = MultiClassImageClassificationInputs + outputs_class: Type[TO] = MultiClassImageClassificationOutputs + + +class SemanticSegmentationInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class SemanticSegmentationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class SemanticSegmentation(AssetNode[SemanticSegmentationInputs, SemanticSegmentationOutputs]): + """ + Semantic segmentation is a computer vision process that involves classifying +each pixel in an image into a predefined category, effectively partitioning the +image into meaningful segments based on the objects or regions they represent. + + InputType: image + OutputType: label + """ + function: str = "semantic-segmentation" + input_type: str = DataType.IMAGE + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SemanticSegmentationInputs + outputs_class: Type[TO] = SemanticSegmentationOutputs + + +class InstanceSegmentationInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class InstanceSegmentationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class InstanceSegmentation(AssetNode[InstanceSegmentationInputs, InstanceSegmentationOutputs]): + """ + Instance segmentation is a computer vision task that involves detecting and +delineating each distinct object within an image, assigning a unique label and +precise boundary to every individual instance of objects, even if they belong +to the same category. + + InputType: image + OutputType: label + """ + function: str = "instance-segmentation" + input_type: str = DataType.IMAGE + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = InstanceSegmentationInputs + outputs_class: Type[TO] = InstanceSegmentationOutputs + + +class ImageColorizationInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class ImageColorizationOutputs(Outputs): + image: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE) + + +class ImageColorization(AssetNode[ImageColorizationInputs, ImageColorizationOutputs]): + """ + Image colorization is a process that involves adding color to grayscale images, +transforming them from black-and-white to full-color representations, often +using advanced algorithms and machine learning techniques to predict and apply +the appropriate hues and shades. + + InputType: image + OutputType: image + """ + function: str = "image-colorization" + input_type: str = DataType.IMAGE + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = ImageColorizationInputs + outputs_class: Type[TO] = ImageColorizationOutputs + + +class AudioGenerationMetricInputs(Inputs): + hypotheses: InputParam = None + references: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.AUDIO, is_required=True) + self.references = self.create_param(code="references", data_type=DataType.AUDIO, is_required=False) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class AudioGenerationMetricOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class AudioGenerationMetric(AssetNode[AudioGenerationMetricInputs, AudioGenerationMetricOutputs]): + """ + The Audio Generation Metric is a quantitative measure used to evaluate the +quality, accuracy, and overall performance of audio generated by artificial +intelligence systems, often considering factors such as fidelity, +intelligibility, and similarity to human-produced audio. + + InputType: text + OutputType: text + """ + function: str = "audio-generation-metric" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = AudioGenerationMetricInputs + outputs_class: Type[TO] = AudioGenerationMetricOutputs + + +class ImageImpaintingInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class ImageImpaintingOutputs(Outputs): + image: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE) + + +class ImageImpainting(AssetNode[ImageImpaintingInputs, ImageImpaintingOutputs]): + """ + Image inpainting is a process that involves filling in missing or damaged parts +of an image in a way that is visually coherent and seamlessly blends with the +surrounding areas, often using advanced algorithms and techniques to restore +the image to its original or intended appearance. + + InputType: image + OutputType: image + """ + function: str = "image-impainting" + input_type: str = DataType.IMAGE + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = ImageImpaintingInputs + outputs_class: Type[TO] = ImageImpaintingOutputs + + +class StyleTransferInputs(Inputs): + image: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=False) + + +class StyleTransferOutputs(Outputs): + image: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE) + + +class StyleTransfer(AssetNode[StyleTransferInputs, StyleTransferOutputs]): + """ + Style Transfer is a technique in artificial intelligence that applies the +visual style of one image (such as the brushstrokes of a famous painting) to +the content of another image, effectively blending the artistic elements of the +first image with the subject matter of the second. + + InputType: image + OutputType: image + """ + function: str = "style-transfer" + input_type: str = DataType.IMAGE + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = StyleTransferInputs + outputs_class: Type[TO] = StyleTransferOutputs + + +class MultiClassTextClassificationInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class MultiClassTextClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class MultiClassTextClassification(AssetNode[MultiClassTextClassificationInputs, MultiClassTextClassificationOutputs]): + """ + Multi Class Text Classification is a natural language processing task that +involves categorizing a given text into one of several predefined classes or +categories based on its content. + + InputType: text + OutputType: label + """ + function: str = "multi-class-text-classification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = MultiClassTextClassificationInputs + outputs_class: Type[TO] = MultiClassTextClassificationOutputs + + +class TextEmbeddingInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class TextEmbeddingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextEmbedding(AssetNode[TextEmbeddingInputs, TextEmbeddingOutputs]): + """ + Text embedding is a process that converts text into numerical vectors, +capturing the semantic meaning and contextual relationships of words or +phrases, enabling machines to understand and analyze natural language more +effectively. + + InputType: text + OutputType: text + """ + function: str = "text-embedding" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextEmbeddingInputs + outputs_class: Type[TO] = TextEmbeddingOutputs + + +class MultiLabelTextClassificationInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class MultiLabelTextClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class MultiLabelTextClassification(AssetNode[MultiLabelTextClassificationInputs, MultiLabelTextClassificationOutputs]): + """ + Multi Label Text Classification is a natural language processing task where a +given text is analyzed and assigned multiple relevant labels or categories from +a predefined set, allowing for the text to belong to more than one category +simultaneously. + + InputType: text + OutputType: label + """ + function: str = "multi-label-text-classification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = MultiLabelTextClassificationInputs + outputs_class: Type[TO] = MultiLabelTextClassificationOutputs + + +class TextReconstructionInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class TextReconstructionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextReconstruction(BaseReconstructor[TextReconstructionInputs, TextReconstructionOutputs]): + """ + Text Reconstruction is a process that involves piecing together fragmented or +incomplete text data to restore it to its original, coherent form. + + InputType: text + OutputType: text + """ + function: str = "text-reconstruction" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextReconstructionInputs + outputs_class: Type[TO] = TextReconstructionOutputs + + +class FactCheckingInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class FactCheckingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class FactChecking(AssetNode[FactCheckingInputs, FactCheckingOutputs]): + """ + Fact Checking is the process of verifying the accuracy and truthfulness of +information, statements, or claims by cross-referencing with reliable sources +and evidence. + + InputType: text + OutputType: label + """ + function: str = "fact-checking" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = FactCheckingInputs + outputs_class: Type[TO] = FactCheckingOutputs + + +class SpeechClassificationInputs(Inputs): + audio: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class SpeechClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class SpeechClassification(AssetNode[SpeechClassificationInputs, SpeechClassificationOutputs]): + """ + Speech Classification is a process that involves analyzing and categorizing +spoken language into predefined categories or classes based on various features +such as tone, pitch, and linguistic content. + + InputType: audio + OutputType: label + """ + function: str = "speech-classification" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SpeechClassificationInputs + outputs_class: Type[TO] = SpeechClassificationOutputs + + +class IntentClassificationInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class IntentClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class IntentClassification(AssetNode[IntentClassificationInputs, IntentClassificationOutputs]): + """ + Intent Classification is a natural language processing task that involves +analyzing and categorizing user text input to determine the underlying purpose +or goal behind the communication, such as booking a flight, asking for weather +information, or setting a reminder. + + InputType: text + OutputType: label + """ + function: str = "intent-classification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = IntentClassificationInputs + outputs_class: Type[TO] = IntentClassificationOutputs + + +class PartOfSpeechTaggingInputs(Inputs): + language: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class PartOfSpeechTaggingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class PartOfSpeechTagging(AssetNode[PartOfSpeechTaggingInputs, PartOfSpeechTaggingOutputs]): + """ + Part of Speech Tagging is a natural language processing task that involves +assigning each word in a sentence its corresponding part of speech, such as +noun, verb, adjective, or adverb, based on its role and context within the +sentence. + + InputType: text + OutputType: label + """ + function: str = "part-of-speech-tagging" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = PartOfSpeechTaggingInputs + outputs_class: Type[TO] = PartOfSpeechTaggingOutputs + + +class MetricAggregationInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class MetricAggregationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class MetricAggregation(AssetNode[MetricAggregationInputs, MetricAggregationOutputs]): + """ + Metric Aggregation is a function that computes and summarizes numerical data by +applying statistical operations, such as averaging, summing, or finding the +minimum and maximum values, to provide insights and facilitate analysis of +large datasets. + + InputType: text + OutputType: text + """ + function: str = "metric-aggregation" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = MetricAggregationInputs + outputs_class: Type[TO] = MetricAggregationOutputs + + +class DialectDetectionInputs(Inputs): + audio: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + + +class DialectDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class DialectDetection(AssetNode[DialectDetectionInputs, DialectDetectionOutputs]): + """ + Dialect Detection is a function that identifies and classifies the specific +regional or social variations of a language spoken or written by an individual, +enabling the recognition of distinct linguistic patterns and nuances associated +with different dialects. + + InputType: audio + OutputType: text + """ + function: str = "dialect-detection" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = DialectDetectionInputs + outputs_class: Type[TO] = DialectDetectionOutputs + + +class InverseTextNormalizationInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=False) + + +class InverseTextNormalizationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class InverseTextNormalization(AssetNode[InverseTextNormalizationInputs, InverseTextNormalizationOutputs]): + """ + Inverse Text Normalization is the process of converting spoken or written +language in its normalized form, such as numbers, dates, and abbreviations, +back into their original, more complex or detailed textual representations. + + InputType: text + OutputType: label + """ + function: str = "inverse-text-normalization" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = InverseTextNormalizationInputs + outputs_class: Type[TO] = InverseTextNormalizationOutputs + + +class TextToAudioInputs(Inputs): + text: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + + +class TextToAudioOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class TextToAudio(AssetNode[TextToAudioInputs, TextToAudioOutputs]): + """ + The Text to Audio function converts written text into spoken words, allowing +users to listen to the content instead of reading it. + + InputType: text + OutputType: audio + """ + function: str = "text-to-audio" + input_type: str = DataType.TEXT + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = TextToAudioInputs + outputs_class: Type[TO] = TextToAudioOutputs + + +class FillTextMaskInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class FillTextMaskOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class FillTextMask(AssetNode[FillTextMaskInputs, FillTextMaskOutputs]): + """ + The "Fill Text Mask" function takes a text input with masked or placeholder +characters and replaces those placeholders with specified or contextually +appropriate characters to generate a complete and coherent text output. + + InputType: text + OutputType: text + """ + function: str = "fill-text-mask" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = FillTextMaskInputs + outputs_class: Type[TO] = FillTextMaskOutputs + + +class VideoContentModerationInputs(Inputs): + video: InputParam = None + min_confidence: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + self.min_confidence = self.create_param(code="min_confidence", data_type=DataType.TEXT, is_required=False) + + +class VideoContentModerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class VideoContentModeration(AssetNode[VideoContentModerationInputs, VideoContentModerationOutputs]): + """ + Video Content Moderation is the process of reviewing, analyzing, and filtering +video content to ensure it adheres to community guidelines, legal standards, +and platform policies, thereby preventing the dissemination of inappropriate, +harmful, or illegal material. + + InputType: video + OutputType: label + """ + function: str = "video-content-moderation" + input_type: str = DataType.VIDEO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = VideoContentModerationInputs + outputs_class: Type[TO] = VideoContentModerationOutputs + + +class ExtractAudioFromVideoInputs(Inputs): + video: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + + +class ExtractAudioFromVideoOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class ExtractAudioFromVideo(AssetNode[ExtractAudioFromVideoInputs, ExtractAudioFromVideoOutputs]): + """ + The "Extract Audio From Video" function allows users to separate and save the +audio track from a video file, enabling them to obtain just the sound without +the accompanying visual content. + + InputType: video + OutputType: audio + """ + function: str = "extract-audio-from-video" + input_type: str = DataType.VIDEO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = ExtractAudioFromVideoInputs + outputs_class: Type[TO] = ExtractAudioFromVideoOutputs + + +class ImageCompressionInputs(Inputs): + image: InputParam = None + apl_qfactor: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + self.apl_qfactor = self.create_param(code="apl_qfactor", data_type=DataType.TEXT, is_required=False) + + +class ImageCompressionOutputs(Outputs): + image: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE) + + +class ImageCompression(AssetNode[ImageCompressionInputs, ImageCompressionOutputs]): + """ + Image compression is a process that reduces the file size of an image by +removing redundant or non-essential data, while maintaining an acceptable level +of visual quality. + + InputType: image + OutputType: image + """ + function: str = "image-compression" + input_type: str = DataType.IMAGE + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = ImageCompressionInputs + outputs_class: Type[TO] = ImageCompressionOutputs + + +class MultilingualSpeechRecognitionInputs(Inputs): + source_audio: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + + +class MultilingualSpeechRecognitionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class MultilingualSpeechRecognition(AssetNode[MultilingualSpeechRecognitionInputs, MultilingualSpeechRecognitionOutputs]): + """ + Multilingual Speech Recognition is a technology that enables the automatic +transcription of spoken language into text across multiple languages, allowing +for seamless communication and understanding in diverse linguistic contexts. + + InputType: audio + OutputType: text + """ + function: str = "multilingual-speech-recognition" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = MultilingualSpeechRecognitionInputs + outputs_class: Type[TO] = MultilingualSpeechRecognitionOutputs + + +class ReferencelessTextGenerationMetricInputs(Inputs): + hypotheses: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.TEXT, is_required=True) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class ReferencelessTextGenerationMetricOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ReferencelessTextGenerationMetric(AssetNode[ReferencelessTextGenerationMetricInputs, ReferencelessTextGenerationMetricOutputs]): + """ + The Referenceless Text Generation Metric is a method for evaluating the quality +of generated text without requiring a reference text for comparison, often +leveraging models or algorithms to assess coherence, relevance, and fluency +based on intrinsic properties of the text itself. + + InputType: text + OutputType: text + """ + function: str = "referenceless-text-generation-metric" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ReferencelessTextGenerationMetricInputs + outputs_class: Type[TO] = ReferencelessTextGenerationMetricOutputs + + +class TextGenerationMetricDefaultInputs(Inputs): + hypotheses: InputParam = None + references: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.TEXT, is_required=True) + self.references = self.create_param(code="references", data_type=DataType.TEXT, is_required=False) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class TextGenerationMetricDefaultOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextGenerationMetricDefault(AssetNode[TextGenerationMetricDefaultInputs, TextGenerationMetricDefaultOutputs]): + """ + The "Text Generation Metric Default" function provides a standard set of +evaluation metrics for assessing the quality and performance of text generation +models. + + InputType: text + OutputType: text + """ + function: str = "text-generation-metric-default" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextGenerationMetricDefaultInputs + outputs_class: Type[TO] = TextGenerationMetricDefaultOutputs + + +class NoiseRemovalInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=False) + + +class NoiseRemovalOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class NoiseRemoval(AssetNode[NoiseRemovalInputs, NoiseRemovalOutputs]): + """ + Noise Removal is a process that involves identifying and eliminating unwanted +random variations or disturbances from an audio signal to enhance the clarity +and quality of the underlying information. + + InputType: audio + OutputType: audio + """ + function: str = "noise-removal" + input_type: str = DataType.AUDIO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = NoiseRemovalInputs + outputs_class: Type[TO] = NoiseRemovalOutputs + + +class AudioReconstructionInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + + +class AudioReconstructionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class AudioReconstruction(BaseReconstructor[AudioReconstructionInputs, AudioReconstructionOutputs]): + """ + Audio Reconstruction is the process of restoring or recreating audio signals +from incomplete, damaged, or degraded recordings to achieve a high-quality, +accurate representation of the original sound. + + InputType: audio + OutputType: audio + """ + function: str = "audio-reconstruction" + input_type: str = DataType.AUDIO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = AudioReconstructionInputs + outputs_class: Type[TO] = AudioReconstructionOutputs + + +class VoiceCloningInputs(Inputs): + text: InputParam = None + audio: InputParam = None + language: InputParam = None + dialect: InputParam = None + voice: InputParam = None + script: InputParam = None + type: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.voice = self.create_param(code="voice", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.type = self.create_param(code="type", data_type=DataType.LABEL, is_required=False) + + +class VoiceCloningOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class VoiceCloning(AssetNode[VoiceCloningInputs, VoiceCloningOutputs]): + """ + Voice cloning is a technology that uses artificial intelligence to create a +digital replica of a person's voice, allowing for the generation of speech that +mimics the tone, pitch, and speaking style of the original speaker. + + InputType: text + OutputType: audio + """ + function: str = "voice-cloning" + input_type: str = DataType.TEXT + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = VoiceCloningInputs + outputs_class: Type[TO] = VoiceCloningOutputs + + +class DiacritizationInputs(Inputs): + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class DiacritizationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class Diacritization(AssetNode[DiacritizationInputs, DiacritizationOutputs]): + """ + Diacritization is the process of adding diacritical marks to letters in a text +to indicate pronunciation, stress, tone, or meaning, often used in languages +such as Arabic, Hebrew, and Vietnamese to provide clarity and accuracy in +written communication. + + InputType: text + OutputType: text + """ + function: str = "diacritization" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = DiacritizationInputs + outputs_class: Type[TO] = DiacritizationOutputs + + +class AudioEmotionDetectionInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=False) + + +class AudioEmotionDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AudioEmotionDetection(AssetNode[AudioEmotionDetectionInputs, AudioEmotionDetectionOutputs]): + """ + Audio Emotion Detection is a technology that analyzes vocal characteristics and +patterns in audio recordings to identify and classify the emotional state of +the speaker. + + InputType: audio + OutputType: label + """ + function: str = "audio-emotion-detection" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AudioEmotionDetectionInputs + outputs_class: Type[TO] = AudioEmotionDetectionOutputs + + +class TextSummarizationInputs(Inputs): + text: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class TextSummarizationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextSummarization(AssetNode[TextSummarizationInputs, TextSummarizationOutputs]): + """ + Text summarization is the process of condensing a large body of text into a +shorter version, capturing the main points and essential information while +maintaining coherence and meaning. + + InputType: text + OutputType: text + """ + function: str = "text-summarization" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextSummarizationInputs + outputs_class: Type[TO] = TextSummarizationOutputs + + +class EntityLinkingInputs(Inputs): + text: InputParam = None + language: InputParam = None + domain: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.domain = self.create_param(code="domain", data_type=DataType.LABEL, is_required=False) + + +class EntityLinkingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class EntityLinking(AssetNode[EntityLinkingInputs, EntityLinkingOutputs]): + """ + Entity Linking is the process of identifying and connecting mentions of +entities within a text to their corresponding entries in a structured knowledge +base, thereby enabling the disambiguation of terms and enhancing the +understanding of the text's context. + + InputType: text + OutputType: label + """ + function: str = "entity-linking" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = EntityLinkingInputs + outputs_class: Type[TO] = EntityLinkingOutputs + + +class TextGenerationMetricInputs(Inputs): + hypotheses: InputParam = None + references: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.TEXT, is_required=True) + self.references = self.create_param(code="references", data_type=DataType.TEXT, is_required=False) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class TextGenerationMetricOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextGenerationMetric(AssetNode[TextGenerationMetricInputs, TextGenerationMetricOutputs]): + """ + A Text Generation Metric is a quantitative measure used to evaluate the quality +and effectiveness of text produced by natural language processing models, often +assessing aspects such as coherence, relevance, fluency, and adherence to given +prompts or instructions. + + InputType: text + OutputType: text + """ + function: str = "text-generation-metric" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextGenerationMetricInputs + outputs_class: Type[TO] = TextGenerationMetricOutputs + + +class SplitOnLinebreakInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class SplitOnLinebreakOutputs(Outputs): + data: OutputParam = None + audio: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO) + + +class SplitOnLinebreak(BaseSegmentor[SplitOnLinebreakInputs, SplitOnLinebreakOutputs]): + """ + The "Split On Linebreak" function divides a given string into a list of +substrings, using linebreaks (newline characters) as the points of separation. + + InputType: text + OutputType: text + """ + function: str = "split-on-linebreak" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SplitOnLinebreakInputs + outputs_class: Type[TO] = SplitOnLinebreakOutputs + + +class SentimentAnalysisInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class SentimentAnalysisOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class SentimentAnalysis(AssetNode[SentimentAnalysisInputs, SentimentAnalysisOutputs]): + """ + Sentiment Analysis is a natural language processing technique used to determine +and classify the emotional tone or subjective information expressed in a piece +of text, such as identifying whether the sentiment is positive, negative, or +neutral. + + InputType: text + OutputType: label + """ + function: str = "sentiment-analysis" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SentimentAnalysisInputs + outputs_class: Type[TO] = SentimentAnalysisOutputs + + +class KeywordSpottingInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=False) + + +class KeywordSpottingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class KeywordSpotting(AssetNode[KeywordSpottingInputs, KeywordSpottingOutputs]): + """ + Keyword Spotting is a function that enables the detection and identification of +specific words or phrases within a stream of audio, often used in voice- +activated systems to trigger actions or commands based on recognized keywords. + + InputType: audio + OutputType: label + """ + function: str = "keyword-spotting" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = KeywordSpottingInputs + outputs_class: Type[TO] = KeywordSpottingOutputs + + +class TextClassificationInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class TextClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextClassification(AssetNode[TextClassificationInputs, TextClassificationOutputs]): + """ + Text Classification is a natural language processing task that involves +categorizing text into predefined labels or classes based on its content, +enabling automated organization, filtering, and analysis of large volumes of +textual data. + + InputType: text + OutputType: label + """ + function: str = "text-classification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TextClassificationInputs + outputs_class: Type[TO] = TextClassificationOutputs + + +class OtherMultipurposeInputs(Inputs): + text: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + + +class OtherMultipurposeOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class OtherMultipurpose(AssetNode[OtherMultipurposeInputs, OtherMultipurposeOutputs]): + """ + The "Other (Multipurpose)" function serves as a versatile category designed to +accommodate a wide range of tasks and activities that do not fit neatly into +predefined classifications, offering flexibility and adaptability for various +needs. + + InputType: text + OutputType: text + """ + function: str = "other-(multipurpose)" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = OtherMultipurposeInputs + outputs_class: Type[TO] = OtherMultipurposeOutputs + + +class SpeechSynthesisInputs(Inputs): + audio: InputParam = None + language: InputParam = None + dialect: InputParam = None + voice: InputParam = None + script: InputParam = None + text: InputParam = None + type: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=False) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.voice = self.create_param(code="voice", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.type = self.create_param(code="type", data_type=DataType.LABEL, is_required=False) + + +class SpeechSynthesisOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class SpeechSynthesis(AssetNode[SpeechSynthesisInputs, SpeechSynthesisOutputs]): + """ + Speech synthesis is the artificial production of human speech, typically +achieved through software or hardware systems that convert text into spoken +words, enabling machines to communicate verbally with users. + + InputType: text + OutputType: audio + """ + function: str = "speech-synthesis" + input_type: str = DataType.TEXT + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = SpeechSynthesisInputs + outputs_class: Type[TO] = SpeechSynthesisOutputs + + +class AudioIntentDetectionInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=False) + + +class AudioIntentDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AudioIntentDetection(AssetNode[AudioIntentDetectionInputs, AudioIntentDetectionOutputs]): + """ + Audio Intent Detection is a process that involves analyzing audio signals to +identify and interpret the underlying intentions or purposes behind spoken +words, enabling systems to understand and respond appropriately to human +speech. + + InputType: audio + OutputType: label + """ + function: str = "audio-intent-detection" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AudioIntentDetectionInputs + outputs_class: Type[TO] = AudioIntentDetectionOutputs + + +class VideoLabelDetectionInputs(Inputs): + video: InputParam = None + min_confidence: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + self.min_confidence = self.create_param(code="min_confidence", data_type=DataType.TEXT, is_required=False) + + +class VideoLabelDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class VideoLabelDetection(AssetNode[VideoLabelDetectionInputs, VideoLabelDetectionOutputs]): + """ + Video Label Detection is a function that automatically identifies and tags +various objects, scenes, activities, and other relevant elements within a +video, providing descriptive labels that enhance searchability and content +organization. + + InputType: video + OutputType: label + """ + function: str = "video-label-detection" + input_type: str = DataType.VIDEO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = VideoLabelDetectionInputs + outputs_class: Type[TO] = VideoLabelDetectionOutputs + + +class AsrQualityEstimationInputs(Inputs): + text: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class AsrQualityEstimationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class AsrQualityEstimation(AssetNode[AsrQualityEstimationInputs, AsrQualityEstimationOutputs]): + """ + ASR Quality Estimation is a process that evaluates the accuracy and reliability +of automatic speech recognition systems by analyzing their performance in +transcribing spoken language into text. + + InputType: text + OutputType: label + """ + function: str = "asr-quality-estimation" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = AsrQualityEstimationInputs + outputs_class: Type[TO] = AsrQualityEstimationOutputs + + +class AudioTranscriptAnalysisInputs(Inputs): + language: InputParam = None + dialect: InputParam = None + source_supplier: InputParam = None + source_audio: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.source_supplier = self.create_param(code="source_supplier", data_type=DataType.LABEL, is_required=False) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class AudioTranscriptAnalysisOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class AudioTranscriptAnalysis(AssetNode[AudioTranscriptAnalysisInputs, AudioTranscriptAnalysisOutputs]): + """ + Audio Transcript Analysis is a process that involves converting spoken language +from audio recordings into written text, followed by examining and interpreting +the transcribed content to extract meaningful insights, identify patterns, and +derive actionable information. + + InputType: audio + OutputType: text + """ + function: str = "audio-transcript-analysis" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = AudioTranscriptAnalysisInputs + outputs_class: Type[TO] = AudioTranscriptAnalysisOutputs + + +class SearchInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class SearchOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class Search(AssetNode[SearchInputs, SearchOutputs]): + """ + The "Search" function allows users to input keywords or phrases to quickly +locate specific information, files, or content within a database, website, or +application. + + InputType: text + OutputType: text + """ + function: str = "search" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SearchInputs + outputs_class: Type[TO] = SearchOutputs + + +class VideoForcedAlignmentInputs(Inputs): + video: InputParam = None + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class VideoForcedAlignmentOutputs(Outputs): + text: OutputParam = None + video: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT) + self.video = self.create_param(code="video", data_type=DataType.VIDEO) + + +class VideoForcedAlignment(AssetNode[VideoForcedAlignmentInputs, VideoForcedAlignmentOutputs]): + """ + Video Forced Alignment is a process that synchronizes video footage with +corresponding audio tracks by precisely aligning the visual and auditory +elements, ensuring that the movements of speakers' lips match the spoken words. + + InputType: video + OutputType: video + """ + function: str = "video-forced-alignment" + input_type: str = DataType.VIDEO + output_type: str = DataType.VIDEO + + inputs_class: Type[TI] = VideoForcedAlignmentInputs + outputs_class: Type[TO] = VideoForcedAlignmentOutputs + + +class VisemeGenerationInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class VisemeGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class VisemeGeneration(AssetNode[VisemeGenerationInputs, VisemeGenerationOutputs]): + """ + Viseme Generation is the process of creating visual representations of +phonemes, which are the distinct units of sound in speech, to synchronize lip +movements with spoken words in animations or virtual avatars. + + InputType: text + OutputType: label + """ + function: str = "viseme-generation" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = VisemeGenerationInputs + outputs_class: Type[TO] = VisemeGenerationOutputs + + +class TopicClassificationInputs(Inputs): + text: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class TopicClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TopicClassification(AssetNode[TopicClassificationInputs, TopicClassificationOutputs]): + """ + Topic Classification is a natural language processing function that categorizes +text into predefined topics or subjects based on its content, enabling +efficient organization and retrieval of information. + + InputType: text + OutputType: label + """ + function: str = "topic-classification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TopicClassificationInputs + outputs_class: Type[TO] = TopicClassificationOutputs + + +class OffensiveLanguageIdentificationInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class OffensiveLanguageIdentificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class OffensiveLanguageIdentification(AssetNode[OffensiveLanguageIdentificationInputs, OffensiveLanguageIdentificationOutputs]): + """ + Offensive Language Identification is a function that analyzes text to detect +and flag language that is abusive, harmful, or inappropriate, helping to +maintain a respectful and safe communication environment. + + InputType: text + OutputType: label + """ + function: str = "offensive-language-identification" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = OffensiveLanguageIdentificationInputs + outputs_class: Type[TO] = OffensiveLanguageIdentificationOutputs + + +class SpeechTranslationInputs(Inputs): + source_audio: InputParam = None + sourcelanguage: InputParam = None + targetlanguage: InputParam = None + dialect: InputParam = None + voice: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.sourcelanguage = self.create_param(code="sourcelanguage", data_type=DataType.LABEL, is_required=True) + self.targetlanguage = self.create_param(code="targetlanguage", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.voice = self.create_param(code="voice", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class SpeechTranslationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class SpeechTranslation(AssetNode[SpeechTranslationInputs, SpeechTranslationOutputs]): + """ + Speech Translation is a technology that converts spoken language in real-time +from one language to another, enabling seamless communication between speakers +of different languages. + + InputType: audio + OutputType: text + """ + function: str = "speech-translation" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SpeechTranslationInputs + outputs_class: Type[TO] = SpeechTranslationOutputs + + +class SpeakerDiarizationAudioInputs(Inputs): + audio: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class SpeakerDiarizationAudioOutputs(Outputs): + data: OutputParam = None + audio: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO) + + +class SpeakerDiarizationAudio(BaseSegmentor[SpeakerDiarizationAudioInputs, SpeakerDiarizationAudioOutputs]): + """ + Speaker Diarization Audio is a process that involves segmenting an audio +recording into distinct sections, each corresponding to a different speaker, in +order to identify and differentiate between multiple speakers within the same +audio stream. + + InputType: audio + OutputType: label + """ + function: str = "speaker-diarization-audio" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SpeakerDiarizationAudioInputs + outputs_class: Type[TO] = SpeakerDiarizationAudioOutputs + + +class AudioTranscriptImprovementInputs(Inputs): + language: InputParam = None + dialect: InputParam = None + source_supplier: InputParam = None + is_medical: InputParam = None + source_audio: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.source_supplier = self.create_param(code="source_supplier", data_type=DataType.LABEL, is_required=False) + self.is_medical = self.create_param(code="is_medical", data_type=DataType.TEXT, is_required=True) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class AudioTranscriptImprovementOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class AudioTranscriptImprovement(AssetNode[AudioTranscriptImprovementInputs, AudioTranscriptImprovementOutputs]): + """ + Audio Transcript Improvement is a function that enhances the accuracy and +clarity of transcribed audio recordings by correcting errors, refining +language, and ensuring the text faithfully represents the original spoken +content. + + InputType: audio + OutputType: text + """ + function: str = "audio-transcript-improvement" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = AudioTranscriptImprovementInputs + outputs_class: Type[TO] = AudioTranscriptImprovementOutputs + + +class SpeechNonSpeechClassificationInputs(Inputs): + audio: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class SpeechNonSpeechClassificationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class SpeechNonSpeechClassification(AssetNode[SpeechNonSpeechClassificationInputs, SpeechNonSpeechClassificationOutputs]): + """ + The function "Speech or Non-Speech Classification" is designed to analyze audio +input and determine whether the sound is human speech or non-speech noise, +enabling applications such as voice recognition systems to filter out +irrelevant background sounds. + + InputType: audio + OutputType: label + """ + function: str = "speech-non-speech-classification" + input_type: str = DataType.AUDIO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SpeechNonSpeechClassificationInputs + outputs_class: Type[TO] = SpeechNonSpeechClassificationOutputs + + +class TextDenormalizationInputs(Inputs): + text: InputParam = None + language: InputParam = None + lowercase_latin: InputParam = None + remove_accents: InputParam = None + remove_punctuation: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.lowercase_latin = self.create_param(code="lowercase_latin", data_type=DataType.TEXT, is_required=False) + self.remove_accents = self.create_param(code="remove_accents", data_type=DataType.TEXT, is_required=False) + self.remove_punctuation = self.create_param(code="remove_punctuation", data_type=DataType.TEXT, is_required=False) + + +class TextDenormalizationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextDenormalization(AssetNode[TextDenormalizationInputs, TextDenormalizationOutputs]): + """ + Text Denormalization is the process of converting abbreviated, contracted, or +otherwise simplified text into its full, standard form, often to improve +readability and ensure consistency in natural language processing tasks. + + InputType: text + OutputType: label + """ + function: str = "text-denormalization" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TextDenormalizationInputs + outputs_class: Type[TO] = TextDenormalizationOutputs + + +class ImageContentModerationInputs(Inputs): + image: InputParam = None + min_confidence: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.image = self.create_param(code="image", data_type=DataType.IMAGE, is_required=True) + self.min_confidence = self.create_param(code="min_confidence", data_type=DataType.TEXT, is_required=False) + + +class ImageContentModerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class ImageContentModeration(AssetNode[ImageContentModerationInputs, ImageContentModerationOutputs]): + """ + Image Content Moderation is a process that involves analyzing and filtering +images to detect and manage inappropriate, harmful, or sensitive content, +ensuring compliance with community guidelines and legal standards. + + InputType: image + OutputType: label + """ + function: str = "image-content-moderation" + input_type: str = DataType.IMAGE + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = ImageContentModerationInputs + outputs_class: Type[TO] = ImageContentModerationOutputs + + +class ReferencelessTextGenerationMetricDefaultInputs(Inputs): + hypotheses: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.TEXT, is_required=True) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class ReferencelessTextGenerationMetricDefaultOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class ReferencelessTextGenerationMetricDefault(AssetNode[ReferencelessTextGenerationMetricDefaultInputs, ReferencelessTextGenerationMetricDefaultOutputs]): + """ + The Referenceless Text Generation Metric Default is a function designed to +evaluate the quality of generated text without relying on reference texts for +comparison. + + InputType: text + OutputType: text + """ + function: str = "referenceless-text-generation-metric-default" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ReferencelessTextGenerationMetricDefaultInputs + outputs_class: Type[TO] = ReferencelessTextGenerationMetricDefaultOutputs + + +class NamedEntityRecognitionInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + domain: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.domain = self.create_param(code="domain", data_type=DataType.LABEL, is_required=False) + + +class NamedEntityRecognitionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class NamedEntityRecognition(AssetNode[NamedEntityRecognitionInputs, NamedEntityRecognitionOutputs]): + """ + Named Entity Recognition (NER) is a natural language processing task that +involves identifying and classifying proper nouns in text into predefined +categories such as names of people, organizations, locations, dates, and other +entities. + + InputType: text + OutputType: label + """ + function: str = "named-entity-recognition" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = NamedEntityRecognitionInputs + outputs_class: Type[TO] = NamedEntityRecognitionOutputs + + +class TextContentModerationInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class TextContentModerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextContentModeration(AssetNode[TextContentModerationInputs, TextContentModerationOutputs]): + """ + Text Content Moderation is the process of reviewing, filtering, and managing +user-generated content to ensure it adheres to community guidelines, legal +standards, and platform policies, thereby maintaining a safe and respectful +online environment. + + InputType: text + OutputType: label + """ + function: str = "text-content-moderation" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TextContentModerationInputs + outputs_class: Type[TO] = TextContentModerationOutputs + + +class SpeakerDiarizationVideoInputs(Inputs): + video: InputParam = None + language: InputParam = None + script: InputParam = None + dialect: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + + +class SpeakerDiarizationVideoOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.VIDEO) + + +class SpeakerDiarizationVideo(AssetNode[SpeakerDiarizationVideoInputs, SpeakerDiarizationVideoOutputs]): + """ + The Speaker Diarization Video function identifies and segments different +speakers in a video, attributing portions of the audio to individual speakers +to facilitate analysis and understanding of multi-speaker conversations. + + InputType: video + OutputType: label + """ + function: str = "speaker-diarization-video" + input_type: str = DataType.VIDEO + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = SpeakerDiarizationVideoInputs + outputs_class: Type[TO] = SpeakerDiarizationVideoOutputs + + +class SplitOnSilenceInputs(Inputs): + audio: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + + +class SplitOnSilenceOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + + +class SplitOnSilence(AssetNode[SplitOnSilenceInputs, SplitOnSilenceOutputs]): + """ + The "Split On Silence" function divides an audio recording into separate +segments based on periods of silence, allowing for easier editing and analysis +of individual sections. + + InputType: audio + OutputType: audio + """ + function: str = "split-on-silence" + input_type: str = DataType.AUDIO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = SplitOnSilenceInputs + outputs_class: Type[TO] = SplitOnSilenceOutputs + + +class EmotionDetectionInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class EmotionDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class EmotionDetection(AssetNode[EmotionDetectionInputs, EmotionDetectionOutputs]): + """ + Emotion Detection is a process that involves analyzing text to identify and +categorize the emotional states or sentiments expressed by individuals, such as +happiness, sadness, anger, or fear. + + InputType: text + OutputType: label + """ + function: str = "emotion-detection" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = EmotionDetectionInputs + outputs_class: Type[TO] = EmotionDetectionOutputs + + +class TextSpamDetectionInputs(Inputs): + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class TextSpamDetectionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextSpamDetection(AssetNode[TextSpamDetectionInputs, TextSpamDetectionOutputs]): + """ + Text Spam Detection is a process that involves analyzing and identifying +unsolicited or irrelevant messages within text communications, typically using +algorithms and machine learning techniques to filter out spam and ensure the +integrity of the communication platform. + + InputType: text + OutputType: label + """ + function: str = "text-spam-detection" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TextSpamDetectionInputs + outputs_class: Type[TO] = TextSpamDetectionOutputs + + +class TranslationInputs(Inputs): + text: InputParam = None + sourcelanguage: InputParam = None + targetlanguage: InputParam = None + script_in: InputParam = None + script_out: InputParam = None + dialect_in: InputParam = None + dialect_out: InputParam = None + context: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.sourcelanguage = self.create_param(code="sourcelanguage", data_type=DataType.LABEL, is_required=True) + self.targetlanguage = self.create_param(code="targetlanguage", data_type=DataType.LABEL, is_required=True) + self.script_in = self.create_param(code="script_in", data_type=DataType.LABEL, is_required=False) + self.script_out = self.create_param(code="script_out", data_type=DataType.LABEL, is_required=False) + self.dialect_in = self.create_param(code="dialect_in", data_type=DataType.LABEL, is_required=False) + self.dialect_out = self.create_param(code="dialect_out", data_type=DataType.LABEL, is_required=False) + self.context = self.create_param(code="context", data_type=DataType.LABEL, is_required=False) + + +class TranslationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class Translation(AssetNode[TranslationInputs, TranslationOutputs]): + """ + Translation is the process of converting text from one language into an +equivalent text in another language, preserving the original meaning and +context. + + InputType: text + OutputType: text + """ + function: str = "translation" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TranslationInputs + outputs_class: Type[TO] = TranslationOutputs + + +class VoiceActivityDetectionInputs(Inputs): + audio: InputParam = None + onset: InputParam = None + offset: InputParam = None + min_duration_on: InputParam = None + min_duration_off: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.onset = self.create_param(code="onset", data_type=DataType.TEXT, is_required=False) + self.offset = self.create_param(code="offset", data_type=DataType.TEXT, is_required=False) + self.min_duration_on = self.create_param(code="min_duration_on", data_type=DataType.TEXT, is_required=False) + self.min_duration_off = self.create_param(code="min_duration_off", data_type=DataType.TEXT, is_required=False) + + +class VoiceActivityDetectionOutputs(Outputs): + data: OutputParam = None + audio: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.AUDIO) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO) + + +class VoiceActivityDetection(BaseSegmentor[VoiceActivityDetectionInputs, VoiceActivityDetectionOutputs]): + """ + Voice Activity Detection (VAD) is a technology that identifies the presence or +absence of human speech within an audio signal, enabling systems to distinguish +between spoken words and background noise. + + InputType: audio + OutputType: audio + """ + function: str = "voice-activity-detection" + input_type: str = DataType.AUDIO + output_type: str = DataType.AUDIO + + inputs_class: Type[TI] = VoiceActivityDetectionInputs + outputs_class: Type[TO] = VoiceActivityDetectionOutputs + + +class SpeechEmbeddingInputs(Inputs): + audio: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.audio = self.create_param(code="audio", data_type=DataType.AUDIO, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class SpeechEmbeddingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class SpeechEmbedding(AssetNode[SpeechEmbeddingInputs, SpeechEmbeddingOutputs]): + """ + Speech Embedding is a process that transforms spoken language into a fixed- +dimensional vector representation, capturing essential features and +characteristics of the speech for tasks such as recognition, classification, +and analysis. + + InputType: audio + OutputType: text + """ + function: str = "speech-embedding" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SpeechEmbeddingInputs + outputs_class: Type[TO] = SpeechEmbeddingOutputs + + +class SubtitlingTranslationInputs(Inputs): + text: InputParam = None + sourcelanguage: InputParam = None + dialect_in: InputParam = None + target_supplier: InputParam = None + targetlanguages: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.sourcelanguage = self.create_param(code="sourcelanguage", data_type=DataType.LABEL, is_required=True) + self.dialect_in = self.create_param(code="dialect_in", data_type=DataType.LABEL, is_required=False) + self.target_supplier = self.create_param(code="target_supplier", data_type=DataType.LABEL, is_required=False) + self.targetlanguages = self.create_param(code="targetlanguages", data_type=DataType.LABEL, is_required=False) + + +class SubtitlingTranslationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class SubtitlingTranslation(AssetNode[SubtitlingTranslationInputs, SubtitlingTranslationOutputs]): + """ + Subtitling Translation is the process of converting spoken dialogue from one +language into written text in another language, which is then displayed on- +screen to aid viewers in understanding the content. + + InputType: text + OutputType: text + """ + function: str = "subtitling-translation" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SubtitlingTranslationInputs + outputs_class: Type[TO] = SubtitlingTranslationOutputs + + +class TextGenerationInputs(Inputs): + text: InputParam = None + prompt: InputParam = None + context: InputParam = None + language: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.prompt = self.create_param(code="prompt", data_type=DataType.TEXT, is_required=False) + self.context = self.create_param(code="context", data_type=DataType.TEXT, is_required=False) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class TextGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class TextGeneration(AssetNode[TextGenerationInputs, TextGenerationOutputs]): + """ + Text Generation is a process in which artificial intelligence models, such as +neural networks, produce coherent and contextually relevant text based on a +given input or prompt, often mimicking human writing styles and patterns. + + InputType: text + OutputType: text + """ + function: str = "text-generation" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = TextGenerationInputs + outputs_class: Type[TO] = TextGenerationOutputs + + +class VideoUnderstandingInputs(Inputs): + video: InputParam = None + text: InputParam = None + language: InputParam = None + dialect: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.video = self.create_param(code="video", data_type=DataType.VIDEO, is_required=True) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class VideoUnderstandingOutputs(Outputs): + text: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT) + + +class VideoUnderstanding(AssetNode[VideoUnderstandingInputs, VideoUnderstandingOutputs]): + """ + Video Understanding is the process of analyzing and interpreting video content +to extract meaningful information, such as identifying objects, actions, +events, and contextual relationships within the footage. + + InputType: video + OutputType: text + """ + function: str = "video-understanding" + input_type: str = DataType.VIDEO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = VideoUnderstandingInputs + outputs_class: Type[TO] = VideoUnderstandingOutputs + + +class TextToVideoGenerationInputs(Inputs): + text: InputParam = None + language: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + + +class TextToVideoGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.VIDEO) + + +class TextToVideoGeneration(AssetNode[TextToVideoGenerationInputs, TextToVideoGenerationOutputs]): + """ + Text To Video Generation is a process that converts written descriptions or +scripts into dynamic, visual video content using advanced algorithms and +artificial intelligence. + + InputType: text + OutputType: video + """ + function: str = "text-to-video-generation" + input_type: str = DataType.TEXT + output_type: str = DataType.VIDEO + + inputs_class: Type[TI] = TextToVideoGenerationInputs + outputs_class: Type[TO] = TextToVideoGenerationOutputs + + +class TextNormalizationInputs(Inputs): + text: InputParam = None + language: InputParam = None + settings: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=False) + self.settings = self.create_param(code="settings", data_type=DataType.TEXT, is_required=False) + + +class TextNormalizationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.LABEL) + + +class TextNormalization(AssetNode[TextNormalizationInputs, TextNormalizationOutputs]): + """ + Text normalization is the process of transforming text into a standard, +consistent format by correcting spelling errors, converting all characters to a +uniform case, removing punctuation, and expanding abbreviations to improve the +text's readability and usability for further processing or analysis. + + InputType: text + OutputType: label + """ + function: str = "text-normalization" + input_type: str = DataType.TEXT + output_type: str = DataType.LABEL + + inputs_class: Type[TI] = TextNormalizationInputs + outputs_class: Type[TO] = TextNormalizationOutputs + + +class SpeechRecognitionInputs(Inputs): + language: InputParam = None + dialect: InputParam = None + voice: InputParam = None + source_audio: InputParam = None + script: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.language = self.create_param(code="language", data_type=DataType.LABEL, is_required=True) + self.dialect = self.create_param(code="dialect", data_type=DataType.LABEL, is_required=False) + self.voice = self.create_param(code="voice", data_type=DataType.LABEL, is_required=False) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.script = self.create_param(code="script", data_type=DataType.LABEL, is_required=False) + + +class SpeechRecognitionOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class SpeechRecognition(AssetNode[SpeechRecognitionInputs, SpeechRecognitionOutputs]): + """ + Speech recognition is a technology that enables a computer or device to +identify and process spoken language, converting it into text. + + InputType: audio + OutputType: text + """ + function: str = "speech-recognition" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SpeechRecognitionInputs + outputs_class: Type[TO] = SpeechRecognitionOutputs + + +class SubtitlingInputs(Inputs): + source_audio: InputParam = None + sourcelanguage: InputParam = None + dialect_in: InputParam = None + source_supplier: InputParam = None + target_supplier: InputParam = None + targetlanguages: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.source_audio = self.create_param(code="source_audio", data_type=DataType.AUDIO, is_required=True) + self.sourcelanguage = self.create_param(code="sourcelanguage", data_type=DataType.LABEL, is_required=True) + self.dialect_in = self.create_param(code="dialect_in", data_type=DataType.LABEL, is_required=False) + self.source_supplier = self.create_param(code="source_supplier", data_type=DataType.LABEL, is_required=False) + self.target_supplier = self.create_param(code="target_supplier", data_type=DataType.LABEL, is_required=False) + self.targetlanguages = self.create_param(code="targetlanguages", data_type=DataType.LABEL, is_required=False) + + +class SubtitlingOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.TEXT) + + +class Subtitling(AssetNode[SubtitlingInputs, SubtitlingOutputs]): + """ + Subtitling is the process of displaying written text on a screen to represent +the spoken dialogue, narration, or other audio elements in a video, typically +to aid viewers who are deaf or hard of hearing, or to provide translations for +audiences who speak different languages. + + InputType: audio + OutputType: text + """ + function: str = "subtitling" + input_type: str = DataType.AUDIO + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = SubtitlingInputs + outputs_class: Type[TO] = SubtitlingOutputs + + +class ClassificationMetricInputs(Inputs): + hypotheses: InputParam = None + references: InputParam = None + lowerIsBetter: InputParam = None + sources: InputParam = None + score_identifier: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.hypotheses = self.create_param(code="hypotheses", data_type=DataType.LABEL, is_required=True) + self.references = self.create_param(code="references", data_type=DataType.LABEL, is_required=True) + self.lowerIsBetter = self.create_param(code="lowerIsBetter", data_type=DataType.TEXT, is_required=False) + self.sources = self.create_param(code="sources", data_type=DataType.TEXT, is_required=False) + self.score_identifier = self.create_param(code="score_identifier", data_type=DataType.TEXT, is_required=True) + + +class ClassificationMetricOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.NUMBER) + + +class ClassificationMetric(AssetNode[ClassificationMetricInputs, ClassificationMetricOutputs]): + """ + A Classification Metric is a quantitative measure used to evaluate the quality +and effectiveness of classification models. + + InputType: text + OutputType: text + """ + function: str = "classification-metric" + input_type: str = DataType.TEXT + output_type: str = DataType.TEXT + + inputs_class: Type[TI] = ClassificationMetricInputs + outputs_class: Type[TO] = ClassificationMetricOutputs + + +class TextToImageGenerationInputs(Inputs): + text: InputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.text = self.create_param(code="text", data_type=DataType.TEXT, is_required=True) + + +class TextToImageGenerationOutputs(Outputs): + data: OutputParam = None + + def __init__(self, node=None): + super().__init__(node=node) + self.data = self.create_param(code="data", data_type=DataType.IMAGE) + + +class TextToImageGeneration(AssetNode[TextToImageGenerationInputs, TextToImageGenerationOutputs]): + """ + Text To Image Generation is a process where a system creates visual images +based on descriptive text input, translating written language into +corresponding graphical representations. + + InputType: text + OutputType: image + """ + function: str = "text-to-image-generation" + input_type: str = DataType.TEXT + output_type: str = DataType.IMAGE + + inputs_class: Type[TI] = TextToImageGenerationInputs + outputs_class: Type[TO] = TextToImageGenerationOutputs + + + +class Pipeline(DefaultPipeline): + + def object_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ObjectDetection: + """ + Object Detection is a computer vision technology that identifies and locates +objects within an image, typically by drawing bounding boxes around the +detected objects and classifying them into predefined categories. + """ + return ObjectDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def language_identification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> LanguageIdentification: + """ + Language Identification is the process of automatically determining the +language in which a given piece of text is written. + """ + return LanguageIdentification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def ocr(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Ocr: + """ + OCR, or Optical Character Recognition, is a technology that converts different +types of documents, such as scanned paper documents, PDFs, or images captured +by a digital camera, into editable and searchable data by recognizing and +extracting text from the images. + """ + return Ocr(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def script_execution(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ScriptExecution: + """ + Script Execution refers to the process of running a set of programmed +instructions or code within a computing environment, enabling the automated +performance of tasks, calculations, or operations as defined by the script. + """ + return ScriptExecution(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_label_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageLabelDetection: + """ + Image Label Detection is a function that automatically identifies and assigns +descriptive tags or labels to objects, scenes, or elements within an image, +enabling easier categorization, search, and analysis of visual content. + """ + return ImageLabelDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_captioning(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageCaptioning: + """ + Image Captioning is a process that involves generating a textual description of +an image, typically using machine learning models to analyze the visual content +and produce coherent and contextually relevant sentences that describe the +objects, actions, and scenes depicted in the image. + """ + return ImageCaptioning(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_language_identification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioLanguageIdentification: + """ + Audio Language Identification is a process that involves analyzing an audio +recording to determine the language being spoken. + """ + return AudioLanguageIdentification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def asr_age_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AsrAgeClassification: + """ + The ASR Age Classification function is designed to analyze audio recordings of +speech to determine the speaker's age group by leveraging automatic speech +recognition (ASR) technology and machine learning algorithms. + """ + return AsrAgeClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def benchmark_scoring_mt(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> BenchmarkScoringMt: + """ + Benchmark Scoring MT is a function designed to evaluate and score machine +translation systems by comparing their output against a set of predefined +benchmarks, thereby assessing their accuracy and performance. + """ + return BenchmarkScoringMt(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def asr_gender_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AsrGenderClassification: + """ + The ASR Gender Classification function analyzes audio recordings to determine +and classify the speaker's gender based on their voice characteristics. + """ + return AsrGenderClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def base_model(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> BaseModel: + """ + The Base-Model function serves as a foundational framework designed to provide +essential features and capabilities upon which more specialized or advanced +models can be built and customized. + """ + return BaseModel(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def language_identification_audio(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> LanguageIdentificationAudio: + """ + The Language Identification Audio function analyzes audio input to determine +and identify the language being spoken. + """ + return LanguageIdentificationAudio(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def loglikelihood(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Loglikelihood: + """ + The Log Likelihood function measures the probability of observing the given +data under a specific statistical model by taking the natural logarithm of the +likelihood function, thereby transforming the product of probabilities into a +sum, which simplifies the process of optimization and parameter estimation. + """ + return Loglikelihood(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_embedding(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoEmbedding: + """ + Video Embedding is a process that transforms video content into a fixed- +dimensional vector representation, capturing essential features and patterns to +facilitate tasks such as retrieval, classification, and recommendation. + """ + return VideoEmbedding(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_segmenation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextSegmenation: + """ + Text Segmentation is the process of dividing a continuous text into meaningful +units, such as words, sentences, or topics, to facilitate easier analysis and +understanding. + """ + return TextSegmenation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_embedding(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageEmbedding: + """ + Image Embedding is a process that transforms an image into a fixed-dimensional +vector representation, capturing its essential features and enabling efficient +comparison, retrieval, and analysis in various machine learning and computer +vision tasks. + """ + return ImageEmbedding(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_manipulation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageManipulation: + """ + Image Manipulation refers to the process of altering or enhancing digital +images using various techniques and tools to achieve desired visual effects, +correct imperfections, or transform the image's appearance. + """ + return ImageManipulation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_to_video_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageToVideoGeneration: + """ + The Image To Video Generation function transforms a series of static images +into a cohesive, dynamic video sequence, often incorporating transitions, +effects, and synchronization with audio to create a visually engaging +narrative. + """ + return ImageToVideoGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_forced_alignment(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioForcedAlignment: + """ + Audio Forced Alignment is a process that synchronizes a given audio recording +with its corresponding transcript by precisely aligning each spoken word or +phoneme to its exact timing within the audio. + """ + return AudioForcedAlignment(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def benchmark_scoring_asr(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> BenchmarkScoringAsr: + """ + Benchmark Scoring ASR is a function that evaluates and compares the performance +of automatic speech recognition systems by analyzing their accuracy, speed, and +other relevant metrics against a standardized set of benchmarks. + """ + return BenchmarkScoringAsr(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def visual_question_answering(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VisualQuestionAnswering: + """ + Visual Question Answering (VQA) is a task in artificial intelligence that +involves analyzing an image and providing accurate, contextually relevant +answers to questions posed about the visual content of that image. + """ + return VisualQuestionAnswering(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def document_image_parsing(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> DocumentImageParsing: + """ + Document Image Parsing is the process of analyzing and converting scanned or +photographed images of documents into structured, machine-readable formats by +identifying and extracting text, layout, and other relevant information. + """ + return DocumentImageParsing(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def document_information_extraction(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> DocumentInformationExtraction: + """ + Document Information Extraction is the process of automatically identifying, +extracting, and structuring relevant data from unstructured or semi-structured +documents, such as invoices, receipts, contracts, and forms, to facilitate +easier data management and analysis. + """ + return DocumentInformationExtraction(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def depth_estimation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> DepthEstimation: + """ + Depth estimation is a computational process that determines the distance of +objects from a viewpoint, typically using visual data from cameras or sensors +to create a three-dimensional understanding of a scene. + """ + return DepthEstimation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoGeneration: + """ + Video Generation is the process of creating video content through automated or +semi-automated means, often utilizing algorithms, artificial intelligence, or +software tools to produce visual and audio elements that can range from simple +animations to complex, realistic scenes. + """ + return VideoGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def referenceless_audio_generation_metric(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ReferencelessAudioGenerationMetric: + """ + The Referenceless Audio Generation Metric is a tool designed to evaluate the +quality of generated audio content without the need for a reference or original +audio sample for comparison. + """ + return ReferencelessAudioGenerationMetric(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def multi_class_image_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> MultiClassImageClassification: + """ + Multi Class Image Classification is a machine learning task where an algorithm +is trained to categorize images into one of several predefined classes or +categories based on their visual content. + """ + return MultiClassImageClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def semantic_segmentation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SemanticSegmentation: + """ + Semantic segmentation is a computer vision process that involves classifying +each pixel in an image into a predefined category, effectively partitioning the +image into meaningful segments based on the objects or regions they represent. + """ + return SemanticSegmentation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def instance_segmentation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> InstanceSegmentation: + """ + Instance segmentation is a computer vision task that involves detecting and +delineating each distinct object within an image, assigning a unique label and +precise boundary to every individual instance of objects, even if they belong +to the same category. + """ + return InstanceSegmentation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_colorization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageColorization: + """ + Image colorization is a process that involves adding color to grayscale images, +transforming them from black-and-white to full-color representations, often +using advanced algorithms and machine learning techniques to predict and apply +the appropriate hues and shades. + """ + return ImageColorization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_generation_metric(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioGenerationMetric: + """ + The Audio Generation Metric is a quantitative measure used to evaluate the +quality, accuracy, and overall performance of audio generated by artificial +intelligence systems, often considering factors such as fidelity, +intelligibility, and similarity to human-produced audio. + """ + return AudioGenerationMetric(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_impainting(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageImpainting: + """ + Image inpainting is a process that involves filling in missing or damaged parts +of an image in a way that is visually coherent and seamlessly blends with the +surrounding areas, often using advanced algorithms and techniques to restore +the image to its original or intended appearance. + """ + return ImageImpainting(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def style_transfer(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> StyleTransfer: + """ + Style Transfer is a technique in artificial intelligence that applies the +visual style of one image (such as the brushstrokes of a famous painting) to +the content of another image, effectively blending the artistic elements of the +first image with the subject matter of the second. + """ + return StyleTransfer(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def multi_class_text_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> MultiClassTextClassification: + """ + Multi Class Text Classification is a natural language processing task that +involves categorizing a given text into one of several predefined classes or +categories based on its content. + """ + return MultiClassTextClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_embedding(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextEmbedding: + """ + Text embedding is a process that converts text into numerical vectors, +capturing the semantic meaning and contextual relationships of words or +phrases, enabling machines to understand and analyze natural language more +effectively. + """ + return TextEmbedding(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def multi_label_text_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> MultiLabelTextClassification: + """ + Multi Label Text Classification is a natural language processing task where a +given text is analyzed and assigned multiple relevant labels or categories from +a predefined set, allowing for the text to belong to more than one category +simultaneously. + """ + return MultiLabelTextClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_reconstruction(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextReconstruction: + """ + Text Reconstruction is a process that involves piecing together fragmented or +incomplete text data to restore it to its original, coherent form. + """ + return TextReconstruction(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def fact_checking(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> FactChecking: + """ + Fact Checking is the process of verifying the accuracy and truthfulness of +information, statements, or claims by cross-referencing with reliable sources +and evidence. + """ + return FactChecking(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechClassification: + """ + Speech Classification is a process that involves analyzing and categorizing +spoken language into predefined categories or classes based on various features +such as tone, pitch, and linguistic content. + """ + return SpeechClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def intent_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> IntentClassification: + """ + Intent Classification is a natural language processing task that involves +analyzing and categorizing user text input to determine the underlying purpose +or goal behind the communication, such as booking a flight, asking for weather +information, or setting a reminder. + """ + return IntentClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def part_of_speech_tagging(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> PartOfSpeechTagging: + """ + Part of Speech Tagging is a natural language processing task that involves +assigning each word in a sentence its corresponding part of speech, such as +noun, verb, adjective, or adverb, based on its role and context within the +sentence. + """ + return PartOfSpeechTagging(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def metric_aggregation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> MetricAggregation: + """ + Metric Aggregation is a function that computes and summarizes numerical data by +applying statistical operations, such as averaging, summing, or finding the +minimum and maximum values, to provide insights and facilitate analysis of +large datasets. + """ + return MetricAggregation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def dialect_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> DialectDetection: + """ + Dialect Detection is a function that identifies and classifies the specific +regional or social variations of a language spoken or written by an individual, +enabling the recognition of distinct linguistic patterns and nuances associated +with different dialects. + """ + return DialectDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def inverse_text_normalization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> InverseTextNormalization: + """ + Inverse Text Normalization is the process of converting spoken or written +language in its normalized form, such as numbers, dates, and abbreviations, +back into their original, more complex or detailed textual representations. + """ + return InverseTextNormalization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_to_audio(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextToAudio: + """ + The Text to Audio function converts written text into spoken words, allowing +users to listen to the content instead of reading it. + """ + return TextToAudio(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def fill_text_mask(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> FillTextMask: + """ + The "Fill Text Mask" function takes a text input with masked or placeholder +characters and replaces those placeholders with specified or contextually +appropriate characters to generate a complete and coherent text output. + """ + return FillTextMask(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_content_moderation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoContentModeration: + """ + Video Content Moderation is the process of reviewing, analyzing, and filtering +video content to ensure it adheres to community guidelines, legal standards, +and platform policies, thereby preventing the dissemination of inappropriate, +harmful, or illegal material. + """ + return VideoContentModeration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def extract_audio_from_video(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ExtractAudioFromVideo: + """ + The "Extract Audio From Video" function allows users to separate and save the +audio track from a video file, enabling them to obtain just the sound without +the accompanying visual content. + """ + return ExtractAudioFromVideo(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_compression(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageCompression: + """ + Image compression is a process that reduces the file size of an image by +removing redundant or non-essential data, while maintaining an acceptable level +of visual quality. + """ + return ImageCompression(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def multilingual_speech_recognition(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> MultilingualSpeechRecognition: + """ + Multilingual Speech Recognition is a technology that enables the automatic +transcription of spoken language into text across multiple languages, allowing +for seamless communication and understanding in diverse linguistic contexts. + """ + return MultilingualSpeechRecognition(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def referenceless_text_generation_metric(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ReferencelessTextGenerationMetric: + """ + The Referenceless Text Generation Metric is a method for evaluating the quality +of generated text without requiring a reference text for comparison, often +leveraging models or algorithms to assess coherence, relevance, and fluency +based on intrinsic properties of the text itself. + """ + return ReferencelessTextGenerationMetric(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_generation_metric_default(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextGenerationMetricDefault: + """ + The "Text Generation Metric Default" function provides a standard set of +evaluation metrics for assessing the quality and performance of text generation +models. + """ + return TextGenerationMetricDefault(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def noise_removal(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> NoiseRemoval: + """ + Noise Removal is a process that involves identifying and eliminating unwanted +random variations or disturbances from an audio signal to enhance the clarity +and quality of the underlying information. + """ + return NoiseRemoval(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_reconstruction(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioReconstruction: + """ + Audio Reconstruction is the process of restoring or recreating audio signals +from incomplete, damaged, or degraded recordings to achieve a high-quality, +accurate representation of the original sound. + """ + return AudioReconstruction(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def voice_cloning(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VoiceCloning: + """ + Voice cloning is a technology that uses artificial intelligence to create a +digital replica of a person's voice, allowing for the generation of speech that +mimics the tone, pitch, and speaking style of the original speaker. + """ + return VoiceCloning(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def diacritization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Diacritization: + """ + Diacritization is the process of adding diacritical marks to letters in a text +to indicate pronunciation, stress, tone, or meaning, often used in languages +such as Arabic, Hebrew, and Vietnamese to provide clarity and accuracy in +written communication. + """ + return Diacritization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_emotion_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioEmotionDetection: + """ + Audio Emotion Detection is a technology that analyzes vocal characteristics and +patterns in audio recordings to identify and classify the emotional state of +the speaker. + """ + return AudioEmotionDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_summarization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextSummarization: + """ + Text summarization is the process of condensing a large body of text into a +shorter version, capturing the main points and essential information while +maintaining coherence and meaning. + """ + return TextSummarization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def entity_linking(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> EntityLinking: + """ + Entity Linking is the process of identifying and connecting mentions of +entities within a text to their corresponding entries in a structured knowledge +base, thereby enabling the disambiguation of terms and enhancing the +understanding of the text's context. + """ + return EntityLinking(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_generation_metric(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextGenerationMetric: + """ + A Text Generation Metric is a quantitative measure used to evaluate the quality +and effectiveness of text produced by natural language processing models, often +assessing aspects such as coherence, relevance, fluency, and adherence to given +prompts or instructions. + """ + return TextGenerationMetric(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def split_on_linebreak(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SplitOnLinebreak: + """ + The "Split On Linebreak" function divides a given string into a list of +substrings, using linebreaks (newline characters) as the points of separation. + """ + return SplitOnLinebreak(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def sentiment_analysis(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SentimentAnalysis: + """ + Sentiment Analysis is a natural language processing technique used to determine +and classify the emotional tone or subjective information expressed in a piece +of text, such as identifying whether the sentiment is positive, negative, or +neutral. + """ + return SentimentAnalysis(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def keyword_spotting(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> KeywordSpotting: + """ + Keyword Spotting is a function that enables the detection and identification of +specific words or phrases within a stream of audio, often used in voice- +activated systems to trigger actions or commands based on recognized keywords. + """ + return KeywordSpotting(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextClassification: + """ + Text Classification is a natural language processing task that involves +categorizing text into predefined labels or classes based on its content, +enabling automated organization, filtering, and analysis of large volumes of +textual data. + """ + return TextClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def other__multipurpose_(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> OtherMultipurpose: + """ + The "Other (Multipurpose)" function serves as a versatile category designed to +accommodate a wide range of tasks and activities that do not fit neatly into +predefined classifications, offering flexibility and adaptability for various +needs. + """ + return OtherMultipurpose(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_synthesis(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechSynthesis: + """ + Speech synthesis is the artificial production of human speech, typically +achieved through software or hardware systems that convert text into spoken +words, enabling machines to communicate verbally with users. + """ + return SpeechSynthesis(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_intent_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioIntentDetection: + """ + Audio Intent Detection is a process that involves analyzing audio signals to +identify and interpret the underlying intentions or purposes behind spoken +words, enabling systems to understand and respond appropriately to human +speech. + """ + return AudioIntentDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_label_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoLabelDetection: + """ + Video Label Detection is a function that automatically identifies and tags +various objects, scenes, activities, and other relevant elements within a +video, providing descriptive labels that enhance searchability and content +organization. + """ + return VideoLabelDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def asr_quality_estimation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AsrQualityEstimation: + """ + ASR Quality Estimation is a process that evaluates the accuracy and reliability +of automatic speech recognition systems by analyzing their performance in +transcribing spoken language into text. + """ + return AsrQualityEstimation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_transcript_analysis(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioTranscriptAnalysis: + """ + Audio Transcript Analysis is a process that involves converting spoken language +from audio recordings into written text, followed by examining and interpreting +the transcribed content to extract meaningful insights, identify patterns, and +derive actionable information. + """ + return AudioTranscriptAnalysis(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def search(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Search: + """ + The "Search" function allows users to input keywords or phrases to quickly +locate specific information, files, or content within a database, website, or +application. + """ + return Search(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_forced_alignment(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoForcedAlignment: + """ + Video Forced Alignment is a process that synchronizes video footage with +corresponding audio tracks by precisely aligning the visual and auditory +elements, ensuring that the movements of speakers' lips match the spoken words. + """ + return VideoForcedAlignment(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def viseme_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VisemeGeneration: + """ + Viseme Generation is the process of creating visual representations of +phonemes, which are the distinct units of sound in speech, to synchronize lip +movements with spoken words in animations or virtual avatars. + """ + return VisemeGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def topic_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TopicClassification: + """ + Topic Classification is a natural language processing function that categorizes +text into predefined topics or subjects based on its content, enabling +efficient organization and retrieval of information. + """ + return TopicClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def offensive_language_identification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> OffensiveLanguageIdentification: + """ + Offensive Language Identification is a function that analyzes text to detect +and flag language that is abusive, harmful, or inappropriate, helping to +maintain a respectful and safe communication environment. + """ + return OffensiveLanguageIdentification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_translation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechTranslation: + """ + Speech Translation is a technology that converts spoken language in real-time +from one language to another, enabling seamless communication between speakers +of different languages. + """ + return SpeechTranslation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speaker_diarization_audio(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeakerDiarizationAudio: + """ + Speaker Diarization Audio is a process that involves segmenting an audio +recording into distinct sections, each corresponding to a different speaker, in +order to identify and differentiate between multiple speakers within the same +audio stream. + """ + return SpeakerDiarizationAudio(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def audio_transcript_improvement(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> AudioTranscriptImprovement: + """ + Audio Transcript Improvement is a function that enhances the accuracy and +clarity of transcribed audio recordings by correcting errors, refining +language, and ensuring the text faithfully represents the original spoken +content. + """ + return AudioTranscriptImprovement(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_non_speech_classification(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechNonSpeechClassification: + """ + The function "Speech or Non-Speech Classification" is designed to analyze audio +input and determine whether the sound is human speech or non-speech noise, +enabling applications such as voice recognition systems to filter out +irrelevant background sounds. + """ + return SpeechNonSpeechClassification(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_denormalization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextDenormalization: + """ + Text Denormalization is the process of converting abbreviated, contracted, or +otherwise simplified text into its full, standard form, often to improve +readability and ensure consistency in natural language processing tasks. + """ + return TextDenormalization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def image_content_moderation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ImageContentModeration: + """ + Image Content Moderation is a process that involves analyzing and filtering +images to detect and manage inappropriate, harmful, or sensitive content, +ensuring compliance with community guidelines and legal standards. + """ + return ImageContentModeration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def referenceless_text_generation_metric_default(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ReferencelessTextGenerationMetricDefault: + """ + The Referenceless Text Generation Metric Default is a function designed to +evaluate the quality of generated text without relying on reference texts for +comparison. + """ + return ReferencelessTextGenerationMetricDefault(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def named_entity_recognition(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> NamedEntityRecognition: + """ + Named Entity Recognition (NER) is a natural language processing task that +involves identifying and classifying proper nouns in text into predefined +categories such as names of people, organizations, locations, dates, and other +entities. + """ + return NamedEntityRecognition(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_content_moderation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextContentModeration: + """ + Text Content Moderation is the process of reviewing, filtering, and managing +user-generated content to ensure it adheres to community guidelines, legal +standards, and platform policies, thereby maintaining a safe and respectful +online environment. + """ + return TextContentModeration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speaker_diarization_video(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeakerDiarizationVideo: + """ + The Speaker Diarization Video function identifies and segments different +speakers in a video, attributing portions of the audio to individual speakers +to facilitate analysis and understanding of multi-speaker conversations. + """ + return SpeakerDiarizationVideo(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def split_on_silence(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SplitOnSilence: + """ + The "Split On Silence" function divides an audio recording into separate +segments based on periods of silence, allowing for easier editing and analysis +of individual sections. + """ + return SplitOnSilence(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def emotion_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> EmotionDetection: + """ + Emotion Detection is a process that involves analyzing text to identify and +categorize the emotional states or sentiments expressed by individuals, such as +happiness, sadness, anger, or fear. + """ + return EmotionDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_spam_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextSpamDetection: + """ + Text Spam Detection is a process that involves analyzing and identifying +unsolicited or irrelevant messages within text communications, typically using +algorithms and machine learning techniques to filter out spam and ensure the +integrity of the communication platform. + """ + return TextSpamDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def translation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Translation: + """ + Translation is the process of converting text from one language into an +equivalent text in another language, preserving the original meaning and +context. + """ + return Translation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def voice_activity_detection(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VoiceActivityDetection: + """ + Voice Activity Detection (VAD) is a technology that identifies the presence or +absence of human speech within an audio signal, enabling systems to distinguish +between spoken words and background noise. + """ + return VoiceActivityDetection(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_embedding(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechEmbedding: + """ + Speech Embedding is a process that transforms spoken language into a fixed- +dimensional vector representation, capturing essential features and +characteristics of the speech for tasks such as recognition, classification, +and analysis. + """ + return SpeechEmbedding(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def subtitling_translation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SubtitlingTranslation: + """ + Subtitling Translation is the process of converting spoken dialogue from one +language into written text in another language, which is then displayed on- +screen to aid viewers in understanding the content. + """ + return SubtitlingTranslation(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextGeneration: + """ + Text Generation is a process in which artificial intelligence models, such as +neural networks, produce coherent and contextually relevant text based on a +given input or prompt, often mimicking human writing styles and patterns. + """ + return TextGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def video_understanding(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> VideoUnderstanding: + """ + Video Understanding is the process of analyzing and interpreting video content +to extract meaningful information, such as identifying objects, actions, +events, and contextual relationships within the footage. + """ + return VideoUnderstanding(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_to_video_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextToVideoGeneration: + """ + Text To Video Generation is a process that converts written descriptions or +scripts into dynamic, visual video content using advanced algorithms and +artificial intelligence. + """ + return TextToVideoGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_normalization(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextNormalization: + """ + Text normalization is the process of transforming text into a standard, +consistent format by correcting spelling errors, converting all characters to a +uniform case, removing punctuation, and expanding abbreviations to improve the +text's readability and usability for further processing or analysis. + """ + return TextNormalization(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def speech_recognition(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> SpeechRecognition: + """ + Speech recognition is a technology that enables a computer or device to +identify and process spoken language, converting it into text. + """ + return SpeechRecognition(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def subtitling(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> Subtitling: + """ + Subtitling is the process of displaying written text on a screen to represent +the spoken dialogue, narration, or other audio elements in a video, typically +to aid viewers who are deaf or hard of hearing, or to provide translations for +audiences who speak different languages. + """ + return Subtitling(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def classification_metric(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> ClassificationMetric: + """ + A Classification Metric is a quantitative measure used to evaluate the quality +and effectiveness of classification models. + """ + return ClassificationMetric(*args, asset_id=asset_id, pipeline=self, **kwargs) + + def text_to_image_generation(self, asset_id: Union[str, asset.Asset], *args, **kwargs) -> TextToImageGeneration: + """ + Text To Image Generation is a process where a system creates visual images +based on descriptive text input, translating written language into +corresponding graphical representations. + """ + return TextToImageGeneration(*args, asset_id=asset_id, pipeline=self, **kwargs) + diff --git a/pyproject.toml b/pyproject.toml index 73980717..5b0ded4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,7 +51,8 @@ dependencies = [ "filetype>=1.2.0", "click>=7.1.2,<8.0.0", "PyYAML>=6.0.1", - "dataclasses-json>=0.5.2" + "dataclasses-json>=0.5.2", + "Jinja2==3.1.4", ] [project.urls] diff --git a/tests/functional/pipelines/data/script.py b/tests/functional/pipelines/data/script.py new file mode 100644 index 00000000..3403fa61 --- /dev/null +++ b/tests/functional/pipelines/data/script.py @@ -0,0 +1,51 @@ +__author__ = "thiagocastroferreira" + +import argparse +import json + + +def main(transcripts, speakers, output_file): + # get the speech recognition json + transcripts = json.load(open(transcripts)) + # get the speaker diarization json + speakers = json.load(open(speakers)) + + # build the response + response = [] + for i, transcript in enumerate(transcripts): + merge = { + "transcript": transcript["attributes"]["data"], + "speaker": speakers[i]["attributes"]["data"]["data"], + } + response.append( + { + "index": i, + "success": True, + "input_type": "text", + "is_url": transcript["is_url"], + "details": {}, + "input_segment_info": transcript["input_segment_info"], + "attributes": {"data": merge, "input": merge}, + } + ) + + # save the response, based on the intermediate representation format, in the output_file + with open(output_file, "w") as f: + json.dump(response, f) + + +if __name__ == "__main__": + # Create the parser + parser = argparse.ArgumentParser() + # Add arguments + parser.add_argument("--transcripts", type=str, required=True) + parser.add_argument("--speakers", type=str, required=True) + parser.add_argument("--output_file", type=str, required=True) + # Parse the argument + args = parser.parse_args() + + transcripts = args.transcripts + speakers = args.speakers + output_file = args.output_file + + main(transcripts, speakers, output_file) diff --git a/tests/functional/pipelines/designer_test.py b/tests/functional/pipelines/designer_test.py new file mode 100644 index 00000000..62f42f7e --- /dev/null +++ b/tests/functional/pipelines/designer_test.py @@ -0,0 +1,248 @@ +import pytest + +from aixplain.enums import DataType +from aixplain.factories import PipelineFactory +from aixplain.modules.pipeline.designer import ( + Link, + Operation, + Route, + RouteType, +) +from aixplain.modules import Pipeline +from aixplain.modules.pipeline.designer import AssetNode +from uuid import uuid4 + + +@pytest.fixture +def pipeline(): + # Setup: Initialize the pipeline + pipeline = PipelineFactory.init( + name=str(uuid4()), + ) + + # Yield control back to the test function + yield pipeline + + # Teardown: Ensure the pipeline is deleted + if pipeline is not None: + pipeline.delete() + + +def test_create_asr_pipeline(pipeline): + # add nodes to the pipeline + input = pipeline.input() + model1 = AssetNode(asset_id="60ddefab8d38c51c5885ee38") + pipeline.add_node(model1) + + model2 = AssetNode(asset_id="60ddefd68d38c51c588608f1") + pipeline.add_node(model2) + + # link the nodes + link1 = Link( + from_node=input, + to_node=model1, + from_param="input", + to_param="source_audio", + ) + pipeline.add_link(link1) + + link2 = Link( + from_node=model1, + to_node=model2, + from_param="data", + to_param="text", + ) + pipeline.add_link(link2) + + # use the output of the last node + model1.use_output("data") + model2.use_output("data") + + # save the pipeline as draft + pipeline.save() + + assert isinstance(pipeline, Pipeline) + assert pipeline.id != "" + + +def test_create_mt_pipeline_and_run(pipeline): + # add nodes to the pipeline + input = pipeline.input() + model1 = pipeline.translation(asset_id="60ddef828d38c51c5885d491") + output = pipeline.output() + + # link the nodes + input.link( + to_node=model1, + from_param=input.outputs.input, + to_param=model1.inputs.text, + ) + + # use the output of the last node + model1.link( + to_node=output, + from_param=model1.outputs.data, + to_param=output.inputs.output, + ) + + # save the pipeline as an asset + pipeline.save(save_as_asset=True) + + assert isinstance(pipeline, Pipeline) + assert pipeline.id != "" + + pipeline = PipelineFactory.get(pipeline.id) + + # run the pipeline + output = pipeline.run( + "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.txt", + **{"batchmode": False, "version": "2.0"}, + ) + assert output["status"] == "SUCCESS" + + +def test_routing_pipeline(pipeline): + + TRANSLATION_ASSET = "60ddefae8d38c51c5885eff7" + SPEECH_RECOGNITION_ASSET = "621cf3fa6442ef511d2830af" + + input = pipeline.input() + translation = pipeline.asset(TRANSLATION_ASSET) + speech_recognition = pipeline.asset(SPEECH_RECOGNITION_ASSET) + + input.route( + translation.inputs.text, speech_recognition.inputs.source_audio + ) + + translation.use_output("data") + speech_recognition.use_output("data") + + pipeline.save() + + output = pipeline.run("This is a sample text!") + + assert output["status"] == "SUCCESS" + assert output.get("data") is not None + assert len(output["data"]) > 0 + assert output["data"][0].get("segments") is not None + assert len(output["data"][0]["segments"]) > 0 + + +def test_scripting_pipeline(pipeline): + + SPEAKER_DIARIZATION_AUDIO_ASSET = "62fab6ecb39cca09ca5bc365" + SPEECH_RECOGNITION_ASSET = "621cf3fa6442ef511d2830af" + + input = pipeline.input() + + segmentor = pipeline.speaker_diarization_audio( + asset_id=SPEAKER_DIARIZATION_AUDIO_ASSET + ) + + speech_recognition = pipeline.speech_recognition( + asset_id=SPEECH_RECOGNITION_ASSET + ) + + script = pipeline.script( + script_path="tests/functional/pipelines/data/script.py" + ) + script.inputs.create_param(code="transcripts", data_type=DataType.TEXT) + script.inputs.create_param(code="speakers", data_type=DataType.LABEL) + script.outputs.create_param(code="data", data_type=DataType.TEXT) + + input.outputs.input.link(segmentor.inputs.audio) + segmentor.outputs.audio.link(speech_recognition.inputs.source_audio) + segmentor.outputs.data.link(script.inputs.speakers) + speech_recognition.outputs.data.link(script.inputs.transcripts) + + script.use_output("data") + + pipeline.save() + + output = pipeline.run( + "s3://aixplain-platform-assets/samples/en/CPAC1x2.wav", + version="2.0", + ) + + assert output["status"] == "SUCCESS" + assert output.get("data") is not None + assert len(output["data"]) > 0 + assert output["data"][0].get("segments") is not None + assert len(output["data"][0]["segments"]) > 0 + + +def test_decision_pipeline(pipeline): + + SENTIMENT_ANALYSIS_ASSET = "6172874f720b09325cbcdc33" + + input = pipeline.input() + + sentiment_analysis = pipeline.sentiment_analysis( + asset_id=SENTIMENT_ANALYSIS_ASSET + ) + + positive_output = pipeline.output() + negative_output = pipeline.output() + decision_node = pipeline.decision( + routes=[ + Route( + type=RouteType.CHECK_VALUE, + operation=Operation.EQUAL, + value="POSITIVE", + path=[positive_output], + ), + Route( + type=RouteType.CHECK_VALUE, + operation=Operation.DIFFERENT, + value="POSITIVE", + path=[negative_output], + ), + ] + ) + + input.outputs.input.link(sentiment_analysis.inputs.text) + sentiment_analysis.outputs.data.link(decision_node.inputs.comparison) + input.outputs.input.link(decision_node.inputs.passthrough) + decision_node.outputs.input.link(positive_output.inputs.output) + decision_node.outputs.input.link(negative_output.inputs.output) + + pipeline.save() + + output = pipeline.run("I feel so bad today!") + + assert output["status"] == "SUCCESS" + assert output.get("data") is not None + assert len(output["data"]) > 0 + assert output["data"][0].get("segments") is not None + assert len(output["data"][0]["segments"]) > 0 + + +def test_reconstructing_pipeline(pipeline): + input = pipeline.input() + + segmentor = pipeline.speaker_diarization_audio( + asset_id="62fab6ecb39cca09ca5bc365" + ) + + speech_recognition = pipeline.speech_recognition( + asset_id="60ddefab8d38c51c5885ee38" + ) + + reconstructor = pipeline.bare_reconstructor() + + input.outputs.input.link(segmentor.inputs.audio) + segmentor.outputs.audio.link(speech_recognition.inputs.source_audio) + speech_recognition.outputs.data.link(reconstructor.inputs.data) + + reconstructor.use_output("data") + + pipeline.save() + + output = pipeline.run( + "s3://aixplain-platform-assets/samples/en/CPAC1x2.wav", + ) + assert output["status"] == "SUCCESS" + assert output.get("data") is not None + assert len(output["data"]) > 0 + assert output["data"][0].get("segments") is not None + assert len(output["data"][0]["segments"]) > 0 diff --git a/tests/unit/designer_test.py b/tests/unit/designer_test.py new file mode 100644 index 00000000..766c7e54 --- /dev/null +++ b/tests/unit/designer_test.py @@ -0,0 +1,707 @@ +import pytest +import unittest.mock as mock + + +from aixplain.enums import DataType +from aixplain.modules.pipeline.designer.base import ( + Node, + Link, + Param, + ParamProxy, + Inputs, + Outputs, + InputParam, + OutputParam, +) + +from aixplain.modules.pipeline.designer.enums import ( + ParamType, + NodeType, +) + +from aixplain.modules.pipeline.designer.mixins import LinkableMixin +from aixplain.modules.pipeline.designer.pipeline import DesignerPipeline + + +def test_create_node(): + + pipeline = DesignerPipeline() + + class BareNode(Node): + pass + + with mock.patch( + "aixplain.modules.pipeline.designer.Node.attach_to" + ) as mock_attach_to: + node = BareNode(number=3, label="FOO") + mock_attach_to.assert_not_called() + assert isinstance(node.inputs, Inputs) + assert isinstance(node.outputs, Outputs) + assert node.number == 3 + assert node.label == "FOO" + + class FooNodeInputs(Inputs): + pass + + class FooNodeOutputs(Outputs): + pass + + class FooNode(Node[FooNodeInputs, FooNodeOutputs]): + inputs_class = FooNodeInputs + outputs_class = FooNodeOutputs + + with mock.patch( + "aixplain.modules.pipeline.designer.Node.attach_to" + ) as mock_attach_to: + node = FooNode(pipeline=pipeline, number=3, label="FOO") + mock_attach_to.assert_called_once_with(pipeline) + assert isinstance(node.inputs, FooNodeInputs) + assert isinstance(node.outputs, FooNodeOutputs) + assert node.number == 3 + assert node.label == "FOO" + + +def test_node_attach_to(): + + pipeline = DesignerPipeline() + + class BareNode(Node): + pass + + node = BareNode() + with pytest.raises(AssertionError) as excinfo: + node.attach_to(pipeline) + + assert "Node type not set" in str(excinfo.value) + + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + c = AssetNode() + d = AssetNode(number=8) + e = AssetNode(number=8) + + a.attach_to(pipeline) + b.attach_to(pipeline) + assert b.pipeline is pipeline + assert b.number == 1 + assert b.label == "ASSET(ID=1)" + assert b in pipeline.nodes + assert len(pipeline.nodes) == 2 + + c.attach_to(pipeline) + assert c.pipeline is pipeline + assert c.number == 2 + assert c.label == "ASSET(ID=2)" + assert c in pipeline.nodes + assert len(pipeline.nodes) == 3 + + d.attach_to(pipeline) + assert d.pipeline is pipeline + assert d.number == 8 + assert d.label == "ASSET(ID=8)" + assert d in pipeline.nodes + assert len(pipeline.nodes) == 4 + + with pytest.raises(AssertionError) as excinfo: + e.attach_to(pipeline) + + assert "Node number already exists" in str(excinfo.value) + + +def test_node_serialize(): + pipeline = DesignerPipeline() + + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + with mock.patch.object(node.inputs, "serialize") as mock_inputs_serialize: + with mock.patch.object( + node.outputs, "serialize" + ) as mock_outputs_serialize: + assert node.serialize() == { + "number": node.number, + "type": NodeType.ASSET, + "inputValues": mock_inputs_serialize.return_value, + "outputValues": mock_outputs_serialize.return_value, + "label": node.label, + } + node.attach_to(pipeline) + mock_inputs_serialize.assert_called_once() + mock_outputs_serialize.assert_called_once() + mock_inputs_serialize.reset_mock() + mock_outputs_serialize.reset_mock() + + assert node.serialize() == { + "number": node.number, + "type": NodeType.ASSET, + "inputValues": mock_inputs_serialize.return_value, + "outputValues": mock_outputs_serialize.return_value, + "label": node.label, + } + mock_inputs_serialize.assert_called_once() + mock_outputs_serialize.assert_called_once() + + +def test_create_param(): + + class TypedParam(Param): + param_type = ParamType.INPUT + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = TypedParam( + code="param", + data_type=DataType.TEXT, + value="foo", + ) + mock_attach_to.assert_not_called() + + assert param.code == "param" + assert param.data_type == DataType.TEXT + assert param.value == "foo" + assert param.param_type == ParamType.INPUT + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = TypedParam( + code="param", + data_type=DataType.TEXT, + value="foo", + param_type=ParamType.OUTPUT, + ) + mock_attach_to.assert_not_called() + + assert param.code == "param" + assert param.data_type == DataType.TEXT + assert param.value == "foo" + assert param.param_type == ParamType.INPUT + + class UnTypedParam(Param): + pass + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = UnTypedParam( + code="param", + data_type=DataType.TEXT, + value="foo", + param_type=ParamType.OUTPUT, + ) + mock_attach_to.assert_not_called() + + assert param.param_type == ParamType.OUTPUT + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = UnTypedParam( + code="param", + data_type=DataType.TEXT, + value="foo", + param_type=ParamType.INPUT, + ) + mock_attach_to.assert_not_called() + + assert param.param_type == ParamType.INPUT + + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = UnTypedParam( + code="param", + data_type=DataType.TEXT, + value="foo", + param_type=ParamType.INPUT, + node=node, + ) + mock_attach_to.assert_called_once_with(node) + + +@pytest.mark.parametrize( + "param_cls, expected_param_type", + [ + (InputParam, ParamType.INPUT), + (OutputParam, ParamType.OUTPUT), + ], +) +def test_create_input_output_param(param_cls, expected_param_type): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + with mock.patch( + "aixplain.modules.pipeline.designer.Param.attach_to" + ) as mock_attach_to: + param = param_cls( + code="param", data_type=DataType.TEXT, value="foo", node=node + ) + mock_attach_to.assert_called_once_with(node) + assert param.code == "param" + assert param.data_type == DataType.TEXT + assert param.value == "foo" + assert param.param_type == expected_param_type + assert not param.node + + +def test_param_attach_to(): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + class NoTypeParam(Param): + pass + + param = NoTypeParam(code="param", data_type=DataType.TEXT, value="foo") + with pytest.raises(AssertionError) as excinfo: + param.attach_to(node) + + assert "Param type not set" in str(excinfo.value) + + input = InputParam(code="input", data_type=DataType.TEXT, value="foo") + + with mock.patch.object(node.inputs, "add_param") as mock_add_param: + input.attach_to(node) + mock_add_param.assert_called_once_with(input) + assert input.node is node + + with pytest.raises(AssertionError) as excinfo: + input.attach_to(node) + + assert "Param already attached to a node" in str(excinfo.value) + + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar") + + with mock.patch.object(node.outputs, "add_param") as mock_add_param: + output.attach_to(node) + mock_add_param.assert_called_once_with(output) + assert output.node is node + + +def test_param_link(): + input = InputParam(code="input", data_type=DataType.TEXT, value="foo") + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar") + + with pytest.raises(AssertionError) as excinfo: + output.link(input) + + assert "Param not attached to a node" in str(excinfo.value) + + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar") + output.node = a + input = InputParam(code="input", data_type=DataType.TEXT, value="foo") + input.node = b + + with pytest.raises(AssertionError) as excinfo: + input.link(output) + + assert "Invalid param type" in str(excinfo.value) + + with pytest.raises(AssertionError) as excinfo: + output.link(input) + + assert "Param not registered as output" in str(excinfo.value) + + output = OutputParam( + code="output", data_type=DataType.TEXT, value="bar", node=a + ) + input = InputParam( + code="input", data_type=DataType.TEXT, value="foo", node=b + ) + + with mock.patch.object(input, "back_link") as mock_back_link: + output.link(input) + mock_back_link.assert_called_once_with(output) + + +def test_param_back_link(): + input = InputParam(code="input", data_type=DataType.TEXT, value="foo") + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar") + + with pytest.raises(AssertionError) as excinfo: + input.back_link(output) + + assert "Param not attached to a node" in str(excinfo.value) + + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar") + output.node = a + input = InputParam(code="input", data_type=DataType.TEXT, value="foo") + input.node = b + + with pytest.raises(AssertionError) as excinfo: + output.back_link(input) + + assert "Invalid param type" in str(excinfo.value) + + with pytest.raises(AssertionError) as excinfo: + input.back_link(output) + + assert "Param not registered as input" in str(excinfo.value) + + output = OutputParam( + code="output", data_type=DataType.TEXT, value="bar", node=a + ) + input = InputParam( + code="input", data_type=DataType.TEXT, value="foo", node=b + ) + + with mock.patch.object(a, "link") as mock_link: + input.back_link(output) + mock_link.assert_called_once_with(b, output, input) + + +def test_create_pipeline(): + pipeline = DesignerPipeline() + + assert pipeline.nodes == [] + assert pipeline.links == [] + assert not pipeline.instance + + +def test_link_create(): + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + + with pytest.raises(AssertionError) as excinfo: + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + assert "Invalid from param" in str(excinfo.value) + + a.outputs.create_param("output", DataType.TEXT, "foo") + + with pytest.raises(AssertionError) as excinfo: + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + assert "Invalid to param" in str(excinfo.value) + + b.inputs.create_param("input", DataType.TEXT, "bar") + + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + assert link.from_node == a + assert link.to_node == b + assert link.from_param == "output" + assert link.to_param == "input" + + pipeline = DesignerPipeline() + + with mock.patch( + "aixplain.modules.pipeline.designer.Link.attach_to" + ) as mock_attach_to: + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + pipeline=pipeline, + ) + mock_attach_to.assert_called_once_with(pipeline) + + +def test_link_attach_to(): + + pipeline = DesignerPipeline() + + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + + a.outputs.create_param("output", DataType.TEXT, "foo") + b.inputs.create_param("input", DataType.TEXT, "bar") + + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + with mock.patch.object(a, "attach_to") as mock_a_attach_to: + with mock.patch.object(b, "attach_to") as mock_b_attach_to: + link.attach_to(pipeline) + mock_a_attach_to.assert_called_once_with(pipeline) + mock_b_attach_to.assert_called_once_with(pipeline) + assert link.pipeline is pipeline + assert link in pipeline.links + + a = AssetNode(pipeline=pipeline) + b = AssetNode(pipeline=pipeline) + a.outputs.create_param("output", DataType.TEXT, "foo") + b.inputs.create_param("input", DataType.TEXT, "bar") + + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + with mock.patch.object(a, "attach_to") as mock_a_attach_to: + with mock.patch.object(b, "attach_to") as mock_b_attach_to: + link.attach_to(pipeline) + mock_a_attach_to.assert_not_called() + mock_b_attach_to.assert_not_called() + assert link.pipeline is pipeline + assert link in pipeline.links + + with pytest.raises(AssertionError) as excinfo: + link.attach_to(pipeline) + + assert "Link already attached to a pipeline" in str(excinfo.value) + + +def test_link_serialize(): + pipeline = DesignerPipeline() + + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + a.outputs.create_param("output", DataType.TEXT, "foo") + b.inputs.create_param("input", DataType.TEXT, "bar") + + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + with pytest.raises(AssertionError) as excinfo: + link.serialize() + + assert "From node number not set" in str(excinfo.value) + a.attach_to(pipeline) + + with pytest.raises(AssertionError) as excinfo: + link.serialize() + + assert "To node number not set" in str(excinfo.value) + b.attach_to(pipeline) + + link = Link( + from_node=a, + to_node=b, + from_param="output", + to_param="input", + ) + + assert link.serialize() == { + "from": a.number, + "to": b.number, + "paramMapping": [ + {"from": "output", "to": "input"}, + ], + } + + +def test_create_param_proxy(): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + param_proxy = ParamProxy(node) + assert param_proxy.node is node + assert param_proxy._params == [] + + +def test_param_proxy_add_param(): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + param_proxy = ParamProxy(node) + + class FooParam(Param): + pass + + param = FooParam(code="foo", data_type=DataType.TEXT) + param_proxy.add_param(param) + assert param in param_proxy._params + assert hasattr(param_proxy, "foo") + assert param_proxy.foo is param + assert param_proxy.foo.code == "foo" + assert param_proxy.foo.data_type == DataType.TEXT + + with pytest.raises(ValueError) as excinfo: + param_proxy.add_param(param) + + assert "Parameter with code 'foo' already exists." in str(excinfo.value) + + +def test_param_proxy_create_param(): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + param_proxy = ParamProxy(node) + + with mock.patch.object(param_proxy, "_create_param") as mock_create_param: + with mock.patch.object(param_proxy, "add_param") as mock_add_param: + param = param_proxy.create_param( + "foo", DataType.TEXT, "bar", is_required=True + ) + mock_create_param.assert_called_once_with( + "foo", DataType.TEXT, "bar" + ) + mock_add_param.assert_called_once_with(param) + assert param.is_required is True + + +def test_param_proxy_attr_access(): + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node = AssetNode() + + param_proxy = ParamProxy(node) + + class FooParam(Param): + pass + + param = FooParam(code="foo", data_type=DataType.TEXT) + param_proxy.add_param(param) + + assert param in param_proxy + assert "foo" in param_proxy + assert param_proxy["foo"] is param + assert param_proxy.foo is param + + with pytest.raises(KeyError) as excinfo: + param_proxy["bar"] + + assert "'bar'" in str(excinfo.value) + + +def test_node_link(): + + class AssetNode(Node, LinkableMixin): + type: NodeType = NodeType.ASSET + + a = AssetNode() + b = AssetNode() + + output = OutputParam( + code="output", data_type=DataType.TEXT, value="bar", node=a + ) + input = InputParam( + code="input", data_type=DataType.TEXT, value="foo", node=b + ) + + # here too lazy to mock Link class properly + # checking the output instance instead + link = a.link(b, from_param=output, to_param=input) + assert isinstance(link, Link) + assert link.from_node == a + assert link.to_node == b + assert link.from_param == "output" + assert link.to_param == "input" + + +def test_pipeline_add_node(): + pipeline = DesignerPipeline() + + class InputNode(Node): + type: NodeType = NodeType.INPUT + + node = InputNode() + pipeline.add_node(node) + assert pipeline.nodes == [node] + assert pipeline.links == [] + + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + node1 = AssetNode() + with mock.patch.object(node1, "attach_to") as mock_attach_to: + pipeline.add_node(node1) + mock_attach_to.assert_called_once_with(pipeline) + + +def test_pipeline_add_nodes(): + pipeline = DesignerPipeline() + + class InputNode(Node): + type: NodeType = NodeType.INPUT + + node = InputNode() + + with mock.patch.object(pipeline, "add_node") as mock_add_node: + pipeline.add_nodes(node) + assert mock_add_node.call_count == 1 + + node1 = InputNode() + node2 = InputNode() + + with mock.patch.object(pipeline, "add_node") as mock_add_node: + pipeline.add_nodes(node1, node2) + assert mock_add_node.call_count == 2 + + +def test_pipeline_add_link(): + pipeline = DesignerPipeline() + + class AssetNode(Node): + type: NodeType = NodeType.ASSET + + a = AssetNode() + a.outputs.create_param("output", DataType.TEXT) + b = AssetNode() + b.inputs.create_param("input", DataType.TEXT) + + link = Link(from_node=a, to_node=b, from_param="output", to_param="input") + pipeline.add_link(link) + + with mock.patch.object(link, "attach_to") as mock_attach_to: + pipeline.add_link(link) + mock_attach_to.assert_called_once_with(pipeline) From ef16dd5ff85ebc23d64bbc3c9674ef9a649c68bb Mon Sep 17 00:00:00 2001 From: mikelam-us-aixplain <131073216+mikelam-us-aixplain@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:31:34 -0700 Subject: [PATCH 19/25] Updated image upload tests (#213) * Updated image upload tests Signed-off-by: root * Added back Hugging Face onboarding Signed-off-by: Michael Lam --------- Signed-off-by: root Signed-off-by: Michael Lam Co-authored-by: root --- aixplain/factories/model_factory.py | 2 +- tests/functional/model/hf_onboarding_test.py | 14 ++++++------- .../model}/image_upload_e2e_test.py | 21 ++++++++----------- .../model}/image_upload_functional_test.py | 18 +++++++--------- tests/test_requests/create_asset_request.json | 9 ++++---- tests/{ => unit}/image_upload_test.py | 16 +++++++------- 6 files changed, 36 insertions(+), 44 deletions(-) rename tests/{ => functional/model}/image_upload_e2e_test.py (72%) rename tests/{ => functional/model}/image_upload_functional_test.py (81%) rename tests/{ => unit}/image_upload_test.py (85%) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index 0fb845f1..c11d837a 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -361,7 +361,7 @@ def create_asset_repo( if function_dict["name"] == function: function_id = function_dict["id"] if function_id is None: - raise Exception("Invalid function name") + raise Exception(f"Invalid function name {function}") create_url = urljoin(config.BACKEND_URL, f"sdk/models/onboard") logging.debug(f"URL: {create_url}") if api_key: diff --git a/tests/functional/model/hf_onboarding_test.py b/tests/functional/model/hf_onboarding_test.py index 47a38361..fa68d2e8 100644 --- a/tests/functional/model/hf_onboarding_test.py +++ b/tests/functional/model/hf_onboarding_test.py @@ -13,7 +13,7 @@ def test_deploy_model(): # Start the deployment model_name = "Test Model" repo_id = "tiiuae/falcon-7b" - response = ModelFactory.deploy_huggingface_model(model_name, repo_id, config.HF_TOKEN) + response = ModelFactory.deploy_huggingface_model(model_name, repo_id, hf_token=config.HF_TOKEN) assert "id" in response.keys() # Check for status @@ -30,31 +30,31 @@ def test_deploy_model(): delete_asset(model_id, config.TEAM_API_KEY) -@pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") +# @pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") def test_nonexistent_model(): # Start the deployment model_name = "Test Model" repo_id = "nonexistent-supplier/nonexistent-model" - response = ModelFactory.deploy_huggingface_model(model_name, repo_id, config.HF_TOKEN) + response = ModelFactory.deploy_huggingface_model(model_name, repo_id, hf_token=config.HF_TOKEN) assert response["statusCode"] == 400 assert response["message"] == "err.unable_to_onboard_model" -@pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") +# @pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") def test_size_limit(): # Start the deployment model_name = "Test Model" repo_id = "tiiuae/falcon-40b" - response = ModelFactory.deploy_huggingface_model(model_name, repo_id, config.HF_TOKEN) + response = ModelFactory.deploy_huggingface_model(model_name, repo_id, hf_token=config.HF_TOKEN) assert response["statusCode"] == 400 assert response["message"] == "err.unable_to_onboard_model" -@pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") +# @pytest.mark.skip(reason="Model Deployment is deactivated for improvements.") def test_gated_model(): # Start the deployment model_name = "Test Model" repo_id = "meta-llama/Llama-2-7b-hf" - response = ModelFactory.deploy_huggingface_model(model_name, repo_id, "mock_key") + response = ModelFactory.deploy_huggingface_model(model_name, repo_id, hf_token="mock_key") assert response["statusCode"] == 400 assert response["message"] == "err.unable_to_onboard_model" diff --git a/tests/image_upload_e2e_test.py b/tests/functional/model/image_upload_e2e_test.py similarity index 72% rename from tests/image_upload_e2e_test.py rename to tests/functional/model/image_upload_e2e_test.py index 0e2ccbc5..7c7efbcc 100644 --- a/tests/image_upload_e2e_test.py +++ b/tests/functional/model/image_upload_e2e_test.py @@ -9,7 +9,6 @@ import pytest -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_create_and_upload_model(): # List the host machines host_response = ModelFactory.list_host_machines() @@ -31,14 +30,15 @@ def test_create_and_upload_model(): # Register the model, and create an image repository for it. with open(Path("tests/test_requests/create_asset_request.json")) as f: - register_payload = json.load(f) - name = register_payload["name"] - host_machine = register_payload["hostingMachine"] - version = register_payload["version"] - description = register_payload["description"] - function = register_payload["function"] - source_language = register_payload["sourceLanguage"] - register_response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) + mock_register_payload = json.load(f) + name = mock_register_payload["name"] + description = mock_register_payload["description"] + function = mock_register_payload["function"] + source_language = mock_register_payload["sourceLanguage"] + input_modality = mock_register_payload["input_modality"] + output_modality = mock_register_payload["output_modality"] + documentation_url = mock_register_payload["documentation_url"] + register_response = ModelFactory.create_asset_repo(name, description, function, source_language, input_modality, output_modality, documentation_url, config.TEAM_API_KEY) assert "id" in register_response.keys() assert "repositoryName" in register_response.keys() model_id = register_response["id"] @@ -56,10 +56,7 @@ def test_create_and_upload_model(): registry = login_response["registry"] # Push an image to ECR - # os.system("aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 535945872701.dkr.ecr.us-east-1.amazonaws.com") low_level_client = docker.APIClient(base_url="unix://var/run/docker.sock") - # low_level_client.pull("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash") - # low_level_client.tag("535945872701.dkr.ecr.us-east-1.amazonaws.com/bash", f"{registry}/{repo_name}") low_level_client.pull("bash") low_level_client.tag("bash", f"{registry}/{repo_name}") low_level_client.push(f"{registry}/{repo_name}", auth_config={"username": username, "password": password}) diff --git a/tests/image_upload_functional_test.py b/tests/functional/model/image_upload_functional_test.py similarity index 81% rename from tests/image_upload_functional_test.py rename to tests/functional/model/image_upload_functional_test.py index b9dd3ebf..60d1d3f0 100644 --- a/tests/image_upload_functional_test.py +++ b/tests/functional/model/image_upload_functional_test.py @@ -1,13 +1,12 @@ __author__ = "michaellam" from pathlib import Path import json -from aixplain.utils import config -from tests.test_utils import delete_asset, delete_service_account from aixplain.factories.model_factory import ModelFactory +from tests.test_utils import delete_asset, delete_service_account +from aixplain.utils import config +import docker import pytest - -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_login(): response = ModelFactory.asset_repo_login() assert response["username"] == "AWS" @@ -18,18 +17,17 @@ def test_login(): delete_service_account(config.TEAM_API_KEY) -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_create_asset_repo(): with open(Path("tests/test_requests/create_asset_request.json")) as f: mock_register_payload = json.load(f) name = mock_register_payload["name"] - host_machine = mock_register_payload["hostingMachine"] - version = mock_register_payload["version"] description = mock_register_payload["description"] function = mock_register_payload["function"] source_language = mock_register_payload["sourceLanguage"] - response = ModelFactory.create_asset_repo(name, host_machine, version, description, function, source_language) - print(response) + input_modality = mock_register_payload["input_modality"] + output_modality = mock_register_payload["output_modality"] + documentation_url = mock_register_payload["documentation_url"] + response = ModelFactory.create_asset_repo(name, description, function, source_language, input_modality, output_modality, documentation_url, config.TEAM_API_KEY) response_dict = dict(response) assert "id" in response_dict.keys() assert "repositoryName" in response_dict.keys() @@ -38,7 +36,6 @@ def test_create_asset_repo(): delete_asset(response["id"], config.TEAM_API_KEY) -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_list_host_machines(): response = ModelFactory.list_host_machines() for hosting_machine_dict in response: @@ -49,7 +46,6 @@ def test_list_host_machines(): assert "hourlyCost" in hosting_machine_dict.keys() -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_get_functions(): # Verbose response = ModelFactory.list_functions(True) diff --git a/tests/test_requests/create_asset_request.json b/tests/test_requests/create_asset_request.json index 4683e526..688dd33a 100644 --- a/tests/test_requests/create_asset_request.json +++ b/tests/test_requests/create_asset_request.json @@ -1,8 +1,9 @@ { "name": "mock_name", - "hostingMachine": "aix-2c-8g-od", - "version": "mock_version", "description": "mock_description", - "function": "Speech Recognition", - "sourceLanguage": "en" + "function": "Text Generation", + "sourceLanguage": "en", + "input_modality": "text", + "output_modality": "text", + "documentation_url": "" } \ No newline at end of file diff --git a/tests/image_upload_test.py b/tests/unit/image_upload_test.py similarity index 85% rename from tests/image_upload_test.py rename to tests/unit/image_upload_test.py index fb919171..4b192292 100644 --- a/tests/image_upload_test.py +++ b/tests/unit/image_upload_test.py @@ -13,7 +13,6 @@ API_FIXED_HEADER = {"x-api-key": f"{config.TEAM_API_KEY}", "Content-Type": "application/json"} -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_login(): url = urljoin(config.BACKEND_URL, f"sdk/ecr/login") with requests_mock.Mocker() as mock: @@ -24,24 +23,26 @@ def test_login(): assert creds == mock_json -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_create_asset_repo(): - url_register = urljoin(config.BACKEND_URL, f"sdk/models/register") + url_register = urljoin(config.BACKEND_URL, f"sdk/models/onboard") url_function = urljoin(config.BACKEND_URL, f"sdk/functions") + print(f"URL_Register {url_register}") with requests_mock.Mocker() as mock: with open(Path("tests/mock_responses/create_asset_repo_response.json")) as f: mock_json_register = json.load(f) - mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json_register) + mock.post(url_register, headers=API_FIXED_HEADER, json=mock_json_register, status_code=201) + with open(Path("tests/mock_responses/list_functions_response.json")) as f: mock_json_functions = json.load(f) mock.get(url_function, headers=AUTH_FIXED_HEADER, json=mock_json_functions) + model_id = ModelFactory.create_asset_repo( - "mock_name", "mock_machines", "mock_version", "mock_description", "Speech Recognition", "en", config.TEAM_API_KEY + "mock_name", "mock_description", "Text Generation", "en", "text", "text", api_key=config.TEAM_API_KEY ) + # print(f"Model ID {model_id}") assert model_id == mock_json_register -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_list_host_machines(): url = urljoin(config.BACKEND_URL, f"sdk/hosting-machines") with requests_mock.Mocker() as mock: @@ -55,8 +56,6 @@ def test_list_host_machines(): for key in machine_dict.keys(): assert machine_dict[key] == mock_json_dict[key] - -@pytest.mark.skip(reason="Model Upload is deactivated for improvements.") def test_get_functions(): url = urljoin(config.BACKEND_URL, f"sdk/functions") with requests_mock.Mocker() as mock: @@ -66,7 +65,6 @@ def test_get_functions(): functions = ModelFactory.list_functions(config.TEAM_API_KEY) assert functions == mock_json - @pytest.mark.skip(reason="Not currently supported.") def test_list_image_repo_tags(): model_id = "mock_id" From d0ad51d2990d291f80cb088cd69ed1d6412d39d3 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 13 Aug 2024 15:03:24 -0300 Subject: [PATCH 20/25] Eng 217 local path (#220) * Content inputs to be processed according to the query. * Add data and query parameters on running agent * Enable processing keys/values in content as well * Agent units tests and tags simolar Jinja2 --- aixplain/modules/agent/__init__.py | 30 ++++++++------ tests/unit/agent_test.py | 63 ++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 tests/unit/agent_test.py diff --git a/aixplain/modules/agent/__init__.py b/aixplain/modules/agent/__init__.py index 8a5cd120..c0604f6a 100644 --- a/aixplain/modules/agent/__init__.py +++ b/aixplain/modules/agent/__init__.py @@ -105,7 +105,7 @@ def run( timeout: float = 300, parameters: Dict = {}, wait_time: float = 0.5, - content: List[Text] = [], + content: Optional[Union[Dict[Text, Text], List[Text]]] = None, ) -> Dict: """Runs an agent call. @@ -118,7 +118,7 @@ def run( timeout (float, optional): total polling time. Defaults to 300. parameters (Dict, optional): optional parameters to the model. Defaults to "{}". wait_time (float, optional): wait time in seconds between polling calls. Defaults to 0.5. - content (List[Text], optional): Content inputs to be processed according to the query. Defaults to []. + content (Union[Dict[Text, Text], List[Text]], optional): Content inputs to be processed according to the query. Defaults to None. Returns: Dict: parsed output from model @@ -156,7 +156,7 @@ def run_async( history: Optional[List[Dict]] = None, name: Text = "model_process", parameters: Dict = {}, - content: List[Text] = [], + content: Optional[Union[Dict[Text, Text], List[Text]]] = None, ) -> Dict: """Runs asynchronously an agent call. @@ -167,7 +167,7 @@ def run_async( history (Optional[List[Dict]], optional): chat history (in case session ID is None). Defaults to None. name (Text, optional): ID given to a call. Defaults to "model_process". parameters (Dict, optional): optional parameters to the model. Defaults to "{}". - content (List[Text], optional): Content inputs to be processed according to the query. Defaults to []. + content (Union[Dict[Text, Text], List[Text]], optional): Content inputs to be processed according to the query. Defaults to None. Returns: dict: polling URL in response @@ -183,19 +183,25 @@ def run_async( session_id = data.get("session_id") if history is None: history = data.get("history") - if len(content) == 0: - content = data.get("content", []) + if content is None: + content = data.get("content") else: query = data # process content inputs - content = list(set(content)) - if len(content) > 0: + if content is not None: assert FileFactory.check_storage_type(query) == StorageType.TEXT, "When providing 'content', query must be text." - assert len(content) <= 3, "The maximum number of content inputs is 3." - for input_link in content: - input_link = FileFactory.to_link(input_link) - query += f"\n{input_link}" + + if isinstance(content, list): + assert len(content) <= 3, "The maximum number of content inputs is 3." + for input_link in content: + input_link = FileFactory.to_link(input_link) + query += f"\n{input_link}" + elif isinstance(content, dict): + for key, value in content.items(): + assert "{{" + key + "}}" in query, f"Key '{key}' not found in query." + value = FileFactory.to_link(value) + query = query.replace("{{" + key + "}}", f"'{value}'") headers = {"x-api-key": self.api_key, "Content-Type": "application/json"} diff --git a/tests/unit/agent_test.py b/tests/unit/agent_test.py new file mode 100644 index 00000000..680fc21a --- /dev/null +++ b/tests/unit/agent_test.py @@ -0,0 +1,63 @@ +import pytest +import requests_mock +from aixplain.modules import Agent +from aixplain.utils import config + + +def test_fail_no_data_query(): + agent = Agent("123", "Test Agent") + with pytest.raises(Exception) as exc_info: + agent.run_async() + assert str(exc_info.value) == "Either 'data' or 'query' must be provided." + + +def test_fail_query_must_be_provided(): + agent = Agent("123", "Test Agent") + with pytest.raises(Exception) as exc_info: + agent.run_async(data={}) + assert str(exc_info.value) == "When providing a dictionary, 'query' must be provided." + + +def test_fail_query_as_text_when_content_not_empty(): + agent = Agent("123", "Test Agent") + with pytest.raises(Exception) as exc_info: + agent.run_async( + data={"query": "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav"}, + content=["https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav"], + ) + assert str(exc_info.value) == "When providing 'content', query must be text." + + +def test_fail_content_exceed_maximum(): + agent = Agent("123", "Test Agent") + with pytest.raises(Exception) as exc_info: + agent.run_async( + data={"query": "Transcribe the audios:"}, + content=[ + "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", + "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", + "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", + "https://aixplain-platform-assets.s3.amazonaws.com/samples/en/CPAC1x2.wav", + ], + ) + assert str(exc_info.value) == "The maximum number of content inputs is 3." + + +def test_fail_key_not_found(): + agent = Agent("123", "Test Agent") + with pytest.raises(Exception) as exc_info: + agent.run_async(data={"query": "Translate the text: {{input1}}"}, content={"input2": "Hello, how are you?"}) + assert str(exc_info.value) == "Key 'input2' not found in query." + + +def test_sucess_query_content(): + agent = Agent("123", "Test Agent") + with requests_mock.Mocker() as mock: + url = agent.url + headers = {"x-api-key": config.TEAM_API_KEY, "Content-Type": "application/json"} + ref_response = {"data": "Hello, how are you?", "status": "IN_PROGRESS"} + mock.post(url, headers=headers, json=ref_response) + + response = agent.run_async(data={"query": "Translate the text: {{input1}}"}, content={"input1": "Hello, how are you?"}) + assert response["status"] == ref_response["status"] + assert response["url"] == ref_response["data"] From dca1a372076ccfb2bbcc949419ab6e6c6b9e4266 Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 13 Aug 2024 17:44:23 -0300 Subject: [PATCH 21/25] Eng 389 fix tests (#222) * Fixing circular import and duplicated names * Fixing the setting of function in agents --- aixplain/factories/agent_factory/__init__.py | 2 +- aixplain/modules/agent/tool/model_tool.py | 6 +- .../functional/agent/agent_functional_test.py | 4 + ...designer_test.py => designer_unit_test.py} | 78 +++++-------------- 4 files changed, 28 insertions(+), 62 deletions(-) rename tests/unit/{designer_test.py => designer_unit_test.py} (89%) diff --git a/aixplain/factories/agent_factory/__init__.py b/aixplain/factories/agent_factory/__init__.py index 6076eef6..0c73637c 100644 --- a/aixplain/factories/agent_factory/__init__.py +++ b/aixplain/factories/agent_factory/__init__.py @@ -67,7 +67,7 @@ def create( if isinstance(tool, ModelTool): tool_payload.append( { - "function": tool.function.value if tool.function is not None else None, + "function": tool.function.value, "type": "model", "description": tool.description, "supplier": tool.supplier.value["code"] if tool.supplier else None, diff --git a/aixplain/modules/agent/tool/model_tool.py b/aixplain/modules/agent/tool/model_tool.py index a5acab30..e15a8bea 100644 --- a/aixplain/modules/agent/tool/model_tool.py +++ b/aixplain/modules/agent/tool/model_tool.py @@ -24,7 +24,6 @@ from aixplain.enums.function import Function from aixplain.enums.supplier import Supplier -from aixplain.factories.model_factory import ModelFactory from aixplain.modules.agent.tool import Tool from aixplain.modules.model import Model @@ -58,7 +57,6 @@ def __init__( if function is not None: if isinstance(function, str): function = Function(function) - self.function = function try: if isinstance(supplier, dict): @@ -68,9 +66,13 @@ def __init__( if model is not None: if isinstance(model, Text) is True: + from aixplain.factories.model_factory import ModelFactory + model = ModelFactory.get(model) + function = model.function if isinstance(model.supplier, Supplier): supplier = model.supplier model = model.id self.supplier = supplier self.model = model + self.function = function diff --git a/tests/functional/agent/agent_functional_test.py b/tests/functional/agent/agent_functional_test.py index cefd34c3..58d421c8 100644 --- a/tests/functional/agent/agent_functional_test.py +++ b/tests/functional/agent/agent_functional_test.py @@ -37,6 +37,9 @@ def run_input_map(request): def test_end2end(run_input_map): + for agent in AgentFactory.list()["results"]: + agent.delete() + tools = [] if "model_tools" in run_input_map: for tool in run_input_map["model_tools"]: @@ -47,6 +50,7 @@ def test_end2end(run_input_map): ]: tool["supplier"] = supplier break + print("TOOL: ", tool) tools.append(AgentFactory.create_model_tool(**tool)) if "pipeline_tools" in run_input_map: for tool in run_input_map["pipeline_tools"]: diff --git a/tests/unit/designer_test.py b/tests/unit/designer_unit_test.py similarity index 89% rename from tests/unit/designer_test.py rename to tests/unit/designer_unit_test.py index 766c7e54..824fd162 100644 --- a/tests/unit/designer_test.py +++ b/tests/unit/designer_unit_test.py @@ -30,9 +30,7 @@ def test_create_node(): class BareNode(Node): pass - with mock.patch( - "aixplain.modules.pipeline.designer.Node.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Node.attach_to") as mock_attach_to: node = BareNode(number=3, label="FOO") mock_attach_to.assert_not_called() assert isinstance(node.inputs, Inputs) @@ -50,9 +48,7 @@ class FooNode(Node[FooNodeInputs, FooNodeOutputs]): inputs_class = FooNodeInputs outputs_class = FooNodeOutputs - with mock.patch( - "aixplain.modules.pipeline.designer.Node.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Node.attach_to") as mock_attach_to: node = FooNode(pipeline=pipeline, number=3, label="FOO") mock_attach_to.assert_called_once_with(pipeline) assert isinstance(node.inputs, FooNodeInputs) @@ -120,9 +116,7 @@ class AssetNode(Node): node = AssetNode() with mock.patch.object(node.inputs, "serialize") as mock_inputs_serialize: - with mock.patch.object( - node.outputs, "serialize" - ) as mock_outputs_serialize: + with mock.patch.object(node.outputs, "serialize") as mock_outputs_serialize: assert node.serialize() == { "number": node.number, "type": NodeType.ASSET, @@ -148,13 +142,10 @@ class AssetNode(Node): def test_create_param(): - class TypedParam(Param): param_type = ParamType.INPUT - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: param = TypedParam( code="param", data_type=DataType.TEXT, @@ -167,9 +158,7 @@ class TypedParam(Param): assert param.value == "foo" assert param.param_type == ParamType.INPUT - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: param = TypedParam( code="param", data_type=DataType.TEXT, @@ -186,9 +175,7 @@ class TypedParam(Param): class UnTypedParam(Param): pass - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: param = UnTypedParam( code="param", data_type=DataType.TEXT, @@ -199,9 +186,7 @@ class UnTypedParam(Param): assert param.param_type == ParamType.OUTPUT - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: param = UnTypedParam( code="param", data_type=DataType.TEXT, @@ -217,9 +202,7 @@ class AssetNode(Node): node = AssetNode() - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: param = UnTypedParam( code="param", data_type=DataType.TEXT, @@ -243,12 +226,8 @@ class AssetNode(Node): node = AssetNode() - with mock.patch( - "aixplain.modules.pipeline.designer.Param.attach_to" - ) as mock_attach_to: - param = param_cls( - code="param", data_type=DataType.TEXT, value="foo", node=node - ) + with mock.patch("aixplain.modules.pipeline.designer.Param.attach_to") as mock_attach_to: + param = param_cls(code="param", data_type=DataType.TEXT, value="foo", node=node) mock_attach_to.assert_called_once_with(node) assert param.code == "param" assert param.data_type == DataType.TEXT @@ -322,12 +301,8 @@ class AssetNode(Node, LinkableMixin): assert "Param not registered as output" in str(excinfo.value) - output = OutputParam( - code="output", data_type=DataType.TEXT, value="bar", node=a - ) - input = InputParam( - code="input", data_type=DataType.TEXT, value="foo", node=b - ) + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar", node=a) + input = InputParam(code="input", data_type=DataType.TEXT, value="foo", node=b) with mock.patch.object(input, "back_link") as mock_back_link: output.link(input) @@ -364,12 +339,8 @@ class AssetNode(Node, LinkableMixin): assert "Param not registered as input" in str(excinfo.value) - output = OutputParam( - code="output", data_type=DataType.TEXT, value="bar", node=a - ) - input = InputParam( - code="input", data_type=DataType.TEXT, value="foo", node=b - ) + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar", node=a) + input = InputParam(code="input", data_type=DataType.TEXT, value="foo", node=b) with mock.patch.object(a, "link") as mock_link: input.back_link(output) @@ -429,9 +400,7 @@ class AssetNode(Node, LinkableMixin): pipeline = DesignerPipeline() - with mock.patch( - "aixplain.modules.pipeline.designer.Link.attach_to" - ) as mock_attach_to: + with mock.patch("aixplain.modules.pipeline.designer.Link.attach_to") as mock_attach_to: link = Link( from_node=a, to_node=b, @@ -588,12 +557,8 @@ class AssetNode(Node): with mock.patch.object(param_proxy, "_create_param") as mock_create_param: with mock.patch.object(param_proxy, "add_param") as mock_add_param: - param = param_proxy.create_param( - "foo", DataType.TEXT, "bar", is_required=True - ) - mock_create_param.assert_called_once_with( - "foo", DataType.TEXT, "bar" - ) + param = param_proxy.create_param("foo", DataType.TEXT, "bar", is_required=True) + mock_create_param.assert_called_once_with("foo", DataType.TEXT, "bar") mock_add_param.assert_called_once_with(param) assert param.is_required is True @@ -624,19 +589,14 @@ class FooParam(Param): def test_node_link(): - class AssetNode(Node, LinkableMixin): type: NodeType = NodeType.ASSET a = AssetNode() b = AssetNode() - output = OutputParam( - code="output", data_type=DataType.TEXT, value="bar", node=a - ) - input = InputParam( - code="input", data_type=DataType.TEXT, value="foo", node=b - ) + output = OutputParam(code="output", data_type=DataType.TEXT, value="bar", node=a) + input = InputParam(code="input", data_type=DataType.TEXT, value="foo", node=b) # here too lazy to mock Link class properly # checking the output instance instead From b1133683768416e3d34d7ea6185a0d14d2519ce3 Mon Sep 17 00:00:00 2001 From: Zaina Abu Shaban Date: Tue, 20 Aug 2024 00:54:20 +0300 Subject: [PATCH 22/25] Tool Validation when creating agents (#226) * Implemented changes * Added test and fixed some issues, pytest-check fails regardless if I include or disclue new tests. * Initial changes, without test * Made changes, pytest-check fails * Fixing mixup * Fixing mixup * Fixing mixup * Handled Exception properly, removed extra space * Removed .DS_Store * Only validate if model is provided by user, not function * Validating Model on ModelTool.__init__ --------- Co-authored-by: xainaz Co-authored-by: Thiago Castro Ferreira --- aixplain/factories/agent_factory/__init__.py | 2 + aixplain/modules/agent/tool/model_tool.py | 19 +++- aixplain/modules/agent/tool/pipeline_tool.py | 8 ++ aixplain/modules/pipeline/asset.py | 96 ++++---------------- tests/unit/agent_test.py | 16 ++++ 5 files changed, 60 insertions(+), 81 deletions(-) diff --git a/aixplain/factories/agent_factory/__init__.py b/aixplain/factories/agent_factory/__init__.py index 6076eef6..4de4e582 100644 --- a/aixplain/factories/agent_factory/__init__.py +++ b/aixplain/factories/agent_factory/__init__.py @@ -65,6 +65,7 @@ def create( tool_payload = [] for tool in tools: if isinstance(tool, ModelTool): + tool.validate() tool_payload.append( { "function": tool.function.value if tool.function is not None else None, @@ -76,6 +77,7 @@ def create( } ) elif isinstance(tool, PipelineTool): + tool.validate() tool_payload.append( { "assetId": tool.pipeline, diff --git a/aixplain/modules/agent/tool/model_tool.py b/aixplain/modules/agent/tool/model_tool.py index 79e4601d..c88f1ee0 100644 --- a/aixplain/modules/agent/tool/model_tool.py +++ b/aixplain/modules/agent/tool/model_tool.py @@ -24,7 +24,6 @@ from aixplain.enums.function import Function from aixplain.enums.supplier import Supplier -from aixplain.factories.model_factory import ModelFactory from aixplain.modules.agent.tool import Tool from aixplain.modules.model import Model @@ -67,13 +66,23 @@ def __init__( if model is not None: if isinstance(model, Text) is True: - from aixplain.factories.model_factory import ModelFactory - - model = ModelFactory.get(model) + self.model = model + model = self.validate() function = model.function if isinstance(model.supplier, Supplier): supplier = model.supplier model = model.id self.supplier = supplier self.model = model - self.function = function \ No newline at end of file + self.function = function + + def validate(self) -> Model: + from aixplain.factories.model_factory import ModelFactory + + try: + model = None + if self.model is not None: + model = ModelFactory.get(self.model) + return model + except Exception: + raise Exception(f"Model Tool Unavailable. Make sure Model '{self.model}' exists or you have access to it.") diff --git a/aixplain/modules/agent/tool/pipeline_tool.py b/aixplain/modules/agent/tool/pipeline_tool.py index a517b198..5ad2915a 100644 --- a/aixplain/modules/agent/tool/pipeline_tool.py +++ b/aixplain/modules/agent/tool/pipeline_tool.py @@ -50,3 +50,11 @@ def __init__( if isinstance(pipeline, Pipeline): pipeline = pipeline.id self.pipeline = pipeline + + def validate(self): + from aixplain.factories.pipeline_factory import PipelineFactory + + try: + PipelineFactory.get(self.pipeline) + except Exception: + raise Exception(f"Pipeline Tool Unavailable. Make sure Pipeline '{self.pipeline}' exists or you have access to it.") diff --git a/aixplain/modules/pipeline/asset.py b/aixplain/modules/pipeline/asset.py index ce168036..ad7cfa1b 100644 --- a/aixplain/modules/pipeline/asset.py +++ b/aixplain/modules/pipeline/asset.py @@ -24,7 +24,6 @@ import time import json import os -import uuid import logging from aixplain.modules.asset import Asset from aixplain.utils import config @@ -101,9 +100,7 @@ def __polling( while not completed and (end - start) < timeout: try: response_body = self.poll(poll_url, name=name) - logging.debug( - f"Polling for Pipeline: Status of polling for {name} : {response_body}" - ) + logging.debug(f"Polling for Pipeline: Status of polling for {name} : {response_body}") completed = response_body["completed"] end = time.time() @@ -112,18 +109,12 @@ def __polling( if wait_time < 60: wait_time *= 1.1 except Exception: - logging.error( - f"Polling for Pipeline: polling for {name} : Continue" - ) + logging.error(f"Polling for Pipeline: polling for {name} : Continue") if response_body and response_body["status"] == "SUCCESS": try: - logging.debug( - f"Polling for Pipeline: Final status of polling for {name} : SUCCESS - {response_body}" - ) + logging.debug(f"Polling for Pipeline: Final status of polling for {name} : SUCCESS - {response_body}") except Exception: - logging.error( - f"Polling for Pipeline: Final status of polling for {name} : ERROR - {response_body}" - ) + logging.error(f"Polling for Pipeline: Final status of polling for {name} : ERROR - {response_body}") else: logging.error( f"Polling for Pipeline: Final status of polling for {name} : No response in {timeout} seconds - {response_body}" @@ -148,9 +139,7 @@ def poll(self, poll_url: Text, name: Text = "pipeline_process") -> Dict: r = _request_with_retry("get", poll_url, headers=headers) try: resp = r.json() - logging.info( - f"Single Poll for Pipeline: Status of polling for {name} : {resp}" - ) + logging.info(f"Single Poll for Pipeline: Status of polling for {name} : {resp}") except Exception: resp = {"status": "FAILED"} return resp @@ -179,18 +168,14 @@ def run( """ start = time.time() try: - response = self.run_async( - data, data_asset=data_asset, name=name, **kwargs - ) + response = self.run_async(data, data_asset=data_asset, name=name, **kwargs) if response["status"] == "FAILED": end = time.time() response["elapsed_time"] = end - start return response poll_url = response["url"] end = time.time() - response = self.__polling( - poll_url, name=name, timeout=timeout, wait_time=wait_time - ) + response = self.__polling(poll_url, name=name, timeout=timeout, wait_time=wait_time) return response except Exception as e: error_message = f"Error in request for {name}: {str(e)}" @@ -240,10 +225,7 @@ def __prepare_payload( try: payload = json.loads(data) if isinstance(payload, dict) is False: - if ( - isinstance(payload, int) is True - or isinstance(payload, float) is True - ): + if isinstance(payload, int) is True or isinstance(payload, float) is True: payload = str(payload) payload = {"data": payload} except Exception: @@ -273,33 +255,15 @@ def __prepare_payload( try: dasset = CorpusFactory.get(str(data_asset[node_label])) asset_payload["dataAsset"]["corpus_id"] = dasset.id - if ( - len( - [ - d - for d in dasset.data - if d.id == data[node_label] - ] - ) - > 0 - ): + if len([d for d in dasset.data if d.id == data[node_label]]) > 0: data_found = True except Exception: try: - dasset = DatasetFactory.get( - str(data_asset[node_label]) - ) + dasset = DatasetFactory.get(str(data_asset[node_label])) asset_payload["dataAsset"]["dataset_id"] = dasset.id if ( - len( - [ - dfield - for dfield in dasset.source_data - if dasset.source_data[dfield].id - == data[node_label] - ] - ) + len([dfield for dfield in dasset.source_data if dasset.source_data[dfield].id == data[node_label]]) > 0 ): data_found = True @@ -332,11 +296,7 @@ def __prepare_payload( return payload def run_async( - self, - data: Union[Text, Dict], - data_asset: Optional[Union[Text, Dict]] = None, - name: Text = "pipeline_process", - **kwargs + self, data: Union[Text, Dict], data_asset: Optional[Union[Text, Dict]] = None, name: Text = "pipeline_process", **kwargs ) -> Dict: """Runs asynchronously a pipeline call. @@ -359,16 +319,12 @@ def run_async( payload = json.dumps(payload) call_url = f"{self.url}/{self.id}" logging.info(f"Start service for {name} - {call_url} - {payload}") - r = _request_with_retry( - "post", call_url, headers=headers, data=payload - ) + r = _request_with_retry("post", call_url, headers=headers, data=payload) resp = None try: resp = r.json() - logging.info( - f"Result of request for {name} - {r.status_code} - {resp}" - ) + logging.info(f"Result of request for {name} - {r.status_code} - {resp}") poll_url = resp["url"] response = {"status": "IN_PROGRESS", "url": poll_url} @@ -405,9 +361,7 @@ def update( for i, node in enumerate(pipeline["nodes"]): if "functionType" in node and node["functionType"] == "AI": - pipeline["nodes"][i]["functionType"] = pipeline["nodes"][ - i - ]["functionType"].lower() + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() # prepare payload status = "draft" if save_as_asset is True: @@ -423,9 +377,7 @@ def update( "Authorization": f"Token {api_key}", "Content-Type": "application/json", } - logging.info( - f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}" - ) + logging.info(f"Start service for PUT Update Pipeline - {url} - {headers} - {json.dumps(payload)}") r = _request_with_retry("put", url, headers=headers, json=payload) response = r.json() logging.info(f"Pipeline {response['id']} Updated.") @@ -440,9 +392,7 @@ def delete(self) -> None: "Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json", } - logging.info( - f"Start service for DELETE Pipeline - {url} - {headers}" - ) + logging.info(f"Start service for DELETE Pipeline - {url} - {headers}") r = _request_with_retry("delete", url, headers=headers) if r.status_code != 200: raise Exception() @@ -451,9 +401,7 @@ def delete(self) -> None: logging.error(message) raise Exception(f"{message}") - def save( - self, save_as_asset: bool = False, api_key: Optional[Text] = None - ): + def save(self, save_as_asset: bool = False, api_key: Optional[Text] = None): """Save Pipeline Args: @@ -468,9 +416,7 @@ def save( for i, node in enumerate(pipeline["nodes"]): if "functionType" in node and node["functionType"] == "AI": - pipeline["nodes"][i]["functionType"] = pipeline["nodes"][ - i - ]["functionType"].lower() + pipeline["nodes"][i]["functionType"] = pipeline["nodes"][i]["functionType"].lower() # prepare payload status = "draft" if save_as_asset is True: @@ -492,9 +438,7 @@ def save( "Authorization": f"Token {api_key}", "Content-Type": "application/json", } - logging.info( - f"Start service for Save Pipeline - {url} - {headers} - {json.dumps(payload)}" - ) + logging.info(f"Start service for Save Pipeline - {url} - {headers} - {json.dumps(payload)}") r = _request_with_retry(method, url, headers=headers, json=payload) response = r.json() self.id = response["id"] diff --git a/tests/unit/agent_test.py b/tests/unit/agent_test.py index 680fc21a..18c92fa3 100644 --- a/tests/unit/agent_test.py +++ b/tests/unit/agent_test.py @@ -2,6 +2,8 @@ import requests_mock from aixplain.modules import Agent from aixplain.utils import config +from aixplain.factories import AgentFactory +from aixplain.modules.agent import PipelineTool, ModelTool def test_fail_no_data_query(): @@ -61,3 +63,17 @@ def test_sucess_query_content(): response = agent.run_async(data={"query": "Translate the text: {{input1}}"}, content={"input1": "Hello, how are you?"}) assert response["status"] == ref_response["status"] assert response["url"] == ref_response["data"] + + +def test_invalid_pipelinetool(): + with pytest.raises(Exception) as exc_info: + AgentFactory.create( + name="Test", tools=[PipelineTool(pipeline="309851793", description="Test")], llm_id="6646261c6eb563165658bbb1" + ) + assert str(exc_info.value) == "Pipeline Tool Unavailable. Make sure Pipeline '309851793' exists or you have access to it." + + +def test_invalid_modeltool(): + with pytest.raises(Exception) as exc_info: + AgentFactory.create(name="Test", tools=[ModelTool(model="309851793")], llm_id="6646261c6eb563165658bbb1") + assert str(exc_info.value) == "Model Tool Unavailable. Make sure Model '309851793' exists or you have access to it." From 0032947f305cb51a1baf7609c6ab0479578ceb24 Mon Sep 17 00:00:00 2001 From: Zaina Abu Shaban Date: Tue, 20 Aug 2024 16:47:48 +0300 Subject: [PATCH 23/25] Eng 398 sdk get users credits - Initial (#232) * Add .DS_Store to .gitignore * Initial commit, added wallet factory and wallet module. No test is added yet * Added test * Formatting Wallet tests --------- Co-authored-by: Thiago Castro Ferreira Co-authored-by: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> --- .gitignore | 2 ++ aixplain/factories/__init__.py | 1 + aixplain/factories/wallet_factoy.py | 26 ++++++++++++++++++++++ aixplain/modules/wallet.py | 34 +++++++++++++++++++++++++++++ tests/unit/wallet_test.py | 16 ++++++++++++++ 5 files changed, 79 insertions(+) create mode 100644 aixplain/factories/wallet_factoy.py create mode 100644 aixplain/modules/wallet.py create mode 100644 tests/unit/wallet_test.py diff --git a/.gitignore b/.gitignore index 843c6556..ad7c16c8 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,5 @@ dmypy.json # Vscode .vscode +.DS_Store + diff --git a/aixplain/factories/__init__.py b/aixplain/factories/__init__.py index 7b876899..08cb8d4a 100644 --- a/aixplain/factories/__init__.py +++ b/aixplain/factories/__init__.py @@ -30,3 +30,4 @@ from .model_factory import ModelFactory from .pipeline_factory import PipelineFactory from .finetune_factory import FinetuneFactory +from .wallet_factoy import WalletFactory diff --git a/aixplain/factories/wallet_factoy.py b/aixplain/factories/wallet_factoy.py new file mode 100644 index 00000000..59ec7c14 --- /dev/null +++ b/aixplain/factories/wallet_factoy.py @@ -0,0 +1,26 @@ +import aixplain.utils.config as config +from aixplain.modules.wallet import Wallet +from aixplain.utils.file_utils import _request_with_retry +import logging + + +class WalletFactory: + aixplain_key = config.AIXPLAIN_API_KEY + backend_url = config.BACKEND_URL + + @classmethod + def get(cls) -> Wallet: + """Get wallet information""" + try: + resp = None + # Check for code 200, other code will be caught when trying to return a Wallet object + url = f"{cls.backend_url}/sdk/billing/wallet" + + headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + logging.info(f"Start fetching billing information from - {url} - {headers}") + headers = {"Content-Type": "application/json", "x-api-key": config.TEAM_API_KEY} + r = _request_with_retry("get", url, headers=headers) + resp = r.json() + return Wallet(total_balance=resp["totalBalance"], reserved_balance=resp["reservedBalance"]) + except Exception as e: + raise Exception(f"Failed to get the wallet credit information. Error: {str(e)}") diff --git a/aixplain/modules/wallet.py b/aixplain/modules/wallet.py new file mode 100644 index 00000000..d7c63524 --- /dev/null +++ b/aixplain/modules/wallet.py @@ -0,0 +1,34 @@ +__author__ = "aixplain" + +""" +Copyright 2024 The aiXplain SDK authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: aiXplain Team +Date: August 20th 2024 +Description: + Wallet Class +""" + + +class Wallet: + def __init__(self, total_balance: float, reserved_balance: float): + """Create a Wallet with the necessary information + + Args: + total_balance (float): total credit balance + reserved_balance (float): reserved credit balance + """ + self.total_balance = total_balance + self.reserved_balance = reserved_balance diff --git a/tests/unit/wallet_test.py b/tests/unit/wallet_test.py new file mode 100644 index 00000000..48ee19ab --- /dev/null +++ b/tests/unit/wallet_test.py @@ -0,0 +1,16 @@ +__author__ = "aixplain" + +from aixplain.factories import WalletFactory +import aixplain.utils.config as config +import requests_mock + + +def test_wallet_service(): + with requests_mock.Mocker() as mock: + url = f"{config.BACKEND_URL}/sdk/billing/wallet" + headers = {"x-api-key": config.TEAM_API_KEY, "Content-Type": "application/json"} + ref_response = {"totalBalance": 5, "reservedBalance": "0"} + mock.get(url, headers=headers, json=ref_response) + wallet = WalletFactory.get() + assert wallet.total_balance == ref_response["totalBalance"] + assert wallet.reserved_balance == ref_response["reservedBalance"] From a5675354a8cda213cd9b52508d921cca980c587f Mon Sep 17 00:00:00 2001 From: Thiago Castro Ferreira <85182544+thiago-aixplain@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:12:22 -0300 Subject: [PATCH 24/25] Eng 398 sdk get users credits (#234) * Add .DS_Store to .gitignore * Initial commit, added wallet factory and wallet module. No test is added yet * Added test * Formatting Wallet tests * wallet_factoy -> wallet_factory --------- Co-authored-by: xainaz --- aixplain/factories/__init__.py | 2 +- aixplain/factories/wallet_factory.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 aixplain/factories/wallet_factory.py diff --git a/aixplain/factories/__init__.py b/aixplain/factories/__init__.py index 08cb8d4a..70361e77 100644 --- a/aixplain/factories/__init__.py +++ b/aixplain/factories/__init__.py @@ -30,4 +30,4 @@ from .model_factory import ModelFactory from .pipeline_factory import PipelineFactory from .finetune_factory import FinetuneFactory -from .wallet_factoy import WalletFactory +from .wallet_factory import WalletFactory diff --git a/aixplain/factories/wallet_factory.py b/aixplain/factories/wallet_factory.py new file mode 100644 index 00000000..59ec7c14 --- /dev/null +++ b/aixplain/factories/wallet_factory.py @@ -0,0 +1,26 @@ +import aixplain.utils.config as config +from aixplain.modules.wallet import Wallet +from aixplain.utils.file_utils import _request_with_retry +import logging + + +class WalletFactory: + aixplain_key = config.AIXPLAIN_API_KEY + backend_url = config.BACKEND_URL + + @classmethod + def get(cls) -> Wallet: + """Get wallet information""" + try: + resp = None + # Check for code 200, other code will be caught when trying to return a Wallet object + url = f"{cls.backend_url}/sdk/billing/wallet" + + headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} + logging.info(f"Start fetching billing information from - {url} - {headers}") + headers = {"Content-Type": "application/json", "x-api-key": config.TEAM_API_KEY} + r = _request_with_retry("get", url, headers=headers) + resp = r.json() + return Wallet(total_balance=resp["totalBalance"], reserved_balance=resp["reservedBalance"]) + except Exception as e: + raise Exception(f"Failed to get the wallet credit information. Error: {str(e)}") From e919fab4f6c3f8f829382e5057541aaa6436e971 Mon Sep 17 00:00:00 2001 From: Zaina Abu Shaban Date: Thu, 22 Aug 2024 00:11:08 +0300 Subject: [PATCH 25/25] Removed wallet_factoy.py (#235) Co-authored-by: xainaz --- aixplain/factories/wallet_factoy.py | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 aixplain/factories/wallet_factoy.py diff --git a/aixplain/factories/wallet_factoy.py b/aixplain/factories/wallet_factoy.py deleted file mode 100644 index 59ec7c14..00000000 --- a/aixplain/factories/wallet_factoy.py +++ /dev/null @@ -1,26 +0,0 @@ -import aixplain.utils.config as config -from aixplain.modules.wallet import Wallet -from aixplain.utils.file_utils import _request_with_retry -import logging - - -class WalletFactory: - aixplain_key = config.AIXPLAIN_API_KEY - backend_url = config.BACKEND_URL - - @classmethod - def get(cls) -> Wallet: - """Get wallet information""" - try: - resp = None - # Check for code 200, other code will be caught when trying to return a Wallet object - url = f"{cls.backend_url}/sdk/billing/wallet" - - headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} - logging.info(f"Start fetching billing information from - {url} - {headers}") - headers = {"Content-Type": "application/json", "x-api-key": config.TEAM_API_KEY} - r = _request_with_retry("get", url, headers=headers) - resp = r.json() - return Wallet(total_balance=resp["totalBalance"], reserved_balance=resp["reservedBalance"]) - except Exception as e: - raise Exception(f"Failed to get the wallet credit information. Error: {str(e)}")