From 519cdce2c22a77f7da3fcddccebd83c71aaa38dc Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 27 Aug 2024 02:36:03 +0300 Subject: [PATCH 1/8] Initial commit for finetune test --- .../finetune/finetune_functional_test.py | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index 7b45613c..dbae2e1b 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -61,39 +61,49 @@ def validate_prompt_input_map(request): def test_end2end(run_input_map): - model = ModelFactory.get(run_input_map["model_id"]) - dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] - train_percentage, dev_percentage = 100, 0 - if run_input_map["required_dev"]: - train_percentage, dev_percentage = 80, 20 - finetune = FinetuneFactory.create( - str(uuid.uuid4()), dataset_list, model, train_percentage=train_percentage, dev_percentage=dev_percentage + models = ModelFactory.list( + function=Function.TEXT_GENERATION, + is_finetunable=True, + #Add date condition here ) - assert type(finetune.cost) is FinetuneCost - cost_map = finetune.cost.to_dict() - assert "trainingCost" in cost_map - assert "hostingCost" in cost_map - assert "inferenceCost" in cost_map - finetune_model = finetune.start() - start, end = time.time(), time.time() - status = finetune_model.check_finetune_status().model_status.value - while status != "onboarded" and (end - start) < TIMEOUT: + for model_id in models: + model = ModelFactory.get(model_id) + dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] + train_percentage, dev_percentage = 100, 0 + if run_input_map["required_dev"]: + train_percentage, dev_percentage = 80, 20 + + finetune = FinetuneFactory.create( + str(uuid.uuid4()), dataset_list, model, train_percentage=train_percentage, dev_percentage=dev_percentage + ) + + assert type(finetune.cost) is FinetuneCost + cost_map = finetune.cost.to_dict() + assert "trainingCost" in cost_map + assert "hostingCost" in cost_map + assert "inferenceCost" in cost_map + + finetune_model = finetune.start() + start, end = time.time(), time.time() status = finetune_model.check_finetune_status().model_status.value - assert status != "failed" - time.sleep(5) - end = time.time() - assert finetune_model.check_finetune_status().model_status.value == "onboarded" - time.sleep(30) - print(f"Model dict: {finetune_model.__dict__}") - result = finetune_model.run(run_input_map["inference_data"]) - print(f"Result: {result}") - assert result is not None - if run_input_map["search_metadata"]: - assert "details" in result - assert len(result["details"]) > 0 - assert "metadata" in result["details"][0] - assert len(result["details"][0]["metadata"]) > 0 - finetune_model.delete() + while status != "onboarded" and (end - start) < TIMEOUT: + status = finetune_model.check_finetune_status().model_status.value + assert status != "failed" + time.sleep(5) + end = time.time() + assert finetune_model.check_finetune_status().model_status.value == "onboarded" + time.sleep(30) + print(f"Model dict: {finetune_model.__dict__}") + result = finetune_model.run(run_input_map["inference_data"]) + print(f"Result: {result}") + assert result is not None + if run_input_map["search_metadata"]: + assert "details" in result + assert len(result["details"]) > 0 + assert "metadata" in result["details"][0] + assert len(result["details"][0]["metadata"]) > 0 + finetune_model.delete() + def test_cost_estimation_text_generation(estimate_cost_input_map): From 7a30afbecd22e13400b57639f72214bb66926e2f Mon Sep 17 00:00:00 2001 From: xainaz Date: Fri, 30 Aug 2024 15:13:12 +0300 Subject: [PATCH 2/8] Added createdAt --- aixplain/factories/model_factory.py | 8 +- aixplain/modules/model/__init__.py | 4 +- .../finetune/finetune_functional_test.py | 73 ++++++++----------- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index c11d837a..7b928329 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -30,7 +30,7 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn - +from datetime import datetime class ModelFactory: """A static class for creating and exploring Model Objects. @@ -66,6 +66,10 @@ def _create_model_from_response(cls, response: Dict) -> Model: if function == Function.TEXT_GENERATION: ModelClass = LLM + created_at = None + if "createdAt" in response and response["createdAt"]: + created_at = datetime.fromisoformat(response["createdAt"].replace("Z", "+00:00")) + return ModelClass( response["id"], response["name"], @@ -73,6 +77,7 @@ def _create_model_from_response(cls, response: Dict) -> Model: api_key=response["api_key"], cost=response["pricing"], function=function, + createdAt=created_at, parameters=parameters, is_subscribed=True if "subscription" in response else False, version=response["version"]["id"], @@ -412,7 +417,6 @@ def asset_repo_login(cls, api_key: Optional[Text] = None) -> Dict: else: headers = {"Authorization": f"Token {config.TEAM_API_KEY}", "Content-Type": "application/json"} response = _request_with_retry("post", login_url, headers=headers) - print(f"Response: {response}") response_dict = json.loads(response.text) return response_dict diff --git a/aixplain/modules/model/__init__.py b/aixplain/modules/model/__init__.py index 8fcd80d2..24027ba0 100644 --- a/aixplain/modules/model/__init__.py +++ b/aixplain/modules/model/__init__.py @@ -30,7 +30,7 @@ from urllib.parse import urljoin from aixplain.utils.file_utils import _request_with_retry from typing import Union, Optional, Text, Dict - +from datetime import datetime class Model(Asset): """This is ready-to-use AI model. This model can be run in both synchronous and asynchronous manner. @@ -61,6 +61,7 @@ def __init__( function: Optional[Function] = None, is_subscribed: bool = False, cost: Optional[Dict] = None, + createdAt: Optional[datetime] = None, # Add createdAt here **additional_info, ) -> None: """Model Init @@ -84,6 +85,7 @@ def __init__( self.backend_url = config.BACKEND_URL self.function = function self.is_subscribed = is_subscribed + self.createdAt = createdAt def to_dict(self) -> Dict: """Get the model info as a Dictionary diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index dbae2e1b..fcede887 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -1,3 +1,4 @@ + __author__ = "lucaspavanelli" """ @@ -61,49 +62,39 @@ def validate_prompt_input_map(request): def test_end2end(run_input_map): - models = ModelFactory.list( - function=Function.TEXT_GENERATION, - is_finetunable=True, - #Add date condition here + model = ModelFactory.get(run_input_map["model_id"]) + dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] + train_percentage, dev_percentage = 100, 0 + if run_input_map["required_dev"]: + train_percentage, dev_percentage = 80, 20 + finetune = FinetuneFactory.create( + str(uuid.uuid4()), dataset_list, model, train_percentage=train_percentage, dev_percentage=dev_percentage ) - for model_id in models: - model = ModelFactory.get(model_id) - dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] - train_percentage, dev_percentage = 100, 0 - if run_input_map["required_dev"]: - train_percentage, dev_percentage = 80, 20 - - finetune = FinetuneFactory.create( - str(uuid.uuid4()), dataset_list, model, train_percentage=train_percentage, dev_percentage=dev_percentage - ) - - assert type(finetune.cost) is FinetuneCost - cost_map = finetune.cost.to_dict() - assert "trainingCost" in cost_map - assert "hostingCost" in cost_map - assert "inferenceCost" in cost_map - - finetune_model = finetune.start() - start, end = time.time(), time.time() + assert type(finetune.cost) is FinetuneCost + cost_map = finetune.cost.to_dict() + assert "trainingCost" in cost_map + assert "hostingCost" in cost_map + assert "inferenceCost" in cost_map + finetune_model = finetune.start() + start, end = time.time(), time.time() + status = finetune_model.check_finetune_status().model_status.value + while status != "onboarded" and (end - start) < TIMEOUT: status = finetune_model.check_finetune_status().model_status.value - while status != "onboarded" and (end - start) < TIMEOUT: - status = finetune_model.check_finetune_status().model_status.value - assert status != "failed" - time.sleep(5) - end = time.time() - assert finetune_model.check_finetune_status().model_status.value == "onboarded" - time.sleep(30) - print(f"Model dict: {finetune_model.__dict__}") - result = finetune_model.run(run_input_map["inference_data"]) - print(f"Result: {result}") - assert result is not None - if run_input_map["search_metadata"]: - assert "details" in result - assert len(result["details"]) > 0 - assert "metadata" in result["details"][0] - assert len(result["details"][0]["metadata"]) > 0 - finetune_model.delete() - + assert status != "failed" + time.sleep(5) + end = time.time() + assert finetune_model.check_finetune_status().model_status.value == "onboarded" + time.sleep(30) + print(f"Model dict: {finetune_model.__dict__}") + result = finetune_model.run(run_input_map["inference_data"]) + print(f"Result: {result}") + assert result is not None + if run_input_map["search_metadata"]: + assert "details" in result + assert len(result["details"]) > 0 + assert "metadata" in result["details"][0] + assert len(result["details"][0]["metadata"]) > 0 + finetune_model.delete() def test_cost_estimation_text_generation(estimate_cost_input_map): From 4172ac43a83b40ea279b759c7b030c02e0f7b2d1 Mon Sep 17 00:00:00 2001 From: xainaz Date: Sat, 31 Aug 2024 18:24:20 +0300 Subject: [PATCH 3/8] Added correct tests --- aixplain/modules/model/__init__.py | 2 +- .../finetune/finetune_functional_test.py | 26 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/aixplain/modules/model/__init__.py b/aixplain/modules/model/__init__.py index 24027ba0..28dd1cb3 100644 --- a/aixplain/modules/model/__init__.py +++ b/aixplain/modules/model/__init__.py @@ -61,7 +61,7 @@ def __init__( function: Optional[Function] = None, is_subscribed: bool = False, cost: Optional[Dict] = None, - createdAt: Optional[datetime] = None, # Add createdAt here + createdAt: Optional[datetime] = None, **additional_info, ) -> None: """Model Init diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index fcede887..0cd67df9 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -1,6 +1,5 @@ __author__ = "lucaspavanelli" - """ Copyright 2022 The aiXplain SDK authors @@ -27,6 +26,7 @@ from aixplain.factories import FinetuneFactory from aixplain.modules.finetune.cost import FinetuneCost from aixplain.enums import Function, Language +from datetime import datetime, timedelta, timezone import pytest @@ -61,8 +61,21 @@ def validate_prompt_input_map(request): return request.param -def test_end2end(run_input_map): - model = ModelFactory.get(run_input_map["model_id"]) +@pytest.fixture(scope="module") +def finetunable_llms_from_last_4_weeks(): + four_weeks_ago = datetime.now(timezone.utc) - timedelta(weeks=4) + models = ModelFactory.list( + function=Function.TEXT_GENERATION, + is_finetunable=True + )["results"] + for model in models: + print(f"Model ID: {model.id}, Created At: {model.createdAt}") + recent_models = [model for model in models if model.createdAt >= four_weeks_ago] + if not recent_models: + return None + return recent_models + +def run_finetune_test(model, run_input_map): dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] train_percentage, dev_percentage = 100, 0 if run_input_map["required_dev"]: @@ -96,6 +109,13 @@ def test_end2end(run_input_map): assert len(result["details"][0]["metadata"]) > 0 finetune_model.delete() +def test_end2end(run_input_map, finetunable_llms_from_last_4_weeks): + if finetunable_llms_from_last_4_weeks: + for model in finetunable_llms_from_last_4_weeks: + run_finetune_test(model, run_input_map) + else: + model = ModelFactory.get(run_input_map["model_id"]) + run_finetune_test(model, run_input_map) def test_cost_estimation_text_generation(estimate_cost_input_map): model = ModelFactory.get(estimate_cost_input_map["model_id"]) From f85ffaad9d28c7431e6ec6f224c200b0e15b41c0 Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 3 Sep 2024 16:26:50 +0300 Subject: [PATCH 4/8] Added test, issue with dev, not passing pytest --- .../finetune/finetune_functional_test.py | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index 0cd67df9..89ad6a78 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -1,4 +1,3 @@ - __author__ = "lucaspavanelli" """ Copyright 2022 The aiXplain SDK authors @@ -61,21 +60,22 @@ def validate_prompt_input_map(request): return request.param -@pytest.fixture(scope="module") -def finetunable_llms_from_last_4_weeks(): +@pytest.fixture(scope="module", params=read_data(RUN_FILE)) +def finetunable_llms_from_last_4_weeks(request): four_weeks_ago = datetime.now(timezone.utc) - timedelta(weeks=4) - models = ModelFactory.list( - function=Function.TEXT_GENERATION, - is_finetunable=True - )["results"] - for model in models: - print(f"Model ID: {model.id}, Created At: {model.createdAt}") + models = ModelFactory.list(function=Function.TEXT_GENERATION, is_finetunable=True)["results"] + recent_models = [model for model in models if model.createdAt >= four_weeks_ago] + if not recent_models: - return None - return recent_models + yield ModelFactory.get(request.param["model_id"]) + else: + for model in recent_models: + yield model + -def run_finetune_test(model, run_input_map): +def test_end2end(run_input_map, finetunable_llms_from_last_4_weeks): + model = finetunable_llms_from_last_4_weeks dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] train_percentage, dev_percentage = 100, 0 if run_input_map["required_dev"]: @@ -104,18 +104,11 @@ def run_finetune_test(model, run_input_map): assert result is not None if run_input_map["search_metadata"]: assert "details" in result - assert len(result["details"]) > 0 + assert len(result["details"]) > 0 assert "metadata" in result["details"][0] assert len(result["details"][0]["metadata"]) > 0 finetune_model.delete() -def test_end2end(run_input_map, finetunable_llms_from_last_4_weeks): - if finetunable_llms_from_last_4_weeks: - for model in finetunable_llms_from_last_4_weeks: - run_finetune_test(model, run_input_map) - else: - model = ModelFactory.get(run_input_map["model_id"]) - run_finetune_test(model, run_input_map) def test_cost_estimation_text_generation(estimate_cost_input_map): model = ModelFactory.get(estimate_cost_input_map["model_id"]) From e7bf397b703c2bba96cd7815299f8e5d08c1a70d Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 3 Sep 2024 17:03:03 +0300 Subject: [PATCH 5/8] Added test, issue with dev, not passing pytest --- aixplain/factories/model_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index ff6bd5c4..cadb5185 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -30,9 +30,9 @@ from aixplain.utils.file_utils import _request_with_retry from urllib.parse import urljoin from warnings import warn -ENG-467-ai-xplain-sdk-update-finetune-functional-tests-to-cover-all-new-finetunable-models from aixplain.enums.function import FunctionInputOutput + class ModelFactory: """A static class for creating and exploring Model Objects. From 07afe47a206f31d37256ecedb1ec7dd7c10d49c6 Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 3 Sep 2024 18:41:03 +0300 Subject: [PATCH 6/8] Added createdAt --- aixplain/modules/model/__init__.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/aixplain/modules/model/__init__.py b/aixplain/modules/model/__init__.py index e18f1896..250fafb2 100644 --- a/aixplain/modules/model/__init__.py +++ b/aixplain/modules/model/__init__.py @@ -30,6 +30,7 @@ from urllib.parse import urljoin from aixplain.utils.file_utils import _request_with_retry from typing import Union, Optional, Text, Dict +from datetime import datetime class Model(Asset): @@ -63,6 +64,7 @@ def __init__( function: Optional[Function] = None, is_subscribed: bool = False, cost: Optional[Dict] = None, + createdAt: Optional[datetime] = None, input_params: Optional[Dict] = None, output_params: Optional[Dict] = None, **additional_info, @@ -88,8 +90,9 @@ def __init__( self.backend_url = config.BACKEND_URL self.function = function self.is_subscribed = is_subscribed - self.input_params = input_params - self.output_params = output_params + self.createdAt = createdAt + self.input_params = input_params + self.output_params = output_params def to_dict(self) -> Dict: """Get the model info as a Dictionary @@ -98,7 +101,14 @@ def to_dict(self) -> Dict: Dict: Model Information """ clean_additional_info = {k: v for k, v in self.additional_info.items() if v is not None} - return {"id": self.id, "name": self.name, "supplier": self.supplier, "additional_info": clean_additional_info, "input_params": self.input_params,"output_params": self.output_params,} + return { + "id": self.id, + "name": self.name, + "supplier": self.supplier, + "additional_info": clean_additional_info, + "input_params": self.input_params, + "output_params": self.output_params, + } def __repr__(self): try: @@ -263,7 +273,9 @@ def run_async(self, data: Union[Text, Dict], name: Text = "model_process", param error = "Validation-related error: Please ensure all required fields are provided and correctly formatted." else: status_code = str(r.status_code) - error = f"Status {status_code}: Unspecified error: An unspecified error occurred while processing your request." + error = ( + f"Status {status_code}: Unspecified error: An unspecified error occurred while processing your request." + ) response = {"status": "FAILED", "error_message": error} logging.error(f"Error in request for {name} - {r.status_code}: {error}") except Exception: From 0363f170c91d2fe62a9c071df13cac78ca58759c Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 3 Sep 2024 18:42:58 +0300 Subject: [PATCH 7/8] Added createdAt --- aixplain/factories/model_factory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aixplain/factories/model_factory.py b/aixplain/factories/model_factory.py index da44600c..c543b973 100644 --- a/aixplain/factories/model_factory.py +++ b/aixplain/factories/model_factory.py @@ -31,6 +31,7 @@ from urllib.parse import urljoin from warnings import warn from aixplain.enums.function import FunctionInputOutput +from datetime import datetime class ModelFactory: @@ -67,6 +68,9 @@ def _create_model_from_response(cls, response: Dict) -> Model: if function == Function.TEXT_GENERATION: ModelClass = LLM + created_at = None + if "createdAt" in response and response["createdAt"]: + created_at = datetime.fromisoformat(response["createdAt"].replace("Z", "+00:00")) function_id = response["function"]["id"] function = Function(function_id) function_io = FunctionInputOutput.get(function_id, None) @@ -80,6 +84,7 @@ def _create_model_from_response(cls, response: Dict) -> Model: api_key=response["api_key"], cost=response["pricing"], function=function, + createdAt=created_at, parameters=parameters, input_params=input_params, output_params=output_params, From b2825380828dbfebb191217891e28d94fdd6056c Mon Sep 17 00:00:00 2001 From: xainaz Date: Tue, 3 Sep 2024 18:45:19 +0300 Subject: [PATCH 8/8] Added createdAt --- .../finetune/finetune_functional_test.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/functional/finetune/finetune_functional_test.py b/tests/functional/finetune/finetune_functional_test.py index 7b45613c..89ad6a78 100644 --- a/tests/functional/finetune/finetune_functional_test.py +++ b/tests/functional/finetune/finetune_functional_test.py @@ -1,5 +1,4 @@ __author__ = "lucaspavanelli" - """ Copyright 2022 The aiXplain SDK authors @@ -26,6 +25,7 @@ from aixplain.factories import FinetuneFactory from aixplain.modules.finetune.cost import FinetuneCost from aixplain.enums import Function, Language +from datetime import datetime, timedelta, timezone import pytest @@ -60,8 +60,22 @@ def validate_prompt_input_map(request): return request.param -def test_end2end(run_input_map): - model = ModelFactory.get(run_input_map["model_id"]) +@pytest.fixture(scope="module", params=read_data(RUN_FILE)) +def finetunable_llms_from_last_4_weeks(request): + four_weeks_ago = datetime.now(timezone.utc) - timedelta(weeks=4) + models = ModelFactory.list(function=Function.TEXT_GENERATION, is_finetunable=True)["results"] + + recent_models = [model for model in models if model.createdAt >= four_weeks_ago] + + if not recent_models: + yield ModelFactory.get(request.param["model_id"]) + else: + for model in recent_models: + yield model + + +def test_end2end(run_input_map, finetunable_llms_from_last_4_weeks): + model = finetunable_llms_from_last_4_weeks dataset_list = [DatasetFactory.list(query=run_input_map["dataset_name"])["results"][0]] train_percentage, dev_percentage = 100, 0 if run_input_map["required_dev"]: @@ -90,7 +104,7 @@ def test_end2end(run_input_map): assert result is not None if run_input_map["search_metadata"]: assert "details" in result - assert len(result["details"]) > 0 + assert len(result["details"]) > 0 assert "metadata" in result["details"][0] assert len(result["details"][0]["metadata"]) > 0 finetune_model.delete()