From 402db5231426d96417c48c0e878927f8dd827f50 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 12 Mar 2025 16:11:37 -0400 Subject: [PATCH 1/5] Changed multi model metadata storage. --- ads/aqua/common/entities.py | 5 ++ ads/aqua/constants.py | 1 + ads/aqua/evaluation/evaluation.py | 24 ++++++- ads/aqua/model/constants.py | 1 + ads/aqua/model/model.py | 58 +++++------------ ads/aqua/modeldeployment/deployment.py | 53 ++++++--------- .../deployment/aqua_multi_model.yaml | 64 ++----------------- .../with_extras/aqua/test_deployment.py | 39 +++++++++-- .../with_extras/aqua/test_evaluation.py | 40 +++++++++++- tests/unitary/with_extras/aqua/test_model.py | 3 +- 10 files changed, 144 insertions(+), 144 deletions(-) diff --git a/ads/aqua/common/entities.py b/ads/aqua/common/entities.py index a8c38c335..bd7b2ede8 100644 --- a/ads/aqua/common/entities.py +++ b/ads/aqua/common/entities.py @@ -153,6 +153,8 @@ class AquaMultiModelRef(Serializable): Number of GPUs required for deployment. env_var : Optional[Dict[str, Any]] Optional environment variables to override during deployment. + artifact_location : Optional[str] + Artifact path of model in the multimodel group. """ model_id: str = Field(..., description="The model OCID to deploy.") @@ -163,6 +165,9 @@ class AquaMultiModelRef(Serializable): env_var: Optional[dict] = Field( default_factory=dict, description="The environment variables of the model." ) + artifact_location: Optional[str] = Field( + None, description="Artifact path of model in the multimodel group." + ) class Config: extra = "ignore" diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index 0f7a501ba..9aff5749c 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -3,6 +3,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ """This module defines constants used in ads.aqua module.""" +UNKNOWN = "" UNKNOWN_VALUE = "" READY_TO_IMPORT_STATUS = "TRUE" UNKNOWN_DICT = {} diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 1a415e474..96954ab0e 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -97,6 +97,7 @@ from ads.model.model_metadata import ( MetadataTaxonomyKeys, ModelCustomMetadata, + ModelCustomMetadataItem, ModelProvenanceMetadata, ModelTaxonomyMetadata, ) @@ -578,6 +579,7 @@ def validate_model_name( This function verifies that: - The model group is not empty. + - The model multi metadata is present in the DataScienceModel metadata. - The user provided a non-empty model name. - The provided model name exists in the DataScienceModel metadata. - The deployment configuration contains core metadata required for validation. @@ -636,10 +638,28 @@ def validate_model_name( f"'{create_aqua_evaluation_details.evaluation_source_id}' does not contain any information about deployed models." ) + multi_model_metadata_value = custom_metadata_list.get( + ModelCustomMetadataFields.MULTIMODEL_METADATA, + ModelCustomMetadataItem(key=ModelCustomMetadataFields.MULTIMODEL_METADATA), + ).value + + if not multi_model_metadata_value: + error_message = ( + "Recreate the model deployment and retry the evaluation. An issue occured when initalizing the model group during deployment." + f"The {ModelCustomMetadataFields.MULTIMODEL_METADATA} is missing from the metadata in evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}." + ) + logger.debug(error_message) + raise AquaRuntimeError(error_message) + + multi_model_metadata = json.loads( + evaluation_source.dsc_model.get_custom_metadata_artifact( + metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA + ).decode("utf-8") + ) + # Build the list of valid model names from custom metadata. model_names = [ - custom_metadata_list.get(f"model-name-{idx}").value - for idx in range(model_group_count) + metadata.get("model_name", UNKNOWN) for metadata in multi_model_metadata ] # Check if the provided model name is among the valid names. diff --git a/ads/aqua/model/constants.py b/ads/aqua/model/constants.py index cff3253b5..9c5859671 100644 --- a/ads/aqua/model/constants.py +++ b/ads/aqua/model/constants.py @@ -19,6 +19,7 @@ class ModelCustomMetadataFields(ExtendedEnum): FINETUNE_CONTAINER = "finetune-container" DEPLOYMENT_CONTAINER_URI = "deployment-container-uri" MULTIMODEL_GROUP_COUNT = "model_group_count" + MULTIMODEL_METADATA = "multi_model_metadata" class ModelTask(ExtendedEnum): diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index f3162575c..880fa5f2a 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # Copyright (c) 2024, 2025 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +import json import os import pathlib from datetime import datetime, timedelta @@ -93,6 +94,7 @@ TENANCY_OCID, ) from ads.model import DataScienceModel +from ads.model.common.utils import MetadataArtifactPathType from ads.model.model_metadata import ( MetadataCustomCategory, ModelCustomMetadata, @@ -279,9 +281,10 @@ def create_multi( selected_models_deployment_containers = set() # Process each model - for idx, model in enumerate(models): + for model in models: source_model = DataScienceModel.from_id(model.model_id) display_name = source_model.display_name + # Update model name in user's input model model.model_name = model.model_name or display_name # TODO Uncomment the section below, if only service models should be allowed for multi-model deployment @@ -310,6 +313,9 @@ def create_multi( "Please register the model first." ) + # Update model artifact location in user's input model + model.artifact_location = model_artifact_path + artifact_list.append(model_artifact_path) # Validate deployment container consistency @@ -328,47 +334,6 @@ def create_multi( selected_models_deployment_containers.add(deployment_container) - # Add model-specific metadata - model_custom_metadata.add( - key=f"model-id-{idx}", - value=source_model.id, - description=f"ID of '{display_name}' in the multimodel group.", - category="Other", - ) - model_custom_metadata.add( - key=f"model-name-{idx}", - value=display_name, - description=f"Name of '{display_name}' in the multimodel group.", - category="Other", - ) - if model.gpu_count: - model_custom_metadata.add( - key=f"model-gpu-count-{idx}", - value=model.gpu_count, - description=f"GPU count of '{display_name}' in the multimodel group.", - category="Other", - ) - user_params = ( - " ".join( - f"{name} {value}" for name, value in model.env_var.items() - ).strip() - if model.env_var - else UNKNOWN - ) - if user_params: - model_custom_metadata.add( - key=f"model-user-params-{idx}", - value=user_params, - description=f"User params of '{display_name}' in the multimodel group.", - category="Other", - ) - model_custom_metadata.add( - key=f"{ModelCustomMetadataFields.ARTIFACT_LOCATION}-{idx}", - value=model_artifact_path, - description=f"Artifact path for '{display_name}' in the multimodel group.", - category="Other", - ) - # Check if the all models in the group shares same container family if len(selected_models_deployment_containers) > 1: raise AquaValueError( @@ -426,6 +391,15 @@ def create_multi( # Finalize creation custom_model.create(model_by_reference=True) + # Create custom metadata for multi model metadata + custom_model.create_custom_metadata_artifact( + metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA, + artifact_path_or_content=json.dumps( + [model.model_dump() for model in models] + ), + path_type=MetadataArtifactPathType.CONTENT, + ) + logger.info( f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}." ) diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index d0d4bc0b3..dc59b1854 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -63,7 +63,7 @@ ) from ads.aqua.modeldeployment.utils import MultiModelDeploymentConfigLoader from ads.common.object_storage_details import ObjectStorageDetails -from ads.common.utils import UNKNOWN, get_log_links +from ads.common.utils import get_log_links from ads.config import ( AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, @@ -550,7 +550,7 @@ def _create_multi( container_params = container_spec.get(ContainerSpec.CLI_PARM, UNKNOWN).strip() - for idx, model in enumerate(create_deployment_details.models): + for model in create_deployment_details.models: user_params = build_params_string(model.env_var) if user_params: restricted_params = self._find_restricted_params( @@ -589,22 +589,13 @@ def _create_multi( params = f"{params} {get_combined_params(config_parameters, user_params)}".strip() break - artifact_location_key = ( - f"{ModelCustomMetadataFields.ARTIFACT_LOCATION}-{idx}" - ) - artifact_path_prefix = aqua_model.custom_metadata_list.get( - artifact_location_key - ).value.rstrip("/") + artifact_path_prefix = model.artifact_location.rstrip("/") if ObjectStorageDetails.is_oci_path(artifact_path_prefix): os_path = ObjectStorageDetails.from_path(artifact_path_prefix) artifact_path_prefix = os_path.filepath.rstrip("/") model_config.append({"params": params, "model_path": artifact_path_prefix}) - - model_name_key = f"model-name-{idx}" - model_name_list.append( - aqua_model.custom_metadata_list.get(model_name_key).value - ) + model_name_list.append(model.model_name) env_var.update({AQUA_MULTI_MODEL_CONFIG: json.dumps({"models": model_config})}) @@ -960,28 +951,24 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": ) aqua_model = DataScienceModel.from_id(aqua_model_id) custom_metadata_list = aqua_model.custom_metadata_list - model_group_count = int( - custom_metadata_list.get( - ModelCustomMetadataFields.MULTIMODEL_GROUP_COUNT - ).value + multi_model_metadata_value = custom_metadata_list.get( + ModelCustomMetadataFields.MULTIMODEL_METADATA, + ModelCustomMetadataItem( + key=ModelCustomMetadataFields.MULTIMODEL_METADATA + ), + ).value + if not multi_model_metadata_value: + raise AquaRuntimeError( + f"Invalid multi model deployment {model_deployment_id}." + f"Make sure the custom metadata {ModelCustomMetadataFields.MULTIMODEL_METADATA} is added to the aqua multi model {aqua_model.display_name}." + ) + multi_model_metadata = json.loads( + aqua_model.dsc_model.get_custom_metadata_artifact( + metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA + ).decode("utf-8") ) aqua_deployment.models = [ - AquaMultiModelRef( - model_id=custom_metadata_list.get(f"model-id-{idx}").value, - model_name=custom_metadata_list.get(f"model-name-{idx}").value, - gpu_count=custom_metadata_list.get( - f"model-gpu-count-{idx}", - ModelCustomMetadataItem(key=f"model-gpu-count-{idx}"), - ).value, - env_var=get_params_dict( - custom_metadata_list.get( - f"model-user-params-{idx}", - ModelCustomMetadataItem(key=f"model-user-params-{idx}"), - ).value - or UNKNOWN - ), - ) - for idx in range(model_group_count) + AquaMultiModelRef(**metadata) for metadata in multi_model_metadata ] return AquaDeploymentDetail( diff --git a/tests/unitary/with_extras/aqua/test_data/deployment/aqua_multi_model.yaml b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_multi_model.yaml index 6cb0df3fc..7a498d035 100644 --- a/tests/unitary/with_extras/aqua/test_data/deployment/aqua_multi_model.yaml +++ b/tests/unitary/with_extras/aqua/test_data/deployment/aqua_multi_model.yaml @@ -4,66 +4,10 @@ spec: compartmentId: ocid1.compartment.oc1.. customMetadataList: data: - - category: Other - description: ID of model_one in the multimodel group. - key: model-id-0 - value: ocid1.compartment.oc1.. - - category: Other - description: Name of model_one in the multimodel group. - key: model-name-0 - value: model_one - - category: Other - description: GPU count of model_one in the multimodel group. - key: model-gpu-count-0 - value: 1 - - category: Other - description: User params of model_one in the multimodel group. - key: model-user-params-0 - value: --test_key_one test_value_one - - category: Other - description: Artifact path for model_one in the multimodel group. - key: artifact_location-0 - value: model_one_path - - category: Other - description: ID of model_two in the multimodel group. - key: model-id-1 - value: ocid1.compartment.oc1.. - - category: Other - description: Name of model_two in the multimodel group. - key: model-name-1 - value: model_two - - category: Other - description: GPU count of model_two in the multimodel group. - key: model-gpu-count-1 - value: 1 - - category: Other - description: User params of model_two in the multimodel group. - key: model-user-params-1 - value: --test_key_two test_value_two - - category: Other - description: Artifact path for model_two in the multimodel group. - key: artifact_location-1 - value: model_two_path - - category: Other - description: ID of model_three in the multimodel group. - key: model-id-2 - value: ocid1.compartment.oc1.. - - category: Other - description: Name of model_three in the multimodel group. - key: model-name-2 - value: model_three - - category: Other - description: GPU count of model_three in the multimodel group. - key: model-gpu-count-2 - value: 1 - - category: Other - description: User params of model_three in the multimodel group. - key: model-user-params-2 - value: --test_key_three test_value_three - - category: Other - description: Artifact path for model_three in the multimodel group. - key: artifact_location-2 - value: model_three_path + - category: null + description: null + key: multi_model_metadata + value: Uploaded - category: Other description: Inference container mapping for multi_model key: deployment-container diff --git a/tests/unitary/with_extras/aqua/test_deployment.py b/tests/unitary/with_extras/aqua/test_deployment.py index 255afe744..0f60e378d 100644 --- a/tests/unitary/with_extras/aqua/test_deployment.py +++ b/tests/unitary/with_extras/aqua/test_deployment.py @@ -15,7 +15,11 @@ import pytest from parameterized import parameterized -from ads.aqua.common.entities import AquaMultiModelRef, ComputeShapeSummary, ModelConfigResult +from ads.aqua.common.entities import ( + AquaMultiModelRef, + ComputeShapeSummary, + ModelConfigResult, +) import ads.aqua.modeldeployment.deployment import ads.config from ads.aqua.common.entities import AquaMultiModelRef @@ -403,19 +407,22 @@ class TestDataset: "env_var": {}, "gpu_count": 2, "model_id": "test_model_id_1", - "model_name": None, + "model_name": "test_model_1", + "artifact_location": "test_location_1", }, { "env_var": {}, "gpu_count": 2, "model_id": "test_model_id_2", - "model_name": None, + "model_name": "test_model_2", + "artifact_location": "test_location_2", }, { "env_var": {}, "gpu_count": 2, "model_id": "test_model_id_3", - "model_name": None, + "model_name": "test_model_3", + "artifact_location": "test_location_3", }, ], "model_id": "ocid1.datasciencemodel.oc1..", @@ -880,18 +887,21 @@ class TestDataset: "gpu_count": 1, "model_id": "ocid1.compartment.oc1..", "model_name": "model_one", + "artifact_location": "artifact_location_one", }, { "env_var": {"--test_key_two": "test_value_two"}, "gpu_count": 1, "model_id": "ocid1.compartment.oc1..", "model_name": "model_two", + "artifact_location": "artifact_location_two", }, { "env_var": {"--test_key_three": "test_value_three"}, "gpu_count": 1, "model_id": "ocid1.compartment.oc1..", "model_name": "model_three", + "artifact_location": "artifact_location_three", }, ] @@ -978,10 +988,16 @@ def test_get_deployment(self, mock_get_resource_name): assert result.log.name == "log-name" assert result.log_group.name == "log-group-name" + @patch( + "ads.model.service.oci_datascience_model.OCIDataScienceModel.get_custom_metadata_artifact" + ) @patch("ads.model.DataScienceModel.from_id") @patch("ads.aqua.modeldeployment.deployment.get_resource_name") def test_get_multi_model_deployment( - self, mock_get_resource_name, mock_model_from_id + self, + mock_get_resource_name, + mock_model_from_id, + mock_get_custom_metadata_artifact, ): multi_model_deployment = copy.deepcopy( TestDataset.multi_model_deployment_object @@ -1012,6 +1028,13 @@ def test_get_multi_model_deployment( uri=aqua_multi_model ) + multi_model_deployment_model_attributes_str = json.dumps( + TestDataset.multi_model_deployment_model_attributes + ).encode("utf-8") + mock_get_custom_metadata_artifact.return_value = ( + multi_model_deployment_model_attributes_str + ) + result = self.app.get(model_deployment_id=TestDataset.MODEL_DEPLOYMENT_ID) expected_attributes = set(AquaDeploymentDetail.__annotations__.keys()) | set( @@ -1588,17 +1611,23 @@ def test_create_deployment_for_multi_model( model_info_1 = AquaMultiModelRef( model_id="test_model_id_1", + model_name="test_model_1", gpu_count=2, + artifact_location="test_location_1", ) model_info_2 = AquaMultiModelRef( model_id="test_model_id_2", + model_name="test_model_2", gpu_count=2, + artifact_location="test_location_2", ) model_info_3 = AquaMultiModelRef( model_id="test_model_id_3", + model_name="test_model_3", gpu_count=2, + artifact_location="test_location_3", ) result = self.app.create( diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 0925dd3e5..d8826e0f4 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -359,6 +359,30 @@ class TestDataset: INVALID_EVAL_ID = "ocid1.datasciencemodel.oc1.phx." MODEL_DEPLOYMENT_ID = "ocid1.datasciencemodeldeployment.oc1.." + multi_model_deployment_model_attributes = [ + { + "env_var": {"--test_key_one": "test_value_one"}, + "gpu_count": 1, + "model_id": "ocid1.compartment.oc1..", + "model_name": "model_one", + "artifact_location": "artifact_location_one", + }, + { + "env_var": {"--test_key_two": "test_value_two"}, + "gpu_count": 1, + "model_id": "ocid1.compartment.oc1..", + "model_name": "model_two", + "artifact_location": "artifact_location_two", + }, + { + "env_var": {"--test_key_three": "test_value_three"}, + "gpu_count": 1, + "model_id": "ocid1.compartment.oc1..", + "model_name": "model_three", + "artifact_location": "artifact_location_three", + }, + ] + class TestAquaEvaluation(unittest.TestCase): """Contains unittests for TestAquaEvaluationApp.""" @@ -551,8 +575,15 @@ def test_create_evaluation( ] ) @patch("ads.aqua.evaluation.evaluation.AquaEvaluationApp.create") + @patch( + "ads.model.datascience_model.OCIDataScienceModel.get_custom_metadata_artifact" + ) def test_validate_model_name( - self, mock_model_parameters, expected_message, mock_model + self, + mock_model_parameters, + expected_message, + mock_get_custom_metadata_artifact, + mock_model, ): curr_dir = os.path.dirname(__file__) @@ -583,6 +614,13 @@ def test_validate_model_name( mock_model = DataScienceModel.from_yaml(uri=aqua_multi_model) + multi_model_deployment_model_attributes_str = json.dumps( + TestDataset.multi_model_deployment_model_attributes + ).encode("utf-8") + mock_get_custom_metadata_artifact.return_value = ( + multi_model_deployment_model_attributes_str + ) + mock_create_aqua_evaluation_details = MagicMock( **create_aqua_evaluation_details, spec=CreateAquaEvaluationDetails ) diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 856936daa..4f59e3fe1 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -362,6 +362,7 @@ def test_create_model(self, mock_from_id, mock_validate, mock_create): assert model.provenance_metadata.training_id == "test_training_id" @patch.object(DataScienceModel, "add_artifact") + @patch.object(DataScienceModel, "create_custom_metadata_artifact") @patch.object(DataScienceModel, "create") @patch("ads.model.datascience_model.validate") @patch("ads.aqua.model.model.get_container_config") @@ -369,9 +370,9 @@ def test_create_model(self, mock_from_id, mock_validate, mock_create): def test_create_multimodel( self, mock_from_id, - mock_get_container_config, mock_validate, mock_create, + mock_create_custom_metadata_artifact, mock_add_artifact, ): mock_get_container_config.return_value = get_container_config() From 3dc9977f2e3209bd8527482473dc7634e75da3b3 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 12 Mar 2025 16:24:36 -0400 Subject: [PATCH 2/5] Updated pr. --- tests/unitary/with_extras/aqua/test_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unitary/with_extras/aqua/test_model.py b/tests/unitary/with_extras/aqua/test_model.py index 4f59e3fe1..f202228fd 100644 --- a/tests/unitary/with_extras/aqua/test_model.py +++ b/tests/unitary/with_extras/aqua/test_model.py @@ -370,6 +370,7 @@ def test_create_model(self, mock_from_id, mock_validate, mock_create): def test_create_multimodel( self, mock_from_id, + mock_get_container_config, mock_validate, mock_create, mock_create_custom_metadata_artifact, From b321d5c49424d90394339b2d3a2817b5df592c9c Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 12 Mar 2025 17:10:35 -0400 Subject: [PATCH 3/5] Updated pr. --- ads/aqua/modeldeployment/deployment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 2ceacb1a2..2a6b342fd 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -62,7 +62,7 @@ ) from ads.aqua.modeldeployment.utils import MultiModelDeploymentConfigLoader from ads.common.object_storage_details import ObjectStorageDetails -from ads.common.utils import get_log_links +from ads.common.utils import UNKNOWN, get_log_links from ads.config import ( AQUA_DEPLOYMENT_CONTAINER_CMD_VAR_METADATA_NAME, AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME, From f9a3064dff7bc09c795ae854e08c17d9765bdf64 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 12 Mar 2025 17:11:33 -0400 Subject: [PATCH 4/5] Updated pr. --- ads/aqua/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index 9aff5749c..0f7a501ba 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -3,7 +3,6 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ """This module defines constants used in ads.aqua module.""" -UNKNOWN = "" UNKNOWN_VALUE = "" READY_TO_IMPORT_STATUS = "TRUE" UNKNOWN_DICT = {} From 740bf3d924be289b792fda0c468b08384c56fb3d Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 12 Mar 2025 18:25:55 -0400 Subject: [PATCH 5/5] Updated pr. --- ads/aqua/evaluation/evaluation.py | 28 ++++++++++++++++++-------- ads/aqua/model/model.py | 8 ++++++-- ads/aqua/modeldeployment/deployment.py | 4 ++-- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 96954ab0e..9f52fbe7b 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -24,6 +24,7 @@ from ads.aqua import logger from ads.aqua.app import AquaApp from ads.aqua.common import utils +from ads.aqua.common.entities import AquaMultiModelRef from ads.aqua.common.enums import ( DataScienceResource, Resource, @@ -645,21 +646,32 @@ def validate_model_name( if not multi_model_metadata_value: error_message = ( - "Recreate the model deployment and retry the evaluation. An issue occured when initalizing the model group during deployment." - f"The {ModelCustomMetadataFields.MULTIMODEL_METADATA} is missing from the metadata in evaluation source ID: {create_aqua_evaluation_details.evaluation_source_id}." + f"Required model metadata is missing for evaluation source ID: {evaluation_source.id}. " + f"A valid multi-model deployment requires {ModelCustomMetadataFields.MULTIMODEL_METADATA}. " + "Please recreate the model deployment and retry the evaluation, as an issue occurred during the initialization of the model group." ) logger.debug(error_message) raise AquaRuntimeError(error_message) - multi_model_metadata = json.loads( - evaluation_source.dsc_model.get_custom_metadata_artifact( - metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA - ).decode("utf-8") - ) + try: + multi_model_metadata = json.loads( + evaluation_source.dsc_model.get_custom_metadata_artifact( + metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA + ).decode("utf-8") + ) + except Exception as ex: + error_message = ( + f"Error fetching {ModelCustomMetadataFields.MULTIMODEL_METADATA} " + f"from custom metadata for evaluation source ID '{evaluation_source.id}'. " + f"Details: {ex}" + ) + logger.error(error_message) + raise AquaRuntimeError(error_message) # Build the list of valid model names from custom metadata. model_names = [ - metadata.get("model_name", UNKNOWN) for metadata in multi_model_metadata + AquaMultiModelRef(**metadata).model_name + for metadata in multi_model_metadata ] # Check if the provided model name is among the valid names. diff --git a/ads/aqua/model/model.py b/ads/aqua/model/model.py index 880fa5f2a..6c2d5b61e 100644 --- a/ads/aqua/model/model.py +++ b/ads/aqua/model/model.py @@ -391,6 +391,10 @@ def create_multi( # Finalize creation custom_model.create(model_by_reference=True) + logger.info( + f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}." + ) + # Create custom metadata for multi model metadata custom_model.create_custom_metadata_artifact( metadata_key_name=ModelCustomMetadataFields.MULTIMODEL_METADATA, @@ -400,8 +404,8 @@ def create_multi( path_type=MetadataArtifactPathType.CONTENT, ) - logger.info( - f"Aqua Model '{custom_model.id}' created with models: {', '.join(display_name_list)}." + logger.debug( + f"Multi model metadata uploaded for Aqua model: {custom_model.id}." ) # Track telemetry event diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 2a6b342fd..0d7b6d2fb 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -958,8 +958,8 @@ def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail": ).value if not multi_model_metadata_value: raise AquaRuntimeError( - f"Invalid multi model deployment {model_deployment_id}." - f"Make sure the custom metadata {ModelCustomMetadataFields.MULTIMODEL_METADATA} is added to the aqua multi model {aqua_model.display_name}." + f"Invalid multi-model deployment: {model_deployment_id}. " + f"Ensure that the required custom metadata `{ModelCustomMetadataFields.MULTIMODEL_METADATA}` is added to the AQUA multi-model `{aqua_model.display_name}` ({aqua_model.id})." ) multi_model_metadata = json.loads( aqua_model.dsc_model.get_custom_metadata_artifact(