diff --git a/ads/aqua/evaluation.py b/ads/aqua/evaluation.py index dfce6e3ee..3f0c25ed4 100644 --- a/ads/aqua/evaluation.py +++ b/ads/aqua/evaluation.py @@ -38,6 +38,7 @@ JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING, NB_SESSION_IDENTIFIER, UNKNOWN, + extract_id_and_name_from_tag, fire_and_forget, get_container_image, is_valid_ocid, @@ -686,7 +687,7 @@ def create( self.telemetry.record_event_async( category="aqua/evaluation", action="create", - detail=evaluation_source.display_name, + detail=self._get_service_model_name(evaluation_source), ) return AquaEvaluationSummary( @@ -776,6 +777,34 @@ def _build_evaluation_runtime( ) return runtime + + @staticmethod + def _get_service_model_name( + source: Union[ModelDeployment, DataScienceModel] + ) -> str: + """Gets the service model name from source. If it's ModelDeployment, needs to check + if its model has been fine tuned or not. + + Parameters + ---------- + source: Union[ModelDeployment, DataScienceModel] + An instance of either ModelDeployment or DataScienceModel + + Returns + ------- + str: + The service model name of source. + """ + if isinstance(source, ModelDeployment): + fine_tuned_model_tag = source.freeform_tags.get( + Tags.AQUA_FINE_TUNED_MODEL_TAG.value, UNKNOWN + ) + if not fine_tuned_model_tag: + return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG.value) + else: + return extract_id_and_name_from_tag(fine_tuned_model_tag)[1] + + return source.display_name @staticmethod def _get_evaluation_container(source_id: str) -> str: diff --git a/ads/aqua/finetune.py b/ads/aqua/finetune.py index bbc0b516c..3bcfc65b7 100644 --- a/ads/aqua/finetune.py +++ b/ads/aqua/finetune.py @@ -469,6 +469,7 @@ def create( **telemetry_kwargs, ) # tracks unique fine-tuned models that were created in the user compartment + # TODO: retrieve the service model name for FT custom models. self.telemetry.record_event_async( category="aqua/service/finetune", action="create", diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 612376c3a..5ea5b6e41 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -16,6 +16,7 @@ from parameterized import parameterized from ads.aqua import utils +from ads.aqua.data import Tags from ads.aqua.evaluation import ( AquaEvalMetrics, AquaEvalReport, @@ -28,9 +29,10 @@ AquaRuntimeError, ) from ads.aqua.extension.base_handler import AquaAPIhandler -from ads.aqua.utils import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD +from ads.aqua.utils import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN from ads.jobs.ads_job import DataScienceJob, DataScienceJobRun, Job from ads.model import DataScienceModel +from ads.model.deployment.model_deployment import ModelDeployment from ads.model.model_version_set import ModelVersionSet null = None @@ -523,6 +525,41 @@ def test_create_evaluation( "time_created": f"{oci_dsc_model.time_created}", } + def test_get_service_model_name(self): + # get service model name from fine tuned model deployment + source = ( + ModelDeployment() + .with_freeform_tags( + **{ + Tags.AQUA_TAG.value: UNKNOWN, + Tags.AQUA_FINE_TUNED_MODEL_TAG.value: "test_service_model_id#test_service_model_name", + Tags.AQUA_MODEL_NAME_TAG.value: "test_fine_tuned_model_name" + } + ) + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + + # get service model name from model deployment + source = ( + ModelDeployment() + .with_freeform_tags( + **{ + Tags.AQUA_TAG.value: "active", + Tags.AQUA_MODEL_NAME_TAG.value: "test_service_model_name" + } + ) + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + + # get service model name from service model + source = DataScienceModel( + display_name="test_service_model_name" + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + @parameterized.expand( [ (