From 621e27716054b487cadb1b2ea1a64c309daa81c8 Mon Sep 17 00:00:00 2001 From: Vipul Date: Mon, 15 Apr 2024 23:24:31 -0700 Subject: [PATCH 01/14] show extension for default region --- ads/aqua/extension/common_handler.py | 5 ++ ads/aqua/utils.py | 13 ++++ .../with_extras/aqua/test_common_handler.py | 64 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/unitary/with_extras/aqua/test_common_handler.py diff --git a/ads/aqua/extension/common_handler.py b/ads/aqua/extension/common_handler.py index 1e6429f79..1061cb20b 100644 --- a/ads/aqua/extension/common_handler.py +++ b/ads/aqua/extension/common_handler.py @@ -9,6 +9,8 @@ from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID from ads.aqua.exception import AquaResourceAccessError +from ads.aqua.utils import known_realm +from ads.aqua.decorator import handle_exceptions class ADSVersionHandler(AquaAPIhandler): @@ -21,9 +23,12 @@ def get(self): class CompatibilityCheckHandler(AquaAPIhandler): """The handler to check if the extension is compatible.""" + @handle_exceptions def get(self): if ODSC_MODEL_COMPARTMENT_OCID: return self.finish(dict(status="ok")) + elif known_realm(): + return self.finish(dict(status="compatible")) else: raise AquaResourceAccessError( f"The AI Quick actions extension is not compatible in the given region." diff --git a/ads/aqua/utils.py b/ads/aqua/utils.py index 90a0cd560..70c52d38e 100644 --- a/ads/aqua/utils.py +++ b/ads/aqua/utils.py @@ -79,6 +79,7 @@ LIFECYCLE_DETAILS_MISSING_JOBRUN = "The asscociated JobRun resource has been deleted." READY_TO_DEPLOY_STATUS = "ACTIVE" READY_TO_FINE_TUNE_STATUS = "TRUE" +AQUA_GA_LIST = ["id19sfcrra6z"] class LifecycleStatus(Enum): @@ -733,3 +734,15 @@ def _is_valid_mvs(mvs: ModelVersionSet, target_tag: str) -> bool: return False return target_tag in mvs.freeform_tags + + +def known_realm(): + """This helper function returns True if the Aqua service is available by default in the + given namespace. + Returns + ------- + bool: + Return True if aqua service is available. + + """ + return os.environ.get("CONDA_BUCKET_NS", "#") in AQUA_GA_LIST diff --git a/tests/unitary/with_extras/aqua/test_common_handler.py b/tests/unitary/with_extras/aqua/test_common_handler.py new file mode 100644 index 000000000..d53d35507 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_common_handler.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-- + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +import os +import json +from importlib import reload +from tornado.web import Application +from tornado.testing import AsyncHTTPTestCase + +import ads.config +import ads.aqua +from ads.aqua.utils import AQUA_GA_LIST +from ads.aqua.extension.common_handler import CompatibilityCheckHandler + + +class TestDataset: + SERVICE_COMPARTMENT_ID = "ocid1.compartment.oc1.." + + +class TestYourHandler(AsyncHTTPTestCase): + def get_app(self): + return Application([(r"/hello", CompatibilityCheckHandler)]) + + def setUp(self): + super().setUp() + os.environ["ODSC_MODEL_COMPARTMENT_OCID"] = TestDataset.SERVICE_COMPARTMENT_ID + reload(ads.config) + reload(ads.aqua) + reload(ads.aqua.extension.common_handler) + + def tearDown(self): + super().tearDown() + os.environ.pop("ODSC_MODEL_COMPARTMENT_OCID", None) + reload(ads.config) + reload(ads.aqua) + reload(ads.aqua.extension.common_handler) + + def test_get_ok(self): + response = self.fetch("/hello", method="GET") + assert json.loads(response.body)["status"] == "ok" + + def test_get_compatible_status(self): + os.environ["ODSC_MODEL_COMPARTMENT_OCID"] = "" + os.environ["CONDA_BUCKET_NS"] = AQUA_GA_LIST[0] + reload(ads.common) + reload(ads.aqua) + reload(ads.aqua.extension.common_handler) + response = self.fetch("/hello", method="GET") + assert json.loads(response.body)["status"] == "compatible" + + def test_raise_not_compatible_error(self): + os.environ["ODSC_MODEL_COMPARTMENT_OCID"] = "" + os.environ["CONDA_BUCKET_NS"] = "test-namespace" + reload(ads.common) + reload(ads.aqua) + reload(ads.aqua.extension.common_handler) + response = self.fetch("/hello", method="GET") + assert ( + json.loads(response.body)["message"] + == "Authorization Failed: The resource you're looking for isn't accessible." + ) From a508b68011da05a884ca71863542387554c34f5c Mon Sep 17 00:00:00 2001 From: Vipul Date: Mon, 15 Apr 2024 23:36:33 -0700 Subject: [PATCH 02/14] space issue --- ads/aqua/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ads/aqua/utils.py b/ads/aqua/utils.py index 70c52d38e..ab72723dc 100644 --- a/ads/aqua/utils.py +++ b/ads/aqua/utils.py @@ -737,8 +737,7 @@ def _is_valid_mvs(mvs: ModelVersionSet, target_tag: str) -> bool: def known_realm(): - """This helper function returns True if the Aqua service is available by default in the - given namespace. + """This helper function returns True if the Aqua service is available by default in the given namespace. Returns ------- bool: From c2fa183b77d65f2b700947c2f7aa3d240328a2c6 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 17 Apr 2024 10:09:40 -0400 Subject: [PATCH 03/14] Updated source name for telemetry. --- ads/aqua/evaluation.py | 31 +++++++++++++- .../with_extras/aqua/test_evaluation.py | 41 ++++++++++++++++++- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/ads/aqua/evaluation.py b/ads/aqua/evaluation.py index bbf8be86c..40afea6e4 100644 --- a/ads/aqua/evaluation.py +++ b/ads/aqua/evaluation.py @@ -38,6 +38,7 @@ JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING, NB_SESSION_IDENTIFIER, UNKNOWN, + extract_id_and_name_from_tag, fire_and_forget, get_container_image, is_valid_ocid, @@ -677,7 +678,7 @@ def create( self.telemetry.record_event_async( category="aqua/evaluation", action="create", - detail=evaluation_source.display_name, + detail=self._get_service_model_name(evaluation_source), ) return AquaEvaluationSummary( @@ -767,6 +768,34 @@ def _build_evaluation_runtime( ) return runtime + + @staticmethod + def _get_service_model_name( + source: Union[ModelDeployment, DataScienceModel] + ) -> str: + """Gets the service model name from source. If it's ModelDeployment, needs to check + if its model has been fine tuned or not. + + Parameters + ---------- + source: Union[ModelDeployment, DataScienceModel] + An instance of either ModelDeployment or DataScienceModel + + Returns + ------- + str: + The service model name of source. + """ + if isinstance(source, ModelDeployment): + fine_tuned_model_tag = source.freeform_tags.get( + Tags.AQUA_FINE_TUNED_MODEL_TAG.value, UNKNOWN + ) + if not fine_tuned_model_tag: + return source.freeform_tags.get(Tags.AQUA_MODEL_NAME_TAG.value) + else: + return extract_id_and_name_from_tag(fine_tuned_model_tag)[1] + + return source.display_name @staticmethod def _get_evaluation_container(source_id: str) -> str: diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index f12ebbab7..f285df791 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -16,6 +16,7 @@ from parameterized import parameterized from ads.aqua import utils +from ads.aqua.data import Tags from ads.aqua.evaluation import ( AquaEvalMetrics, AquaEvalReport, @@ -28,9 +29,10 @@ AquaRuntimeError, ) from ads.aqua.extension.base_handler import AquaAPIhandler -from ads.aqua.utils import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD +from ads.aqua.utils import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN from ads.jobs.ads_job import DataScienceJob, DataScienceJobRun, Job from ads.model import DataScienceModel +from ads.model.deployment.model_deployment import ModelDeployment from ads.model.model_version_set import ModelVersionSet null = None @@ -355,6 +357,8 @@ def setUpClass(cls): utils.is_valid_ocid = MagicMock(return_value=True) def setUp(self): + import ads + ads.set_auth("security_token") self.app = AquaEvaluationApp() self._query_resources = utils.query_resources @@ -523,6 +527,41 @@ def test_create_evaluation( "time_created": f"{oci_dsc_model.time_created}", } + def test_get_service_model_name(self): + # get service model name from fine tuned model deployment + source = ( + ModelDeployment() + .with_freeform_tags( + **{ + Tags.AQUA_TAG.value: UNKNOWN, + Tags.AQUA_FINE_TUNED_MODEL_TAG.value: "test_service_model_id#test_service_model_name", + Tags.AQUA_MODEL_NAME_TAG.value: "test_fine_tuned_model_name" + } + ) + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + + # get service model name from model deployment + source = ( + ModelDeployment() + .with_freeform_tags( + **{ + Tags.AQUA_TAG.value: "active", + Tags.AQUA_MODEL_NAME_TAG.value: "test_service_model_name" + } + ) + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + + # get service model name from service model + source = DataScienceModel( + display_name="test_service_model_name" + ) + service_model_name = self.app._get_service_model_name(source) + assert service_model_name == "test_service_model_name" + @parameterized.expand( [ ( From 7eee820c69e26f47e576002cdac01b7a134499dc Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 17 Apr 2024 10:54:48 -0400 Subject: [PATCH 04/14] Updated pr, --- tests/unitary/with_extras/aqua/test_evaluation.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index f285df791..bd7fc0154 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -357,8 +357,6 @@ def setUpClass(cls): utils.is_valid_ocid = MagicMock(return_value=True) def setUp(self): - import ads - ads.set_auth("security_token") self.app = AquaEvaluationApp() self._query_resources = utils.query_resources From 8dd0704dfe1679312645cdfd08a63adce9b3e2d3 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 17 Apr 2024 08:43:10 -0700 Subject: [PATCH 05/14] change assert statements --- tests/unitary/with_extras/aqua/test_common_handler.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unitary/with_extras/aqua/test_common_handler.py b/tests/unitary/with_extras/aqua/test_common_handler.py index d53d35507..c8bc913b5 100644 --- a/tests/unitary/with_extras/aqua/test_common_handler.py +++ b/tests/unitary/with_extras/aqua/test_common_handler.py @@ -58,7 +58,9 @@ def test_raise_not_compatible_error(self): reload(ads.aqua) reload(ads.aqua.extension.common_handler) response = self.fetch("/hello", method="GET") + body = json.loads(response.body) + assert body["status"] == 404 assert ( - json.loads(response.body)["message"] - == "Authorization Failed: The resource you're looking for isn't accessible." + body["reason"] + == "The AI Quick actions extension is not compatible in the given region." ) From 31c77fb2009ad01688b565cfe39594f2f30ca9be Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 17 Apr 2024 17:04:03 -0400 Subject: [PATCH 06/14] Added default compartment id for evaluation list. --- ads/aqua/evaluation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ads/aqua/evaluation.py b/ads/aqua/evaluation.py index bbf8be86c..3a56a16f3 100644 --- a/ads/aqua/evaluation.py +++ b/ads/aqua/evaluation.py @@ -916,6 +916,7 @@ def list( List[AquaEvaluationSummary]: The list of the `ads.aqua.evalution.AquaEvaluationSummary`. """ + compartment_id = compartment_id or COMPARTMENT_OCID logger.info(f"Fetching evaluations from compartment {compartment_id}.") models = utils.query_resources( compartment_id=compartment_id, From 0efe6fd9a49ce1ce9eaca74700e2d17b73002f5c Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Wed, 17 Apr 2024 17:04:20 -0700 Subject: [PATCH 07/14] Adds supporting BLEU score metric for evaluation. --- ads/aqua/evaluation.py | 60 ++++++++++++++++++++++++++++++------------ ads/config.py | 1 + 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/ads/aqua/evaluation.py b/ads/aqua/evaluation.py index bbf8be86c..c33db2392 100644 --- a/ads/aqua/evaluation.py +++ b/ads/aqua/evaluation.py @@ -78,7 +78,7 @@ class EvaluationJobExitCode(Enum): SUCCESS = 0 COMMON_ERROR = 1 - # Configuration-related issues + # Configuration-related issues 10-19 INVALID_EVALUATION_CONFIG = 10 EVALUATION_CONFIG_NOT_PROVIDED = 11 INVALID_OUTPUT_DIR = 12 @@ -87,7 +87,7 @@ class EvaluationJobExitCode(Enum): INVALID_TARGET_EVALUATION_ID = 15 INVALID_EVALUATION_CONFIG_VALIDATION = 16 - # Evaluation process issues + # Evaluation process issues 20-39 OUTPUT_DIR_NOT_FOUND = 20 INVALID_INPUT_DATASET = 21 INPUT_DATA_NOT_FOUND = 22 @@ -100,6 +100,7 @@ class EvaluationJobExitCode(Enum): MODEL_INFERENCE_WRONG_RESPONSE_FORMAT = 29 UNSUPPORTED_METRICS = 30 METRIC_CALCULATION_FAILURE = 31 + EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED = 32 EVALUATION_JOB_EXIT_CODE_MESSAGE = { @@ -124,6 +125,11 @@ class EvaluationJobExitCode(Enum): EvaluationJobExitCode.MODEL_INFERENCE_WRONG_RESPONSE_FORMAT.value: "Evaluation encountered unsupported, or unexpected model output, verify the target evaluation model is compatible and produces the correct format.", EvaluationJobExitCode.UNSUPPORTED_METRICS.value: "None of the provided metrics are supported by the framework.", EvaluationJobExitCode.METRIC_CALCULATION_FAILURE.value: "All attempted metric calculations were unsuccessful. Please review the metric configurations and input data.", + EvaluationJobExitCode.EVALUATION_MODEL_CATALOG_RECORD_CREATION_FAILED.value: ( + "Failed to create a Model Catalog record for the evaluation. " + "This could be due to missing required permissions. " + "Please check the log for more information." + ), } @@ -849,13 +855,17 @@ def get(self, eval_id) -> AquaEvaluationDetail: loggroup_id = "" loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id) - log_url = get_log_links( - region=self.region, - log_group_id=loggroup_id, - log_id=log_id, - compartment_id=job_run_details.compartment_id, - source_id=jobrun_id - ) if job_run_details else "" + log_url = ( + get_log_links( + region=self.region, + log_group_id=loggroup_id, + log_id=log_id, + compartment_id=job_run_details.compartment_id, + source_id=jobrun_id, + ) + if job_run_details + else "" + ) log_name = None loggroup_name = None @@ -931,7 +941,6 @@ def list( evaluations = [] async_tasks = [] for model in models: - if model.identifier in self._eval_cache.keys(): logger.debug(f"Retrieving evaluation {model.identifier} from cache.") evaluations.append(self._eval_cache.get(model.identifier)) @@ -1049,13 +1058,17 @@ def get_status(self, eval_id: str) -> dict: loggroup_id = "" loggroup_url = get_log_links(region=self.region, log_group_id=loggroup_id) - log_url = get_log_links( - region=self.region, - log_group_id=loggroup_id, - log_id=log_id, - compartment_id=job_run_details.compartment_id, - source_id=jobrun_id - ) if job_run_details else "" + log_url = ( + get_log_links( + region=self.region, + log_group_id=loggroup_id, + log_id=log_id, + compartment_id=job_run_details.compartment_id, + source_id=jobrun_id, + ) + if job_run_details + else "" + ) return dict( id=eval_id, @@ -1100,6 +1113,19 @@ def get_supported_metrics(self) -> dict: ), "args": {}, }, + { + "use_case": ["text_generation"], + "key": "bleu", + "name": "bleu", + "description": ( + "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the " + "quality of text which has been machine-translated from one natural language to another. " + "Quality is considered to be the correspondence between a machine's output and that of a " + "human: 'the closer a machine translation is to a professional human translation, " + "the better it is'." + ), + "args": {}, + }, ] @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua") diff --git a/ads/config.py b/ads/config.py index 382ffadd7..e4d76e703 100644 --- a/ads/config.py +++ b/ads/config.py @@ -79,6 +79,7 @@ "AQUA_TELEMETRY_BUCKET", "service-managed-models" ) AQUA_TELEMETRY_BUCKET_NS = os.environ.get("AQUA_TELEMETRY_BUCKET_NS", CONDA_BUCKET_NS) + DEBUG_TELEMETRY = os.environ.get("DEBUG_TELEMETRY", None) AQUA_SERVICE_NAME = "aqua" DATA_SCIENCE_SERVICE_NAME = "data-science" From afea502495c9f83d809f6ee715725a948931ec44 Mon Sep 17 00:00:00 2001 From: Vipul Date: Wed, 17 Apr 2024 18:26:39 -0700 Subject: [PATCH 08/14] update md telemetry --- ads/aqua/deployment.py | 70 ++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/ads/aqua/deployment.py b/ads/aqua/deployment.py index 184bfdf27..db38cc83f 100644 --- a/ads/aqua/deployment.py +++ b/ads/aqua/deployment.py @@ -391,40 +391,30 @@ def create( .with_runtime(container_runtime) ).deploy(wait_for_completion=False) - if is_fine_tuned_model: - # tracks unique deployments that were created in the user compartment - self.telemetry.record_event_async( - category="aqua/custom/deployment", action="create", detail=model_name - ) - # tracks the shape used for deploying the custom models - self.telemetry.record_event_async( - category="aqua/custom/deployment/create", - action="shape", - detail=instance_shape, - ) - # tracks the shape used for deploying the custom models by name - self.telemetry.record_event_async( - category=f"aqua/custom/{model_name}/deployment/create", - action="shape", - detail=instance_shape, - ) - else: - # tracks unique deployments that were created in the user compartment - self.telemetry.record_event_async( - category="aqua/service/deployment", action="create", detail=model_name - ) - # tracks the shape used for deploying the service models - self.telemetry.record_event_async( - category="aqua/service/deployment/create", - action="shape", - detail=instance_shape, - ) - # tracks the shape used for deploying the service models by name - self.telemetry.record_event_async( - category=f"aqua/service/{model_name}/deployment/create", - action="shape", - detail=instance_shape, - ) + model_type = "custom" if is_fine_tuned_model else "service" + deployment_id = deployment.dsc_model_deployment.id + telemetry_kwargs = ( + {"ocid": deployment_id[-8:]} if len(deployment_id) > 8 else {} + ) + # tracks unique deployments that were created in the user compartment + self.telemetry.record_event_async( + category=f"aqua/{model_type}/deployment", + action="create", + detail=model_name, + **telemetry_kwargs, + ) + # tracks the shape used for deploying the custom or service models + self.telemetry.record_event_async( + category=f"aqua/{model_type}/deployment/create", + action="shape", + detail=instance_shape, + ) + # tracks the shape used for deploying the custom or service models by name + self.telemetry.record_event_async( + category=f"aqua/{model_type}/{model_name}/deployment/create", + action="shape", + detail=instance_shape, + ) return AquaDeployment.from_oci_model_deployment( deployment.dsc_model_deployment, self.region @@ -471,6 +461,18 @@ def list(self, **kwargs) -> List["AquaDeployment"]: ) ) + # log telemetry if MD is in active or failed state + deployment_id = model_deployment.id + state = model_deployment.lifecycle_state.upper() + if state in ["ACTIVE", "FAILED"]: + # tracks unique deployments that were listed in the user compartment + self.telemetry.record_event_async( + category=f"aqua/deployment", + action="list", + detail=deployment_id[-8:] if len(deployment_id) > 8 else "", + value=state, + ) + # tracks number of times deployment listing was called self.telemetry.record_event_async(category="aqua/deployment", action="list") From 3e7fd13c19eb33ac05f5a6ebfbb410a6fc600c7a Mon Sep 17 00:00:00 2001 From: MING KANG Date: Thu, 18 Apr 2024 10:36:21 -0400 Subject: [PATCH 09/14] Improvements on handling error raised in AQUA --- ads/aqua/decorator.py | 8 + ads/aqua/extension/base_handler.py | 26 ++- ads/aqua/extension/common_handler.py | 5 +- ads/aqua/extension/deployment_handler.py | 5 +- ads/aqua/extension/evaluation_handler.py | 3 +- ads/aqua/extension/finetune_handler.py | 1 - ads/aqua/extension/ui_handler.py | 13 +- ads/aqua/extension/utils.py | 8 +- .../with_extras/aqua/test_decorator.py | 193 ++++++++++++++++++ .../aqua/test_evaluation_handler.py | 4 +- .../unitary/with_extras/aqua/test_handlers.py | 51 +++-- 11 files changed, 266 insertions(+), 51 deletions(-) create mode 100644 tests/unitary/with_extras/aqua/test_decorator.py diff --git a/ads/aqua/decorator.py b/ads/aqua/decorator.py index 851fadc19..520e930d9 100644 --- a/ads/aqua/decorator.py +++ b/ads/aqua/decorator.py @@ -17,6 +17,7 @@ RequestException, ServiceError, ) +from tornado.web import HTTPError from ads.aqua.exception import AquaError from ads.aqua.extension.base_handler import AquaAPIhandler @@ -58,6 +59,7 @@ def inner_function(self: AquaAPIhandler, *args, **kwargs): except ServiceError as error: self.write_error( status_code=error.status or 500, + message=error.message, reason=error.message, service_payload=error.args[0] if error.args else None, exc_info=sys.exc_info(), @@ -91,6 +93,12 @@ def inner_function(self: AquaAPIhandler, *args, **kwargs): service_payload=error.service_payload, exc_info=sys.exc_info(), ) + except HTTPError as e: + self.write_error( + status_code=e.status_code, + reason=e.log_message, + exc_info=sys.exc_info(), + ) except Exception as ex: self.write_error( status_code=500, diff --git a/ads/aqua/extension/base_handler.py b/ads/aqua/extension/base_handler.py index f0b7d937b..73c676cdc 100644 --- a/ads/aqua/extension/base_handler.py +++ b/ads/aqua/extension/base_handler.py @@ -8,14 +8,16 @@ import traceback import uuid from dataclasses import asdict, is_dataclass +from http.client import responses from typing import Any from notebook.base.handlers import APIHandler -from tornado.web import HTTPError, Application from tornado import httputil -from ads.telemetry.client import TelemetryClient -from ads.config import AQUA_TELEMETRY_BUCKET, AQUA_TELEMETRY_BUCKET_NS +from tornado.web import Application, HTTPError + from ads.aqua import logger +from ads.config import AQUA_TELEMETRY_BUCKET, AQUA_TELEMETRY_BUCKET_NS +from ads.telemetry.client import TelemetryClient class AquaAPIhandler(APIHandler): @@ -66,12 +68,15 @@ def finish(self, payload=None): # pylint: disable=W0221 def write_error(self, status_code, **kwargs): """AquaAPIhandler errors are JSON, not human pages.""" - self.set_header("Content-Type", "application/json") reason = kwargs.get("reason") self.set_status(status_code, reason=reason) service_payload = kwargs.get("service_payload", {}) - message = self.get_default_error_messages(service_payload, str(status_code)) + default_msg = responses.get(status_code, "Unknown HTTP Error") + message = self.get_default_error_messages( + service_payload, str(status_code), kwargs.get("message", default_msg) + ) + reply = { "status": status_code, "message": message, @@ -84,7 +89,7 @@ def write_error(self, status_code, **kwargs): e = exc_info[1] if isinstance(e, HTTPError): reply["message"] = e.log_message or message - reply["reason"] = e.reason + reply["reason"] = e.reason if e.reason else reply["reason"] reply["request_id"] = str(uuid.uuid4()) else: reply["request_id"] = str(uuid.uuid4()) @@ -102,7 +107,11 @@ def write_error(self, status_code, **kwargs): self.finish(json.dumps(reply)) @staticmethod - def get_default_error_messages(service_payload: dict, status_code: str): + def get_default_error_messages( + service_payload: dict, + status_code: str, + default_msg: str = "Unknown HTTP Error.", + ): """Method that maps the error messages based on the operation performed or the status codes encountered.""" messages = { @@ -110,7 +119,6 @@ def get_default_error_messages(service_payload: dict, status_code: str): "403": "We're having trouble processing your request with the information provided.", "404": "Authorization Failed: The resource you're looking for isn't accessible.", "408": "Server is taking too long to response, please try again.", - "500": "An error occurred while creating the resource.", "create": "Authorization Failed: Could not create resource.", "get": "Authorization Failed: The resource you're looking for isn't accessible.", } @@ -128,7 +136,7 @@ def get_default_error_messages(service_payload: dict, status_code: str): if status_code in messages: return messages[status_code] else: - return "Unknown HTTP Error." + return default_msg # todo: remove after error handler is implemented diff --git a/ads/aqua/extension/common_handler.py b/ads/aqua/extension/common_handler.py index 1e6429f79..8cd26ccd9 100644 --- a/ads/aqua/extension/common_handler.py +++ b/ads/aqua/extension/common_handler.py @@ -6,14 +6,16 @@ from importlib import metadata -from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID +from ads.aqua.decorator import handle_exceptions from ads.aqua.exception import AquaResourceAccessError +from ads.aqua.extension.base_handler import AquaAPIhandler class ADSVersionHandler(AquaAPIhandler): """The handler to get the current version of the ADS.""" + @handle_exceptions def get(self): self.finish({"data": metadata.version("oracle_ads")}) @@ -21,6 +23,7 @@ def get(self): class CompatibilityCheckHandler(AquaAPIhandler): """The handler to check if the extension is compatible.""" + @handle_exceptions def get(self): if ODSC_MODEL_COMPARTMENT_OCID: return self.finish(dict(status="ok")) diff --git a/ads/aqua/extension/deployment_handler.py b/ads/aqua/extension/deployment_handler.py index c44828bba..1e74fdcf4 100644 --- a/ads/aqua/extension/deployment_handler.py +++ b/ads/aqua/extension/deployment_handler.py @@ -7,10 +7,10 @@ from tornado.web import HTTPError +from ads.aqua.decorator import handle_exceptions from ads.aqua.deployment import AquaDeploymentApp, MDInferenceResponse, ModelParams from ads.aqua.extension.base_handler import AquaAPIhandler, Errors from ads.config import COMPARTMENT_OCID, PROJECT_OCID -from ads.aqua.decorator import handle_exceptions class AquaDeploymentHandler(AquaAPIhandler): @@ -110,12 +110,10 @@ def post(self, *args, **kwargs): ) ) - @handle_exceptions def read(self, id): """Read the information of an Aqua model deployment.""" return self.finish(AquaDeploymentApp().get(model_deployment_id=id)) - @handle_exceptions def list(self): """List Aqua models.""" # If default is not specified, @@ -129,7 +127,6 @@ def list(self): ) ) - @handle_exceptions def get_deployment_config(self, model_id): """Gets the deployment config for Aqua model.""" return self.finish(AquaDeploymentApp().get_deployment_config(model_id=model_id)) diff --git a/ads/aqua/extension/evaluation_handler.py b/ads/aqua/extension/evaluation_handler.py index 4755bd92a..1555fb827 100644 --- a/ads/aqua/extension/evaluation_handler.py +++ b/ads/aqua/extension/evaluation_handler.py @@ -5,11 +5,10 @@ from urllib.parse import urlparse -from requests import HTTPError +from tornado.web import HTTPError from ads.aqua.decorator import handle_exceptions from ads.aqua.evaluation import AquaEvaluationApp, CreateAquaEvaluationDetails -from ads.aqua.exception import AquaError from ads.aqua.extension.base_handler import AquaAPIhandler, Errors from ads.aqua.extension.utils import validate_function_parameters from ads.config import COMPARTMENT_OCID diff --git a/ads/aqua/extension/finetune_handler.py b/ads/aqua/extension/finetune_handler.py index 928653ed4..1809742ef 100644 --- a/ads/aqua/extension/finetune_handler.py +++ b/ads/aqua/extension/finetune_handler.py @@ -54,7 +54,6 @@ def post(self, *args, **kwargs): self.finish(AquaFineTuningApp().create(CreateFineTuningDetails(**input_data))) - @handle_exceptions def get_finetuning_config(self, model_id): """Gets the finetuning config for Aqua model.""" return self.finish(AquaFineTuningApp().get_finetuning_config(model_id=model_id)) diff --git a/ads/aqua/extension/ui_handler.py b/ads/aqua/extension/ui_handler.py index e880abf34..05cf0d0c7 100644 --- a/ads/aqua/extension/ui_handler.py +++ b/ads/aqua/extension/ui_handler.py @@ -34,6 +34,7 @@ class AquaUIHandler(AquaAPIhandler): HTTPError: For various failure scenarios such as invalid input format, missing data, etc. """ + @handle_exceptions def get(self, id=""): """Handle GET request.""" url_parse = urlparse(self.request.path) @@ -76,7 +77,6 @@ def delete(self, id=""): else: raise HTTPError(400, f"The request {self.request.path} is invalid.") - @handle_exceptions def list_log_groups(self, **kwargs): """Lists all log groups for the specified compartment or tenancy.""" compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID) @@ -84,22 +84,18 @@ def list_log_groups(self, **kwargs): AquaUIApp().list_log_groups(compartment_id=compartment_id, **kwargs) ) - @handle_exceptions def list_logs(self, log_group_id: str, **kwargs): """Lists the specified log group's log objects.""" return self.finish(AquaUIApp().list_logs(log_group_id=log_group_id, **kwargs)) - @handle_exceptions def list_compartments(self): """Lists the compartments in a compartment specified by ODSC_MODEL_COMPARTMENT_OCID env variable.""" return self.finish(AquaUIApp().list_compartments()) - @handle_exceptions def get_default_compartment(self): """Returns user compartment ocid.""" return self.finish(AquaUIApp().get_default_compartment()) - @handle_exceptions def list_model_version_sets(self, **kwargs): """Lists all model version sets for the specified compartment or tenancy.""" @@ -112,7 +108,6 @@ def list_model_version_sets(self, **kwargs): ) ) - @handle_exceptions def list_experiments(self, **kwargs): """Lists all experiments for the specified compartment or tenancy.""" @@ -125,7 +120,6 @@ def list_experiments(self, **kwargs): ) ) - @handle_exceptions def list_buckets(self, **kwargs): """Lists all model version sets for the specified compartment or tenancy.""" compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID) @@ -138,7 +132,6 @@ def list_buckets(self, **kwargs): ) ) - @handle_exceptions def list_job_shapes(self, **kwargs): """Lists job shapes available in the specified compartment.""" compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID) @@ -146,7 +139,6 @@ def list_job_shapes(self, **kwargs): AquaUIApp().list_job_shapes(compartment_id=compartment_id, **kwargs) ) - @handle_exceptions def list_vcn(self, **kwargs): """Lists the virtual cloud networks (VCNs) in the specified compartment.""" compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID) @@ -154,7 +146,6 @@ def list_vcn(self, **kwargs): AquaUIApp().list_vcn(compartment_id=compartment_id, **kwargs) ) - @handle_exceptions def list_subnets(self, **kwargs): """Lists the subnets in the specified VCN and the specified compartment.""" compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID) @@ -165,7 +156,6 @@ def list_subnets(self, **kwargs): ) ) - @handle_exceptions def get_shape_availability(self, **kwargs): """For a given compartmentId, resource limit name, and scope, returns the number of available resources associated with the given limit.""" @@ -178,7 +168,6 @@ def get_shape_availability(self, **kwargs): ) ) - @handle_exceptions def is_bucket_versioned(self): """For a given compartmentId, resource limit name, and scope, returns the number of available resources associated with the given limit.""" diff --git a/ads/aqua/extension/utils.py b/ads/aqua/extension/utils.py index 6ced07e77..5f8320498 100644 --- a/ads/aqua/extension/utils.py +++ b/ads/aqua/extension/utils.py @@ -4,16 +4,16 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ from dataclasses import fields from typing import Dict, Optional -from requests import HTTPError + +from tornado.web import HTTPError from ads.aqua.extension.base_handler import Errors def validate_function_parameters(data_class, input_data: Dict): - """Validates if the required parameters are provided in input data.""" + """Validates if the required parameters are provided in input data.""" required_parameters = [ - field.name for field in fields(data_class) - if field.type != Optional[field.type] + field.name for field in fields(data_class) if field.type != Optional[field.type] ] for required_parameter in required_parameters: diff --git a/tests/unitary/with_extras/aqua/test_decorator.py b/tests/unitary/with_extras/aqua/test_decorator.py new file mode 100644 index 000000000..8617e898c --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_decorator.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-- + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +import json +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from notebook.base.handlers import IPythonHandler +from oci.exceptions import ( + UPLOAD_MANAGER_DEBUG_INFORMATION_LOG, + CompositeOperationError, + ConfigFileNotFound, + ConnectTimeout, + MissingEndpointForNonRegionalServiceClientError, + MultipartUploadError, + RequestException, + ServiceError, +) +from parameterized import parameterized +from tornado.web import HTTPError + +from ads.aqua.exception import AquaError +from ads.aqua.extension.base_handler import AquaAPIhandler + + +class TestDataset: + mock_request_id = "1234" + + +class TestAquaDecorators(TestCase): + """Tests the all aqua common decorators.""" + + @patch.object(IPythonHandler, "__init__") + def setUp(self, ipython_init_mock) -> None: + ipython_init_mock.return_value = None + self.test_instance = AquaAPIhandler(MagicMock(), MagicMock()) + self.test_instance.finish = MagicMock() + self.test_instance.set_header = MagicMock() + self.test_instance.set_status = MagicMock() + + @parameterized.expand( + [ + [ + "oci ServiceError", + ServiceError( + status=500, + code="InternalError", + message="An internal error occurred.", + headers={}, + ), + { + "status": 500, + "message": "An internal error occurred.", + "service_payload": { + "target_service": None, + "status": 500, + "code": "InternalError", + "opc-request-id": None, + "message": "An internal error occurred.", + "operation_name": None, + "timestamp": None, + "client_version": None, + "request_endpoint": None, + "logging_tips": "To get more info on the failing request, refer to https://docs.oracle.com/en-us/iaas/tools/python/latest/logging.html for ways to log the request/response details.", + "troubleshooting_tips": "See https://docs.oracle.com/iaas/Content/API/References/apierrors.htm#apierrors_500__500_internalerror for more information about resolving this error. If you are unable to resolve this None issue, please contact Oracle support and provide them this full error message.", + }, + "reason": "An internal error occurred.", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci ClientError", + ConfigFileNotFound("Could not find config file at the given path."), + { + "status": 400, + "message": "Something went wrong with your request.", + "service_payload": {}, + "reason": "ConfigFileNotFound: Could not find config file at the given path.", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci MissingEndpointForNonRegionalServiceClientError", + MissingEndpointForNonRegionalServiceClientError( + "An endpoint must be provided for a non-regional service client" + ), + { + "status": 400, + "message": "Something went wrong with your request.", + "service_payload": {}, + "reason": "MissingEndpointForNonRegionalServiceClientError: An endpoint must be provided for a non-regional service client", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci RequestException", + RequestException("An exception occurred when making the request"), + { + "status": 400, + "message": "Something went wrong with your request.", + "service_payload": {}, + "reason": "RequestException: An exception occurred when making the request", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci ConnectTimeout", + ConnectTimeout( + "The request timed out while trying to connect to the remote server." + ), + { + "status": 408, + "message": "Server is taking too long to response, please try again.", + "service_payload": {}, + "reason": "ConnectTimeout: The request timed out while trying to connect to the remote server.", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci MultipartUploadError", + MultipartUploadError(), + { + "status": 500, + "message": "Internal Server Error", + "service_payload": {}, + "reason": f"MultipartUploadError: MultipartUploadError exception has occured. {UPLOAD_MANAGER_DEBUG_INFORMATION_LOG}", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "oci CompositeOperationError", + CompositeOperationError(), + { + "status": 500, + "message": "Internal Server Error", + "service_payload": {}, + "reason": "CompositeOperationError: ", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "AquaError", + AquaError(reason="Mocking AQUA error.", status=403, service_payload={}), + { + "status": 403, + "message": "We're having trouble processing your request with the information provided.", + "service_payload": {}, + "reason": "Mocking AQUA error.", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "HTTPError", + HTTPError(400, "The request `/test` is invalid."), + { + "status": 400, + "message": "The request `/test` is invalid.", + "service_payload": {}, + "reason": "The request `/test` is invalid.", + "request_id": TestDataset.mock_request_id, + }, + ], + [ + "ADS Error", + ValueError("Mocking ADS internal error."), + { + "status": 500, + "message": "Internal Server Error", + "service_payload": {}, + "reason": "ValueError: Mocking ADS internal error.", + "request_id": TestDataset.mock_request_id, + }, + ], + ] + ) + @patch("uuid.uuid4") + def test_handle_exceptions(self, name, error, expected_reply, mock_uuid): + """Tests handling error decorator.""" + from ads.aqua.decorator import handle_exceptions + + mock_uuid.return_value = TestDataset.mock_request_id + expected_call = json.dumps(expected_reply) + + @handle_exceptions + def mock_function(self): + raise error + + mock_function(self.test_instance) + + self.test_instance.finish.assert_called_with(expected_call) diff --git a/tests/unitary/with_extras/aqua/test_evaluation_handler.py b/tests/unitary/with_extras/aqua/test_evaluation_handler.py index 5a7fc4ab1..4a9cb3135 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation_handler.py +++ b/tests/unitary/with_extras/aqua/test_evaluation_handler.py @@ -68,9 +68,6 @@ def test_post(self, mock_create): (dict(return_value=None), 400, Errors.NO_INPUT_DATA), ] ) - @unittest.skip( - "Need a fix in `handle_exceptions` decorator before enabling this test." - ) def test_post_fail( self, mock_get_json_body_response, expected_status_code, expected_error_msg ): @@ -87,6 +84,7 @@ def test_post_fail( self.test_instance.write_error.call_args[1].get("status_code") == expected_status_code ), "Raised wrong status code." + assert expected_error_msg in self.test_instance.write_error.call_args[1].get( "reason" ), "Error message is incorrect." diff --git a/tests/unitary/with_extras/aqua/test_handlers.py b/tests/unitary/with_extras/aqua/test_handlers.py index af8e5046d..fa9ca5cdf 100644 --- a/tests/unitary/with_extras/aqua/test_handlers.py +++ b/tests/unitary/with_extras/aqua/test_handlers.py @@ -54,20 +54,22 @@ def setUp(self, mock_init) -> None: @parameterized.expand( [ - (None, None), - ([1, 2, 3], {"data": [1, 2, 3]}), + ("with None", None, None), + ("with list", [1, 2, 3], {"data": [1, 2, 3]}), ( + "with DataClassSerializable", AquaResourceIdentifier(id="123", name="myname"), {"id": "123", "name": "myname", "url": ""}, ), ( + "with dataclass", TestDataset.mock_dataclass_obj, asdict(TestDataset.mock_dataclass_obj), ), ] ) @patch.object(APIHandler, "finish") - def test_finish(self, payload, expected_call, mock_super_finish): + def test_finish(self, name, payload, expected_call, mock_super_finish): """Tests AquaAPIhandler.finish""" mock_super_finish.return_value = None @@ -79,23 +81,25 @@ def test_finish(self, payload, expected_call, mock_super_finish): @parameterized.expand( [ - ( + [ + "HTTPError", dict( status_code=400, exc_info=(None, HTTPError(400, "Bad Request"), None), ), "Bad Request", - ), - ( + ], + [ + "ADS Error", dict( status_code=500, reason="Testing ADS Internal Error.", exc_info=(None, ValueError("Invalid parameter."), None), ), - "An error occurred while creating the resource.", - # This error message doesn't make sense. Need to revisit the code to fix. - ), - ( + "Internal Server Error", + ], + [ + "AQUA Error", dict( status_code=404, reason="Testing AquaError happen during create operation.", @@ -111,8 +115,9 @@ def test_finish(self, payload, expected_call, mock_super_finish): ), ), "Authorization Failed: Could not create resource.", - ), - ( + ], + [ + "oci ServiceError", dict( status_code=404, reason="Testing ServiceError happen when get_job_run.", @@ -127,12 +132,12 @@ def test_finish(self, payload, expected_call, mock_super_finish): ), ), "Authorization Failed: The resource you're looking for isn't accessible. Operation Name: get_job_run.", - ), + ], ] ) @patch("ads.aqua.extension.base_handler.logger") @patch("uuid.uuid4") - def test_write_error(self, input, expected_msg, mock_uuid, mock_logger): + def test_write_error(self, name, input, expected_msg, mock_uuid, mock_logger): """Tests AquaAPIhandler.write_error""" mock_uuid.return_value = "1234" self.test_instance.set_header = MagicMock() @@ -200,6 +205,7 @@ def tearDownClass(cls): @parameterized.expand( [ ( + "AquaEvaluationConfigHandler", AquaEvaluationConfigHandler, AquaEvaluationApp, "load_evaluation_config", @@ -207,6 +213,7 @@ def tearDownClass(cls): TestDataset.MOCK_OCID, ), ( + "AquaEvaluationMetricsHandler", AquaEvaluationMetricsHandler, AquaEvaluationApp, "load_metrics", @@ -214,6 +221,7 @@ def tearDownClass(cls): TestDataset.MOCK_OCID, ), ( + "AquaEvaluationReportHandler", AquaEvaluationReportHandler, AquaEvaluationApp, "download_report", @@ -221,6 +229,7 @@ def tearDownClass(cls): TestDataset.MOCK_OCID, ), ( + "AquaEvaluationStatusHandler", AquaEvaluationStatusHandler, AquaEvaluationApp, "get_status", @@ -228,6 +237,7 @@ def tearDownClass(cls): TestDataset.MOCK_OCID, ), ( + "ADSVersionHandler", ADSVersionHandler, None, None, @@ -235,6 +245,7 @@ def tearDownClass(cls): {"data": metadata.version("oracle_ads")}, ), ( + "CompatibilityCheckHandler", CompatibilityCheckHandler, None, None, @@ -242,6 +253,7 @@ def tearDownClass(cls): dict(status="ok"), ), ( + "AquaModelLicenseHandler", AquaModelLicenseHandler, AquaModelApp, "load_license", @@ -249,18 +261,27 @@ def tearDownClass(cls): TestDataset.MOCK_OCID, ), ( + "AquaModelHandler", AquaModelHandler, AquaModelApp, "get", TestDataset.MOCK_OCID, TestDataset.MOCK_OCID, ), - (AquaModelHandler, AquaModelApp, "list", "", (None, None)), + ( + "AquaModelHandler_list", + AquaModelHandler, + AquaModelApp, + "list", + "", + (None, None), + ), ] ) @patch.object(IPythonHandler, "__init__") def test_get( self, + name, target_handler, target_app, associated_api, From 2915053d304fe195a870f9dbb2cf20e1a3d49c61 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Thu, 18 Apr 2024 12:41:44 -0400 Subject: [PATCH 10/14] Added support to force overwrite extsing file in object storage. --- ads/aqua/evaluation.py | 7 +++++-- ads/aqua/finetune.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ads/aqua/evaluation.py b/ads/aqua/evaluation.py index bbf8be86c..c9ad08800 100644 --- a/ads/aqua/evaluation.py +++ b/ads/aqua/evaluation.py @@ -311,6 +311,8 @@ class CreateAquaEvaluationDetails(DataClassSerializable): The log id for the evaluation job infrastructure. metrics: (list, optional). Defaults to `None`. The metrics for the evaluation. + force_overwrite: (bool, optional). Defaults to `False`. + Whether to force overwrite the existing file in object storage. """ evaluation_source_id: str @@ -331,6 +333,7 @@ class CreateAquaEvaluationDetails(DataClassSerializable): log_group_id: Optional[str] = None log_id: Optional[str] = None metrics: Optional[List] = None + force_overwrite: Optional[bool] = False class AquaEvaluationApp(AquaApp): @@ -434,12 +437,12 @@ def create( src_uri=evaluation_dataset_path, dst_uri=dst_uri, auth=default_signer(), - force_overwrite=False, + force_overwrite=create_aqua_evaluation_details.force_overwrite, ) except FileExistsError: raise AquaFileExistsError( f"Dataset {dataset_file} already exists in {create_aqua_evaluation_details.report_path}. " - "Please use a new dataset file name or report path." + "Please use a new dataset file name, report path or set `force_overwrite` as True." ) logger.debug( f"Uploaded local file {evaluation_dataset_path} to object storage {dst_uri}." diff --git a/ads/aqua/finetune.py b/ads/aqua/finetune.py index 35deab53b..bbc0b516c 100644 --- a/ads/aqua/finetune.py +++ b/ads/aqua/finetune.py @@ -122,6 +122,8 @@ class CreateFineTuningDetails(DataClassSerializable): The log group id for fine tuning job infrastructure. log_id: (str, optional). Defaults to `None`. The log id for fine tuning job infrastructure. + force_overwrite: (bool, optional). Defaults to `False`. + Whether to force overwrite the existing file in object storage. """ ft_source_id: str @@ -142,6 +144,7 @@ class CreateFineTuningDetails(DataClassSerializable): subnet_id: Optional[str] = None log_id: Optional[str] = None log_group_id: Optional[str] = None + force_overwrite: Optional[bool] = False class AquaFineTuningApp(AquaApp): @@ -273,12 +276,12 @@ def create( src_uri=ft_dataset_path, dst_uri=dst_uri, auth=default_signer(), - force_overwrite=False, + force_overwrite=create_fine_tuning_details.force_overwrite, ) except FileExistsError: raise AquaFileExistsError( f"Dataset {dataset_file} already exists in {create_fine_tuning_details.report_path}. " - "Please use a new dataset file name or report path." + "Please use a new dataset file name, report path or set `force_overwrite` as True." ) logger.debug( f"Uploaded local file {ft_dataset_path} to object storage {dst_uri}." From 5ffb5de3391a10d2fe0f101086ad02a73a792d3b Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 18 Apr 2024 11:37:11 -0700 Subject: [PATCH 11/14] update error message --- ads/aqua/extension/base_handler.py | 2 +- tests/unitary/with_extras/aqua/test_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ads/aqua/extension/base_handler.py b/ads/aqua/extension/base_handler.py index 73c676cdc..b92e2deab 100644 --- a/ads/aqua/extension/base_handler.py +++ b/ads/aqua/extension/base_handler.py @@ -127,7 +127,7 @@ def get_default_error_messages( operation_name = service_payload["operation_name"] if operation_name: if operation_name.startswith("create"): - return messages["create"] + return messages["create"] + f" Operation Name: {operation_name}." elif operation_name.startswith("list") or operation_name.startswith( "get" ): diff --git a/tests/unitary/with_extras/aqua/test_handlers.py b/tests/unitary/with_extras/aqua/test_handlers.py index fa9ca5cdf..e74b99d4f 100644 --- a/tests/unitary/with_extras/aqua/test_handlers.py +++ b/tests/unitary/with_extras/aqua/test_handlers.py @@ -114,7 +114,7 @@ def test_finish(self, name, payload, expected_call, mock_super_finish): None, ), ), - "Authorization Failed: Could not create resource.", + "Authorization Failed: Could not create resource. Operation Name: create_resources.", ], [ "oci ServiceError", From 88bb9c6dba37ad064ff5e845393a952a22626f08 Mon Sep 17 00:00:00 2001 From: Vipul Date: Thu, 18 Apr 2024 14:17:52 -0700 Subject: [PATCH 12/14] review comments --- ads/aqua/extension/common_handler.py | 13 +++++++++++++ ads/aqua/utils.py | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ads/aqua/extension/common_handler.py b/ads/aqua/extension/common_handler.py index 2ea3e1545..676481756 100644 --- a/ads/aqua/extension/common_handler.py +++ b/ads/aqua/extension/common_handler.py @@ -26,6 +26,19 @@ class CompatibilityCheckHandler(AquaAPIhandler): @handle_exceptions def get(self): + """This method provides the availability status of Aqua. If ODSC_MODEL_COMPARTMENT_OCID environment variable + is set, then status `ok` is returned. For regions where Aqua is available but the environment variable is not + set due to accesses/policies, we return the `compatible` status to indicate that the extension can be enabled + for the selected notebook session. + + Returns + ------- + status dict: + ok or compatible + Raises: + AquaResourceAccessError: raised when aqua is not accessible in the given session/region. + + """ if ODSC_MODEL_COMPARTMENT_OCID: return self.finish(dict(status="ok")) elif known_realm(): diff --git a/ads/aqua/utils.py b/ads/aqua/utils.py index ab72723dc..d3369ba33 100644 --- a/ads/aqua/utils.py +++ b/ads/aqua/utils.py @@ -744,4 +744,4 @@ def known_realm(): Return True if aqua service is available. """ - return os.environ.get("CONDA_BUCKET_NS", "#") in AQUA_GA_LIST + return os.environ.get("CONDA_BUCKET_NS") in AQUA_GA_LIST From a4e36a954a589ec01c8f21fef4e729e9dfe0a445 Mon Sep 17 00:00:00 2001 From: MING KANG Date: Fri, 19 Apr 2024 13:41:45 -0400 Subject: [PATCH 13/14] Add test cases for checking content decoding (#791) --- .../aqua/test_data/valid_eval_artifact/report.html | 1 + tests/unitary/with_extras/aqua/test_evaluation.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unitary/with_extras/aqua/test_data/valid_eval_artifact/report.html b/tests/unitary/with_extras/aqua/test_data/valid_eval_artifact/report.html index 12a593f1f..00ca412fe 100644 --- a/tests/unitary/with_extras/aqua/test_data/valid_eval_artifact/report.html +++ b/tests/unitary/with_extras/aqua/test_data/valid_eval_artifact/report.html @@ -1 +1,2 @@ This is a sample evaluation report.html. +Standard deviation (σ) diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index f12ebbab7..d3cad7063 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -624,9 +624,10 @@ def test_download_report( mock_dsc_model_from_id.assert_called_with(TestDataset.EVAL_ID) self.print_expected_response(response, "DOWNLOAD REPORT") self.assert_payload(response, AquaEvalReport) - read_content = base64.b64decode(response.content) + read_content = base64.b64decode(response.content).decode() assert ( - read_content == b"This is a sample evaluation report.html.\n" + read_content + == "This is a sample evaluation report.html.\nStandard deviation (σ)\n" ), read_content assert self.app._report_cache.currsize == 1 From ff217d85d673bea87173fbb68dc78b4ef860e042 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 19 Apr 2024 13:51:47 -0400 Subject: [PATCH 14/14] Updated comment. --- ads/aqua/finetune.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ads/aqua/finetune.py b/ads/aqua/finetune.py index 35deab53b..6222cd621 100644 --- a/ads/aqua/finetune.py +++ b/ads/aqua/finetune.py @@ -466,6 +466,7 @@ def create( **telemetry_kwargs, ) # tracks unique fine-tuned models that were created in the user compartment + # TODO: retrieve the service model name for FT custom models. self.telemetry.record_event_async( category="aqua/service/finetune", action="create",