From 2174b3633c90f6f23401fd37680f1a4fd27636e2 Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 12:41:31 +0530 Subject: [PATCH 01/19] HF look ahead search --- Makefile | 4 ++++ ads/aqua/common/utils.py | 11 +++++++++++ ads/aqua/extension/model_handler.py | 25 ++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 61c041089..3d1d5c304 100644 --- a/Makefile +++ b/Makefile @@ -11,3 +11,7 @@ clean: @find ./ -name 'Thumbs.db' -exec rm -f {} \; @find ./ -name '*~' -exec rm -f {} \; @find ./ -name '.DS_Store' -exec rm -f {} \; +test: + pip install -e . + jupyter server extension enable --py ads.aqua.extension + jupyter lab --NotebookApp.disable_check_xsrf=True --no-browser \ No newline at end of file diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index daf71973c..0e2b66865 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -57,6 +57,7 @@ VLLM_INFERENCE_RESTRICTED_PARAMS, ) from ads.aqua.data import AquaResourceIdentifier +from ads.aqua.model.constants import ModelTask from ads.common.auth import AuthState, default_signer from ads.common.extended_enum import ExtendedEnumMeta from ads.common.object_storage_details import ObjectStorageDetails @@ -64,6 +65,7 @@ from ads.common.utils import copy_file, get_console_link, upload_to_os from ads.config import AQUA_SERVICE_MODELS_BUCKET, CONDA_BUCKET_NS, TENANCY_OCID from ads.model import DataScienceModel, ModelVersionSet +from tests.unitary.with_extras.model.score import model_name logger = logging.getLogger("ads.aqua") @@ -1062,3 +1064,12 @@ def get_hf_model_info(repo_id: str) -> ModelInfo: return HfApi().model_info(repo_id=repo_id) except HfHubHTTPError as err: raise format_hf_custom_error_message(err) from err + +def list_hf_models(query:str) -> List[str]: + try: + models= HfApi().list_models(model_name=query,task=ModelTask.TEXT_GENERATION) + return [model.id for model in models if model.disabled is None] + except HfHubHTTPError as err: + raise format_hf_custom_error_message(err) from err + + diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 01af714c2..8d56e8b85 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -9,7 +9,7 @@ from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.errors import AquaRuntimeError, AquaValueError -from ads.aqua.common.utils import get_hf_model_info +from ads.aqua.common.utils import get_hf_model_info, list_hf_models from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors from ads.aqua.model import AquaModelApp @@ -177,6 +177,29 @@ def _find_matching_aqua_model(model_id: str) -> Optional[AquaModelSummary]: return None + @handle_exceptions + def get(self): + """ + Finds a list of matching models from hugging face based on query string provided from users. + + Parameters + ---------- + query (str): The Hugging Face model name to search for. + + Returns + ------- + List[AquaModelSummary] + Returns the matching AquaModelSummary object if found, else None. + """ + + query=self.get_argument("query",default=None) + if not query: + raise HTTPError(400,Errors.MISSING_REQUIRED_PARAMETER.format("query")) + models=list_hf_models(query) + return self.finish({"models":models}) + + + @handle_exceptions def post(self, *args, **kwargs): """Handles post request for the HF Models APIs From 52077ceb57723f2bc275c2930aff74ff7de28c44 Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 13:52:31 +0530 Subject: [PATCH 02/19] Reverting changes --- Makefile | 6 +----- ads/aqua/common/utils.py | 8 -------- ads/aqua/extension/model_handler.py | 23 ----------------------- 3 files changed, 1 insertion(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 3d1d5c304..37a0b426f 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,4 @@ clean: @find ./ -name '*.pyc' -exec rm -f {} \; @find ./ -name 'Thumbs.db' -exec rm -f {} \; @find ./ -name '*~' -exec rm -f {} \; - @find ./ -name '.DS_Store' -exec rm -f {} \; -test: - pip install -e . - jupyter server extension enable --py ads.aqua.extension - jupyter lab --NotebookApp.disable_check_xsrf=True --no-browser \ No newline at end of file + @find ./ -name '.DS_Store' -exec rm -f {} \; \ No newline at end of file diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index 0e2b66865..0831c18e7 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -57,7 +57,6 @@ VLLM_INFERENCE_RESTRICTED_PARAMS, ) from ads.aqua.data import AquaResourceIdentifier -from ads.aqua.model.constants import ModelTask from ads.common.auth import AuthState, default_signer from ads.common.extended_enum import ExtendedEnumMeta from ads.common.object_storage_details import ObjectStorageDetails @@ -65,7 +64,6 @@ from ads.common.utils import copy_file, get_console_link, upload_to_os from ads.config import AQUA_SERVICE_MODELS_BUCKET, CONDA_BUCKET_NS, TENANCY_OCID from ads.model import DataScienceModel, ModelVersionSet -from tests.unitary.with_extras.model.score import model_name logger = logging.getLogger("ads.aqua") @@ -1065,11 +1063,5 @@ def get_hf_model_info(repo_id: str) -> ModelInfo: except HfHubHTTPError as err: raise format_hf_custom_error_message(err) from err -def list_hf_models(query:str) -> List[str]: - try: - models= HfApi().list_models(model_name=query,task=ModelTask.TEXT_GENERATION) - return [model.id for model in models if model.disabled is None] - except HfHubHTTPError as err: - raise format_hf_custom_error_message(err) from err diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 8d56e8b85..1c2e68cbc 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -177,29 +177,6 @@ def _find_matching_aqua_model(model_id: str) -> Optional[AquaModelSummary]: return None - @handle_exceptions - def get(self): - """ - Finds a list of matching models from hugging face based on query string provided from users. - - Parameters - ---------- - query (str): The Hugging Face model name to search for. - - Returns - ------- - List[AquaModelSummary] - Returns the matching AquaModelSummary object if found, else None. - """ - - query=self.get_argument("query",default=None) - if not query: - raise HTTPError(400,Errors.MISSING_REQUIRED_PARAMETER.format("query")) - models=list_hf_models(query) - return self.finish({"models":models}) - - - @handle_exceptions def post(self, *args, **kwargs): """Handles post request for the HF Models APIs From 833b26906d67bb90752dc4186f5a7d8a35e3c69f Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 13:53:19 +0530 Subject: [PATCH 03/19] Revert --- ads/aqua/extension/model_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 1c2e68cbc..01af714c2 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -9,7 +9,7 @@ from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.errors import AquaRuntimeError, AquaValueError -from ads.aqua.common.utils import get_hf_model_info, list_hf_models +from ads.aqua.common.utils import get_hf_model_info from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors from ads.aqua.model import AquaModelApp From 83ce1dfe733d4526d600ffad250c2fb9216f2aa5 Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 14:06:29 +0530 Subject: [PATCH 04/19] HF look ahead search --- ads/aqua/common/utils.py | 8 ++++++++ ads/aqua/extension/model_handler.py | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index 0831c18e7..b3361b312 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -1064,4 +1064,12 @@ def get_hf_model_info(repo_id: str) -> ModelInfo: raise format_hf_custom_error_message(err) from err +@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) +def list_hf_models(query:str) -> List[str]: + try: + models= HfApi().list_models(model_name=query,task="text-generation") + return [model.id for model in models if model.disabled is None] + except HfHubHTTPError as err: + raise format_hf_custom_error_message(err) from err + diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 01af714c2..5887150c6 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -9,7 +9,7 @@ from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.errors import AquaRuntimeError, AquaValueError -from ads.aqua.common.utils import get_hf_model_info +from ads.aqua.common.utils import get_hf_model_info, list_hf_models from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors from ads.aqua.model import AquaModelApp @@ -177,6 +177,31 @@ def _find_matching_aqua_model(model_id: str) -> Optional[AquaModelSummary]: return None + + + @handle_exceptions + def get(self,*args, **kwargs): + """ + Finds a list of matching models from hugging face based on query string provided from users. + + Parameters + ---------- + query (str): The Hugging Face model name to search for. + + Returns + ------- + List[AquaModelSummary] + Returns the matching AquaModelSummary object if found, else None. + """ + + query=self.get_argument("query",default=None) + if not query: + raise HTTPError(400,Errors.MISSING_REQUIRED_PARAMETER.format("query")) + models=list_hf_models(query) + return self.finish({"models":models}) + + + @handle_exceptions def post(self, *args, **kwargs): """Handles post request for the HF Models APIs From 611796f4910bcc99f25f9d9b5af4785dbfed666d Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 14:17:41 +0530 Subject: [PATCH 05/19] updating filters --- ads/aqua/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index b3361b312..93d583ba7 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -1067,7 +1067,7 @@ def get_hf_model_info(repo_id: str) -> ModelInfo: @cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) def list_hf_models(query:str) -> List[str]: try: - models= HfApi().list_models(model_name=query,task="text-generation") + models= HfApi().list_models(model_name=query,task="text-generation",sort="downloads",direction=-1,limit=20) return [model.id for model in models if model.disabled is None] except HfHubHTTPError as err: raise format_hf_custom_error_message(err) from err From 5d5026a94bd66874d82d2d4acbb42749219c5f36 Mon Sep 17 00:00:00 2001 From: kumar-shivam-ranjan Date: Fri, 30 Aug 2024 19:23:47 +0530 Subject: [PATCH 06/19] updating doc string --- Makefile | 2 +- ads/aqua/extension/model_handler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 37a0b426f..61c041089 100644 --- a/Makefile +++ b/Makefile @@ -10,4 +10,4 @@ clean: @find ./ -name '*.pyc' -exec rm -f {} \; @find ./ -name 'Thumbs.db' -exec rm -f {} \; @find ./ -name '*~' -exec rm -f {} \; - @find ./ -name '.DS_Store' -exec rm -f {} \; \ No newline at end of file + @find ./ -name '.DS_Store' -exec rm -f {} \; diff --git a/ads/aqua/extension/model_handler.py b/ads/aqua/extension/model_handler.py index 5887150c6..258396c3e 100644 --- a/ads/aqua/extension/model_handler.py +++ b/ads/aqua/extension/model_handler.py @@ -190,8 +190,8 @@ def get(self,*args, **kwargs): Returns ------- - List[AquaModelSummary] - Returns the matching AquaModelSummary object if found, else None. + List[str] + Returns the matching model ids string """ query=self.get_argument("query",default=None) From 56abb0646cea816a945155e8abf7a5b02968e59d Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Sun, 1 Sep 2024 22:45:38 -0700 Subject: [PATCH 07/19] Added the EvaluationServiceConfig class along with supporting structures to manage evaluation service configurations. --- .gitignore | 3 + ads/aqua/common/utils.py | 6 +- ads/aqua/config/__init__.py | 4 + ads/aqua/config/config.py | 24 ++ ads/aqua/config/evaluation/__init__.py | 4 + .../evaluation/evaluation_service_config.py | 317 ++++++++++++++++ .../evaluation_service_model_config.py | 8 + ads/aqua/config/utils/__init__.py | 4 + ads/aqua/config/utils/serializer.py | 339 ++++++++++++++++++ ads/aqua/constants.py | 3 +- pyproject.toml | 1 + tests/unitary/with_extras/aqua/test_config.py | 29 ++ .../test_data/config/evaluation_config.json | 333 +++++++++++++++++ ...evaluation_config_with_default_params.json | 24 ++ .../aqua/test_evaluation_service_config.py | 149 ++++++++ 15 files changed, 1244 insertions(+), 4 deletions(-) create mode 100644 ads/aqua/config/__init__.py create mode 100644 ads/aqua/config/evaluation/__init__.py create mode 100644 ads/aqua/config/evaluation/evaluation_service_config.py create mode 100644 ads/aqua/config/evaluation/evaluation_service_model_config.py create mode 100644 ads/aqua/config/utils/__init__.py create mode 100644 ads/aqua/config/utils/serializer.py create mode 100644 tests/unitary/with_extras/aqua/test_config.py create mode 100644 tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json create mode 100644 tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json create mode 100644 tests/unitary/with_extras/aqua/test_evaluation_service_config.py diff --git a/.gitignore b/.gitignore index ac478a955..8abb0d36e 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ logs/ # Python Wheel *.whl + +# The demo folder +.demo diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index daf71973c..fadf53b04 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -536,14 +536,14 @@ def _build_job_identifier( return AquaResourceIdentifier() -def container_config_path(): +def service_config_path(): return f"oci://{AQUA_SERVICE_MODELS_BUCKET}@{CONDA_BUCKET_NS}/service_models/config" @cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) def get_container_config(): config = load_config( - file_path=container_config_path(), + file_path=service_config_path(), config_file_name=CONTAINER_INDEX, ) @@ -568,7 +568,7 @@ def get_container_image( """ config = config_file_name or get_container_config() - config_file_name = container_config_path() + config_file_name = service_config_path() if container_type not in config: raise AquaValueError( diff --git a/ads/aqua/config/__init__.py b/ads/aqua/config/__init__.py new file mode 100644 index 000000000..1427168e1 --- /dev/null +++ b/ads/aqua/config/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index 5ece62c50..1eb3f3104 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -3,6 +3,30 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +from datetime import datetime, timedelta + +from cachetools import TTLCache, cached + +from ads.aqua.common.utils import service_config_path +from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig +from ads.aqua.constants import EVALUATION_SERVICE_CONFIG + + +@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) +def evaluation_service_config() -> EvaluationServiceConfig: + """ + Retrieves the common evaluation configuration. + + Returns + ------- + EvaluationServiceConfig: The evaluation common config. + """ + + return EvaluationServiceConfig.from_json( + uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}" + ) + + # TODO: move this to global config.json in object storage def get_finetuning_config_defaults(): """Generate and return the fine-tuning default configuration dictionary.""" diff --git a/ads/aqua/config/evaluation/__init__.py b/ads/aqua/config/evaluation/__init__.py new file mode 100644 index 000000000..1427168e1 --- /dev/null +++ b/ads/aqua/config/evaluation/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py new file mode 100644 index 000000000..6626abef9 --- /dev/null +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Union + +from pydantic import Field + +from ads.aqua.config.utils.serializer import Serializable + +# Constants +INFERENCE_RPS = 25 # Max RPS for inferencing deployed model. +INFERENCE_TIMEOUT = 120 +INFERENCE_MAX_THREADS = 10 # Maximum parallel threads for model inference. +INFERENCE_RETRIES = 3 +INFERENCE_BACKOFF_FACTOR = 3 +INFERENCE_DELAY = 0 + + +class ModelParamItem(Serializable): + """Represents min, max, and default values for a model parameter.""" + + min: Optional[Union[int, float]] = None + max: Optional[Union[int, float]] = None + default: Optional[Union[int, float]] = None + + class Config: + extra = "ignore" + + +class ModelParamsOverrides(Serializable): + """Defines overrides for model parameters, including exclusions and additional inclusions.""" + + exclude: Optional[List[str]] = Field(default_factory=list) + include: Optional[Dict[str, Any]] = Field(default_factory=dict) + + class Config: + extra = "ignore" + + +class ModelParamsVersion(Serializable): + """Handles version-specific model parameter overrides.""" + + overrides: Optional[ModelParamsOverrides] = Field( + default_factory=ModelParamsOverrides + ) + + class Config: + extra = "ignore" + + +class ModelDefaultParams(Serializable): + """Defines default parameters for a model within a specific framework.""" + + model: Optional[str] = None + max_tokens: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + temperature: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + top_p: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + top_k: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + presence_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + frequency_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) + stop: List[str] = Field(default_factory=list) + + class Config: + extra = "allow" + + +class ModelFramework(Serializable): + """Represents a framework's model configuration, including tasks, defaults, and versions.""" + + framework: Optional[str] = None + task: Optional[List[str]] = Field(default_factory=list) + default: Optional[ModelDefaultParams] = Field(default_factory=ModelDefaultParams) + versions: Optional[Dict[str, ModelParamsVersion]] = Field(default_factory=dict) + + class Config: + extra = "ignore" + + +class InferenceParams(Serializable): + """Contains inference-related parameters with defaults.""" + + inference_rps: Optional[int] = INFERENCE_RPS + inference_timeout: Optional[int] = INFERENCE_TIMEOUT + inference_max_threads: Optional[int] = INFERENCE_MAX_THREADS + inference_retries: Optional[int] = INFERENCE_RETRIES + inference_backoff_factor: Optional[float] = INFERENCE_BACKOFF_FACTOR + inference_delay: Optional[float] = INFERENCE_DELAY + + class Config: + extra = "allow" + + +class InferenceFramework(Serializable): + """Represents the inference parameters specific to a framework.""" + + framework: Optional[str] = None + params: Optional[Dict[str, Any]] = Field(default_factory=dict) + + class Config: + extra = "ignore" + + +class ReportParams(Serializable): + """Handles the report-related parameters.""" + + default: Optional[Dict[str, Any]] = Field(default_factory=dict) + + class Config: + extra = "ignore" + + +class InferenceParamsConfig(Serializable): + """Combines default inference parameters with framework-specific configurations.""" + + default: Optional[InferenceParams] = Field(default_factory=InferenceParams) + frameworks: Optional[List[InferenceFramework]] = Field(default_factory=list) + + def get_merged_params(self, framework_name: str) -> InferenceParams: + """ + Merges default inference params with those specific to the given framework. + + Parameters + ---------- + framework_name (str): The name of the framework. + + Returns + ------- + InferenceParams: The merged inference parameters. + """ + merged_params = self.default.to_dict() + for framework in self.frameworks: + if framework.framework.lower() == framework_name.lower(): + merged_params.update(framework.params or {}) + break + return InferenceParams(**merged_params) + + class Config: + extra = "ignore" + + +class ModelParamsConfig(Serializable): + """Encapsulates the model parameters for different frameworks.""" + + default: Optional[Dict[str, Any]] = Field(default_factory=dict) + frameworks: Optional[List[ModelFramework]] = Field(default_factory=list) + + def get_model_params( + self, + framework_name: str, + version: Optional[str] = None, + task: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Gets the model parameters for a given framework, version, and tasks, + merged with the defaults. + + Parameters + ---------- + framework_name (str): The name of the framework. + version (Optional[str]): The specific version of the framework. + task (Optional[str]): The specific task. + + Returns + ------- + Dict[str, Any]: The merged model parameters. + """ + params = deepcopy(self.default) + + for framework in self.frameworks: + if framework.framework.lower() == framework_name.lower() and ( + not task or task.lower() in framework.task + ): + params.update(framework.default.to_dict()) + + if version and version in framework.versions: + version_overrides = framework.versions[version].overrides + if version_overrides: + if version_overrides.include: + params.update(version_overrides.include) + if version_overrides.exclude: + for key in version_overrides.exclude: + params.pop(key, None) + break + + return params + + class Config: + extra = "ignore" + + +class ShapeFilterConfig(Serializable): + """Represents the filtering options for a specific shape.""" + + evaluation_container: Optional[List[str]] = Field(default_factory=list) + evaluation_target: Optional[List[str]] = Field(default_factory=list) + + class Config: + extra = "ignore" + + +class ShapeConfig(Serializable): + """Defines the configuration for a specific shape.""" + + name: Optional[str] = None + ocpu: Optional[int] = None + memory_in_gbs: Optional[int] = None + block_storage_size: Optional[int] = None + filter: Optional[ShapeFilterConfig] = Field(default_factory=ShapeFilterConfig) + + class Config: + extra = "allow" + + +class MetricConfig(Serializable): + """Handles metric configuration including task, key, and additional details.""" + + task: Optional[List[str]] = Field(default_factory=list) + key: Optional[str] = None + name: Optional[str] = None + description: Optional[str] = None + args: Optional[Dict[str, Any]] = Field(default_factory=dict) + tags: Optional[List[str]] = Field(default_factory=list) + + class Config: + extra = "ignore" + + +class EvaluationServiceConfig(Serializable): + """ + Root configuration class for evaluation setup including model, + inference, and shape configurations. + """ + + version: Optional[str] = "1.0" + kind: Optional[str] = "evaluation" + report_params: Optional[ReportParams] = Field(default_factory=ReportParams) + inference_params: Optional[InferenceParamsConfig] = Field( + default_factory=InferenceParamsConfig + ) + model_params: Optional[ModelParamsConfig] = Field(default_factory=ModelParamsConfig) + shapes: List[ShapeConfig] = Field(default_factory=list) + metrics: List[MetricConfig] = Field(default_factory=list) + + def get_merged_inference_params(self, framework_name: str) -> InferenceParams: + """ + Merges default inference params with those specific to the given framework. + + Params + ------ + framework_name (str): The name of the framework. + + Returns + ------- + InferenceParams: The merged inference parameters. + """ + return self.inference_params.get_merged_params(framework_name=framework_name) + + def get_merged_model_params( + self, + framework_name: str, + version: Optional[str] = None, + task: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Gets the model parameters for a given framework, version, and task, merged with the defaults. + + Parameters + ---------- + framework_name (str): The name of the framework. + version (Optional[str]): The specific version of the framework. + task (Optional[str]): The task. + + Returns + ------- + Dict[str, Any]: The merged model parameters. + """ + return self.model_params.get_model_params( + framework_name=framework_name, version=version, task=task + ) + + def search_shapes( + self, + evaluation_container: Optional[str] = None, + evaluation_target: Optional[str] = None, + ) -> List[ShapeConfig]: + """ + Searches for shapes that match the given filters. + + Parameters + ---------- + evaluation_container (Optional[str]): Filter for evaluation_container. + evaluation_target (Optional[str]): Filter for evaluation_target. + + Returns + ------- + List[ShapeConfig]: A list of shapes that match the filters. + """ + results = [] + for shape in self.shapes: + if ( + evaluation_container + and evaluation_container not in shape.filter.evaluation_container + ): + continue + if ( + evaluation_target + and evaluation_target not in shape.filter.evaluation_target + ): + continue + results.append(shape) + return results + + class Config: + extra = "ignore" diff --git a/ads/aqua/config/evaluation/evaluation_service_model_config.py b/ads/aqua/config/evaluation/evaluation_service_model_config.py new file mode 100644 index 000000000..911fe3176 --- /dev/null +++ b/ads/aqua/config/evaluation/evaluation_service_model_config.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +""" +This serves as a future template for implementing model-specific evaluation configurations. +""" diff --git a/ads/aqua/config/utils/__init__.py b/ads/aqua/config/utils/__init__.py new file mode 100644 index 000000000..1427168e1 --- /dev/null +++ b/ads/aqua/config/utils/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ diff --git a/ads/aqua/config/utils/serializer.py b/ads/aqua/config/utils/serializer.py new file mode 100644 index 000000000..a0558ec8f --- /dev/null +++ b/ads/aqua/config/utils/serializer.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python + +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +import json +from typing import Union +from urllib.parse import urlparse + +import fsspec +import yaml +from pydantic import BaseModel +from yaml import SafeLoader as Loader + +from ads.common.auth import default_signer + + +class Serializable(BaseModel): + """Base class that represents a serializable item. + + Methods + ------- + to_json(self, uri=None, **kwargs) + Returns object serialized as a JSON string + from_json(cls, json_string=None, uri=None, **kwargs) + Creates an object from JSON string provided or from URI location containing JSON string + to_yaml(self, uri=None, **kwargs) + Returns object serialized as a YAML string + from_yaml(cls, yaml_string=None, uri=None, **kwargs) + Creates an object from YAML string provided or from URI location containing YAML string + """ + + @staticmethod + def _write_to_file(s: str, uri: str, **kwargs) -> None: + """Write string s into location specified by uri. + + Parameters + ---------- + s: (string) + content + uri: (string) + URI location to save string s + kwargs : dict + keyword arguments to be passed into fsspec.open(). + For OCI object storage, this can be config="path/to/.oci/config". + + Returns + ------- + None + Nothing + """ + + overwrite = kwargs.pop("overwrite", True) + if not overwrite: + dst_path_scheme = urlparse(uri).scheme or "file" + if fsspec.filesystem(dst_path_scheme, **kwargs).exists(uri): + raise FileExistsError( + f"The `{uri}` is already exists. Set `overwrite` to True " + "if you wish to overwrite." + ) + + with fsspec.open(uri, "w", **kwargs) as f: + f.write(s) + + @staticmethod + def _read_from_file(uri: str, **kwargs) -> str: + """Returns contents from location specified by URI + + Parameters + ---------- + uri: (string) + URI location + kwargs : dict + keyword arguments to be passed into fsspec.open(). + For OCI object storage, this can be config="path/to/.oci/config". + + Returns + ------- + string: Contents in file specified by URI + """ + # Add default signer if the uri is an object storage uri, and + # the user does not specify config or signer. + if ( + uri.startswith("oci://") + and "config" not in kwargs + and "signer" not in kwargs + ): + kwargs.update(default_signer()) + with fsspec.open(uri, "r", **kwargs) as f: + return f.read() + + def to_json( + self, + uri: str = None, + encoder: callable = json.JSONEncoder, + default: callable = None, + **kwargs, + ) -> str: + """Returns object serialized as a JSON string + + Parameters + ---------- + uri: (string, optional) + URI location to save the JSON string. Defaults to None. + encoder: (callable, optional) + Encoder for custom data structures. Defaults to JSONEncoder. + default: (callable, optional) + A function that gets called for objects that can't otherwise be serialized. + It should return JSON-serializable version of the object or original object. + + kwargs + ------ + overwrite: (bool, optional). Defaults to True. + Whether to overwrite existing file or not. + + keyword arguments to be passed into fsspec.open(). + For OCI object storage, this could be config="path/to/.oci/config". + For other storage connections consider e.g. host, port, username, password, etc. + + Returns + ------- + Union[str, None] + Serialized version of object. + `None` in case when `uri` provided. + """ + json_string = json.dumps( + self.model_dump(exclude_none=kwargs.pop("exclude_none", False)), + cls=encoder, + default=default, + ) + if uri: + self._write_to_file(s=json_string, uri=uri, **kwargs) + return None + return json_string + + def to_dict(self) -> dict: + """Returns object serialized as a dictionary + + Returns + ------- + dict + Serialized version of object + """ + return json.loads(self.to_json()) + + @classmethod + def from_json( + cls, + json_string: str = None, + uri: str = None, + decoder: callable = json.JSONDecoder, + **kwargs, + ): + """Creates an object from JSON string provided or from URI location containing JSON string + + Parameters + ---------- + json_string: (string, optional) + JSON string. Defaults to None. + uri: (string, optional) + URI location of file containing JSON string. Defaults to None. + decoder: (callable, optional) + Custom decoder. Defaults to simple JSON decoder. + kwargs + ------ + keyword arguments to be passed into fsspec.open(). For OCI object storage, this should be config="path/to/.oci/config". + For other storage connections consider e.g. host, port, username, password, etc. + + Raises + ------ + ValueError + Raised if neither string nor uri is provided + + Returns + ------- + cls + Returns instance of the class + """ + if json_string: + return cls(**json.loads(json_string, cls=decoder)) + if uri: + return cls(**json.loads(cls._read_from_file(uri, **kwargs), cls=decoder)) + raise ValueError("Must provide either JSON string or URI location") + + def to_yaml( + self, uri: str = None, dumper: callable = yaml.SafeDumper, **kwargs + ) -> Union[str, None]: + """Returns object serialized as a YAML string + + Parameters + ---------- + uri : str, optional + URI location to save the YAML string, by default None + dumper : callable, optional + Custom YAML Dumper, by default yaml.SafeDumper + kwargs : dict + overwrite: (bool, optional). Defaults to True. + Whether to overwrite existing file or not. + note: (str, optional) + The note that needs to be added in the beginning of the YAML. + It will be added as is without any formatting. + side_effect: Optional[SideEffect] + side effect to take on the dictionary. The side effect can be either + convert the dictionary keys to "lower" (SideEffect.CONVERT_KEYS_TO_LOWER.value) + or "upper"(SideEffect.CONVERT_KEYS_TO_UPPER.value) cases. + + The other keyword arguments to be passed into fsspec.open(). + For OCI object storage, this could be config="path/to/.oci/config". + + Returns + ------- + Union[str, None] + Serialized version of object. + `None` in case when `uri` provided. + """ + note = kwargs.pop("note", "") + + yaml_string = f"{note}\n" + yaml.dump( + self.model_dump(exclude_none=kwargs.pop("exclude_none", False)), + Dumper=dumper, + ) + if uri: + self._write_to_file(s=yaml_string, uri=uri, **kwargs) + return None + + return yaml_string + + @classmethod + def from_yaml( + cls, + yaml_string: str = None, + uri: str = None, + loader: callable = Loader, + **kwargs, + ): + """Creates an object from YAML string provided or from URI location containing YAML string + + Parameters + ---------- + yaml_string (string, optional) + YAML string. Defaults to None. + uri (string, optional) + URI location of file containing YAML string. Defaults to None. + loader (callable, optional) + Custom YAML loader. Defaults to CLoader/SafeLoader. + kwargs (dict) + keyword arguments to be passed into fsspec.open(). + For OCI object storage, this should be config="path/to/.oci/config". + For other storage connections consider e.g. host, port, username, password, etc. + + Raises + ------ + ValueError + Raised if neither string nor uri is provided + + Returns + ------- + cls + Returns instance of the class + """ + if yaml_string: + return cls(**yaml.load(yaml_string, Loader=loader)) + if uri: + return cls( + **yaml.load(cls._read_from_file(uri=uri, **kwargs), Loader=loader) + ) + raise ValueError("Must provide either YAML string or URI location") + + @classmethod + def schema_to_yaml(cls, uri: str = None, **kwargs) -> Union[str, None]: + """Returns the schema serialized as a YAML string + + Parameters + ---------- + uri : str, optional + URI location to save the YAML string, by default None + dumper : callable, optional + Custom YAML Dumper, by default yaml.SafeDumper + kwargs : dict + overwrite: (bool, optional). Defaults to True. + Whether to overwrite existing file or not. + Returns + ------- + Union[str, None] + Serialized schema. + `None` in case when `uri` provided. + """ + yaml_string = yaml.dump(cls.model_json_schema(), sort_keys=False) + + if uri: + cls._write_to_file(s=yaml_string, uri=uri, **kwargs) + return None + + return yaml_string + + @classmethod + def schema_to_json( + cls, + uri: str = None, + encoder: callable = json.JSONEncoder, + default: callable = None, + **kwargs, + ) -> Union[str, None]: + """Returns the schema serialized as a JSON string + + Parameters + ---------- + uri: (string, optional) + URI location to save the JSON string. Defaults to None. + encoder: (callable, optional) + Encoder for custom data structures. Defaults to JSONEncoder. + default: (callable, optional) + A function that gets called for objects that can't otherwise be serialized. + It should return JSON-serializable version of the object or original object. + + kwargs + ------ + overwrite: (bool, optional). Defaults to True. + Whether to overwrite existing file or not. + + keyword arguments to be passed into fsspec.open(). + For OCI object storage, this could be config="path/to/.oci/config". + For other storage connections consider e.g. host, port, username, password, etc. + + Returns + ------- + Union[str, None] + Serialized version of object. + `None` in case when `uri` provided. + """ + json_string = json.dumps( + cls.model_json_schema(), + cls=encoder, + default=default, + ) + if uri: + cls._write_to_file(s=json_string, uri=uri, **kwargs) + return None + return json_string diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index 55b5cf7e9..d04bed9ed 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -15,6 +15,7 @@ EVALUATION_REPORT_JSON = "report.json" EVALUATION_REPORT_MD = "report.md" EVALUATION_REPORT = "report.html" +EVALUATION_SERVICE_CONFIG = "evaluation.json" UNKNOWN_JSON_STR = "{}" FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20" DEFAULT_FT_BLOCK_STORAGE_SIZE = 750 @@ -24,7 +25,7 @@ MAXIMUM_ALLOWED_DATASET_IN_BYTE = 52428800 # 1024 x 1024 x 50 = 50MB JOB_INFRASTRUCTURE_TYPE_DEFAULT_NETWORKING = "ME_STANDALONE" NB_SESSION_IDENTIFIER = "NB_SESSION_OCID" -LIFECYCLE_DETAILS_MISSING_JOBRUN = "The asscociated JobRun resource has been deleted." +LIFECYCLE_DETAILS_MISSING_JOBRUN = "The associated JobRun resource has been deleted." READY_TO_DEPLOY_STATUS = "ACTIVE" READY_TO_FINE_TUNE_STATUS = "TRUE" AQUA_GA_LIST = ["id19sfcrra6z"] diff --git a/pyproject.toml b/pyproject.toml index ff134a266..feb8cf355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ dependencies = [ "scikit-learn>=1.0", "tabulate>=0.8.9", "tqdm>=4.59.0", + "pydantic>=2.6.3", ] [project.optional-dependencies] diff --git a/tests/unitary/with_extras/aqua/test_config.py b/tests/unitary/with_extras/aqua/test_config.py new file mode 100644 index 000000000..bbdbd1297 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_config.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + + +from unittest.mock import MagicMock, patch + +from ads.aqua.common.utils import service_config_path +from ads.aqua.config.config import evaluation_config +from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig +from ads.aqua.constants import EVALUATION_SERVICE_CONFIG + + +class TestConfig: + """Unit tests for AQUA common configurations.""" + + @patch.object(EvaluationServiceConfig, "from_json") + def test_evaluation_service_config(self, mock_from_json): + """Ensures that the common evaluation configuration can be successfully retrieved.""" + + expected_result = MagicMock() + mock_from_json.return_value = expected_result + + test_result = evaluation_config() + + mock_from_json.assert_called_with( + uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}" + ) + assert test_result == expected_result diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json new file mode 100644 index 000000000..794fb00d7 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json @@ -0,0 +1,333 @@ +{ + "inference_params": { + "default": { + "inference_backoff_factor": 3.0, + "inference_delay": 0.0, + "inference_max_threads": 10, + "inference_retries": 3, + "inference_rps": 25, + "inference_timeout": 120 + }, + "frameworks": [ + { + "framework": "vllm", + "params": {} + }, + { + "framework": "tgi", + "params": {} + }, + { + "framework": "llama-cpp", + "params": { + "inference_delay": 1, + "inference_max_threads": 1 + } + } + ] + }, + "kind": "evaluation", + "metrics": [ + { + "args": {}, + "description": "BERT Score.", + "key": "bertscore", + "name": "BERT Score", + "tags": [], + "task": [ + "text-generation" + ] + }, + { + "args": {}, + "description": "ROUGE scores.", + "key": "rouge", + "name": "ROUGE Score", + "tags": [], + "task": [ + "text-generation" + ] + }, + { + "args": {}, + "description": "BLEU (Bilingual Evaluation Understudy).", + "key": "bleu", + "name": "BLEU Score", + "tags": [], + "task": [ + "text-generation" + ] + }, + { + "args": {}, + "description": "Perplexity is a metric to evaluate the quality of language models.", + "key": "perplexity_score", + "name": "Perplexity Score", + "tags": [], + "task": [ + "text-generation" + ] + }, + { + "args": {}, + "description": "Text quality/readability metrics.", + "key": "text_readability", + "name": "Text Readability", + "tags": [], + "task": [ + "text-generation" + ] + } + ], + "model_params": { + "default": { + "some_default_param": "some_default_param" + }, + "frameworks": [ + { + "default": { + "add_generation_prompt": false, + "frequency_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "max_tokens": { + "default": 500, + "max": 4096, + "min": 50 + }, + "model": "odsc-llm", + "presence_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "stop": [], + "temperature": { + "default": 0.7, + "max": 2.0, + "min": 0.0 + }, + "top_k": { + "default": 50, + "max": 1000, + "min": 1 + }, + "top_p": { + "default": 0.9, + "max": 1.0, + "min": 0.0 + } + }, + "framework": "vllm", + "task": [ + "text-generation", + "image-text-to-text" + ], + "versions": { + "0.5.1": { + "overrides": { + "exclude": [ + "max_tokens", + "frequency_penalty" + ], + "include": { + "some_other_param": "some_other_param_value" + } + } + }, + "0.5.3.post1": { + "overrides": { + "exclude": [ + "add_generation_prompt" + ], + "include": {} + } + } + } + }, + { + "default": { + "add_generation_prompt": false, + "frequency_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "max_tokens": { + "default": 500, + "max": 4096, + "min": 50 + }, + "model": "odsc-llm", + "presence_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "stop": [], + "temperature": { + "default": 0.7, + "max": 2.0, + "min": 0.0 + }, + "top_k": { + "default": 50, + "max": 1000, + "min": 1 + }, + "top_p": { + "default": 0.9, + "max": 1.0, + "min": 0.0 + } + }, + "framework": "tgi", + "task": [ + "text-generation", + "image-text-to-text" + ], + "versions": { + "2.0.1.4": { + "overrides": { + "exclude": [ + "max_tokens", + "frequency_penalty" + ], + "include": { + "some_other_param": "some_other_param_value" + } + } + } + } + }, + { + "default": { + "add_generation_prompt": false, + "frequency_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "max_tokens": { + "default": 500, + "max": 4096, + "min": 50 + }, + "model": "odsc-llm", + "presence_penalty": { + "default": 0.0, + "max": 2.0, + "min": -2.0 + }, + "stop": [], + "temperature": { + "default": 0.7, + "max": 2.0, + "min": 0.0 + }, + "top_k": { + "default": 50, + "max": 1000, + "min": 1 + }, + "top_p": { + "default": 0.9, + "max": 1.0, + "min": 0.0 + } + }, + "framework": "llama-cpp", + "task": [ + "text-generation", + "image-text-to-text" + ], + "versions": { + "0.2.78.0": { + "overrides": { + "exclude": [], + "include": {} + } + } + } + } + ] + }, + "report_params": { + "default": {} + }, + "shapes": [ + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard.E3.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard.E4.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard3.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Optimized3.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodel" + ] + }, + "memory_in_gbs": null, + "name": "VM.GPU.A10.2", + "ocpu": null + } + ], + "version": "1.0" +} diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json new file mode 100644 index 000000000..930c4a0d1 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json @@ -0,0 +1,24 @@ +{ + "inference_params": { + "default": { + "inference_backoff_factor": 3, + "inference_delay": 0, + "inference_max_threads": 10, + "inference_retries": 3, + "inference_rps": 25, + "inference_timeout": 120 + }, + "frameworks": [] + }, + "kind": "evaluation", + "metrics": [], + "model_params": { + "default": {}, + "frameworks": [] + }, + "report_params": { + "default": {} + }, + "shapes": [], + "version": "1.0" +} diff --git a/tests/unitary/with_extras/aqua/test_evaluation_service_config.py b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py new file mode 100644 index 000000000..83a9d0986 --- /dev/null +++ b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# Copyright (c) 2024 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + + +import json +import os + +import pytest + +from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig + + +class TestEvaluationServiceConfig: + """Unit tests for EvaluationServiceConfig class.""" + + def setup_class(cls): + cls.curr_dir = os.path.dirname(os.path.abspath(__file__)) + cls.artifact_dir = os.path.join(cls.curr_dir, "test_data", "config") + + def teardown_class(cls): ... + + def setup_method(self): + self.mock_config: EvaluationServiceConfig = EvaluationServiceConfig.from_json( + uri=os.path.join(self.artifact_dir, "evaluation_config.json") + ) + + def test_init(self): + """Ensures the config can be instantiated with the default params""" + test_config = EvaluationServiceConfig() + + with open( + os.path.join( + self.artifact_dir, "evaluation_config_with_default_params.json" + ) + ) as file: + expected_config = json.load(file) + + assert test_config.to_dict() == expected_config + + def test_read_config(self): + """Ensures the config can be read from the JSON file.""" + + with open(os.path.join(self.artifact_dir, "evaluation_config.json")) as file: + expected_config = json.load(file) + + assert self.mock_config.to_dict() == expected_config + + @pytest.mark.parametrize( + "framework_name, extra_params", + [ + ("vllm", {}), + ("VLLM", {}), + ("tgi", {}), + ("llama-cpp", {"inference_max_threads": 1, "inference_delay": 1}), + ("none-exist", {}), + ], + ) + def test_get_merged_inference_params(self, framework_name, extra_params): + """Tests merging default inference params with those specific to the given framework.""" + + test_result = self.mock_config.get_merged_inference_params( + framework_name=framework_name + ) + expected_result = { + "inference_rps": 25, + "inference_timeout": 120, + "inference_max_threads": 10, + "inference_retries": 3, + "inference_backoff_factor": 3.0, + "inference_delay": 0.0, + } + expected_result.update(extra_params) + + assert test_result.to_dict() == expected_result + + @pytest.mark.parametrize( + "framework_name, version, task, exclude, include", + [ + ("vllm", "0.5.3.post1", "text-generation", ["add_generation_prompt"], {}), + ( + "vllm", + "0.5.1", + "image-text-to-text", + ["max_tokens", "frequency_penalty"], + {"some_other_param": "some_other_param_value"}, + ), + ("vllm", "0.5.1", "none-exist", [], {}), + ("vllm", "none-exist", "text-generation", [], {}), + ("tgi", None, "text-generation", [], {}), + ("tgi", "none-exist", "text-generation", [], {}), + ( + "tgi", + "2.0.1.4", + "text-generation", + ["max_tokens", "frequency_penalty"], + {"some_other_param": "some_other_param_value"}, + ), + ("llama-cpp", "0.2.78.0", "text-generation", [], {}), + ("none-exist", "none-exist", "text-generation", [], {}), + ], + ) + def test_get_merged_model_params( + self, framework_name, version, task, exclude, include + ): + expected_result = {"some_default_param": "some_default_param"} + if task != "none-exist" and framework_name != "none-exist": + expected_result.update( + { + "model": "odsc-llm", + "add_generation_prompt": False, + "max_tokens": {"min": 50, "max": 4096, "default": 500}, + "temperature": {"min": 0.0, "max": 2.0, "default": 0.7}, + "top_p": {"min": 0.0, "max": 1.0, "default": 0.9}, + "top_k": {"min": 1, "max": 1000, "default": 50}, + "presence_penalty": {"min": -2.0, "max": 2.0, "default": 0.0}, + "frequency_penalty": {"min": -2.0, "max": 2.0, "default": 0.0}, + "stop": [], + } + ) + expected_result.update(include) + for key in exclude: + expected_result.pop(key, None) + + test_result = self.mock_config.get_merged_model_params( + framework_name=framework_name, version=version, task=task + ) + + assert test_result == expected_result + + @pytest.mark.parametrize( + "evaluation_container, evaluation_target, shapes_found", + [ + (None, None, 5), + ("odsc-llm-evaluate", None, 5), + ("odsc-llm-evaluate", "datasciencemodeldeployment", 4), + ("odsc-llm-evaluate", "datasciencemodel", 1), + ("none", None, 0), + (None, "none", 0), + ], + ) + def test_search_shapes(self, evaluation_container, evaluation_target, shapes_found): + """Ensures searching shapes that match the given filters.""" + test_result = self.mock_config.search_shapes( + evaluation_container=evaluation_container, + evaluation_target=evaluation_target, + ) + + assert len(test_result) == shapes_found From c55ce2295f31bf7cf93534ee3f467f932f6ec507 Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Mon, 2 Sep 2024 19:10:33 -0700 Subject: [PATCH 08/19] ODSC-61883: Externalize Supported Metrics List to Service Config --- ads/aqua/evaluation/evaluation.py | 57 +++---------------- .../with_extras/aqua/test_evaluation.py | 34 ++++++++--- tests/unitary/with_extras/aqua/utils.py | 11 ---- 3 files changed, 36 insertions(+), 66 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index b484cff4c..10579346d 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -45,6 +45,7 @@ is_valid_ocid, upload_local_to_os, ) +from ads.aqua.config.config import evaluation_service_config from ads.aqua.constants import ( CONSOLE_LINK_RESOURCE_TYPE_MAPPING, EVALUATION_REPORT, @@ -191,7 +192,7 @@ def create( enable_spec=True ).inference for container in inference_config.values(): - if container.name == runtime.image[:runtime.image.rfind(":")]: + if container.name == runtime.image[: runtime.image.rfind(":")]: eval_inference_configuration = ( container.spec.evaluation_configuration ) @@ -416,9 +417,11 @@ def create( report_path=create_aqua_evaluation_details.report_path, model_parameters=create_aqua_evaluation_details.model_parameters, metrics=create_aqua_evaluation_details.metrics, - inference_configuration=eval_inference_configuration.to_filtered_dict() - if eval_inference_configuration - else {}, + inference_configuration=( + eval_inference_configuration.to_filtered_dict() + if eval_inference_configuration + else {} + ), ) ).create(**kwargs) ## TODO: decide what parameters will be needed logger.debug( @@ -901,49 +904,7 @@ def get_status(self, eval_id: str) -> dict: def get_supported_metrics(self) -> dict: """Gets a list of supported metrics for evaluation.""" - # TODO: implement it when starting to support more metrics. - return [ - { - "use_case": ["text_generation"], - "key": "bertscore", - "name": "bertscore", - "description": ( - "BERT Score is a metric for evaluating the quality of text " - "generation models, such as machine translation or summarization. " - "It utilizes pre-trained BERT contextual embeddings for both the " - "generated and reference texts, and then calculates the cosine " - "similarity between these embeddings." - ), - "args": {}, - }, - { - "use_case": ["text_generation"], - "key": "rouge", - "name": "rouge", - "description": ( - "ROUGE scores compare a candidate document to a collection of " - "reference documents to evaluate the similarity between them. " - "The metrics range from 0 to 1, with higher scores indicating " - "greater similarity. ROUGE is more suitable for models that don't " - "include paraphrasing and do not generate new text units that don't " - "appear in the references." - ), - "args": {}, - }, - { - "use_case": ["text_generation"], - "key": "bleu", - "name": "bleu", - "description": ( - "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the " - "quality of text which has been machine-translated from one natural language to another. " - "Quality is considered to be the correspondence between a machine's output and that of a " - "human: 'the closer a machine translation is to a professional human translation, " - "the better it is'." - ), - "args": {}, - }, - ] + return [item.to_dict() for item in evaluation_service_config().metrics] @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua") def load_metrics(self, eval_id: str) -> AquaEvalMetrics: @@ -1225,7 +1186,7 @@ def _delete_job_and_model(job, model): f"Exception message: {ex}" ) - def load_evaluation_config(self, eval_id): + def load_evaluation_config(self): """Loads evaluation config.""" return { "model_params": { diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index b75986d3d..2440acd03 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -22,6 +22,10 @@ AquaMissingKeyError, AquaRuntimeError, ) +from ads.aqua.config.evaluation.evaluation_service_config import ( + EvaluationServiceConfig, + MetricConfig, +) from ads.aqua.constants import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN from ads.aqua.evaluation import AquaEvaluationApp from ads.aqua.evaluation.entities import ( @@ -875,17 +879,33 @@ def test_extract_job_lifecycle_details(self, input, expect_output): msg = self.app._extract_job_lifecycle_details(input) assert msg == expect_output, msg - def test_get_supported_metrics(self): - """Tests getting a list of supported metrics for evaluation. - This method currently hardcoded the return value. + @patch("ads.aqua.evaluation.evaluation.evaluation_service_config") + def test_get_supported_metrics(self, mock_evaluation_service_config): + """ + Tests getting a list of supported metrics for evaluation. """ - from .utils import SupportMetricsFormat as metric_schema - from .utils import check + test_evaluation_service_config = EvaluationServiceConfig( + metrics=[ + MetricConfig( + **{ + "args": {}, + "description": "BERT Score.", + "key": "bertscore", + "name": "BERT Score", + "tags": [], + "task": ["text-generation"], + }, + ) + ] + ) + mock_evaluation_service_config.return_value = test_evaluation_service_config response = self.app.get_supported_metrics() assert isinstance(response, list) - for metric in response: - assert check(metric_schema, metric) + assert len(response) == len(test_evaluation_service_config.metrics) + assert response == [ + item.to_dict() for item in test_evaluation_service_config.metrics + ] def test_load_evaluation_config(self): """Tests loading default config for evaluation. diff --git a/tests/unitary/with_extras/aqua/utils.py b/tests/unitary/with_extras/aqua/utils.py index f1b25fdf4..97ae857cb 100644 --- a/tests/unitary/with_extras/aqua/utils.py +++ b/tests/unitary/with_extras/aqua/utils.py @@ -76,17 +76,6 @@ def __post_init__(self): ) -@dataclass -class SupportMetricsFormat(BaseFormat): - """Format for supported evaluation metrics.""" - - use_case: list - key: str - name: str - description: str - args: dict - - @dataclass class EvaluationConfigFormat(BaseFormat): """Evaluation config format.""" From e5710ba11af65b80984787907a4856edd02d9a3f Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Wed, 4 Sep 2024 18:45:11 -0700 Subject: [PATCH 09/19] Enhance the evaluation service config. --- ads/aqua/config/config.py | 17 +- .../evaluation/evaluation_service_config.py | 200 ++++---- ads/aqua/constants.py | 1 - tests/unitary/with_extras/aqua/test_config.py | 42 +- .../test_data/config/evaluation_config.json | 439 +++++++----------- ...evaluation_config_with_default_params.json | 21 +- .../aqua/test_evaluation_service_config.py | 75 ++- 7 files changed, 356 insertions(+), 439 deletions(-) diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index 1eb3f3104..1cdda1ac5 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -4,16 +4,21 @@ from datetime import datetime, timedelta +from typing import Optional from cachetools import TTLCache, cached -from ads.aqua.common.utils import service_config_path +from ads.aqua.common.entities import ContainerSpec +from ads.aqua.common.utils import get_container_config from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig -from ads.aqua.constants import EVALUATION_SERVICE_CONFIG + +DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" @cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) -def evaluation_service_config() -> EvaluationServiceConfig: +def evaluation_service_config( + container: Optional[str] = DEFAULT_EVALUATION_CONTAINER, +) -> EvaluationServiceConfig: """ Retrieves the common evaluation configuration. @@ -22,8 +27,10 @@ def evaluation_service_config() -> EvaluationServiceConfig: EvaluationServiceConfig: The evaluation common config. """ - return EvaluationServiceConfig.from_json( - uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}" + return EvaluationServiceConfig( + **get_container_config() + .get(ContainerSpec.CONTAINER_SPEC, {}) + .get(container, {}) ) diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py index 6626abef9..4242ab608 100644 --- a/ads/aqua/config/evaluation/evaluation_service_config.py +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -4,7 +4,7 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ from copy import deepcopy -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional from pydantic import Field @@ -19,17 +19,6 @@ INFERENCE_DELAY = 0 -class ModelParamItem(Serializable): - """Represents min, max, and default values for a model parameter.""" - - min: Optional[Union[int, float]] = None - max: Optional[Union[int, float]] = None - default: Optional[Union[int, float]] = None - - class Config: - extra = "ignore" - - class ModelParamsOverrides(Serializable): """Defines overrides for model parameters, including exclusions and additional inclusions.""" @@ -51,28 +40,11 @@ class Config: extra = "ignore" -class ModelDefaultParams(Serializable): - """Defines default parameters for a model within a specific framework.""" - - model: Optional[str] = None - max_tokens: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - temperature: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - top_p: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - top_k: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - presence_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - frequency_penalty: Optional[ModelParamItem] = Field(default_factory=ModelParamItem) - stop: List[str] = Field(default_factory=list) +class ModelParamsContainer(Serializable): + """Represents a container's model configuration, including tasks, defaults, and versions.""" - class Config: - extra = "allow" - - -class ModelFramework(Serializable): - """Represents a framework's model configuration, including tasks, defaults, and versions.""" - - framework: Optional[str] = None - task: Optional[List[str]] = Field(default_factory=list) - default: Optional[ModelDefaultParams] = Field(default_factory=ModelDefaultParams) + name: Optional[str] = None + default: Optional[Dict[str, Any]] = Field(default_factory=dict) versions: Optional[Dict[str, ModelParamsVersion]] = Field(default_factory=dict) class Config: @@ -93,10 +65,10 @@ class Config: extra = "allow" -class InferenceFramework(Serializable): - """Represents the inference parameters specific to a framework.""" +class InferenceContainer(Serializable): + """Represents the inference parameters specific to a container.""" - framework: Optional[str] = None + name: Optional[str] = None params: Optional[Dict[str, Any]] = Field(default_factory=dict) class Config: @@ -113,27 +85,27 @@ class Config: class InferenceParamsConfig(Serializable): - """Combines default inference parameters with framework-specific configurations.""" + """Combines default inference parameters with container-specific configurations.""" default: Optional[InferenceParams] = Field(default_factory=InferenceParams) - frameworks: Optional[List[InferenceFramework]] = Field(default_factory=list) + containers: Optional[List[InferenceContainer]] = Field(default_factory=list) - def get_merged_params(self, framework_name: str) -> InferenceParams: + def get_merged_params(self, container_name: str) -> InferenceParams: """ - Merges default inference params with those specific to the given framework. + Merges default inference params with those specific to the given container. Parameters ---------- - framework_name (str): The name of the framework. + container_name (str): The name of the container. Returns ------- InferenceParams: The merged inference parameters. """ merged_params = self.default.to_dict() - for framework in self.frameworks: - if framework.framework.lower() == framework_name.lower(): - merged_params.update(framework.params or {}) + for containers in self.containers: + if containers.name.lower() == container_name.lower(): + merged_params.update(containers.params or {}) break return InferenceParams(**merged_params) @@ -141,27 +113,25 @@ class Config: extra = "ignore" -class ModelParamsConfig(Serializable): - """Encapsulates the model parameters for different frameworks.""" +class InferenceModelParamsConfig(Serializable): + """Encapsulates the model parameters for different containers.""" default: Optional[Dict[str, Any]] = Field(default_factory=dict) - frameworks: Optional[List[ModelFramework]] = Field(default_factory=list) + containers: Optional[List[ModelParamsContainer]] = Field(default_factory=list) - def get_model_params( + def get_merged_model_params( self, - framework_name: str, + container_name: str, version: Optional[str] = None, - task: Optional[str] = None, ) -> Dict[str, Any]: """ - Gets the model parameters for a given framework, version, and tasks, + Gets the model parameters for a given container, version, merged with the defaults. Parameters ---------- - framework_name (str): The name of the framework. - version (Optional[str]): The specific version of the framework. - task (Optional[str]): The specific task. + container_name (str): The name of the container. + version (Optional[str]): The specific version of the container. Returns ------- @@ -169,14 +139,12 @@ def get_model_params( """ params = deepcopy(self.default) - for framework in self.frameworks: - if framework.framework.lower() == framework_name.lower() and ( - not task or task.lower() in framework.task - ): - params.update(framework.default.to_dict()) + for container in self.containers: + if container.name.lower() == container_name.lower(): + params.update(container.default) - if version and version in framework.versions: - version_overrides = framework.versions[version].overrides + if version and version in container.versions: + version_overrides = container.versions[version].overrides if version_overrides: if version_overrides.include: params.update(version_overrides.include) @@ -228,59 +196,17 @@ class Config: extra = "ignore" -class EvaluationServiceConfig(Serializable): - """ - Root configuration class for evaluation setup including model, - inference, and shape configurations. - """ +class ModelParamsConfig(Serializable): + """Encapsulates the default model parameters.""" - version: Optional[str] = "1.0" - kind: Optional[str] = "evaluation" - report_params: Optional[ReportParams] = Field(default_factory=ReportParams) - inference_params: Optional[InferenceParamsConfig] = Field( - default_factory=InferenceParamsConfig - ) + default: Optional[Dict[str, Any]] = Field(default_factory=dict) + + +class UIConfig(Serializable): model_params: Optional[ModelParamsConfig] = Field(default_factory=ModelParamsConfig) shapes: List[ShapeConfig] = Field(default_factory=list) metrics: List[MetricConfig] = Field(default_factory=list) - def get_merged_inference_params(self, framework_name: str) -> InferenceParams: - """ - Merges default inference params with those specific to the given framework. - - Params - ------ - framework_name (str): The name of the framework. - - Returns - ------- - InferenceParams: The merged inference parameters. - """ - return self.inference_params.get_merged_params(framework_name=framework_name) - - def get_merged_model_params( - self, - framework_name: str, - version: Optional[str] = None, - task: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Gets the model parameters for a given framework, version, and task, merged with the defaults. - - Parameters - ---------- - framework_name (str): The name of the framework. - version (Optional[str]): The specific version of the framework. - task (Optional[str]): The task. - - Returns - ------- - Dict[str, Any]: The merged model parameters. - """ - return self.model_params.get_model_params( - framework_name=framework_name, version=version, task=task - ) - def search_shapes( self, evaluation_container: Optional[str] = None, @@ -315,3 +241,59 @@ def search_shapes( class Config: extra = "ignore" + + +class EvaluationServiceConfig(Serializable): + """ + Root configuration class for evaluation setup including model, + inference, and shape configurations. + """ + + version: Optional[str] = "1.0" + kind: Optional[str] = "evaluation" + report_params: Optional[ReportParams] = Field(default_factory=ReportParams) + inference_params: Optional[InferenceParamsConfig] = Field( + default_factory=InferenceParamsConfig + ) + inference_model_params: Optional[InferenceModelParamsConfig] = Field( + default_factory=InferenceModelParamsConfig + ) + ui_config: Optional[UIConfig] = Field(default_factory=UIConfig) + + def get_merged_inference_params(self, container_name: str) -> InferenceParams: + """ + Merges default inference params with those specific to the given container. + + Params + ------ + container_name (str): The name of the container. + + Returns + ------- + InferenceParams: The merged inference parameters. + """ + return self.inference_params.get_merged_params(container_name=container_name) + + def get_merged_inference_model_params( + self, + container_name: str, + version: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Gets the model parameters for a given container, version, and task, merged with the defaults. + + Parameters + ---------- + container_name (str): The name of the container. + version (Optional[str]): The specific version of the container. + + Returns + ------- + Dict[str, Any]: The merged model parameters. + """ + return self.inference_model_params.get_merged_model_params( + container_name=container_name, version=version + ) + + class Config: + extra = "ignore" diff --git a/ads/aqua/constants.py b/ads/aqua/constants.py index d04bed9ed..958b161bd 100644 --- a/ads/aqua/constants.py +++ b/ads/aqua/constants.py @@ -15,7 +15,6 @@ EVALUATION_REPORT_JSON = "report.json" EVALUATION_REPORT_MD = "report.md" EVALUATION_REPORT = "report.html" -EVALUATION_SERVICE_CONFIG = "evaluation.json" UNKNOWN_JSON_STR = "{}" FINE_TUNING_RUNTIME_CONTAINER = "iad.ocir.io/ociodscdev/aqua_ft_cuda121:0.3.17.20" DEFAULT_FT_BLOCK_STORAGE_SIZE = 750 diff --git a/tests/unitary/with_extras/aqua/test_config.py b/tests/unitary/with_extras/aqua/test_config.py index bbdbd1297..381b03116 100644 --- a/tests/unitary/with_extras/aqua/test_config.py +++ b/tests/unitary/with_extras/aqua/test_config.py @@ -2,28 +2,38 @@ # Copyright (c) 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +import json +import os +from unittest.mock import patch -from unittest.mock import MagicMock, patch - -from ads.aqua.common.utils import service_config_path -from ads.aqua.config.config import evaluation_config -from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig -from ads.aqua.constants import EVALUATION_SERVICE_CONFIG +from ads.aqua.common.entities import ContainerSpec +from ads.aqua.config.config import evaluation_service_config class TestConfig: """Unit tests for AQUA common configurations.""" - @patch.object(EvaluationServiceConfig, "from_json") - def test_evaluation_service_config(self, mock_from_json): - """Ensures that the common evaluation configuration can be successfully retrieved.""" - - expected_result = MagicMock() - mock_from_json.return_value = expected_result + def setup_class(cls): + cls.curr_dir = os.path.dirname(os.path.abspath(__file__)) + cls.artifact_dir = os.path.join(cls.curr_dir, "test_data", "config") - test_result = evaluation_config() + @patch("ads.aqua.config.config.get_container_config") + def test_evaluation_service_config(self, mock_get_container_config): + """Ensures that the common evaluation configuration can be successfully retrieved.""" - mock_from_json.assert_called_with( - uri=f"{service_config_path()}/{EVALUATION_SERVICE_CONFIG}" + with open( + os.path.join( + self.artifact_dir, "evaluation_config_with_default_params.json" + ) + ) as file: + expected_result = { + ContainerSpec.CONTAINER_SPEC: {"test_container": json.load(file)} + } + + mock_get_container_config.return_value = expected_result + + test_result = evaluation_service_config(container="test_container") + assert ( + test_result.to_dict() + == expected_result[ContainerSpec.CONTAINER_SPEC]["test_container"] ) - assert test_result == expected_result diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json index 794fb00d7..4a1dd2b30 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json @@ -1,130 +1,11 @@ { - "inference_params": { - "default": { - "inference_backoff_factor": 3.0, - "inference_delay": 0.0, - "inference_max_threads": 10, - "inference_retries": 3, - "inference_rps": 25, - "inference_timeout": 120 - }, - "frameworks": [ - { - "framework": "vllm", - "params": {} - }, - { - "framework": "tgi", - "params": {} - }, - { - "framework": "llama-cpp", - "params": { - "inference_delay": 1, - "inference_max_threads": 1 - } - } - ] - }, - "kind": "evaluation", - "metrics": [ - { - "args": {}, - "description": "BERT Score.", - "key": "bertscore", - "name": "BERT Score", - "tags": [], - "task": [ - "text-generation" - ] - }, - { - "args": {}, - "description": "ROUGE scores.", - "key": "rouge", - "name": "ROUGE Score", - "tags": [], - "task": [ - "text-generation" - ] - }, - { - "args": {}, - "description": "BLEU (Bilingual Evaluation Understudy).", - "key": "bleu", - "name": "BLEU Score", - "tags": [], - "task": [ - "text-generation" - ] - }, - { - "args": {}, - "description": "Perplexity is a metric to evaluate the quality of language models.", - "key": "perplexity_score", - "name": "Perplexity Score", - "tags": [], - "task": [ - "text-generation" - ] - }, - { - "args": {}, - "description": "Text quality/readability metrics.", - "key": "text_readability", - "name": "Text Readability", - "tags": [], - "task": [ - "text-generation" - ] - } - ], - "model_params": { - "default": { - "some_default_param": "some_default_param" - }, - "frameworks": [ + "inference_model_params": { + "containers": [ { "default": { - "add_generation_prompt": false, - "frequency_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "max_tokens": { - "default": 500, - "max": 4096, - "min": 50 - }, - "model": "odsc-llm", - "presence_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "stop": [], - "temperature": { - "default": 0.7, - "max": 2.0, - "min": 0.0 - }, - "top_k": { - "default": 50, - "max": 1000, - "min": 1 - }, - "top_p": { - "default": 0.9, - "max": 1.0, - "min": 0.0 - } + "add_generation_prompt": false }, - "framework": "vllm", - "task": [ - "text-generation", - "image-text-to-text" - ], + "name": "odsc-vllm-serving", "versions": { "0.5.1": { "overrides": { @@ -149,45 +30,9 @@ }, { "default": { - "add_generation_prompt": false, - "frequency_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "max_tokens": { - "default": 500, - "max": 4096, - "min": 50 - }, - "model": "odsc-llm", - "presence_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "stop": [], - "temperature": { - "default": 0.7, - "max": 2.0, - "min": 0.0 - }, - "top_k": { - "default": 50, - "max": 1000, - "min": 1 - }, - "top_p": { - "default": 0.9, - "max": 1.0, - "min": 0.0 - } + "add_generation_prompt": false }, - "framework": "tgi", - "task": [ - "text-generation", - "image-text-to-text" - ], + "name": "odsc-tgi-serving", "versions": { "2.0.1.4": { "overrides": { @@ -204,45 +49,9 @@ }, { "default": { - "add_generation_prompt": false, - "frequency_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "max_tokens": { - "default": 500, - "max": 4096, - "min": 50 - }, - "model": "odsc-llm", - "presence_penalty": { - "default": 0.0, - "max": 2.0, - "min": -2.0 - }, - "stop": [], - "temperature": { - "default": 0.7, - "max": 2.0, - "min": 0.0 - }, - "top_k": { - "default": 50, - "max": 1000, - "min": 1 - }, - "top_p": { - "default": 0.9, - "max": 1.0, - "min": 0.0 - } + "add_generation_prompt": false }, - "framework": "llama-cpp", - "task": [ - "text-generation", - "image-text-to-text" - ], + "name": "odsc-llama-cpp-serving", "versions": { "0.2.78.0": { "overrides": { @@ -252,82 +61,188 @@ } } } - ] + ], + "default": { + "add_generation_prompt": false, + "frequency_penalty": 0.0, + "max_tokens": 500, + "model": "odsc-llm", + "presence_penalty": 0.0, + "some_default_param": "some_default_param", + "stop": [], + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9 + } + }, + "inference_params": { + "containers": [ + { + "name": "odsc-vllm-serving", + "params": {} + }, + { + "name": "odsc-tgi-serving", + "params": {} + }, + { + "name": "odsc-llama-cpp-serving", + "params": { + "inference_delay": 1, + "inference_max_threads": 1 + } + } + ], + "default": { + "inference_backoff_factor": 3, + "inference_delay": 0, + "inference_max_threads": 10, + "inference_retries": 3, + "inference_rps": 25, + "inference_timeout": 120 + } }, + "kind": "evaluation", "report_params": { "default": {} }, - "shapes": [ - { - "block_storage_size": 200, - "filter": { - "evaluation_container": [ - "odsc-llm-evaluate" - ], - "evaluation_target": [ - "datasciencemodeldeployment" + "ui_config": { + "metrics": [ + { + "args": {}, + "description": "BERT Score is a metric for evaluating the quality of text generation models, such as machine translation or summarization. It utilizes pre-trained BERT contextual embeddings for both the generated and reference texts, and then calculates the cosine similarity between these embeddings.", + "key": "bertscore", + "name": "BERT Score", + "tags": [], + "task": [ + "text-generation" ] }, - "memory_in_gbs": 128, - "name": "VM.Standard.E3.Flex", - "ocpu": 8 - }, - { - "block_storage_size": 200, - "filter": { - "evaluation_container": [ - "odsc-llm-evaluate" - ], - "evaluation_target": [ - "datasciencemodeldeployment" + { + "args": {}, + "description": "ROUGE scores compare a candidate document to a collection of reference documents to evaluate the similarity between them. The metrics range from 0 to 1, with higher scores indicating greater similarity. ROUGE is more suitable for models that don't include paraphrasing and do not generate new text units that don't appear in the references.", + "key": "rouge", + "name": "ROUGE Score", + "tags": [], + "task": [ + "text-generation" ] }, - "memory_in_gbs": 128, - "name": "VM.Standard.E4.Flex", - "ocpu": 8 - }, - { - "block_storage_size": 200, - "filter": { - "evaluation_container": [ - "odsc-llm-evaluate" - ], - "evaluation_target": [ - "datasciencemodeldeployment" + { + "args": {}, + "description": "BLEU (Bilingual Evaluation Understudy) is an algorithm for evaluating the quality of text which has been machine-translated from one natural language to another. Quality is considered to be the correspondence between a machine's output and that of a human: 'the closer a machine translation is to a professional human translation, the better it is'.", + "key": "bleu", + "name": "BLEU Score", + "tags": [], + "task": [ + "text-generation" ] }, - "memory_in_gbs": 128, - "name": "VM.Standard3.Flex", - "ocpu": 8 - }, - { - "block_storage_size": 200, - "filter": { - "evaluation_container": [ - "odsc-llm-evaluate" - ], - "evaluation_target": [ - "datasciencemodeldeployment" + { + "args": {}, + "description": "Perplexity is a metric to evaluate the quality of language models, particularly for \"Text Generation\" task type. Perplexity quantifies how well a LLM can predict the next word in a sequence of words. A high perplexity score indicates that the LLM is not confident in its text generation \u2014 that is, the model is \"perplexed\" \u2014 whereas a low perplexity score indicates that the LLM is confident in its generation.", + "key": "perplexity_score", + "name": "Perplexity Score", + "tags": [], + "task": [ + "text-generation" ] }, - "memory_in_gbs": 128, - "name": "VM.Optimized3.Flex", - "ocpu": 8 - }, - { - "block_storage_size": 200, - "filter": { - "evaluation_container": [ - "odsc-llm-evaluate" - ], - "evaluation_target": [ - "datasciencemodel" + { + "args": {}, + "description": "Text quality/readability metrics offer valuable insights into the quality and suitability of generated responses. Monitoring these metrics helps ensure that Language Model (LLM) outputs are clear, concise, and appropriate for the target audience. Evaluating text complexity and grade level helps tailor the generated content to the intended readers. By considering aspects such as sentence structure, vocabulary, and domain-specific needs, we can make sure the LLM produces responses that match the desired reading level and professional context. Additionally, metrics like syllable count, word count, and character count allow you to keep track of the length and structure of the generated text.", + "key": "text_readability", + "name": "Text Readability", + "tags": [], + "task": [ + "text-generation" ] + } + ], + "model_params": { + "default": { + "frequency_penalty": 0.0, + "max_tokens": 500, + "model": "odsc-llm", + "presence_penalty": 0.0, + "stop": [], + "temperature": 0.7, + "top_k": 50, + "top_p": 0.9 + } + }, + "shapes": [ + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard.E3.Flex", + "ocpu": 8 }, - "memory_in_gbs": null, - "name": "VM.GPU.A10.2", - "ocpu": null - } - ], + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard.E4.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Standard3.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodeldeployment" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Optimized3.Flex", + "ocpu": 8 + }, + { + "block_storage_size": 200, + "filter": { + "evaluation_container": [ + "odsc-llm-evaluate" + ], + "evaluation_target": [ + "datasciencemodel" + ] + }, + "memory_in_gbs": 128, + "name": "VM.Optimized3.Flex", + "ocpu": 8 + } + ] + }, "version": "1.0" } diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json index 930c4a0d1..0ef9983c9 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json @@ -1,5 +1,10 @@ { + "inference_model_params": { + "containers": [], + "default": {} + }, "inference_params": { + "containers": [], "default": { "inference_backoff_factor": 3, "inference_delay": 0, @@ -7,18 +12,18 @@ "inference_retries": 3, "inference_rps": 25, "inference_timeout": 120 - }, - "frameworks": [] + } }, "kind": "evaluation", - "metrics": [], - "model_params": { - "default": {}, - "frameworks": [] - }, "report_params": { "default": {} }, - "shapes": [], + "ui_config": { + "metrics": [], + "model_params": { + "default": {} + }, + "shapes": [] + }, "version": "1.0" } diff --git a/tests/unitary/with_extras/aqua/test_evaluation_service_config.py b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py index 83a9d0986..a090f6ad2 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation_service_config.py +++ b/tests/unitary/with_extras/aqua/test_evaluation_service_config.py @@ -47,20 +47,23 @@ def test_read_config(self): assert self.mock_config.to_dict() == expected_config @pytest.mark.parametrize( - "framework_name, extra_params", + "container_name, extra_params", [ - ("vllm", {}), - ("VLLM", {}), - ("tgi", {}), - ("llama-cpp", {"inference_max_threads": 1, "inference_delay": 1}), + ("odsc-vllm-serving", {}), + ("odsc-vllm-serving", {}), + ("odsc-tgi-serving", {}), + ( + "odsc-llama-cpp-serving", + {"inference_max_threads": 1, "inference_delay": 1}, + ), ("none-exist", {}), ], ) - def test_get_merged_inference_params(self, framework_name, extra_params): + def test_get_merged_inference_params(self, container_name, extra_params): """Tests merging default inference params with those specific to the given framework.""" test_result = self.mock_config.get_merged_inference_params( - framework_name=framework_name + container_name=container_name ) expected_result = { "inference_rps": 25, @@ -75,55 +78,51 @@ def test_get_merged_inference_params(self, framework_name, extra_params): assert test_result.to_dict() == expected_result @pytest.mark.parametrize( - "framework_name, version, task, exclude, include", + "container_name, version, exclude, include", [ - ("vllm", "0.5.3.post1", "text-generation", ["add_generation_prompt"], {}), + ("odsc-vllm-serving", "0.5.3.post1", ["add_generation_prompt"], {}), ( - "vllm", + "odsc-vllm-serving", "0.5.1", - "image-text-to-text", ["max_tokens", "frequency_penalty"], {"some_other_param": "some_other_param_value"}, ), - ("vllm", "0.5.1", "none-exist", [], {}), - ("vllm", "none-exist", "text-generation", [], {}), - ("tgi", None, "text-generation", [], {}), - ("tgi", "none-exist", "text-generation", [], {}), + ("odsc-vllm-serving", "none-exist", [], {}), + ("odsc-tgi-serving", None, [], {}), + ("odsc-tgi-serving", "none-exist", [], {}), ( - "tgi", + "odsc-tgi-serving", "2.0.1.4", - "text-generation", ["max_tokens", "frequency_penalty"], {"some_other_param": "some_other_param_value"}, ), - ("llama-cpp", "0.2.78.0", "text-generation", [], {}), - ("none-exist", "none-exist", "text-generation", [], {}), + ("odsc-llama-cpp-serving", "0.2.78.0", [], {}), + ("none-exist", "none-exist", [], {}), ], ) - def test_get_merged_model_params( - self, framework_name, version, task, exclude, include + def test_get_merged_inference_model_params( + self, container_name, version, exclude, include ): expected_result = {"some_default_param": "some_default_param"} - if task != "none-exist" and framework_name != "none-exist": - expected_result.update( - { - "model": "odsc-llm", - "add_generation_prompt": False, - "max_tokens": {"min": 50, "max": 4096, "default": 500}, - "temperature": {"min": 0.0, "max": 2.0, "default": 0.7}, - "top_p": {"min": 0.0, "max": 1.0, "default": 0.9}, - "top_k": {"min": 1, "max": 1000, "default": 50}, - "presence_penalty": {"min": -2.0, "max": 2.0, "default": 0.0}, - "frequency_penalty": {"min": -2.0, "max": 2.0, "default": 0.0}, - "stop": [], - } - ) + expected_result.update( + { + "model": "odsc-llm", + "add_generation_prompt": False, + "max_tokens": 500, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop": [], + } + ) expected_result.update(include) for key in exclude: expected_result.pop(key, None) - test_result = self.mock_config.get_merged_model_params( - framework_name=framework_name, version=version, task=task + test_result = self.mock_config.get_merged_inference_model_params( + container_name=container_name, version=version ) assert test_result == expected_result @@ -141,7 +140,7 @@ def test_get_merged_model_params( ) def test_search_shapes(self, evaluation_container, evaluation_target, shapes_found): """Ensures searching shapes that match the given filters.""" - test_result = self.mock_config.search_shapes( + test_result = self.mock_config.ui_config.search_shapes( evaluation_container=evaluation_container, evaluation_target=evaluation_target, ) From 44814b42621732bef44e8576c9339f949cdf1843 Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Wed, 4 Sep 2024 18:58:40 -0700 Subject: [PATCH 10/19] Adjusts the evaluation metrics config. --- ads/aqua/evaluation/evaluation.py | 4 ++- .../with_extras/aqua/test_evaluation.py | 31 ++++++++++--------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 10579346d..5b8ed14fb 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -904,7 +904,9 @@ def get_status(self, eval_id: str) -> dict: def get_supported_metrics(self) -> dict: """Gets a list of supported metrics for evaluation.""" - return [item.to_dict() for item in evaluation_service_config().metrics] + return [ + item.to_dict() for item in evaluation_service_config().ui_config.metrics + ] @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua") def load_metrics(self, eval_id: str) -> AquaEvalMetrics: diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 2440acd03..4468a4e44 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -25,6 +25,7 @@ from ads.aqua.config.evaluation.evaluation_service_config import ( EvaluationServiceConfig, MetricConfig, + UIConfig, ) from ads.aqua.constants import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN from ads.aqua.evaluation import AquaEvaluationApp @@ -886,25 +887,27 @@ def test_get_supported_metrics(self, mock_evaluation_service_config): """ test_evaluation_service_config = EvaluationServiceConfig( - metrics=[ - MetricConfig( - **{ - "args": {}, - "description": "BERT Score.", - "key": "bertscore", - "name": "BERT Score", - "tags": [], - "task": ["text-generation"], - }, - ) - ] + ui_config=UIConfig( + metrics=[ + MetricConfig( + **{ + "args": {}, + "description": "BERT Score.", + "key": "bertscore", + "name": "BERT Score", + "tags": [], + "task": ["text-generation"], + }, + ) + ] + ) ) mock_evaluation_service_config.return_value = test_evaluation_service_config response = self.app.get_supported_metrics() assert isinstance(response, list) - assert len(response) == len(test_evaluation_service_config.metrics) + assert len(response) == len(test_evaluation_service_config.ui_config.metrics) assert response == [ - item.to_dict() for item in test_evaluation_service_config.metrics + item.to_dict() for item in test_evaluation_service_config.ui_config.metrics ] def test_load_evaluation_config(self): From 81494c231251e02ca10bc3012ddf83887ed338b0 Mon Sep 17 00:00:00 2001 From: Lu Peng <118394507+lu-ohai@users.noreply.github.com> Date: Thu, 5 Sep 2024 14:37:10 -0400 Subject: [PATCH 11/19] Fix override container family (#938) --- ads/aqua/modeldeployment/deployment.py | 82 ++++++++++---------------- 1 file changed, 31 insertions(+), 51 deletions(-) diff --git a/ads/aqua/modeldeployment/deployment.py b/ads/aqua/modeldeployment/deployment.py index 2970e6192..654e00dc8 100644 --- a/ads/aqua/modeldeployment/deployment.py +++ b/ads/aqua/modeldeployment/deployment.py @@ -146,7 +146,7 @@ def create( env_var : dict, optional Environment variable for the deployment, by default None. container_family: str - The image family of model deployment container runtime. Required for unverified Aqua models. + The image family of model deployment container runtime. memory_in_gbs: float The memory in gbs for the shape selected. ocpus: float @@ -230,41 +230,14 @@ def create( env_var.update({"FT_MODEL": f"{fine_tune_output_path}"}) - is_custom_container = False - try: - container_type_key = aqua_model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME - ).value - except ValueError as err: - message = ( - f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field " - f"for model {aqua_model.id}." - ) - logger.debug(message) - if not container_family: - raise AquaValueError( - f"{message}. For unverified Aqua models, container_family parameter should be " - f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}." - ) from err - container_type_key = container_family - try: - # Check if the container override flag is set. If set, then the user has chosen custom image - if aqua_model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_OVERRIDE_FLAG_METADATA_NAME - ).value: - is_custom_container = True - except Exception: - pass + container_type_key = self._get_container_type_key( + model=aqua_model, + container_family=container_family + ) # fetch image name from config - # If the image is of type custom, then `container_type_key` is the inference image - container_image = ( - get_container_image( - container_type=container_type_key, - ) - if not is_custom_container - else container_type_key - ) + container_image = get_container_image(container_type=container_type_key) + logging.info( f"Aqua Image used for deploying {aqua_model.id} : {container_image}" ) @@ -433,6 +406,26 @@ def create( deployment.dsc_model_deployment, self.region ) + @staticmethod + def _get_container_type_key(model: DataScienceModel, container_family: str) -> str: + container_type_key = UNKNOWN + if container_family: + container_type_key = container_family + else: + try: + container_type_key = model.custom_metadata_list.get( + AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME + ).value + except ValueError as err: + raise AquaValueError( + f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field " + f"for model {model.id}. For unverified Aqua models, {AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} should be" + f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}." + ) from err + + return container_type_key + + @telemetry(entry_point="plugin=deployment&action=list", name="aqua") def list(self, **kwargs) -> List["AquaDeployment"]: """List Aqua model deployments in a given compartment and under certain project. @@ -672,23 +665,10 @@ def validate_deployment_params( restricted_params = [] if params: model = DataScienceModel.from_id(model_id) - try: - container_type_key = model.custom_metadata_list.get( - AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME - ).value - except ValueError as err: - message = ( - f"{AQUA_DEPLOYMENT_CONTAINER_METADATA_NAME} key is not available in the custom metadata field " - f"for model {model_id}." - ) - logger.debug(message) - - if not container_family: - raise AquaValueError( - f"{message}. For unverified Aqua models, container_family parameter should be " - f"set and value can be one of {', '.join(InferenceContainerTypeFamily.values())}." - ) from err - container_type_key = container_family + container_type_key = self._get_container_type_key( + model=model, + container_family=container_family + ) container_config = get_container_config() container_spec = container_config.get(ContainerSpec.CONTAINER_SPEC, {}).get( From 9f69f375625c6598ac6c348ee5dd5d51d0a433e2 Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Thu, 5 Sep 2024 17:07:36 -0700 Subject: [PATCH 12/19] ODSC-61986: Get evaluation shapes list form the service config. --- ads/aqua/config/config.py | 3 +- .../evaluation/evaluation_service_config.py | 2 +- ads/aqua/evaluation/evaluation.py | 79 +++++++---------- ads/aqua/extension/evaluation_handler.py | 7 +- ads/aqua/ui.py | 52 +++++------ .../test_data/config/evaluation_config.json | 2 +- ...evaluation_config_with_default_params.json | 2 +- .../with_extras/aqua/test_evaluation.py | 87 +++++++++++++++++-- tests/unitary/with_extras/aqua/test_ui.py | 24 ----- tests/unitary/with_extras/aqua/utils.py | 9 -- 10 files changed, 150 insertions(+), 117 deletions(-) diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index 1cdda1ac5..95f3e0d78 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -15,7 +15,7 @@ DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" -@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) +@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=1), timer=datetime.now)) def evaluation_service_config( container: Optional[str] = DEFAULT_EVALUATION_CONTAINER, ) -> EvaluationServiceConfig: @@ -27,6 +27,7 @@ def evaluation_service_config( EvaluationServiceConfig: The evaluation common config. """ + container = container or DEFAULT_EVALUATION_CONTAINER return EvaluationServiceConfig( **get_container_config() .get(ContainerSpec.CONTAINER_SPEC, {}) diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py index 4242ab608..053f507c6 100644 --- a/ads/aqua/config/evaluation/evaluation_service_config.py +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -250,7 +250,7 @@ class EvaluationServiceConfig(Serializable): """ version: Optional[str] = "1.0" - kind: Optional[str] = "evaluation" + kind: Optional[str] = "evaluation_service_config" report_params: Optional[ReportParams] = Field(default_factory=ReportParams) inference_params: Optional[InferenceParamsConfig] = Field( default_factory=InferenceParamsConfig diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index b484cff4c..f3eb5be2b 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -11,7 +11,7 @@ from datetime import datetime, timedelta from pathlib import Path from threading import Lock -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import oci from cachetools import TTLCache @@ -45,6 +45,7 @@ is_valid_ocid, upload_local_to_os, ) +from ads.aqua.config.config import evaluation_service_config from ads.aqua.constants import ( CONSOLE_LINK_RESOURCE_TYPE_MAPPING, EVALUATION_REPORT, @@ -171,7 +172,7 @@ def create( "Specify either a model or model deployment id." ) evaluation_source = None - eval_inference_configuration = None + eval_inference_configuration: Dict = {} if ( DataScienceResource.MODEL_DEPLOYMENT in create_aqua_evaluation_details.evaluation_source_id @@ -187,17 +188,26 @@ def create( runtime = ModelDeploymentContainerRuntime.from_dict( evaluation_source.runtime.to_dict() ) - inference_config = AquaContainerConfig.from_container_index_json( + container_config = AquaContainerConfig.from_container_index_json( enable_spec=True - ).inference - for container in inference_config.values(): - if container.name == runtime.image[:runtime.image.rfind(":")]: + ) + for ( + inference_container_family, + inference_container_info, + ) in container_config.inference.items(): + if ( + inference_container_info.name + == runtime.image[: runtime.image.rfind(":")] + ): eval_inference_configuration = ( - container.spec.evaluation_configuration + evaluation_service_config() + .get_merged_inference_params(inference_container_family) + .to_dict() ) + except Exception: logger.debug( - f"Could not load inference config details for the evaluation id: " + f"Could not load inference config details for the evaluation source id: " f"{create_aqua_evaluation_details.evaluation_source_id}. Please check if the container" f" runtime has the correct SMC image information." ) @@ -416,9 +426,7 @@ def create( report_path=create_aqua_evaluation_details.report_path, model_parameters=create_aqua_evaluation_details.model_parameters, metrics=create_aqua_evaluation_details.metrics, - inference_configuration=eval_inference_configuration.to_filtered_dict() - if eval_inference_configuration - else {}, + inference_configuration=eval_inference_configuration or {}, ) ).create(**kwargs) ## TODO: decide what parameters will be needed logger.debug( @@ -1225,45 +1233,24 @@ def _delete_job_and_model(job, model): f"Exception message: {ex}" ) - def load_evaluation_config(self, eval_id): + def load_evaluation_config(self, container: Optional[str] = None) -> Dict: """Loads evaluation config.""" + + # retrieve the evaluation config by container family name + evaluation_config = evaluation_service_config(container) + + # convert the new config representation to the old one return { - "model_params": { - "max_tokens": 500, - "temperature": 0.7, - "top_p": 1.0, - "top_k": 50, - "presence_penalty": 0.0, - "frequency_penalty": 0.0, - "stop": [], - }, + "model_params": evaluation_config.ui_config.model_params.default, "shape": { - "VM.Standard.E3.Flex": { - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, - }, - "VM.Standard.E4.Flex": { - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, - }, - "VM.Standard3.Flex": { - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, - }, - "VM.Optimized3.Flex": { - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, - }, - }, - "default": { - "ocpu": 8, - "memory_in_gbs": 128, - "block_storage_size": 200, + shape.name: shape.to_dict() + for shape in evaluation_config.ui_config.shapes }, + "default": ( + evaluation_config.ui_config.shapes[0].to_dict() + if len(evaluation_config.ui_config.shapes) > 0 + else {} + ), } def _get_attribute_from_model_metadata( diff --git a/ads/aqua/extension/evaluation_handler.py b/ads/aqua/extension/evaluation_handler.py index e3f2b5512..ed040f5c4 100644 --- a/ads/aqua/extension/evaluation_handler.py +++ b/ads/aqua/extension/evaluation_handler.py @@ -2,6 +2,7 @@ # Copyright (c) 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ +from typing import Optional from urllib.parse import urlparse from tornado.web import HTTPError @@ -30,7 +31,7 @@ def get(self, eval_id=""): return self.read(eval_id) @handle_exceptions - def post(self, *args, **kwargs): + def post(self, *args, **kwargs): # noqa """Handles post request for the evaluation APIs Raises @@ -117,10 +118,10 @@ class AquaEvaluationConfigHandler(AquaAPIhandler): """Handler for Aqua Evaluation Config REST APIs.""" @handle_exceptions - def get(self, model_id): + def get(self, container: Optional[str] = None, **kwargs): # noqa """Handle GET request.""" - return self.finish(AquaEvaluationApp().load_evaluation_config(model_id)) + return self.finish(AquaEvaluationApp().load_evaluation_config(container)) __handlers__ = [ diff --git a/ads/aqua/ui.py b/ads/aqua/ui.py index 07d8ae443..fd7fd91a2 100644 --- a/ads/aqua/ui.py +++ b/ads/aqua/ui.py @@ -84,9 +84,6 @@ class AquaContainerConfigSpec(DataClassSerializable): health_check_port: str = None env_vars: List[dict] = None restricted_params: List[str] = None - evaluation_configuration: AquaContainerEvaluationConfig = field( - default_factory=AquaContainerEvaluationConfig - ) @dataclass(repr=False) @@ -184,32 +181,37 @@ def from_container_index_json( family=container_type, platforms=platforms, model_formats=model_formats, - spec=AquaContainerConfigSpec( - cli_param=container_spec.get(ContainerSpec.CLI_PARM, ""), - server_port=container_spec.get( - ContainerSpec.SERVER_PORT, "" - ), - health_check_port=container_spec.get( - ContainerSpec.HEALTH_CHECK_PORT, "" - ), - env_vars=container_spec.get(ContainerSpec.ENV_VARS, []), - restricted_params=container_spec.get( - ContainerSpec.RESTRICTED_PARAMS, [] - ), - evaluation_configuration=AquaContainerEvaluationConfig.from_config( - container_spec.get( - ContainerSpec.EVALUATION_CONFIGURATION, {} - ) - ), - ) - if container_spec - else None, + spec=( + AquaContainerConfigSpec( + cli_param=container_spec.get( + ContainerSpec.CLI_PARM, "" + ), + server_port=container_spec.get( + ContainerSpec.SERVER_PORT, "" + ), + health_check_port=container_spec.get( + ContainerSpec.HEALTH_CHECK_PORT, "" + ), + env_vars=container_spec.get(ContainerSpec.ENV_VARS, []), + restricted_params=container_spec.get( + ContainerSpec.RESTRICTED_PARAMS, [] + ), + ) + if container_spec + else None + ), ) if container.get("type") == "inference": inference_items[container_type] = container_item - elif container_type == "odsc-llm-fine-tuning": + elif ( + container.get("type") == "fine-tune" + or container_type == "odsc-llm-fine-tuning" + ): finetune_items[container_type] = container_item - elif container_type == "odsc-llm-evaluate": + elif ( + container.get("type") == "evaluate" + or container_type == "odsc-llm-evaluate" + ): evaluate_items[container_type] = container_item return AquaContainerConfig( diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json index 4a1dd2b30..f108073f0 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config.json @@ -102,7 +102,7 @@ "inference_timeout": 120 } }, - "kind": "evaluation", + "kind": "evaluation_service_config", "report_params": { "default": {} }, diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json index 0ef9983c9..f6f7e4803 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json @@ -14,7 +14,7 @@ "inference_timeout": 120 } }, - "kind": "evaluation", + "kind": "evaluation_service_config", "report_params": { "default": {} }, diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index b75986d3d..f9c0123eb 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -22,6 +22,12 @@ AquaMissingKeyError, AquaRuntimeError, ) +from ads.aqua.config.evaluation.evaluation_service_config import ( + EvaluationServiceConfig, + ModelParamsConfig, + ShapeConfig, + UIConfig, +) from ads.aqua.constants import EVALUATION_REPORT_JSON, EVALUATION_REPORT_MD, UNKNOWN from ads.aqua.evaluation import AquaEvaluationApp from ads.aqua.evaluation.entities import ( @@ -887,16 +893,85 @@ def test_get_supported_metrics(self): for metric in response: assert check(metric_schema, metric) - def test_load_evaluation_config(self): - """Tests loading default config for evaluation. + @patch("ads.aqua.evaluation.evaluation.evaluation_service_config") + def test_load_evaluation_config(self, mock_evaluation_service_config): + """ + Tests loading default config for evaluation. This method currently hardcoded the return value. """ - from .utils import EvaluationConfigFormat as config_schema - from .utils import check - response = self.app.load_evaluation_config(eval_id=TestDataset.EVAL_ID) + test_evaluation_service_config = EvaluationServiceConfig( + ui_config=UIConfig( + model_params=ModelParamsConfig( + **{ + "default": { + "model": "odsc-llm", + "max_tokens": 500, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop": [], + } + } + ), + shapes=[ + ShapeConfig( + **{ + "name": "VM.Standard.E3.Flex", + "ocpu": 8, + "memory_in_gbs": 128, + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, + } + ) + ], + ) + ) + mock_evaluation_service_config.return_value = test_evaluation_service_config + + expected_result = { + "model_params": { + "model": "odsc-llm", + "max_tokens": 500, + "temperature": 0.7, + "top_p": 0.9, + "top_k": 50, + "presence_penalty": 0.0, + "frequency_penalty": 0.0, + "stop": [], + }, + "shape": { + "VM.Standard.E3.Flex": { + "name": "VM.Standard.E3.Flex", + "ocpu": 8, + "memory_in_gbs": 128, + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, + } + }, + "default": { + "name": "VM.Standard.E3.Flex", + "ocpu": 8, + "memory_in_gbs": 128, + "block_storage_size": 200, + "filter": { + "evaluation_container": ["odsc-llm-evaluate"], + "evaluation_target": ["datasciencemodeldeployment"], + }, + }, + } + + response = self.app.load_evaluation_config() assert isinstance(response, dict) - assert check(config_schema, response) + assert response == expected_result class TestAquaEvaluationList(unittest.TestCase): diff --git a/tests/unitary/with_extras/aqua/test_ui.py b/tests/unitary/with_extras/aqua/test_ui.py index 4a0c9b3b5..459b9bafd 100644 --- a/tests/unitary/with_extras/aqua/test_ui.py +++ b/tests/unitary/with_extras/aqua/test_ui.py @@ -502,14 +502,6 @@ def test_list_containers(self, mock_get_container_config): "health_check_port": "8080", "restricted_params": [], "server_port": "8080", - "evaluation_configuration": { - "inference_max_threads": 1, - "inference_rps": None, - "inference_timeout": None, - "inference_backoff_factor": None, - "inference_delay": 1, - "inference_retries": None, - }, }, }, { @@ -536,14 +528,6 @@ def test_list_containers(self, mock_get_container_config): "--trust-remote-code", ], "server_port": "8080", - "evaluation_configuration": { - "inference_max_threads": None, - "inference_rps": None, - "inference_timeout": None, - "inference_backoff_factor": None, - "inference_delay": None, - "inference_retries": None, - }, }, }, { @@ -569,14 +553,6 @@ def test_list_containers(self, mock_get_container_config): "--seed", ], "server_port": "8080", - "evaluation_configuration": { - "inference_max_threads": None, - "inference_rps": None, - "inference_timeout": None, - "inference_backoff_factor": None, - "inference_delay": None, - "inference_retries": None, - }, }, }, ], diff --git a/tests/unitary/with_extras/aqua/utils.py b/tests/unitary/with_extras/aqua/utils.py index f1b25fdf4..0a4299748 100644 --- a/tests/unitary/with_extras/aqua/utils.py +++ b/tests/unitary/with_extras/aqua/utils.py @@ -87,15 +87,6 @@ class SupportMetricsFormat(BaseFormat): args: dict -@dataclass -class EvaluationConfigFormat(BaseFormat): - """Evaluation config format.""" - - model_params: dict - shape: Dict[str, dict] - default: Dict[str, int] - - def check(conf_schema, conf): """Check if the format of the output dictionary is correct.""" try: From b868f2d1382e7baccf56b1dc766e83c6b15deb6b Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 6 Sep 2024 09:09:38 -0700 Subject: [PATCH 13/19] Implement Pass-Through Mechanism for Model Sampling Parameters --- ads/aqua/evaluation/entities.py | 1 + ads/aqua/evaluation/evaluation.py | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/ads/aqua/evaluation/entities.py b/ads/aqua/evaluation/entities.py index 6a00c0753..d626995a6 100644 --- a/ads/aqua/evaluation/entities.py +++ b/ads/aqua/evaluation/entities.py @@ -102,6 +102,7 @@ class ModelParams(DataClassSerializable): presence_penalty: Optional[float] = 0.0 frequency_penalty: Optional[float] = 0.0 stop: Optional[Union[str, List[str]]] = field(default_factory=list) + model: Optional[str] = "odsc-llm" @dataclass(repr=False) diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index f3eb5be2b..cb37a8964 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -46,6 +46,7 @@ upload_local_to_os, ) from ads.aqua.config.config import evaluation_service_config +from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig from ads.aqua.constants import ( CONSOLE_LINK_RESOURCE_TYPE_MAPPING, EVALUATION_REPORT, @@ -171,8 +172,19 @@ def create( f"Invalid evaluation source {create_aqua_evaluation_details.evaluation_source_id}. " "Specify either a model or model deployment id." ) + + # The model to evaluate evaluation_source = None + # The evaluation service config + evaluation_config: EvaluationServiceConfig = evaluation_service_config() + # The evaluation inference configuration. The inference configuration will be extracted + # based on the inferencing container family. eval_inference_configuration: Dict = {} + # The evaluation inference model sampling params. The system parameters that will not be + # visible for user, but will be applied implicitly for evaluation. The service model params + # will be extracted based on the container family and version. + eval_inference_service_model_params: Dict = {} + if ( DataScienceResource.MODEL_DEPLOYMENT in create_aqua_evaluation_details.evaluation_source_id @@ -200,9 +212,15 @@ def create( == runtime.image[: runtime.image.rfind(":")] ): eval_inference_configuration = ( - evaluation_service_config() - .get_merged_inference_params(inference_container_family) - .to_dict() + evaluation_config.get_merged_inference_params( + inference_container_family + ).to_dict() + ) + eval_inference_service_model_params = ( + evaluation_config.get_merged_inference_model_params( + inference_container_family, + inference_container_info.version, + ) ) except Exception: @@ -424,7 +442,10 @@ def create( container_image=container_image, dataset_path=evaluation_dataset_path, report_path=create_aqua_evaluation_details.report_path, - model_parameters=create_aqua_evaluation_details.model_parameters, + model_parameters={ + **eval_inference_service_model_params, + **create_aqua_evaluation_details.model_parameters, + }, metrics=create_aqua_evaluation_details.metrics, inference_configuration=eval_inference_configuration or {}, ) From ea10a423d2ff8d1a68641c2610f27fb3a4ec6c5d Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 6 Sep 2024 12:08:40 -0700 Subject: [PATCH 14/19] Removes the evaluation inference default parameters from the code. --- .../evaluation/evaluation_service_config.py | 39 ++++++------------- ...evaluation_config_with_default_params.json | 9 +---- 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/ads/aqua/config/evaluation/evaluation_service_config.py b/ads/aqua/config/evaluation/evaluation_service_config.py index 4242ab608..963b29e4c 100644 --- a/ads/aqua/config/evaluation/evaluation_service_config.py +++ b/ads/aqua/config/evaluation/evaluation_service_config.py @@ -10,14 +10,6 @@ from ads.aqua.config.utils.serializer import Serializable -# Constants -INFERENCE_RPS = 25 # Max RPS for inferencing deployed model. -INFERENCE_TIMEOUT = 120 -INFERENCE_MAX_THREADS = 10 # Maximum parallel threads for model inference. -INFERENCE_RETRIES = 3 -INFERENCE_BACKOFF_FACTOR = 3 -INFERENCE_DELAY = 0 - class ModelParamsOverrides(Serializable): """Defines overrides for model parameters, including exclusions and additional inclusions.""" @@ -54,13 +46,6 @@ class Config: class InferenceParams(Serializable): """Contains inference-related parameters with defaults.""" - inference_rps: Optional[int] = INFERENCE_RPS - inference_timeout: Optional[int] = INFERENCE_TIMEOUT - inference_max_threads: Optional[int] = INFERENCE_MAX_THREADS - inference_retries: Optional[int] = INFERENCE_RETRIES - inference_backoff_factor: Optional[float] = INFERENCE_BACKOFF_FACTOR - inference_delay: Optional[float] = INFERENCE_DELAY - class Config: extra = "allow" @@ -224,20 +209,18 @@ def search_shapes( ------- List[ShapeConfig]: A list of shapes that match the filters. """ - results = [] - for shape in self.shapes: - if ( - evaluation_container - and evaluation_container not in shape.filter.evaluation_container - ): - continue + return [ + shape + for shape in self.shapes if ( - evaluation_target - and evaluation_target not in shape.filter.evaluation_target - ): - continue - results.append(shape) - return results + not evaluation_container + or evaluation_container in shape.filter.evaluation_container + ) + and ( + not evaluation_target + or evaluation_target in shape.filter.evaluation_target + ) + ] class Config: extra = "ignore" diff --git a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json index 0ef9983c9..315e70ad1 100644 --- a/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json +++ b/tests/unitary/with_extras/aqua/test_data/config/evaluation_config_with_default_params.json @@ -5,14 +5,7 @@ }, "inference_params": { "containers": [], - "default": { - "inference_backoff_factor": 3, - "inference_delay": 0, - "inference_max_threads": 10, - "inference_retries": 3, - "inference_rps": 25, - "inference_timeout": 120 - } + "default": {} }, "kind": "evaluation", "report_params": { From 900069122746b870c8d41db8886e1d4b54f674cf Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 6 Sep 2024 13:03:32 -0700 Subject: [PATCH 15/19] Removes caching from the retrieving evaluation config. --- ads/aqua/config/config.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index 95f3e0d78..c56d0c3f0 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -3,11 +3,8 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ -from datetime import datetime, timedelta from typing import Optional -from cachetools import TTLCache, cached - from ads.aqua.common.entities import ContainerSpec from ads.aqua.common.utils import get_container_config from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig @@ -15,7 +12,6 @@ DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" -@cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=1), timer=datetime.now)) def evaluation_service_config( container: Optional[str] = DEFAULT_EVALUATION_CONTAINER, ) -> EvaluationServiceConfig: From 9a9569b00ef8e3fe262a69db1c4f30286a8aea6a Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Tue, 10 Sep 2024 10:05:16 -0700 Subject: [PATCH 16/19] Update THIRD_PARTY_LICENSES.txt --- THIRD_PARTY_LICENSES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index 1bd2e036c..6c320bca9 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -453,6 +453,12 @@ mlforecast * Source code: https://github.com/Nixtla/mlforecast * Project home: https://github.com/Nixtla/mlforecast +pydantic +* Copyright (c) 2017 to present Pydantic Services Inc. and individual contributors. +* License: The MIT License (MIT) +* Source code: https://github.com/pydantic/pydantic +* Project home: https://docs.pydantic.dev/latest/ + ======= =============================== Licenses =============================== ------------------------------------------------------------------------ From e3de1d09ec64ac444fe373fbe21c0645471026fa Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 13 Sep 2024 00:03:34 -0700 Subject: [PATCH 17/19] Fixes evaluation unit tests. --- ads/aqua/config/config.py | 2 +- ads/aqua/evaluation/evaluation.py | 8 ++++---- tests/unitary/with_extras/aqua/test_config.py | 4 ++-- .../unitary/with_extras/aqua/test_evaluation.py | 16 ++++++++++------ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/ads/aqua/config/config.py b/ads/aqua/config/config.py index c56d0c3f0..1cabc203c 100644 --- a/ads/aqua/config/config.py +++ b/ads/aqua/config/config.py @@ -12,7 +12,7 @@ DEFAULT_EVALUATION_CONTAINER = "odsc-llm-evaluate" -def evaluation_service_config( +def get_evaluation_service_config( container: Optional[str] = DEFAULT_EVALUATION_CONTAINER, ) -> EvaluationServiceConfig: """ diff --git a/ads/aqua/evaluation/evaluation.py b/ads/aqua/evaluation/evaluation.py index 3bdaf49dd..7f7349beb 100644 --- a/ads/aqua/evaluation/evaluation.py +++ b/ads/aqua/evaluation/evaluation.py @@ -45,7 +45,7 @@ is_valid_ocid, upload_local_to_os, ) -from ads.aqua.config.config import evaluation_service_config +from ads.aqua.config.config import get_evaluation_service_config from ads.aqua.config.evaluation.evaluation_service_config import EvaluationServiceConfig from ads.aqua.constants import ( CONSOLE_LINK_RESOURCE_TYPE_MAPPING, @@ -176,7 +176,7 @@ def create( # The model to evaluate evaluation_source = None # The evaluation service config - evaluation_config: EvaluationServiceConfig = evaluation_service_config() + evaluation_config: EvaluationServiceConfig = get_evaluation_service_config() # The evaluation inference configuration. The inference configuration will be extracted # based on the inferencing container family. eval_inference_configuration: Dict = {} @@ -931,7 +931,7 @@ def get_status(self, eval_id: str) -> dict: def get_supported_metrics(self) -> dict: """Gets a list of supported metrics for evaluation.""" return [ - item.to_dict() for item in evaluation_service_config().ui_config.metrics + item.to_dict() for item in get_evaluation_service_config().ui_config.metrics ] @telemetry(entry_point="plugin=evaluation&action=load_metrics", name="aqua") @@ -1218,7 +1218,7 @@ def load_evaluation_config(self, container: Optional[str] = None) -> Dict: """Loads evaluation config.""" # retrieve the evaluation config by container family name - evaluation_config = evaluation_service_config(container) + evaluation_config = get_evaluation_service_config(container) # convert the new config representation to the old one return { diff --git a/tests/unitary/with_extras/aqua/test_config.py b/tests/unitary/with_extras/aqua/test_config.py index 381b03116..4994fabe7 100644 --- a/tests/unitary/with_extras/aqua/test_config.py +++ b/tests/unitary/with_extras/aqua/test_config.py @@ -7,7 +7,7 @@ from unittest.mock import patch from ads.aqua.common.entities import ContainerSpec -from ads.aqua.config.config import evaluation_service_config +from ads.aqua.config.config import get_evaluation_service_config class TestConfig: @@ -32,7 +32,7 @@ def test_evaluation_service_config(self, mock_get_container_config): mock_get_container_config.return_value = expected_result - test_result = evaluation_service_config(container="test_container") + test_result = get_evaluation_service_config(container="test_container") assert ( test_result.to_dict() == expected_result[ContainerSpec.CONTAINER_SPEC]["test_container"] diff --git a/tests/unitary/with_extras/aqua/test_evaluation.py b/tests/unitary/with_extras/aqua/test_evaluation.py index 1d39e21b2..0a64732f7 100644 --- a/tests/unitary/with_extras/aqua/test_evaluation.py +++ b/tests/unitary/with_extras/aqua/test_evaluation.py @@ -426,6 +426,7 @@ def assert_payload(self, response, response_type): continue assert rdict.get(attr), f"{attr} is empty" + @patch("ads.aqua.evaluation.evaluation.get_evaluation_service_config") @patch.object(Job, "run") @patch("ads.jobs.ads_job.Job.name", new_callable=PropertyMock) @patch("ads.jobs.ads_job.Job.id", new_callable=PropertyMock) @@ -444,6 +445,7 @@ def test_create_evaluation( mock_job_id, mock_job_name, mock_job_run, + mock_get_evaluation_service_config, ): foundation_model = MagicMock() foundation_model.display_name = "test_foundation_model" @@ -473,6 +475,8 @@ def test_create_evaluation( evaluation_job_run.lifecycle_state = "IN_PROGRESS" mock_job_run.return_value = evaluation_job_run + mock_get_evaluation_service_config.return_value = EvaluationServiceConfig() + self.app.ds_client.update_model = MagicMock() self.app.ds_client.update_model_provenance = MagicMock() @@ -883,8 +887,8 @@ def test_extract_job_lifecycle_details(self, input, expect_output): msg = self.app._extract_job_lifecycle_details(input) assert msg == expect_output, msg - @patch("ads.aqua.evaluation.evaluation.evaluation_service_config") - def test_get_supported_metrics(self, mock_evaluation_service_config): + @patch("ads.aqua.evaluation.evaluation.get_evaluation_service_config") + def test_get_supported_metrics(self, mock_get_evaluation_service_config): """ Tests getting a list of supported metrics for evaluation. """ @@ -905,7 +909,7 @@ def test_get_supported_metrics(self, mock_evaluation_service_config): ] ) ) - mock_evaluation_service_config.return_value = test_evaluation_service_config + mock_get_evaluation_service_config.return_value = test_evaluation_service_config response = self.app.get_supported_metrics() assert isinstance(response, list) assert len(response) == len(test_evaluation_service_config.ui_config.metrics) @@ -913,8 +917,8 @@ def test_get_supported_metrics(self, mock_evaluation_service_config): item.to_dict() for item in test_evaluation_service_config.ui_config.metrics ] - @patch("ads.aqua.evaluation.evaluation.evaluation_service_config") - def test_load_evaluation_config(self, mock_evaluation_service_config): + @patch("ads.aqua.evaluation.evaluation.get_evaluation_service_config") + def test_load_evaluation_config(self, mock_get_evaluation_service_config): """ Tests loading default config for evaluation. This method currently hardcoded the return value. @@ -952,7 +956,7 @@ def test_load_evaluation_config(self, mock_evaluation_service_config): ], ) ) - mock_evaluation_service_config.return_value = test_evaluation_service_config + mock_get_evaluation_service_config.return_value = test_evaluation_service_config expected_result = { "model_params": { From 882a215332d4d31f0c85e41aa0df9b26193822c0 Mon Sep 17 00:00:00 2001 From: Vipul Date: Fri, 20 Sep 2024 17:30:38 -0700 Subject: [PATCH 18/19] added compatibility check --- ads/aqua/common/utils.py | 14 ++++++++++---- ads/aqua/extension/common_handler.py | 5 ++--- ads/aqua/extension/common_ws_msg_handler.py | 4 ++-- ads/aqua/extension/utils.py | 10 +++++++++- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/ads/aqua/common/utils.py b/ads/aqua/common/utils.py index a59dac646..ede4ddb88 100644 --- a/ads/aqua/common/utils.py +++ b/ads/aqua/common/utils.py @@ -58,6 +58,7 @@ ) from ads.aqua.data import AquaResourceIdentifier from ads.common.auth import AuthState, default_signer +from ads.common.decorator.threaded import threaded from ads.common.extended_enum import ExtendedEnumMeta from ads.common.object_storage_details import ObjectStorageDetails from ads.common.oci_resource import SEARCH_TYPE, OCIResource @@ -225,6 +226,7 @@ def read_file(file_path: str, **kwargs) -> str: return UNKNOWN +@threaded() def load_config(file_path: str, config_file_name: str, **kwargs) -> dict: artifact_path = f"{file_path.rstrip('/')}/{config_file_name}" signer = default_signer() if artifact_path.startswith("oci://") else {} @@ -1065,11 +1067,15 @@ def get_hf_model_info(repo_id: str) -> ModelInfo: @cached(cache=TTLCache(maxsize=1, ttl=timedelta(hours=5), timer=datetime.now)) -def list_hf_models(query:str) -> List[str]: +def list_hf_models(query: str) -> List[str]: try: - models= HfApi().list_models(model_name=query,task="text-generation",sort="downloads",direction=-1,limit=20) + models = HfApi().list_models( + model_name=query, + task="text-generation", + sort="downloads", + direction=-1, + limit=20, + ) return [model.id for model in models if model.disabled is None] except HfHubHTTPError as err: raise format_hf_custom_error_message(err) from err - - diff --git a/ads/aqua/extension/common_handler.py b/ads/aqua/extension/common_handler.py index c114b3a14..cc9a2f663 100644 --- a/ads/aqua/extension/common_handler.py +++ b/ads/aqua/extension/common_handler.py @@ -11,16 +11,15 @@ from huggingface_hub.utils import LocalTokenNotFoundError from tornado.web import HTTPError -from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.errors import AquaResourceAccessError, AquaRuntimeError from ads.aqua.common.utils import ( - fetch_service_compartment, get_huggingface_login_timeout, known_realm, ) from ads.aqua.extension.base_handler import AquaAPIhandler from ads.aqua.extension.errors import Errors +from ads.aqua.extension.utils import ui_compatability_check class ADSVersionHandler(AquaAPIhandler): @@ -51,7 +50,7 @@ def get(self): AquaResourceAccessError: raised when aqua is not accessible in the given session/region. """ - if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment(): + if ui_compatability_check(): return self.finish({"status": "ok"}) elif known_realm(): return self.finish({"status": "compatible"}) diff --git a/ads/aqua/extension/common_ws_msg_handler.py b/ads/aqua/extension/common_ws_msg_handler.py index 71cb545f4..cc54af1de 100644 --- a/ads/aqua/extension/common_ws_msg_handler.py +++ b/ads/aqua/extension/common_ws_msg_handler.py @@ -7,7 +7,6 @@ from importlib import metadata from typing import List, Union -from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID, fetch_service_compartment from ads.aqua.common.decorator import handle_exceptions from ads.aqua.common.errors import AquaResourceAccessError from ads.aqua.common.utils import known_realm @@ -17,6 +16,7 @@ CompatibilityCheckResponse, RequestResponseType, ) +from ads.aqua.extension.utils import ui_compatability_check class AquaCommonWsMsgHandler(AquaWSMsgHandler): @@ -39,7 +39,7 @@ def process(self) -> Union[AdsVersionResponse, CompatibilityCheckResponse]: ) return response if request.get("kind") == "CompatibilityCheck": - if ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment(): + if ui_compatability_check(): return CompatibilityCheckResponse( message_id=request.get("message_id"), kind=RequestResponseType.CompatibilityCheck, diff --git a/ads/aqua/extension/utils.py b/ads/aqua/extension/utils.py index c757d91e2..e39b35b53 100644 --- a/ads/aqua/extension/utils.py +++ b/ads/aqua/extension/utils.py @@ -1,12 +1,15 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # Copyright (c) 2024 Oracle and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ from dataclasses import fields +from datetime import datetime, timedelta from typing import Dict, Optional +from cachetools import TTLCache, cached from tornado.web import HTTPError +from ads.aqua import ODSC_MODEL_COMPARTMENT_OCID +from ads.aqua.common.utils import fetch_service_compartment from ads.aqua.extension.errors import Errors @@ -21,3 +24,8 @@ def validate_function_parameters(data_class, input_data: Dict): raise HTTPError( 400, Errors.MISSING_REQUIRED_PARAMETER.format(required_parameter) ) + + +@cached(cache=TTLCache(maxsize=1, ttl=timedelta(minutes=1), timer=datetime.now)) +def ui_compatability_check(): + return ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment() From 2a27ef6708c2cf1deca779fc2541917904fb29b8 Mon Sep 17 00:00:00 2001 From: Vipul Date: Mon, 23 Sep 2024 10:37:09 -0700 Subject: [PATCH 19/19] add comments and fix tests --- ads/aqua/extension/utils.py | 3 +++ .../with_extras/aqua/test_common_handler.py | 25 ++++++++++++++----- .../unitary/with_extras/aqua/test_handlers.py | 3 ++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ads/aqua/extension/utils.py b/ads/aqua/extension/utils.py index e39b35b53..90787beb6 100644 --- a/ads/aqua/extension/utils.py +++ b/ads/aqua/extension/utils.py @@ -28,4 +28,7 @@ def validate_function_parameters(data_class, input_data: Dict): @cached(cache=TTLCache(maxsize=1, ttl=timedelta(minutes=1), timer=datetime.now)) def ui_compatability_check(): + """This method caches the service compartment OCID details that is set by either the environment variable or if + fetched from the configuration. The cached result is returned when multiple calls are made in quick succession + from the UI to avoid multiple config file loads.""" return ODSC_MODEL_COMPARTMENT_OCID or fetch_service_compartment() diff --git a/tests/unitary/with_extras/aqua/test_common_handler.py b/tests/unitary/with_extras/aqua/test_common_handler.py index 88e3e6e06..ec0590b07 100644 --- a/tests/unitary/with_extras/aqua/test_common_handler.py +++ b/tests/unitary/with_extras/aqua/test_common_handler.py @@ -15,6 +15,7 @@ import ads.config from ads.aqua.constants import AQUA_GA_LIST from ads.aqua.extension.common_handler import CompatibilityCheckHandler +from ads.aqua.extension.utils import ui_compatability_check class TestDataset: @@ -28,6 +29,9 @@ def setUp(self, ipython_init_mock) -> None: self.common_handler = CompatibilityCheckHandler(MagicMock(), MagicMock()) self.common_handler.request = MagicMock() + def tearDown(self) -> None: + ui_compatability_check.cache_clear() + def test_get_ok(self): """Test to check if ok is returned when ODSC_MODEL_COMPARTMENT_OCID is set.""" with patch.dict( @@ -36,15 +40,22 @@ def test_get_ok(self): ): reload(ads.config) reload(ads.aqua) + reload(ads.aqua.extension.utils) reload(ads.aqua.extension.common_handler) with patch( "ads.aqua.extension.base_handler.AquaAPIhandler.finish" ) as mock_finish: - mock_finish.side_effect = lambda x: x - self.common_handler.request.path = "aqua/hello" - result = self.common_handler.get() - assert result["status"] == "ok" + with patch( + "ads.aqua.extension.utils.fetch_service_compartment" + ) as mock_fetch_service_compartment: + mock_fetch_service_compartment.return_value = ( + TestDataset.SERVICE_COMPARTMENT_ID + ) + mock_finish.side_effect = lambda x: x + self.common_handler.request.path = "aqua/hello" + result = self.common_handler.get() + assert result["status"] == "ok" def test_get_compatible_status(self): """Test to check if compatible is returned when ODSC_MODEL_COMPARTMENT_OCID is not set @@ -55,12 +66,13 @@ def test_get_compatible_status(self): ): reload(ads.config) reload(ads.aqua) + reload(ads.aqua.extension.utils) reload(ads.aqua.extension.common_handler) with patch( "ads.aqua.extension.base_handler.AquaAPIhandler.finish" ) as mock_finish: with patch( - "ads.aqua.extension.common_handler.fetch_service_compartment" + "ads.aqua.extension.utils.fetch_service_compartment" ) as mock_fetch_service_compartment: mock_fetch_service_compartment.return_value = None mock_finish.side_effect = lambda x: x @@ -77,12 +89,13 @@ def test_raise_not_compatible_error(self): ): reload(ads.config) reload(ads.aqua) + reload(ads.aqua.extension.utils) reload(ads.aqua.extension.common_handler) with patch( "ads.aqua.extension.base_handler.AquaAPIhandler.finish" ) as mock_finish: with patch( - "ads.aqua.extension.common_handler.fetch_service_compartment" + "ads.aqua.extension.utils.fetch_service_compartment" ) as mock_fetch_service_compartment: mock_fetch_service_compartment.return_value = None mock_finish.side_effect = lambda x: x diff --git a/tests/unitary/with_extras/aqua/test_handlers.py b/tests/unitary/with_extras/aqua/test_handlers.py index 97c5660f7..74b9853b4 100644 --- a/tests/unitary/with_extras/aqua/test_handlers.py +++ b/tests/unitary/with_extras/aqua/test_handlers.py @@ -13,7 +13,6 @@ from notebook.base.handlers import APIHandler, IPythonHandler from oci.exceptions import ServiceError from parameterized import parameterized -from tornado.httpserver import HTTPRequest from tornado.httputil import HTTPServerRequest from tornado.web import Application, HTTPError @@ -191,6 +190,7 @@ def setUpClass(cls): reload(ads.config) reload(ads.aqua) + reload(ads.aqua.extension.utils) reload(ads.aqua.extension.common_handler) @classmethod @@ -200,6 +200,7 @@ def tearDownClass(cls): reload(ads.config) reload(ads.aqua) + reload(ads.aqua.extension.utils) reload(ads.aqua.extension.common_handler) @parameterized.expand(