From 1433210710b577704eb7b99fd15f8ecab51b938f Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Tue, 30 May 2023 14:47:36 -0700 Subject: [PATCH 01/20] Added validation for driver and executor shape details. --- ads/jobs/builders/infrastructure/dataflow.py | 12 ++++ .../default_setup/jobs/test_jobs_dataflow.py | 57 +++++++++++++++++-- 2 files changed, 63 insertions(+), 6 deletions(-) diff --git a/ads/jobs/builders/infrastructure/dataflow.py b/ads/jobs/builders/infrastructure/dataflow.py index 3f70c6ba9..ddff9c604 100644 --- a/ads/jobs/builders/infrastructure/dataflow.py +++ b/ads/jobs/builders/infrastructure/dataflow.py @@ -860,6 +860,18 @@ def create(self, runtime: DataFlowRuntime, **kwargs) -> "DataFlow": raise ValueError( "Compartment id is required. Specify compartment id via 'with_compartment_id()'." ) + if "executor_shape" not in payload: + payload["executor_shape"] = DEFAULT_SHAPE + if "driver_shape" not in payload: + payload["driver_shape"] = DEFAULT_SHAPE + executor_shape = payload["executor_shape"] + executor_shape_config = payload.get("executor_shape_config", {}) + driver_shape = payload["driver_shape"] + driver_shape_config = payload.get("driver_shape_config", {}) + if executor_shape != driver_shape: + raise ValueError("`executor_shape` and `driver_shape` must be from the same shape family.") + if (not executor_shape.endswith("Flex") and executor_shape_config) or (not driver_shape.endswith("Flex") and driver_shape_config): + raise ValueError("Shape config is not required for non flex shape from user end.") payload.pop("id", None) logger.debug(f"Creating a DataFlow Application with payload {payload}") self.df_app = DataFlowApp(**payload).create() diff --git a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py index bd6d220f9..deb36f3bd 100644 --- a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py +++ b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py @@ -37,9 +37,9 @@ arguments=["test-df"], compartment_id="ocid1.compartment.oc1..", display_name="test-df", - driver_shape="VM.Standard2.1", + driver_shape="VM.Standard.E4.Flex", driver_shape_config={"memory_in_gbs": 1, "ocpus": 16}, - executor_shape="VM.Standard2.1", + executor_shape="VM.Standard.E4.Flex", executor_shape_config={"memory_in_gbs": 1, "ocpus": 16}, file_uri="oci://test_bucket@test_namespace/test-dataflow/test-dataflow.py", num_executors=1, @@ -124,7 +124,7 @@ def test_create_delete(self, mock_to_dict, mock_client): df.lifecycle_state == oci.data_flow.models.Application.LIFECYCLE_STATE_DELETED ) - assert len(df.to_yaml()) == 557 + assert len(df.to_yaml()) == 567 def test_create_df_app_with_default_display_name( self, @@ -403,8 +403,8 @@ def test_create_from_id(self, mock_from_ocid): mock_from_ocid.return_value = Application(**SAMPLE_PAYLOAD) df = DataFlow.from_id("ocid1.datasciencejob.oc1.iad.") assert df.name == "test-df" - assert df.driver_shape == "VM.Standard2.1" - assert df.executor_shape == "VM.Standard2.1" + assert df.driver_shape == "VM.Standard.E4.Flex" + assert df.executor_shape == "VM.Standard.E4.Flex" assert df.private_endpoint_id == "test_private_endpoint" assert ( @@ -424,7 +424,7 @@ def test_create_from_id(self, mock_from_ocid): def test_to_and_from_dict(self, df): df_dict = df.to_dict() assert df_dict["spec"]["numExecutors"] == 2 - assert df_dict["spec"]["driverShape"] == "VM.Standard2.1" + assert df_dict["spec"]["driverShape"] == "VM.Standard.E4.Flex" assert df_dict["spec"]["logsBucketUri"] == "oci://test_bucket@test_namespace/" assert df_dict["spec"]["privateEndpointId"] == "test_private_endpoint" assert df_dict["spec"]["driverShapeConfig"] == {"memoryInGBs": 1, "ocpus": 16} @@ -444,6 +444,51 @@ def test_to_and_from_dict(self, df): assert df3_dict["spec"]["sparkVersion"] == "3.2.1" assert df3_dict["spec"]["numExecutors"] == 2 + def test_shape_and_details(self, mock_to_dict, mock_client, df): + df.with_driver_shape( + "VM.Standard2.1" + ).with_executor_shape( + "VM.Standard2.8" + ) + + rt = ( + DataFlowRuntime() + .with_script_uri(SAMPLE_PAYLOAD["file_uri"]) + .with_archive_uri(SAMPLE_PAYLOAD["archive_uri"]) + .with_custom_conda( + "oci://my_bucket@my_namespace/conda_environments/cpu/PySpark 3.0 and Data Flow/5.0/pyspark30_p37_cpu_v5" + ) + .with_overwrite(True) + ) + + with pytest.raises( + ValueError, + match="`executor_shape` and `driver_shape` must be from the same shape family." + ): + with patch.object(DataFlowApp, "client", mock_client): + with patch.object(DataFlowApp, "to_dict", mock_to_dict): + df.create(rt) + + df.with_driver_shape( + "VM.Standard2.1" + ).with_driver_shape_config( + memory_in_gbs=SAMPLE_PAYLOAD["driver_shape_config"]["memory_in_gbs"], + ocpus=SAMPLE_PAYLOAD["driver_shape_config"]["ocpus"], + ).with_executor_shape( + "VM.Standard2.1" + ).with_executor_shape_config( + memory_in_gbs=SAMPLE_PAYLOAD["executor_shape_config"]["memory_in_gbs"], + ocpus=SAMPLE_PAYLOAD["executor_shape_config"]["ocpus"], + ) + + with pytest.raises( + ValueError, + match="Shape config is not required for non flex shape from user end." + ): + with patch.object(DataFlowApp, "client", mock_client): + with patch.object(DataFlowApp, "to_dict", mock_to_dict): + df.create(rt) + class TestDataFlowNotebookRuntime: @pytest.mark.skipif( From 6ed266e41ee7a0347e170789b6c97c5b357b801a Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 2 Jun 2023 13:17:34 -0700 Subject: [PATCH 02/20] Updated pr. --- ads/jobs/builders/infrastructure/dataflow.py | 41 +++++++++++++++---- .../default_setup/jobs/test_jobs_dataflow.py | 4 +- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/ads/jobs/builders/infrastructure/dataflow.py b/ads/jobs/builders/infrastructure/dataflow.py index ddff9c604..bc36b555e 100644 --- a/ads/jobs/builders/infrastructure/dataflow.py +++ b/ads/jobs/builders/infrastructure/dataflow.py @@ -42,6 +42,13 @@ DEFAULT_SPARK_VERSION = "3.2.1" DEFAULT_NUM_EXECUTORS = 1 DEFAULT_SHAPE = "VM.Standard.E3.Flex" +DATAFLOW_SHAPE_FAMILY = [ + "Standard.E3", + "Standard.E4", + "Standard3", + "Standard.A1", + "Standard2" +] def conda_pack_name_to_dataflow_config(conda_uri): @@ -860,6 +867,15 @@ def create(self, runtime: DataFlowRuntime, **kwargs) -> "DataFlow": raise ValueError( "Compartment id is required. Specify compartment id via 'with_compartment_id()'." ) + self._validate_shapes(payload) + payload.pop("id", None) + logger.debug(f"Creating a DataFlow Application with payload {payload}") + self.df_app = DataFlowApp(**payload).create() + self.with_id(self.df_app.id) + return self + + @staticmethod + def _validate_shapes(payload: Dict): if "executor_shape" not in payload: payload["executor_shape"] = DEFAULT_SHAPE if "driver_shape" not in payload: @@ -868,15 +884,22 @@ def create(self, runtime: DataFlowRuntime, **kwargs) -> "DataFlow": executor_shape_config = payload.get("executor_shape_config", {}) driver_shape = payload["driver_shape"] driver_shape_config = payload.get("driver_shape_config", {}) - if executor_shape != driver_shape: - raise ValueError("`executor_shape` and `driver_shape` must be from the same shape family.") - if (not executor_shape.endswith("Flex") and executor_shape_config) or (not driver_shape.endswith("Flex") and driver_shape_config): - raise ValueError("Shape config is not required for non flex shape from user end.") - payload.pop("id", None) - logger.debug(f"Creating a DataFlow Application with payload {payload}") - self.df_app = DataFlowApp(**payload).create() - self.with_id(self.df_app.id) - return self + same_shape_family = False + for shape in DATAFLOW_SHAPE_FAMILY: + if shape in executor_shape and shape in driver_shape: + same_shape_family = True + break + if not same_shape_family: + raise ValueError( + "`executor_shape` and `driver_shape` must be from the same shape family." + ) + if ( + (not executor_shape.endswith("Flex") and executor_shape_config) + or (not driver_shape.endswith("Flex") and driver_shape_config) + ): + raise ValueError( + "Shape config is not required for non flex shape from user end." + ) @staticmethod def _upload_file(local_path, bucket, overwrite=False): diff --git a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py index deb36f3bd..739a4cae6 100644 --- a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py +++ b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py @@ -448,7 +448,7 @@ def test_shape_and_details(self, mock_to_dict, mock_client, df): df.with_driver_shape( "VM.Standard2.1" ).with_executor_shape( - "VM.Standard2.8" + "VM.Standard.E4.Flex" ) rt = ( @@ -475,7 +475,7 @@ def test_shape_and_details(self, mock_to_dict, mock_client, df): memory_in_gbs=SAMPLE_PAYLOAD["driver_shape_config"]["memory_in_gbs"], ocpus=SAMPLE_PAYLOAD["driver_shape_config"]["ocpus"], ).with_executor_shape( - "VM.Standard2.1" + "VM.Standard2.16" ).with_executor_shape_config( memory_in_gbs=SAMPLE_PAYLOAD["executor_shape_config"]["memory_in_gbs"], ocpus=SAMPLE_PAYLOAD["executor_shape_config"]["ocpus"], From 5294fef1eb31cd506fb9ab053449a00600aa6ad9 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 2 Jun 2023 15:48:53 -0700 Subject: [PATCH 03/20] Support uploading large artifact for model deployment. --- ads/model/deployment/model_deployment.py | 40 +++- .../deployment/model_deployment_runtime.py | 201 ++++++++++++++++++ .../test_model_deployment_v2.py | 138 ++++++++---- 3 files changed, 323 insertions(+), 56 deletions(-) diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index cc533e6d2..e04bc8f03 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -44,7 +44,7 @@ ) from ads.common import utils as ads_utils from .common import utils -from .common.utils import OCIClientManager, State +from .common.utils import State from .model_deployment_properties import ModelDeploymentProperties from oci.data_science.models import ( LogDetails, @@ -70,6 +70,7 @@ MODEL_DEPLOYMENT_INSTANCE_COUNT = 1 MODEL_DEPLOYMENT_BANDWIDTH_MBPS = 10 +MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB class ModelDeploymentLogType: PREDICT = "predict" @@ -200,7 +201,10 @@ class ModelDeployment(Builder): ... .with_health_check_port() ... .with_env({"key":"value"}) ... .with_deployment_mode("HTTPS_ONLY") - ... .with_model_uri()) + ... .with_model_uri() + ... .with_bucket_uri() + ... .with_auth() + ... .with_timeout()) ... ) ... ) >>> ds_model_deployment.deploy() @@ -1563,15 +1567,31 @@ def _build_model_deployment_configuration_details(self) -> Dict: model_id = runtime.model_uri if not model_id.startswith("ocid"): - model_id = OCIClientManager().prepare_artifact( - model_uri=runtime.model_uri, - properties=dict( - display_name=self.display_name, - compartment_id=self.infrastructure.compartment_id - or COMPARTMENT_OCID, - project_id=self.infrastructure.project_id or PROJECT_OCID, - ), + if ads_utils.folder_size(runtime.model_uri) > MAX_ARTIFACT_SIZE_IN_BYTES: + if not runtime.bucket_uri: + raise ValueError( + f"The model artifacts size is greater than `{MAX_ARTIFACT_SIZE_IN_BYTES}`. " + "The `bucket_uri` needs to be specified to copy artifacts to the object storage bucket. " + "Example: `runtime.with_bucket_uri(oci://@/prefix/)`" + ) + + from ads.model.datascience_model import DataScienceModel + + dsc_model = DataScienceModel( + name=self.display_name, + compartment_id=self.infrastructure.compartment_id + or COMPARTMENT_OCID, + project_id=self.infrastructure.project_id or PROJECT_OCID, + artifact=runtime.model_uri, + ).upload_artifact( + bucket_uri=runtime.bucket_uri, + auth=runtime.auth, + region=runtime.region, + overwrite_existing_artifact=runtime.overwrite_existing_artifact, + remove_existing_artifact=runtime.remove_existing_artifact, + timeout=runtime.timeout ) + model_id = dsc_model.id model_configuration_details = { infrastructure.CONST_BANDWIDTH_MBPS: infrastructure.bandwidth_mbps diff --git a/ads/model/deployment/model_deployment_runtime.py b/ads/model/deployment/model_deployment_runtime.py index 6be401336..e6c743652 100644 --- a/ads/model/deployment/model_deployment_runtime.py +++ b/ads/model/deployment/model_deployment_runtime.py @@ -36,6 +36,18 @@ class ModelDeploymentRuntime(Builder): The output stream ids of model deployment. model_uri: str The model uri of model deployment. + bucket_uri: str + The OCI Object Storage URI where large size model artifacts will be copied to. + auth: Dict + The default authentication is set using `ads.set_auth` API. + region: str + The destination Object Storage bucket region. + overwrite_existing_artifact: bool + Whether overwrite existing target bucket artifact or not. + remove_existing_artifact: bool + Whether artifacts uploaded to object storage bucket need to be removed or not. + timeout: int + The connection timeout in seconds for the client. Methods ------- @@ -49,6 +61,18 @@ class ModelDeploymentRuntime(Builder): Sets the output stream ids of model deployment with_model_uri(model_uri) Sets the model uri of model deployment + with_bucket_uri(bucket_uri) + Sets the bucket uri when uploading large size model. + with_auth(auth) + Sets the default authentication when uploading large size model. + with_region(region) + Sets the region when uploading large size model. + with_overwrite_existing_artifact(overwrite_existing_artifact) + Sets whether to overwrite existing artifact when uploading large size model. + with_remove_existing_artifact(remove_existing_artifact) + Sets whether to remove existing artifact when uploading large size model. + with_timeout(timeout) + Sets the connection timeout when uploading large size model. """ CONST_MODEL_ID = "modelId" @@ -60,6 +84,12 @@ class ModelDeploymentRuntime(Builder): CONST_INPUT_STREAM_IDS = "inputStreamIds" CONST_OUTPUT_STREAM_IDS = "outputStreamIds" CONST_ENVIRONMENT_CONFIG_DETAILS = "environmentConfigurationDetails" + CONST_BUCKET_URI = "bucketUri" + CONST_AUTH = "auth" + CONST_REGION = "region" + CONST_OVERWRITE_EXISTING_ARTIFACT = "overwriteExistingArtifact" + CONST_REMOVE_EXISTING_ARTIFACT = "removeExistingArtifact" + CONST_TIMEOUT = "timeout" attribute_map = { CONST_ENV: "env", @@ -68,6 +98,12 @@ class ModelDeploymentRuntime(Builder): CONST_OUTPUT_STREAM_IDS: "output_stream_ids", CONST_DEPLOYMENT_MODE: "deployment_mode", CONST_MODEL_URI: "model_uri", + CONST_BUCKET_URI: "bucket_uri", + CONST_AUTH: "auth", + CONST_REGION: "region", + CONST_OVERWRITE_EXISTING_ARTIFACT: "overwrite_existing_artifact", + CONST_REMOVE_EXISTING_ARTIFACT: "remove_existing_artifact", + CONST_TIMEOUT: "timeout" } ENVIRONMENT_CONFIG_DETAILS_PATH = ( @@ -236,7 +272,172 @@ def with_model_uri(self, model_uri: str) -> "ModelDeploymentRuntime": The ModelDeploymentRuntime instance (self). """ return self.set_spec(self.CONST_MODEL_URI, model_uri) + + @property + def bucket_uri(self) -> str: + """The bucket uri of model. + + Returns + ------- + str + The bucket uri of model. + """ + return self.get_spec(self.CONST_BUCKET_URI, None) + + def with_bucket_uri(self, bucket_uri: str) -> "ModelDeploymentRuntime": + """Sets the bucket uri of model. + + Parameters + ---------- + bucket_uri: str + The bucket uri of model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_BUCKET_URI, bucket_uri) + + @property + def auth(self) -> Dict: + """The auth when uploading large-size model. + + Returns + ------- + Dict + The auth when uploading large-size model. + """ + return self.get_spec(self.CONST_AUTH, {}) + + def with_auth(self, auth: Dict) -> "ModelDeploymentRuntime": + """Sets the auth when uploading large-size model. + + Parameters + ---------- + auth: Dict + The auth when uploading large-size model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_AUTH, auth) + + @property + def region(self) -> str: + """The region when uploading large-size model. + Returns + ------- + str + The region when uploading large-size model. + """ + return self.get_spec(self.CONST_REGION, None) + + def with_region(self, region: str) -> "ModelDeploymentRuntime": + """Sets the region when uploading large-size model. + + Parameters + ---------- + region: str + The region when uploading large-size model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_REGION, region) + + @property + def overwrite_existing_artifact(self) -> bool: + """Overwrite existing artifact when uploading large size model. + + Returns + ------- + bool + Overwrite existing artifact when uploading large size model. + """ + return self.get_spec(self.CONST_OVERWRITE_EXISTING_ARTIFACT, True) + + def with_overwrite_existing_artifact( + self, + overwrite_existing_artifact: bool + ) -> "ModelDeploymentRuntime": + """Sets whether to overwrite existing artifact when uploading large size model. + + Parameters + ---------- + overwrite_existing_artifact: bool + Overwrite existing artifact when uploading large size model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec( + self.CONST_OVERWRITE_EXISTING_ARTIFACT, + overwrite_existing_artifact + ) + + @property + def remove_existing_artifact(self) -> bool: + """Remove existing artifact when uploading large size model. + + Returns + ------- + bool + Remove existing artifact when uploading large size model. + """ + return self.get_spec(self.CONST_REMOVE_EXISTING_ARTIFACT, True) + + def with_remove_existing_artifact( + self, + remove_existing_artifact: bool + ) -> "ModelDeploymentRuntime": + """Sets whether to remove existing artifact when uploading large size model. + + Parameters + ---------- + remove_existing_artifact: bool + Remove existing artifact when uploading large size model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_REMOVE_EXISTING_ARTIFACT, remove_existing_artifact) + + @property + def timeout(self) -> int: + """The timeout when uploading large-size model. + + Returns + ------- + int + The timeout when uploading large-size model. + """ + return self.get_spec(self.CONST_TIMEOUT, None) + + def with_timeout(self, timeout: int) -> "ModelDeploymentRuntime": + """Sets the timeout when uploading large-size model. + + Parameters + ---------- + timeout: int + The timeout when uploading large-size model. + + Returns + ------- + ModelDeploymentRuntime + The ModelDeploymentRuntime instance (self). + """ + return self.set_spec(self.CONST_TIMEOUT, timeout) + def init(self) -> "ModelDeploymentRuntime": """Initializes a starter specification for the runtime. diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index fe2e4e5a5..57bb5e241 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -14,8 +14,9 @@ from ads.common.oci_datascience import OCIDataScienceMixin from ads.common.oci_logging import ConsolidatedLog, OCILog from ads.common.oci_mixin import OCIModelMixin -from ads.model.deployment.common.utils import OCIClientManager, State +from ads.model.deployment.common.utils import State from ads.common.oci_datascience import DSCNotebookSession +from ads.model.datascience_model import DataScienceModel from ads.model.deployment.model_deployment import ( ModelDeployment, @@ -534,22 +535,17 @@ def test_model_deployment_to_dict(self): }, } - @patch.object(OCIClientManager, "prepare_artifact") - def test_build_model_deployment_configuration_details(self, mock_prepare_artifact): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + @patch.object(DataScienceModel, "upload_artifact") + def test_build_model_deployment_configuration_details(self, mock_upload_artifact): + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model model_deployment = self.initialize_model_deployment() model_deployment_configuration_details = ( model_deployment._build_model_deployment_configuration_details() ) - mock_prepare_artifact.assert_called_with( - model_uri=model_deployment.runtime.model_uri, - properties={ - "display_name": model_deployment.display_name, - "compartment_id": model_deployment.infrastructure.compartment_id, - "project_id": model_deployment.infrastructure.project_id, - }, - ) + mock_upload_artifact.assert_called() assert model_deployment_configuration_details == { "deploymentType": "SINGLE_MODEL", "modelConfigurationDetails": { @@ -595,22 +591,17 @@ def test_build_category_log_details(self): }, } - @patch.object(OCIClientManager, "prepare_artifact") - def test_build_model_deployment_details(self, mock_prepare_artifact): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + @patch.object(DataScienceModel, "upload_artifact") + def test_build_model_deployment_details(self, mock_upload_artifact): + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model model_deployment = self.initialize_model_deployment() create_model_deployment_details = ( model_deployment._build_model_deployment_details() ) - mock_prepare_artifact.assert_called_with( - model_uri=model_deployment.runtime.model_uri, - properties={ - "display_name": model_deployment.display_name, - "compartment_id": model_deployment.infrastructure.compartment_id, - "project_id": model_deployment.infrastructure.project_id, - }, - ) + mock_upload_artifact.assert_called() assert isinstance( create_model_deployment_details, @@ -893,22 +884,17 @@ def test_model_deployment_from_dict(self): assert new_model_deployment.to_dict() == model_deployment.to_dict() - @patch.object(OCIClientManager, "prepare_artifact") - def test_update_model_deployment_details(self, mock_prepare_artifact): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + @patch.object(DataScienceModel, "upload_artifact") + def test_update_model_deployment_details(self, mock_upload_artifact): + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model model_deployment = self.initialize_model_deployment() update_model_deployment_details = ( model_deployment._update_model_deployment_details() ) - mock_prepare_artifact.assert_called_with( - model_uri=model_deployment.runtime.model_uri, - properties={ - "display_name": model_deployment.display_name, - "compartment_id": model_deployment.infrastructure.compartment_id, - "project_id": model_deployment.infrastructure.project_id, - }, - ) + mock_upload_artifact.assert_called() assert isinstance( update_model_deployment_details, @@ -1142,11 +1128,13 @@ def test_from_ocid(self, mock_from_ocid): oci.data_science.DataScienceClient, "create_model_deployment", ) - @patch.object(OCIClientManager, "prepare_artifact") + @patch.object(DataScienceModel, "upload_artifact") def test_deploy( - self, mock_prepare_artifact, mock_create_model_deployment, mock_sync + self, mock_upload_artifact, mock_create_model_deployment, mock_sync ): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model response = MagicMock() response.data = OCI_MODEL_DEPLOYMENT_RESPONSE mock_create_model_deployment.return_value = response @@ -1156,7 +1144,7 @@ def test_deploy( model_deployment._build_model_deployment_details() ) model_deployment.deploy(wait_for_completion=False) - mock_prepare_artifact.assert_called() + mock_upload_artifact.assert_called() mock_create_model_deployment.assert_called_with(create_model_deployment_details) mock_sync.assert_called() @@ -1165,11 +1153,13 @@ def test_deploy( oci.data_science.DataScienceClient, "create_model_deployment", ) - @patch.object(OCIClientManager, "prepare_artifact") + @patch.object(DataScienceModel, "upload_artifact") def test_deploy_failed( - self, mock_prepare_artifact, mock_create_model_deployment, mock_sync + self, mock_upload_artifact, mock_create_model_deployment, mock_sync ): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model response = oci.response.Response( status=MagicMock(), headers=MagicMock(), @@ -1190,7 +1180,7 @@ def test_deploy_failed( match=f"Model deployment {response.data.id} failed to deploy: {response.data.lifecycle_details}", ): model_deployment.deploy(wait_for_completion=False) - mock_prepare_artifact.assert_called() + mock_upload_artifact.assert_called() mock_create_model_deployment.assert_called_with( create_model_deployment_details ) @@ -1246,14 +1236,16 @@ def test_delete(self, mock_delete): oci.data_science.DataScienceClientCompositeOperations, "update_model_deployment_and_wait_for_state", ) - @patch.object(OCIClientManager, "prepare_artifact") + @patch.object(DataScienceModel, "upload_artifact") def test_update( self, - mock_prepare_artifact, + mock_upload_artifact, mock_update_model_deployment_and_wait_for_state, mock_sync, ): - mock_prepare_artifact.return_value = "fakeid.datasciencemodel.oc1.iad.xxx" + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model response = MagicMock() response.data = OCI_MODEL_DEPLOYMENT_RESPONSE mock_update_model_deployment_and_wait_for_state.return_value = response @@ -1263,7 +1255,7 @@ def test_update( model_deployment._update_model_deployment_details() ) model_deployment.update(wait_for_completion=True) - mock_prepare_artifact.assert_called() + mock_upload_artifact.assert_called() mock_update_model_deployment_and_wait_for_state.assert_called_with( "test_model_deployment_id", update_model_deployment_details, @@ -1389,3 +1381,57 @@ def test_model_deployment_with_subnet_id(self): model_deployment.infrastructure.with_subnet_id("test_id") assert model_deployment.infrastructure.subnet_id == "test_id" + + @patch.object(OCIDataScienceMixin, "sync") + @patch.object( + oci.data_science.DataScienceClient, + "create_model_deployment", + ) + @patch.object(DataScienceModel, "upload_artifact") + def test_model_deployment_with_large_size_artifact( + self, + mock_upload_artifact, + mock_create_model_deployment, + mock_sync + ): + dsc_model = MagicMock() + dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" + mock_upload_artifact.return_value = dsc_model + model_deployment = self.initialize_model_deployment() + ( + model_deployment.runtime + .with_bucket_uri("test_bucket_uri") + .with_auth({"test_key":"test_value"}) + .with_region("test_region") + .with_overwrite_existing_artifact(True) + .with_remove_existing_artifact(True) + .with_timeout(100) + ) + + runtime_dict = model_deployment.runtime.to_dict()["spec"] + assert runtime_dict["bucketUri"] == "test_bucket_uri" + assert runtime_dict["auth"] == {"test_key": "test_value"} + assert runtime_dict["region"] == "test_region" + assert runtime_dict["overwriteExistingArtifact"] == True + assert runtime_dict["removeExistingArtifact"] == True + assert runtime_dict["timeout"] == 100 + + response = MagicMock() + response.data = OCI_MODEL_DEPLOYMENT_RESPONSE + mock_create_model_deployment.return_value = response + model_deployment = self.initialize_model_deployment() + model_deployment.set_spec(model_deployment.CONST_ID, "test_model_deployment_id") + create_model_deployment_details = ( + model_deployment._build_model_deployment_details() + ) + model_deployment.deploy(wait_for_completion=False) + mock_upload_artifact.assert_called_with( + bucket_uri="test_bucket_uri", + auth={"test_key":"test_value"}, + region="test_region", + overwrite_existing_artifact=True, + remove_existing_artifact=True, + timeout=100 + ) + mock_create_model_deployment.assert_called_with(create_model_deployment_details) + mock_sync.assert_called() From 122445a2ac4c211d2fe118b4127e3ad56e6eb9f0 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 2 Jun 2023 16:02:57 -0700 Subject: [PATCH 04/20] Updated unit test. --- .../model_deployment/test_model_deployment_v2.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index 57bb5e241..ebd762ae3 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -11,6 +11,7 @@ import unittest import pandas from unittest.mock import MagicMock, patch +from ads.common import utils from ads.common.oci_datascience import OCIDataScienceMixin from ads.common.oci_logging import ConsolidatedLog, OCILog from ads.common.oci_mixin import OCIModelMixin @@ -22,6 +23,7 @@ ModelDeployment, ModelDeploymentLogType, ModelDeploymentFailedError, + MAX_ARTIFACT_SIZE_IN_BYTES, ) from ads.model.deployment.model_deployment_infrastructure import ( ModelDeploymentInfrastructure, @@ -1388,8 +1390,10 @@ def test_model_deployment_with_subnet_id(self): "create_model_deployment", ) @patch.object(DataScienceModel, "upload_artifact") + @patch.object(utils, "folder_size") def test_model_deployment_with_large_size_artifact( self, + mock_folder_size, mock_upload_artifact, mock_create_model_deployment, mock_sync @@ -1400,7 +1404,6 @@ def test_model_deployment_with_large_size_artifact( model_deployment = self.initialize_model_deployment() ( model_deployment.runtime - .with_bucket_uri("test_bucket_uri") .with_auth({"test_key":"test_value"}) .with_region("test_region") .with_overwrite_existing_artifact(True) @@ -1409,7 +1412,6 @@ def test_model_deployment_with_large_size_artifact( ) runtime_dict = model_deployment.runtime.to_dict()["spec"] - assert runtime_dict["bucketUri"] == "test_bucket_uri" assert runtime_dict["auth"] == {"test_key": "test_value"} assert runtime_dict["region"] == "test_region" assert runtime_dict["overwriteExistingArtifact"] == True @@ -1421,6 +1423,14 @@ def test_model_deployment_with_large_size_artifact( mock_create_model_deployment.return_value = response model_deployment = self.initialize_model_deployment() model_deployment.set_spec(model_deployment.CONST_ID, "test_model_deployment_id") + + mock_folder_size.return_value = MAX_ARTIFACT_SIZE_IN_BYTES + 1 + with pytest.raises(ValueError): + model_deployment.deploy(wait_for_completion=False) + + model_deployment.runtime.with_bucket_uri("test_bucket_uri") + runtime_dict = model_deployment.runtime.to_dict()["spec"] + assert runtime_dict["bucketUri"] == "test_bucket_uri" create_model_deployment_details = ( model_deployment._build_model_deployment_details() ) From af126984877286a5d1c21b2b6c91cff0c931b173 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Mon, 5 Jun 2023 13:12:13 -0700 Subject: [PATCH 05/20] Updated pr. --- ads/model/deployment/model_deployment.py | 2 +- .../test_model_deployment_v2.py | 56 +++++++++---------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index e04bc8f03..b3e56f780 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -1583,7 +1583,7 @@ def _build_model_deployment_configuration_details(self) -> Dict: or COMPARTMENT_OCID, project_id=self.infrastructure.project_id or PROJECT_OCID, artifact=runtime.model_uri, - ).upload_artifact( + ).create( bucket_uri=runtime.bucket_uri, auth=runtime.auth, region=runtime.region, diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index ebd762ae3..41aa61543 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -537,17 +537,17 @@ def test_model_deployment_to_dict(self): }, } - @patch.object(DataScienceModel, "upload_artifact") - def test_build_model_deployment_configuration_details(self, mock_upload_artifact): + @patch.object(DataScienceModel, "create") + def test_build_model_deployment_configuration_details(self, mock_create): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model model_deployment = self.initialize_model_deployment() model_deployment_configuration_details = ( model_deployment._build_model_deployment_configuration_details() ) - mock_upload_artifact.assert_called() + mock_create.assert_called() assert model_deployment_configuration_details == { "deploymentType": "SINGLE_MODEL", "modelConfigurationDetails": { @@ -593,17 +593,17 @@ def test_build_category_log_details(self): }, } - @patch.object(DataScienceModel, "upload_artifact") - def test_build_model_deployment_details(self, mock_upload_artifact): + @patch.object(DataScienceModel, "create") + def test_build_model_deployment_details(self, mock_create): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model model_deployment = self.initialize_model_deployment() create_model_deployment_details = ( model_deployment._build_model_deployment_details() ) - mock_upload_artifact.assert_called() + mock_create.assert_called() assert isinstance( create_model_deployment_details, @@ -886,17 +886,17 @@ def test_model_deployment_from_dict(self): assert new_model_deployment.to_dict() == model_deployment.to_dict() - @patch.object(DataScienceModel, "upload_artifact") - def test_update_model_deployment_details(self, mock_upload_artifact): + @patch.object(DataScienceModel, "create") + def test_update_model_deployment_details(self, mock_create): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model model_deployment = self.initialize_model_deployment() update_model_deployment_details = ( model_deployment._update_model_deployment_details() ) - mock_upload_artifact.assert_called() + mock_create.assert_called() assert isinstance( update_model_deployment_details, @@ -1130,13 +1130,13 @@ def test_from_ocid(self, mock_from_ocid): oci.data_science.DataScienceClient, "create_model_deployment", ) - @patch.object(DataScienceModel, "upload_artifact") + @patch.object(DataScienceModel, "create") def test_deploy( - self, mock_upload_artifact, mock_create_model_deployment, mock_sync + self, mock_create, mock_create_model_deployment, mock_sync ): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model response = MagicMock() response.data = OCI_MODEL_DEPLOYMENT_RESPONSE mock_create_model_deployment.return_value = response @@ -1146,7 +1146,7 @@ def test_deploy( model_deployment._build_model_deployment_details() ) model_deployment.deploy(wait_for_completion=False) - mock_upload_artifact.assert_called() + mock_create.assert_called() mock_create_model_deployment.assert_called_with(create_model_deployment_details) mock_sync.assert_called() @@ -1155,13 +1155,13 @@ def test_deploy( oci.data_science.DataScienceClient, "create_model_deployment", ) - @patch.object(DataScienceModel, "upload_artifact") + @patch.object(DataScienceModel, "create") def test_deploy_failed( - self, mock_upload_artifact, mock_create_model_deployment, mock_sync + self, mock_create, mock_create_model_deployment, mock_sync ): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model response = oci.response.Response( status=MagicMock(), headers=MagicMock(), @@ -1182,7 +1182,7 @@ def test_deploy_failed( match=f"Model deployment {response.data.id} failed to deploy: {response.data.lifecycle_details}", ): model_deployment.deploy(wait_for_completion=False) - mock_upload_artifact.assert_called() + mock_create.assert_called() mock_create_model_deployment.assert_called_with( create_model_deployment_details ) @@ -1238,16 +1238,16 @@ def test_delete(self, mock_delete): oci.data_science.DataScienceClientCompositeOperations, "update_model_deployment_and_wait_for_state", ) - @patch.object(DataScienceModel, "upload_artifact") + @patch.object(DataScienceModel, "create") def test_update( self, - mock_upload_artifact, + mock_create, mock_update_model_deployment_and_wait_for_state, mock_sync, ): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model response = MagicMock() response.data = OCI_MODEL_DEPLOYMENT_RESPONSE mock_update_model_deployment_and_wait_for_state.return_value = response @@ -1257,7 +1257,7 @@ def test_update( model_deployment._update_model_deployment_details() ) model_deployment.update(wait_for_completion=True) - mock_upload_artifact.assert_called() + mock_create.assert_called() mock_update_model_deployment_and_wait_for_state.assert_called_with( "test_model_deployment_id", update_model_deployment_details, @@ -1389,18 +1389,18 @@ def test_model_deployment_with_subnet_id(self): oci.data_science.DataScienceClient, "create_model_deployment", ) - @patch.object(DataScienceModel, "upload_artifact") + @patch.object(DataScienceModel, "create") @patch.object(utils, "folder_size") def test_model_deployment_with_large_size_artifact( self, mock_folder_size, - mock_upload_artifact, + mock_create, mock_create_model_deployment, mock_sync ): dsc_model = MagicMock() dsc_model.id = "fakeid.datasciencemodel.oc1.iad.xxx" - mock_upload_artifact.return_value = dsc_model + mock_create.return_value = dsc_model model_deployment = self.initialize_model_deployment() ( model_deployment.runtime @@ -1435,7 +1435,7 @@ def test_model_deployment_with_large_size_artifact( model_deployment._build_model_deployment_details() ) model_deployment.deploy(wait_for_completion=False) - mock_upload_artifact.assert_called_with( + mock_create.assert_called_with( bucket_uri="test_bucket_uri", auth={"test_key":"test_value"}, region="test_region", From 000210a9a2965db5d38d4289b7167332b4643310 Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Tue, 6 Jun 2023 12:16:55 -0700 Subject: [PATCH 06/20] Added docs. --- .../model_registration/model_deploy_byoc.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/source/user_guide/model_registration/model_deploy_byoc.rst b/docs/source/user_guide/model_registration/model_deploy_byoc.rst index 00143ccd7..6717a55ac 100644 --- a/docs/source/user_guide/model_registration/model_deploy_byoc.rst +++ b/docs/source/user_guide/model_registration/model_deploy_byoc.rst @@ -114,7 +114,13 @@ Below is an example of deploying model on container runtime using ``ModelDeploym .with_health_check_port(5000) .with_env({"key":"value"}) .with_deployment_mode("HTTPS_ONLY") - .with_model_uri("") + .with_model_uri("") + .with_auth({"auth_key":"auth_value"}) + .with_region("us-ashburn-1") + .with_overwrite_existing_artifact(True) + .with_remove_existing_artifact(True) + .with_timeout(100) + .with_bucket_uri("oci://@/") ) # configure model deployment @@ -169,7 +175,7 @@ Below is an example of deploying model on container runtime using ``ModelDeploym kind: runtime type: container spec: - modelUri: + modelUri: image: iad.ocir.io//: imageDigest: entrypoint: ["python","/opt/ds/model/deployed_model/api.py"] @@ -177,6 +183,13 @@ Below is an example of deploying model on container runtime using ``ModelDeploym healthCheckPort: 5000 env: WEB_CONCURRENCY: "10" + auth: + auth_key: auth_value + region: us-ashburn-1 + overwriteExistingArtifact: True + removeExistingArtifact: True + timeout: 100 + bucketUri: oci://@/ deploymentMode: HTTPS_ONLY """ From 589d93abf4aff52a0e32751e014759cb312a210d Mon Sep 17 00:00:00 2001 From: Liuda Rudenka Date: Tue, 6 Jun 2023 15:31:37 -0500 Subject: [PATCH 07/20] Fix jobs integration test (#219) --- tests/integration/jobs/test_dsc_job.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/jobs/test_dsc_job.py b/tests/integration/jobs/test_dsc_job.py index fdd2837c8..61ae273b4 100644 --- a/tests/integration/jobs/test_dsc_job.py +++ b/tests/integration/jobs/test_dsc_job.py @@ -314,6 +314,10 @@ def test_create_job_with_dsc_infra_config(self): for sk, sv in v.items(): expected_infra_spec[attr_map[k]][attr_map[sk]] = sv infra = builder_method(**v) + # Shape config is not required for non flex shape from user end. + # When we have shape config, we need Flex shape also: + if k == "shape_config_details": + infra = infra.with_shape_name("VM.Standard.E3.Flex") else: expected_infra_spec[attr_map[k]] = v infra = builder_method(v) From 5fe97d45650cc6ea3f6ac7769716b24c5da898ca Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Wed, 7 Jun 2023 10:43:05 -0700 Subject: [PATCH 08/20] Updated pr. --- ads/model/deployment/model_deployment.py | 8 -------- .../model_deployment/test_model_deployment_v2.py | 13 ++----------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index b3e56f780..85459f5ec 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -70,7 +70,6 @@ MODEL_DEPLOYMENT_INSTANCE_COUNT = 1 MODEL_DEPLOYMENT_BANDWIDTH_MBPS = 10 -MAX_ARTIFACT_SIZE_IN_BYTES = 2147483648 # 2GB class ModelDeploymentLogType: PREDICT = "predict" @@ -1567,13 +1566,6 @@ def _build_model_deployment_configuration_details(self) -> Dict: model_id = runtime.model_uri if not model_id.startswith("ocid"): - if ads_utils.folder_size(runtime.model_uri) > MAX_ARTIFACT_SIZE_IN_BYTES: - if not runtime.bucket_uri: - raise ValueError( - f"The model artifacts size is greater than `{MAX_ARTIFACT_SIZE_IN_BYTES}`. " - "The `bucket_uri` needs to be specified to copy artifacts to the object storage bucket. " - "Example: `runtime.with_bucket_uri(oci://@/prefix/)`" - ) from ads.model.datascience_model import DataScienceModel diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index 41aa61543..48cc76e65 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -11,7 +11,6 @@ import unittest import pandas from unittest.mock import MagicMock, patch -from ads.common import utils from ads.common.oci_datascience import OCIDataScienceMixin from ads.common.oci_logging import ConsolidatedLog, OCILog from ads.common.oci_mixin import OCIModelMixin @@ -23,7 +22,6 @@ ModelDeployment, ModelDeploymentLogType, ModelDeploymentFailedError, - MAX_ARTIFACT_SIZE_IN_BYTES, ) from ads.model.deployment.model_deployment_infrastructure import ( ModelDeploymentInfrastructure, @@ -1390,10 +1388,8 @@ def test_model_deployment_with_subnet_id(self): "create_model_deployment", ) @patch.object(DataScienceModel, "create") - @patch.object(utils, "folder_size") def test_model_deployment_with_large_size_artifact( self, - mock_folder_size, mock_create, mock_create_model_deployment, mock_sync @@ -1409,6 +1405,7 @@ def test_model_deployment_with_large_size_artifact( .with_overwrite_existing_artifact(True) .with_remove_existing_artifact(True) .with_timeout(100) + .with_bucket_uri("test_bucket_uri") ) runtime_dict = model_deployment.runtime.to_dict()["spec"] @@ -1417,6 +1414,7 @@ def test_model_deployment_with_large_size_artifact( assert runtime_dict["overwriteExistingArtifact"] == True assert runtime_dict["removeExistingArtifact"] == True assert runtime_dict["timeout"] == 100 + assert runtime_dict["bucketUri"] == "test_bucket_uri" response = MagicMock() response.data = OCI_MODEL_DEPLOYMENT_RESPONSE @@ -1424,13 +1422,6 @@ def test_model_deployment_with_large_size_artifact( model_deployment = self.initialize_model_deployment() model_deployment.set_spec(model_deployment.CONST_ID, "test_model_deployment_id") - mock_folder_size.return_value = MAX_ARTIFACT_SIZE_IN_BYTES + 1 - with pytest.raises(ValueError): - model_deployment.deploy(wait_for_completion=False) - - model_deployment.runtime.with_bucket_uri("test_bucket_uri") - runtime_dict = model_deployment.runtime.to_dict()["spec"] - assert runtime_dict["bucketUri"] == "test_bucket_uri" create_model_deployment_details = ( model_deployment._build_model_deployment_details() ) From e008fad3abae7a3874cd46af023151c8609d1edf Mon Sep 17 00:00:00 2001 From: Liuda Rudenka Date: Wed, 7 Jun 2023 13:44:08 -0500 Subject: [PATCH 09/20] Change link to api_keys.ipynb with instruction using OCI approach (#220) --- docs/source/user_guide/cli/authentication.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/source/user_guide/cli/authentication.rst b/docs/source/user_guide/cli/authentication.rst index f53a3f339..d6c5d85cc 100644 --- a/docs/source/user_guide/cli/authentication.rst +++ b/docs/source/user_guide/cli/authentication.rst @@ -37,9 +37,7 @@ You can choose to use the resource principal to authenticate while using the Acc Use API Key setup when you are working from a local workstation or on platform which does not support resource principals. -This is the default method of authentication. You can also authenticate as your own personal IAM user by creating or uploading OCI configuration and API key files inside your notebook session environment. The OCI configuration file contains the necessary credentials to authenticate your user against the model catalog and other OCI services like Object Storage. The example notebook, `api_keys.ipynb` demonstrates how to create these files. - -You can follow the steps in `api_keys.ipynb `_ for step by step instruction on setting up API Keys. +This is the default method of authentication. You can also authenticate as your own personal IAM user by creating or uploading OCI configuration and API key files inside your notebook session environment. The OCI configuration file contains the necessary credentials to authenticate your user against the model catalog and other OCI services like Object Storage. You can create this file using a setup dialog or manually using a text editor. See `Setting up the Configuration File `_ for steps to follow. .. note:: If you already have an OCI configuration file (``config``) and associated keys, you can upload them directly to the ``/home/datascience/.oci`` directory using the JupyterLab **Upload Files** or the drag-and-drop option. From 62dca37aed961491c85149cd07a13d12ddd5c578 Mon Sep 17 00:00:00 2001 From: MING KANG Date: Wed, 7 Jun 2023 11:49:36 -0700 Subject: [PATCH 10/20] bugfix/ODSC-43563/entryscript_mounted_as_directory (#218) --- ads/opctl/backend/local.py | 49 ++++++++++++------- ads/opctl/cli.py | 8 ++- .../cli/opctl/localdev/local_deployment.rst | 15 +++--- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/ads/opctl/backend/local.py b/ads/opctl/backend/local.py index 596792109..b30ea6db7 100644 --- a/ads/opctl/backend/local.py +++ b/ads/opctl/backend/local.py @@ -54,6 +54,7 @@ from ads.common.oci_client import OCIClientFactory from ads.config import NO_CONTAINER + class CondaPackNotFound(Exception): # pragma: no cover pass @@ -219,7 +220,9 @@ def _run_with_conda_pack( ) if os.path.exists(os.path.join(conda_pack_path, "spark-defaults.conf")): env_vars["SPARK_CONF_DIR"] = os.path.join(DEFAULT_IMAGE_CONDA_DIR, slug) - logger.info(f"Running with conda pack in a container with command {command}") + logger.info( + f"Running with conda pack in a container with command {command}" + ) return self._activate_conda_env_and_run( image, slug, command, bind_volumes, env_vars ) @@ -419,8 +422,7 @@ def _activate_conda_env_and_run( build_image("job-local", gpu=False) else: build_image("job-local", gpu=True) - - with tempfile.TemporaryDirectory() as td: + with tempfile.TemporaryDirectory(dir=os.path.expanduser("~")) as td: with open(os.path.join(td, "entryscript.sh"), "w") as f: f.write( f""" @@ -681,11 +683,11 @@ def predict(self) -> None: None Nothing. """ - + # model artifact in artifact directory artifact_directory = self.config["execution"].get("artifact_directory") ocid = self.config["execution"].get("ocid") - + model_folder = os.path.expanduser( self.config["execution"].get("model_save_folder", DEFAULT_MODEL_FOLDER) ) @@ -711,20 +713,23 @@ def predict(self) -> None: timeout=timeout, force_overwrite=True, ) - + # conda - conda_slug, conda_path = self.config["execution"].get("conda_slug"), self.config["execution"].get("conda_path") + conda_slug = self.config["execution"].get("conda_slug") + conda_path = self.config["execution"].get("conda_path") if not conda_slug and not conda_path and ocid: conda_slug, conda_path = self._get_conda_info_from_custom_metadata(ocid) if not conda_slug and not conda_path: conda_slug, conda_path = self._get_conda_info_from_runtime( artifact_dir=artifact_directory ) - if 'conda_slug' not in self.config["execution"]: - self.config["execution"]["conda_slug"] = conda_path.split("/")[-1] if conda_path else conda_slug + if "conda_slug" not in self.config["execution"]: + self.config["execution"]["conda_slug"] = ( + conda_path.split("/")[-1] if conda_path else conda_slug + ) self.config["execution"]["image"] = ML_JOB_IMAGE - + # bind_volumnes bind_volumes = {} SCRIPT = "script.py" @@ -735,39 +740,45 @@ def predict(self) -> None: os.path.dirname(self.config["execution"]["oci_config"]) ): {"bind": os.path.join(DEFAULT_IMAGE_HOME_DIR, ".oci")} } - + self.config["execution"]["source_folder"] = os.path.abspath( os.path.join(dir_path, "..") ) self.config["execution"]["entrypoint"] = SCRIPT bind_volumes[artifact_directory] = {"bind": DEFAULT_MODEL_DEPLOYMENT_FOLDER} - + # extra cmd data = self.config["execution"].get("payload") extra_cmd = f"--payload '{data}' " + f"--auth {self.auth_type} " if self.auth_type != "resource_principal": extra_cmd += f"--profile {self.profile}" - + if is_in_notebook_session() or NO_CONTAINER: # _run_with_conda_pack has code to handle notebook session case, - # however, it activate the conda pack and then run the script. + # however, it activate the conda pack and then run the script. # For the deployment, we just take the current conda env and run it. # Hence we just handle the notebook case directly here. script_path = os.path.join(os.path.join(dir_path, ".."), SCRIPT) - cmd = f"python {script_path} " + f"--artifact-directory {artifact_directory} " + extra_cmd + cmd = ( + f"python {script_path} " + + f"--artifact-directory {artifact_directory} " + + extra_cmd + ) logger.info(f"Running in a notebook or NO_CONTAINER with command {cmd}") run_command(cmd=cmd, shell=True) else: - extra_cmd = f"--artifact-directory {DEFAULT_MODEL_DEPLOYMENT_FOLDER} "+ extra_cmd + extra_cmd = ( + f"--artifact-directory {DEFAULT_MODEL_DEPLOYMENT_FOLDER} " + extra_cmd + ) exit_code = self._run_with_conda_pack( - bind_volumes, extra_cmd, install=True, conda_uri=conda_path - ) + bind_volumes, extra_cmd, install=True, conda_uri=conda_path + ) if exit_code != 0: raise RuntimeError( f"`predict` did not complete successfully. Exit code: {exit_code}. " f"Run with the --debug argument to view container logs." ) - + def _get_conda_info_from_custom_metadata(self, ocid): """ Get conda env info from custom metadata from model catalog. diff --git a/ads/opctl/cli.py b/ads/opctl/cli.py index 57dd03ce9..6e500513c 100644 --- a/ads/opctl/cli.py +++ b/ads/opctl/cli.py @@ -30,8 +30,12 @@ from ads.opctl.cmds import run_diagnostics as run_diagnostics_cmd from ads.opctl.cmds import watch as watch_cmd from ads.opctl.config.merger import ConfigMerger -from ads.opctl.constants import (BACKEND_NAME, DEFAULT_MODEL_FOLDER, - RESOURCE_TYPE, RUNTIME_TYPE) +from ads.opctl.constants import ( + BACKEND_NAME, + DEFAULT_MODEL_FOLDER, + RESOURCE_TYPE, + RUNTIME_TYPE, +) from ads.opctl.utils import build_image as build_image_cmd from ads.opctl.utils import publish_image as publish_image_cmd from ads.opctl.utils import suppress_traceback diff --git a/docs/source/user_guide/cli/opctl/localdev/local_deployment.rst b/docs/source/user_guide/cli/opctl/localdev/local_deployment.rst index ea880c903..d3a7cafc5 100644 --- a/docs/source/user_guide/cli/opctl/localdev/local_deployment.rst +++ b/docs/source/user_guide/cli/opctl/localdev/local_deployment.rst @@ -34,13 +34,13 @@ This example below demonstrates how to run a local predict using an installed co .. code-block:: shell - ads opctl predict --artifact-directory /folder/your/model/artifacts/are/stored --payload '[[-1.68671955,2.25814541,-0.5068027,0.25248417,0.62665134,0.23441123]]' --conda-slug myconda_p38_cpu_v1 + ads opctl predict --ocid "ocid1.datasciencemodel.oc1.iad." --payload '[[-1.68671955,2.25814541,-0.5068027,0.25248417,0.62665134,0.23441123]]' --conda-slug myconda_p38_cpu_v1 Parameter explanation: - - ``--ocid ocid1.datasciencemodel.oc1.iad.******``: Run the predict locally in a docker container when you pass in a model id. If you pass in a deployment id, e.g. ``ocid1.datasciencemodeldeployment.oc1.iad.``, it will actually predict against the remote endpoint. In that case, only ``--ocid`` and ``--payload`` are needed. - - ``--conda-slug myconda_p38_cpu_v1``: Use the ``myconda_p38_cpu_v1`` conda environment. Note that you must publish this conda environment to you own bucket first if you are actually using a service conda pack for your real deployment. However, you dont have to install it locally. We will find to auto detect the conda information from the custom metadata, then the runtime.yaml from your model artifact if --conda-slug and --conda-path is not provided. However, if detected conda are service packs, we will throw an error asking you to publish it first. You can also provide the conda information directly through --conda-slug or --conda-path. Note, in order to test whether deployemnt will work, you should provide the slug that you will use in real deployment. The local conda environment directory will be automatically mounted into the container and activated before the entrypoint is executed. - - ``--payload``: the payload to be passed to your model. - - ``--bucket-uri``: is used to download large model artifact to your local machine. Extra policy needs to be placed for this bucket. Check this :doc:`link <./user_guide/model_registration/model_load.html#large-model-artifacts>` for more details. Small artifact does not require this. + - ``--ocid``: Run the predict locally in a docker container when you pass in a model id. If you pass in a deployment id, e.g. ``ocid1.datasciencemodeldeployment.oc1.iad.``, it will actually predict against the remote endpoint. In that case, only ``--ocid`` and ``--payload`` are needed. + - ``--conda-slug myconda_p38_cpu_v1``: Use the ``myconda_p38_cpu_v1`` conda environment. The environment should be installed in your local already. If you haven't installed it, you can provide the path by ``--conda-path``, for example, ``--conda-path "oci://my-bucket@mytenancy/.../myconda_p38_cpu_v1"``, it will download and install the conda environment in your local. Note that you must publish this conda environment to you own bucket first if you are actually using a service conda pack for your real deployment. We will find to auto detect the conda information from the custom metadata, then the runtime.yaml from your model artifact if ``--conda-slug`` and ``--conda-path`` is not provided. However, if detected conda are service packs, we will throw an error asking you to publish it first. Note, in order to test whether deployemnt will work, you should provide the slug that you will use in real deployment. The local conda environment directory will be automatically mounted into the container and activated before the entrypoint is executed. + - ``--payload``: The payload to be passed to your model. + - ``--bucket-uri``: Used to download large model artifact to your local machine. Extra policy needs to be placed for this bucket. Check this :doc:`link <./user_guide/model_registration/model_load.html#large-model-artifacts>` for more details. Small artifact does not require this. .. code-block:: shell @@ -54,10 +54,9 @@ Running your Predict Against the Deployment Endpoint ---------------------------------------------------- .. code-block:: shell - ads opctl predict ocid1.datasciencemodeldeployment.oc1.iad.***** --payload '[[-1.68671955,2.25814541,-0.5068027,0.25248417,0.62665134,0.23441123]]' + ads opctl predict --ocid "ocid1.datasciencemodeldeployment.oc1.iad." --payload '[[-1.68671955,2.25814541,-0.5068027,0.25248417,0.62665134,0.23441123]]' Parameter explanation: - - ``--ocid ocid1.datasciencemodeldeployment.oc1.iad.******``: Run the predict remotely against the remote endpoint. + - ``--ocid``: Run the predict remotely against the remote endpoint. - ``--payload``: The payload to be passed to your model. - \ No newline at end of file From c7985700b9578f2845af5b6773e38b3be2cee55f Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Thu, 8 Jun 2023 16:16:01 -0700 Subject: [PATCH 11/20] Deprecated ModelDeploymentProperties, ModelDeployer and updated ModelDeployment class. --- ads/model/deployment/model_deployer.py | 12 ++- ads/model/deployment/model_deployment.py | 81 +++++++++++++++++-- .../deployment/model_deployment_properties.py | 12 +++ ads/model/generic_model.py | 36 +++++++-- .../test_model_deployment_v2.py | 44 ++++++++++ .../with_extras/model/test_generic_model.py | 30 +++---- 6 files changed, 185 insertions(+), 30 deletions(-) diff --git a/ads/model/deployment/model_deployer.py b/ads/model/deployment/model_deployer.py index 3f3b324ff..e2f9e04bd 100644 --- a/ads/model/deployment/model_deployer.py +++ b/ads/model/deployment/model_deployer.py @@ -51,6 +51,16 @@ from .model_deployment import DEFAULT_POLL_INTERVAL, DEFAULT_WAIT_TIME, ModelDeployment from .model_deployment_properties import ModelDeploymentProperties +warnings.warn( + ( + "The `ads.model.deployment.model_deployer` is deprecated in `oracle-ads 2.8.7` and will be removed in `oracle-ads 3.0`." + "Use `ModelDeployment` class in `ads.model.deployment` module for initializing and deploy model deployment. " + "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/introduction.html" + ), + DeprecationWarning, + stacklevel=2, +) + class ModelDeployer: """ModelDeployer is the class responsible for deploying the ModelDeployment @@ -96,7 +106,7 @@ def __init__(self, config: dict = None, ds_client: DataScienceClient = None): """ if config: warnings.warn( - "`config` will be deprecated in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." + "`config` will be removed in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." ) self.ds_client = None diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index cc533e6d2..5e0bcefa7 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -70,6 +70,11 @@ MODEL_DEPLOYMENT_INSTANCE_COUNT = 1 MODEL_DEPLOYMENT_BANDWIDTH_MBPS = 10 +MODEL_DEPLOYMENT_RUNTIMES = { + ModelDeploymentRuntimeType.CONDA: ModelDeploymentCondaRuntime, + ModelDeploymentRuntimeType.CONTAINER: ModelDeploymentContainerRuntime, +} + class ModelDeploymentLogType: PREDICT = "predict" @@ -292,17 +297,17 @@ def __init__( if config: warnings.warn( - "`config` will be deprecated in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." + "Parameter `config` will be removed from ModelDeployment constructor in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." ) if properties: warnings.warn( - "`properties` will be deprecated in 3.0.0. Please use `spec` or the builder pattern to initialize model deployment instance." + "Parameter `properties` will be removed from ModelDeployment constructor in 3.0.0. Please use `spec` or the builder pattern to initialize model deployment instance." ) if model_deployment_url or model_deployment_id: warnings.warn( - "`model_deployment_url` and `model_deployment_id` will be deprecated in 3.0.0 and will be ignored now. These two fields will be auto-populated from the service side." + "Parameter `model_deployment_url` and `model_deployment_id` will be removed from ModelDeployment constructor in 3.0.0 and will be ignored now. These two fields will be auto-populated from the service side." ) initialize_spec = {} @@ -694,6 +699,11 @@ def update( ModelDeployment The instance of ModelDeployment. """ + if properties: + warnings.warn( + "Parameter `properties` will be removed from ModelDeployment update() in 3.0.0. Please use the builder pattern to update model deployment instance." + ) + updated_properties = properties if not isinstance(properties, ModelDeploymentProperties): updated_properties = ModelDeploymentProperties( @@ -703,7 +713,7 @@ def update( update_model_deployment_details = ( updated_properties.to_update_deployment() if properties or updated_properties.oci_model_deployment or kwargs - else self._update_model_deployment_details() + else self._update_model_deployment_details(**kwargs) ) response = self.dsc_model_deployment.update( @@ -1487,7 +1497,7 @@ def _build_model_deployment_details(self) -> CreateModelDeploymentDetails: **create_model_deployment_details ).to_oci_model(CreateModelDeploymentDetails) - def _update_model_deployment_details(self) -> UpdateModelDeploymentDetails: + def _update_model_deployment_details(self, **kwargs) -> UpdateModelDeploymentDetails: """Builds UpdateModelDeploymentDetails from model deployment instance. Returns @@ -1499,7 +1509,7 @@ def _update_model_deployment_details(self) -> UpdateModelDeploymentDetails: raise ValueError( "Missing parameter runtime or infrastructure. Try reruning it after parameters are fully configured." ) - + self._update_spec(**kwargs) update_model_deployment_details = { self.CONST_DISPLAY_NAME: self.display_name, self.CONST_DESCRIPTION: self.description, @@ -1511,6 +1521,65 @@ def _update_model_deployment_details(self) -> UpdateModelDeploymentDetails: return OCIDataScienceModelDeployment( **update_model_deployment_details ).to_oci_model(UpdateModelDeploymentDetails) + + def _update_spec(self, **kwargs) -> "ModelDeployment": + """Updates model deployment specs from kwargs. + + Parameters + ---------- + kwargs: + display_name: (str) + Model deployment display name + description: (str) + Model deployment description + freeform_tags: (dict) + Model deployment freeform tags + defined_tags: (dict) + Model deployment defined tags + + Additional kwargs arguments. + Can be any attribute that `ads.model.deployment.ModelDeploymentCondaRuntime`, `ads.model.deployment.ModelDeploymentContainerRuntime` + and `ads.model.deployment.ModelDeploymentInfrastructure` accepts. + + Returns + ------- + ModelDeployment + The instance of ModelDeployment. + """ + if not kwargs: + return self + + converted_specs = ads_utils.batch_convert_case(kwargs, "camel") + specs = { + "self": self._spec, + "runtime": self.runtime._spec, + "infrastructure": self.infrastructure._spec + } + sub_set = { + self.infrastructure.CONST_ACCESS_LOG, + self.infrastructure.CONST_PREDICT_LOG, + self.infrastructure.CONST_SHAPE_CONFIG_DETAILS + } + for spec_value in specs.values(): + for key in spec_value: + if key in converted_specs: + if key in sub_set: + for sub_key in converted_specs[key]: + converted_sub_key = ads_utils.snake_to_camel(sub_key) + spec_value[key][converted_sub_key] = converted_specs[key][sub_key] + else: + spec_value[key] = copy.deepcopy(converted_specs[key]) + self = ( + ModelDeployment(spec=specs["self"]) + .with_runtime( + MODEL_DEPLOYMENT_RUNTIMES[self.runtime.type](spec=specs["runtime"]) + ) + .with_infrastructure( + ModelDeploymentInfrastructure(spec=specs["infrastructure"]) + ) + ) + + return self def _build_model_deployment_configuration_details(self) -> Dict: """Builds model deployment configuration details from model deployment instance. diff --git a/ads/model/deployment/model_deployment_properties.py b/ads/model/deployment/model_deployment_properties.py index 9734b9cdb..67e039498 100644 --- a/ads/model/deployment/model_deployment_properties.py +++ b/ads/model/deployment/model_deployment_properties.py @@ -12,6 +12,18 @@ from .common.utils import OCIClientManager +import warnings + +warnings.warn( + ( + "The `ads.model.deployment.model_deployment_properties` is deprecated in `oracle-ads 2.8.7` and will be removed in `oracle-ads 3.0`." + "Use `ModelDeploymentInfrastructure` and `ModelDeploymentRuntime` classes in `ads.model.deployment` module for configuring model deployment. " + "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/introduction.html" + ), + DeprecationWarning, + stacklevel=2, +) + class ModelDeploymentProperties( OCIDataScienceMixin, data_science_models.ModelDeployment diff --git a/ads/model/generic_model.py b/ads/model/generic_model.py index 5b4f81b40..82c4de0be 100644 --- a/ads/model/generic_model.py +++ b/ads/model/generic_model.py @@ -10,6 +10,7 @@ import tempfile from enum import Enum from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union +import warnings import numpy as np import pandas as pd @@ -44,7 +45,6 @@ from ads.model.deployment import ( DEFAULT_POLL_INTERVAL, DEFAULT_WAIT_TIME, - ModelDeployer, ModelDeployment, ModelDeploymentMode, ModelDeploymentProperties, @@ -1565,9 +1565,7 @@ def from_model_deployment( "Only SparkPipelineModel framework supports object storage path as `artifact_dir`." ) - model_deployment = ModelDeployer(config=auth).get_model_deployment( - model_deployment_id=model_deployment_id - ) + model_deployment = ModelDeployment.from_id(model_deployment_id) current_state = model_deployment.state.name.upper() if current_state != ModelDeploymentState.ACTIVE.name: @@ -1618,6 +1616,7 @@ def update_deployment( Examples -------- >>> # Update access log id, freeform tags and description for the model deployment + >>> # Deprecated >>> model.update_deployment( >>> properties=ModelDeploymentProperties( >>> access_log_id=, @@ -1625,6 +1624,14 @@ def update_deployment( >>> freeform_tags={"key": "value"}, >>> ) >>> ) + >>> # New way to update access log id, freeform tags and description for the model deployment + >>> model.update_deployment( + ... access_log={ + ... log_id= + ... }, + ... description="Description for Custom Model", + ... freeform_tags={"key": "value"}, + ... ) Parameters ---------- @@ -1647,12 +1654,28 @@ def update_deployment( If you need to override the default, use the `ads.common.auth.api_keys` or `ads.common.auth.resource_principal` to create appropriate authentication signer and kwargs required to instantiate IdentityClient object. + display_name: (str) + Model deployment display name + description: (str) + Model deployment description + freeform_tags: (dict) + Model deployment freeform tags + defined_tags: (dict) + Model deployment defined tags + + Additional kwargs arguments. + Can be any attribute that `ads.model.deployment.ModelDeploymentCondaRuntime`, `ads.model.deployment.ModelDeploymentContainerRuntime` + and `ads.model.deployment.ModelDeploymentInfrastructure` accepts. Returns ------- ModelDeployment An instance of ModelDeployment class. """ + if properties: + warnings.warn( + "Parameter `properties` will be removed from GenericModel `update_deployment()` in 3.0.0. Please use kwargs to update model deployment." + ) if not inspect.isclass(cls): if cls.model_deployment: return cls.model_deployment.update( @@ -1666,10 +1689,7 @@ def update_deployment( if not model_deployment_id: raise ValueError("Parameter `model_deployment_id` must be provided.") - auth = kwargs.pop("auth", authutil.default_signer()) - model_deployment = ModelDeployer(config=auth).get_model_deployment( - model_deployment_id=model_deployment_id - ) + model_deployment = ModelDeployment.from_id(model_deployment_id) return model_deployment.update( properties=properties, wait_for_completion=wait_for_completion, diff --git a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py index fe2e4e5a5..348f6167e 100644 --- a/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py +++ b/tests/unitary/default_setup/model_deployment/test_model_deployment_v2.py @@ -1389,3 +1389,47 @@ def test_model_deployment_with_subnet_id(self): model_deployment.infrastructure.with_subnet_id("test_id") assert model_deployment.infrastructure.subnet_id == "test_id" + + def test_update_spec(self): + model_deployment = self.initialize_model_deployment() + model_deployment._update_spec( + display_name="test_updated_name", + freeform_tags={"test_updated_key":"test_updated_value"}, + access_log={ + "log_id": "test_updated_access_log_id" + }, + predict_log={ + "log_group_id": "test_updated_predict_log_group_id" + }, + shape_config_details={ + "ocpus": 100, + "memoryInGBs": 200 + }, + replica=20, + image="test_updated_image", + env={ + "test_updated_env_key":"test_updated_env_value" + } + ) + + assert model_deployment.display_name == "test_updated_name" + assert model_deployment.freeform_tags == { + "test_updated_key":"test_updated_value" + } + assert model_deployment.infrastructure.access_log == { + "logId": "test_updated_access_log_id", + "logGroupId": "fakeid.loggroup.oc1.iad.xxx" + } + assert model_deployment.infrastructure.predict_log == { + "logId": "fakeid.log.oc1.iad.xxx", + "logGroupId": "test_updated_predict_log_group_id" + } + assert model_deployment.infrastructure.shape_config_details == { + "ocpus": 100, + "memoryInGBs": 200 + } + assert model_deployment.infrastructure.replica == 20 + assert model_deployment.runtime.image == "test_updated_image" + assert model_deployment.runtime.env == { + "test_updated_env_key":"test_updated_env_value" + } diff --git a/tests/unitary/with_extras/model/test_generic_model.py b/tests/unitary/with_extras/model/test_generic_model.py index 8f126ca4a..e25d88f7e 100644 --- a/tests/unitary/with_extras/model/test_generic_model.py +++ b/tests/unitary/with_extras/model/test_generic_model.py @@ -958,14 +958,14 @@ def test_from_model_catalog( return_value=ModelDeploymentState.ACTIVE, ) @patch.object(GenericModel, "from_model_catalog") - @patch.object(ModelDeployer, "get_model_deployment") + @patch.object(ModelDeployment, "from_id") @patch("ads.common.auth.default_signer") @patch("ads.common.oci_client.OCIClientFactory") def test_from_model_deployment( self, mock_client, mock_default_signer, - mock_get_model_deployment, + mock_from_id, mock_from_model_catalog, mock_model_deployment_state, mock_update_status, @@ -977,7 +977,7 @@ def test_from_model_deployment( test_model_id = "model_ocid" md_props = ModelDeploymentProperties(model_id=test_model_id) md = ModelDeployment(properties=md_props) - mock_get_model_deployment.return_value = md + mock_from_id.return_value = md test_model = MagicMock(model_deployment=md, _summary_status=SummaryStatus()) mock_from_model_catalog.return_value = test_model @@ -994,8 +994,8 @@ def test_from_model_deployment( compartment_id="test_compartment_id", ) - mock_get_model_deployment.assert_called_with( - model_deployment_id=test_model_deployment_id + mock_from_id.assert_called_with( + test_model_deployment_id ) mock_from_model_catalog.assert_called_with( model_id=test_model_id, @@ -1018,14 +1018,14 @@ def test_from_model_deployment( new_callable=PropertyMock, return_value=ModelDeploymentState.FAILED, ) - @patch.object(ModelDeployer, "get_model_deployment") + @patch.object(ModelDeployment, "from_id") @patch("ads.common.auth.default_signer") @patch("ads.common.oci_client.OCIClientFactory") def test_from_model_deployment_fail( self, mock_client, mock_default_signer, - mock_get_model_deployment, + mock_from_id, mock_model_deployment_state, ): """Tests loading model from model deployment.""" @@ -1035,7 +1035,7 @@ def test_from_model_deployment_fail( test_model_id = "model_ocid" md_props = ModelDeploymentProperties(model_id=test_model_id) md = ModelDeployment(properties=md_props) - mock_get_model_deployment.return_value = md + mock_from_id.return_value = md with pytest.raises(NotActiveDeploymentError): GenericModel.from_model_deployment( @@ -1049,21 +1049,21 @@ def test_from_model_deployment_fail( remove_existing_artifact=True, compartment_id="test_compartment_id", ) - mock_get_model_deployment.assert_called_with( - model_deployment_id=test_model_deployment_id + mock_from_id.assert_called_with( + test_model_deployment_id ) @patch.object(ModelDeployment, "update") - @patch.object(ModelDeployer, "get_model_deployment") + @patch.object(ModelDeployment, "from_id") @patch("ads.common.auth.default_signer") @patch("ads.common.oci_client.OCIClientFactory") def test_update_deployment_class_level( - self, mock_client, mock_signer, mock_get_model_deployment, mock_update + self, mock_client, mock_signer, mock_from_id, mock_update ): test_model_deployment_id = "xxxx.datasciencemodeldeployment.xxxx" md_props = ModelDeploymentProperties(model_id=test_model_deployment_id) md = ModelDeployment(properties=md_props) - mock_get_model_deployment.return_value = md + mock_from_id.return_value = md test_model = MagicMock(model_deployment=md, _summary_status=SummaryStatus()) mock_update.return_value = test_model @@ -1086,8 +1086,8 @@ def test_update_deployment_class_level( poll_interval=200, ) - mock_get_model_deployment.assert_called_with( - model_deployment_id=test_model_deployment_id + mock_from_id.assert_called_with( + test_model_deployment_id ) mock_update.assert_called_with( From 60080772b09ec8052c2970579fc4638ed42f805b Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 9 Jun 2023 10:30:51 -0700 Subject: [PATCH 12/20] Updated pr. --- ads/model/deployment/model_deployer.py | 2 +- ads/model/deployment/model_deployment.py | 2 +- ads/model/generic_model.py | 15 +++++++-------- ads/model/service/oci_datascience_model.py | 6 ++---- .../model_registration/model_deploy.rst | 14 +++++++------- 5 files changed, 18 insertions(+), 21 deletions(-) diff --git a/ads/model/deployment/model_deployer.py b/ads/model/deployment/model_deployer.py index e2f9e04bd..aba49e325 100644 --- a/ads/model/deployment/model_deployer.py +++ b/ads/model/deployment/model_deployer.py @@ -106,7 +106,7 @@ def __init__(self, config: dict = None, ds_client: DataScienceClient = None): """ if config: warnings.warn( - "`config` will be removed in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." + "`config` will be deprecated in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." ) self.ds_client = None diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index 5e0bcefa7..6bd27e836 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -701,7 +701,7 @@ def update( """ if properties: warnings.warn( - "Parameter `properties` will be removed from ModelDeployment update() in 3.0.0. Please use the builder pattern to update model deployment instance." + "Parameter `properties` will be removed from ModelDeployment `update()` in 3.0.0. Please use the builder pattern or kwargs to update model deployment instance." ) updated_properties = properties diff --git a/ads/model/generic_model.py b/ads/model/generic_model.py index 82c4de0be..3445c5d75 100644 --- a/ads/model/generic_model.py +++ b/ads/model/generic_model.py @@ -299,12 +299,12 @@ class GenericModel(MetadataMixin, Introspectable, EvaluatorMixin): >>> model.deploy() >>> # Update access log id, freeform tags and description for the model deployment >>> model.update_deployment( - >>> properties=ModelDeploymentProperties( - >>> access_log_id=, - >>> description="Description for Custom Model", - >>> freeform_tags={"key": "value"}, - >>> ) - >>> ) + ... access_log={ + ... log_id= + ... }, + ... description="Description for Custom Model", + ... freeform_tags={"key": "value"}, + ... ) >>> model.predict(2) >>> # Uncomment the line below to delete the model and the associated model deployment >>> # model.delete(delete_associated_model_deployment = True) @@ -1615,8 +1615,7 @@ def update_deployment( Examples -------- - >>> # Update access log id, freeform tags and description for the model deployment - >>> # Deprecated + >>> # Deprecated way to update access log id, freeform tags and description for the model deployment >>> model.update_deployment( >>> properties=ModelDeploymentProperties( >>> access_log_id=, diff --git a/ads/model/service/oci_datascience_model.py b/ads/model/service/oci_datascience_model.py index 8f0da620e..865bbf108 100644 --- a/ads/model/service/oci_datascience_model.py +++ b/ads/model/service/oci_datascience_model.py @@ -17,7 +17,7 @@ from ads.common.oci_mixin import OCIWorkRequestMixin from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.common.utils import extract_region -from ads.model.deployment.model_deployer import ModelDeployer +from ads.model.deployment import ModelDeployment from oci.data_science.models import ( ArtifactExportDetailsObjectStorage, ArtifactImportDetailsObjectStorage, @@ -469,9 +469,7 @@ def delete( logger.info( f"Deleting model deployment `{oci_model_deployment.identifier}`." ) - ModelDeployer(ds_client=self.client).delete( - model_deployment_id=oci_model_deployment.identifier - ) + ModelDeployment.from_id(oci_model_deployment.identifier).delete() logger.info(f"Deleting model `{self.id}`.") self.client.delete_model(self.id) diff --git a/docs/source/user_guide/model_registration/model_deploy.rst b/docs/source/user_guide/model_registration/model_deploy.rst index 46d17c9ea..098d66529 100644 --- a/docs/source/user_guide/model_registration/model_deploy.rst +++ b/docs/source/user_guide/model_registration/model_deploy.rst @@ -106,10 +106,10 @@ You can update the existing Model Deployment by using ``.update_deployment()`` m .. code-block:: python3 lightgbm_model.update_deployment( - properties=ModelDeploymentProperties( - access_log_id="ocid1.log.oc1.xxx.xxxxx", - description="Description for Custom Model", - freeform_tags={"key": "value"}, - ) - wait_for_completion = True, - ) + wait_for_completion = True, + access_log={ + log_id="ocid1.log.oc1.xxx.xxxxx", + }, + description="Description for Custom Model", + freeform_tags={"key": "value"}, + ) From cad3d4647ab4d6a3be3d3e22fb04e7c15569bb1b Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 9 Jun 2023 10:55:21 -0700 Subject: [PATCH 13/20] Updated pr. --- .../default_setup/model/test_oci_datascience_model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unitary/default_setup/model/test_oci_datascience_model.py b/tests/unitary/default_setup/model/test_oci_datascience_model.py index 5894973a1..29ab8afd4 100644 --- a/tests/unitary/default_setup/model/test_oci_datascience_model.py +++ b/tests/unitary/default_setup/model/test_oci_datascience_model.py @@ -24,7 +24,6 @@ from ads.common.oci_resource import SEARCH_TYPE, OCIResource from ads.dataset.progress import TqdmProgressBar from ads.model.datascience_model import _MAX_ARTIFACT_SIZE_IN_BYTES -from ads.model.deployment.model_deployer import ModelDeployer from ads.model.service.oci_datascience_model import ( ModelArtifactNotFoundError, ModelProvenanceNotFoundError, @@ -230,10 +229,10 @@ def test_delete_success(self, mock_client): mock_model_deployment.return_value = [ MagicMock(lifecycle_state="ACTIVE", identifier="md_id") ] - with patch.object(ModelDeployer, "delete") as mock_md_delete: + with patch("ads.model.deployment.ModelDeployment.from_id") as mock_from_id: with patch.object(OCIDataScienceModel, "sync") as mock_sync: self.mock_model.delete(delete_associated_model_deployment=True) - mock_md_delete.assert_called_with(model_deployment_id="md_id") + mock_from_id.assert_called_with("md_id") mock_client.delete_model.assert_called_with(self.mock_model.id) mock_sync.assert_called() From 60d5b7ec8402c184f4e810c91c4ba3deb938f58e Mon Sep 17 00:00:00 2001 From: Lu Peng Date: Fri, 9 Jun 2023 13:06:55 -0700 Subject: [PATCH 14/20] Updated pr. --- ads/model/deployment/model_deployer.py | 4 ++-- ads/model/deployment/model_deployment.py | 12 ++++++++---- .../deployment/model_deployment_properties.py | 2 +- ads/model/generic_model.py | 14 ++++---------- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/ads/model/deployment/model_deployer.py b/ads/model/deployment/model_deployer.py index aba49e325..0f40d7941 100644 --- a/ads/model/deployment/model_deployer.py +++ b/ads/model/deployment/model_deployer.py @@ -53,8 +53,8 @@ warnings.warn( ( - "The `ads.model.deployment.model_deployer` is deprecated in `oracle-ads 2.8.7` and will be removed in `oracle-ads 3.0`." - "Use `ModelDeployment` class in `ads.model.deployment` module for initializing and deploy model deployment. " + "The `ads.model.deployment.model_deployer` is deprecated in `oracle-ads 2.8.6` and will be removed in `oracle-ads 3.0`." + "Use `ModelDeployment` class in `ads.model.deployment` module for initializing and deploying model deployment. " "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/introduction.html" ), DeprecationWarning, diff --git a/ads/model/deployment/model_deployment.py b/ads/model/deployment/model_deployment.py index 6bd27e836..547333a17 100644 --- a/ads/model/deployment/model_deployment.py +++ b/ads/model/deployment/model_deployment.py @@ -297,17 +297,20 @@ def __init__( if config: warnings.warn( - "Parameter `config` will be removed from ModelDeployment constructor in 3.0.0 and will be ignored now. Please use `ads.set_auth()` to config the auth information." + "Parameter `config` was deprecated in 2.8.2 from ModelDeployment constructor and will be removed in 3.0.0. Please use `ads.set_auth()` to config the auth information. " + "Check: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/authentication.html" ) if properties: warnings.warn( - "Parameter `properties` will be removed from ModelDeployment constructor in 3.0.0. Please use `spec` or the builder pattern to initialize model deployment instance." + "Parameter `properties` was deprecated in 2.8.2 from ModelDeployment constructor and will be removed in 3.0.0. Please use `spec` or the builder pattern to initialize model deployment instance. " + "Check: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html" ) if model_deployment_url or model_deployment_id: warnings.warn( - "Parameter `model_deployment_url` and `model_deployment_id` will be removed from ModelDeployment constructor in 3.0.0 and will be ignored now. These two fields will be auto-populated from the service side." + "Parameter `model_deployment_url` and `model_deployment_id` were deprecated in 2.8.2 from ModelDeployment constructor and will be removed in 3.0.0. These two fields will be auto-populated from the service side. " + "Check: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html" ) initialize_spec = {} @@ -701,7 +704,8 @@ def update( """ if properties: warnings.warn( - "Parameter `properties` will be removed from ModelDeployment `update()` in 3.0.0. Please use the builder pattern or kwargs to update model deployment instance." + "Parameter `properties` is deprecated from ModelDeployment `update()` in 2.8.6 and will be removed in 3.0.0. Please use the builder pattern or kwargs to update model deployment instance. " + "Check: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/quick_start.html" ) updated_properties = properties diff --git a/ads/model/deployment/model_deployment_properties.py b/ads/model/deployment/model_deployment_properties.py index 67e039498..54a1c5740 100644 --- a/ads/model/deployment/model_deployment_properties.py +++ b/ads/model/deployment/model_deployment_properties.py @@ -16,7 +16,7 @@ warnings.warn( ( - "The `ads.model.deployment.model_deployment_properties` is deprecated in `oracle-ads 2.8.7` and will be removed in `oracle-ads 3.0`." + "The `ads.model.deployment.model_deployment_properties` is deprecated in `oracle-ads 2.8.6` and will be removed in `oracle-ads 3.0`." "Use `ModelDeploymentInfrastructure` and `ModelDeploymentRuntime` classes in `ads.model.deployment` module for configuring model deployment. " "Check https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/introduction.html" ), diff --git a/ads/model/generic_model.py b/ads/model/generic_model.py index 3445c5d75..f39aa27dc 100644 --- a/ads/model/generic_model.py +++ b/ads/model/generic_model.py @@ -1615,15 +1615,7 @@ def update_deployment( Examples -------- - >>> # Deprecated way to update access log id, freeform tags and description for the model deployment - >>> model.update_deployment( - >>> properties=ModelDeploymentProperties( - >>> access_log_id=, - >>> description="Description for Custom Model", - >>> freeform_tags={"key": "value"}, - >>> ) - >>> ) - >>> # New way to update access log id, freeform tags and description for the model deployment + >>> # Update access log id, freeform tags and description for the model deployment >>> model.update_deployment( ... access_log={ ... log_id= @@ -1673,8 +1665,10 @@ def update_deployment( """ if properties: warnings.warn( - "Parameter `properties` will be removed from GenericModel `update_deployment()` in 3.0.0. Please use kwargs to update model deployment." + "Parameter `properties` is deprecated from GenericModel `update_deployment()` in 2.8.6 and will be removed in 3.0.0. Please use kwargs to update model deployment. " + "Check: https://accelerated-data-science.readthedocs.io/en/latest/user_guide/model_registration/introduction.html" ) + if not inspect.isclass(cls): if cls.model_deployment: return cls.model_deployment.update( From f46439d9305a5e04e40a6967448c74c8820d24f3 Mon Sep 17 00:00:00 2001 From: MING KANG Date: Fri, 9 Jun 2023 16:39:18 -0700 Subject: [PATCH 15/20] feature/ODSC-41635/Support Data Flow Pools (#212) --- ads/jobs/builders/infrastructure/dataflow.py | 41 +++++++++++++++-- .../apachespark/dataflow-spark-magic.rst | 2 +- .../user_guide/apachespark/dataflow.rst | 22 ++++----- .../default_setup/jobs/test_jobs_dataflow.py | 45 ++++++++++--------- 4 files changed, 74 insertions(+), 36 deletions(-) diff --git a/ads/jobs/builders/infrastructure/dataflow.py b/ads/jobs/builders/infrastructure/dataflow.py index c41697608..3caae3372 100644 --- a/ads/jobs/builders/infrastructure/dataflow.py +++ b/ads/jobs/builders/infrastructure/dataflow.py @@ -391,6 +391,7 @@ class DataFlow(Infrastructure): CONST_OCPUS = "ocpus" CONST_ID = "id" CONST_PRIVATE_ENDPOINT_ID = "private_endpoint_id" + CONST_POOL_ID = "pool_id" CONST_FREEFORM_TAGS = "freeform_tags" CONST_DEFINED_TAGS = "defined_tags" @@ -411,8 +412,9 @@ class DataFlow(Infrastructure): CONST_OCPUS: CONST_OCPUS, CONST_ID: CONST_ID, CONST_PRIVATE_ENDPOINT_ID: "privateEndpointId", + CONST_POOL_ID: "poolId", CONST_FREEFORM_TAGS: "freeformTags", - CONST_DEFINED_TAGS: "definedTags" + CONST_DEFINED_TAGS: "definedTags", } def __init__(self, spec: dict = None, **kwargs): @@ -425,8 +427,10 @@ def __init__(self, spec: dict = None, **kwargs): spec = { k: v for k, v in spec.items() - if (f"with_{camel_to_snake(k)}" in self.__dir__() - or (k == "defined_tags" or "freeform_tags")) + if ( + f"with_{camel_to_snake(k)}" in self.__dir__() + or (k == "defined_tags" or "freeform_tags") + ) and v is not None } defaults.update(spec) @@ -809,10 +813,34 @@ def with_defined_tag(self, **kwargs) -> "DataFlow": """ return self.set_spec(self.CONST_DEFINED_TAGS, kwargs) + def with_pool_id(self, pool_id: str) -> "DataFlow": + """ + Set the Data Flow Pool Id for a Data Flow job. + + Parameters + ---------- + pool_id: str + The OCID of a Data Flow Pool. + + Returns + ------- + DataFlow + the Data Flow instance itself + """ + if not hasattr(CreateApplicationDetails, "pool_id"): + raise EnvironmentError( + "Data Flow Pool has not been supported in the current OCI SDK installed." + ) + return self.set_spec(self.CONST_POOL_ID, pool_id) + def __getattr__(self, item): if item == self.CONST_DEFINED_TAGS or item == self.CONST_FREEFORM_TAGS: return self.get_spec(item) - elif f"with_{item}" in self.__dir__() and item != "defined_tag" and item != "freeform_tag": + elif ( + f"with_{item}" in self.__dir__() + and item != "defined_tag" + and item != "freeform_tag" + ): return self.get_spec(item) raise AttributeError(f"Attribute {item} not found.") @@ -832,6 +860,11 @@ def create(self, runtime: DataFlowRuntime, **kwargs) -> "DataFlow": DataFlow a Data Flow job instance """ + if self.pool_id: + if not hasattr(CreateApplicationDetails, "pool_id"): + raise EnvironmentError( + "Data Flow Pool has not been supported in the current OCI SDK installed." + ) # Set default display_name if not specified - randomly generated easy to remember name if not self.name: self.name = utils.get_random_name_for_resource() diff --git a/docs/source/user_guide/apachespark/dataflow-spark-magic.rst b/docs/source/user_guide/apachespark/dataflow-spark-magic.rst index dfbe0a6d6..8ec7b88f3 100644 --- a/docs/source/user_guide/apachespark/dataflow-spark-magic.rst +++ b/docs/source/user_guide/apachespark/dataflow-spark-magic.rst @@ -32,7 +32,7 @@ Data Flow Sessions are accessible through the following conda environment: * PySpark 3.2 and Data Flow 2.0 (pyspark32_p38_cpu_v2) -You can customize **pypspark32_p38_cpu_v1**, publish it, and use it as a runtime environment for a Data Flow Session. +You can customize **pypspark32_p38_cpu_v2**, publish it, and use it as a runtime environment for a Data Flow Session. Policies ******** diff --git a/docs/source/user_guide/apachespark/dataflow.rst b/docs/source/user_guide/apachespark/dataflow.rst index 48ce64045..51868e050 100644 --- a/docs/source/user_guide/apachespark/dataflow.rst +++ b/docs/source/user_guide/apachespark/dataflow.rst @@ -159,9 +159,9 @@ You could submit a notebook using ADS SDK APIs. Here is an example to submit a n "ocid1.compartment.oc1." ) .with_driver_shape("VM.Standard.E4.Flex") - .with_driver_shape_config(ocpus=2, memory_in_gbs=32) - .with_executor_shape("VM.Standard.E4.Flex") - .with_executor_shape_config(ocpus=4, memory_in_gbs=64) + .with_driver_shape_config(ocpus=2, memory_in_gbs=32) + .with_executor_shape("VM.Standard.E4.Flex") + .with_executor_shape_config(ocpus=4, memory_in_gbs=64) .with_logs_bucket_uri("oci://mybucket@mytenancy/") .with_private_endpoint_id("ocid1.dataflowprivateendpoint.oc1.iad.") .with_configuration({ @@ -231,8 +231,8 @@ create applications. In the following "hello-world" example, ``DataFlow`` is populated with ``compartment_id``, ``driver_shape``, ``driver_shape_config``, ``executor_shape``, ``executor_shape_config`` -, ``spark_version``, ``defined_tags`` and ``freeform_tags``. ``DataFlowRuntime`` is -populated with ``script_uri`` and ``script_bucket``. The ``script_uri`` specifies the +, ``spark_version``, ``defined_tags`` and ``freeform_tags``. ``DataFlowRuntime`` is +populated with ``script_uri`` and ``script_bucket``. The ``script_uri`` specifies the path to the script. It can be local or remote (an Object Storage path). If the path is local, then ``script_bucket`` must be specified additionally because Data Flow requires a script to be available in Object Storage. ADS @@ -270,9 +270,9 @@ accepted. In the next example, the prefix is given for ``script_bucket``. .with_compartment_id("oci.xx.") .with_logs_bucket_uri("oci://mybucket@mynamespace/dflogs") .with_driver_shape("VM.Standard.E4.Flex") - .with_driver_shape_config(ocpus=2, memory_in_gbs=32) - .with_executor_shape("VM.Standard.E4.Flex") - .with_executor_shape_config(ocpus=4, memory_in_gbs=64) + .with_driver_shape_config(ocpus=2, memory_in_gbs=32) + .with_executor_shape("VM.Standard.E4.Flex") + .with_executor_shape_config(ocpus=4, memory_in_gbs=64) .with_spark_version("3.0.2") .with_defined_tag( **{"Oracle-Tags": {"CreatedBy": "test_name@oracle.com"}} @@ -391,9 +391,9 @@ In the next example, ``archive_uri`` is given as an Object Storage location. .with_compartment_id("oci1.xxx.") .with_logs_bucket_uri("oci://mybucket@mynamespace/prefix") .with_driver_shape("VM.Standard.E4.Flex") - .with_driver_shape_config(ocpus=2, memory_in_gbs=32) - .with_executor_shape("VM.Standard.E4.Flex") - .with_executor_shape_config(ocpus=4, memory_in_gbs=64) + .with_driver_shape_config(ocpus=2, memory_in_gbs=32) + .with_executor_shape("VM.Standard.E4.Flex") + .with_executor_shape_config(ocpus=4, memory_in_gbs=64) .with_spark_version("3.0.2") .with_configuration({ "spark.driverEnv.myEnvVariable": "value1", diff --git a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py index 1f59f6fc1..9e6d89a28 100644 --- a/tests/unitary/default_setup/jobs/test_jobs_dataflow.py +++ b/tests/unitary/default_setup/jobs/test_jobs_dataflow.py @@ -28,6 +28,7 @@ DataFlowRuntime, DataFlowNotebookRuntime, ) +from oci.data_flow.models import CreateApplicationDetails logger.setLevel(logging.DEBUG) @@ -47,7 +48,13 @@ language="PYTHON", logs_bucket_uri="oci://test_bucket@test_namespace/", private_endpoint_id="test_private_endpoint", + pool_id="ocid1.dataflowpool.oc1..", ) +EXPECTED_YAML_LENGTH = 614 +if not hasattr(CreateApplicationDetails, "pool_id"): + SAMPLE_PAYLOAD.pop("pool_id") + EXPECTED_YAML_LENGTH = 567 + random_seed = 42 @@ -124,7 +131,7 @@ def test_create_delete(self, mock_to_dict, mock_client): df.lifecycle_state == oci.data_flow.models.Application.LIFECYCLE_STATE_DELETED ) - assert len(df.to_yaml()) == 567 + assert len(df.to_yaml()) == EXPECTED_YAML_LENGTH def test_create_df_app_with_default_display_name( self, @@ -319,7 +326,7 @@ def df(self): ).with_num_executors( 2 ).with_private_endpoint_id( - "test_private_endpoint" + SAMPLE_PAYLOAD["private_endpoint_id"] ).with_freeform_tag( test_freeform_tags_key="test_freeform_tags_value", ).with_defined_tag( @@ -327,6 +334,8 @@ def df(self): "test_defined_tags_key": "test_defined_tags_value" } ) + if SAMPLE_PAYLOAD.get("pool_id", None): + df.with_pool_id(SAMPLE_PAYLOAD["pool_id"]) return df def test_create_with_builder_pattern(self, mock_to_dict, mock_client, df): @@ -341,6 +350,8 @@ def test_create_with_builder_pattern(self, mock_to_dict, mock_client, df): "test_defined_tags_key": "test_defined_tags_value" } } + if SAMPLE_PAYLOAD.get("pool_id", None): + assert df.pool_id == SAMPLE_PAYLOAD["pool_id"] rt = ( DataFlowRuntime() @@ -483,50 +494,44 @@ def test_to_and_from_dict(self, df): assert df3_dict["spec"]["numExecutors"] == 2 def test_shape_and_details(self, mock_to_dict, mock_client, df): - df.with_driver_shape( - "VM.Standard2.1" - ).with_executor_shape( + df.with_driver_shape("VM.Standard2.1").with_executor_shape( "VM.Standard.E4.Flex" ) rt = ( DataFlowRuntime() - .with_script_uri(SAMPLE_PAYLOAD["file_uri"]) - .with_archive_uri(SAMPLE_PAYLOAD["archive_uri"]) - .with_custom_conda( - "oci://my_bucket@my_namespace/conda_environments/cpu/PySpark 3.0 and Data Flow/5.0/pyspark30_p37_cpu_v5" - ) - .with_overwrite(True) + .with_script_uri(SAMPLE_PAYLOAD["file_uri"]) + .with_archive_uri(SAMPLE_PAYLOAD["archive_uri"]) + .with_custom_conda( + "oci://my_bucket@my_namespace/conda_environments/cpu/PySpark 3.0 and Data Flow/5.0/pyspark30_p37_cpu_v5" + ) + .with_overwrite(True) ) with pytest.raises( ValueError, - match="`executor_shape` and `driver_shape` must be from the same shape family." + match="`executor_shape` and `driver_shape` must be from the same shape family.", ): with patch.object(DataFlowApp, "client", mock_client): with patch.object(DataFlowApp, "to_dict", mock_to_dict): df.create(rt) - df.with_driver_shape( - "VM.Standard2.1" - ).with_driver_shape_config( + df.with_driver_shape("VM.Standard2.1").with_driver_shape_config( memory_in_gbs=SAMPLE_PAYLOAD["driver_shape_config"]["memory_in_gbs"], ocpus=SAMPLE_PAYLOAD["driver_shape_config"]["ocpus"], - ).with_executor_shape( - "VM.Standard2.16" - ).with_executor_shape_config( + ).with_executor_shape("VM.Standard2.16").with_executor_shape_config( memory_in_gbs=SAMPLE_PAYLOAD["executor_shape_config"]["memory_in_gbs"], ocpus=SAMPLE_PAYLOAD["executor_shape_config"]["ocpus"], ) with pytest.raises( ValueError, - match="Shape config is not required for non flex shape from user end." + match="Shape config is not required for non flex shape from user end.", ): with patch.object(DataFlowApp, "client", mock_client): with patch.object(DataFlowApp, "to_dict", mock_to_dict): df.create(rt) - + class TestDataFlowNotebookRuntime: @pytest.mark.skipif( From c46e8a378c0fcbad2b86cbed3ab2986b017c5915 Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 9 Jun 2023 17:02:37 -0700 Subject: [PATCH 16/20] Adds a decorator to generate watch command for the run operation. --- ads/opctl/backend/ads_dataflow.py | 10 ++++++---- ads/opctl/backend/ads_ml_job.py | 13 +++++++------ ads/opctl/backend/ads_ml_pipeline.py | 12 +++++++++--- ads/opctl/backend/base.py | 6 +++--- ads/opctl/cmds.py | 1 + docs/source/user_guide/cli/opctl/_template/jobs.rst | 12 ++++++------ 6 files changed, 32 insertions(+), 22 deletions(-) diff --git a/ads/opctl/backend/ads_dataflow.py b/ads/opctl/backend/ads_dataflow.py index fd311d965..1f5cca128 100644 --- a/ads/opctl/backend/ads_dataflow.py +++ b/ads/opctl/backend/ads_dataflow.py @@ -11,6 +11,7 @@ from typing import Dict, Union from ads.opctl.backend.base import Backend +from ads.opctl.decorator.common import print_watch_command from ads.common.auth import create_signer, AuthContext from ads.common.oci_client import OCIClientFactory @@ -114,21 +115,22 @@ def init( **kwargs, ) - def apply(self): + def apply(self) -> Dict: """ Create DataFlow and DataFlow Run from YAML. """ # TODO add the logic for build dataflow and dataflow run from YAML. raise NotImplementedError(f"`apply` hasn't been supported for data flow yet.") - def run(self) -> None: + @print_watch_command + def run(self) -> Dict: """ Create DataFlow and DataFlow Run from OCID or cli parameters. """ with AuthContext(auth=self.auth_type, profile=self.profile): if self.config["execution"].get("ocid", None): - data_flow_id = self.config["execution"]["ocid"] - run_id = Job.from_dataflow_job(data_flow_id).run().id + job_id = self.config["execution"]["ocid"] + run_id = Job.from_dataflow_job(job_id).run().id else: infra = self.config.get("infrastructure", {}) if any(k not in infra for k in REQUIRED_FIELDS): diff --git a/ads/opctl/backend/ads_ml_job.py b/ads/opctl/backend/ads_ml_job.py index 84077c26a..4be274800 100644 --- a/ads/opctl/backend/ads_ml_job.py +++ b/ads/opctl/backend/ads_ml_job.py @@ -27,12 +27,10 @@ ScriptRuntime, ) from ads.opctl import logger -from ads.opctl.backend.base import ( - Backend, - RuntimeFactory, -) +from ads.opctl.backend.base import Backend, RuntimeFactory from ads.opctl.config.resolver import ConfigResolver from ads.opctl.constants import DEFAULT_IMAGE_SCRIPT_DIR +from ads.opctl.decorator.common import print_watch_command from ads.opctl.distributed.common.cluster_config_helper import ( ClusterConfigToJobSpecConverter, ) @@ -126,7 +124,8 @@ def init( **kwargs, ) - def apply(self) -> None: + @print_watch_command + def apply(self) -> Dict: """ Create Job and Job Run from YAML. """ @@ -136,8 +135,10 @@ def apply(self) -> None: job_run = job.run() print("JOB OCID:", job.id) print("JOB RUN OCID:", job_run.id) + return {"job_id": job.id, "run_id": job_run.id} - def run(self) -> None: + @print_watch_command + def run(self) -> Dict: """ Create Job and Job Run from OCID or cli parameters. """ diff --git a/ads/opctl/backend/ads_ml_pipeline.py b/ads/opctl/backend/ads_ml_pipeline.py index e53b771b2..b0328205f 100644 --- a/ads/opctl/backend/ads_ml_pipeline.py +++ b/ads/opctl/backend/ads_ml_pipeline.py @@ -9,6 +9,7 @@ from ads.common.oci_client import OCIClientFactory from ads.opctl.backend.base import Backend from ads.opctl.backend.ads_ml_job import JobRuntimeFactory +from ads.opctl.decorator.common import print_watch_command from ads.pipeline import Pipeline, PipelineRun, PipelineStep, CustomScriptStep from ads.jobs import PythonRuntime @@ -34,7 +35,8 @@ def __init__(self, config: Dict) -> None: self.profile = config["execution"].get("oci_profile", None) self.client = OCIClientFactory(**self.oci_auth).data_science - def apply(self) -> None: + @print_watch_command + def apply(self) -> Dict: """ Create Pipeline and Pipeline Run from YAML. """ @@ -44,8 +46,10 @@ def apply(self) -> None: pipeline_run = pipeline.run() print("PIPELINE OCID:", pipeline.id) print("PIPELINE RUN OCID:", pipeline_run.id) + return {"job_id": pipeline.id, "run_id": pipeline_run.id} - def run(self) -> None: + @print_watch_command + def run(self) -> Dict: """ Create Pipeline and Pipeline Run from OCID. """ @@ -53,7 +57,9 @@ def run(self) -> None: with AuthContext(auth=self.auth_type, profile=self.profile): pipeline = Pipeline.from_ocid(ocid=pipeline_id) pipeline_run = pipeline.run() + print("PIPELINE OCID:", pipeline.id) print("PIPELINE RUN OCID:", pipeline_run.id) + return {"job_id": pipeline.id, "run_id": pipeline_run.id} def delete(self) -> None: """ @@ -84,7 +90,7 @@ def watch(self) -> None: Watch Pipeline Run from OCID. """ run_id = self.config["execution"]["run_id"] - log_type = self.config["execution"]["log_type"] + log_type = self.config["execution"].get("log_type") with AuthContext(auth=self.auth_type, profile=self.profile): PipelineRun.from_ocid(run_id).watch(log_type=log_type) diff --git a/ads/opctl/backend/base.py b/ads/opctl/backend/base.py index 6b01b1c79..0df2e3a13 100644 --- a/ads/opctl/backend/base.py +++ b/ads/opctl/backend/base.py @@ -32,7 +32,7 @@ def run(self) -> Dict: Returns ------- - None + Dict """ def delete(self) -> None: @@ -62,13 +62,13 @@ def cancel(self) -> None: None """ - def apply(self) -> None: + def apply(self) -> Dict: """ Initiate Data Science service from YAML. Returns ------- - None + Dict """ def activate(self) -> None: diff --git a/ads/opctl/cmds.py b/ads/opctl/cmds.py index 57633ecde..6c899f123 100644 --- a/ads/opctl/cmds.py +++ b/ads/opctl/cmds.py @@ -246,6 +246,7 @@ def run(config: Dict, **kwargs) -> Dict: if ( "kind" in p.config and p.config["execution"].get("backend", None) != BACKEND_NAME.LOCAL.value + and "ocid" not in p.config["execution"] ): p.config["execution"]["backend"] = p.config["kind"] return _BackendFactory(p.config).backend.apply() diff --git a/docs/source/user_guide/cli/opctl/_template/jobs.rst b/docs/source/user_guide/cli/opctl/_template/jobs.rst index 222de2eb9..db2cbc6d6 100644 --- a/docs/source/user_guide/cli/opctl/_template/jobs.rst +++ b/docs/source/user_guide/cli/opctl/_template/jobs.rst @@ -7,12 +7,12 @@ Prerequisite :doc:`Install ADS CLI <../../quickstart>` -Running a Pre Defined Job +Running a Pre Defined Job ------------------------- .. code-block:: shell - ads opctl run -j + ads opctl run --ocid Delete Job or Job Run --------------------- @@ -36,15 +36,15 @@ Stop a running cluster using ``cancel`` subcommand. **Option 1: Using Job OCID and Work Dir** .. code-block:: shell - - ads opctl cancel -j --work-dir + + ads opctl cancel --work-dir **Option 2: Using cluster info file** Cluster info file is a yaml file with output generated from ``ads opctl run -f`` .. code-block:: shell - - ads opctl cancel -j --work-dir + + ads opctl cancel --work-dir This command requires an api key or resource principal setup. The logs are streamed from the logging service. If your job is not attached to logging service, this option will show only the lifecycle state. From b7277508002250ec2d2e3342e6f58f0a84f6cec1 Mon Sep 17 00:00:00 2001 From: Dmitrii Cherkasov Date: Fri, 9 Jun 2023 17:20:11 -0700 Subject: [PATCH 17/20] Adds a decorator to generate watch command for the run operation. --- ads/opctl/decorator/__init__.py | 5 +++++ ads/opctl/decorator/common.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 ads/opctl/decorator/__init__.py create mode 100644 ads/opctl/decorator/common.py diff --git a/ads/opctl/decorator/__init__.py b/ads/opctl/decorator/__init__.py new file mode 100644 index 000000000..53955a5a2 --- /dev/null +++ b/ads/opctl/decorator/__init__.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python +# -*- coding: utf-8; -*- + +# Copyright (c) 2023 Oracle and its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ diff --git a/ads/opctl/decorator/common.py b/ads/opctl/decorator/common.py new file mode 100644 index 000000000..98100977c --- /dev/null +++ b/ads/opctl/decorator/common.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8; -*- + +# Copyright (c) 2023 Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +from typing import Dict, Callable +from functools import wraps + +RUN_ID_FIELD = "run_id" + +def print_watch_command(func: callable)->Callable: + """The decorator to help build the `opctl watch` command.""" + @wraps(func) + def wrapper(*args, **kwargs)->Dict: + result = func(*args, **kwargs) + if result and isinstance(result, Dict) and RUN_ID_FIELD in result: + msg_header = ( + f"{'*' * 40} To monitor the progress of a task, execute the following command {'*' * 40}" + ) + print(msg_header) + print(f"ads opctl watch {result[RUN_ID_FIELD]}") + print("*" * len(msg_header)) + return result + return wrapper \ No newline at end of file From be1e35f67d7a71e9c040961a3a39f0e83d9caa07 Mon Sep 17 00:00:00 2001 From: Liuda Rudenka Date: Mon, 12 Jun 2023 13:39:22 -0500 Subject: [PATCH 18/20] Update THIRD_PARTY_LICENSES.txt (#223) --- .github/workflows/add-3plicense-warning.yml | 43 ++ THIRD_PARTY_LICENSES.txt | 520 +++++++++++--------- 2 files changed, 323 insertions(+), 240 deletions(-) create mode 100644 .github/workflows/add-3plicense-warning.yml diff --git a/.github/workflows/add-3plicense-warning.yml b/.github/workflows/add-3plicense-warning.yml new file mode 100644 index 000000000..4d6ea0aef --- /dev/null +++ b/.github/workflows/add-3plicense-warning.yml @@ -0,0 +1,43 @@ +name: "Add 3P License Warning to PR" + +on: + pull_request: + paths: + - setup.py + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + +jobs: + warning-message-3plicense: + name: Add ⚠️ Warning about 3p license + runs-on: ubuntu-latest + if: ${{ success() }} && ${{ github.event.issue.pull_request }} + steps: + - run: | + BODY_MSG=$(cat << EOF + ⚠️ This PR changed **setup.py** file. ⚠️ + - PR Creator must update 📃 THIRD_PARTY_LICENSES.txt, if any 📚 library added/removed in **setup.py**. + - PR Approver must confirm 📃 THIRD_PARTY_LICENSES.txt updated, if any 📚 library added/removed in **setup.py**. + EOF + ) + echo "BODY_MSG<> $GITHUB_ENV + echo "$BODY_MSG" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + - uses: actions/github-script@v6 + with: + github-token: ${{ github.token }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `${{ env.BODY_MSG }}` + }) diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index 5f4b710ac..a6b0aab38 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -2,137 +2,14 @@ Advanced Data Science and Data Services toolkit for Python Third Party License F ------------------------ Third Party Components ------------------------ ------------------------------- Licenses ------------------------------- -- MIT License - Apache License 2.0 - BSD 3-Clause "New" or "Revised" License +- Google Protobuf License +- ISC License (ISCL) (ISC) +- Matplotlib License +- MIT License - Mozilla Public License 2.0 (MPL 2.0) - Universal Permissive License (UPL) -- Matplotlib License --------------------------------- Notices ------------------------------- -=== NOTICE file corresponding to the section 4 d of the Apache License, === -=== Version 2.0, in this case for fdk-python code. ======================== - - Fn Project - ============ - - Copyright (c) 2019, 2020 Oracle and/or its affiliates. All rights reserved. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - ========================================================================== - Third Party Dependencies - ========================================================================== - - This project includes or depends on code from third party projects. - Attributions are contained in THIRD_PARTY_LICENSES.txt - - -=== NOTICE file corresponding to the section 4 d of the Apache License, === -=== Version 2.0, in this case for oci code. =============================== - - (c) 2016, 2020, Oracle and/or its affiliates. - - -=== NOTICE file corresponding to the section 4 d of the Apache License, === -=== Version 2.0, in this case for pyarrow code. =========================== - - - Apache Arrow - Copyright 2016-2019 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - This product includes software from the SFrame project (BSD, 3-clause). - * Copyright (C) 2015 Dato, Inc. - * Copyright (c) 2009 Carnegie Mellon University. - - This product includes software from the Feather project (Apache 2.0) - https://github.com/wesm/feather - - This product includes software from the DyND project (BSD 2-clause) - https://github.com/libdynd - - This product includes software from the LLVM project - * distributed under the University of Illinois Open Source - - This product includes software from the google-lint project - * Copyright (c) 2009 Google Inc. All rights reserved. - - This product includes software from the mman-win32 project - * Copyright https://code.google.com/p/mman-win32/ - * Licensed under the MIT License; - - This product includes software from the LevelDB project - * Copyright (c) 2011 The LevelDB Authors. All rights reserved. - * Use of this source code is governed by a BSD-style license that can be - * Moved from Kudu http://github.com/cloudera/kudu - - This product includes software from the CMake project - * Copyright 2001-2009 Kitware, Inc. - * Copyright 2012-2014 Continuum Analytics, Inc. - * All rights reserved. - - This product includes software from https://github.com/matthew-brett/multibuild (BSD 2-clause) - * Copyright (c) 2013-2016, Matt Terry and Matthew Brett; all rights reserved. - - This product includes software from the Ibis project (Apache 2.0) - * Copyright (c) 2015 Cloudera, Inc. - * https://github.com/cloudera/ibis - - This product includes software from Dremio (Apache 2.0) - * Copyright (C) 2017-2018 Dremio Corporation - * https://github.com/dremio/dremio-oss - - This product includes software from Google Guava (Apache 2.0) - * Copyright (C) 2007 The Guava Authors - * https://github.com/google/guava - - This product include software from CMake (BSD 3-Clause) - * CMake - Cross Platform Makefile Generator - * Copyright 2000-2019 Kitware, Inc. and Contributors - - The web site includes files generated by Jekyll. - - -------------------------------------------------------------------------------- - - This product includes code from Apache Kudu, which includes the following in - its NOTICE file: - - Apache Kudu - Copyright 2016 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (http://www.apache.org/). - - Portions of this software were developed at - Cloudera, Inc (http://www.cloudera.com/). - - -------------------------------------------------------------------------------- - - This product includes code from Apache ORC, which includes the following in - its NOTICE file: - - Apache ORC - Copyright 2013-2019 The Apache Software Foundation - - This product includes software developed by The Apache Software - Foundation (http://www.apache.org/). - - This product includes software developed by Hewlett-Packard: - (c) Copyright [2014-2015] Hewlett-Packard Development Company, L.P - ------------------------------------------------------------------------- ======================== Third Party Components ======================== asteval @@ -154,35 +31,35 @@ cerberus * Project home: http://docs.python-cerberus.org/ cloudpickle -* Copyright (c) 2015, Cloudpickle contributors. +* Copyright (c) 2015, Cloudpickle contributors. * License: BSD 3-Clause "New" or "Revised" License * Source code: https://github.com/cloudpipe/cloudpickle * Project home: https://github.com/cloudpipe/cloudpickle +conda-pack +* Copyright (c) 2017, Jim Crist and contributors +* License: BSD-3-Clause license +* Source code: https://github.com/conda/conda-pack +* Project home: https://conda.github.io/conda-pack/ + datefinder * Copyright (c) 2015 Alec Koumjian * License: MIT License * Source code: https://github.com/akoumjian/datefinder * Project home: https://github.com/akoumjian/datefinder +docker +* Copyright 2016 Docker, Inc. +* License: Apache-2.0 license +* Source code: https://github.com/docker +* Project home: https://www.docker.com/ + fastavro * Copyright (c) 2011 Miki Tebeka * License: MIT License * Source code: https://github.com/fastavro/fastavro * Project home: https://github.com/fastavro/fastavro -fastparquet -* Copyright -* License: Apache License 2.0 -* Source code: https://github.com/dask/fastparquet/ -* Project home: https://github.com/dask/fastparquet/ - -fdk-python -* Copyright: see notices section above -* License: Apache License 2.0 -* Source code: https://github.com/fnproject/fdk-python -* Project home: https://fnproject.io/ - folium * Copyright (C) 2013, Rob Story * License: MIT License @@ -213,12 +90,31 @@ graphviz * Source code: https://github.com/xflr6/graphviz * Project home: https://github.com/xflr6/graphviz +hdfs +* Copyright (c) 2013, Matthieu Monsch. +* License: MIT License +* Source code: https://github.com/mtth/hdfs +* Project home: https://hdfscli.readthedocs.io/en/latest/ + htmllistparse * Copyright (c) 2016-2017 Dingyuan Wang * License: MIT License * Source code: https://github.com/gumblex/htmllisting-parser * Project home: https://github.com/gumblex/htmllisting-parser +ibisframework + +* Copyright 2015 Cloudera Inc. +* License: Apache-2.0 license +* Source code: https://github.com/ibis-project/ibis +* Project home: https://ibis-project.org/ + +inflection +* Copyright (C) 2012-2020 Janne Vanhala +* License: MIT license +* Source code: https://github.com/jpvanhal/inflection +* Project home: https://github.com/jpvanhal/inflection + ipython * Copyright (c) 2008-Present, IPython Development Team * License: BSD 3-Clause "New" or "Revised" License @@ -237,17 +133,29 @@ jinja2 * Source code: https://github.com/pallets/jinja/ * Project home: https://palletsprojects.com/p/jinja/ +lightgbm +* Copyright (c) 2023 Microsoft Corporation +* License: MIT license +* Source code: https://github.com/microsoft/LightGBM +* Project home: https://github.com/microsoft/LightGBM + matplotlib * Copyright * License: Matplotlib License * Source code: https://github.com/matplotlib/matplotlib * Project home: https://matplotlib.org/ -numexpr -* Copyright (c) 2011- See AUTHORS.txt [in the numexpr Source Code] -* License: MIT License -* Source code: https://github.com/pydata/numexpr -* Project home: https://github.com/pydata/numexpr +nbconvert +* Copyright (c) 2001-2015, IPython Development Team, Copyright (c) 2015-, Jupyter Development Team +* License: BSD-3-Clause license +* Source code: https://github.com/jupyter/nbconvert +* Project home: https://jupyter.org/ + +nbformat +* Copyright (c) 2001-2015, IPython Development Team, Copyright (c) 2015-, Jupyter Development Team +* License: BSD-3-Clause license +* Source code: https://github.com/jupyter/nbconvert +* Project home: https://jupyter.org/ numpy * Copyright (c) 2005-2021, NumPy Developers. @@ -255,6 +163,12 @@ numpy * Source code: https://github.com/numpy/numpy * Project home: https://www.numpy.org/ +oci-cli +* Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. +* License: The Universal Permissive License (UPL), Version 1.0 +* Source code: https://github.com/oracle/oci-cli +* Project home: https://docs.oracle.com/en-us/iaas/Content/API/Concepts/cliconcepts.htm + oci * Copyright (c) 2016, 2020, Oracle and/or its affiliates. [see notices section above] * License: Apache License 2.0, Universal Permissive License (UPL) @@ -268,13 +182,13 @@ ocifs * Project home: https://pypi.org/project/ocifs/ onnx -* Copyright +* Copyright (c) 2023 ONNX Project Contributors. * License: Apache License 2.0 * Source code: https://github.com/onnx/onnx/ * Project home: https://onnxruntime.ai/ onnxmltools -* Copyright +* Copyright (c) Microsoft Corporation * License: Apache License 2.0 * Source code: https://github.com/onnx/onnxmltools * Project home: https://github.com/onnx/onnxmltools @@ -297,6 +211,12 @@ optuna * Source code: https://github.com/optuna/optuna * Project home: https://optuna.org/ +oracledb +* Copyright (c) 2016, 2022 Oracle and/or its affiliates. +* License: Apache Software License Version 2.0, Universal Permissive License (UPL) (Apache and/or UPL) +* Source code: https://github.com/oracle/python-oracledb +* Project home: https://oracle.github.io/python-oracledb/ + pandas * Copyright (c) 2011-2021, Open source contributors. * License: BSD 3-Clause "New" or "Revised" License @@ -309,17 +229,23 @@ pandavro * Source code: https://github.com/ynqa/pandavro * Project home: https://github.com/ynqa/pandavro -py-cpuinfo -* Copyright (c) 2014-2021 Matthew Brennan Jones -* License: MIT License -* Source code: https://github.com/workhorsy/py-cpuinfo -* Project home: https://github.com/workhorsy/py-cpuinfo +protobuf +* Copyright 2008 Google Inc. All rights reserved. +* License: Google Protobuf License +* Source code: https://github.com/protocolbuffers/protobuf +* Project home: https://protobuf.dev/ -pyarrow -* Copyright: see notices section above -* License: Apache License 2.0 -* Source code: https://github.com/apache/arrow -* Project home: https://arrow.apache.org/ +psutil +* Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola. All rights reserved. +* License: BSD-3-Clause license +* Source code: https://github.com/giampaolo/psutil +* Project home: https://github.com/giampaolo/psutil + +pyspark +* Copyright 2014 and onwards The Apache Software Foundation. +* License: Apache-2.0 LICENSE +* Source code: https://github.com/apache/spark/tree/master/python +* Project home: https://spark.apache.org/ python_jsonschema_objects * Copyright (c) 2014 Chris Wacek @@ -327,11 +253,17 @@ python_jsonschema_objects * Source code: https://github.com/cwacek/python-jsonschema-objects * Project home: python-jsonschema-objects.readthedocs.org -python-snappy -* Copyright (c) 2011-2017, Andres Moreira -* License: BSD 3-Clause "New" or "Revised" License -* Source code: https://github.com/andrix/python-snappy -* Project home: https://github.com/andrix/python-snappy +PyYAML +* Copyright (c) 2017-2021 Ingy döt Net, Copyright (c) 2006-2016 Kirill Simonov +* License: MIT license +* Source code: https://github.com/yaml/pyyaml/ +* Project home: https://pyyaml.org/ + +requests +* Copyright 2023 Kenneth Reitz +* License: Apache-2.0 license +* Source code: https://github.com/psf/requests +* Project home: https://requests.readthedocs.io/en/latest/ scikit-learn * Copyright (c) 2007-2021 The scikit-learn developers @@ -351,11 +283,17 @@ seaborn * Source code: https://github.com/mwaskom/seaborn * Project home: https://seaborn.pydata.org/ -six -* Copyright (c) 2010-2020 Benjamin Peterson -* License: MIT License -* Source code: https://github.com/benjaminp/six -* Project home: six.readthedocs.io/ +skl2onnx +* Copyright (c) 2018-2022 Microsoft Corporation +* License: Apache-2.0 license +* Source code: https://github.com/onnx/sklearn-onnx +* Project home: https://onnx.ai/sklearn-onnx/ + +spacy +* Copyright (C) 2016-2022 ExplosionAI GmbH, 2016 spaCy GmbH, 2015 Matthew Honnibal +* License: The MIT License +* Source code: https://github.com/explosion/spaCy +* Project home: https://spacy.io/ sqlalchemy * Copyright 2005-2021 SQLAlchemy authors and contributors. @@ -369,38 +307,63 @@ tabulate * Source code: https://github.com/astanin/python-tabulate * Project home: https://github.com/astanin/python-tabulate +tensorflow +* Copyright (c) 2023 Google, Inc. +* License: Apache-2.0 license +* Source code: https://github.com/tensorflow/tensorflow +* Project home: https://www.tensorflow.org/ + +tf2onnx +* No listed copyright holder +* License: Apache-2.0 license +* Source code: https://github.com/onnx/tensorflow-onnx +* Project home: https://github.com/onnx/tensorflow-onnx + +torch +* Copyright (c) 2016- Facebook, Inc (Adam Paszke) +* Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +* Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +* Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +* Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +* Copyright (c) 2011-2013 NYU (Clement Farabet) +* Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +* Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +* Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) +* License: BSD-3 +* Source code: https://github.com/pytorch/pytorch +* Project home: https://pytorch.org/ + +torchvision +* Copyright (c) Soumith Chintala 2016, All rights reserved. +* License: BSD-3-Clause license +* Source code: https://github.com/pytorch/vision +* Project home: https://pytorch.org/vision/stable/index.html + tqdm * Copyright (c) 2013 noamraph, 2016 (c) [PR #96] on behalf of Google Inc., 2015-2021 (c) Casper da Costa-Luis * License: MIT License, Mozilla Public License 2.0 (MPL 2.0) * Source code: https://github.com/tqdm/tqdm * Project home: https://tqdm.github.io/ +transformers +* Copyright 2018- The Hugging Face team. All rights reserved. +* License: Apache-2.0 license +* Source code: https://github.com/huggingface/transformers +* Project home: https://huggingface.co/transformers + wordcloud * Copyright (c) 2012 Andreas Christian Mueller * License: MIT License * Source code: https://github.com/amueller/word_cloud * Project home: http://amueller.github.io/word_cloud/ -=============================== Licenses =============================== - ------------------------------- MIT License ----------------------------- -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +xgboost +* Copyright 2022, xgboost developers +* License: Apache-2.0 license +* Source code: https://github.com/dmlc/xgboost +* Project home: https://xgboost.ai/ -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. +=============================== Licenses =============================== ------------------------------------------------------------------------ -------------------------- Apache License 2.0 -------------------------- @@ -609,6 +572,137 @@ OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTE HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------ + +--------------------- Google Protobuf License -------------------------- + +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +------------------------------------------------------------------------ + +-------------------------- ISC License --------------------------------- +ISC License + +Copyright (c) 2012-2016 Nicola Iarocci. + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + +------------------------------------------------------------------------ + +------------------- Matplotlib License --------------------------------- +License agreement for matplotlib versions 1.3.0 and later +========================================================= + +1. This LICENSE AGREEMENT is between the Matplotlib Development Team +("MDT"), and the Individual or Organization ("Licensee") accessing and +otherwise using matplotlib software in source or binary form and its +associated documentation. + +2. Subject to the terms and conditions of this License Agreement, MDT +hereby grants Licensee a nonexclusive, royalty-free, world-wide license +to reproduce, analyze, test, perform and/or display publicly, prepare +derivative works, distribute, and otherwise use matplotlib +alone or in any derivative version, provided, however, that MDT's +License Agreement and MDT's notice of copyright, i.e., "Copyright (c) +2012- Matplotlib Development Team; All Rights Reserved" are retained in +matplotlib alone or in any derivative version prepared by +Licensee. + +3. In the event Licensee prepares a derivative work that is based on or +incorporates matplotlib or any part thereof, and wants to +make the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to matplotlib . + +4. MDT is making matplotlib available to Licensee on an "AS +IS" basis. MDT MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, MDT MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB +WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. MDT SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR +LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING +MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF +THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between MDT and +Licensee. This License Agreement does not grant permission to use MDT +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using matplotlib , +Licensee agrees to be bound by the terms and conditions of this License +Agreement. + +------------------------------------------------------------------------ + +------------------------------ MIT License ----------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to +deal in the Software without restriction, including without limitation the +rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +IN THE SOFTWARE. + ------------------------------------------------------------------------ ----------------------- Mozilla Public License ------------------------- @@ -954,57 +1048,3 @@ PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ------------------------------------------------------------------------- - -------------------- Matplotlib License --------------------------------- -License agreement for matplotlib versions 1.3.0 and later -========================================================= - -1. This LICENSE AGREEMENT is between the Matplotlib Development Team -("MDT"), and the Individual or Organization ("Licensee") accessing and -otherwise using matplotlib software in source or binary form and its -associated documentation. - -2. Subject to the terms and conditions of this License Agreement, MDT -hereby grants Licensee a nonexclusive, royalty-free, world-wide license -to reproduce, analyze, test, perform and/or display publicly, prepare -derivative works, distribute, and otherwise use matplotlib -alone or in any derivative version, provided, however, that MDT's -License Agreement and MDT's notice of copyright, i.e., "Copyright (c) -2012- Matplotlib Development Team; All Rights Reserved" are retained in -matplotlib alone or in any derivative version prepared by -Licensee. - -3. In the event Licensee prepares a derivative work that is based on or -incorporates matplotlib or any part thereof, and wants to -make the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to matplotlib . - -4. MDT is making matplotlib available to Licensee on an "AS -IS" basis. MDT MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, MDT MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB -WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - -5. MDT SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR -LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING -MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF -THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between MDT and -Licensee. This License Agreement does not grant permission to use MDT -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using matplotlib , -Licensee agrees to be bound by the terms and conditions of this License -Agreement. ------------------------------------------------------------------------- From bb86b6ef329e215b86febaa8fc0fb2c48b478ea1 Mon Sep 17 00:00:00 2001 From: Liuda Rudenka Date: Tue, 13 Jun 2023 14:08:56 -0500 Subject: [PATCH 19/20] ADS release v2.8.6 (#230) --- ads/ads_version.json | 2 +- docs/source/release_notes.rst | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ads/ads_version.json b/ads/ads_version.json index 0e4879eca..2e82ab006 100644 --- a/ads/ads_version.json +++ b/ads/ads_version.json @@ -1,3 +1,3 @@ { - "version": "2.8.5" + "version": "2.8.6" } diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index bc7c41dd3..63aece4fa 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -2,6 +2,22 @@ Release Notes ============= +2.8.6 +----- +Release date: June 13, 2023 + +* Resolved an issue in ``ads opctl build-image job-local`` when the build of ``job-local`` would get stuck. Updated the Python version to 3.8 in the base environment of the ``job-local`` image. +* Fixed a bug that prevented the support of defined tags for Data Science job runs. +* Fixed a bug in the ``entryscript.sh`` of ``ads opctl`` that attempted to create a temporary folder in the ``/var/folders`` directory. +* Added support for defined tags in the Data Flow application and application run. +* Deprecated the old :py:class:`~ads.model.ModelDeploymentProperties` and :py:class:`~ads.model.ModelDeployer` classes, and their corresponding APIs. +* Enabled the uploading of large size model artifacts for the :py:class:`~ads.model.ModelDeployment` class. +* Implemented validation for shape name and shape configuration details in Data Science jobs and Data Flow applications. +* Added the capability to create ``ADSDataset`` using the Pandas accessor. +* Made ``Docker`` dependency optional for ``ads opctl run``. +* Provided a prebuilt watch command for monitoring Data Science jobs with ``ads opctl``. +* Eliminated the legacy ``ads.dataflow`` package from ADS. + 2.8.5 ----- Release date: May 17, 2023 From 02815a9e98af4d7ebabeeeed64acf89c09edf7b2 Mon Sep 17 00:00:00 2001 From: Kshitiz Lohia Date: Thu, 15 Jun 2023 19:46:16 +0530 Subject: [PATCH 20/20] added licenses --- THIRD_PARTY_LICENSES.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/THIRD_PARTY_LICENSES.txt b/THIRD_PARTY_LICENSES.txt index a6b0aab38..3a381914f 100644 --- a/THIRD_PARTY_LICENSES.txt +++ b/THIRD_PARTY_LICENSES.txt @@ -363,6 +363,18 @@ xgboost * Source code: https://github.com/dmlc/xgboost * Project home: https://xgboost.ai/ +great-expectations +* No listed copyright holder +* License: Apache License 2.0 +* Source code: https://github.com/great-expectations/great_expectations +* Project home: https://greatexpectations.io/ + +delta +* Copyright (2021) The Delta Lake Project Authors. All rights reserved. +* License: Apache License 2.0 +* Source code: https://github.com/delta-io/delta/ +* Project home: https://delta.io/ + =============================== Licenses =============================== ------------------------------------------------------------------------