From 803729845f9370a99cd3cef047cb40d3ee75af30 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Wed, 21 Apr 2021 11:02:33 -0400 Subject: [PATCH 01/11] checkpoint --- README.rst | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/README.rst b/README.rst index 209b577ead..97d133f66c 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,91 @@ Python Client for Cloud AI Platform .. _Client Library Documentation: https://googleapis.dev/python/aiplatform/latest .. _Product Documentation: https://cloud.google.com/ai-platform-unified/docs +Overview +~~~~~~~~ +Importing +^^^^^^^^^^^^^^^^^^^^ +All SDK functionality can be used from the root of the package: + +.. code-block:: Python + + from google.cloud import aiplatform + + +Initialization +^^^^^^^^^^^^^^^^^^^^ +Initialize the SDK to store common configurations that will be used throughout the SDK. + +.. code-block:: Python + + aiplatform.init( + # your GCP project ID or number + # environment default used is not set + project='my-project', + + # the AI platform region you will use + # defaults to us-central1 + location='us-central1', + + # bucket in same region as location + # used to stage artifacts + staging_bucket='gs://my_staging_bucket', + + # custom google.auth.credentials.Credentials + # environment default creds used if not set + credentials=my_credentials, + + # customer managed encryption key resource name + # will be applied to all AI Platform resources if set + encryption_spec_key_name=my_encryption_key_name + ) + +Datasets +^^^^^^^^ +AI Platform provides managed Tabular, Text, Image, and Video datasets. In the SDK, Datasets can be used downstream to +train models. + +To create a Tabular dataset: + +.. code-block:: Python + + my_dataset = aiplatform.TabularDataset.create( + display_name="my-dataset", gcs_source=['gs://path/to/my/dataset.csv']) + +You can also create and import a dataset in separate steps: + +.. code-block:: Python + + from google.cloud import aiplatform + + my_dataset = aiplatform.TextDataset.create( + display_name="my-dataset") + + my_dataset.import( + gcs_source=['gs://path/to/my/dataset.csv'] + import_schema_uri=aiplatform.schema.dataset.ioformat.text.multi_label_classification + ) + +AI Platform supports a variety of dataset schemas. References to these schemas are available under the +:code:`aiplatform.schema.dataset` namespace. For more information on the supported dataset schemas please refer to the +`Preparing data docs`_. + +.. _Preparing data docs: https://cloud.google.com/ai-platform-unified/docs/datasets/prepare + +Training +^^^^^^^^ +The AI Platform SDK allows you train Custom and AutoML Models. + +Custom models can be trained using a custom Python script, custom Python package, or container. + +Preparing Your Custom Code +-------------------------- +AI Platform custom training enables you to train on AI Platform Datasets and produce AI Platform Models. To do so your +script must adhere to the following contract: + +1. It must read dataset from the given environment variables: + + Quick Start ----------- From 6a65a0e107c0506e6bf29b35a36befe3a0338f51 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 14:21:39 -0400 Subject: [PATCH 02/11] feat: add tensorboard support to custom job and hyperparameter tuning job --- google/cloud/aiplatform/jobs.py | 58 ++++++++- tests/unit/aiplatform/test_custom_job.py | 71 ++++++++++- .../test_hyperparameter_tuning_job.py | 113 ++++++++++++++++-- 3 files changed, 232 insertions(+), 10 deletions(-) diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 7b1f5cccc5..a3c69ad73a 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -45,11 +45,13 @@ batch_prediction_job_v1 as gca_bp_job_v1, batch_prediction_job_v1beta1 as gca_bp_job_v1beta1, custom_job as gca_custom_job_compat, + custom_job_v1beta1 as gca_custom_job_v1beta1, explanation_v1beta1 as gca_explanation_v1beta1, io as gca_io_compat, io_v1beta1 as gca_io_v1beta1, job_state as gca_job_state, hyperparameter_tuning_job as gca_hyperparameter_tuning_job_compat, + hyperparameter_tuning_job_v1beta1 as gca_hyperparameter_tuning_job_v1beta1, machine_resources as gca_machine_resources_compat, machine_resources_v1beta1 as gca_machine_resources_v1beta1, study as gca_study_compat, @@ -1132,6 +1134,7 @@ def run( network: Optional[str] = None, timeout: Optional[int] = None, restart_job_on_worker_restart: bool = False, + tensorboard: Optional[str] = None, sync: bool = True, ) -> None: """Run this configured CustomJob. @@ -1152,6 +1155,20 @@ def run( gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + tensorboard (str): + Optional. The name of an AI Platform + [Tensorboard][google.cloud.aiplatform.v1beta1.Tensorboard] + resource to which this CustomJob will upload Tensorboard + logs. Format: + ``projects/{project}/locations/{location}/tensorboards/{tensorboard}`` + + The training script should write Tensorboard to following AI Platform environment + variable: + + AIP_TENSORBOARD_LOG_DIR + + `service_account` is required with provided `tensorboard`. + (TODO: add documentation when released) sync (bool): Whether to execute this method synchronously. If False, this method will unblock and it will be executed in a concurrent Future. @@ -1170,9 +1187,18 @@ def run( restart_job_on_worker_restart=restart_job_on_worker_restart, ) + if tensorboard: + v1beta1_gca_resource = gca_custom_job_v1beta1.CustomJob() + v1beta1_gca_resource._pb.MergeFromString( + self._gca_resource._pb.SerializeToString() + ) + self._gca_resource = v1beta1_gca_resource + self._gca_resource.job_spec.tensorboard = tensorboard + _LOGGER.log_create_with_lro(self.__class__) - self._gca_resource = self.api_client.create_custom_job( + version = "v1beta1" if tensorboard else "v1" + self._gca_resource = self.api_client.select_version(version).create_custom_job( parent=self._parent, custom_job=self._gca_resource ) @@ -1415,6 +1441,7 @@ def run( network: Optional[str] = None, timeout: Optional[int] = None, # seconds restart_job_on_worker_restart: bool = False, + tensorboard: Optional[str] = None, sync: bool = True, ) -> None: """Run this configured CustomJob. @@ -1435,6 +1462,20 @@ def run( gets restarted. This feature can be used by distributed training jobs that are not resilient to workers leaving and joining a job. + tensorboard (str): + Optional. The name of an AI Platform + [Tensorboard][google.cloud.aiplatform.v1beta1.Tensorboard] + resource to which this CustomJob will upload Tensorboard + logs. Format: + ``projects/{project}/locations/{location}/tensorboards/{tensorboard}`` + + The training script should write Tensorboard to following AI Platform environment + variable: + + AIP_TENSORBOARD_LOG_DIR + + `service_account` is required with provided `tensorboard`. + (TODO: add documentation when released) sync (bool): Whether to execute this method synchronously. If False, this method will unblock and it will be executed in a concurrent Future. @@ -1453,9 +1494,22 @@ def run( restart_job_on_worker_restart=restart_job_on_worker_restart, ) + if tensorboard: + v1beta1_gca_resource = ( + gca_hyperparameter_tuning_job_v1beta1.HyperparameterTuningJob() + ) + v1beta1_gca_resource._pb.MergeFromString( + self._gca_resource._pb.SerializeToString() + ) + self._gca_resource = v1beta1_gca_resource + self._gca_resource.trial_job_spec.tensorboard = tensorboard + _LOGGER.log_create_with_lro(self.__class__) - self._gca_resource = self.api_client.create_hyperparameter_tuning_job( + version = "v1beta1" if tensorboard else "v1" + self._gca_resource = self.api_client.select_version( + version + ).create_hyperparameter_tuning_job( parent=self._parent, hyperparameter_tuning_job=self._gca_resource ) diff --git a/tests/unit/aiplatform/test_custom_job.py b/tests/unit/aiplatform/test_custom_job.py index 37c2ac3df0..7797e0edef 100644 --- a/tests/unit/aiplatform/test_custom_job.py +++ b/tests/unit/aiplatform/test_custom_job.py @@ -29,12 +29,18 @@ from google.cloud import aiplatform from google.cloud.aiplatform.compat.types import custom_job as gca_custom_job_compat +from google.cloud.aiplatform.compat.types import ( + custom_job_v1beta1 as gca_custom_job_v1beta1, +) from google.cloud.aiplatform.compat.types import io as gca_io_compat from google.cloud.aiplatform.compat.types import job_state as gca_job_state_compat from google.cloud.aiplatform.compat.types import ( encryption_spec as gca_encryption_spec_compat, ) from google.cloud.aiplatform_v1.services.job_service import client as job_service_client +from google.cloud.aiplatform_v1beta1.services.job_service import ( + client as job_service_client_v1beta1, +) _TEST_PROJECT = "test-project" _TEST_LOCATION = "us-central1" @@ -44,6 +50,7 @@ _TEST_PARENT = f"projects/{_TEST_PROJECT}/locations/{_TEST_LOCATION}" _TEST_CUSTOM_JOB_NAME = f"{_TEST_PARENT}/customJobs/{_TEST_ID}" +_TEST_TENSORBOARD_NAME = f"{_TEST_PARENT}/tensorboards/{_TEST_ID}" _TEST_TRAINING_CONTAINER_IMAGE = "gcr.io/test-training/container:image" @@ -97,11 +104,20 @@ ) -def _get_custom_job_proto(state=None, name=None, error=None): +def _get_custom_job_proto(state=None, name=None, error=None, version="v1"): custom_job_proto = copy.deepcopy(_TEST_BASE_CUSTOM_JOB_PROTO) custom_job_proto.name = name custom_job_proto.state = state custom_job_proto.error = error + + if version == "v1beta1": + v1beta1_custom_job_proto = gca_custom_job_v1beta1.CustomJob() + v1beta1_custom_job_proto._pb.MergeFromString( + custom_job_proto._pb.SerializeToString() + ) + custom_job_proto = v1beta1_custom_job_proto + custom_job_proto.job_spec.tensorboard = _TEST_TENSORBOARD_NAME + return custom_job_proto @@ -162,6 +178,19 @@ def create_custom_job_mock(): yield create_custom_job_mock +@pytest.fixture +def create_custom_job_v1beta1_mock(): + with mock.patch.object( + job_service_client_v1beta1.JobServiceClient, "create_custom_job" + ) as create_custom_job_mock: + create_custom_job_mock.return_value = _get_custom_job_proto( + name=_TEST_CUSTOM_JOB_NAME, + state=gca_job_state_compat.JobState.JOB_STATE_PENDING, + version="v1beta1", + ) + yield create_custom_job_mock + + class TestCustomJob: def setup_method(self): reload(aiplatform.initializer) @@ -321,3 +350,43 @@ def test_create_from_local_script_raises_with_no_staging_bucket( script_path=test_training_jobs._TEST_LOCAL_SCRIPT_FILE_NAME, container_uri=_TEST_TRAINING_CONTAINER_IMAGE, ) + + @pytest.mark.parametrize("sync", [True, False]) + def test_create_custom_job_with_tensorboard( + self, create_custom_job_v1beta1_mock, get_custom_job_mock, sync + ): + + aiplatform.init( + project=_TEST_PROJECT, + location=_TEST_LOCATION, + staging_bucket=_TEST_STAGING_BUCKET, + encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, + ) + + job = aiplatform.CustomJob( + display_name=_TEST_DISPLAY_NAME, worker_pool_specs=_TEST_WORKER_POOL_SPEC + ) + + job.run( + service_account=_TEST_SERVICE_ACCOUNT, + tensorboard=_TEST_TENSORBOARD_NAME, + network=_TEST_NETWORK, + timeout=_TEST_TIMEOUT, + restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART, + sync=sync, + ) + + job.wait() + + expected_custom_job = _get_custom_job_proto(version="v1beta1") + + create_custom_job_v1beta1_mock.assert_called_once_with( + parent=_TEST_PARENT, custom_job=expected_custom_job + ) + + expected_custom_job = _get_custom_job_proto() + + assert job.job_spec == expected_custom_job.job_spec + assert ( + job._gca_resource.state == gca_job_state_compat.JobState.JOB_STATE_SUCCEEDED + ) diff --git a/tests/unit/aiplatform/test_hyperparameter_tuning_job.py b/tests/unit/aiplatform/test_hyperparameter_tuning_job.py index fcd15f93ac..f4102fc3bb 100644 --- a/tests/unit/aiplatform/test_hyperparameter_tuning_job.py +++ b/tests/unit/aiplatform/test_hyperparameter_tuning_job.py @@ -31,9 +31,13 @@ ) from google.cloud.aiplatform.compat.types import ( hyperparameter_tuning_job as gca_hyperparameter_tuning_job_compat, + hyperparameter_tuning_job_v1beta1 as gca_hyperparameter_tuning_job_v1beta1, ) from google.cloud.aiplatform.compat.types import study as gca_study_compat from google.cloud.aiplatform_v1.services.job_service import client as job_service_client +from google.cloud.aiplatform_v1beta1.services.job_service import ( + client as job_service_client_v1beta1, +) import test_custom_job @@ -122,12 +126,29 @@ ) -def _get_hyperparameter_tuning_job_proto(state=None, name=None, error=None): - custom_job_proto = copy.deepcopy(_TEST_BASE_HYPERPARAMETER_TUNING_JOB_PROTO) - custom_job_proto.name = name - custom_job_proto.state = state - custom_job_proto.error = error - return custom_job_proto +def _get_hyperparameter_tuning_job_proto( + state=None, name=None, error=None, version="v1" +): + hyperparameter_tuning_job_proto = copy.deepcopy( + _TEST_BASE_HYPERPARAMETER_TUNING_JOB_PROTO + ) + hyperparameter_tuning_job_proto.name = name + hyperparameter_tuning_job_proto.state = state + hyperparameter_tuning_job_proto.error = error + + if version == "v1beta1": + v1beta1_hyperparameter_tuning_job_proto = ( + gca_hyperparameter_tuning_job_v1beta1.HyperparameterTuningJob() + ) + v1beta1_hyperparameter_tuning_job_proto._pb.MergeFromString( + hyperparameter_tuning_job_proto._pb.SerializeToString() + ) + hyperparameter_tuning_job_proto = v1beta1_hyperparameter_tuning_job_proto + hyperparameter_tuning_job_proto.trial_job_spec.tensorboard = ( + test_custom_job._TEST_TENSORBOARD_NAME + ) + + return hyperparameter_tuning_job_proto @pytest.fixture @@ -187,7 +208,20 @@ def create_hyperparameter_tuning_job_mock(): yield create_hyperparameter_tuning_job_mock -class TestCustomJob: +@pytest.fixture +def create_hyperparameter_tuning_job_v1beta1_mock(): + with mock.patch.object( + job_service_client_v1beta1.JobServiceClient, "create_hyperparameter_tuning_job" + ) as create_hyperparameter_tuning_job_mock: + create_hyperparameter_tuning_job_mock.return_value = _get_hyperparameter_tuning_job_proto( + name=_TEST_HYPERPARAMETERTUNING_JOB_NAME, + state=gca_job_state_compat.JobState.JOB_STATE_PENDING, + version="v1beta1", + ) + yield create_hyperparameter_tuning_job_mock + + +class TestHyperparameterTuningJob: def setup_method(self): reload(aiplatform.initializer) reload(aiplatform) @@ -366,3 +400,68 @@ def test_get_hyperparameter_tuning_job(self, get_hyperparameter_tuning_job_mock) assert ( job._gca_resource.state == gca_job_state_compat.JobState.JOB_STATE_PENDING ) + + @pytest.mark.parametrize("sync", [True, False]) + def test_create_hyperparameter_tuning_job_with_tensorboard( + self, + create_hyperparameter_tuning_job_v1beta1_mock, + get_hyperparameter_tuning_job_mock, + sync, + ): + + aiplatform.init( + project=_TEST_PROJECT, + location=_TEST_LOCATION, + staging_bucket=_TEST_STAGING_BUCKET, + encryption_spec_key_name=_TEST_DEFAULT_ENCRYPTION_KEY_NAME, + ) + + custom_job = aiplatform.CustomJob( + display_name=test_custom_job._TEST_DISPLAY_NAME, + worker_pool_specs=test_custom_job._TEST_WORKER_POOL_SPEC, + ) + + job = aiplatform.HyperparameterTuningJob( + display_name=_TEST_DISPLAY_NAME, + custom_job=custom_job, + metric_spec={_TEST_METRIC_SPEC_KEY: _TEST_METRIC_SPEC_VALUE}, + parameter_spec={ + "lr": hpt.DoubleParameterSpec(min=0.001, max=0.1, scale="log"), + "units": hpt.IntegerParameterSpec(min=4, max=1028, scale="linear"), + "activation": hpt.CategoricalParameterSpec( + values=["relu", "sigmoid", "elu", "selu", "tanh"] + ), + "batch_size": hpt.DiscreteParameterSpec( + values=[16, 32], scale="linear" + ), + }, + parallel_trial_count=_TEST_PARALLEL_TRIAL_COUNT, + max_trial_count=_TEST_MAX_TRIAL_COUNT, + max_failed_trial_count=_TEST_MAX_FAILED_TRIAL_COUNT, + search_algorithm=_TEST_SEARCH_ALGORITHM, + measurement_selection=_TEST_MEASUREMENT_SELECTION, + ) + + job.run( + service_account=_TEST_SERVICE_ACCOUNT, + network=_TEST_NETWORK, + timeout=_TEST_TIMEOUT, + restart_job_on_worker_restart=_TEST_RESTART_JOB_ON_WORKER_RESTART, + tensorboard=test_custom_job._TEST_TENSORBOARD_NAME, + sync=sync, + ) + + job.wait() + + expected_hyperparameter_tuning_job = _get_hyperparameter_tuning_job_proto( + version="v1beta1" + ) + + create_hyperparameter_tuning_job_v1beta1_mock.assert_called_once_with( + parent=_TEST_PARENT, + hyperparameter_tuning_job=expected_hyperparameter_tuning_job, + ) + + assert ( + job._gca_resource.state == gca_job_state_compat.JobState.JOB_STATE_SUCCEEDED + ) From a79c4a1ed683b850fa660f4378c215351a68579e Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 14:53:42 -0400 Subject: [PATCH 03/11] chore: docstring with Vertex AI --- .github/CODEOWNERS | 2 +- google/cloud/aiplatform/base.py | 24 +- .../cloud/aiplatform/datasets/_datasources.py | 6 +- google/cloud/aiplatform/datasets/dataset.py | 8 +- .../aiplatform/datasets/image_dataset.py | 2 +- .../aiplatform/datasets/tabular_dataset.py | 2 +- .../cloud/aiplatform/datasets/text_dataset.py | 2 +- .../datasets/time_series_dataset.py | 2 +- .../aiplatform/datasets/video_dataset.py | 2 +- google/cloud/aiplatform/initializer.py | 4 +- google/cloud/aiplatform/jobs.py | 34 +-- google/cloud/aiplatform/metadata/artifact.py | 2 +- google/cloud/aiplatform/metadata/context.py | 2 +- google/cloud/aiplatform/metadata/execution.py | 2 +- .../aiplatform/metadata/metadata_store.py | 2 +- google/cloud/aiplatform/metadata/resource.py | 2 +- google/cloud/aiplatform/models.py | 12 +- google/cloud/aiplatform/schema.py | 2 +- google/cloud/aiplatform/training_jobs.py | 210 +++++++++--------- google/cloud/aiplatform/utils/__init__.py | 8 +- .../aiplatform/utils/worker_spec_utils.py | 8 +- 21 files changed, 169 insertions(+), 169 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ccdc098900..f0c14d5c2b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,7 +10,7 @@ # The AI Platform GAPIC libraries are owned by Cloud AI DPE /google/cloud/aiplatform* @googleapis/cdpe-cloudai -# The AI Platform SDK is owned by Model Builder SDK Dev team +# The Vertex SDK is owned by Model Builder SDK Dev team /google/cloud/aiplatform/* @googleapis/cloud-aiplatform-model-builder-sdk /tests/unit/aiplatform/* @googleapis/cloud-aiplatform-model-builder-sdk diff --git a/google/cloud/aiplatform/base.py b/google/cloud/aiplatform/base.py index a9fcef24bd..620dbcc3fa 100644 --- a/google/cloud/aiplatform/base.py +++ b/google/cloud/aiplatform/base.py @@ -67,7 +67,7 @@ def log_create_with_lro( Args: cls (AiPlatformResourceNoune): - AI Platform Resource Noun class that is being created. + Vertex AI Resource Noun class that is being created. lro (operation.Operation): Optional. Backing LRO for creation. """ @@ -90,9 +90,9 @@ def log_create_complete( Args: cls (AiPlatformResourceNoun): - AI Platform Resource Noun class that is being created. + Vertex AI Resource Noun class that is being created. resource (proto.Message): - AI Platform Resourc proto.Message + Vertex AI Resourc proto.Message variable_name (str): Name of variable to use for code snippet """ self._logger.info(f"{cls.__name__} created. Resource name: {resource.name}") @@ -113,9 +113,9 @@ def log_create_complete_with_getter( Args: cls (AiPlatformResourceNoun): - AI Platform Resource Noun class that is being created. + Vertex AI Resource Noun class that is being created. resource (proto.Message): - AI Platform Resourc proto.Message + Vertex AI Resourc proto.Message variable_name (str): Name of variable to use for code snippet """ self._logger.info(f"{cls.__name__} created. Resource name: {resource.name}") @@ -386,7 +386,7 @@ def __repr__(self) -> str: class AiPlatformResourceNoun(metaclass=abc.ABCMeta): - """Base class the AI Platform resource nouns. + """Base class the Vertex AI resource nouns. Subclasses require two class attributes: @@ -715,7 +715,7 @@ def wrapper(*args, **kwargs): class AiPlatformResourceNounWithFutureManager(AiPlatformResourceNoun, FutureManager): - """Allows optional asynchronous calls to this AI Platform Resource + """Allows optional asynchronous calls to this Vertex AI Resource Nouns.""" def __init__( @@ -816,7 +816,7 @@ def _construct_sdk_resource_from_gapic( Args: gapic_resource (proto.Message): - A GAPIC representation of an AI Platform resource, usually + A GAPIC representation of an Vertex AI resource, usually retrieved by a get_* or in a list_* API call. project (str): Optional. Project to construct SDK object from. If not set, @@ -850,7 +850,7 @@ def _list( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ) -> List[AiPlatformResourceNoun]: - """Private method to list all instances of this AI Platform Resource, + """Private method to list all instances of this Vertex AI Resource, takes a `cls_filter` arg to filter to a particular SDK resource subclass. @@ -919,7 +919,7 @@ def _list_with_local_order( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ) -> List[AiPlatformResourceNoun]: - """Private method to list all instances of this AI Platform Resource, + """Private method to list all instances of this Vertex AI Resource, takes a `cls_filter` arg to filter to a particular SDK resource subclass. Provides client-side sorting when a list API doesn't support `order_by`. @@ -981,7 +981,7 @@ def list( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ) -> List[AiPlatformResourceNoun]: - """List all instances of this AI Platform Resource. + """List all instances of this Vertex AI Resource. Example Usage: @@ -1023,7 +1023,7 @@ def list( @optional_sync() def delete(self, sync: bool = True) -> None: - """Deletes this AI Platform resource. WARNING: This deletion is + """Deletes this Vertex AI resource. WARNING: This deletion is permament. Args: diff --git a/google/cloud/aiplatform/datasets/_datasources.py b/google/cloud/aiplatform/datasets/_datasources.py index ea436eb91b..9323f40382 100644 --- a/google/cloud/aiplatform/datasets/_datasources.py +++ b/google/cloud/aiplatform/datasets/_datasources.py @@ -46,7 +46,7 @@ def import_data_config(self): class TabularDatasource(Datasource): - """Datasource for creating a tabular dataset for AI Platform.""" + """Datasource for creating a tabular dataset for Vertex AI.""" def __init__( self, @@ -99,7 +99,7 @@ def dataset_metadata(self) -> Optional[Dict]: class NonTabularDatasource(Datasource): - """Datasource for creating an empty non-tabular dataset for AI Platform.""" + """Datasource for creating an empty non-tabular dataset for Vertex AI.""" @property def dataset_metadata(self) -> Optional[Dict]: @@ -107,7 +107,7 @@ def dataset_metadata(self) -> Optional[Dict]: class NonTabularDatasourceImportable(NonTabularDatasource, DatasourceImportable): - """Datasource for creating a non-tabular dataset for AI Platform and + """Datasource for creating a non-tabular dataset for Vertex AI and importing data to the dataset.""" def __init__( diff --git a/google/cloud/aiplatform/datasets/dataset.py b/google/cloud/aiplatform/datasets/dataset.py index 44dadc4ee4..a4db10c693 100644 --- a/google/cloud/aiplatform/datasets/dataset.py +++ b/google/cloud/aiplatform/datasets/dataset.py @@ -36,7 +36,7 @@ class _Dataset(base.AiPlatformResourceNounWithFutureManager): - """Managed dataset resource for AI Platform.""" + """Managed dataset resource for Vertex AI.""" client_class = utils.DatasetClientWithOverride _is_client_prediction_client = False @@ -264,7 +264,7 @@ def _create_and_import( that can be used here are found in gs://google-cloud- aiplatform/schema/dataset/metadata/. datasource (_datasources.Datasource): - Required. Datasource for creating a dataset for AI Platform. + Required. Datasource for creating a dataset for Vertex AI. project (str): Required. Project to upload this model to. Overrides project set in aiplatform.init. @@ -368,7 +368,7 @@ def _create( that can be used here are found in gs://google-cloud- aiplatform/schema/dataset/metadata/. datasource (_datasources.Datasource): - Required. Datasource for creating a dataset for AI Platform. + Required. Datasource for creating a dataset for Vertex AI. request_metadata (Sequence[Tuple[str, str]]): Strings which should be sent along with the create_dataset request as metadata. Usually to specify special dataset config. @@ -401,7 +401,7 @@ def _import( Args: datasource (_datasources.DatasourceImportable): - Required. Datasource for importing data to an existing dataset for AI Platform. + Required. Datasource for importing data to an existing dataset for Vertex AI. Returns: operation (Operation): diff --git a/google/cloud/aiplatform/datasets/image_dataset.py b/google/cloud/aiplatform/datasets/image_dataset.py index c2b3ca68b5..506338c915 100644 --- a/google/cloud/aiplatform/datasets/image_dataset.py +++ b/google/cloud/aiplatform/datasets/image_dataset.py @@ -27,7 +27,7 @@ class ImageDataset(datasets._Dataset): - """Managed image dataset resource for AI Platform.""" + """Managed image dataset resource for Vertex AI.""" _supported_metadata_schema_uris: Optional[Tuple[str]] = ( schema.dataset.metadata.image, diff --git a/google/cloud/aiplatform/datasets/tabular_dataset.py b/google/cloud/aiplatform/datasets/tabular_dataset.py index b80266cf00..95f1b16f98 100644 --- a/google/cloud/aiplatform/datasets/tabular_dataset.py +++ b/google/cloud/aiplatform/datasets/tabular_dataset.py @@ -33,7 +33,7 @@ class TabularDataset(datasets._Dataset): - """Managed tabular dataset resource for AI Platform.""" + """Managed tabular dataset resource for Vertex AI.""" _supported_metadata_schema_uris: Optional[Tuple[str]] = ( schema.dataset.metadata.tabular, diff --git a/google/cloud/aiplatform/datasets/text_dataset.py b/google/cloud/aiplatform/datasets/text_dataset.py index 6f6fd57bda..85676ed2ed 100644 --- a/google/cloud/aiplatform/datasets/text_dataset.py +++ b/google/cloud/aiplatform/datasets/text_dataset.py @@ -27,7 +27,7 @@ class TextDataset(datasets._Dataset): - """Managed text dataset resource for AI Platform.""" + """Managed text dataset resource for Vertex AI.""" _supported_metadata_schema_uris: Optional[Tuple[str]] = ( schema.dataset.metadata.text, diff --git a/google/cloud/aiplatform/datasets/time_series_dataset.py b/google/cloud/aiplatform/datasets/time_series_dataset.py index 92d8e60c37..d5aa3dcbf2 100644 --- a/google/cloud/aiplatform/datasets/time_series_dataset.py +++ b/google/cloud/aiplatform/datasets/time_series_dataset.py @@ -27,7 +27,7 @@ class TimeSeriesDataset(datasets._Dataset): - """Managed time series dataset resource for AI Platform""" + """Managed time series dataset resource for Vertex AI""" _supported_metadata_schema_uris: Optional[Tuple[str]] = ( schema.dataset.metadata.time_series, diff --git a/google/cloud/aiplatform/datasets/video_dataset.py b/google/cloud/aiplatform/datasets/video_dataset.py index 7064c8b7cf..594a4ac407 100644 --- a/google/cloud/aiplatform/datasets/video_dataset.py +++ b/google/cloud/aiplatform/datasets/video_dataset.py @@ -27,7 +27,7 @@ class VideoDataset(datasets._Dataset): - """Managed video dataset resource for AI Platform.""" + """Managed video dataset resource for Vertex AI.""" _supported_metadata_schema_uris: Optional[Tuple[str]] = ( schema.dataset.metadata.video, diff --git a/google/cloud/aiplatform/initializer.py b/google/cloud/aiplatform/initializer.py index 9adae3be9a..a6f0786857 100644 --- a/google/cloud/aiplatform/initializer.py +++ b/google/cloud/aiplatform/initializer.py @@ -257,13 +257,13 @@ def create_client( Args: client_class (utils.AiPlatformServiceClientWithOverride): - (Required) An AI Platform Service Client with optional overrides. + (Required) An Vertex AI Service Client with optional overrides. credentials (auth_credentials.Credentials): Custom auth credentials. If not provided will use the current config. location_override (str): Optional location override. prediction_client (str): Optional flag to use a prediction endpoint. Returns: - client: Instantiated AI Platform Service client with optional overrides + client: Instantiated Vertex AI Service client with optional overrides """ gapic_version = pkg_resources.get_distribution( "google-cloud-aiplatform", diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index a3c69ad73a..61320235f7 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -74,7 +74,7 @@ class _Job(base.AiPlatformResourceNounWithFutureManager): - """Class that represents a general Job resource in AI Platform (Unified). + """Class that represents a general Job resource in Vertex AI. Cannot be directly instantiated. Serves as base class to specific Job types, i.e. BatchPredictionJob or @@ -130,7 +130,7 @@ def state(self) -> gca_job_state.JobState: Returns: state (job_state.JobState): - Enum that describes the state of a AI Platform job. + Enum that describes the state of a Vertex AI job. """ # Fetch the Job again for most up-to-date job state @@ -347,7 +347,7 @@ def create( or "file-list". Default is "jsonl" when using `gcs_source`. If a `bigquery_source` is provided, this is overriden to "bigquery". predictions_format (str): - Required. The format in which AI Platform gives the + Required. The format in which Vertex AI gives the predictions, must be one of "jsonl", "csv", or "bigquery". Default is "jsonl" when using `gcs_destination_prefix`. If a `bigquery_destination_prefix` is provided, this is overriden to @@ -417,7 +417,7 @@ def create( `machine_type`. Only used if `machine_type` is set. starting_replica_count (Optional[int]): The number of machine replicas used at the start of the batch - operation. If not set, AI Platform decides starting number, not + operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. max_replica_count (Optional[int]): @@ -647,7 +647,7 @@ def _create( Required. An instance of DatasetServiceClient with the correct api_endpoint already set based on user's preferences. batch_prediction_job (gca_bp_job.BatchPredictionJob): - Required. a batch prediction job proto for creating a batch prediction job on AI Platform. + Required. a batch prediction job proto for creating a batch prediction job on Vertex AI. generate_explanation (bool): Required. Generate explanation along with the batch prediction results. @@ -673,7 +673,7 @@ def _create( ValueError: If no or multiple source or destinations are provided. Also, if provided instances_format or predictions_format are not supported - by AI Platform. + by Vertex AI. """ # select v1beta1 if explain else use default v1 if generate_explanation: @@ -841,7 +841,7 @@ def get( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, ) -> "_RunnableJob": - """Get an AI Platform Job for the given resource_name. + """Get an Vertex AI Job for the given resource_name. Args: resource_name (str): @@ -857,7 +857,7 @@ def get( credentials set in aiplatform.init. Returns: - An AI Platform Job. + An Vertex AI Job. """ self = cls._empty_constructor( project=project, @@ -882,7 +882,7 @@ class DataLabelingJob(_Job): class CustomJob(_RunnableJob): - """AI Platform (Unified) Custom Job.""" + """Vertex AI Custom Job.""" _resource_noun = "customJobs" _getter_method = "get_custom_job" @@ -1156,13 +1156,13 @@ def run( distributed training jobs that are not resilient to workers leaving and joining a job. tensorboard (str): - Optional. The name of an AI Platform + Optional. The name of an Vertex AI [Tensorboard][google.cloud.aiplatform.v1beta1.Tensorboard] resource to which this CustomJob will upload Tensorboard logs. Format: ``projects/{project}/locations/{location}/tensorboards/{tensorboard}`` - The training script should write Tensorboard to following AI Platform environment + The training script should write Tensorboard to following Vertex AI environment variable: AIP_TENSORBOARD_LOG_DIR @@ -1228,7 +1228,7 @@ def job_spec(self): class HyperparameterTuningJob(_RunnableJob): - """AI Platform (Unified) Hyperparameter Tuning Job.""" + """Vertex AI Hyperparameter Tuning Job.""" _resource_noun = "hyperparameterTuningJobs" _getter_method = "get_hyperparameter_tuning_job" @@ -1348,13 +1348,13 @@ def __init__( max_failed_trial_count (int): Optional. The number of failed Trials that need to be seen before failing the HyperparameterTuningJob. - If set to 0, AI Platform decides how many Trials + If set to 0, Vertex AI decides how many Trials must fail before the whole job fails. search_algorithm (str): The search algorithm specified for the Study. Accepts one of the following: `None` - If you do not specify an algorithm, your job uses - the default AI Platform algorithm. The default algorithm + the default Vertex AI algorithm. The default algorithm applies Bayesian optimization to arrive at the optimal solution with a more effective search over the parameter space. @@ -1362,7 +1362,7 @@ def __init__( option is particularly useful if you want to specify a quantity of trials that is greater than the number of points in the feasible space. In such cases, if you do not specify a grid - search, the AI Platform default algorithm may generate duplicate + search, the Vertex AI default algorithm may generate duplicate suggestions. To use grid search, all parameter specs must be of type `IntegerParameterSpec`, `CategoricalParameterSpace`, or `DiscreteParameterSpec`. @@ -1463,13 +1463,13 @@ def run( distributed training jobs that are not resilient to workers leaving and joining a job. tensorboard (str): - Optional. The name of an AI Platform + Optional. The name of an Vertex AI [Tensorboard][google.cloud.aiplatform.v1beta1.Tensorboard] resource to which this CustomJob will upload Tensorboard logs. Format: ``projects/{project}/locations/{location}/tensorboards/{tensorboard}`` - The training script should write Tensorboard to following AI Platform environment + The training script should write Tensorboard to following Vertex AI environment variable: AIP_TENSORBOARD_LOG_DIR diff --git a/google/cloud/aiplatform/metadata/artifact.py b/google/cloud/aiplatform/metadata/artifact.py index 98eefacc5f..b3ef6e09a2 100644 --- a/google/cloud/aiplatform/metadata/artifact.py +++ b/google/cloud/aiplatform/metadata/artifact.py @@ -26,7 +26,7 @@ class _Artifact(_Resource): - """Metadata Artifact resource for AI Platform""" + """Metadata Artifact resource for Vertex AI""" _resource_noun = "artifacts" _getter_method = "get_artifact" diff --git a/google/cloud/aiplatform/metadata/context.py b/google/cloud/aiplatform/metadata/context.py index cb3340499b..ddd583bbdf 100644 --- a/google/cloud/aiplatform/metadata/context.py +++ b/google/cloud/aiplatform/metadata/context.py @@ -26,7 +26,7 @@ class _Context(_Resource): - """Metadata Context resource for AI Platform""" + """Metadata Context resource for Vertex AI""" _resource_noun = "contexts" _getter_method = "get_context" diff --git a/google/cloud/aiplatform/metadata/execution.py b/google/cloud/aiplatform/metadata/execution.py index 39fc7a74b3..3605efdb4f 100644 --- a/google/cloud/aiplatform/metadata/execution.py +++ b/google/cloud/aiplatform/metadata/execution.py @@ -29,7 +29,7 @@ class _Execution(_Resource): - """Metadata Execution resource for AI Platform""" + """Metadata Execution resource for Vertex AI""" _resource_noun = "executions" _getter_method = "get_execution" diff --git a/google/cloud/aiplatform/metadata/metadata_store.py b/google/cloud/aiplatform/metadata/metadata_store.py index 2a55f066a8..e712f611a8 100644 --- a/google/cloud/aiplatform/metadata/metadata_store.py +++ b/google/cloud/aiplatform/metadata/metadata_store.py @@ -28,7 +28,7 @@ class _MetadataStore(base.AiPlatformResourceNounWithFutureManager): - """Managed MetadataStore resource for AI Platform""" + """Managed MetadataStore resource for Vertex AI""" client_class = utils.MetadataClientWithOverride _is_client_prediction_client = False diff --git a/google/cloud/aiplatform/metadata/resource.py b/google/cloud/aiplatform/metadata/resource.py index 11f03b7af1..ba11f8eb07 100644 --- a/google/cloud/aiplatform/metadata/resource.py +++ b/google/cloud/aiplatform/metadata/resource.py @@ -34,7 +34,7 @@ class _Resource(base.AiPlatformResourceNounWithFutureManager, abc.ABC): - """Metadata Resource for AI Platform""" + """Metadata Resource for Vertex AI""" client_class = utils.MetadataClientWithOverride _is_client_prediction_client = False diff --git a/google/cloud/aiplatform/models.py b/google/cloud/aiplatform/models.py index cecc992644..c106c019ee 100644 --- a/google/cloud/aiplatform/models.py +++ b/google/cloud/aiplatform/models.py @@ -1181,7 +1181,7 @@ def undeploy_all(self, sync: bool = True) -> "Endpoint": return self def delete(self, force: bool = False, sync: bool = True) -> None: - """Deletes this AI Platform Endpoint resource. If force is set to True, + """Deletes this Vertex AI Endpoint resource. If force is set to True, all models on this Endpoint will be undeployed prior to deletion. Args: @@ -1325,11 +1325,11 @@ def upload( serving_container_predict_route (str): Optional. An HTTP path to send prediction requests to the container, and which must be supported by it. If not specified a default HTTP path will - be used by AI Platform. + be used by Vertex AI. serving_container_health_route (str): Optional. An HTTP path to send health check requests to the container, and which must be supported by it. If not specified a standard HTTP path will be - used by AI Platform. + used by Vertex AI. description (str): The description of the model. serving_container_command: Optional[Sequence[str]]=None, @@ -1353,7 +1353,7 @@ def upload( and values are environment variable values for those names. serving_container_ports: Optional[Sequence[int]]=None, Declaration of ports that are exposed by the container. This field is - primarily informational, it gives AI Platform information about the + primarily informational, it gives Vertex AI information about the network connections the container uses. Listing or not a port here has no impact on whether the port is actually exposed, any port listening on the default "0.0.0.0" address inside a container will be accessible from @@ -1898,7 +1898,7 @@ def batch_predict( ```google.rpc.Status`` `__ represented as a STRUCT, and containing only ``code`` and ``message``. predictions_format: str = "jsonl" - Required. The format in which AI Platform gives the + Required. The format in which Vertex AI gives the predictions, must be one of "jsonl", "csv", or "bigquery". Default is "jsonl" when using `gcs_destination_prefix`. If a `bigquery_destination_prefix` is provided, this is overriden to @@ -1919,7 +1919,7 @@ def batch_predict( `machine_type`. Only used if `machine_type` is set. starting_replica_count: Optional[int] = None The number of machine replicas used at the start of the batch - operation. If not set, AI Platform decides starting number, not + operation. If not set, Vertex AI decides starting number, not greater than `max_replica_count`. Only used if `machine_type` is set. max_replica_count: Optional[int] = None diff --git a/google/cloud/aiplatform/schema.py b/google/cloud/aiplatform/schema.py index 6b2a3d7d66..a1da75d9e6 100644 --- a/google/cloud/aiplatform/schema.py +++ b/google/cloud/aiplatform/schema.py @@ -15,7 +15,7 @@ # limitations under the License. # -"""Namespaced AI Platform Schemas.""" +"""Namespaced Vertex AI Schemas.""" class training_job: diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 470e30bf56..039898c955 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -166,7 +166,7 @@ def get( doesn't match the custom training task definition. Returns: - An AI Platform Training Job + An Vertex AI Training Job """ # Create job with dummy parameters @@ -277,7 +277,7 @@ def _create_input_data_config( gcs_destination_uri_prefix (str): Optional. The Google Cloud Storage location. - The AI Platform environment variables representing Google + The Vertex AI environment variables representing Google Cloud Storage data URIs will always be represented in the Google Cloud Storage wildcard format to support sharded data. @@ -449,14 +449,14 @@ def _run_job( does not support uploading a Model as part of the pipeline. When the Pipeline's state becomes ``PIPELINE_STATE_SUCCEEDED`` and the trained Model had been - uploaded into AI Platform, then the model_to_upload's + uploaded into Vertex AI, then the model_to_upload's resource ``name`` is populated. The Model is always uploaded into the Project and Location in which this pipeline is. gcs_destination_uri_prefix (str): Optional. The Google Cloud Storage location. - The AI Platform environment variables representing Google + The Vertex AI environment variables representing Google Cloud Storage data URIs will always be represented in the Google Cloud Storage wildcard format to support sharded data. @@ -546,7 +546,7 @@ def state(self) -> Optional[gca_pipeline_state.PipelineState]: return self._gca_resource.state def get_model(self, sync=True) -> models.Model: - """AI Platform Model produced by this training, if one was produced. + """Vertex AI Model produced by this training, if one was produced. Args: sync (bool): @@ -555,7 +555,7 @@ def get_model(self, sync=True) -> models.Model: be immediately returned and synced when the Future has completed. Returns: - model: AI Platform Model produced by this training + model: Vertex AI Model produced by this training Raises: RuntimeError: If training failed or if a model was not produced by this training. @@ -569,7 +569,7 @@ def get_model(self, sync=True) -> models.Model: @base.optional_sync() def _force_get_model(self, sync: bool = True) -> models.Model: - """AI Platform Model produced by this training, if one was produced. + """Vertex AI Model produced by this training, if one was produced. Args: sync (bool): @@ -578,7 +578,7 @@ def _force_get_model(self, sync: bool = True) -> models.Model: be immediately returned and synced when the Future has completed. Returns: - model: AI Platform Model produced by this training + model: Vertex AI Model produced by this training Raises: RuntimeError: If training failed or if a model was not produced by this training. @@ -594,7 +594,7 @@ def _get_model(self) -> Optional[models.Model]: """Helper method to get and instantiate the Model to Upload. Returns: - model: AI Platform Model if training succeeded and produced an AI Platform + model: Vertex AI Model if training succeeded and produced an Vertex AI Model. None otherwise. Raises: @@ -806,15 +806,15 @@ def __init__( container_uri (str): Required: Uri of the training container image in the GCR. model_serving_container_image_uri (str): - If the training produces a managed AI Platform Model, the URI of the + If the training produces a managed Vertex AI Model, the URI of the Model serving container suitable for serving the model produced by the training script. model_serving_container_predict_route (str): - If the training produces a managed AI Platform Model, An HTTP path to + If the training produces a managed Vertex AI Model, An HTTP path to send prediction requests to the container, and which must be supported - by it. If not specified a default HTTP path will be used by AI Platform. + by it. If not specified a default HTTP path will be used by Vertex AI. model_serving_container_health_route (str): - If the training produces a managed AI Platform Model, an HTTP path to + If the training produces a managed Vertex AI Model, an HTTP path to send health check requests to the container, and which must be supported by it. If not specified a standard HTTP path will be used by AI Platform. @@ -839,7 +839,7 @@ def __init__( and values are environment variable values for those names. model_serving_container_ports (Sequence[int]): Declaration of ports that are exposed by the container. This field is - primarily informational, it gives AI Platform information about the + primarily informational, it gives Vertex AI information about the network connections the container uses. Listing or not a port here has no impact on whether the port is actually exposed, any port listening on the default "0.0.0.0" address inside a container will be accessible from @@ -1010,7 +1010,7 @@ def _prepare_and_validate_run( Args: model_display_name (str): - If the script produces a managed AI Platform Model. The display name of + If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -1130,10 +1130,10 @@ def _model_upload_fail_string(self) -> str: # TODO(b/172368325) add scheduling, custom_job.Scheduling class CustomTrainingJob(_CustomTrainingJob): - """Class to launch a Custom Training Job in AI Platform using a script. + """Class to launch a Custom Training Job in Vertex AI using a script. Takes a training implementation as a python script and executes that - script in Cloud AI Platform Training. + script in Cloud Vertex AI Training. """ def __init__( @@ -1184,7 +1184,7 @@ def __init__( TODO(b/169782082) add documentation about traning utilities - To ensure your model gets saved in AI Platform, write your saved model to + To ensure your model gets saved in Vertex AI, write your saved model to os.environ["AIP_MODEL_DIR"] in your provided training script. @@ -1197,15 +1197,15 @@ def __init__( requirements (Sequence[str]): List of python packages dependencies of script. model_serving_container_image_uri (str): - If the training produces a managed AI Platform Model, the URI of the + If the training produces a managed Vertex AI Model, the URI of the Model serving container suitable for serving the model produced by the training script. model_serving_container_predict_route (str): - If the training produces a managed AI Platform Model, An HTTP path to + If the training produces a managed Vertex AI Model, An HTTP path to send prediction requests to the container, and which must be supported - by it. If not specified a default HTTP path will be used by AI Platform. + by it. If not specified a default HTTP path will be used by Vertex AI. model_serving_container_health_route (str): - If the training produces a managed AI Platform Model, an HTTP path to + If the training produces a managed Vertex AI Model, an HTTP path to send health check requests to the container, and which must be supported by it. If not specified a standard HTTP path will be used by AI Platform. @@ -1230,7 +1230,7 @@ def __init__( and values are environment variable values for those names. model_serving_container_ports (Sequence[int]): Declaration of ports that are exposed by the container. This field is - primarily informational, it gives AI Platform information about the + primarily informational, it gives Vertex AI information about the network connections the container uses. Listing or not a port here has no impact on whether the port is actually exposed, any port listening on the default "0.0.0.0" address inside a container will be accessible from @@ -1386,7 +1386,7 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform.If none of the fractions are set, by default roughly 80% + decided by Vertex AI.If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -1398,7 +1398,7 @@ def run( datasets.VideoDataset, ] ): - AI Platform to fit this training against. Custom training script should + Vertex AI to fit this training against. Custom training script should retrieve datasets through passed in environment variables uris: os.environ["AIP_TRAINING_DATA_URI"] @@ -1431,7 +1431,7 @@ def run( and ``annotation_schema_uri``. model_display_name (str): - If the script produces a managed AI Platform Model. The display name of + If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -1440,7 +1440,7 @@ def run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -1518,8 +1518,8 @@ def run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ worker_pool_specs, managed_model = self._prepare_and_validate_run( model_display_name=model_display_name, @@ -1593,7 +1593,7 @@ def _run( datasets.VideoDataset, ] ): - AI Platform to fit this training against. + Vertex AI to fit this training against. annotation_schema_uri (str): Google Cloud Storage URI points to a YAML file describing annotation schema. @@ -1617,7 +1617,7 @@ def _run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -1671,8 +1671,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ package_gcs_uri = python_packager.package_and_copy_to_gcs( gcs_staging_dir=self._staging_bucket, @@ -1724,7 +1724,7 @@ def _run( class CustomContainerTrainingJob(_CustomTrainingJob): - """Class to launch a Custom Training Job in AI Platform using a + """Class to launch a Custom Training Job in Vertex AI using a Container.""" def __init__( @@ -1773,7 +1773,7 @@ def __init__( TODO(b/169782082) add documentation about traning utilities - To ensure your model gets saved in AI Platform, write your saved model to + To ensure your model gets saved in Vertex AI, write your saved model to os.environ["AIP_MODEL_DIR"] in your provided training script. @@ -1786,15 +1786,15 @@ def __init__( The command to be invoked when the container is started. It overrides the entrypoint instruction in Dockerfile when provided model_serving_container_image_uri (str): - If the training produces a managed AI Platform Model, the URI of the + If the training produces a managed Vertex AI Model, the URI of the Model serving container suitable for serving the model produced by the training script. model_serving_container_predict_route (str): - If the training produces a managed AI Platform Model, An HTTP path to + If the training produces a managed Vertex AI Model, An HTTP path to send prediction requests to the container, and which must be supported - by it. If not specified a default HTTP path will be used by AI Platform. + by it. If not specified a default HTTP path will be used by Vertex AI. model_serving_container_health_route (str): - If the training produces a managed AI Platform Model, an HTTP path to + If the training produces a managed Vertex AI Model, an HTTP path to send health check requests to the container, and which must be supported by it. If not specified a standard HTTP path will be used by AI Platform. @@ -1819,7 +1819,7 @@ def __init__( and values are environment variable values for those names. model_serving_container_ports (Sequence[int]): Declaration of ports that are exposed by the container. This field is - primarily informational, it gives AI Platform information about the + primarily informational, it gives Vertex AI information about the network connections the container uses. Listing or not a port here has no impact on whether the port is actually exposed, any port listening on the default "0.0.0.0" address inside a container will be accessible from @@ -1974,12 +1974,12 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: dataset (Union[datasets.ImageDataset,datasets.TabularDataset,datasets.TextDataset,datasets.VideoDataset]): - AI Platform to fit this training against. Custom training script should + Vertex AI to fit this training against. Custom training script should retrieve datasets through passed in environment variables uris: os.environ["AIP_TRAINING_DATA_URI"] @@ -2012,7 +2012,7 @@ def run( and ``annotation_schema_uri``. model_display_name (str): - If the script produces a managed AI Platform Model. The display name of + If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -2021,7 +2021,7 @@ def run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -2099,8 +2099,8 @@ def run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. Raises: RuntimeError: If Training job has already been run, staging_bucket has not @@ -2169,7 +2169,7 @@ def _run( datasets.VideoDataset, ] ): - AI Platform to fit this training against. + Vertex AI to fit this training against. annotation_schema_uri (str): Google Cloud Storage URI points to a YAML file describing annotation schema. @@ -2193,7 +2193,7 @@ def _run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -2246,8 +2246,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ for spec in worker_pool_specs: @@ -2440,7 +2440,7 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -2491,7 +2491,7 @@ def run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. If the script produces a managed AI Platform Model. The display name of + Optional. If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -2507,8 +2507,8 @@ def run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. Raises: RuntimeError: If Training job has already been run or is waiting to run. @@ -2555,7 +2555,7 @@ def _run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -2606,7 +2606,7 @@ def _run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. If the script produces a managed AI Platform Model. The display name of + Optional. If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -2623,8 +2623,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ training_task_definition = schema.training_job.definition.automl_tabular @@ -2883,7 +2883,7 @@ def run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. If the script produces a managed AI Platform Model. The display name of + Optional. If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -2893,8 +2893,8 @@ def run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. Raises: RuntimeError if Training job has already been run or is waiting to run. @@ -3068,7 +3068,7 @@ def _run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. If the script produces a managed AI Platform Model. The display name of + Optional. If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -3078,8 +3078,8 @@ def _run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ training_task_definition = schema.training_job.definition.automl_forecasting @@ -3311,7 +3311,7 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -3345,7 +3345,7 @@ def run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. The display name of the managed AI Platform Model. The name + Optional. The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. disable_early_stopping: bool = False @@ -3359,8 +3359,8 @@ def run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. Raises: RuntimeError: If Training job has already been run or is waiting to run. @@ -3403,7 +3403,7 @@ def _run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -3443,7 +3443,7 @@ def _run( will error. The minimum value is 1000 and the maximum is 72000. model_display_name (str): - Optional. The display name of the managed AI Platform Model. The name + Optional. The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If a `base_model` was provided, the display_name in the base_model will be overritten with this value. If not provided upon @@ -3460,8 +3460,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ # Retrieve the objective-specific training task schema based on prediction_type @@ -3491,7 +3491,7 @@ def _run( model_tbt.description = getattr(base_model._gca_resource, "description") model_tbt.labels = getattr(base_model._gca_resource, "labels") - # Set ID of AI Platform Model to base this training job off of + # Set ID of Vertex AI Model to base this training job off of training_task_inputs_dict["baseModelId"] = base_model.name return self._run_job( @@ -3514,11 +3514,11 @@ def _model_upload_fail_string(self) -> str: class CustomPythonPackageTrainingJob(_CustomTrainingJob): - """Class to launch a Custom Training Job in AI Platform using a Python + """Class to launch a Custom Training Job in Vertex AI using a Python Package. Takes a training implementation as a python package and executes - that package in Cloud AI Platform Training. + that package in Cloud Vertex AI Training. """ def __init__( @@ -3576,7 +3576,7 @@ def __init__( model_display_name='my-trained-model' ) - To ensure your model gets saved in AI Platform, write your saved model to + To ensure your model gets saved in Vertex AI, write your saved model to os.environ["AIP_MODEL_DIR"] in your provided training script. Args: @@ -3589,15 +3589,15 @@ def __init__( container_uri (str): Required: Uri of the training container image in the GCR. model_serving_container_image_uri (str): - If the training produces a managed AI Platform Model, the URI of the + If the training produces a managed Vertex AI Model, the URI of the Model serving container suitable for serving the model produced by the training script. model_serving_container_predict_route (str): - If the training produces a managed AI Platform Model, An HTTP path to + If the training produces a managed Vertex AI Model, An HTTP path to send prediction requests to the container, and which must be supported - by it. If not specified a default HTTP path will be used by AI Platform. + by it. If not specified a default HTTP path will be used by Vertex AI. model_serving_container_health_route (str): - If the training produces a managed AI Platform Model, an HTTP path to + If the training produces a managed Vertex AI Model, an HTTP path to send health check requests to the container, and which must be supported by it. If not specified a standard HTTP path will be used by AI Platform. @@ -3622,7 +3622,7 @@ def __init__( and values are environment variable values for those names. model_serving_container_ports (Sequence[int]): Declaration of ports that are exposed by the container. This field is - primarily informational, it gives AI Platform information about the + primarily informational, it gives Vertex AI information about the network connections the container uses. Listing or not a port here has no impact on whether the port is actually exposed, any port listening on the default "0.0.0.0" address inside a container will be accessible from @@ -3776,12 +3776,12 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform.If none of the fractions are set, by default roughly 80% + decided by Vertex AI.If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: dataset (Union[datasets.ImageDataset,datasets.TabularDataset,datasets.TextDataset,datasets.VideoDataset,]): - AI Platform to fit this training against. Custom training script should + Vertex AI to fit this training against. Custom training script should retrieve datasets through passed in environment variables uris: os.environ["AIP_TRAINING_DATA_URI"] @@ -3814,7 +3814,7 @@ def run( and ``annotation_schema_uri``. model_display_name (str): - If the script produces a managed AI Platform Model. The display name of + If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -3823,7 +3823,7 @@ def run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -3901,8 +3901,8 @@ def run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ worker_pool_specs, managed_model = self._prepare_and_validate_run( model_display_name=model_display_name, @@ -3967,7 +3967,7 @@ def _run( datasets.VideoDataset, ] ): - AI Platform to fit this training against. + Vertex AI to fit this training against. annotation_schema_uri (str): Google Cloud Storage URI points to a YAML file describing annotation schema. @@ -3991,7 +3991,7 @@ def _run( GCS output directory of job. If not provided a timestamped directory in the staging directory will be used. - AI Platform sets the following environment variables when it runs your training code: + Vertex AI sets the following environment variables when it runs your training code: - AIP_MODEL_DIR: a Cloud Storage URI of a directory intended for saving model artifacts, i.e. /model/ - AIP_CHECKPOINT_DIR: a Cloud Storage URI of a directory intended for saving checkpoints, i.e. /checkpoints/ @@ -4030,8 +4030,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ for spec in worker_pool_specs: spec["python_package_spec"] = { @@ -4228,7 +4228,7 @@ def run( Required. The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. model_display_name (str): - Optional. The display name of the managed AI Platform Model. The name + Optional. The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If not provided upon creation, the job's display_name is used. sync: bool = True @@ -4236,8 +4236,8 @@ def run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. Raises: RuntimeError: If Training job has already been run or is waiting to run. @@ -4289,7 +4289,7 @@ def _run( Required. The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. model_display_name (str): - Optional. The display name of the managed AI Platform Model. The name + Optional. The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. If a `base_model` was provided, the display_name in the base_model will be overritten with this value. If not provided upon @@ -4300,8 +4300,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ # Retrieve the objective-specific training task schema based on prediction_type @@ -4364,7 +4364,7 @@ def __init__( The type of prediction the Model is to produce, one of: "classification" - A classification model analyzes text data and returns a list of categories that apply to the text found in the data. - AI Platform offers both single-label and multi-label text classification models. + Vertex AI offers both single-label and multi-label text classification models. "extraction" - An entity extraction model inspects text data for known entities referenced in the data and labels those entities in the text. @@ -4478,7 +4478,7 @@ def run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -4498,7 +4498,7 @@ def run( Required. The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. model_display_name (str): - Optional. The display name of the managed AI Platform Model. + Optional. The display name of the managed Vertex AI Model. The name can be up to 128 characters long and can consist of any UTF-8 characters. @@ -4508,7 +4508,7 @@ def run( will be executed in concurrent Future and any downstream object will be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource. + model: The trained Vertex AI Model resource. Raises: RuntimeError: If Training job has already been run or is waiting to run. @@ -4545,7 +4545,7 @@ def _run( Any of ``training_fraction_split``, ``validation_fraction_split`` and ``test_fraction_split`` may optionally be provided, they must sum to up to 1. If the provided ones sum to less than 1, the remainder is assigned to sets as - decided by AI Platform. If none of the fractions are set, by default roughly 80% + decided by Vertex AI. If none of the fractions are set, by default roughly 80% of data will be used for training, 10% for validation, and 10% for test. Args: @@ -4567,7 +4567,7 @@ def _run( Required. The fraction of the input data that is to be used to evaluate the Model. This is ignored if Dataset is not provided. model_display_name (str): - Optional. If the script produces a managed AI Platform Model. The display name of + Optional. If the script produces a managed Vertex AI Model. The display name of the Model. The name can be up to 128 characters long and can be consist of any UTF-8 characters. @@ -4578,8 +4578,8 @@ def _run( be immediately returned and synced when the Future has completed. Returns: - model: The trained AI Platform Model resource or None if training did not - produce an AI Platform Model. + model: The trained Vertex AI Model resource or None if training did not + produce an Vertex AI Model. """ if model_display_name is None: diff --git a/google/cloud/aiplatform/utils/__init__.py b/google/cloud/aiplatform/utils/__init__.py index 22a4d985bb..08687bf99b 100644 --- a/google/cloud/aiplatform/utils/__init__.py +++ b/google/cloud/aiplatform/utils/__init__.py @@ -110,7 +110,7 @@ def extract_fields_from_resource_name( Args: resource_name (str): - Required. A fully-qualified AI Platform (Unified) resource name + Required. A fully-qualified Vertex AI resource name resource_noun (str): A resource noun to validate the resource name against. @@ -145,7 +145,7 @@ def full_resource_name( Args: resource_name (str): - Required. A fully-qualified AI Platform (Unified) resource name or + Required. A fully-qualified Vertex AI resource name or resource ID. resource_noun (str): A resource noun to validate the resource name against. @@ -163,7 +163,7 @@ def full_resource_name( Returns: resource_name (str): - A fully-qualified AI Platform (Unified) resource name. + A fully-qualified Vertex AI resource name. Raises: ValueError: @@ -257,7 +257,7 @@ def validate_region(region: str) -> bool: region = region.lower() if region not in constants.SUPPORTED_REGIONS: raise ValueError( - f"Unsupported region for AI Platform, select from {constants.SUPPORTED_REGIONS}" + f"Unsupported region for Vertex AI, select from {constants.SUPPORTED_REGIONS}" ) return True diff --git a/google/cloud/aiplatform/utils/worker_spec_utils.py b/google/cloud/aiplatform/utils/worker_spec_utils.py index 9a681d3b98..385ac83979 100644 --- a/google/cloud/aiplatform/utils/worker_spec_utils.py +++ b/google/cloud/aiplatform/utils/worker_spec_utils.py @@ -87,7 +87,7 @@ def is_empty(self) -> bool: class _DistributedTrainingSpec(NamedTuple): """Configuration for distributed training worker pool specs. - AI Platform Training expects configuration in this order: + Vertex AI Training expects configuration in this order: [ chief spec, # can only have one replica worker spec, @@ -122,15 +122,15 @@ class _DistributedTrainingSpec(NamedTuple): def pool_specs( self, ) -> List[Dict[str, Union[int, str, Dict[str, Union[int, str]]]]]: - """Return each pools spec in correct order for AI Platform as a list of + """Return each pools spec in correct order for Vertex AI as a list of dicts. Also removes specs if they are empty but leaves specs in if there unusual - specifications to not break the ordering in AI Platform Training. + specifications to not break the ordering in Vertex AI Training. ie. 0 chief replica, 10 worker replica, 3 ps replica Returns: - Order list of worker pool specs suitable for AI Platform Training. + Order list of worker pool specs suitable for Vertex AI Training. """ if self.chief_spec.replica_count > 1: raise ValueError("Chief spec replica count cannot be greater than 1.") From 52f787a596c473ab2b7b832b1f616cf3124e4ad5 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 15:21:26 -0400 Subject: [PATCH 04/11] chore: Rename AIPlatform classes to Vertex AI and rename README --- README.rst | 16 ++--- google/cloud/aiplatform/base.py | 64 +++++++++---------- google/cloud/aiplatform/datasets/dataset.py | 6 +- google/cloud/aiplatform/initializer.py | 8 +-- google/cloud/aiplatform/jobs.py | 8 +-- .../aiplatform/metadata/metadata_store.py | 2 +- google/cloud/aiplatform/metadata/resource.py | 2 +- google/cloud/aiplatform/models.py | 4 +- google/cloud/aiplatform/training_jobs.py | 6 +- google/cloud/aiplatform/utils/__init__.py | 14 ++-- 10 files changed, 65 insertions(+), 65 deletions(-) diff --git a/README.rst b/README.rst index 209b577ead..5dca33198c 100644 --- a/README.rst +++ b/README.rst @@ -1,10 +1,10 @@ -Python Client for Cloud AI Platform +Python Client for Vertex AI ================================================= |beta| |pypi| |versions| -`Cloud AI Platform`_: Google Cloud AI Platform is an integrated suite of machine learning tools and services for building and using ML models with AutoML or custom code. It offers both novices and experts the best workbench for the entire machine learning development lifecycle. +`Vertex AI`_: Google Vertex AI is an integrated suite of machine learning tools and services for building and using ML models with AutoML or custom code. It offers both novices and experts the best workbench for the entire machine learning development lifecycle. - `Client Library Documentation`_ - `Product Documentation`_ @@ -15,7 +15,7 @@ Python Client for Cloud AI Platform :target: https://pypi.org/project/google-cloud-aiplatform/ .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-aiplatform.svg :target: https://pypi.org/project/google-cloud-aiplatform/ -.. _Cloud AI Platform: https://cloud.google.com/ai-platform-unified/docs +.. _Vertex AI: https://cloud.google.com/ai-platform-unified/docs .. _Client Library Documentation: https://googleapis.dev/python/aiplatform/latest .. _Product Documentation: https://cloud.google.com/ai-platform-unified/docs @@ -26,12 +26,12 @@ In order to use this library, you first need to go through the following steps: 1. `Select or create a Cloud Platform project.`_ 2. `Enable billing for your project.`_ -3. `Enable the Cloud AI Platform API.`_ +3. `Enable the Vertex AI API.`_ 4. `Setup Authentication.`_ .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the Cloud AI Platform API.: https://cloud.google.com/ai-platform/docs +.. _Enable the Vertex AI API.: https://cloud.google.com/ai-platform/docs .. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation @@ -72,12 +72,12 @@ Windows Next Steps ~~~~~~~~~~ -- Read the `Client Library Documentation`_ for Cloud AI Platform +- Read the `Client Library Documentation`_ for Vertex AI API to see other available methods on the client. -- Read the `Cloud AI Platform API Product documentation`_ to learn +- Read the `Vertex AI API Product documentation`_ to learn more about the product and see How-to Guides. - View this `README`_ to see the full list of Cloud APIs that we cover. -.. _Cloud AI Platform API Product documentation: https://cloud.google.com/ai-platform-unified/docs +.. _Vertex AI API Product documentation: https://cloud.google.com/ai-platform-unified/docs .. _README: https://github.com/googleapis/google-cloud-python/blob/master/README.rst \ No newline at end of file diff --git a/google/cloud/aiplatform/base.py b/google/cloud/aiplatform/base.py index 620dbcc3fa..845b7d3753 100644 --- a/google/cloud/aiplatform/base.py +++ b/google/cloud/aiplatform/base.py @@ -60,13 +60,13 @@ def __init__(self, name: str = ""): def log_create_with_lro( self, - cls: Type["AiPlatformResourceNoun"], + cls: Type["VertexAiResourceNoun"], lro: Optional[operation.Operation] = None, ): """Logs create event with LRO. Args: - cls (AiPlatformResourceNoune): + cls (VertexAiResourceNoun): Vertex AI Resource Noun class that is being created. lro (operation.Operation): Optional. Backing LRO for creation. @@ -80,7 +80,7 @@ def log_create_with_lro( def log_create_complete( self, - cls: Type["AiPlatformResourceNoun"], + cls: Type["VertexAiResourceNoun"], resource: proto.Message, variable_name: str, ): @@ -89,7 +89,7 @@ def log_create_complete( Will also include code snippet to instantiate resource in SDK. Args: - cls (AiPlatformResourceNoun): + cls (VertexAiResourceNoun): Vertex AI Resource Noun class that is being created. resource (proto.Message): Vertex AI Resourc proto.Message @@ -103,7 +103,7 @@ def log_create_complete( def log_create_complete_with_getter( self, - cls: Type["AiPlatformResourceNoun"], + cls: Type["VertexAiResourceNoun"], resource: proto.Message, variable_name: str, ): @@ -112,7 +112,7 @@ def log_create_complete_with_getter( Will also include code snippet to instantiate resource in SDK. Args: - cls (AiPlatformResourceNoun): + cls (VertexAiResourceNoun): Vertex AI Resource Noun class that is being created. resource (proto.Message): Vertex AI Resourc proto.Message @@ -125,14 +125,14 @@ def log_create_complete_with_getter( ) def log_action_start_against_resource( - self, action: str, noun: str, resource_noun_obj: "AiPlatformResourceNoun" + self, action: str, noun: str, resource_noun_obj: "VertexAiResourceNoun" ): """Logs intention to start an action against a resource. Args: action (str): Action to complete against the resource ie: "Deploying". Can be empty string. noun (str): Noun the action acts on against the resource. Can be empty string. - resource_noun_obj (AiPlatformResourceNoun): + resource_noun_obj (VertexAiResourceNoun): Resource noun object the action is acting against. """ self._logger.info( @@ -143,7 +143,7 @@ def log_action_started_against_resource_with_lro( self, action: str, noun: str, - cls: Type["AiPlatformResourceNoun"], + cls: Type["VertexAiResourceNoun"], lro: operation.Operation, ): """Logs an action started against a resource with lro. @@ -151,7 +151,7 @@ def log_action_started_against_resource_with_lro( Args: action (str): Action started against resource. ie: "Deploy". Can be empty string. noun (str): Noun the action acts on against the resource. Can be empty string. - cls (AiPlatformResourceNoun): + cls (VertexAiResourceNoun): Resource noun object the action is acting against. lro (operation.Operation): Backing LRO for action. """ @@ -160,14 +160,14 @@ def log_action_started_against_resource_with_lro( ) def log_action_completed_against_resource( - self, noun: str, action: str, resource_noun_obj: "AiPlatformResourceNoun" + self, noun: str, action: str, resource_noun_obj: "VertexAiResourceNoun" ): """Logs action completed against resource. Args: noun (str): Noun the action acts on against the resource. Can be empty string. action (str): Action started against resource. ie: "Deployed". Can be empty string. - resource_noun_obj (AiPlatformResourceNoun): + resource_noun_obj (VertexAiResourceNoun): Resource noun object the action is acting against """ self._logger.info( @@ -385,7 +385,7 @@ def __repr__(self) -> str: return object.__repr__(self) -class AiPlatformResourceNoun(metaclass=abc.ABCMeta): +class VertexAiResourceNoun(metaclass=abc.ABCMeta): """Base class the Vertex AI resource nouns. Subclasses require two class attributes: @@ -400,7 +400,7 @@ class AiPlatformResourceNoun(metaclass=abc.ABCMeta): @property @classmethod @abc.abstractmethod - def client_class(cls) -> Type[utils.AiPlatformServiceClientWithOverride]: + def client_class(cls) -> Type[utils.VertexAiServiceClientWithOverride]: """Client class required to interact with resource with optional overrides.""" pass @@ -464,7 +464,7 @@ def _instantiate_client( cls, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> utils.AiPlatformServiceClientWithOverride: + ) -> utils.VertexAiServiceClientWithOverride: """Helper method to instantiate service client for resource noun. Args: @@ -473,7 +473,7 @@ def _instantiate_client( Optional custom credentials to use when accessing interacting with resource noun. Returns: - client (utils.AiPlatformServiceClientWithOverride): + client (utils.VertexAiServiceClientWithOverride): Initialized service client for this service noun with optional overrides. """ return initializer.global_config.create_client( @@ -580,7 +580,7 @@ def optional_sync( return_input_arg: Optional[str] = None, bind_future_to_self: bool = True, ): - """Decorator for AiPlatformResourceNounWithFutureManager with optional sync + """Decorator for VertexAIResourceNounWithFutureManager with optional sync support. Methods with this decorator should include a "sync" argument that defaults to @@ -714,7 +714,7 @@ def wrapper(*args, **kwargs): return optional_run_in_thread -class AiPlatformResourceNounWithFutureManager(AiPlatformResourceNoun, FutureManager): +class VertexAIResourceNounWithFutureManager(VertexAiResourceNoun, FutureManager): """Allows optional asynchronous calls to this Vertex AI Resource Nouns.""" @@ -735,7 +735,7 @@ def __init__( resource noun. resource_name(str): A fully-qualified resource name or ID. """ - AiPlatformResourceNoun.__init__( + VertexAiResourceNoun.__init__( self, project=project, location=location, @@ -751,7 +751,7 @@ def _empty_constructor( location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, resource_name: Optional[str] = None, - ) -> "AiPlatformResourceNounWithFutureManager": + ) -> "VertexAIResourceNounWithFutureManager": """Initializes with all attributes set to None. The attributes should be populated after a future is complete. This allows @@ -768,7 +768,7 @@ def _empty_constructor( An instance of this class with attributes set to None. """ self = cls.__new__(cls) - AiPlatformResourceNoun.__init__( + VertexAiResourceNoun.__init__( self, project=project, location=location, @@ -780,12 +780,12 @@ def _empty_constructor( return self def _sync_object_with_future_result( - self, result: "AiPlatformResourceNounWithFutureManager" + self, result: "VertexAIResourceNounWithFutureManager" ): """Populates attributes from a Future result to this object. Args: - result: AiPlatformResourceNounWithFutureManager + result: VertexAIResourceNounWithFutureManager Required. Result of future with same type as this object. """ sync_attributes = [ @@ -811,7 +811,7 @@ def _construct_sdk_resource_from_gapic( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> AiPlatformResourceNoun: + ) -> VertexAiResourceNoun: """Given a GAPIC resource object, return the SDK representation. Args: @@ -829,7 +829,7 @@ def _construct_sdk_resource_from_gapic( Overrides credentials set in aiplatform.init. Returns: - AiPlatformResourceNoun: + VertexAiResourceNoun: An initialized SDK object that represents GAPIC type. """ sdk_resource = self._empty_constructor( @@ -849,7 +849,7 @@ def _list( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List[AiPlatformResourceNoun]: + ) -> List[VertexAiResourceNoun]: """Private method to list all instances of this Vertex AI Resource, takes a `cls_filter` arg to filter to a particular SDK resource subclass. @@ -878,7 +878,7 @@ def _list( credentials set in aiplatform.init. Returns: - List[AiPlatformResourceNoun] - A list of SDK resource objects + List[VertexAiResourceNoun] - A list of SDK resource objects """ self = cls._empty_constructor( project=project, location=location, credentials=credentials @@ -918,7 +918,7 @@ def _list_with_local_order( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List[AiPlatformResourceNoun]: + ) -> List[VertexAiResourceNoun]: """Private method to list all instances of this Vertex AI Resource, takes a `cls_filter` arg to filter to a particular SDK resource subclass. Provides client-side sorting when a list API doesn't support @@ -948,7 +948,7 @@ def _list_with_local_order( credentials set in aiplatform.init. Returns: - List[AiPlatformResourceNoun] - A list of SDK resource objects + List[VertexAiResourceNoun] - A list of SDK resource objects """ li = cls._list( @@ -980,7 +980,7 @@ def list( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List[AiPlatformResourceNoun]: + ) -> List[VertexAiResourceNoun]: """List all instances of this Vertex AI Resource. Example Usage: @@ -1010,7 +1010,7 @@ def list( credentials set in aiplatform.init. Returns: - List[AiPlatformResourceNoun] - A list of SDK resource objects + List[VertexAiResourceNoun] - A list of SDK resource objects """ return cls._list( @@ -1042,7 +1042,7 @@ def delete(self, sync: bool = True) -> None: def __repr__(self) -> str: if self._gca_resource: - return AiPlatformResourceNoun.__repr__(self) + return VertexAiResourceNoun.__repr__(self) return FutureManager.__repr__(self) diff --git a/google/cloud/aiplatform/datasets/dataset.py b/google/cloud/aiplatform/datasets/dataset.py index a4db10c693..ee046b6585 100644 --- a/google/cloud/aiplatform/datasets/dataset.py +++ b/google/cloud/aiplatform/datasets/dataset.py @@ -35,7 +35,7 @@ _LOGGER = base.Logger(__name__) -class _Dataset(base.AiPlatformResourceNounWithFutureManager): +class _Dataset(base.VertexAIResourceNounWithFutureManager): """Managed dataset resource for Vertex AI.""" client_class = utils.DatasetClientWithOverride @@ -528,7 +528,7 @@ def list( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List[base.AiPlatformResourceNoun]: + ) -> List[base.VertexAiResourceNoun]: """List all instances of this Dataset resource. Example Usage: @@ -557,7 +557,7 @@ def list( credentials set in aiplatform.init. Returns: - List[base.AiPlatformResourceNoun] - A list of Dataset resource objects + List[base.VertexAiResourceNoun] - A list of Dataset resource objects """ dataset_subclass_filter = ( diff --git a/google/cloud/aiplatform/initializer.py b/google/cloud/aiplatform/initializer.py index a6f0786857..18341bde46 100644 --- a/google/cloud/aiplatform/initializer.py +++ b/google/cloud/aiplatform/initializer.py @@ -247,16 +247,16 @@ def common_location_path( def create_client( self, - client_class: Type[utils.AiPlatformServiceClientWithOverride], + client_class: Type[utils.VertexAiServiceClientWithOverride], credentials: Optional[auth_credentials.Credentials] = None, location_override: Optional[str] = None, prediction_client: bool = False, - ) -> utils.AiPlatformServiceClientWithOverride: - """Instantiates a given AiPlatformServiceClient with optional + ) -> utils.VertexAiServiceClientWithOverride: + """Instantiates a given VertexAiServiceClient with optional overrides. Args: - client_class (utils.AiPlatformServiceClientWithOverride): + client_class (utils.VertexAiServiceClientWithOverride): (Required) An Vertex AI Service Client with optional overrides. credentials (auth_credentials.Credentials): Custom auth credentials. If not provided will use the current config. diff --git a/google/cloud/aiplatform/jobs.py b/google/cloud/aiplatform/jobs.py index 61320235f7..a0aa81679f 100644 --- a/google/cloud/aiplatform/jobs.py +++ b/google/cloud/aiplatform/jobs.py @@ -73,7 +73,7 @@ ) -class _Job(base.AiPlatformResourceNounWithFutureManager): +class _Job(base.VertexAIResourceNounWithFutureManager): """Class that represents a general Job resource in Vertex AI. Cannot be directly instantiated. @@ -209,7 +209,7 @@ def list( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List[base.AiPlatformResourceNoun]: + ) -> List[base.VertexAiResourceNoun]: """List all instances of this Job Resource. Example Usage: @@ -237,7 +237,7 @@ def list( credentials set in aiplatform.init. Returns: - List[AiPlatformResourceNoun] - A list of Job resource objects + List[VertexAiResourceNoun] - A list of Job resource objects """ return cls._list_with_local_order( @@ -804,7 +804,7 @@ def __init__( credentials to use when accessing interacting with resource noun. """ - base.AiPlatformResourceNounWithFutureManager.__init__( + base.VertexAIResourceNounWithFutureManager.__init__( self, project=project, location=location, credentials=credentials ) diff --git a/google/cloud/aiplatform/metadata/metadata_store.py b/google/cloud/aiplatform/metadata/metadata_store.py index e712f611a8..2b9112813d 100644 --- a/google/cloud/aiplatform/metadata/metadata_store.py +++ b/google/cloud/aiplatform/metadata/metadata_store.py @@ -27,7 +27,7 @@ from google.cloud.aiplatform_v1beta1.types import metadata_store as gca_metadata_store -class _MetadataStore(base.AiPlatformResourceNounWithFutureManager): +class _MetadataStore(base.VertexAIResourceNounWithFutureManager): """Managed MetadataStore resource for Vertex AI""" client_class = utils.MetadataClientWithOverride diff --git a/google/cloud/aiplatform/metadata/resource.py b/google/cloud/aiplatform/metadata/resource.py index ba11f8eb07..4b5227c4fb 100644 --- a/google/cloud/aiplatform/metadata/resource.py +++ b/google/cloud/aiplatform/metadata/resource.py @@ -33,7 +33,7 @@ from google.cloud.aiplatform_v1beta1 import Execution as GapicExecution -class _Resource(base.AiPlatformResourceNounWithFutureManager, abc.ABC): +class _Resource(base.VertexAIResourceNounWithFutureManager, abc.ABC): """Metadata Resource for Vertex AI""" client_class = utils.MetadataClientWithOverride diff --git a/google/cloud/aiplatform/models.py b/google/cloud/aiplatform/models.py index c106c019ee..0e27638b6a 100644 --- a/google/cloud/aiplatform/models.py +++ b/google/cloud/aiplatform/models.py @@ -73,7 +73,7 @@ class Prediction(NamedTuple): explanations: Optional[Sequence[gca_explanation_v1beta1.Explanation]] = None -class Endpoint(base.AiPlatformResourceNounWithFutureManager): +class Endpoint(base.VertexAIResourceNounWithFutureManager): client_class = utils.EndpointClientWithOverride _is_client_prediction_client = False @@ -1201,7 +1201,7 @@ def delete(self, force: bool = False, sync: bool = True) -> None: super().delete(sync=sync) -class Model(base.AiPlatformResourceNounWithFutureManager): +class Model(base.VertexAIResourceNounWithFutureManager): client_class = utils.ModelClientWithOverride _is_client_prediction_client = False diff --git a/google/cloud/aiplatform/training_jobs.py b/google/cloud/aiplatform/training_jobs.py index 039898c955..1c00c901cb 100644 --- a/google/cloud/aiplatform/training_jobs.py +++ b/google/cloud/aiplatform/training_jobs.py @@ -61,7 +61,7 @@ ) -class _TrainingJob(base.AiPlatformResourceNounWithFutureManager): +class _TrainingJob(base.VertexAIResourceNounWithFutureManager): client_class = utils.PipelineClientWithOverride _is_client_prediction_client = False @@ -709,7 +709,7 @@ def list( project: Optional[str] = None, location: Optional[str] = None, credentials: Optional[auth_credentials.Credentials] = None, - ) -> List["base.AiPlatformResourceNoune"]: + ) -> List["base.VertexAiResourceNoun"]: """List all instances of this TrainingJob resource. Example Usage: @@ -738,7 +738,7 @@ def list( credentials set in aiplatform.init. Returns: - List[AiPlatformResourceNoun] - A list of TrainingJob resource objects + List[VertexAiResourceNoun] - A list of TrainingJob resource objects """ training_job_subclass_filter = ( diff --git a/google/cloud/aiplatform/utils/__init__.py b/google/cloud/aiplatform/utils/__init__.py index 08687bf99b..4404defb21 100644 --- a/google/cloud/aiplatform/utils/__init__.py +++ b/google/cloud/aiplatform/utils/__init__.py @@ -56,8 +56,8 @@ accelerator_type as gca_accelerator_type, ) -AiPlatformServiceClient = TypeVar( - "AiPlatformServiceClient", +VertexAiServiceClient = TypeVar( + "VertexAiServiceClient", # v1beta1 dataset_service_client_v1beta1.DatasetServiceClient, endpoint_service_client_v1beta1.EndpointServiceClient, @@ -324,7 +324,7 @@ class WrappedClient: def __init__( self, - client_class: Type[AiPlatformServiceClient], + client_class: Type[VertexAiServiceClient], client_options: client_options.ClientOptions, client_info: gapic_v1.client_info.ClientInfo, credentials: Optional[auth_credentials.Credentials] = None, @@ -332,7 +332,7 @@ def __init__( """Stores parameters needed to instantiate client. Args: - client_class (AiPlatformServiceClient): + client_class (VertexAiServiceClient): Required. Class of the client to use. client_options (client_options.ClientOptions): Required. Client options to pass to client. @@ -410,7 +410,7 @@ def __getattr__(self, name: str) -> Any: """Instantiates client and returns attribute of the client.""" return getattr(self._clients[self._default_version], name) - def select_version(self, version: str) -> AiPlatformServiceClient: + def select_version(self, version: str) -> VertexAiServiceClient: return self._clients[version] @@ -484,8 +484,8 @@ class TensorboardClientWithOverride(ClientWithOverride): ) -AiPlatformServiceClientWithOverride = TypeVar( - "AiPlatformServiceClientWithOverride", +VertexAiServiceClientWithOverride = TypeVar( + "VertexAiServiceClientWithOverride", DatasetClientWithOverride, EndpointClientWithOverride, JobClientWithOverride, From 8f19f22854c01053633a83fae1202304b607b55e Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 15:35:07 -0400 Subject: [PATCH 05/11] chore: update README to point to vertex-ai docs --- README.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 5dca33198c..a3e30be1cc 100644 --- a/README.rst +++ b/README.rst @@ -15,9 +15,9 @@ Python Client for Vertex AI :target: https://pypi.org/project/google-cloud-aiplatform/ .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-aiplatform.svg :target: https://pypi.org/project/google-cloud-aiplatform/ -.. _Vertex AI: https://cloud.google.com/ai-platform-unified/docs +.. _Vertex AI: https://cloud.google.com/vertex-ai/docs .. _Client Library Documentation: https://googleapis.dev/python/aiplatform/latest -.. _Product Documentation: https://cloud.google.com/ai-platform-unified/docs +.. _Product Documentation: https://cloud.google.com/vertex-ai/docs Quick Start ----------- @@ -79,5 +79,5 @@ Next Steps - View this `README`_ to see the full list of Cloud APIs that we cover. -.. _Vertex AI API Product documentation: https://cloud.google.com/ai-platform-unified/docs +.. _Vertex AI API Product documentation: https://cloud.google.com/vertex-ai/docs .. _README: https://github.com/googleapis/google-cloud-python/blob/master/README.rst \ No newline at end of file From 0d4cc0a6af6af5f97e4663d5503c1d962c795afa Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 15:46:21 -0400 Subject: [PATCH 06/11] checkpoint --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 97d133f66c..6163317bf6 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,7 @@ Overview ~~~~~~~~ Importing ^^^^^^^^^^^^^^^^^^^^ -All SDK functionality can be used from the root of the package: +SDK functionality can be used from the root of the package: .. code-block:: Python From dd4a7a7d855bb4e233b98d7cc069a64445867559 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 15:53:27 -0400 Subject: [PATCH 07/11] chore: update to Vertex SDK for Python --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a3e30be1cc..cfaca2bf90 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -Python Client for Vertex AI +Vertex SDK for Python ================================================= |beta| |pypi| |versions| From 483f0e25a8813324f293ff2c3099cbe5370cde19 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 16:22:58 -0400 Subject: [PATCH 08/11] checkpoint --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9f4d621ff7..9e941058fc 100644 --- a/README.rst +++ b/README.rst @@ -41,7 +41,7 @@ Initialize the SDK to store common configurations that will be used throughout t # environment default used is not set project='my-project', - # the AI platform region you will use + # the Vertex AI region you will use # defaults to us-central1 location='us-central1', From 89fe93033aa81619a1f9711830233d06b19af637 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Tue, 18 May 2021 18:05:47 -0400 Subject: [PATCH 09/11] checkpoint --- README.rst | 126 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 41 deletions(-) diff --git a/README.rst b/README.rst index 9e941058fc..ba3b739fca 100644 --- a/README.rst +++ b/README.rst @@ -19,6 +19,57 @@ Vertex SDK for Python .. _Client Library Documentation: https://googleapis.dev/python/aiplatform/latest .. _Product Documentation: https://cloud.google.com/vertex-ai/docs +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. `Enable the Vertex AI API.`_ +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Enable the Vertex AI API.: https://cloud.google.com/ai-platform/docs +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + pip install virtualenv + virtualenv + source /bin/activate + /bin/pip install google-cloud-aiplatform + + +Windows +^^^^^^^ + +.. code-block:: console + + pip install virtualenv + virtualenv + \Scripts\activate + \Scripts\pip.exe install google-cloud-aiplatform + + Overview ~~~~~~~~ Importing @@ -55,7 +106,14 @@ Initialize the SDK to store common configurations that will be used throughout t # customer managed encryption key resource name # will be applied to all AI Platform resources if set - encryption_spec_key_name=my_encryption_key_name + encryption_spec_key_name=my_encryption_key_name, + + # the name of the experiment to use to track + # logged metrics and parameters + experiment='my-experiment', + + # description of the experiment above + experiment_description='my experiment decsription' ) Datasets @@ -84,7 +142,7 @@ You can also create and import a dataset in separate steps: import_schema_uri=aiplatform.schema.dataset.ioformat.text.multi_label_classification ) -AI Platform supports a variety of dataset schemas. References to these schemas are available under the +Vertex AI supports a variety of dataset schemas. References to these schemas are available under the :code:`aiplatform.schema.dataset` namespace. For more information on the supported dataset schemas please refer to the `Preparing data docs`_. @@ -92,67 +150,53 @@ AI Platform supports a variety of dataset schemas. References to these schemas a Training ^^^^^^^^ -The AI Platform SDK allows you train Custom and AutoML Models. +The Vertex SDK allows you train Custom and AutoML Models. Custom models can be trained using a custom Python script, custom Python package, or container. Preparing Your Custom Code -------------------------- -AI Platform custom training enables you to train on AI Platform Datasets and produce AI Platform Models. To do so your +Vertex AI custom training enables you to train on AI Platform Datasets and produce AI Platform Models. To do so your script must adhere to the following contract: -1. It must read dataset from the given environment variables: +1. It must read dataset from the environment variables populated by the training service: +.. code-block:: Python -Quick Start ------------ + os.environ['AIP_DATA_FORMAT'] # provides format of data + os.environ['AIP_TRAINING_DATA_URI'] # uri to training split + os.environ['AIP_VALIDATION_DATA_URI'] # uri to validation split + os.environ['AIP_TEST_DATA_URI'] # uri to test split -In order to use this library, you first need to go through the following steps: +Please visit `Using a managed dataset in a custom training application`_ for a detailed overview information. -1. `Select or create a Cloud Platform project.`_ -2. `Enable billing for your project.`_ -3. `Enable the Vertex AI API.`_ -4. `Setup Authentication.`_ +.. _Using a managed dataset in a custom training application: https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets -.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project -.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the Vertex AI API.: https://cloud.google.com/ai-platform/docs -.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html +2. It must write the model artifact to the environment variable populated by the traing service: -Installation -~~~~~~~~~~~~ +.. code-block:: Python -Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to -create isolated Python environments. The basic problem it addresses is one of -dependencies and versions, and indirectly permissions. - -With `virtualenv`_, it's possible to install this library without needing system -install permissions, and without clashing with the installed system -dependencies. + os.environ['AIP_MODEL_DIR'] -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +Running Training +---------------- +.. code-block:: Python -Mac/Linux -^^^^^^^^^ - -.. code-block:: console + job = aiplatform.CustomTrainingJob( + display_name="my-training-job", + script_path="training_script.py", + container_uri="gcr.io/cloud-aiplatform/training/tf-cpu.2-2:latest", + requirements=["gcsfs==0.7.1"], + model_serving_container_image_uri="gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest", + ) - pip install virtualenv - virtualenv - source /bin/activate - /bin/pip install google-cloud-aiplatform + model = job.run(my_dataset, replica_count=1) +In the code block above my_dataset is managed dataset created in the `Datasets` section above. `model` is a Managed Vertex AI Model that the be deployed or exported. -Windows -^^^^^^^ -.. code-block:: console - pip install virtualenv - virtualenv - \Scripts\activate - \Scripts\pip.exe install google-cloud-aiplatform Next Steps ~~~~~~~~~~ From 4bbc48a2e7d7b7ee7b3ab63bdaee16c703ff661e Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Wed, 19 May 2021 11:53:12 -0400 Subject: [PATCH 10/11] chore: update README --- README.rst | 129 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 8df1c20129..de9233d388 100644 --- a/README.rst +++ b/README.rst @@ -45,7 +45,7 @@ With `virtualenv`_, it's possible to install this library without needing system install permissions, and without clashing with the installed system dependencies. -.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +.. _virtualenv: https://virtualenv.pypa.io/en/latest/ Mac/Linux @@ -72,8 +72,12 @@ Windows Overview ~~~~~~~~ +This section provides a brief overview of the Vertex SDK for Python. You can also reference the notebooks in `vertex-samples`_ for examples. + +.. _vertex-samples: https://github.com/GoogleCloudPlatform/ai-platform-samples/tree/master/ai-platform-unified/notebooks/unofficial/sdk + Importing -^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^ SDK functionality can be used from the root of the package: .. code-block:: Python @@ -82,7 +86,7 @@ SDK functionality can be used from the root of the package: Initialization -^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^ Initialize the SDK to store common configurations that will be used throughout the SDK. .. code-block:: Python @@ -142,6 +146,12 @@ You can also create and import a dataset in separate steps: import_schema_uri=aiplatform.schema.dataset.ioformat.text.multi_label_classification ) +To get a previously created Dataset: + +.. code-block:: Python + + dataset = aiplatform.ImageDataset('projects/my-project/location/us-central1/datasets/{DATASET_ID}') + Vertex AI supports a variety of dataset schemas. References to these schemas are available under the :code:`aiplatform.schema.dataset` namespace. For more information on the supported dataset schemas please refer to the `Preparing data docs`_. @@ -168,7 +178,7 @@ script must adhere to the following contract: os.environ['AIP_VALIDATION_DATA_URI'] # uri to validation split os.environ['AIP_TEST_DATA_URI'] # uri to test split -Please visit `Using a managed dataset in a custom training application`_ for a detailed overview information. +Please visit `Using a managed dataset in a custom training application`_ for a detailed overview. .. _Using a managed dataset in a custom training application: https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets @@ -191,11 +201,120 @@ Running Training model_serving_container_image_uri="gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest", ) - model = job.run(my_dataset, replica_count=1) + model = job.run(my_dataset, + replica_count=1, + machine_type="n1-standard-4", + accelerator_type='NVIDIA_TESLA_K80', + accelerator_count=1) In the code block above my_dataset is managed dataset created in the `Datasets` section above. `model` is a Managed Vertex AI Model that the be deployed or exported. +AutoMLs +------- +The Vertex SDK for Python supports AutoML Tabular, Image, Text, Video, and Forecasting. + +To train an AutoML Tabular model: + +.. code-block:: Python + + dataset = aiplatform.TabularDataset('projects/my-project/location/us-central1/datasets/{DATASET_ID}') + + job = aiplatform.AutoMLTabularTrainingJob( + display_name="train-automl", + optimization_prediction_type="regression", + optimization_objective="minimize-rmse", + ) + + model = job.run( + dataset=dataset, + target_column="target_column_name", + training_fraction_split=0.6, + validation_fraction_split=0.2, + test_fraction_split=0.2, + budget_milli_node_hours=1000, + model_display_name="my-automl-model", + disable_early_stopping=False, + ) + + +Models +------ + +To deploy a model: + + +.. code-block:: Python + + endpoint = model.deploy(machine_type="n1-standard-4", + min_replica_count=1, + max_replica_count=5 + machine_type='n1-standard-4', + accelerator_type='NVIDIA_TESLA_K80', + accelerator_count=1) + + +To upload a Model: + +.. code-block:: Python + + model = aiplatform.Model.upload( + display_name='my-model', + artifact_uri="gs://python/to/my/model/dir", + serving_container_image_uri="gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest", + ) + +To get a Model: + +.. code-block:: Python + + model = aiplatform.Model('/projects/my-project/locations/us-central1/models/{MODEL_ID}') + +Please visit `Importing models to Vertex AI`_ for a detailed overview: + +.. _Importing models to Vertex AI: https://cloud.google.com/vertex-ai/docs/general/import-model + + +Endpoints +--------- + +To get predictions from endpoints: + +.. code-block:: Python + + endpoint.predict(instances=[[6.7, 3.1, 4.7, 1.5], [4.6, 3.1, 1.5, 0.2]]) + + +To create an Endpoint + +.. code-block:: Python + + endpoint = endpoint.create(display_name='my-endpoint') + +To deploy a Model to a created Endpoint: + +.. code-block:: Python + + model = aiplatform.Model('/projects/my-project/locations/us-central1/models/{MODEL_ID}') + + endpoint.deploy(model, + min_replica_count=1, + max_replica_count=5 + machine_type='n1-standard-4', + accelerator_type='NVIDIA_TESLA_K80', + accelerator_count=1) + +To undeploy models from an Endpoint: + +.. code-block:: Python + + endpoint.undeploy_all() + +To delete an Endpoint: + +.. code-block:: Python + + endpoint.delete() Next Steps From 69624ce5c5befdfb63979ad7dddc0fee0c9be731 Mon Sep 17 00:00:00 2001 From: Sasha Sobran Date: Wed, 19 May 2021 12:52:14 -0400 Subject: [PATCH 11/11] chore: address reviewer comments and README linting --- README.rst | 51 +++++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index de9233d388..57ead60fea 100644 --- a/README.rst +++ b/README.rst @@ -72,9 +72,9 @@ Windows Overview ~~~~~~~~ -This section provides a brief overview of the Vertex SDK for Python. You can also reference the notebooks in `vertex-samples`_ for examples. +This section provides a brief overview of the Vertex SDK for Python. You can also reference the notebooks in `vertex-ai-samples`_ for examples. -.. _vertex-samples: https://github.com/GoogleCloudPlatform/ai-platform-samples/tree/master/ai-platform-unified/notebooks/unofficial/sdk +.. _vertex-ai-samples: https://github.com/GoogleCloudPlatform/ai-platform-samples/tree/master/ai-platform-unified/notebooks/unofficial/sdk Importing ^^^^^^^^^ @@ -87,12 +87,12 @@ SDK functionality can be used from the root of the package: Initialization ^^^^^^^^^^^^^^ -Initialize the SDK to store common configurations that will be used throughout the SDK. +Initialize the SDK to store common configurations that you use with the SDK. .. code-block:: Python aiplatform.init( - # your GCP project ID or number + # your Google Cloud Project ID or number # environment default used is not set project='my-project', @@ -100,7 +100,7 @@ Initialize the SDK to store common configurations that will be used throughout t # defaults to us-central1 location='us-central1', - # bucket in same region as location + # Googlge Cloud Stoage bucket in same region as location # used to stage artifacts staging_bucket='gs://my_staging_bucket', @@ -109,7 +109,7 @@ Initialize the SDK to store common configurations that will be used throughout t credentials=my_credentials, # customer managed encryption key resource name - # will be applied to all AI Platform resources if set + # will be applied to all Vertex AI resources if set encryption_spec_key_name=my_encryption_key_name, # the name of the experiment to use to track @@ -122,10 +122,10 @@ Initialize the SDK to store common configurations that will be used throughout t Datasets ^^^^^^^^ -AI Platform provides managed Tabular, Text, Image, and Video datasets. In the SDK, Datasets can be used downstream to +Vertex AI provides managed tabular, text, image, and video datasets. In the SDK, datasets can be used downstream to train models. -To create a Tabular dataset: +To create a tabular dataset: .. code-block:: Python @@ -160,16 +160,16 @@ Vertex AI supports a variety of dataset schemas. References to these schemas are Training ^^^^^^^^ -The Vertex SDK allows you train Custom and AutoML Models. +The Vertex SDK for Python allows you train Custom and AutoML Models. -Custom models can be trained using a custom Python script, custom Python package, or container. +You can train custom models using a custom Python script, custom Python package, or container. -Preparing Your Custom Code --------------------------- -Vertex AI custom training enables you to train on AI Platform Datasets and produce AI Platform Models. To do so your +**Preparing Your Custom Code** + +Vertex AI custom training enables you to train on Vertex AI datasets and produce Vertex AI models. To do so your script must adhere to the following contract: -1. It must read dataset from the environment variables populated by the training service: +It must read datasets from the environment variables populated by the training service: .. code-block:: Python @@ -182,14 +182,13 @@ Please visit `Using a managed dataset in a custom training application`_ for a d .. _Using a managed dataset in a custom training application: https://cloud.google.com/vertex-ai/docs/training/using-managed-datasets -2. It must write the model artifact to the environment variable populated by the traing service: +It must write the model artifact to the environment variable populated by the traing service: .. code-block:: Python os.environ['AIP_MODEL_DIR'] -Running Training ----------------- +**Running Training** .. code-block:: Python @@ -207,14 +206,14 @@ Running Training accelerator_type='NVIDIA_TESLA_K80', accelerator_count=1) -In the code block above my_dataset is managed dataset created in the `Datasets` section above. `model` is a Managed Vertex AI Model that the be deployed or exported. +In the code block above `my_dataset` is managed dataset created in the `Dataset` section above. The `model` variable is a managed Vertex AI model that can be deployed or exported. AutoMLs ------- -The Vertex SDK for Python supports AutoML Tabular, Image, Text, Video, and Forecasting. +The Vertex SDK for Python supports AutoML tabular, image, text, video, and forecasting. -To train an AutoML Tabular model: +To train an AutoML tabular model: .. code-block:: Python @@ -254,7 +253,7 @@ To deploy a model: accelerator_count=1) -To upload a Model: +To upload a model: .. code-block:: Python @@ -264,7 +263,7 @@ To upload a Model: serving_container_image_uri="gcr.io/cloud-aiplatform/prediction/tf2-cpu.2-2:latest", ) -To get a Model: +To get a model: .. code-block:: Python @@ -285,13 +284,13 @@ To get predictions from endpoints: endpoint.predict(instances=[[6.7, 3.1, 4.7, 1.5], [4.6, 3.1, 1.5, 0.2]]) -To create an Endpoint +To create an endpoint .. code-block:: Python endpoint = endpoint.create(display_name='my-endpoint') -To deploy a Model to a created Endpoint: +To deploy a model to a created endpoint: .. code-block:: Python @@ -304,13 +303,13 @@ To deploy a Model to a created Endpoint: accelerator_type='NVIDIA_TESLA_K80', accelerator_count=1) -To undeploy models from an Endpoint: +To undeploy models from an endpoint: .. code-block:: Python endpoint.undeploy_all() -To delete an Endpoint: +To delete an endpoint: .. code-block:: Python