examples/vertex_pipeline/notebook/hpt_pipeline_development.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6f938540",
   "metadata": {},
   "source": [
    "# An end-to-end Vertex HPT Pipeline Demonstration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62f7de01",
   "metadata": {},
   "source": [
    "Finally, check that you have correctly installed the packages. The KFP SDK version should be >=1.6:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9222df33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFP SDK version: 1.8.18\n"
     ]
    }
   ],
   "source": [
    "!python3 -c \"import kfp; print('KFP SDK version: {}'.format(kfp.__version__))\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "164ff1c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "from functools import partial\n",
    "\n",
    "import kfp\n",
    "import pprint\n",
    "import yaml\n",
    "from jinja2 import Template\n",
    "from kfp.v2 import dsl\n",
    "from kfp.v2.compiler import compiler\n",
    "from kfp.v2.dsl import Dataset, Input, Metrics, Model, Output, component\n",
    "from kfp.v2.google.client import AIPlatformClient\n",
    "\n",
    "from google.cloud import aiplatform, firestore\n",
    "from datetime import datetime\n",
    "from google_cloud_pipeline_components.experimental import hyperparameter_tuning_job\n",
    "from google_cloud_pipeline_components.v1.hyperparameter_tuning_job import HyperparameterTuningJobRunOp\n",
    "from google_cloud_pipeline_components.v1.model import ModelUploadOp\n",
    "from google_cloud_pipeline_components.types import artifact_types\n",
    "from kfp.v2.components import importer_node\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dbe612cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "project_id='hyu-ml-sandbox'\n",
    "project_number='439762652'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "27224ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "af_registry_location='us-central1'\n",
    "af_registry_name='mlops-vertex-kit'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cfe4d02b",
   "metadata": {},
   "outputs": [],
   "source": [
    "components_dir='../components/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "82030efb",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def _load_custom_component(project_id: str,\n",
    "                           af_registry_location: str,\n",
    "                           af_registry_name: str,\n",
    "                           components_dir: str,\n",
    "                           component_name: str):\n",
    "  component_path = os.path.join(components_dir,\n",
    "                                component_name,\n",
    "                                'component.yaml.jinja')\n",
    "  with open(component_path, 'r') as f:\n",
    "    component_text = Template(f.read()).render(\n",
    "      project_id=project_id,\n",
    "      af_registry_location=af_registry_location,\n",
    "      af_registry_name=af_registry_name)\n",
    "\n",
    "  return kfp.components.load_component_from_text(component_text)\n",
    "\n",
    "load_custom_component = partial(_load_custom_component,\n",
    "                                project_id=project_id,\n",
    "                                af_registry_location=af_registry_location,\n",
    "                                af_registry_name=af_registry_name,\n",
    "                                components_dir=components_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "495502ee",
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "preprocess_op = load_custom_component(component_name='data_preprocess')\n",
    "train_op = load_custom_component(component_name='train_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0026cb75",
   "metadata": {},
   "source": [
    "Then define the pipeline using the following function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8d36a16a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_region='us-central1'\n",
    "pipeline_root='gs://vertex_pipeline_demo_root_hyu/pipeline_root'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "954a7af2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_region='us-central1'\n",
    "input_dataset_uri='bq://hyu-ml-sandbox.vertex_pipeline_demo.banknote_authentication'\n",
    "gcs_data_output_folder='gs://vertex_pipeline_demo_root_hyu/datasets/training'\n",
    "training_data_schema='VWT:float;SWT:float;KWT:float;Entropy:float;Class:int'\n",
    "\n",
    "data_pipeline_root='gs://vertex_pipeline_demo_root_hyu/compute_root'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e6812d61",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_container_image_uri=f'{af_registry_location}-docker.pkg.dev/{project_id}/{af_registry_name}/training:latest'\n",
    "serving_container_image_uri=f'{af_registry_location}-docker.pkg.dev/{project_id}/{af_registry_name}/serving:latest'\n",
    "hpt_container_image_uri=f'{af_registry_location}-docker.pkg.dev/{project_id}/{af_registry_name}/hpt:latest'\n",
    "custom_job_service_account=f'{project_number}-compute@developer.gserviceaccount.com'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4574529c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('us-central1-docker.pkg.dev/hyu-ml-sandbox/mlops-vertex-kit/training:latest',\n",
       " 'us-central1-docker.pkg.dev/hyu-ml-sandbox/mlops-vertex-kit/serving:latest',\n",
       " '439762652-compute@developer.gserviceaccount.com',\n",
       " 'us-central1-docker.pkg.dev/hyu-ml-sandbox/mlops-vertex-kit/hpt:latest')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "training_container_image_uri,serving_container_image_uri,custom_job_service_account,hpt_container_image_uri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7eb569d8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'{\"num_leaves_hp_param_min\": 6, \"num_leaves_hp_param_max\": 11, \"max_depth_hp_param_min\": -1, \"max_depth_hp_param_max\": 4}'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_additional_args = json.dumps({\n",
    "    'num_leaves_hp_param_min': 6,\n",
    "    'num_leaves_hp_param_max': 11,\n",
    "    'max_depth_hp_param_min': -1,\n",
    "    'max_depth_hp_param_max': 4\n",
    "})\n",
    "train_additional_args"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0d61fc89-15e3-4160-8eb6-8d4aafed5da5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from kfp.v2.dsl import component\n",
    "from kfp.v2.dsl import Dataset, Input, Metrics, Model, Output\n",
    "\n",
    "@component\n",
    "def worker_pool_specs(project_id: str,\n",
    "    data_region: str,\n",
    "    data_pipeline_root: str,\n",
    "    hpt_container_image_uri: str,\n",
    "    custom_job_service_account: str,\n",
    "    input_dataset: Input[Dataset],\n",
    "    input_data_schema: str) -> list:\n",
    "    \"\"\"\n",
    "    Pass the preprocessed data uri to hpt as a worker pool argument. The vanilla hpt API \n",
    "    doesn't support 'input data' so it's done this way.\n",
    "    \n",
    "    data_preprocess -> dataset.uri -> CMDARGS -> worker_pool_specs -> hpt\n",
    "    \"\"\"\n",
    "    \n",
    "    display_name = 'hpt-pipeline-template'\n",
    "    fields = [field.split(':')[0] for field in input_data_schema.split(';')]\n",
    "    label = fields[-1]\n",
    "    features = ','.join(fields[0:-1])\n",
    "    CMDARGS = [\n",
    "    \"--training_data_uri=\"+str(input_dataset.uri),\n",
    "    \"--training_data_schema=\"+input_data_schema,\n",
    "    \"--label=\"+label,\n",
    "    \"--features=\"+features\n",
    "]\n",
    "\n",
    "    # The spec of the worker pools including machine type and Docker image\n",
    "    worker_pool_specs = [\n",
    "    {\n",
    "        \"machine_spec\": {\n",
    "            \"machine_type\": \"n1-standard-4\",\n",
    "        },\n",
    "        \"replica_count\": 1,\n",
    "        \"container_spec\": {\"image_uri\": hpt_container_image_uri, \"args\": CMDARGS},\n",
    "    }\n",
    "    ]\n",
    "    \n",
    "    return worker_pool_specs\n",
    "\n",
    "\n",
    "@component(\n",
    "    packages_to_install=['google-cloud-aiplatform', \n",
    "                         'google-cloud-pipeline-components',\n",
    "                         'protobuf'], base_image='python:3.7')\n",
    "def GetBestTrialOp(gcp_resources: str, study_spec_metrics: list) -> str:\n",
    "\n",
    "    from google.cloud import aiplatform\n",
    "    from google_cloud_pipeline_components.proto.gcp_resources_pb2 import GcpResources\n",
    "    from google.protobuf.json_format import Parse\n",
    "    from google.cloud.aiplatform_v1.types import study\n",
    "\n",
    "    api_endpoint_suffix = '-aiplatform.googleapis.com'\n",
    "    gcp_resources_proto = Parse(gcp_resources, GcpResources())\n",
    "    gcp_resources_split = gcp_resources_proto.resources[0].resource_uri.partition(\n",
    "      'projects')\n",
    "    resource_name = gcp_resources_split[1] + gcp_resources_split[2]\n",
    "    prefix_str = gcp_resources_split[0]\n",
    "    prefix_str = prefix_str[:prefix_str.find(api_endpoint_suffix)]\n",
    "    api_endpoint = prefix_str[(prefix_str.rfind('//') + 2):] + api_endpoint_suffix\n",
    "\n",
    "    client_options = {'api_endpoint': api_endpoint}\n",
    "    job_client = aiplatform.gapic.JobServiceClient(client_options=client_options)\n",
    "    response = job_client.get_hyperparameter_tuning_job(name=resource_name)\n",
    "    \n",
    "    trials = [study.Trial.to_json(trial) for trial in response.trials]\n",
    "\n",
    "    if len(study_spec_metrics) > 1:\n",
    "        raise RuntimeError('Unable to determine best parameters for multi-objective'\n",
    "                       ' hyperparameter tuning.')\n",
    "    trials_list = [study.Trial.from_json(trial) for trial in trials]\n",
    "    best_trial = None\n",
    "    goal = study_spec_metrics[0]['goal']\n",
    "    best_fn = None\n",
    "    if goal == study.StudySpec.MetricSpec.GoalType.MAXIMIZE:\n",
    "        best_fn = max\n",
    "    elif goal == study.StudySpec.MetricSpec.GoalType.MINIMIZE:\n",
    "        best_fn = min\n",
    "    best_trial = best_fn(\n",
    "      trials_list, key=lambda trial: trial.final_measurement.metrics[0].value)\n",
    "\n",
    "    return study.Trial.to_json(best_trial)\n",
    "\n",
    "@component(packages_to_install=['google-cloud-firestore==2.3'])\n",
    "def best_hpt_to_args(hpt_best: str,\n",
    "                    project_id: str,\n",
    "                    solution_name: str) -> str:\n",
    "    \"\"\"\n",
    "    Write the best hpt params to firestore. \n",
    "    We keep the output to chain this component to the conditional training\n",
    "    \"\"\"\n",
    "    \n",
    "    import json\n",
    "    from datetime import datetime\n",
    "    from google.cloud import firestore\n",
    "    hpt_best = json.loads(hpt_best.replace(\"'\", '\"'))\n",
    "\n",
    "    hpt_best_dict = {}\n",
    "    \n",
    "    for i in hpt_best['parameters']:\n",
    "        hpt_best_dict.update({i['parameterId']: i['value']})\n",
    "    \n",
    "    for i in hpt_best['finalMeasurement']['metrics']:\n",
    "        hpt_best_dict.update({i['metricId']: i['value']})\n",
    "    \n",
    "    db = firestore.Client(project=project_id)\n",
    "    task_flag=datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "    db.collection(solution_name).document(task_flag).set(hpt_best_dict,merge=True)\n",
    "    \n",
    "    return \"true\"\n",
    "\n",
    "def hpt(project_id,\n",
    "         data_region,\n",
    "         data_pipeline_root,\n",
    "         preprocess_task,\n",
    "         display_name,\n",
    "         metric_spec,\n",
    "         parameter_spec,\n",
    "         max_trial_count,\n",
    "         parallel_trial_count,\n",
    "         study_spec_algorithm,\n",
    "         hpt_container_image_uri,\n",
    "         training_data_schema):\n",
    "    \n",
    "    worker_pool_specs_op = worker_pool_specs(project_id=project_id,\n",
    "    data_region=data_region,\n",
    "    data_pipeline_root=data_pipeline_root,\n",
    "    hpt_container_image_uri=hpt_container_image_uri,\n",
    "    custom_job_service_account=custom_job_service_account,                               \n",
    "    input_dataset=preprocess_task.outputs['output_dataset'],\n",
    "    input_data_schema=training_data_schema\n",
    "    )\n",
    "\n",
    "    tuning_op = HyperparameterTuningJobRunOp(\n",
    "    display_name=display_name,\n",
    "    project=project_id,\n",
    "    location=data_region,\n",
    "    worker_pool_specs=worker_pool_specs_op.output,\n",
    "    study_spec_metrics=metric_spec,\n",
    "    study_spec_parameters=parameter_spec,\n",
    "    max_trial_count=max_trial_count,\n",
    "    parallel_trial_count=parallel_trial_count,\n",
    "    base_output_directory=data_pipeline_root,\n",
    "    study_spec_algorithm=study_spec_algorithm\n",
    "    )\n",
    "\n",
    "    best_trial_op = GetBestTrialOp(\n",
    "        gcp_resources=tuning_op.outputs[\"gcp_resources\"], study_spec_metrics=metric_spec\n",
    "    )\n",
    "    \n",
    "    best_hpt_to_args_op = best_hpt_to_args(best_trial_op.output,\n",
    "                                          project_id=project_id,               \n",
    "                                          solution_name=display_name)\n",
    "    return best_hpt_to_args_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a36e0154",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEARCH_SPACE={\"num_boost_round\": [100, 200],\n",
    "             \"min_data_in_leaf\": [5, 10]}\n",
    "\n",
    "@dsl.pipeline(name='hpt-pipeline-template')\n",
    "def pipeline(project_id: str,\n",
    "             data_region: str,\n",
    "             gcs_data_output_folder: str,\n",
    "             input_dataset_uri: str,\n",
    "             training_data_schema: str,\n",
    "             data_pipeline_root: str,\n",
    "             \n",
    "             training_container_image_uri: str,\n",
    "             hpt_container_image_uri: str,\n",
    "             train_additional_args: str,\n",
    "             serving_container_image_uri: str,\n",
    "             custom_job_service_account: str,\n",
    "             hptune_region: str,\n",
    "             hp_config_suggestions_per_request: int,\n",
    "             hp_config_max_trials: int,\n",
    "             \n",
    "             metrics_name: str,\n",
    "             metrics_threshold: float,\n",
    "             \n",
    "             endpoint_machine_type: str,\n",
    "             endpoint_min_replica_count: int,\n",
    "             endpoint_max_replica_count: int,\n",
    "             endpoint_test_instances: str,\n",
    "             \n",
    "             monitoring_user_emails: str,\n",
    "             monitoring_log_sample_rate: float,\n",
    "             monitor_interval: int,\n",
    "             monitoring_default_threshold: float,\n",
    "             monitoring_custom_skew_thresholds: str,\n",
    "             monitoring_custom_drift_thresholds: str,\n",
    "             \n",
    "             machine_type: str = \"n1-standard-4\",\n",
    "             accelerator_count: int = 0,\n",
    "             accelerator_type: str = 'ACCELERATOR_TYPE_UNSPECIFIED',\n",
    "             vpc_network: str = \"\",\n",
    "             enable_model_monitoring: str = 'False'):\n",
    "    \n",
    "    display_name = 'hpt-pipeline-template'\n",
    "    metric_spec = hyperparameter_tuning_job.serialize_metrics({\"accuracy\": \"maximize\"})\n",
    "    parameter = {}\n",
    "    for i in SEARCH_SPACE.keys():\n",
    "        parameter.update({i: aiplatform.hyperparameter_tuning.DiscreteParameterSpec(\n",
    "            values=SEARCH_SPACE[i], scale=None)})\n",
    "    parameter_spec = hyperparameter_tuning_job.serialize_parameters(parameter)\n",
    "    max_trial_count=int(np.prod([len(i) for i in SEARCH_SPACE.values()]))\n",
    "    parallel_trial_count=max_trial_count\n",
    "    study_spec_algorithm='GRID_SEARCH'\n",
    "\n",
    "    dataset_importer = kfp.v2.dsl.importer(\n",
    "      artifact_uri=input_dataset_uri,\n",
    "      artifact_class=Dataset,\n",
    "      reimport=False)\n",
    "\n",
    "    preprocess_task = preprocess_op(\n",
    "      project_id=project_id,\n",
    "      data_region=data_region,\n",
    "      gcs_output_folder=gcs_data_output_folder,\n",
    "      gcs_output_format=\"CSV\",\n",
    "      input_dataset=dataset_importer.output)\n",
    "    \n",
    "    hpt_op = hpt(project_id,\n",
    "             data_region,\n",
    "             data_pipeline_root,\n",
    "             preprocess_task,\n",
    "             display_name,\n",
    "             metric_spec,\n",
    "             parameter_spec,\n",
    "             max_trial_count,\n",
    "             parallel_trial_count,\n",
    "             study_spec_algorithm,\n",
    "             hpt_container_image_uri,\n",
    "             training_data_schema)\n",
    "    \n",
    "    with dsl.Condition(\n",
    "        hpt_op.output==\"true\",\n",
    "        name=\"train_model\"\n",
    "    ):\n",
    "\n",
    "        train_task = train_op(\n",
    "          project_id=project_id,\n",
    "          data_region=data_region,\n",
    "          data_pipeline_root=data_pipeline_root,\n",
    "          input_data_schema=training_data_schema,\n",
    "          training_container_image_uri=training_container_image_uri,\n",
    "          train_additional_args=train_additional_args,\n",
    "          serving_container_image_uri=serving_container_image_uri,\n",
    "          custom_job_service_account=custom_job_service_account,\n",
    "          input_dataset=preprocess_task.outputs['output_dataset'],\n",
    "          machine_type=machine_type,\n",
    "          accelerator_count=accelerator_count,\n",
    "          accelerator_type=accelerator_type,\n",
    "          hptune_region=hptune_region,\n",
    "          hp_config_max_trials=hp_config_max_trials,\n",
    "          hp_config_suggestions_per_request=hp_config_suggestions_per_request,\n",
    "          vpc_network=vpc_network)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff5fba56",
   "metadata": {},
   "source": [
    "### Compile and run the end-to-end ML pipeline\n",
    "With our full pipeline defined, it's time to compile it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "794c18bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/kfp/v2/compiler/compiler.py:1293: FutureWarning: APIs imported from the v1 namespace (e.g. kfp.dsl, kfp.components, etc) will not be supported by the v2 compiler since v2.0.0\n",
      "  category=FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "compiler.Compiler().compile(\n",
    "    pipeline_func=pipeline, \n",
    "    package_path=\"hpt_pipeline_job.json\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "575b575f",
   "metadata": {},
   "source": [
    "Next, instantiate an API client:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5ab349ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/kfp/v2/google/client/client.py:173: FutureWarning: AIPlatformClient will be deprecated in v2.0.0. Please use PipelineJob https://googleapis.dev/python/aiplatform/latest/_modules/google/cloud/aiplatform/pipeline_jobs.html in Vertex SDK. Install the SDK using \"pip install google-cloud-aiplatform\"\n",
      "  category=FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "api_client = AIPlatformClient(\n",
    "    project_id=project_id,\n",
    "    region=pipeline_region)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "951f42d8",
   "metadata": {},
   "source": [
    "Next, kick off a pipeline run:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0423941b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[{\"VWT\": 3.6216, \"SWT\": 8.6661, \"KWT\": -2.8073, \"Entropy\": -0.44699, \"Class\": \"0\"}, {\"VWT\": 4.5459, \"SWT\": 8.1674, \"KWT\": -2.4586, \"Entropy\": -1.4621, \"Class\": \"0\"}, {\"VWT\": 3.866, \"SWT\": -2.6383, \"KWT\": 1.9242, \"Entropy\": 0.10645, \"Class\": \"0\"}, {\"VWT\": -3.7503, \"SWT\": -13.4586, \"KWT\": 17.5932, \"Entropy\": -2.7771, \"Class\": \"1\"}, {\"VWT\": -3.5637, \"SWT\": -8.3827, \"KWT\": 12.393, \"Entropy\": -1.2823, \"Class\": \"1\"}, {\"VWT\": -2.5419, \"SWT\": -0.65804, \"KWT\": 2.6842, \"Entropy\": 1.1952, \"Class\": \"1\"}]'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_instances = json.dumps([\n",
    "\t\t{\"VWT\":3.6216,\"SWT\":8.6661,\"KWT\":-2.8073,\"Entropy\":-0.44699,\"Class\":\"0\"},\n",
    "\t\t{\"VWT\":4.5459,\"SWT\":8.1674,\"KWT\":-2.4586,\"Entropy\":-1.4621,\"Class\":\"0\"},\n",
    "\t\t{\"VWT\":3.866,\"SWT\":-2.6383,\"KWT\":1.9242,\"Entropy\":0.10645,\"Class\":\"0\"},\n",
    "\t\t{\"VWT\":-3.7503,\"SWT\":-13.4586,\"KWT\":17.5932,\"Entropy\":-2.7771,\"Class\":\"1\"},\n",
    "\t\t{\"VWT\":-3.5637,\"SWT\":-8.3827,\"KWT\":12.393,\"Entropy\":-1.2823,\"Class\":\"1\"},\n",
    "\t\t{\"VWT\":-2.5419,\"SWT\":-0.65804,\"KWT\":2.6842,\"Entropy\":1.1952,\"Class\":\"1\"}\n",
    "\t\t])\n",
    "test_instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "5a3fe7dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "See the Pipeline job <a href=\"https://console.cloud.google.com/vertex-ai/locations/us-central1/pipelines/runs/hpt-pipeline-template-20230311092541?project=hyu-ml-sandbox\" target=\"_blank\" >here</a>."
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline_params = {\n",
    "    'project_id': project_id,\n",
    "    'data_region': data_region,\n",
    "    'gcs_data_output_folder': gcs_data_output_folder,\n",
    "    'input_dataset_uri': input_dataset_uri,\n",
    "    'training_data_schema': training_data_schema,\n",
    "    'data_pipeline_root': data_pipeline_root,\n",
    "    \n",
    "    'training_container_image_uri': training_container_image_uri,\n",
    "    'hpt_container_image_uri': hpt_container_image_uri,\n",
    "    'train_additional_args': train_additional_args,\n",
    "    'serving_container_image_uri': serving_container_image_uri,\n",
    "    'custom_job_service_account': custom_job_service_account,\n",
    "    'hptune_region':\"\",\n",
    "    'hp_config_suggestions_per_request': 5,\n",
    "    'hp_config_max_trials': 30,\n",
    "    \n",
    "    'metrics_name': 'au_prc',\n",
    "    'metrics_threshold': 0.4,\n",
    "    \n",
    "    'endpoint_machine_type': 'n1-standard-4',\n",
    "    'endpoint_min_replica_count': 1,\n",
    "    'endpoint_max_replica_count': 1,\n",
    "    'endpoint_test_instances': test_instances,\n",
    "    \n",
    "    'monitoring_user_emails': 'simon19891101@google.com',\n",
    "    'monitoring_log_sample_rate': 0.8,\n",
    "    'monitor_interval': 3600,\n",
    "    'monitoring_default_threshold': 0.3,\n",
    "    'monitoring_custom_skew_thresholds': 'VWT:.5,SWT:.2,KWT:.7,Entropy:.4',\n",
    "    'monitoring_custom_drift_thresholds': 'VWT:.5,SWT:.2,KWT:.7,Entropy:.4',\n",
    "    'enable_model_monitoring': 'True'\n",
    "}\n",
    "\n",
    "response = api_client.create_run_from_job_spec(\n",
    "    job_spec_path=\"hpt_pipeline_job.json\", \n",
    "    pipeline_root=pipeline_root,\n",
    "    parameter_values=pipeline_params,\n",
    "    service_account=custom_job_service_account,\n",
    "    enable_caching=False)"
   ]
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-6.m103",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-6:m103"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}