diff --git a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh index 35bfb6aded..0cf1f472f7 100644 --- a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh +++ b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh @@ -9,8 +9,8 @@ ENDPOINT_SUFIX=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-5} | head ENDPOINT_NAME="heart-classifier-$ENDPOINT_SUFIX" # -MODEL_NAME='heart-classifier' -az ml model create --name $MODEL_NAME --type "mlflow_model" --path "model" +MODEL_NAME='heart-classifier-sklpipe' +az ml model create --name $MODEL_NAME --type "custom_model" --path "model" # echo "Creating compute" @@ -25,7 +25,7 @@ az ml batch-endpoint create -n $ENDPOINT_NAME -f endpoint.yml echo "Creating batch deployment $DEPLOYMENT_NAME for endpoint $ENDPOINT_NAME" # -az ml batch-deployment create --file deployment-parquet/deployment.yml --endpoint-name $ENDPOINT_NAME --set-default +az ml batch-deployment create --file deployment.yml --endpoint-name $ENDPOINT_NAME --set-default # echo "Update the batch deployment as default for the endpoint" diff --git a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml index c15c2a7ddc..4375652ec0 100644 --- a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml +++ b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml @@ -2,9 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/batchDeployment.schema.json endpoint_name: heart-classifier-batch name: classifier-xgboost-custom description: A heart condition classifier based on XGBoost and Scikit-Learn pipelines that append predictions on parquet files. -model: - path: model - name: heart-classifier-sklpipe +model: azureml:heart-classifier-sklpipe@latest environment: name: batch-mlflow-xgboost image: mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest diff --git a/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh b/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh index 3352a9a9ee..a1464e41ca 100644 --- a/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh +++ b/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh @@ -32,7 +32,7 @@ az ml batch-endpoint create --file endpoint.yml --name $ENDPOINT_NAME echo "Creating batch deployment for endpoint $ENDPOINT_NAME" # -az ml batch-deployment create --file deployment.yml --endpoint-name $ENDPOINT_NAME --set-default +az ml batch-deployment create --file deployment-by-file.yml --endpoint-name $ENDPOINT_NAME --set-default # echo "Showing details of the batch endpoint" @@ -46,9 +46,18 @@ DEPLOYMENT_NAME="imagenet-classifier-resnetv2" az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NAME # +# +wget https://azuremlexampledata.blob.core.windows.net/data/imagenet-1000.zip +unzip imagenet-1000.zip -d data +# + +# +az ml data create -f imagenet-sample-unlabeled.yml +# + echo "Invoking batch endpoint with local data" # -JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input data --input-type uri_folder --query name -o tsv) +JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:imagenet-sample-unlabeled@latest --query name -o tsv) # echo "Showing job detail" @@ -77,9 +86,45 @@ else fi # +# +az ml job download --name $JOB_NAME --output-name score --download-path . +# + +echo "Creating batch deployment for endpoint $ENDPOINT_NAME with high throughput" +# +az ml batch-deployment create --file deployment-by-batch.yml --endpoint-name $ENDPOINT_NAME --default +# + +echo "Invoking batch endpoint with local data" +# +JOB_NAME=$(az ml batch-endpoint invoke --name $ENDPOINT_NAME --input azureml:imagenet-sample-unlabeled@latest --query name -o tsv) +# + +echo "Stream job logs to console" +# +az ml job stream -n $JOB_NAME +# + +# +STATUS=$(az ml job show -n $JOB_NAME --query status -o tsv) +echo $STATUS +if [[ $STATUS == "Completed" ]] +then + echo "Job completed" +elif [[ $STATUS == "Failed" ]] +then + echo "Job failed" + exit 1 +else + echo "Job status not failed or completed" + exit 2 +fi +# + # az ml batch-endpoint delete --name $ENDPOINT_NAME --yes # echo "Clean temp files" find ./model -exec rm -rf {} + +find ./data -exec rm -rf {} + diff --git a/cli/endpoints/batch/deploy-models/imagenet-classifier/deployment-by-batch.yml b/cli/endpoints/batch/deploy-models/imagenet-classifier/deployment-by-batch.yml index eae21b8435..100090b88d 100644 --- a/cli/endpoints/batch/deploy-models/imagenet-classifier/deployment-by-batch.yml +++ b/cli/endpoints/batch/deploy-models/imagenet-classifier/deployment-by-batch.yml @@ -13,6 +13,9 @@ code_configuration: scoring_script: batch_driver.py resources: instance_count: 2 +tags: + device_acceleration: CUDA + device_batching: 16 max_concurrency_per_instance: 1 mini_batch_size: 5 output_action: append_row diff --git a/cli/endpoints/batch/deploy-models/imagenet-classifier/imagenet-sample-unlabeled.yml b/cli/endpoints/batch/deploy-models/imagenet-classifier/imagenet-sample-unlabeled.yml new file mode 100644 index 0000000000..f572daf284 --- /dev/null +++ b/cli/endpoints/batch/deploy-models/imagenet-classifier/imagenet-sample-unlabeled.yml @@ -0,0 +1,5 @@ +$schema: https://azuremlschemas.azureedge.net/latest/data.schema.json +name: imagenet-sample-unlabeled +description: A sample of 1000 images from the original ImageNet dataset. Download content from https://azuremlexampledata.blob.core.windows.net/data/imagenet-1000.zip. +type: uri_folder +path: data \ No newline at end of file diff --git a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-keras/deployment.yml b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-keras/deployment.yml index b07f01a7a0..d0921cd3d6 100644 --- a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-keras/deployment.yml +++ b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-keras/deployment.yml @@ -3,6 +3,7 @@ name: mnist-keras-dpl description: A deployment using Keras with TensorFlow to solve the MNIST classification dataset. endpoint_name: mnist-batch model: + name: mnist-classifier-keras path: model code_configuration: code: code diff --git a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py index e061fb60e2..a77ec6e907 100644 --- a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py +++ b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py @@ -1,6 +1,3 @@ -# Copyright (c) Microsoft. All rights reserved. -# Licensed under the MIT license. - import os import pandas as pd import torch @@ -47,8 +44,8 @@ def run(mini_batch: List[str]) -> pd.DataFrame: results.append( { "file": basename(image_path), - "class": predicted_class.numpy(), - "probability": predicted_prob.numpy(), + "class": predicted_class.numpy()[0], + "probability": predicted_prob.numpy()[0], } ) diff --git a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml index 524a533531..165d7d6a7a 100644 --- a/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml +++ b/cli/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/deployment.yml @@ -2,7 +2,8 @@ $schema: https://azuremlschemas.azureedge.net/latest/batchDeployment.schema.json name: mnist-torch-dpl description: A deployment using Torch to solve the MNIST classification dataset. endpoint_name: mnist-batch -model: +model: + name: mnist-classifier-torch path: model code_configuration: code: code diff --git a/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb index 653cf27655..7694bc0d63 100644 --- a/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/custom-outputs-parquet/custom-output-batch.ipynb @@ -120,7 +120,7 @@ "\n", "### 2.1 About the model\n", "\n", - "This example shows how you can deploy an MLflow model to a batch endpoint to perform batch predictions. This example uses a model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence).\n", + "This example shows how you can deploy a model to a batch endpoint to perform batch predictions. This example uses a model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence).\n", "\n", "The model has been trained using an `XGBBoost` classifier and all the required preprocessing has been packaged as a `scikit-learn` pipeline, making this model an end-to-end pipeline that goes from raw data to predictions.\n", "\n", @@ -145,7 +145,7 @@ }, "outputs": [], "source": [ - "model_name = \"heart-classifier\"\n", + "model_name = \"heart-classifier-sklpipe\"\n", "model_description = \"A heart condition classifier trained with XGBoosts and Scikit-Learn for feature processing.\"\n", "model_local_path = \"model\"" ] @@ -490,6 +490,7 @@ "outputs": [], "source": [ "environment = Environment(\n", + " name=\"batch-mlflow-xgboost\",\n", " conda_file=\"environment/conda.yml\",\n", " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n", ")" diff --git a/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb b/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb index 06b3d4969f..36f726eb07 100644 --- a/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb @@ -1,911 +1,918 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "# Use MLflow models in batch deployments\n", - "\n", - "In this article, learn how to deploy your MLflow model to Azure ML for both batch inference using batch endpoints. Azure Machine Learning supports no-code deployment of models created and logged with MLflow. This means that you don't have to provide a scoring script or an environment.\n", - "\n", - "For no-code-deployment, Azure Machine Learning\n", - "\n", - "* Provides a MLflow base image/curated environment that contains the required dependencies to run an Azure Machine Learning Batch job.\n", - "* Creates a batch job pipeline with a scoring script for you that can be used to process data using parallelization." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Connect to Azure Machine Learning Workspace\n", - "\n", - "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", - "\n", - "### 1.1. Import the required libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from azure.ai.ml import MLClient, Input\n", - "from azure.ai.ml.entities import BatchEndpoint, BatchDeployment, Model, AmlCompute, Data\n", - "from azure.ai.ml.constants import AssetTypes\n", - "from azure.identity import DefaultAzureCredential" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2. Configure workspace details and get a handle to the workspace\n", - "\n", - "To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. We will use these details in the `MLClient` from `azure.ai.ml` to get a handle to the required Azure Machine Learning workspace. We use the default [default azure authentication](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for this tutorial. Check the [configuration notebook](../../jobs/configuration.ipynb) for more details on how to configure credentials and connect to a workspace." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Use MLflow models in batch deployments\n", + "\n", + "In this article, learn how to deploy your MLflow model to Azure ML for both batch inference using batch endpoints. Azure Machine Learning supports no-code deployment of models created and logged with MLflow. This means that you don't have to provide a scoring script or an environment.\n", + "\n", + "For no-code-deployment, Azure Machine Learning\n", + "\n", + "* Provides a MLflow base image/curated environment that contains the required dependencies to run an Azure Machine Learning Batch job.\n", + "* Creates a batch job pipeline with a scoring script for you that can be used to process data using parallelization." + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "subscription_id = \"\"\n", - "resource_group = \"\"\n", - "workspace = \"\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client = MLClient(\n", - " DefaultAzureCredential(), subscription_id, resource_group, workspace\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## 2. Registering the model\n", - "\n", - "### 2.1 About the model\n", - "\n", - "This example shows how you can deploy an MLflow model to a batch endpoint to perform batch predictions. This example uses an MLflow model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence).\n", - "\n", - "The model has been trained using an `XGBBoost` classifier and all the required preprocessing has been packaged as a `scikit-learn` pipeline, making this model an end-to-end pipeline that goes from raw data to predictions.\n", - "\n", - "### 2.2 Registering the model in the workspace\n", - "\n", - "Let's verify if the model we want to deploy, `heart-classifier`, is registered in the model registry. If not, we will register it from a local version we have in the repository:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "## 1. Connect to Azure Machine Learning Workspace\n", + "\n", + "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", + "\n", + "### 1.1. Import the required libraries" + ], + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model_name = \"heart-classifier\"\n", - "model_local_path = \"model\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "from azure.ai.ml import MLClient, Input\n", + "from azure.ai.ml.entities import BatchEndpoint, BatchDeployment, Model, AmlCompute, Data\n", + "from azure.ai.ml.constants import AssetTypes\n", + "from azure.identity import DefaultAzureCredential" + ], + "outputs": [], + "execution_count": null, + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "if not any(filter(lambda m: m.name == model_name, ml_client.models.list())):\n", - " print(f\"Model {model_name} is not registered. Creating...\")\n", - " model = ml_client.models.create_or_update(\n", - " Model(name=model_name, path=model_local_path, type=AssetTypes.MLFLOW_MODEL)\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Let's get the model:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "### 1.2. Configure workspace details and get a handle to the workspace\n", + "\n", + "To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. We will use these details in the `MLClient` from `azure.ai.ml` to get a handle to the required Azure Machine Learning workspace. We use the default [default azure authentication](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for this tutorial. Check the [configuration notebook](../../jobs/configuration.ipynb) for more details on how to configure credentials and connect to a workspace." + ], + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "model = ml_client.models.get(name=model_name, label=\"latest\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "## 3 Create Batch Endpoint\n", - "\n", - "Batch endpoints are endpoints that are used batch inferencing on large volumes of data over a period of time. Batch endpoints receive pointers to data and run jobs asynchronously to process the data in parallel on compute clusters. Batch endpoints store outputs to a data store for further analysis.\n", - "\n", - "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", - "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", - "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", - "- `defaults` - Default settings for the endpoint.\n", - " - `deployment_name` - Name of the deployment that will serve as the default deployment for the endpoint.\n", - "- `description`- Description of the endpoint.\n", - "\n", - "### 3.1 Configure the endpoint\n", - "\n", - "First, let's create the endpoint that is going to host the batch deployments. To ensure that our endpoint name is unique, let's create a random suffix to append to it. \n", - "\n", - "> In general, you won't need to use this technique but you will use more meaningful names. Please skip the following cell if your case:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "subscription_id = \"\"\n", + "resource_group = \"\"\n", + "workspace = \"\"" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "import random\n", - "import string\n", - "\n", - "# Creating a unique endpoint name by including a random suffix\n", - "allowed_chars = string.ascii_lowercase + string.digits\n", - "endpoint_suffix = \"\".join(random.choice(allowed_chars) for x in range(5))\n", - "endpoint_name = \"heart-classifier-\" + endpoint_suffix" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's configure the endpoint:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "endpoint = BatchEndpoint(\n", - " name=endpoint_name,\n", - " description=\"A heart condition classifier for batch inference\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2 Create the endpoint\n", - "Using the `MLClient` created earlier, we will now create the Endpoint in the workspace. This command will start the endpoint creation and return a confirmation response while the endpoint creation continues." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "ml_client = MLClient(\n", + " DefaultAzureCredential(), subscription_id, resource_group, workspace\n", + ")" + ], + "outputs": [], + "execution_count": null, + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "ml_client.batch_endpoints.begin_create_or_update(endpoint).result()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Create a batch deployment\n", - "\n", - "A deployment is a set of resources required for hosting the model that does the actual inferencing. We will create a deployment for our endpoint using the `BatchDeployment` class.\n", - "\n", - "### 4.1 Creating an scoring script to work with the model\n", - "\n", - "> MLflow models don't require an scoring script." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 4.2 Creating the compute\n", - "\n", - "Batch deployments can run on any Azure ML compute that already exists in the workspace. That means that multiple batch deployments can share the same compute infrastructure. In this example, we are going to work on an AzureML compute cluster called `cpu-cluster`. Let's verify the compute exists on the workspace or create it otherwise." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "## 2. Registering the model\n", + "\n", + "### 2.1 About the model\n", + "\n", + "This example shows how you can deploy an MLflow model to a batch endpoint to perform batch predictions. This example uses an MLflow model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence).\n", + "\n", + "The model has been trained using an `XGBBoost` classifier and all the required preprocessing has been packaged as a `scikit-learn` pipeline, making this model an end-to-end pipeline that goes from raw data to predictions.\n", + "\n", + "### 2.2 Registering the model in the workspace\n", + "\n", + "Let's verify if the model we want to deploy, `heart-classifier`, is registered in the model registry. If not, we will register it from a local version we have in the repository:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "compute_name = \"cpu-cluster\"\n", - "if not any(filter(lambda m: m.name == compute_name, ml_client.compute.list())):\n", - " print(f\"Compute {compute_name} is not created. Creating...\")\n", - " compute_cluster = AmlCompute(\n", - " name=compute_name, description=\"amlcompute\", min_instances=0, max_instances=5\n", - " )\n", - " ml_client.begin_create_or_update(compute_cluster).result()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Compute may take time to be created. Let's wait for it:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "model_name = \"heart-classifier-mlflow\"\n", + "model_local_path = \"model\"" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from time import sleep\n", - "\n", - "print(\"Waiting for compute\", end=\"\")\n", - "while ml_client.compute.get(name=compute_name).provisioning_state == \"Creating\":\n", - " sleep(1)\n", - " print(\".\", end=\"\")\n", - "\n", - "print(\" [DONE]\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.3 Creating the environment\n", - "\n", - "> MLflow models don't require an environment." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 4.4 Configuring the deployment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "if not any(filter(lambda m: m.name == model_name, ml_client.models.list())):\n", + " print(f\"Model {model_name} is not registered. Creating...\")\n", + " model = ml_client.models.create_or_update(\n", + " Model(name=model_name, path=model_local_path, type=AssetTypes.MLFLOW_MODEL)\n", + " )" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "from azure.ai.ml.constants import BatchDeploymentOutputAction\n", - "from azure.ai.ml.entities import BatchRetrySettings\n", - "\n", - "deployment = BatchDeployment(\n", - " name=\"classifier-xgboost\",\n", - " description=\"A heart condition classifier based on XGBoost\",\n", - " endpoint_name=endpoint.name,\n", - " model=model,\n", - " compute=compute_name,\n", - " instance_count=2,\n", - " max_concurrency_per_instance=2,\n", - " mini_batch_size=10,\n", - " output_action=BatchDeploymentOutputAction.APPEND_ROW,\n", - " output_file_name=\"predictions.csv\",\n", - " retry_settings=BatchRetrySettings(max_retries=3, timeout=300),\n", - " logging_level=\"info\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.5 Create the deployment\n", - "Using the `MLClient` created earlier, we will now create the deployment in the workspace. This command will start the deployment creation and return a confirmation response while the deployment creation continues." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "Let's get the model:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "ml_client.batch_deployments.begin_create_or_update(deployment).result()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Once created, let's configure this new deployment as the default one:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "model = ml_client.models.get(name=model_name, label=\"latest\")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "endpoint = ml_client.batch_endpoints.get(endpoint.name)\n", - "endpoint.defaults.deployment_name = deployment.name\n", - "ml_client.batch_endpoints.begin_create_or_update(endpoint).result()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "We can see the endpoint URL as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "attachments": {}, + "cell_type": "markdown", + "source": [ + "## 3 Create Batch Endpoint\n", + "\n", + "Batch endpoints are endpoints that are used batch inferencing on large volumes of data over a period of time. Batch endpoints receive pointers to data and run jobs asynchronously to process the data in parallel on compute clusters. Batch endpoints store outputs to a data store for further analysis.\n", + "\n", + "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", + "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", + "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", + "- `description`- Description of the endpoint.\n", + "\n", + "### 3.1 Configure the endpoint\n", + "\n", + "First, let's create the endpoint that is going to host the batch deployments. To ensure that our endpoint name is unique, let's create a random suffix to append to it. \n", + "\n", + "> In general, you won't need to use this technique but you will use more meaningful names. Please skip the following cell if your case:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "print(f\"The default deployment is {endpoint.defaults.deployment_name}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "### 4.6 Testing the deployment\n", - "\n", - "Once the deployment is created, it is ready to recieve jobs.\n", - "\n", - "#### 4.6.1 Creating a data asset\n", - "\n", - "Let's first register a data asset so we can run the job against it. This data asset is a folder containing 1000 images from the original ImageNet dataset. We are going to download it first and then create the data asset:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "import random\n", + "import string\n", + "\n", + "# Creating a unique endpoint name by including a random suffix\n", + "allowed_chars = string.ascii_lowercase + string.digits\n", + "endpoint_suffix = \"\".join(random.choice(allowed_chars) for x in range(5))\n", + "endpoint_name = \"heart-classifier-\" + endpoint_suffix" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "data_path = \"data\"\n", - "dataset_name = \"heart-dataset-unlabeled\"\n", - "\n", - "heart_dataset_unlabeled = Data(\n", - " path=data_path,\n", - " type=AssetTypes.URI_FOLDER,\n", - " description=\"An unlabeled dataset for heart classification\",\n", - " name=dataset_name,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.data.create_or_update(heart_dataset_unlabeled)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's get a reference of the new data asset:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "Let's configure the endpoint:" + ], + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "heart_dataset_unlabeled = ml_client.data.get(name=dataset_name, label=\"latest\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### 4.6.2 Creating an input for the deployment" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "endpoint = BatchEndpoint(\n", + " name=endpoint_name,\n", + " description=\"A heart condition classifier for batch inference\",\n", + ")" + ], + "outputs": [], + "execution_count": null, + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "input = Input(type=AssetTypes.URI_FOLDER, path=heart_dataset_unlabeled.id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 4.6.3 Invoke the deployment\n", - "\n", - "Using the `MLClient` created earlier, we will get a handle to the endpoint. The endpoint can be invoked using the `invoke` command with the following parameters:\n", - "- `name` - Name of the endpoint\n", - "- `input_path` - Path where input data is present" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "### 3.2 Create the endpoint\n", + "Using the `MLClient` created earlier, we will now create the Endpoint in the workspace. This command will start the endpoint creation and return a confirmation response while the endpoint creation continues." + ], + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "job = ml_client.batch_endpoints.invoke(endpoint_name=endpoint.name, input=input)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "Notice how we are not indicating the deployment name in the invoke operation. That's because the endpoint automatically routes the job to the default deployment. Since our endpoint only has one deployment, then that one is the default one. You can target an specific deployment by indicating the argument/parameter `deployment_name`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "ml_client.batch_endpoints.begin_create_or_update(endpoint).result()" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "job = ml_client.batch_endpoints.invoke(\n", - " deployment_name=deployment.name, endpoint_name=endpoint.name, input=input\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - }, - "source": [ - "#### 4.6.4 Get the details of the invoked job\n", - "\n", - "Let us get details and logs of the invoked job:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "## 4. Create a batch deployment\n", + "\n", + "A deployment is a set of resources required for hosting the model that does the actual inferencing. We will create a deployment for our endpoint using the `BatchDeployment` class.\n", + "\n", + "### 4.1 Creating an scoring script to work with the model\n", + "\n", + "> MLflow models don't require an scoring script." + ], + "metadata": {} }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "ml_client.jobs.get(job.name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can wait for the job to finish using the following code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Waiting for batch job deployment {job.name}\", end=\"\")\n", - "while ml_client.jobs.get(name=job.name).status not in [\"Completed\", \"Failed\"]:\n", - " sleep(10)\n", - " print(\".\", end=\"\")\n", - "\n", - "print(\" [DONE]\")" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.7 Exploring the results\n", - "\n", - "The deployment creates a child job that executes the scoring. We can get the details of it using the following code:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "### 4.2 Creating the compute\n", + "\n", + "Batch deployments can run on any Azure ML compute that already exists in the workspace. That means that multiple batch deployments can share the same compute infrastructure. In this example, we are going to work on an AzureML compute cluster called `cpu-cluster`. Let's verify the compute exists on the workspace or create it otherwise." + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "scoring_job = list(ml_client.jobs.list(parent_job_name=job.name))[0]" - ] - }, - { - "attachments": {}, - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 4.7.1 Download the results\n", - "\n", - "The outputs generated by the deployment job will be placed in an output named `score`:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.jobs.download(name=scoring_job.name, download_path=\".\", output_name=\"score\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can read this data using pandas library:\n", - "\n", - "> Tip: Notice how the CSV is read as opposite to use directly `pd.read_csv`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "code", + "source": [ + "compute_name = \"cpu-cluster\"\n", + "if not any(filter(lambda m: m.name == compute_name, ml_client.compute.list())):\n", + " print(f\"Compute {compute_name} is not created. Creating...\")\n", + " compute_cluster = AmlCompute(\n", + " name=compute_name, description=\"amlcompute\", min_instances=0, max_instances=5\n", + " )\n", + " ml_client.begin_create_or_update(compute_cluster).result()" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } }, - "nteract": { - "transient": { - "deleting": false - } - } - }, - "outputs": [], - "source": [ - "with open(\"named-outputs/score/predictions.csv\", \"r\") as f:\n", - " data = f.read()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "jupyter": { - "outputs_hidden": false, - "source_hidden": false + { + "cell_type": "markdown", + "source": [ + "Compute may take time to be created. Let's wait for it:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "from time import sleep\n", + "\n", + "print(\"Waiting for compute\", end=\"\")\n", + "while ml_client.compute.get(name=compute_name).provisioning_state == \"Creating\":\n", + " sleep(1)\n", + " print(\".\", end=\"\")\n", + "\n", + "print(\" [DONE]\")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### 4.3 Creating the environment\n", + "\n", + "> MLflow models don't require an environment." + ], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "### 4.4 Configuring the deployment" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "from azure.ai.ml.constants import BatchDeploymentOutputAction\n", + "from azure.ai.ml.entities import BatchRetrySettings\n", + "\n", + "deployment = BatchDeployment(\n", + " name=\"classifier-xgboost\",\n", + " description=\"A heart condition classifier based on XGBoost\",\n", + " endpoint_name=endpoint.name,\n", + " model=model,\n", + " compute=compute_name,\n", + " instance_count=2,\n", + " max_concurrency_per_instance=2,\n", + " mini_batch_size=10,\n", + " output_action=BatchDeploymentOutputAction.APPEND_ROW,\n", + " output_file_name=\"predictions.csv\",\n", + " retry_settings=BatchRetrySettings(max_retries=3, timeout=300),\n", + " logging_level=\"info\",\n", + ")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### 4.5 Create the deployment\n", + "Using the `MLClient` created earlier, we will now create the deployment in the workspace. This command will start the deployment creation and return a confirmation response while the deployment creation continues." + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "ml_client.batch_deployments.begin_create_or_update(deployment).result()" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Once created, let's configure this new deployment as the default one:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "endpoint = ml_client.batch_endpoints.get(endpoint.name)\n", + "endpoint.defaults.deployment_name = deployment.name\n", + "ml_client.batch_endpoints.begin_create_or_update(endpoint).result()" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "We can see the endpoint URL as follows:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "print(f\"The default deployment is {endpoint.defaults.deployment_name}\")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "### 4.6 Testing the deployment\n", + "\n", + "Once the deployment is created, it is ready to recieve jobs.\n", + "\n", + "#### 4.6.1 Creating a data asset\n", + "\n", + "Let's first register a data asset so we can run the job against it. This data asset is a folder containing 1000 images from the original ImageNet dataset. We are going to download it first and then create the data asset:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "data_path = \"data\"\n", + "dataset_name = \"heart-dataset-unlabeled\"\n", + "\n", + "heart_dataset_unlabeled = Data(\n", + " path=data_path,\n", + " type=AssetTypes.URI_FOLDER,\n", + " description=\"An unlabeled dataset for heart classification\",\n", + " name=dataset_name,\n", + ")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "ml_client.data.create_or_update(heart_dataset_unlabeled)" + ], + "outputs": [], + "execution_count": null, + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "Let's get a reference of the new data asset:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "heart_dataset_unlabeled = ml_client.data.get(name=dataset_name, label=\"latest\")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "#### 4.6.2 Creating an input for the deployment" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "input = Input(type=AssetTypes.URI_FOLDER, path=heart_dataset_unlabeled.id)" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "#### 4.6.3 Invoke the deployment\n", + "\n", + "Using the `MLClient` created earlier, we will get a handle to the endpoint. The endpoint can be invoked using the `invoke` command with the following parameters:\n", + "- `name` - Name of the endpoint\n", + "- `input_path` - Path where input data is present" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "job = ml_client.batch_endpoints.invoke(endpoint_name=endpoint.name, input=input)" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "Notice how we are not indicating the deployment name in the invoke operation. That's because the endpoint automatically routes the job to the default deployment. Since our endpoint only has one deployment, then that one is the default one. You can target an specific deployment by indicating the argument/parameter `deployment_name`." + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "job = ml_client.batch_endpoints.invoke(\n", + " deployment_name=deployment.name, endpoint_name=endpoint.name, input=input\n", + ")" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "#### 4.6.4 Get the details of the invoked job\n", + "\n", + "Let us get details and logs of the invoked job:" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "ml_client.jobs.get(job.name)" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "We can wait for the job to finish using the following code:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "print(f\"Waiting for batch job deployment {job.name}\", end=\"\")\n", + "while ml_client.jobs.get(name=job.name).status not in [\"Completed\", \"Failed\"]:\n", + " sleep(10)\n", + " print(\".\", end=\"\")\n", + "\n", + "print(\" [DONE]\")" + ], + "outputs": [], + "execution_count": null, + "metadata": {} + }, + { + "attachments": {}, + "cell_type": "markdown", + "source": [ + "### 4.7 Exploring the results\n", + "\n", + "The deployment creates a child job that executes the scoring. We can get the details of it using the following code:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "scoring_job = list(ml_client.jobs.list(parent_job_name=job.name))[0]" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "attachments": {}, + "cell_type": "markdown", + "source": [ + "#### 4.7.1 Download the results\n", + "\n", + "The outputs generated by the deployment job will be placed in an output named `score`:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "ml_client.jobs.download(name=scoring_job.name, download_path=\".\", output_name=\"score\")" + ], + "outputs": [], + "execution_count": null, + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "We can read this data using pandas library:\n", + "\n", + "> Tip: Notice how the CSV is read as opposite to use directly `pd.read_csv`." + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "with open(\"named-outputs/score/predictions.csv\", \"r\") as f:\n", + " data = f.read()" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1680223027101 + } + } + }, + { + "cell_type": "code", + "source": [ + "from ast import literal_eval\n", + "import pandas as pd\n", + "\n", + "score = pd.DataFrame(\n", + " literal_eval(data.replace(\"\\n\", \",\")), columns=[\"file\", \"prediction\"]\n", + ")\n", + "score" + ], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "markdown", + "source": [ + "## 5. Clean up resources\n", + "\n", + "Clean-up the resources created. " + ], + "metadata": {} + }, + { + "cell_type": "code", + "source": [ + "ml_client.batch_endpoints.begin_delete(endpoint_name)" + ], + "outputs": [], + "execution_count": null, + "metadata": {} + } + ], + "metadata": { + "kernel_info": { + "name": "python310-sdkv2" + }, + "kernelspec": { + "name": "python310-sdkv2", + "language": "python", + "display_name": "Python 3.10 - SDK v2" + }, + "language_info": { + "name": "python", + "version": "3.10.9", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" }, "nteract": { - "transient": { - "deleting": false - } + "version": "nteract-front-end@1.0.0" + }, + "vscode": { + "interpreter": { + "hash": "8d732042e5e620df2ddb4aad7f460808ed1754fa045785bdb47941e58456b253" + } + }, + "microsoft": { + "ms_spell_check": { + "ms_spell_check_language": "en" + } } - }, - "outputs": [], - "source": [ - "from ast import literal_eval\n", - "import pandas as pd\n", - "\n", - "score = pd.DataFrame(\n", - " literal_eval(data.replace(\"\\n\", \",\")), columns=[\"file\", \"prediction\"]\n", - ")\n", - "score" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Clean up resources\n", - "\n", - "Clean-up the resources created. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ml_client.batch_endpoints.begin_delete(endpoint_name)" - ] - } - ], - "metadata": { - "kernel_info": { - "name": "amlv2" - }, - "kernelspec": { - "display_name": "Python 3.10 - SDK V2", - "language": "python", - "name": "python310-sdkv2" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.15" - }, - "nteract": { - "version": "nteract-front-end@1.0.0" }, - "vscode": { - "interpreter": { - "hash": "8d732042e5e620df2ddb4aad7f460808ed1754fa045785bdb47941e58456b253" - } - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb index 53763a1818..6e9cee0ac3 100644 --- a/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/huggingface-text-summarization/text-summarization-batch.ipynb @@ -288,6 +288,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -298,8 +299,6 @@ "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", - "- `defaults` - Default settings for the endpoint.\n", - " - `deployment_name` - Name of the deployment that will serve as the default deployment for the endpoint.\n", "- `description`- Description of the endpoint.\n", "\n", "### 3.1 Configure the endpoint\n", @@ -1068,9 +1067,9 @@ ], "metadata": { "kernelspec": { - "display_name": "azureml_py310_sdkv2", + "display_name": "Python 3.10 - SDK V2", "language": "python", - "name": "python3" + "name": "python310-sdkv2" }, "language_info": { "name": "python", diff --git a/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-batch.ipynb index 8a277affda..383a854abd 100644 --- a/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-batch.ipynb @@ -110,11 +110,9 @@ }, "outputs": [], "source": [ - "# ml_client = MLClient(\n", - "# DefaultAzureCredential(), subscription_id, resource_group, workspace\n", - "# )\n", - "\n", - "ml_client = MLClient.from_config(DefaultAzureCredential())" + "ml_client = MLClient(\n", + " DefaultAzureCredential(), subscription_id, resource_group, workspace\n", + ")" ] }, { @@ -421,6 +419,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": { "nteract": { @@ -437,8 +436,6 @@ "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", - "- `defaults` - Default settings for the endpoint.\n", - " - `deployment_name` - Name of the deployment that will serve as the default deployment for the endpoint.\n", "- `description`- Description of the endpoint.\n", "\n", "### 3.1 Configure the endpoint\n", @@ -699,6 +696,7 @@ "outputs": [], "source": [ "environment = Environment(\n", + " name=\"tensorflow27-cuda11-gpu\",\n", " conda_file=\"environment/conda.yml\",\n", " image=\"mcr.microsoft.com/azureml/curated/tensorflow-2.7-ubuntu20.04-py38-cuda11-gpu:latest\",\n", ")" diff --git a/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-mlflow.ipynb b/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-mlflow.ipynb index a4c0a0707c..85601865ce 100644 --- a/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-mlflow.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/imagenet-classifier/imagenet-classifier-mlflow.ipynb @@ -110,11 +110,9 @@ }, "outputs": [], "source": [ - "# ml_client = MLClient(\n", - "# DefaultAzureCredential(), subscription_id, resource_group, workspace\n", - "# )\n", - "\n", - "ml_client = MLClient.from_config(DefaultAzureCredential())" + "ml_client = MLClient(\n", + " DefaultAzureCredential(), subscription_id, resource_group, workspace\n", + ")" ] }, { @@ -463,6 +461,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": { "nteract": { @@ -479,8 +478,6 @@ "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", - "- `defaults` - Default settings for the endpoint.\n", - " - `deployment_name` - Name of the deployment that will serve as the default deployment for the endpoint.\n", "- `description`- Description of the endpoint.\n", "\n", "### 3.1 Configure the endpoint\n", @@ -531,6 +528,7 @@ "endpoint = BatchEndpoint(\n", " name=endpoint_name,\n", " description=\"An batch service to perform ImageNet image classification\",\n", + " tags={\"input-type\": \"tabular\"},\n", ")" ] }, @@ -830,6 +828,8 @@ }, "language_info": { "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", "version": "3.10.9" }, "orig_nbformat": 4 diff --git a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py index e061fb60e2..a77ec6e907 100644 --- a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py +++ b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/deployment-torch/code/batch_driver.py @@ -1,6 +1,3 @@ -# Copyright (c) Microsoft. All rights reserved. -# Licensed under the MIT license. - import os import pandas as pd import torch @@ -47,8 +44,8 @@ def run(mini_batch: List[str]) -> pd.DataFrame: results.append( { "file": basename(image_path), - "class": predicted_class.numpy(), - "probability": predicted_prob.numpy(), + "class": predicted_class.numpy()[0], + "probability": predicted_prob.numpy()[0], } ) diff --git a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb index 1ce7b07c25..f6d1a7ed82 100644 --- a/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb +++ b/sdk/python/endpoints/batch/deploy-models/mnist-classifier/mnist-batch.ipynb @@ -1,26 +1,13 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# Create and manage a batch endpoint for inferencing\n", "\n", - "**Requirements** - In order to benefit from this tutorial, you will need:\n", - "- A basic understanding of Machine Learning\n", - "- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", - "- An Azure ML workspace with computer cluster - [Configure workspace](../../jobs/configuration.ipynb) \n", - "\n", - "- A python environment\n", - "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../../README.md) - check the getting started section\n", - "\n", - "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", - "- Connect to your AML workspace from the Python SDK\n", - "- Create a batch endpoint from Python SDK\n", - "- Create deployments on that endpoint from Python SDK\n", - "- Test a deployment with a sample request\n", - "\n", - "**Motivations** - This notebook explains how to create an online endpoint and manage deployments on that endpoint. An endpoint is an HTTPS endpoint that clients can call to receive the inferencing (scoring) output of a trained model. Online endpoints are endpoints that are used for online (real-time) inferencing. " + "**Motivations** - In this example, we're going to deploy a model to solve the classic MNIST (\"Modified National Institute of Standards and Technology\") digit recognition problem to perform batch inferencing over large amounts of data (image files). In the first section of this tutorial, we're going to create a batch deployment with a model created using Torch. Such deployment will become our default one in the endpoint. In the second half, we're going to see how we can create a second deployment using a model created with TensorFlow (Keras), test it out, and then switch the endpoint to start using the new deployment as default once we confirm it is working.. " ] }, { @@ -50,6 +37,7 @@ " BatchDeployment,\n", " Model,\n", " Environment,\n", + " AmlCompute,\n", " BatchRetrySettings,\n", " CodeConfiguration,\n", ")\n", @@ -98,6 +86,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -107,9 +96,6 @@ "To create an online endpoint we will use `BatchEndpoint`. This class allows user to configure the following key aspects:\n", "- `name` - Name of the endpoint. Needs to be unique at the Azure region level\n", "- `auth_mode` - The authentication method for the endpoint. Currently only Azure Active Directory (Azure AD) token-based (`aad_token`) authentication is supported. \n", - "- `identity`- The managed identity configuration for accessing Azure resources for endpoint provisioning and inference.\n", - "- `defaults` - Default settings for the endpoint.\n", - " - `deployment_name` - Name of the deployment that will serve as the default deployment for the endpoint.\n", "- `description`- Description of the endpoint.\n", "\n", "## 2.1 Configure the endpoint" @@ -197,7 +183,7 @@ "metadata": {}, "outputs": [], "source": [ - "model_name = \"mnist-classification-torch\"\n", + "model_name = \"mnist-classifier-torch\"\n", "model_local_path = \"deployment-torch/model/\"" ] }, @@ -210,7 +196,12 @@ "if not any(filter(lambda m: m.name == model_name, ml_client.models.list())):\n", " print(f\"Model {model_name} is not registered. Creating...\")\n", " model = ml_client.models.create_or_update(\n", - " Model(name=model_name, path=model_local_path, type=AssetTypes.CUSTOM_MODEL)\n", + " Model(\n", + " name=model_name,\n", + " path=model_local_path,\n", + " type=AssetTypes.CUSTOM_MODEL,\n", + " tags={\"task\": \"classification\", \"framework\": \"torch\"},\n", + " )\n", " )" ] }, @@ -297,8 +288,8 @@ " \n", " results.append({ \n", " \"file\": basename(image_path),\n", - " \"class\": predicted_class.numpy(),\n", - " \"probability\": predicted_prob.numpy()\n", + " \"class\": predicted_class.numpy()[0],\n", + " \"probability\": predicted_prob.numpy()[0]\n", " })\n", "\n", " return pd.DataFrame(results)" @@ -347,6 +338,7 @@ "outputs": [], "source": [ "env = Environment(\n", + " name=\"batch-torch-py38\",\n", " conda_file=\"deployment-torch/environment/conda.yml\",\n", " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n", ")" @@ -509,6 +501,7 @@ "source": [ "job = ml_client.batch_endpoints.invoke(\n", " endpoint_name=endpoint_name,\n", + " deployment_name=deployment.name,\n", " input=input,\n", ")" ] @@ -643,6 +636,50 @@ "Let's now create a second deployment, but this time we will solve the same problem using a model trained with TensorFlow and Keras." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model_name = \"mnist-classifier-keras\"\n", + "model_local_path = \"deployment-keras/model/\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if not any(filter(lambda m: m.name == model_name, ml_client.models.list())):\n", + " print(f\"Model {model_name} is not registered. Creating...\")\n", + " model = ml_client.models.create_or_update(\n", + " Model(\n", + " name=model_name,\n", + " path=model_local_path,\n", + " type=AssetTypes.CUSTOM_MODEL,\n", + " tags={\"task\": \"classification\", \"framework\": \"tensorflow\"},\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's get a reference to the model:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = ml_client.models.get(name=model_name, label=\"latest\")" + ] + }, { "attachments": {}, "cell_type": "markdown", @@ -730,6 +767,7 @@ "outputs": [], "source": [ "env = Environment(\n", + " name=\"batch-tensorflow-py38\",\n", " conda_file=\"deployment-keras/environment/conda.yml\",\n", " image=\"mcr.microsoft.com/azureml/openmpi4.1.0-ubuntu20.04:latest\",\n", ")" @@ -1026,7 +1064,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.15" + "version": "3.10.9" }, "nteract": { "version": "nteract-front-end@1.0.0" diff --git a/sdk/python/endpoints/batch/readme.md b/sdk/python/endpoints/batch/readme.md index 7578332c01..6378ea8a73 100644 --- a/sdk/python/endpoints/batch/readme.md +++ b/sdk/python/endpoints/batch/readme.md @@ -12,7 +12,7 @@ Batch endpoints provide a convenient way to run inference over large volumes of Example | Description | Input data type | Notebook -|-|-|- [Batch score an MLflow model for the Heart Disease Classification problem](deploy-models/heart-classifier-mlflow) | This example shows how you can deploy an MLflow model to a batch endpoint to perform batch predictions. This example uses an MLflow model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence). The model has been trained using an XGBBoost classifier and all the required preprocessing has been packaged as a scikit-learn pipeline, making this model an end-to-end pipeline that goes from raw data to predictions. | Tabular | [See notebook](deploy-models/heart-classifier-mlflow/mlflow-for-batch-tabular.ipynb) -[Batch score an XGBoost model for the Heart Disease Classification problem and write predictions on parquet files](deploy-models/custom-outputs-parquet) | This example shows how you can deploy an MLflow model to a batch endpoint to perform batch predictions. This example uses an MLflow model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence). The model has been trained using an XGBBoost classifier and all the required preprocessing has been packaged as a scikit-learn pipeline, making this model an end-to-end pipeline that goes from raw data to predictions. This example also customizes the way the endpoint write predictions. | Tabular | [See notebook](deploy-models/custom-outputs-parquet/custom-output-batch.ipynb) +[Batch score an XGBoost model for the Heart Disease Classification problem and write predictions on parquet files](deploy-models/custom-outputs-parquet) | This example shows how you can deploy a model to a batch endpoint to perform batch predictions. This example uses a model based on the UCI Heart Disease Data Set. The database contains 76 attributes, but we are using a subset of 14 of them. The model tries to predict the presence of heart disease in a patient. It is integer valued from 0 (no presence) to 1 (presence). The model has been trained using an XGBBoost classifier and all the required preprocessing has been packaged as a scikit-learn pipeline, making this model an end-to-end pipeline that goes from raw data to predictions. This example also customizes the way the endpoint write predictions. | Tabular | [See notebook](deploy-models/custom-outputs-parquet/custom-output-batch.ipynb) [Batch score a model for MNIST classification with multiple deployments](deploy-models/mnist-classifier) | In this example, we're going to deploy a model to solve the classic MNIST ("Modified National Institute of Standards and Technology") digit recognition problem to perform batch inferencing over large amounts of data (image files). In the first section of this tutorial, we're going to create a batch deployment with a model created using Torch. Such deployment will become our default one in the endpoint. In the second half, we're going to see how we can create a second deployment using a model created with TensorFlow (Keras), test it out, and then switch the endpoint to start using the new deployment as default. | Images | [See notebook](deploy-models/mnist-classifier/mnist-batch.ipynb) [Batch score and classify images using a ResNet50 model for the ImageNet dataset](deploy-models/imagenet-classifier) | The model we are going to work with was built using TensorFlow along with the RestNet architecture (Identity Mappings in Deep Residual Networks). This example shows also how to perform high performance inference over batches of images on GPU. | Images | [See notebook](deploy-models/imagenet-classifier/imagenet-classifier-batch.ipynb) [Batch score and classify images using a ResNet50 model for the ImageNet dataset (MLflow)](deploy-models/imagenet-classifier) | The model we are going to work with was built using TensorFlow along with the RestNet architecture (Identity Mappings in Deep Residual Networks). This example shows you can package the model as MLflow and deploy later. | Images | [See notebook](deploy-models/imagenet-classifier/imagenet-classifier-mlflow.ipynb)