Santiagxf/batch pup (#2269)

* pipelines * fix * fixes * snppets * tests * models * models * example * example * name * fixes * sdk * tags * fix * batchv2 * fixes * fixes * fix * fixes * fixes * fixes * build * changes * fixes * fixes * build * black * wait * compute * pandas * review * build * readme * sample * scripts * review --------- Co-authored-by: santiagxf <fasantia@microsfot.com>
Azure · May 17, 2023 · 7a7ad56 · 7a7ad56
1 parent 0c56b57
commit 7a7ad56
Show file tree

Hide file tree

Showing 103 changed files with 8,251 additions and 53,177 deletions.
diff --git a/...ws/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml b/...ws/sdk-endpoints-batch-deploy-models-heart-classifier-mlflow-mlflow-for-batch-tabular.yml
@@ -34,6 +34,8 @@ jobs:
         python-version: "3.8"
     - name: pip install notebook reqs
       run: pip install -r sdk/python/dev-requirements.txt
+    - name: pip install mlflow reqs
+      run: pip install -r sdk/python/mlflow-requirements.txt
     - name: azure login
       uses: azure/login@v1
       with:

diff --git a/...endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml b/...endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml
@@ -0,0 +1,75 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "29 5/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/**
+      - .github/workflows/sdk-endpoints-batch-deploy-pipelines-batch-scoring-with-preprocessing-sdk-deploy-and-test.yml
+      - sdk/python/dev-requirements.txt
+      - infra/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing/sdk-deploy-and-test.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb
+      working-directory: sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: sdk-deploy-and-test
+        path: sdk/python/endpoints/batch/deploy-pipelines/batch-scoring-with-preprocessing
diff --git a/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml b/.github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml
@@ -0,0 +1,75 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "29 5/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/endpoints/batch/deploy-pipelines/hello-batch/**
+      - .github/workflows/sdk-endpoints-batch-deploy-pipelines-hello-batch-sdk-deploy-and-test.yml
+      - sdk/python/dev-requirements.txt
+      - infra/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run endpoints/batch/deploy-pipelines/hello-batch/sdk-deploy-and-test.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb
+      working-directory: sdk/python/endpoints/batch/deploy-pipelines/hello-batch
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: sdk-deploy-and-test
+        path: sdk/python/endpoints/batch/deploy-pipelines/hello-batch
diff --git a/...ows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml b/...ows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml
@@ -0,0 +1,75 @@
+# This code is autogenerated.
+# Code is generated by running custom script: python3 readme.py
+# Any manual changes to this file may cause incorrect behavior.
+# Any manual changes will be overwritten if the code is regenerated.
+
+name: sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test
+# This file is created by sdk/python/readme.py.
+# Please do not edit directly.
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "29 5/12 * * *"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - sdk/python/endpoints/batch/deploy-pipelines/training-with-components/**
+      - .github/workflows/sdk-endpoints-batch-deploy-pipelines-training-with-components-sdk-deploy-and-test.yml
+      - sdk/python/dev-requirements.txt
+      - infra/**
+      - sdk/python/setup.sh
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: check out repo
+      uses: actions/checkout@v2
+    - name: setup python
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.8"
+    - name: pip install notebook reqs
+      run: pip install -r sdk/python/dev-requirements.txt
+    - name: azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
+    - name: bootstrap resources
+      run: |
+          echo '${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}';
+          bash bootstrap.sh
+      working-directory: infra
+      continue-on-error: false
+    - name: setup SDK
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: sdk/python
+      continue-on-error: true
+    - name: setup-cli
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash setup.sh
+      working-directory: cli
+      continue-on-error: true
+    - name: run endpoints/batch/deploy-pipelines/training-with-components/sdk-deploy-and-test.ipynb
+      run: |
+          source "${{ github.workspace }}/infra/sdk_helpers.sh";
+          source "${{ github.workspace }}/infra/init_environment.sh";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" generate_workspace_config "../../.azureml/config.json";
+          bash "${{ github.workspace }}/infra/sdk_helpers.sh" replace_template_values "sdk-deploy-and-test.ipynb";
+          [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
+          papermill -k python sdk-deploy-and-test.ipynb sdk-deploy-and-test.output.ipynb
+      working-directory: sdk/python/endpoints/batch/deploy-pipelines/training-with-components
+    - name: upload notebook's working folder as an artifact
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v2
+      with:
+        name: sdk-deploy-and-test
+        path: sdk/python/endpoints/batch/deploy-pipelines/training-with-components
diff --git a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deploy-and-run.sh
@@ -4,9 +4,13 @@ set -e
 export ENDPOINT_NAME="<YOUR_ENDPOINT_NAME>"
 # </set_variables>
 
+# <name_endpoint>
+ENDPOINT_NAME="heart-classifier-custom"
+# </name_endpoint>
+
 # The following code ensures the created deployment has a unique name
 ENDPOINT_SUFIX=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w ${1:-5} | head -n 1)
-ENDPOINT_NAME="heart-classifier-$ENDPOINT_SUFIX"
+ENDPOINT_NAME="$ENDPOINT_NAME-$ENDPOINT_SUFIX"
 
 # <register_model>
 MODEL_NAME='heart-classifier-sklpipe'
@@ -19,30 +23,30 @@ az ml compute create -n batch-cluster --type amlcompute --min-instances 0 --max-
 # </create_compute>
 
 echo "Creating batch endpoint $ENDPOINT_NAME"
-# <create_batch_endpoint>
+# <create_endpoint>
 az ml batch-endpoint create -n $ENDPOINT_NAME -f endpoint.yml
-# </create_batch_endpoint>
+# </create_endpoint>
+
+echo "Showing details of the batch endpoint"
+# <query_endpoint>
+az ml batch-endpoint show --name $ENDPOINT_NAME
+# </query_endpoint>
 
 echo "Creating batch deployment $DEPLOYMENT_NAME for endpoint $ENDPOINT_NAME"
-# <create_batch_deployment_set_default>
+# <create_deployment>
 az ml batch-deployment create --file deployment.yml --endpoint-name $ENDPOINT_NAME --set-default
-# </create_batch_deployment_set_default>
+# </create_deployment>
 
 echo "Update the batch deployment as default for the endpoint"
-# <update_default_deployment>
+# <set_default_deployment>
 DEPLOYMENT_NAME="classifier-xgboost-custom"
 az ml batch-endpoint update --name $ENDPOINT_NAME --set defaults.deployment_name=$DEPLOYMENT_NAME
-# </update_default_deployment>
-
-echo "Showing details of the batch endpoint"
-# <check_batch_endpooint_detail>
-az ml batch-endpoint show --name $ENDPOINT_NAME
-# </check_batch_endpooint_detail>
+# </set_default_deployment>
 
 echo "Showing details of the batch deployment"
-# <check_batch_deployment_detail>
+# <query_deployment>
 az ml batch-deployment show --name $DEPLOYMENT_NAME --endpoint-name $ENDPOINT_NAME
-# </check_batch_deployment_detail>
+# </query_deployment>
 
 echo "Invoking batch endpoint"
 # <start_batch_scoring_job>
@@ -60,9 +64,9 @@ az ml batch-deployment list-jobs --name $DEPLOYMENT_NAME --endpoint-name $ENDPOI
 # </list_all_jobs>
 
 echo "Stream job logs to console"
-# <stream_job_logs_to_console>
+# <stream_job_logs>
 az ml job stream -n $JOB_NAME
-# </stream_job_logs_to_console>
+# </stream_job_logs>
 
 # <check_job_status>
 STATUS=$(az ml job show -n $JOB_NAME --query status -o tsv)
@@ -81,9 +85,9 @@ fi
 # </check_job_status>
 
 echo "Download scores to local path"
-# <download_scores>
+# <download_outputs>
 az ml job download --name $JOB_NAME --output-name score --download-path ./
-# </download_scores>
+# </download_outputs>
 
 echo "Delete resources"
 # <delete_endpoint>

diff --git a/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml b/cli/endpoints/batch/deploy-models/custom-outputs-parquet/deployment.yml
@@ -2,6 +2,7 @@ $schema: https://azuremlschemas.azureedge.net/latest/batchDeployment.schema.json
 endpoint_name: heart-classifier-batch
 name: classifier-xgboost-custom
 description: A heart condition classifier based on XGBoost and Scikit-Learn pipelines that append predictions on parquet files.
+type: model
 model: azureml:heart-classifier-sklpipe@latest
 environment:
   name: batch-mlflow-xgboost
@@ -13,11 +14,12 @@ code_configuration:
 compute: azureml:batch-cluster
 resources:
   instance_count: 2
-max_concurrency_per_instance: 2
-mini_batch_size: 2
-output_action: summary_only
-retry_settings:
-  max_retries: 3
-  timeout: 300
-error_threshold: -1
-logging_level: info
+settings:
+  max_concurrency_per_instance: 2
+  mini_batch_size: 2
+  output_action: summary_only
+  retry_settings:
+    max_retries: 3
+    timeout: 300
+  error_threshold: -1
+  logging_level: info