Added validation for forecast automl notebooks

.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau-validations.json

@@ -0,0 +1,10 @@
+{
+  "validations": [
+  {
+    "name": "check notebook output",
+    "params": {
+      "check": "warning"
+     }
+  }
+   ]
+}

.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow-validations.json

@@ -0,0 +1,21 @@
+{
+  "validations": [
+  {
+    "name": "check notebook output",
+    "params": {
+      "check": "warning stderr"
+     }
+  },
+  {
+    "name": "check v2 experiment result",
+    "params": {
+      "experiment_name": "dpv2-forecasting-experiment",
+      "minimum_median_score": "0.01",
+      "maximum_median_score": "0.3",
+      "metric_name": "normalized_root_mean_squared_error",
+      "absolute_minimum_score": "0.0",
+      "absolute_maximum_score": "1.0"
+    }
+  }
+  ]
+}

.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share-validations.json

@@ -0,0 +1,22 @@
+{
+  "validations": [
+  {
+    "name": "check notebook output",
+    "params": {
+      "check": "warning"
+     }
+  },
+  {
+    "name": "check v2 experiment result",
+    "params": {
+      "experiment_name": "dpv2-bike-test",
+      "minimum_median_score": "0.01",
+      "maximum_median_score": "0.3",
+      "metric_name": "normalized_root_mean_squared_error",
+      "absolute_minimum_score": "0.0",
+      "absolute_maximum_score": "1.0"
+    }
+  }
+  ]
+}
+

.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml

@@ -69,6 +69,13 @@ jobs:
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-github-dau.ipynb auto-ml-forecasting-github-dau.output.ipynb
       working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau
+    - name: check notebook output
+      run: |
+         python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \
+                --file_name auto-ml-forecasting-github-dau.output.ipynb \
+                --folder . \
+                --check warning \
+      working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v2

.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml

@@ -71,6 +71,25 @@ jobs:
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           papermill -k python -p compute_name automl-cpu-cluster automl-forecasting-orange-juice-sales-mlflow.ipynb automl-forecasting-orange-juice-sales-mlflow.output.ipynb
       working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales
+    - name: check notebook output
+      run: |
+         python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \
+                --file_name automl-forecasting-orange-juice-sales-mlflow.output.ipynb \
+                --folder . \
+                --check warning stderr \
+      working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales
+    - name: check v2 experiment result
+      run: |
+         python ${{ github.workspace }}/v1/scripts/validation/check_v2_experiment_result.py \
+                --file_name automl-forecasting-orange-juice-sales-mlflow.output.ipynb \
+                --folder . \
+                --experiment_name dpv2-forecasting-experiment \
+                --minimum_median_score 0.01 \
+                --maximum_median_score 0.3 \
+                --metric_name normalized_root_mean_squared_error \
+                --absolute_minimum_score 0.0 \
+                --absolute_maximum_score 1.0 \
+      working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v2

.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml

@@ -71,6 +71,25 @@ jobs:
           [ -f "../../.azureml/config" ] && cat "../../.azureml/config";
           papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-bike-share.ipynb auto-ml-forecasting-bike-share.output.ipynb
       working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share
+    - name: check notebook output
+      run: |
+         python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \
+                --file_name auto-ml-forecasting-bike-share.output.ipynb \
+                --folder . \
+                --check warning \
+      working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share
+    - name: check v2 experiment result
+      run: |
+         python ${{ github.workspace }}/v1/scripts/validation/check_v2_experiment_result.py \
+                --file_name auto-ml-forecasting-bike-share.output.ipynb \
+                --folder . \
+                --experiment_name dpv2-bike-test \
+                --minimum_median_score 0.01 \
+                --maximum_median_score 0.3 \
+                --metric_name normalized_root_mean_squared_error \
+                --absolute_minimum_score 0.0 \
+                --absolute_maximum_score 1.0 \
+      working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share
     - name: upload notebook's working folder as an artifact
       if: ${{ always() }}
       uses: actions/upload-artifact@v2

sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb

@@ -445,6 +445,7 @@
    "outputs": [],
    "source": [
     "from mlflow.tracking.client import MlflowClient\n",
+    "from mlflow.artifacts import download_artifacts\n",
     "\n",
     "# Initialize MLFlow client\n",
     "mlflow_client = MlflowClient()"
@@ -559,8 +560,8 @@
    "outputs": [],
    "source": [
     "# Download run's artifacts/outputs\n",
-    "local_path = mlflow_client.download_artifacts(\n",
-    "    best_run.info.run_id, \"outputs\", local_dir\n",
+    "local_path = download_artifacts(\n",
+    "    run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n",
     ")\n",
     "print(\"Artifacts downloaded in: {}\".format(local_path))\n",
     "print(\"Artifacts: {}\".format(os.listdir(local_path)))"

sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb

@@ -632,6 +632,7 @@
    "outputs": [],
    "source": [
     "from mlflow.tracking.client import MlflowClient\n",
+    "from mlflow.artifacts import download_artifacts\n",
     "\n",
     "# Initialize MLFlow client\n",
     "mlflow_client = MlflowClient()"
@@ -744,8 +745,8 @@
    "outputs": [],
    "source": [
     "# Download run's artifacts/outputs\n",
-    "local_path = mlflow_client.download_artifacts(\n",
-    "    best_run.info.run_id, \"outputs\", local_dir\n",
+    "local_path = download_artifacts(\n",
+    "    run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n",
     ")\n",
     "print(\"Artifacts downloaded in: {}\".format(local_path))\n",
     "print(\"Artifacts: {}\".format(os.listdir(local_path)))"

sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share.ipynb

@@ -500,6 +500,7 @@
    "outputs": [],
    "source": [
     "from mlflow.tracking.client import MlflowClient\n",
+    "from mlflow.artifacts import download_artifacts\n",
     "\n",
     "# Initialize MLFlow client\n",
     "mlflow_client = MlflowClient()"
@@ -613,8 +614,8 @@
    "outputs": [],
    "source": [
     "# Download run's artifacts/outputs\n",
-    "local_path = mlflow_client.download_artifacts(\n",
-    "    best_run.info.run_id, \"outputs\", local_dir\n",
+    "local_path = download_artifacts(\n",
+    "    run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n",
     ")\n",
     "print(\"Artifacts downloaded in: {}\".format(local_path))\n",
     "print(\"Artifacts: {}\".format(os.listdir(local_path)))"

v1/scripts/validation/check_notebook_output.py

@@ -50,6 +50,8 @@
     "Check: endpoint",
     "data_collector is not a known attribute of class",
     "Readonly attribute primary_metric will be ignored",
+    "Downloading artifact ",
+    "Warnings:",
 ]
 
 with open(full_name, "r") as notebook_file:

v1/scripts/validation/check_v2_experiment_result.py

@@ -103,7 +103,7 @@ def checkExperimentResult(
             for iteration in children:
                 iteration_status = iteration.info.status
                 print(iteration.info.run_id + ": " + iteration_status)
-                assert iteration_status == "FINISHED" or iteration_status == "CANCELED"
+                assert iteration_status in ["FINISHED", "CANCELED", "KILLED"]
                 if iteration_status == "FINISHED":
                     metrics = iteration.data.metrics
                     print(metric_name + " = " + str(metrics[metric_name]))