diff --git a/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau-validations.json b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau-validations.json new file mode 100644 index 0000000000..deafc70ab6 --- /dev/null +++ b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau-validations.json @@ -0,0 +1,10 @@ +{ + "validations": [ + { + "name": "check notebook output", + "params": { + "check": "warning" + } + } + ] +} diff --git a/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow-validations.json b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow-validations.json new file mode 100644 index 0000000000..c3f4953190 --- /dev/null +++ b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow-validations.json @@ -0,0 +1,21 @@ +{ + "validations": [ + { + "name": "check notebook output", + "params": { + "check": "warning stderr" + } + }, + { + "name": "check v2 experiment result", + "params": { + "experiment_name": "dpv2-forecasting-experiment", + "minimum_median_score": "0.01", + "maximum_median_score": "0.3", + "metric_name": "normalized_root_mean_squared_error", + "absolute_minimum_score": "0.0", + "absolute_maximum_score": "1.0" + } + } + ] +} diff --git a/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share-validations.json b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share-validations.json new file mode 100644 index 0000000000..f9840fab37 --- /dev/null +++ b/.github/validate/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share-validations.json @@ -0,0 +1,22 @@ +{ + "validations": [ + { + "name": "check notebook output", + "params": { + "check": "warning" + } + }, + { + "name": "check v2 experiment result", + "params": { + "experiment_name": "dpv2-bike-test", + "minimum_median_score": "0.01", + "maximum_median_score": "0.3", + "metric_name": "normalized_root_mean_squared_error", + "absolute_minimum_score": "0.0", + "absolute_maximum_score": "1.0" + } + } + ] +} + \ No newline at end of file diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml index 749225336c..5347e1e4bf 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-github-dau-auto-ml-forecasting-github-dau.yml @@ -69,6 +69,13 @@ jobs: [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-github-dau.ipynb auto-ml-forecasting-github-dau.output.ipynb working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau + - name: check notebook output + run: | + python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \ + --file_name auto-ml-forecasting-github-dau.output.ipynb \ + --folder . \ + --check warning \ + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau - name: upload notebook's working folder as an artifact if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml index 6618e9795c..3e83a7acd8 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-orange-juice-sales-automl-forecasting-orange-juice-sales-mlflow.yml @@ -71,6 +71,25 @@ jobs: [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; papermill -k python -p compute_name automl-cpu-cluster automl-forecasting-orange-juice-sales-mlflow.ipynb automl-forecasting-orange-juice-sales-mlflow.output.ipynb working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales + - name: check notebook output + run: | + python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \ + --file_name automl-forecasting-orange-juice-sales-mlflow.output.ipynb \ + --folder . \ + --check warning stderr \ + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales + - name: check v2 experiment result + run: | + python ${{ github.workspace }}/v1/scripts/validation/check_v2_experiment_result.py \ + --file_name automl-forecasting-orange-juice-sales-mlflow.output.ipynb \ + --folder . \ + --experiment_name dpv2-forecasting-experiment \ + --minimum_median_score 0.01 \ + --maximum_median_score 0.3 \ + --metric_name normalized_root_mean_squared_error \ + --absolute_minimum_score 0.0 \ + --absolute_maximum_score 1.0 \ + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales - name: upload notebook's working folder as an artifact if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml index 31782e0e61..7a7290615d 100644 --- a/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml +++ b/.github/workflows/sdk-jobs-automl-standalone-jobs-automl-forecasting-task-bike-share-auto-ml-forecasting-bike-share.yml @@ -71,6 +71,25 @@ jobs: [ -f "../../.azureml/config" ] && cat "../../.azureml/config"; papermill -k python -p compute_name automl-cpu-cluster auto-ml-forecasting-bike-share.ipynb auto-ml-forecasting-bike-share.output.ipynb working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share + - name: check notebook output + run: | + python ${{ github.workspace }}/v1/scripts/validation/check_notebook_output.py \ + --file_name auto-ml-forecasting-bike-share.output.ipynb \ + --folder . \ + --check warning \ + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share + - name: check v2 experiment result + run: | + python ${{ github.workspace }}/v1/scripts/validation/check_v2_experiment_result.py \ + --file_name auto-ml-forecasting-bike-share.output.ipynb \ + --folder . \ + --experiment_name dpv2-bike-test \ + --minimum_median_score 0.01 \ + --maximum_median_score 0.3 \ + --metric_name normalized_root_mean_squared_error \ + --absolute_minimum_score 0.0 \ + --absolute_maximum_score 1.0 \ + working-directory: sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share - name: upload notebook's working folder as an artifact if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb index 694bae7172..2b60c4052d 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-github-dau/auto-ml-forecasting-github-dau.ipynb @@ -445,6 +445,7 @@ "outputs": [], "source": [ "from mlflow.tracking.client import MlflowClient\n", + "from mlflow.artifacts import download_artifacts\n", "\n", "# Initialize MLFlow client\n", "mlflow_client = MlflowClient()" @@ -559,8 +560,8 @@ "outputs": [], "source": [ "# Download run's artifacts/outputs\n", - "local_path = mlflow_client.download_artifacts(\n", - " best_run.info.run_id, \"outputs\", local_dir\n", + "local_path = download_artifacts(\n", + " run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n", ")\n", "print(\"Artifacts downloaded in: {}\".format(local_path))\n", "print(\"Artifacts: {}\".format(os.listdir(local_path)))" diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb index 55ae29f31c..12ba59d13b 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-orange-juice-sales/automl-forecasting-orange-juice-sales-mlflow.ipynb @@ -632,6 +632,7 @@ "outputs": [], "source": [ "from mlflow.tracking.client import MlflowClient\n", + "from mlflow.artifacts import download_artifacts\n", "\n", "# Initialize MLFlow client\n", "mlflow_client = MlflowClient()" @@ -744,8 +745,8 @@ "outputs": [], "source": [ "# Download run's artifacts/outputs\n", - "local_path = mlflow_client.download_artifacts(\n", - " best_run.info.run_id, \"outputs\", local_dir\n", + "local_path = download_artifacts(\n", + " run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n", ")\n", "print(\"Artifacts downloaded in: {}\".format(local_path))\n", "print(\"Artifacts: {}\".format(os.listdir(local_path)))" diff --git a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share.ipynb b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share.ipynb index bdda7137c6..94d4862988 100644 --- a/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share.ipynb +++ b/sdk/python/jobs/automl-standalone-jobs/automl-forecasting-task-bike-share/auto-ml-forecasting-bike-share.ipynb @@ -500,6 +500,7 @@ "outputs": [], "source": [ "from mlflow.tracking.client import MlflowClient\n", + "from mlflow.artifacts import download_artifacts\n", "\n", "# Initialize MLFlow client\n", "mlflow_client = MlflowClient()" @@ -613,8 +614,8 @@ "outputs": [], "source": [ "# Download run's artifacts/outputs\n", - "local_path = mlflow_client.download_artifacts(\n", - " best_run.info.run_id, \"outputs\", local_dir\n", + "local_path = download_artifacts(\n", + " run_id=best_run.info.run_id, artifact_path=\"outputs\", dst_path=local_dir\n", ")\n", "print(\"Artifacts downloaded in: {}\".format(local_path))\n", "print(\"Artifacts: {}\".format(os.listdir(local_path)))" diff --git a/v1/scripts/validation/check_notebook_output.py b/v1/scripts/validation/check_notebook_output.py index eba8d1c05b..e0e4c9514b 100644 --- a/v1/scripts/validation/check_notebook_output.py +++ b/v1/scripts/validation/check_notebook_output.py @@ -50,6 +50,8 @@ "Check: endpoint", "data_collector is not a known attribute of class", "Readonly attribute primary_metric will be ignored", + "Downloading artifact ", + "Warnings:", ] with open(full_name, "r") as notebook_file: diff --git a/v1/scripts/validation/check_v2_experiment_result.py b/v1/scripts/validation/check_v2_experiment_result.py index 6283ddc738..9f5c490040 100644 --- a/v1/scripts/validation/check_v2_experiment_result.py +++ b/v1/scripts/validation/check_v2_experiment_result.py @@ -103,7 +103,7 @@ def checkExperimentResult( for iteration in children: iteration_status = iteration.info.status print(iteration.info.run_id + ": " + iteration_status) - assert iteration_status == "FINISHED" or iteration_status == "CANCELED" + assert iteration_status in ["FINISHED", "CANCELED", "KILLED"] if iteration_status == "FINISHED": metrics = iteration.data.metrics print(metric_name + " = " + str(metrics[metric_name]))