diff --git a/.pipelines/azdo-abtest-pipeline.yml b/.pipelines/azdo-abtest-pipeline.yml index a74cf330..9d2415a6 100644 --- a/.pipelines/azdo-abtest-pipeline.yml +++ b/.pipelines/azdo-abtest-pipeline.yml @@ -40,7 +40,7 @@ stages: inlineScript: | set -e export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt + python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt displayName: 'Create Scoring Image' name: 'buildscoringimage' diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml index 926b404f..c45a7d23 100644 --- a/.pipelines/azdo-base-pipeline.yml +++ b/.pipelines/azdo-base-pipeline.yml @@ -10,7 +10,7 @@ steps: enabled: 'true' - script: | - pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit + python -m pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit displayName: 'Run unit tests' enabled: 'true' env: diff --git a/.pipelines/diabetes_regression-ci-build-train.yml b/.pipelines/diabetes_regression-ci-build-train.yml index 45ecc244..bcc1249c 100644 --- a/.pipelines/diabetes_regression-ci-build-train.yml +++ b/.pipelines/diabetes_regression-ci-build-train.yml @@ -31,11 +31,12 @@ stages: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline - python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }} + python -m ml_service.pipelines.diabetes_regression_build_train_pipeline displayName: 'Publish Azure Machine Learning Pipeline' - stage: 'Trigger_AML_Pipeline' @@ -53,10 +54,11 @@ stages: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution + python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution # Set AMLPIPELINEID variable for next AML Pipeline task in next job AMLPIPELINEID="$(cat pipeline_id.txt)" echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID" @@ -95,7 +97,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - stage: 'Deploy_ACI' displayName: 'Deploy to ACI' dependsOn: Trigger_AML_Pipeline @@ -108,7 +110,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -129,7 +131,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)" + python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)" - stage: 'Deploy_AKS' displayName: 'Deploy to AKS' @@ -143,7 +145,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -165,7 +167,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)" + python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" - stage: 'Deploy_Webapp' displayName: 'Deploy to Webapp' @@ -179,7 +181,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: AzureCLI@1 displayName: 'Create scoring image and set IMAGE_LOCATION variable' inputs: @@ -188,7 +190,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt + python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt # Output image location to Azure DevOps job IMAGE_LOCATION="$(cat image_location.txt)" echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" @@ -207,4 +209,4 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" + python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" diff --git a/.pipelines/diabetes_regression-ci-image.yml b/.pipelines/diabetes_regression-ci-image.yml index 04c02253..0da7875e 100644 --- a/.pipelines/diabetes_regression-ci-image.yml +++ b/.pipelines/diabetes_regression-ci-image.yml @@ -28,9 +28,10 @@ steps: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py + python3 -m ml_service.util.create_scoring_image displayName: 'Create Scoring Image' diff --git a/.pipelines/azdo-template-get-model-version.yml b/.pipelines/diabetes_regression-template-get-model-version.yml similarity index 75% rename from .pipelines/azdo-template-get-model-version.yml rename to .pipelines/diabetes_regression-template-get-model-version.yml index f69f3366..8fc29b2c 100644 --- a/.pipelines/azdo-template-get-model-version.yml +++ b/.pipelines/diabetes_regression-template-get-model-version.yml @@ -6,7 +6,7 @@ steps: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" + python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" # Output model version to Azure DevOps job MODEL_VERSION="$(cat model_version.txt)" echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION" diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index cbac5bb9..ed234b7a 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -24,47 +24,54 @@ POSSIBILITY OF SUCH DAMAGE. """ import os -import sys -from azureml.core import Run, Workspace, Experiment +from azureml.core import Run import argparse import traceback +from util.model_helper import get_model_by_tag run = Run.get_context() -if (run.id.startswith('OfflineRun')): - from dotenv import load_dotenv - # For local development, set values in this section - load_dotenv() - sources_dir = os.environ.get("SOURCES_DIR_TRAIN") - if (sources_dir is None): - sources_dir = 'diabetes_regression' - path_to_util = os.path.join(".", sources_dir, "util") - sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 - from model_helper import get_model_by_tag - workspace_name = os.environ.get("WORKSPACE_NAME") - experiment_name = os.environ.get("EXPERIMENT_NAME") - resource_group = os.environ.get("RESOURCE_GROUP") - subscription_id = os.environ.get("SUBSCRIPTION_ID") - tenant_id = os.environ.get("TENANT_ID") - model_name = os.environ.get("MODEL_NAME") - app_id = os.environ.get('SP_APP_ID') - app_secret = os.environ.get('SP_APP_SECRET') - build_id = os.environ.get('BUILD_BUILDID') - # run_id useful to query previous runs - run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70" - aml_workspace = Workspace.get( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group - ) - ws = aml_workspace - exp = Experiment(ws, experiment_name) -else: - sys.path.append(os.path.abspath("./util")) # NOQA: E402 - from model_helper import get_model_by_tag - exp = run.experiment - ws = run.experiment.workspace - run_id = 'amlcompute' +# if you would like to run this script on a local computer +# the following code is a good starting point for you +# use +# python -m evaluate.evaluate_model +# in diabetes_regression folder context + +# if (run.id.startswith('OfflineRun')): +# from dotenv import load_dotenv +# # For local development, set values in this section +# load_dotenv() +# sources_dir = os.environ.get("SOURCES_DIR_TRAIN") +# if (sources_dir is None): +# sources_dir = 'diabetes_regression' +# path_to_util = os.path.join(".", sources_dir, "util") +# sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 +# from model_helper import get_model_by_tag +# workspace_name = os.environ.get("WORKSPACE_NAME") +# experiment_name = os.environ.get("EXPERIMENT_NAME") +# resource_group = os.environ.get("RESOURCE_GROUP") +# subscription_id = os.environ.get("SUBSCRIPTION_ID") +# tenant_id = os.environ.get("TENANT_ID") +# model_name = os.environ.get("MODEL_NAME") +# app_id = os.environ.get('SP_APP_ID') +# app_secret = os.environ.get('SP_APP_SECRET') +# build_id = os.environ.get('BUILD_BUILDID') +# # run_id useful to query previous runs +# run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70" + +# aml_workspace = Workspace.get( +# name=workspace_name, +# subscription_id=subscription_id, +# resource_group=resource_group +# ) +# ws = aml_workspace +# exp = Experiment(ws, experiment_name) + +# comment the following three lines +# if you would like to use Offline mode +exp = run.experiment +ws = run.experiment.workspace +run_id = 'amlcompute' parser = argparse.ArgumentParser("evaluate") parser.add_argument( diff --git a/diabetes_regression/util/__init__.py b/diabetes_regression/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/code_description.md b/docs/code_description.md index 5a1af307..5f323ad6 100644 --- a/docs/code_description.md +++ b/docs/code_description.md @@ -24,7 +24,7 @@ - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py` : builds and publishes an ML training pipeline. It uses R on ML Compute. - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` : builds and publishes an ML training pipeline. It uses R on Databricks Compute. - `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline (Python on ML Compute) via REST API. -- `ml_service/pipelines/verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model. +- `ml_service/pipelines/diabetes_regression_verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model. - `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline. ### Code diff --git a/ml_service/pipelines/__init__.py b/ml_service/pipelines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py index 0e963d96..66913420 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py @@ -4,11 +4,8 @@ from azureml.core import Workspace from azureml.core.runconfig import RunConfiguration, CondaDependencies from azureml.core import Dataset, Datastore -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute -from env_variables import Env +from ml_service.util.attach_compute import get_compute +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py index 509224e7..cb47cdf5 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py @@ -2,11 +2,8 @@ from azureml.pipeline.core import Pipeline from azureml.core import Workspace from azureml.core.runconfig import RunConfiguration, CondaDependencies -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute -from env_variables import Env +from ml_service.util.attach_compute import get_compute +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py index 7e435659..ae607b3b 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py @@ -1,11 +1,8 @@ from azureml.pipeline.core import Pipeline from azureml.core import Workspace -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute +from ml_service.util.attach_compute import get_compute from azureml.pipeline.steps import DatabricksStep -from env_variables import Env +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/pipelines/verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py similarity index 81% rename from ml_service/pipelines/verify_train_pipeline.py rename to ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index 2824accb..f0a4c965 100644 --- a/ml_service/pipelines/verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -1,24 +1,21 @@ -import os -import sys import argparse +import sys +import os from azureml.core import Run, Experiment, Workspace -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env +from diabetes_regression.util.model_helper import get_model_by_tag def main(): run = Run.get_context() + if (run.id.startswith('OfflineRun')): from dotenv import load_dotenv - # For local development, set values in this section load_dotenv() sources_dir = os.environ.get("SOURCES_DIR_TRAIN") if (sources_dir is None): sources_dir = 'diabetes_regression' - path_to_util = os.path.join(".", sources_dir, "util") - sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 - from model_helper import get_model_by_tag workspace_name = os.environ.get("WORKSPACE_NAME") experiment_name = os.environ.get("EXPERIMENT_NAME") resource_group = os.environ.get("RESOURCE_GROUP") @@ -32,9 +29,6 @@ def main(): ws = aml_workspace exp = Experiment(ws, experiment_name) else: - sys.path.append(os.path.abspath("./util")) # NOQA: E402 - from model_helper import get_model_by_tag - ws = run.experiment.workspace exp = run.experiment e = Env() diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 50bf6e65..f5dba4fd 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -1,10 +1,7 @@ from azureml.pipeline.core import PublishedPipeline from azureml.core import Workspace -import os -import sys import argparse -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/util/__init__.py b/ml_service/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ml_service/util/attach_compute.py b/ml_service/util/attach_compute.py index cc88be95..bcff58da 100644 --- a/ml_service/util/attach_compute.py +++ b/ml_service/util/attach_compute.py @@ -2,7 +2,7 @@ from azureml.core.compute import AmlCompute from azureml.core.compute import ComputeTarget from azureml.exceptions import ComputeTargetException -from env_variables import Env +from ml_service.util.env_variables import Env def get_compute( diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py index 4b3887fe..daa3878e 100644 --- a/ml_service/util/create_scoring_image.py +++ b/ml_service/util/create_scoring_image.py @@ -1,11 +1,9 @@ import os -import sys import argparse from azureml.core import Workspace from azureml.core.image import ContainerImage, Image from azureml.core.model import Model -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env e = Env() diff --git a/ml_service/util/smoke_test_scoring_service.py b/ml_service/util/smoke_test_scoring_service.py index 753ef23e..58d075aa 100644 --- a/ml_service/util/smoke_test_scoring_service.py +++ b/ml_service/util/smoke_test_scoring_service.py @@ -1,12 +1,9 @@ -import os -import sys import argparse import requests import time from azureml.core import Workspace from azureml.core.webservice import AksWebservice, AciWebservice -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env import secrets diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index c7b10182..6b6d60ac 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -1,11 +1,7 @@ -import sys -import os import numpy as np from azureml.core.run import Run from unittest.mock import Mock -sys.path.append(os.path.abspath( - "./diabetes_regression/training")) # NOQA: E402 -from train import train_model +from diabetes_regression.training.train import train_model def test_train_model():