From c7dd3de5324df2adce94ea9c488d642fe53f5c24 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Wed, 29 Jan 2020 09:31:53 -0800 Subject: [PATCH 01/11] apply module approach to step 1 --- .pipelines/diabetes_regression-ci-build-train.yml | 3 ++- ml_service/pipelines/__init__.py | 0 .../pipelines/diabetes_regression_build_train_pipeline.py | 7 ++----- .../diabetes_regression_build_train_pipeline_with_r.py | 7 ++----- ...es_regression_build_train_pipeline_with_r_on_dbricks.py | 7 ++----- ml_service/pipelines/run_train_pipeline.py | 5 +---- ml_service/util/attach_compute.py | 2 +- 7 files changed, 10 insertions(+), 21 deletions(-) create mode 100644 ml_service/pipelines/__init__.py diff --git a/.pipelines/diabetes_regression-ci-build-train.yml b/.pipelines/diabetes_regression-ci-build-train.yml index b8889b87..292414b4 100644 --- a/.pipelines/diabetes_regression-ci-build-train.yml +++ b/.pipelines/diabetes_regression-ci-build-train.yml @@ -31,11 +31,12 @@ stages: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline - python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }} + python -m ml_service.pipelines.diabetes_regression_build_train_pipeline displayName: 'Publish Azure Machine Learning Pipeline' - stage: 'Trigger_AML_Pipeline' diff --git a/ml_service/pipelines/__init__.py b/ml_service/pipelines/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py index 0e963d96..1b1bc8e7 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py @@ -4,11 +4,8 @@ from azureml.core import Workspace from azureml.core.runconfig import RunConfiguration, CondaDependencies from azureml.core import Dataset, Datastore -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute -from env_variables import Env +from ml_service.util.attach_compute import get_compute +from ml_service.util.Env_variables import Env def main(): diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py index 509224e7..cb47cdf5 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py @@ -2,11 +2,8 @@ from azureml.pipeline.core import Pipeline from azureml.core import Workspace from azureml.core.runconfig import RunConfiguration, CondaDependencies -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute -from env_variables import Env +from ml_service.util.attach_compute import get_compute +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py index 7e435659..ae607b3b 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py @@ -1,11 +1,8 @@ from azureml.pipeline.core import Pipeline from azureml.core import Workspace -import os -import sys -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from attach_compute import get_compute +from ml_service.util.attach_compute import get_compute from azureml.pipeline.steps import DatabricksStep -from env_variables import Env +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/pipelines/run_train_pipeline.py b/ml_service/pipelines/run_train_pipeline.py index 50bf6e65..f5dba4fd 100644 --- a/ml_service/pipelines/run_train_pipeline.py +++ b/ml_service/pipelines/run_train_pipeline.py @@ -1,10 +1,7 @@ from azureml.pipeline.core import PublishedPipeline from azureml.core import Workspace -import os -import sys import argparse -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env def main(): diff --git a/ml_service/util/attach_compute.py b/ml_service/util/attach_compute.py index cc88be95..bcff58da 100644 --- a/ml_service/util/attach_compute.py +++ b/ml_service/util/attach_compute.py @@ -2,7 +2,7 @@ from azureml.core.compute import AmlCompute from azureml.core.compute import ComputeTarget from azureml.exceptions import ComputeTargetException -from env_variables import Env +from ml_service.util.env_variables import Env def get_compute( From 021e040d9b4fcfa87715910e90b25f9f71f8781f Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Wed, 29 Jan 2020 19:44:05 -0800 Subject: [PATCH 02/11] remove sys.path for create_image and smoke_test --- .pipelines/azdo-base-pipeline.yml | 2 +- .pipelines/diabetes_regression-ci-build-train.yml | 11 ++++++----- .pipelines/diabetes_regression-ci-image.yml | 3 ++- ml_service/util/__init__.py | 0 ml_service/util/create_scoring_image.py | 4 +--- ml_service/util/smoke_test_scoring_service.py | 5 +---- tests/unit/__init__.py | 0 tests/unit/code_test.py | 6 +----- 8 files changed, 12 insertions(+), 19 deletions(-) create mode 100644 ml_service/util/__init__.py create mode 100644 tests/unit/__init__.py diff --git a/.pipelines/azdo-base-pipeline.yml b/.pipelines/azdo-base-pipeline.yml index 926b404f..c45a7d23 100644 --- a/.pipelines/azdo-base-pipeline.yml +++ b/.pipelines/azdo-base-pipeline.yml @@ -10,7 +10,7 @@ steps: enabled: 'true' - script: | - pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit + python -m pytest --junitxml=$(Build.BinariesDirectory)/unit-testresults.xml $(Build.SourcesDirectory)/tests/unit displayName: 'Run unit tests' enabled: 'true' env: diff --git a/.pipelines/diabetes_regression-ci-build-train.yml b/.pipelines/diabetes_regression-ci-build-train.yml index 292414b4..b9e2d4d5 100644 --- a/.pipelines/diabetes_regression-ci-build-train.yml +++ b/.pipelines/diabetes_regression-ci-build-train.yml @@ -54,10 +54,11 @@ stages: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/run_train_pipeline.py --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution + python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution # Set AMLPIPELINEID variable for next AML Pipeline task in next job AMLPIPELINEID="$(cat pipeline_id.txt)" echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID" @@ -129,7 +130,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)" + python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)" - stage: 'Deploy_AKS' displayName: 'Deploy to AKS' @@ -165,7 +166,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)" + python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" - stage: 'Deploy_Webapp' displayName: 'Deploy to Webapp' @@ -188,7 +189,7 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt + python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt # Output image location to Azure DevOps job IMAGE_LOCATION="$(cat image_location.txt)" echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" @@ -207,4 +208,4 @@ stages: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" + python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" diff --git a/.pipelines/diabetes_regression-ci-image.yml b/.pipelines/diabetes_regression-ci-image.yml index 04c02253..0da7875e 100644 --- a/.pipelines/diabetes_regression-ci-image.yml +++ b/.pipelines/diabetes_regression-ci-image.yml @@ -28,9 +28,10 @@ steps: inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) inlineScript: | set -e export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python3 $(Build.SourcesDirectory)/ml_service/util/create_scoring_image.py + python3 -m ml_service.util.create_scoring_image displayName: 'Create Scoring Image' diff --git a/ml_service/util/__init__.py b/ml_service/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py index 4b3887fe..daa3878e 100644 --- a/ml_service/util/create_scoring_image.py +++ b/ml_service/util/create_scoring_image.py @@ -1,11 +1,9 @@ import os -import sys import argparse from azureml.core import Workspace from azureml.core.image import ContainerImage, Image from azureml.core.model import Model -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env e = Env() diff --git a/ml_service/util/smoke_test_scoring_service.py b/ml_service/util/smoke_test_scoring_service.py index 753ef23e..58d075aa 100644 --- a/ml_service/util/smoke_test_scoring_service.py +++ b/ml_service/util/smoke_test_scoring_service.py @@ -1,12 +1,9 @@ -import os -import sys import argparse import requests import time from azureml.core import Workspace from azureml.core.webservice import AksWebservice, AciWebservice -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env import secrets diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/code_test.py b/tests/unit/code_test.py index c7b10182..6b6d60ac 100644 --- a/tests/unit/code_test.py +++ b/tests/unit/code_test.py @@ -1,11 +1,7 @@ -import sys -import os import numpy as np from azureml.core.run import Run from unittest.mock import Mock -sys.path.append(os.path.abspath( - "./diabetes_regression/training")) # NOQA: E402 -from train import train_model +from diabetes_regression.training.train import train_model def test_train_model(): From fb4abeadf849b882d5a9f8de1be2ed87a9c0a215 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Wed, 29 Jan 2020 20:14:45 -0800 Subject: [PATCH 03/11] fix for evaluate model (in DevOps only) --- diabetes_regression/evaluate/evaluate_model.py | 3 +-- diabetes_regression/util/__init__.py | 0 2 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 diabetes_regression/util/__init__.py diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index cbac5bb9..2ae58637 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -60,8 +60,7 @@ ws = aml_workspace exp = Experiment(ws, experiment_name) else: - sys.path.append(os.path.abspath("./util")) # NOQA: E402 - from model_helper import get_model_by_tag + from util.model_helper import get_model_by_tag exp = run.experiment ws = run.experiment.workspace run_id = 'amlcompute' diff --git a/diabetes_regression/util/__init__.py b/diabetes_regression/util/__init__.py new file mode 100644 index 00000000..e69de29b From 44385161cd9eb49e5814ab4150f36b53c130a2f0 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 10:07:23 -0800 Subject: [PATCH 04/11] fix all other sys.path --- .../diabetes_regression-ci-build-train.yml | 8 ++-- ...regression-template-get-model-version.yml} | 2 +- .../evaluate/evaluate_model.py | 38 ++----------------- ...betes_regression_verify_train_pipeline.py} | 32 ++-------------- 4 files changed, 12 insertions(+), 68 deletions(-) rename .pipelines/{azdo-template-get-model-version.yml => diabetes_regression-template-get-model-version.yml} (75%) rename ml_service/pipelines/{verify_train_pipeline.py => diabetes_regression_verify_train_pipeline.py} (51%) diff --git a/.pipelines/diabetes_regression-ci-build-train.yml b/.pipelines/diabetes_regression-ci-build-train.yml index b9e2d4d5..0337044c 100644 --- a/.pipelines/diabetes_regression-ci-build-train.yml +++ b/.pipelines/diabetes_regression-ci-build-train.yml @@ -96,7 +96,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - stage: 'Deploy_ACI' displayName: 'Deploy to ACI' dependsOn: Trigger_AML_Pipeline @@ -109,7 +109,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -144,7 +144,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 displayName: 'Azure ML Model Deploy' inputs: @@ -180,7 +180,7 @@ stages: container: mcr.microsoft.com/mlops/python:latest timeoutInMinutes: 0 steps: - - template: azdo-template-get-model-version.yml + - template: diabetes_regression-template-get-model-version.yml - task: AzureCLI@1 displayName: 'Create scoring image and set IMAGE_LOCATION variable' inputs: diff --git a/.pipelines/azdo-template-get-model-version.yml b/.pipelines/diabetes_regression-template-get-model-version.yml similarity index 75% rename from .pipelines/azdo-template-get-model-version.yml rename to .pipelines/diabetes_regression-template-get-model-version.yml index f69f3366..8fc29b2c 100644 --- a/.pipelines/azdo-template-get-model-version.yml +++ b/.pipelines/diabetes_regression-template-get-model-version.yml @@ -6,7 +6,7 @@ steps: inlineScript: | set -e # fail on error export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" + python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(Build.BuildId) --output_model_version_file "model_version.txt" # Output model version to Azure DevOps job MODEL_VERSION="$(cat model_version.txt)" echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION" diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index 2ae58637..fe911f02 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -28,42 +28,12 @@ from azureml.core import Run, Workspace, Experiment import argparse import traceback +from util.model_helper import get_model_by_tag run = Run.get_context() -if (run.id.startswith('OfflineRun')): - from dotenv import load_dotenv - # For local development, set values in this section - load_dotenv() - sources_dir = os.environ.get("SOURCES_DIR_TRAIN") - if (sources_dir is None): - sources_dir = 'diabetes_regression' - path_to_util = os.path.join(".", sources_dir, "util") - sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 - from model_helper import get_model_by_tag - workspace_name = os.environ.get("WORKSPACE_NAME") - experiment_name = os.environ.get("EXPERIMENT_NAME") - resource_group = os.environ.get("RESOURCE_GROUP") - subscription_id = os.environ.get("SUBSCRIPTION_ID") - tenant_id = os.environ.get("TENANT_ID") - model_name = os.environ.get("MODEL_NAME") - app_id = os.environ.get('SP_APP_ID') - app_secret = os.environ.get('SP_APP_SECRET') - build_id = os.environ.get('BUILD_BUILDID') - # run_id useful to query previous runs - run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70" - - aml_workspace = Workspace.get( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group - ) - ws = aml_workspace - exp = Experiment(ws, experiment_name) -else: - from util.model_helper import get_model_by_tag - exp = run.experiment - ws = run.experiment.workspace - run_id = 'amlcompute' +exp = run.experiment +ws = run.experiment.workspace +run_id = 'amlcompute' parser = argparse.ArgumentParser("evaluate") parser.add_argument( diff --git a/ml_service/pipelines/verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py similarity index 51% rename from ml_service/pipelines/verify_train_pipeline.py rename to ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index 2824accb..22f36a02 100644 --- a/ml_service/pipelines/verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -2,40 +2,14 @@ import sys import argparse from azureml.core import Run, Experiment, Workspace -sys.path.append(os.path.abspath("./ml_service/util")) # NOQA: E402 -from env_variables import Env +from ml_service.util.env_variables import Env +from diabetes_regression.util.model_helper import get_model_by_tag def main(): run = Run.get_context() - if (run.id.startswith('OfflineRun')): - from dotenv import load_dotenv - # For local development, set values in this section - load_dotenv() - sources_dir = os.environ.get("SOURCES_DIR_TRAIN") - if (sources_dir is None): - sources_dir = 'diabetes_regression' - path_to_util = os.path.join(".", sources_dir, "util") - sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 - from model_helper import get_model_by_tag - workspace_name = os.environ.get("WORKSPACE_NAME") - experiment_name = os.environ.get("EXPERIMENT_NAME") - resource_group = os.environ.get("RESOURCE_GROUP") - subscription_id = os.environ.get("SUBSCRIPTION_ID") - build_id = os.environ.get('BUILD_BUILDID') - aml_workspace = Workspace.get( - name=workspace_name, - subscription_id=subscription_id, - resource_group=resource_group - ) - ws = aml_workspace - exp = Experiment(ws, experiment_name) - else: - sys.path.append(os.path.abspath("./util")) # NOQA: E402 - from model_helper import get_model_by_tag - ws = run.experiment.workspace - exp = run.experiment + exp = run.experiment e = Env() From ae82b99607ae3401f85fa9a8650c3f24560f9035 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 10:18:29 -0800 Subject: [PATCH 05/11] abtest fix --- .pipelines/azdo-abtest-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/azdo-abtest-pipeline.yml b/.pipelines/azdo-abtest-pipeline.yml index a74cf330..9d2415a6 100644 --- a/.pipelines/azdo-abtest-pipeline.yml +++ b/.pipelines/azdo-abtest-pipeline.yml @@ -40,7 +40,7 @@ stages: inlineScript: | set -e export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt + python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt displayName: 'Create Scoring Image' name: 'buildscoringimage' From f4da7563298d12fdb5dcc24780a67fcbe725e0f2 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 11:19:18 -0800 Subject: [PATCH 06/11] OfflineRun in comments for smoke --- ...abetes_regression_verify_train_pipeline.py | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index 22f36a02..da8206a9 100644 --- a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -1,5 +1,3 @@ -import os -import sys import argparse from azureml.core import Run, Experiment, Workspace from ml_service.util.env_variables import Env @@ -9,6 +7,36 @@ def main(): run = Run.get_context() + +# if you would like to run this script on a local computer +# the following code is a good starting point for you +# use +# python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline + + # if (run.id.startswith('OfflineRun')): + # from dotenv import load_dotenv + # # For local development, set values in this section + # load_dotenv() + # sources_dir = os.environ.get("SOURCES_DIR_TRAIN") + # if (sources_dir is None): + # sources_dir = 'diabetes_regression' + # path_to_util = os.path.join(".", sources_dir, "util") + # sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 + # from model_helper import get_model_by_tag + # workspace_name = os.environ.get("WORKSPACE_NAME") + # experiment_name = os.environ.get("EXPERIMENT_NAME") + # resource_group = os.environ.get("RESOURCE_GROUP") + # subscription_id = os.environ.get("SUBSCRIPTION_ID") + # build_id = os.environ.get('BUILD_BUILDID') + # aml_workspace = Workspace.get( + # name=workspace_name, + # subscription_id=subscription_id, + # resource_group=resource_group + # ) + # ws = aml_workspace + # exp = Experiment(ws, experiment_name) + +# comment this line if you are running offline exp = run.experiment e = Env() From 4313103c2b31a7b379b156a4098502daffdf3f70 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 11:25:51 -0800 Subject: [PATCH 07/11] add offline mode to evaluate --- .../evaluate/evaluate_model.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index fe911f02..b899e7aa 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -24,13 +24,51 @@ POSSIBILITY OF SUCH DAMAGE. """ import os -import sys from azureml.core import Run, Workspace, Experiment import argparse import traceback from util.model_helper import get_model_by_tag run = Run.get_context() + +# if you would like to run this script on a local computer +# the following code is a good starting point for you +# use +# python -m evaluate.evaluate_model +# in diabetes_regression folder context + +# if (run.id.startswith('OfflineRun')): +# from dotenv import load_dotenv +# # For local development, set values in this section +# load_dotenv() +# sources_dir = os.environ.get("SOURCES_DIR_TRAIN") +# if (sources_dir is None): +# sources_dir = 'diabetes_regression' +# path_to_util = os.path.join(".", sources_dir, "util") +# sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 +# from model_helper import get_model_by_tag +# workspace_name = os.environ.get("WORKSPACE_NAME") +# experiment_name = os.environ.get("EXPERIMENT_NAME") +# resource_group = os.environ.get("RESOURCE_GROUP") +# subscription_id = os.environ.get("SUBSCRIPTION_ID") +# tenant_id = os.environ.get("TENANT_ID") +# model_name = os.environ.get("MODEL_NAME") +# app_id = os.environ.get('SP_APP_ID') +# app_secret = os.environ.get('SP_APP_SECRET') +# build_id = os.environ.get('BUILD_BUILDID') +# # run_id useful to query previous runs +# run_id = "57fee47f-5ae8-441c-bc0c-d4c371f32d70" + +# aml_workspace = Workspace.get( +# name=workspace_name, +# subscription_id=subscription_id, +# resource_group=resource_group +# ) +# ws = aml_workspace +# exp = Experiment(ws, experiment_name) + +# comment the following three lines +# if you would like to use Offline mode exp = run.experiment ws = run.experiment.workspace run_id = 'amlcompute' From 7a34927126c1a002565dbe6888c360b1b49a4603 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 11:29:06 -0800 Subject: [PATCH 08/11] docs --- docs/code_description.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/code_description.md b/docs/code_description.md index 5a1af307..5f323ad6 100644 --- a/docs/code_description.md +++ b/docs/code_description.md @@ -24,7 +24,7 @@ - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py` : builds and publishes an ML training pipeline. It uses R on ML Compute. - `ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py` : builds and publishes an ML training pipeline. It uses R on Databricks Compute. - `ml_service/pipelines/run_train_pipeline.py` : invokes a published ML training pipeline (Python on ML Compute) via REST API. -- `ml_service/pipelines/verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model. +- `ml_service/pipelines/diabetes_regression_verify_train_pipeline.py` : determines whether the evaluate_model.py step of the training pipeline registered a new model. - `ml_service/util` : contains common utility functions used to build and publish an ML training pipeline. ### Code From 4cd9014a46f89e2f4e481a73777094988954ec45 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 11:50:38 -0800 Subject: [PATCH 09/11] linting --- diabetes_regression/evaluate/evaluate_model.py | 6 +++--- .../pipelines/diabetes_regression_verify_train_pipeline.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index b899e7aa..ed234b7a 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -24,7 +24,7 @@ POSSIBILITY OF SUCH DAMAGE. """ import os -from azureml.core import Run, Workspace, Experiment +from azureml.core import Run import argparse import traceback from util.model_helper import get_model_by_tag @@ -33,7 +33,7 @@ # if you would like to run this script on a local computer # the following code is a good starting point for you -# use +# use # python -m evaluate.evaluate_model # in diabetes_regression folder context @@ -67,7 +67,7 @@ # ws = aml_workspace # exp = Experiment(ws, experiment_name) -# comment the following three lines +# comment the following three lines # if you would like to use Offline mode exp = run.experiment ws = run.experiment.workspace diff --git a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index da8206a9..2da0876d 100644 --- a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -1,5 +1,6 @@ import argparse -from azureml.core import Run, Experiment, Workspace +import sys +from azureml.core import Run from ml_service.util.env_variables import Env from diabetes_regression.util.model_helper import get_model_by_tag @@ -10,7 +11,7 @@ def main(): # if you would like to run this script on a local computer # the following code is a good starting point for you -# use +# use # python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline # if (run.id.startswith('OfflineRun')): From 05dd3c482071bfb5ea4600e7c61954992439fe27 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 11:58:43 -0800 Subject: [PATCH 10/11] typo --- .../pipelines/diabetes_regression_build_train_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py index 1b1bc8e7..66913420 100644 --- a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py @@ -5,7 +5,7 @@ from azureml.core.runconfig import RunConfiguration, CondaDependencies from azureml.core import Dataset, Datastore from ml_service.util.attach_compute import get_compute -from ml_service.util.Env_variables import Env +from ml_service.util.env_variables import Env def main(): From 599be38295cb6571a135bc2d22b022170a69ad84 Mon Sep 17 00:00:00 2001 From: Sergii Baidachnyi Date: Thu, 30 Jan 2020 12:20:31 -0800 Subject: [PATCH 11/11] return offline Run back for verify train because it was designed not for development as it was commented. out pipelines need it --- ...abetes_regression_verify_train_pipeline.py | 53 ++++++++----------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index 2da0876d..f0a4c965 100644 --- a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -1,6 +1,7 @@ import argparse import sys -from azureml.core import Run +import os +from azureml.core import Run, Experiment, Workspace from ml_service.util.env_variables import Env from diabetes_regression.util.model_helper import get_model_by_tag @@ -9,36 +10,26 @@ def main(): run = Run.get_context() -# if you would like to run this script on a local computer -# the following code is a good starting point for you -# use -# python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline - - # if (run.id.startswith('OfflineRun')): - # from dotenv import load_dotenv - # # For local development, set values in this section - # load_dotenv() - # sources_dir = os.environ.get("SOURCES_DIR_TRAIN") - # if (sources_dir is None): - # sources_dir = 'diabetes_regression' - # path_to_util = os.path.join(".", sources_dir, "util") - # sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 - # from model_helper import get_model_by_tag - # workspace_name = os.environ.get("WORKSPACE_NAME") - # experiment_name = os.environ.get("EXPERIMENT_NAME") - # resource_group = os.environ.get("RESOURCE_GROUP") - # subscription_id = os.environ.get("SUBSCRIPTION_ID") - # build_id = os.environ.get('BUILD_BUILDID') - # aml_workspace = Workspace.get( - # name=workspace_name, - # subscription_id=subscription_id, - # resource_group=resource_group - # ) - # ws = aml_workspace - # exp = Experiment(ws, experiment_name) - -# comment this line if you are running offline - exp = run.experiment + if (run.id.startswith('OfflineRun')): + from dotenv import load_dotenv + load_dotenv() + sources_dir = os.environ.get("SOURCES_DIR_TRAIN") + if (sources_dir is None): + sources_dir = 'diabetes_regression' + workspace_name = os.environ.get("WORKSPACE_NAME") + experiment_name = os.environ.get("EXPERIMENT_NAME") + resource_group = os.environ.get("RESOURCE_GROUP") + subscription_id = os.environ.get("SUBSCRIPTION_ID") + build_id = os.environ.get('BUILD_BUILDID') + aml_workspace = Workspace.get( + name=workspace_name, + subscription_id=subscription_id, + resource_group=resource_group + ) + ws = aml_workspace + exp = Experiment(ws, experiment_name) + else: + exp = run.experiment e = Env()