diff --git a/.env.example b/.env.example index 3c63d969..c6f0e317 100644 --- a/.env.example +++ b/.env.example @@ -47,10 +47,10 @@ DB_CLUSTER_ID = '' IMAGE_NAME = 'mltrained' # Run Evaluation Step in AML pipeline -RUN_EVALUATION = 'true' +# RUN_EVALUATION = 'true' # Set to true cancels the Azure ML pipeline run when evaluation criteria are not met. -ALLOW_RUN_CANCEL = 'true' +# ALLOW_RUN_CANCEL = 'false' # Flag to allow rebuilding the AML Environment after it was built for the first time. This enables dependency updates from conda_dependencies.yaml. AML_REBUILD_ENVIRONMENT = 'false' diff --git a/.gitignore b/.gitignore index 3ab04e2f..7aab9c01 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,4 @@ condaenv.* .mypy_cache/ .DS_Store +*.csv diff --git a/.pipelines/diabetes_regression-cicd.yml b/.pipelines/diabetes_regression-cicd.yml new file mode 100644 index 00000000..4ad1e1c4 --- /dev/null +++ b/.pipelines/diabetes_regression-cicd.yml @@ -0,0 +1,244 @@ +# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model. + +resources: + containers: + - container: mlops + image: mcr.microsoft.com/mlops/python:latest + +pr: none +trigger: + branches: + include: + - master + paths: + include: + - diabetes_regression/ + - ml_service/pipelines/diabetes_regression_build_train_pipeline.py + - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py + - ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py + +variables: +- template: diabetes_regression-variables-template.yml +- group: devopsforai-aml-vg + + +# Runtime parameters to select artifacts +parameters: +- name : artifactBuildId + displayName: Model Train CI Build ID. Default is 'latest'. + type: string + default: latest + +pool: + vmImage: ubuntu-latest + +stages: +- stage: 'Model_CI' + displayName: 'Model CI' + jobs: + - job: "Model_CI_Pipeline" + displayName: "Model CI Pipeline" + container: mlops + timeoutInMinutes: 0 + steps: + - template: code-quality-template.yml + - task: AzureCLI@1 + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + # Invoke the Python building and publishing a training pipeline + python -m ml_service.pipelines.diabetes_regression_build_train_pipeline + displayName: 'Publish Azure Machine Learning Pipeline' + +- stage: 'Trigger_AML_Pipeline' + displayName: 'Train and evaluate model' + condition: succeeded() + variables: + BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)' + jobs: + - job: "Get_Pipeline_ID" + condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true')) + displayName: "Get Pipeline ID for execution" + container: mlops + timeoutInMinutes: 0 + steps: + - task: AzureCLI@1 + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.pipelines.run_train_pipeline --output_pipeline_id_file "pipeline_id.txt" --skip_train_execution + # Set AMLPIPELINEID variable for next AML Pipeline task in next job + AMLPIPELINEID="$(cat pipeline_id.txt)" + echo "##vso[task.setvariable variable=AMLPIPELINEID;isOutput=true]$AMLPIPELINEID" + name: 'getpipelineid' + displayName: 'Get Pipeline ID' + - job: "Run_ML_Pipeline" + dependsOn: "Get_Pipeline_ID" + displayName: "Trigger ML Training Pipeline" + timeoutInMinutes: 0 + pool: server + variables: + AMLPIPELINE_ID: $[ dependencies.Get_Pipeline_ID.outputs['getpipelineid.AMLPIPELINEID'] ] + steps: + - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 + displayName: 'Invoke ML pipeline' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + PipelineId: '$(AMLPIPELINE_ID)' + ExperimentName: '$(EXPERIMENT_NAME)' + PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)"}, "tags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}, "StepTags": {"BuildId": "$(Build.BuildId)", "BuildUri": "$(BUILD_URI)"}' + - job: "Training_Run_Report" + dependsOn: "Run_ML_Pipeline" + condition: always() + displayName: "Publish artifact if new model was registered" + container: mlops + timeoutInMinutes: 0 + steps: + - template: diabetes_regression-publish-model-artifact-template.yml + +- stage: 'Deploy_ACI' + displayName: 'Deploy to ACI' + condition: variables['ACI_DEPLOYMENT_NAME'] + jobs: + - job: "Deploy_ACI" + displayName: "Deploy to ACI" + container: mlops + timeoutInMinutes: 0 + steps: + - download: current + - script: echo $(System.TeamProjectId) # this step uses the mapped-in variable + - script: echo $(Build.BuildId) # this step uses the mapped-in variable + # - script: echo $(parameters.artifactBuildId) # this step uses the mapped-in variable + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + # projectId: '$(AMLPIPELINE_ID)' + # projectId: 'b13f2931-712c-4f57-b7ec-ad1e35397470' + projectId: '$(System.TeamProjectId)' + # pipelineId: '49' + pipelineId: '$(Build.BuildId)' + # artifactBuildId: ${{ parameters.artifactBuildId }} + artifactBuildId: '$(Build.BuildId)' + - task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' + - task: AzureCLI@1 + displayName: "Deploy to ACI (CLI)" + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring + inlineScript: | + set -e # fail on error + + az ml model deploy --name $(ACI_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(get_model.MODEL_VERSION)' \ + --ic inference_config.yml \ + --dc deployment_config_aci.yml \ + -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \ + --overwrite -v + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)" + +- stage: 'Deploy_AKS' + displayName: 'Deploy to AKS' + dependsOn: Deploy_ACI + condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME']) + jobs: + - job: "Deploy_AKS" + displayName: "Deploy to AKS" + container: mlops + timeoutInMinutes: 0 + steps: + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} + - task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' + - task: AzureCLI@1 + displayName: "Deploy to AKS (CLI)" + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring + inlineScript: | + set -e # fail on error + + az ml model deploy --name $(AKS_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(get_model.MODEL_VERSION)' \ + --compute-target $(AKS_COMPUTE_NAME) \ + --ic inference_config.yml \ + --dc deployment_config_aks.yml \ + -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \ + --overwrite -v + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" + +- stage: 'Deploy_Webapp' + displayName: 'Deploy to Webapp' + condition: variables['WEBAPP_DEPLOYMENT_NAME'] + jobs: + - job: "Deploy_Webapp" + displayName: "Package and deploy model" + container: mlops + timeoutInMinutes: 0 + steps: + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} + - template: diabetes_regression-package-model-template.yml + parameters: + modelId: $(MODEL_NAME):$(get_model.MODEL_VERSION) + scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py' + condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml' + - script: echo $(IMAGE_LOCATION) >image_location.txt + displayName: "Write image location file" + - task: AzureWebAppContainer@1 + name: WebAppDeploy + displayName: 'Azure Web App on Container Deploy' + inputs: + azureSubscription: '$(AZURE_RM_SVC_CONNECTION)' + appName: '$(WEBAPP_DEPLOYMENT_NAME)' + resourceGroupName: '$(RESOURCE_GROUP)' + imageName: '$(IMAGE_LOCATION)' + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" diff --git a/.pipelines/diabetes_regression-get-model-id-artifact-template.yml b/.pipelines/diabetes_regression-get-model-id-artifact-template.yml index b9e61306..285a7503 100644 --- a/.pipelines/diabetes_regression-get-model-id-artifact-template.yml +++ b/.pipelines/diabetes_regression-get-model-id-artifact-template.yml @@ -13,8 +13,10 @@ parameters: steps: - download: none + - script: echo $(parameters.projectId) # this step uses the mapped-in variable corrected + - script: echo $(parameters.pipelineId) # this step uses the mapped-in variable - task: DownloadPipelineArtifact@2 - displayName: Download Pipeline Artifacts + displayName: Download Pipeline Artifacts from template inputs: source: 'specific' project: '${{ parameters.projectId }}' diff --git a/.pipelines/diabetes_regression-variables-template.yml b/.pipelines/diabetes_regression-variables-template.yml index 502753fb..71d58982 100644 --- a/.pipelines/diabetes_regression-variables-template.yml +++ b/.pipelines/diabetes_regression-variables-template.yml @@ -59,11 +59,11 @@ variables: # These are the default values set in ml_service\util\env_variables.py. Uncomment and override if desired. # Set to false to disable the evaluation step in the ML pipeline and register the newly trained model unconditionally. - # - name: RUN_EVALUATION - # value: "true" + - name: RUN_EVALUATION + value: "true" # Set to false to register the model regardless of the outcome of the evaluation step in the ML pipeline. - # - name: ALLOW_RUN_CANCEL - # value: "true" + - name: ALLOW_RUN_CANCEL + value: "false" # Flag to allow rebuilding the AML Environment after it was built for the first time. This enables dependency updates from conda_dependencies.yaml. # - name: AML_REBUILD_ENVIRONMENT diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index 02f51bbc..23251a3e 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -82,27 +82,27 @@ def validate_args(self): def replace_project_name(project_dir, project_name, rename_name): # Replace instances of rename_name within files with project_name files = [r".env.example", - r".pipelines/code-quality-template.yml", - r".pipelines/pr.yml", - r".pipelines/diabetes_regression-cd.yml", - r".pipelines/diabetes_regression-ci.yml", - r".pipelines/abtest.yml", - r".pipelines/diabetes_regression-ci-image.yml", - r".pipelines/diabetes_regression-publish-model-artifact-template.yml", # NOQA: E501 - r".pipelines/diabetes_regression-get-model-id-artifact-template.yml", # NOQA: E501 - r".pipelines/diabetes_regression-batchscoring-ci.yml", - r".pipelines/diabetes_regression-variables-template.yml", - r"environment_setup/Dockerfile", - r"environment_setup/install_requirements.sh", - r"ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py", # NOQA: E501 - r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py", # NOQA: E501 - r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py", # NOQA: E501 - r"ml_service/pipelines/diabetes_regression_build_train_pipeline.py", # NOQA: E501 - r"ml_service/util/create_scoring_image.py", - r"diabetes_regression/conda_dependencies.yml", - r"diabetes_regression/evaluate/evaluate_model.py", - r"diabetes_regression/register/register_model.py", - r"diabetes_regression/training/test_train.py"] + r".pipelines/code-quality-template.yml", + r".pipelines/pr.yml", + r".pipelines/diabetes_regression-cd.yml", + r".pipelines/diabetes_regression-ci.yml", + r".pipelines/abtest.yml", + r".pipelines/diabetes_regression-ci-image.yml", + r".pipelines/diabetes_regression-publish-model-artifact-template.yml", # NOQA: E501 + r".pipelines/diabetes_regression-get-model-id-artifact-template.yml", # NOQA: E501 + r".pipelines/diabetes_regression-batchscoring-ci.yml", + r".pipelines/diabetes_regression-variables-template.yml", + r"environment_setup/Dockerfile", + r"environment_setup/install_requirements.sh", + r"ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py", # NOQA: E501 + r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r_on_dbricks.py", # NOQA: E501 + r"ml_service/pipelines/diabetes_regression_build_train_pipeline_with_r.py", # NOQA: E501 + r"ml_service/pipelines/diabetes_regression_build_train_pipeline.py", # NOQA: E501 + r"ml_service/util/create_scoring_image.py", + r"diabetes_regression/conda_dependencies.yml", + r"diabetes_regression/evaluate/evaluate_model.py", + r"diabetes_regression/register/register_model.py", + r"diabetes_regression/training/test_train.py"] for file in files: path = os.path.join(project_dir, os.path.normpath(file)) diff --git a/diabetes_regression/conda_dependencies.yml b/diabetes_regression/conda_dependencies.yml index 49405c47..8bcc6636 100644 --- a/diabetes_regression/conda_dependencies.yml +++ b/diabetes_regression/conda_dependencies.yml @@ -31,6 +31,8 @@ dependencies: # Training deps - scikit-learn + - lightgbm + - pandas # Scoring deps - inference-schema[numpy-support] diff --git a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py b/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py index 5a0f0125..0ac264c2 100644 --- a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py @@ -145,8 +145,7 @@ def get_fallback_input_dataset(ws: Workspace, env: Env) -> Dataset: if not os.path.exists(env.scoring_datastore_input_filename): error_message = ( - "Could not find CSV dataset for scoring at {}. " - + "No alternate data store location was provided either.".format( + "Could not find CSV dataset for scoring at {}. No alternate data store location was provided either.".format( # NOQA: E501 env.scoring_datastore_input_filename ) # NOQA: E501 ) diff --git a/mlopsendtoend.yml b/mlopsendtoend.yml new file mode 100644 index 00000000..27a798ad --- /dev/null +++ b/mlopsendtoend.yml @@ -0,0 +1,19 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: ubuntu-latest + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script'