diff --git a/.env b/.env new file mode 100644 index 00000000..47311d1e --- /dev/null +++ b/.env @@ -0,0 +1,81 @@ +# Azure Subscription Variables +SUBSCRIPTION_ID = '' +LOCATION = '' +TENANT_ID = '' +BASE_NAME = '' +SP_APP_ID = '' +SP_APP_SECRET = '' +RESOURCE_GROUP = 'mlops-RG' + +# Mock build/release ID for local testing +BUILD_BUILDID = '001' + +# Azure ML Workspace Variables +WORKSPACE_NAME = 'mlops-aml-ws' +EXPERIMENT_NAME = 'mlopspython' + +# AML Compute Cluster Config +AML_ENV_NAME='diabetes_regression_training_env' +AML_ENV_TRAIN_CONDA_DEP_FILE="conda_dependencies.yml" +AML_COMPUTE_CLUSTER_NAME = 'train-cluster' +AML_COMPUTE_CLUSTER_CPU_SKU = 'STANDARD_DS2_V2' +AML_CLUSTER_MAX_NODES = '4' +AML_CLUSTER_MIN_NODES = '0' +AML_CLUSTER_PRIORITY = 'lowpriority' +# Training Config +MODEL_NAME = 'diabetes_regression_model.pkl' +MODEL_VERSION = '1' +TRAIN_SCRIPT_PATH = 'training/train_aml.py' + + +# AML Pipeline Config +TRAINING_PIPELINE_NAME = 'Training Pipeline' +MODEL_PATH = '' +EVALUATE_SCRIPT_PATH = 'evaluate/evaluate_model.py' +REGISTER_SCRIPT_PATH = 'register/register_model.py' +SOURCES_DIR_TRAIN = 'diabetes_regression' +DATASET_NAME = 'diabetes_ds' +DATASET_VERSION = 'latest' +# Optional. Set it if you have configured non default datastore to point to your data +DATASTORE_NAME = '' +SCORE_SCRIPT = 'scoring/score.py' + +# Optional. Used by a training pipeline with R on Databricks +DB_CLUSTER_ID = '' + +# Optional. Container Image name for image creation +IMAGE_NAME = 'mltrained' + +# Run Evaluation Step in AML pipeline +RUN_EVALUATION = 'true' + +# Set to true cancels the Azure ML pipeline run when evaluation criteria are not met. +ALLOW_RUN_CANCEL = 'true' + +# Flag to allow rebuilding the AML Environment after it was built for the first time. This enables dependency updates from conda_dependencies.yaml. +AML_REBUILD_ENVIRONMENT = 'false' + + + +USE_GPU_FOR_SCORING = "false" +AML_ENV_SCORE_CONDA_DEP_FILE="conda_dependencies_scoring.yml" +AML_ENV_SCORECOPY_CONDA_DEP_FILE="conda_dependencies_scorecopy.yml" +# AML Compute Cluster Config for parallel batch scoring +AML_ENV_NAME_SCORING='diabetes_regression_scoring_env' +AML_ENV_NAME_SCORE_COPY='diabetes_regression_score_copy_env' +AML_COMPUTE_CLUSTER_NAME_SCORING = 'score-cluster' +AML_COMPUTE_CLUSTER_CPU_SKU_SCORING = 'STANDARD_DS2_V2' +AML_CLUSTER_MAX_NODES_SCORING = '4' +AML_CLUSTER_MIN_NODES_SCORING = '0' +AML_CLUSTER_PRIORITY_SCORING = 'lowpriority' +AML_REBUILD_ENVIRONMENT_SCORING = 'true' +BATCHSCORE_SCRIPT_PATH = 'scoring/parallel_batchscore.py' +BATCHSCORE_COPY_SCRIPT_PATH = 'scoring/parallel_batchscore_copyoutput.py' + + +SCORING_DATASTORE_INPUT_CONTAINER = 'input' +SCORING_DATASTORE_INPUT_FILENAME = 'diabetes_scoring_input.csv' +SCORING_DATASTORE_OUTPUT_CONTAINER = 'output' +SCORING_DATASTORE_OUTPUT_FILENAME = 'diabetes_scoring_output.csv' +SCORING_DATASET_NAME = 'diabetes_scoring_ds' +SCORING_PIPELINE_NAME = 'diabetes-scoring-pipeline' diff --git a/.pipelines/diabetes_regression-ci.yml b/.pipelines/diabetes_regression-ci.yml index 5a539af0..481df400 100644 --- a/.pipelines/diabetes_regression-ci.yml +++ b/.pipelines/diabetes_regression-ci.yml @@ -19,7 +19,7 @@ trigger: variables: - template: diabetes_regression-variables-template.yml -- group: devopsforai-aml-vg +- group: mlops-vg pool: vmImage: ubuntu-latest diff --git a/azure-pipelines-1.yml b/azure-pipelines-1.yml new file mode 100644 index 00000000..ced3416c --- /dev/null +++ b/azure-pipelines-1.yml @@ -0,0 +1,30 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: ubuntu-latest +# Starter pipeline + + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' +- task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: "$(AZURE_RM_SVC_CONNECTION)" + action: "Create Or Update Resource Group" + resourceGroupName: "$(RESOURCE_GROUP)" + location: $(LOCATION) + templateLocation: "Linked artifact" + deploymentMode: "Incremental" + displayName: "Deploy MLOps resources to Azure" + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' diff --git a/azure-pipelines-2.yml b/azure-pipelines-2.yml new file mode 100644 index 00000000..27a798ad --- /dev/null +++ b/azure-pipelines-2.yml @@ -0,0 +1,19 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- master + +pool: + vmImage: ubuntu-latest + +steps: +- script: echo Hello, world! + displayName: 'Run a one-line script' + +- script: | + echo Add other tasks to build, test, and deploy your project. + echo See https://aka.ms/yaml + displayName: 'Run a multi-line script' diff --git a/azure-pipelines-3.yml b/azure-pipelines-3.yml new file mode 100644 index 00000000..55905e25 --- /dev/null +++ b/azure-pipelines-3.yml @@ -0,0 +1,29 @@ +# Docker +# Build a Docker image +# https://docs.microsoft.com/azure/devops/pipelines/languages/docker + +trigger: +- master + +resources: +- repo: self + +variables: + tag: '$(Build.BuildId)' + +stages: +- stage: Build + displayName: Build image + jobs: + - job: Build + displayName: Build + pool: + vmImage: ubuntu-latest + steps: + - task: Docker@2 + displayName: Build an image + inputs: + command: build + dockerfile: '$(Build.SourcesDirectory)/environment_setup/Dockerfile' + tags: | + $(tag) diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..d0db0886 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,36 @@ +# CI/PR Pipeline that deploys an ARM template to create or update the resources needed by the other pipelines. +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* +pr: + branches: + include: + - master + paths: + include: + - environment_setup/arm-templates/* + +pool: + vmImage: "ubuntu-latest" + +variables: + - group: mlops-vg + - name: WORKSPACE_SKU # https://docs.microsoft.com/en-us/azure/machine-learning/overview-what-is-azure-ml#sku + value: basic + +steps: + - task: AzureResourceGroupDeployment@2 + inputs: + azureSubscription: "$(AZURE_RM_SVC_CONNECTION)" + action: "Create Or Update Resource Group" + resourceGroupName: "$(RESOURCE_GROUP)" + location: $(LOCATION) + templateLocation: "Linked artifact" + csmFile: "$(Build.SourcesDirectory)/environment_setup/arm-templates/cloud-environment.json" + overrideParameters: "-baseName $(BASE_NAME) -location $(LOCATION) -workspace $(WORKSPACE_NAME) -sku $(WORKSPACE_SKU)" + deploymentMode: "Incremental" + displayName: "Deploy MLOps resources to Azure" diff --git a/environment_setup/iac-create-environment-pipeline-arm.yml b/environment_setup/iac-create-environment-pipeline-arm.yml index 0b9f474c..d0db0886 100644 --- a/environment_setup/iac-create-environment-pipeline-arm.yml +++ b/environment_setup/iac-create-environment-pipeline-arm.yml @@ -18,7 +18,7 @@ pool: vmImage: "ubuntu-latest" variables: - - group: devopsforai-aml-vg + - group: mlops-vg - name: WORKSPACE_SKU # https://docs.microsoft.com/en-us/azure/machine-learning/overview-what-is-azure-ml#sku value: basic diff --git a/environment_setup/iac-create-environment-pipeline-tf.yml b/environment_setup/iac-create-environment-pipeline-tf.yml index ef184546..85525f29 100644 --- a/environment_setup/iac-create-environment-pipeline-tf.yml +++ b/environment_setup/iac-create-environment-pipeline-tf.yml @@ -18,7 +18,7 @@ pool: vmImage: 'ubuntu-latest' variables: -- group: devopsforai-aml-vg +- group: mlops-vg steps: - task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-installer.TerraformInstaller@0