Skip to content

Commit

Permalink
replaced infra files
Browse files Browse the repository at this point in the history
  • Loading branch information
cindyweng committed Dec 1, 2022
2 parents 64dc8f4 + 5044983 commit a8852de
Show file tree
Hide file tree
Showing 46 changed files with 374 additions and 350 deletions.

This file was deleted.

4 changes: 2 additions & 2 deletions classical/aml-cli-v2/data-science/src/prep.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def main(args):
val.to_parquet((Path(args.val_data) / "val.parquet"))
test.to_parquet((Path(args.test_data) / "test.parquet"))

if (args.enable_monitoring.lower == 'true' or args.enable_monitoring == '1' or args.enable_monitoring.lower == 'yes'):
if (args.enable_monitoring.lower() == 'true' or args.enable_monitoring == '1' or args.enable_monitoring.lower() == 'yes'):
log_training_data(data, args.table_name)


Expand Down Expand Up @@ -129,4 +129,4 @@ def main(args):

mlflow.end_run()



Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@ $schema: https://azuremlschemas.azureedge.net/latest/managedOnlineDeployment.sch
name: blue
endpoint_name: taxi-fare-online
model: azureml:taxi-model@latest
instance_type: Standard_DS2_v2
instance_type: Standard_DS3_v2
instance_count: 1
4 changes: 2 additions & 2 deletions classical/aml-cli-v2/mlops/azureml/train/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ inputs:
input: #using local data, will create an anonymous data asset
type: uri_folder
path: ../../../data/
enable_monitoring: "true"
enable_monitoring: 'false'
table_name: 'taximonitoring'

outputs:
Expand Down Expand Up @@ -98,4 +98,4 @@ jobs:
evaluation_output: ${{parent.jobs.evaluate_model.outputs.evaluation_output}}
outputs:
model_info_output_path: ${{parent.outputs.model_info_output_path}}
# </jobs>
# </jobs>
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,18 @@ stages:
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
parameters:
build_type: conda
environment_name: taxi-train-env
environment_file: mlops/azureml/train/train-env.yml
enable_monitoring: $(enable_monitoring)
- template: templates/${{ variables.version }}/create-compute.yml@mlops-templates
parameters:
cluster_name: cpu-cluster
size: Standard_DS3_v2
min_instances: 0
max_instances: 4
cluster_tier: LowPriority
- template: templates/${{ variables.version }}/run-pipeline.yml@mlops-templates
parameters:
pipeline_file: mlops/azureml/train/pipeline.yml
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
enable_monitoring: $(enable_monitoring)
enable_monitoring: $(enable_monitoring)
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,21 @@ jobs:
conda_file: data-science/environment/train-conda.yml
secrets:
creds: ${{secrets.AZURE_CREDENTIALS}}
run-pipeline:
create-compute:
needs: [get-config,register-environment]
uses: Azure/mlops-templates/.github/workflows/create-compute.yml@main
with:
cluster_name: cpu-cluster
size: Standard_DS3_v2
min_instances: 0
max_instances: 4
cluster_tier: LowPriority
resource_group: ${{ needs.get-config.outputs.resource_group }}
workspace_name: ${{ needs.get-config.outputs.aml_workspace }}
secrets:
creds: ${{secrets.AZURE_CREDENTIALS}}
run-pipeline:
needs: [get-config,register-environment,create-compute]
uses: Azure/mlops-templates/.github/workflows/run-pipeline.yml@main
with:
resource_group: ${{ needs.get-config.outputs.resource_group }}
Expand Down

This file was deleted.

4 changes: 2 additions & 2 deletions classical/rai-aml-cli-v2/data-science/src/prep/prep.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def main(args):
val.to_parquet((Path(args.val_data) / "val.parquet"))
test.to_parquet((Path(args.test_data) / "test.parquet"))

if (args.enable_monitoring.lower == 'true' or args.enable_monitoring == '1' or args.enable_monitoring.lower == 'yes'):
if (args.enable_monitoring.lower() == 'true' or args.enable_monitoring == '1' or args.enable_monitoring.lower() == 'yes'):
log_training_data(data, args.table_name)


Expand Down Expand Up @@ -129,4 +129,4 @@ def main(args):

mlflow.end_run()



Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,8 @@ stages:
- template: templates/${{ variables.version }}/connect-to-workspace.yml@mlops-templates
- template: templates/${{ variables.version }}/register-environment.yml@mlops-templates
parameters:
build_type: conda
environment_name: taxi-train-env
environment_file: mlops/azureml/train/train-env.yml
enable_monitoring: $(enable_monitoring)
- checkout: rai-vnext-preview
path: s/
- template: register-rai-components.yml
Expand All @@ -65,4 +63,4 @@ stages:
pipeline_file: mlops/azureml/train/pipeline.yml
experiment_name: $(environment)_taxi_fare_train_$(Build.SourceBranchName)
display_name: $(environment)_taxi_fare_run_$(Build.BuildID)
enable_monitoring: $(enable_monitoring)
enable_monitoring: $(enable_monitoring)
2 changes: 1 addition & 1 deletion config-infra-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ variables:

# For terraform reference
terraform_version: 0.14.7
terraform_workingdir: infrastructure/terraform
terraform_workingdir: infrastructure
terraform_st_resource_group: rg-$(namespace)-$(postfix)$(environment)-tf
terraform_st_storage_account: st$(namespace)$(postfix)$(environment)tf
terraform_st_container_name: default
Expand Down
28 changes: 28 additions & 0 deletions infrastructure/bicep/bicepconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
"analyzers": {
"core": {
"enabled": true,
"verbose": true,
"rules": {
"no-hardcoded-env-urls": {
"level": "error"
},
"no-unused-params": {
"level": "error"
},
"no-unused-vars": {
"level": "error"
},
"prefer-interpolation": {
"level": "error"
},
"secure-parameter-default": {
"level": "error"
},
"simplify-interpolation": {
"level": "error"
}
}
}
}
}
93 changes: 93 additions & 0 deletions infrastructure/bicep/main.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
targetScope = 'subscription'

param location string = 'westus2'
param prefix string
param postfix string
param env string

param tags object = {
Owner: 'mlops-v2'
Project: 'mlops-v2'
Environment: env
Toolkit: 'bicep'
Name: prefix
}

var baseName = '${prefix}-${postfix}${env}'
var resourceGroupName = 'rg-${baseName}'

resource rg 'Microsoft.Resources/resourceGroups@2020-06-01' = {
name: resourceGroupName
location: location

tags: tags
}

// Storage Account
module st './modules/storage_account.bicep' = {
name: 'st'
scope: resourceGroup(rg.name)
params: {
baseName: '${uniqueString(rg.id)}${env}'
location: location
tags: tags
}
}

// Key Vault
module kv './modules/key_vault.bicep' = {
name: 'kv'
scope: resourceGroup(rg.name)
params: {
baseName: baseName
location: location
tags: tags
}
}

// App Insights
module appi './modules/application_insights.bicep' = {
name: 'appi'
scope: resourceGroup(rg.name)
params: {
baseName: baseName
location: location
tags: tags
}
}

// Container Registry
module cr './modules/container_registry.bicep' = {
name: 'cr'
scope: resourceGroup(rg.name)
params: {
baseName: '${uniqueString(rg.id)}${env}'
location: location
tags: tags
}
}

// AML workspace
module mlw './modules/aml_workspace.bicep' = {
name: 'mlw'
scope: resourceGroup(rg.name)
params: {
baseName: baseName
location: location
stoacctid: st.outputs.stoacctOut
kvid: kv.outputs.kvOut
appinsightid: appi.outputs.appinsightOut
crid: cr.outputs.crOut
tags: tags
}
}

// AML compute cluster
module mlwcc './modules/aml_computecluster.bicep' = {
name: 'mlwcc'
scope: resourceGroup(rg.name)
params: {
location: location
workspaceName: mlw.outputs.amlsName
}
}
20 changes: 20 additions & 0 deletions infrastructure/bicep/modules/aml_computecluster.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
param location string
param computeClusterName string = 'cpu-cluster'
param workspaceName string

resource amlci 'Microsoft.MachineLearningServices/workspaces/computes@2020-09-01-preview' = {
name: '${workspaceName}/${computeClusterName}'
location: location
properties: {
computeType: 'AmlCompute'
properties: {
vmSize: 'Standard_DS3_v2'
subnet: json('null')
osType: 'Linux'
scaleSettings: {
maxNodeCount: 4
minNodeCount: 0
}
}
}
}
37 changes: 37 additions & 0 deletions infrastructure/bicep/modules/aml_workspace.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
param baseName string
param location string
param stoacctid string
param kvid string
param appinsightid string
param crid string
param tags object

// AML workspace
resource amls 'Microsoft.MachineLearningServices/workspaces@2020-09-01-preview' = {
name: 'mlw-${baseName}'
location: location
identity: {
type: 'SystemAssigned'
}
sku: {
tier: 'basic'
name: 'basic'
}
properties: {
storageAccount: stoacctid
keyVault: kvid
applicationInsights: appinsightid
containerRegistry: crid
encryption: {
status: 'Disabled'
keyVaultProperties: {
keyIdentifier: ''
keyVaultArmId: ''
}
}
}

tags: tags
}

output amlsName string = amls.name
17 changes: 17 additions & 0 deletions infrastructure/bicep/modules/application_insights.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
param baseName string
param location string
param tags object

// App Insights
resource appinsight 'Microsoft.Insights/components@2020-02-02-preview' = {
name: 'appi-${baseName}'
location: location
kind: 'web'
properties: {
Application_Type: 'web'
}

tags: tags
}

output appinsightOut string = appinsight.id
19 changes: 19 additions & 0 deletions infrastructure/bicep/modules/container_registry.bicep
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
param baseName string
param location string
param tags object

resource cr 'Microsoft.ContainerRegistry/registries@2020-11-01-preview' = {
name: 'cr${baseName}'
location: location
sku: {
name: 'Standard'
}

properties: {
adminUserEnabled: true
}

tags: tags
}

output crOut string = cr.id
Loading

0 comments on commit a8852de

Please sign in to comment.