azure-databricks-cicd/devops/ado-databricks-cicd.yml at main · globalbao/azure-databricks-cicd · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
# Databricks Config pipeline for Azure DevOps
pr:
  branches:
    include:
      - main
  paths:
    exclude:
      - devops

trigger:
  branches:
    include:
      - main
  paths:
    exclude:
      - devops

# Global Variables. For environment-specific variables, see the individual stages further below e.g. Release_Dev, Release_Uat, etc.
variables:
  - name: AGENT_POOL
    value: "This is a placeholder for your private agent pool" #The name of the Azure DevOps agent pool
  - name: condition
    value: "and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))" #Check if the build has succeeded and the source branch is main
  - name: DATABRICKS_SDK_VERSION # the version of the Databricks SDK to install https://pypi.org/project/databricks-sdk/#history
    value: "0.28.0"
  - name: DATABRICKS_LOG_LEVEL #Logging level for Databricks CLI e.g. 'debug', 'warn', 'info' https://docs.databricks.com/en/dev-tools/cli/troubleshooting.html
    value: "warn" #log messages at the warn and error levels.

#Import Azure DevOps variable group values containing secrets and other variables
  - group: placeholder-for-devops-library-name #replace with the name of your Azure DevOps library

stages:
  - stage: Build_Dev
    displayName: "Build Dev"
    dependsOn: []
    jobs:
      - template: /devops/templates/ado-build-template.yml@self
        parameters:
          BUNDLE_TARGET: "dev"
          ADB_HOST: $(ADB_DEV_HOST)
          ADB_CLIENT_ID: $(ADB_DEV_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_DEV_SPN_SECRET)
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADO_ENVIRONMENT: "Dev-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Release_Dev
    displayName: "Release Dev"
    dependsOn: [Build_Dev]
    condition: ${{ variables.Condition }}
    jobs:
      - template: /devops/templates/ado-release-template.yml@self
        parameters:
          BUNDLE_TARGET: "dev"
          ADB_HOST: $(ADB_DEV_HOST)
          ADB_CLIENT_ID: $(ADB_DEV_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_DEV_SPN_SECRET)
          ADB_ENABLE_IP_ACCESS_LISTS: false #Update to true if you want to enable IP access lists for the Databricks workspace
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADB_JOB_NAMES: "init_job" #Update this to desired init job name defined in your YAML file
          ADO_ENVIRONMENT: "Dev-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Build_Uat
    displayName: "Build Uat"
    dependsOn: []
    jobs:
      - template: /devops/templates/ado-build-template.yml@self
        parameters:
          BUNDLE_TARGET: "uat"
          ADB_HOST: $(ADB_UAT_HOST)
          ADB_CLIENT_ID: $(ADB_UAT_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_UAT_SPN_SECRET)
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADO_ENVIRONMENT: "Uat-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Release_Uat
    displayName: "Release Uat"
    dependsOn: [Build_Uat]
    condition: succeeded()
    jobs:
      - template: /devops/templates/ado-release-template.yml@self
        parameters:
          BUNDLE_TARGET: "uat"
          ADB_HOST: $(ADB_UAT_HOST)
          ADB_CLIENT_ID: $(ADB_UAT_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_UAT_SPN_SECRET)
          ADB_ENABLE_IP_ACCESS_LISTS: false #Update to true if you want to enable IP access lists for the Databricks workspace
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADB_JOB_NAMES: "init_job" #Update this to desired init job name defined in your YAML file
          ADO_ENVIRONMENT: "Uat-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Build_PFX
    displayName: "Build PFX"
    dependsOn: []
    jobs:
      - template: /devops/templates/ado-build-template.yml@self
        parameters:
          BUNDLE_TARGET: "pfx"
          ADB_HOST: $(ADB_PFX_HOST)
          ADB_CLIENT_ID: $(ADB_PFX_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_PFX_SPN_SECRET)
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADO_ENVIRONMENT: "ProdFix-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Release_PFX
    displayName: "Release PFX"
    dependsOn: [Build_PFX]
    condition: ${{ variables.Condition }}
    jobs:
      - template: /devops/templates/ado-release-template.yml@self
        parameters:
          BUNDLE_TARGET: "pfx"
          ADB_HOST: $(ADB_PFX_HOST)
          ADB_CLIENT_ID: $(ADB_PFX_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_PFX_SPN_SECRET)
          ADB_ENABLE_IP_ACCESS_LISTS: false #Update to true if you want to enable IP access lists for the Databricks workspace
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADB_JOB_NAMES: "init_job" #Update this to desired init job name defined in your YAML file
          ADO_ENVIRONMENT: "ProdFix-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Build_PRD
    displayName: "Build PRD"
    dependsOn: []
    jobs:
      - template: /devops/templates/ado-build-template.yml@self
        parameters:
          BUNDLE_TARGET: "prd"
          ADB_HOST: $(ADB_PRD_HOST)
          ADB_CLIENT_ID: $(ADB_PRD_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_PRD_SPN_SECRET)
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADO_ENVIRONMENT: "Prod-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}

  - stage: Release_PRD
    displayName: "Release PRD"
    dependsOn: [Build_PRD]
    condition: ${{ variables.Condition }}
    jobs:
      - template: /devops/templates/ado-release-template.yml@self
        parameters:
          BUNDLE_TARGET: "prd"
          ADB_HOST: $(ADB_PRD_HOST)
          ADB_CLIENT_ID: $(ADB_PRD_SPN_ID)
          ADB_CLIENT_SECRET: $(ADB_PRD_SPN_SECRET)
          ADB_ENABLE_IP_ACCESS_LISTS: false #Update to true if you want to enable IP access lists for the Databricks workspace
          DATABRICKS_SDK_VERSION: ${{ variables.DATABRICKS_SDK_VERSION }}
          DATABRICKS_LOG_LEVEL: ${{ variables.DATABRICKS_LOG_LEVEL }}
          ADB_JOB_NAMES: "init_job" #Update this to desired init job name defined in your YAML file
          ADO_ENVIRONMENT: "Prod-Databricks" #Update this to desired environment name in Azure DevOps
          AGENT_POOL: ${{ variables.AGENT_POOL }}