-
Notifications
You must be signed in to change notification settings - Fork 1
/
adf-pipeline.yml
178 lines (148 loc) · 7.17 KB
/
adf-pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
trigger:
batch: false
branches:
include:
- develop #collaboration branch
- mainn
pr:
branches:
include:
- mainn
- develop
pool:
vmImage: 'ubuntu-latest'
variables:
- ${{ if eq(variables['build.SourceBranchName'], 'develop') }}:
- group: demo
- ${{ if eq(variables['build.SourceBranchName'], 'main') }}:
- group: demo1
- name: adf_code_path
value: "$(Build.SourcesDirectory)/data_ops/adf"
- name: adf_package_file_path
value: "$(Build.SourcesDirectory)/build/"
# Installs Node and the npm packages saved in your package.json files in the .pipelines folder
stages:
- stage: Validate_ADF_Code
pool:
vmImage: 'windows-latest'
jobs:
- job: Build_ADF_Code
displayName: 'Validating the ADF Code'
steps:
- task: BuildADFTask@1
inputs:
DataFactoryCodePath: '$(adf_code_path)'
Action: 'Build'
continueOnError: true
- stage: Build_And_Publish_ADF_Artifacts
jobs:
- job: Build_Adf_Arm_Templates
displayName: 'Generate ADF Artifacts'
steps:
- task: NodeTool@0
inputs:
versionSpec: '10.x'
displayName: 'Install Node.js'
- task: Npm@1
inputs:
command: 'install'
workingDir: '$(adf_package_file_path)'
verbose: true
displayName: 'Install npm package'
- task: Npm@1
inputs:
command: 'custom'
workingDir: '$(adf_package_file_path)'
customCommand: 'run build validate $(adf_code_path) /subscriptions/$(azure_subscription_id)/resourceGroups/$(resource_group_name)/providers/Microsoft.DataFactory/factories/$(azure_data_factory_name)' # Change "ADFIntegration" to the name of your root folder, if there is not root folder, remove that part and keep Build.Repository.LocalPath only.
displayName: 'Validate'
- task: Npm@1
inputs:
command: 'custom'
workingDir: '$(adf_package_file_path)'
customCommand: 'run build export $(adf_code_path) /subscriptions/$(azure_subscription_id)/resourceGroups/$(resource_group_name)/providers/Microsoft.DataFactory/factories/$(azure_data_factory_name) "ArmTemplate"' # Change "ADFIntegration" to the name of your root folder, if there is not root folder, remove that part.
displayName: 'Validate and Generate ARM template'
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.SourcesDirectory)/build/ArmTemplate'
artifact: 'adf-artifact-$(Build.BuildNumber)'
publishLocation: 'pipeline'
- stage: Deploy_to_Dev
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['build.SourceBranchName'], 'develop'))
displayName: Deploy Dev Stage
dependsOn: Build_And_Publish_ADF_Artifacts
jobs:
- job: Deploy_Dev
displayName: 'Deployment - PreProd'
steps:
- task: DownloadPipelineArtifact@2
displayName: Download Build Artifacts - ADF ARM templates
inputs:
artifactName: 'adf-artifact-$(Build.BuildNumber)'
targetPath: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)'
- task: toggle-adf-trigger@2
inputs:
azureSubscription: '$(azure_service_connection_name)'
ResourceGroupName: '$(resource_group_name)'
DatafactoryName: '$(azure_data_factory_name)'
TriggerFilter: 'Dataload_Trigger'
TriggerStatus: 'stop'
- task: AzureResourceManagerTemplateDeployment@3
inputs:
deploymentScope: 'Resource Group'
azureResourceManagerConnection: '$(azure_service_connection_name)'
subscriptionId: '$(azure_subscription_id)'
action: 'Create Or Update Resource Group'
resourceGroupName: '$(resource_group_name)'
location: '$(location)'
templateLocation: 'Linked artifact'
csmFile: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)/ARMTemplateForFactory.json'
csmParametersFile: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)/ARMTemplateParametersForFactory.json'
overrideParameters: '-factoryName "$(azure_data_factory_name)" -adls_connection_properties_typeProperties_url "https://$(azure_storage_account_name).dfs.core.windows.net/" -databricks_connection_properties_typeProperties_existingClusterId $(azure_databricks_cluster_id) -keyvault_connection_properties_typeProperties_baseUrl "https://$(azure_keyvault_name).vault.azure.net/"'
deploymentMode: 'Incremental'
- task: toggle-adf-trigger@2
inputs:
azureSubscription: '$(azure_service_connection_name)'
ResourceGroupName: '$(resource_group_name)'
DatafactoryName: '$(azure_data_factory_name)'
TriggerFilter: 'Dataload_Trigger'
TriggerStatus: 'start'
- stage: Deploy_to_PreProd
condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['build.SourceBranchName'], 'mainn'))
displayName: Deploy Pre Prod Stage
dependsOn: Build_And_Publish_ADF_Artifacts
jobs:
- job: Deploy_PreProd
displayName: 'Deployment - PreProd'
steps:
- task: DownloadPipelineArtifact@2
displayName: Download Build Artifacts - ADF ARM templates
inputs:
artifactName: 'adf-artifact-$(Build.BuildNumber)'
targetPath: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)'
- task: toggle-adf-trigger@2
inputs:
azureSubscription: '$(azure_service_connection_name)'
ResourceGroupName: '$(resource_group_name)'
DatafactoryName: '$(azure_data_factory_name)'
TriggerFilter: 'Dataload_Trigger'
TriggerStatus: 'stop'
- task: AzureResourceManagerTemplateDeployment@3
inputs:
deploymentScope: 'Resource Group'
azureResourceManagerConnection: '$(azure_service_connection_name)'
subscriptionId: '$(azure_subscription_id)'
action: 'Create Or Update Resource Group'
resourceGroupName: '$(resource_group_name)'
location: '$(location)'
templateLocation: 'Linked artifact'
csmFile: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)/ARMTemplateForFactory.json'
csmParametersFile: '$(Pipeline.Workspace)/adf-artifact-$(Build.BuildNumber)/ARMTemplateParametersForFactory.json'
overrideParameters: '-factoryName "$(azure_data_factory_name)" -adls_connection_properties_typeProperties_url "https://$(azure_storage_account_name).dfs.core.windows.net/" -databricks_connection_properties_typeProperties_existingClusterId $(azure_databricks_cluster_id) -keyvault_connection_properties_typeProperties_baseUrl "https://$(azure_keyvault_name).vault.azure.net/"'
deploymentMode: 'Incremental'
- task: toggle-adf-trigger@2
inputs:
azureSubscription: '$(azure_service_connection_name)'
ResourceGroupName: '$(resource_group_name)'
DatafactoryName: '$(azure_data_factory_name)'
TriggerFilter: 'Dataload_Trigger'
TriggerStatus: 'start'