/
.azure-pipelines.yml
213 lines (203 loc) · 9.25 KB
/
.azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
# This is the configuration file for Azure Pipelines, used to run tests on
# macOS. Documentation to help understand this setup:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/multiple-phases
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates
# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables
# https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/index
#
# In addition to this configuration file, some setup in the Azure DevOps
# project is required:
# - The "Build pull requests from forks of this repository" setting must be
# enabled: https://docs.microsoft.com/en-us/azure/devops/pipelines/repos/github#validate-contributions-from-forks
# - A scheduled build needs to be set up for the the epochs/daily branch.
# - To get results from scheduled builds into wpt.fyi, a service connection
# named wpt.fyi with URL https://wpt.fyi is needed.
# - Self-hosted agents for Windows 10 are used:
# - 'Hosted Windows Client' is the latest Windows 10
# - 'Hosted Windows Client Next' is Windows 10 Insider Preview
# Documention for the setup of these agents:
# https://docs.microsoft.com/en-us/azure/devops/pipelines/agents/v2-windows
jobs:
# The affected tests jobs are unconditional for speed, as most PRs have one or
# more affected tests: https://github.com/web-platform-tests/wpt/issues/13936.
- job: affected_safari_preview
displayName: 'affected tests (Safari Technology Preview)'
condition: eq(variables['Build.Reason'], 'PullRequest')
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/affected_tests.yml
parameters:
artifactName: 'safari-preview-affected-tests'
- job: affected_without_changes_safari_preview
displayName: 'affected tests without changes (Safari Technology Preview)'
condition: eq(variables['Build.Reason'], 'PullRequest')
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/affected_tests.yml
parameters:
checkoutCommit: 'HEAD^1'
affectedRange: 'HEAD@{1}'
artifactName: 'safari-preview-affected-tests-without-changes'
# The decision jobs runs `./wpt test-jobs` to determine which jobs to run,
# and all following jobs wait for it to finish and depend on its output.
- job: decision
displayName: './wpt test-jobs'
condition: eq(variables['Build.Reason'], 'PullRequest')
pool:
vmImage: 'ubuntu-16.04'
steps:
- template: tools/ci/azure/checkout.yml
- script: |
./wpt test-jobs | while read job; do
echo "$job"
echo "##vso[task.setvariable variable=$job;isOutput=true]true";
done
name: test_jobs
displayName: 'Run ./wpt test-jobs'
- job: infrastructure_macOS
displayName: 'infrastructure/ tests (macOS)'
dependsOn: decision
condition: dependencies.decision.outputs['test_jobs.wptrunner_infrastructure']
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/pip_install.yml
parameters:
packages: virtualenv
- template: tools/ci/azure/install_fonts.yml
- template: tools/ci/azure/install_certs.yml
- template: tools/ci/azure/install_chrome.yml
- template: tools/ci/azure/install_firefox.yml
- template: tools/ci/azure/install_safari.yml
- template: tools/ci/azure/update_hosts.yml
- template: tools/ci/azure/update_manifest.yml
- script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/chrome.tbpl.log --log-tbpl-level info --channel dev chrome infrastructure/
displayName: 'Run tests (Chrome Dev)'
- script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/firefox.tbpl.log --log-tbpl-level info --channel nightly firefox infrastructure/
displayName: 'Run tests (Firefox Nightly)'
- script: no_proxy='*' ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/safari.tbpl.log --log-tbpl-level info --channel preview safari infrastructure/
displayName: 'Run tests (Safari Technology Preview)'
- task: PublishBuildArtifacts@1
displayName: 'Publish results'
inputs:
artifactName: 'infrastructure'
condition: succeededOrFailed()
- job: tools_unittest_macOS
displayName: 'tools/ unittests (macOS)'
dependsOn: decision
condition: dependencies.decision.outputs['test_jobs.tools_unittest']
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/tox_pytest.yml
parameters:
directory: tools/
toxenv: py27
- job: wptrunner_unittest_macOS
displayName: 'tools/wptrunner/ unittests (macOS)'
dependsOn: decision
condition: dependencies.decision.outputs['test_jobs.wptrunner_unittest']
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/tox_pytest.yml
parameters:
directory: tools/wptrunner/
- job: wpt_integration_macOS
displayName: 'tools/wpt/ tests (macOS)'
dependsOn: decision
condition: dependencies.decision.outputs['test_jobs.wpt_integration']
pool:
vmImage: 'macOS-10.13'
steps:
# full checkout required
- template: tools/ci/azure/install_chrome.yml
- template: tools/ci/azure/install_firefox.yml
- template: tools/ci/azure/update_hosts.yml
- template: tools/ci/azure/update_manifest.yml
- template: tools/ci/azure/tox_pytest.yml
parameters:
directory: tools/wpt/
- job: infrastructure_win10
displayName: 'infrastructure/ tests (Windows 10)'
# This job is only triggered manually until it has been shown to be robust.
condition: and(eq(variables['Build.Reason'], 'Manual'), variables['run_infrastructure_win10'])
pool:
name: 'Hosted Windows Client'
steps:
- template: tools/ci/azure/system_info.yml
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/install_python.yml
- template: tools/ci/azure/pip_install.yml
parameters:
packages: virtualenv
- template: tools/ci/azure/install_certs.yml
- template: tools/ci/azure/update_hosts.yml
- template: tools/ci/azure/update_manifest.yml
- script: python ./wpt run --yes --no-manifest-update --install-fonts --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-tbpl $(Build.ArtifactStagingDirectory)/edge.tbpl.log --log-tbpl-level info edge_webdriver infrastructure/
displayName: 'Run tests (Edge)'
- task: PublishBuildArtifacts@1
displayName: 'Publish results'
inputs:
artifactName: 'infrastructure'
condition: succeededOrFailed()
- template: tools/ci/azure/cleanup_win10.yml
- job: results_edge
displayName: 'all tests (Edge)'
# This job is only triggered manually until it has been shown to be robust.
condition: and(eq(variables['Build.Reason'], 'Manual'), variables['run_all_edge'])
# There are 5 agents in the pool, but use more jobs so that each takes <1h.
strategy:
parallel: 20
timeoutInMinutes: 360
pool:
name: 'Hosted Windows Client'
steps:
- template: tools/ci/azure/system_info.yml
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/install_python.yml
- template: tools/ci/azure/pip_install.yml
parameters:
packages: virtualenv
- template: tools/ci/azure/install_certs.yml
- template: tools/ci/azure/update_hosts.yml
- template: tools/ci/azure/update_manifest.yml
- script: python ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --install-fonts --test-types reftest testharness --this-chunk $(System.JobPositionInPhase) --total-chunks $(System.TotalJobsInPhase) --chunk-type hash --log-tbpl - --log-tbpl-level info --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json edge_webdriver
displayName: 'Run tests'
- task: PublishBuildArtifacts@1
displayName: 'Publish results'
inputs:
artifactName: 'edge-results'
- template: tools/ci/azure/cleanup_win10.yml
# All `./wpt run` tests are run from epochs/* branches on a schedule. See
# documentation at the top of this file for required setup.
- job: results_safari_preview
displayName: 'all tests (Safari Technology Preview)'
condition: eq(variables['Build.Reason'], 'Schedule')
strategy:
parallel: 4 # chosen to make runtime ~2h
timeoutInMinutes: 360
pool:
vmImage: 'macOS-10.13'
steps:
- template: tools/ci/azure/checkout.yml
- template: tools/ci/azure/pip_install.yml
parameters:
packages: virtualenv
- template: tools/ci/azure/install_fonts.yml
- template: tools/ci/azure/install_certs.yml
- template: tools/ci/azure/install_safari.yml
- template: tools/ci/azure/update_hosts.yml
- template: tools/ci/azure/update_manifest.yml
- script: no_proxy='*' ./wpt run --no-manifest-update --no-restart-on-unexpected --no-fail-on-unexpected --this-chunk=$(System.JobPositionInPhase) --total-chunks=$(System.TotalJobsInPhase) --chunk-type hash --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_$(System.JobPositionInPhase).json --channel preview safari
displayName: 'Run tests'
- task: PublishBuildArtifacts@1
displayName: 'Publish results'
inputs:
artifactName: 'safari-preview-results'