/
azure-pipelines.yml
321 lines (316 loc) · 14.8 KB
/
azure-pipelines.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
# azure-pipelines.yml
# Copyright (c) 2013-2019 Pablo Acosta-Serafini
# See LICENSE for details
# yamllint disable rule:document-start
# yamllint disable rule:indentation
# yamllint disable rule:line-length
# yamllint disable rule:comments
trigger:
- master
jobs:
# [[[cog
# import os
# PKG_NAME = os.path.basename(os.path.dirname(os.path.abspath(cog.inFile)))
# exec("from " + PKG_NAME.replace("-", "_") + ".pkgdata import SUPPORTED_INTERPS")
# cog.outl("- job: 'Test'")
# cog.outl(" strategy:")
# cog.outl(" matrix:")
# PLATFORMS = {"linux":"ubuntu-16.04", "macos":"macos-10.13", "windows":"vs2017-win2016"}
# for PLATFORM, IMAGE in PLATFORMS.items():
# for SUPPORTED_INTERP in SUPPORTED_INTERPS:
# cog.outl(" {0}_python{1}:".format(PLATFORM, str(SUPPORTED_INTERP).replace(".", "")))
# cog.outl(" imageName: '{0}'".format(IMAGE))
# cog.outl(" python.version: '{0}'".format(str(SUPPORTED_INTERP)))
# cog.outl(" aname: '{0}'".format(PLATFORM))
# cog.outl(" pyname: 'py{0}'".format(str(SUPPORTED_INTERP).replace(".", "")))
# ]]]
- job: 'Test'
strategy:
matrix:
macos_python35:
imageName: 'macos-10.13'
python.version: '3.5'
aname: 'macos'
pyname: 'py35'
macos_python36:
imageName: 'macos-10.13'
python.version: '3.6'
aname: 'macos'
pyname: 'py36'
macos_python37:
imageName: 'macos-10.13'
python.version: '3.7'
aname: 'macos'
pyname: 'py37'
linux_python35:
imageName: 'ubuntu-16.04'
python.version: '3.5'
aname: 'linux'
pyname: 'py35'
linux_python36:
imageName: 'ubuntu-16.04'
python.version: '3.6'
aname: 'linux'
pyname: 'py36'
linux_python37:
imageName: 'ubuntu-16.04'
python.version: '3.7'
aname: 'linux'
pyname: 'py37'
windows_python35:
imageName: 'vs2017-win2016'
python.version: '3.5'
aname: 'windows'
pyname: 'py35'
windows_python36:
imageName: 'vs2017-win2016'
python.version: '3.6'
aname: 'windows'
pyname: 'py36'
windows_python37:
imageName: 'vs2017-win2016'
python.version: '3.7'
aname: 'windows'
pyname: 'py37'
# [[[end]]]
pool:
vmImage: $(imageName)
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: '$(python.version)'
architecture: 'x64'
- script: python --version
displayName: Verify python version
- bash: |
git submodule init
git submodule update
displayName: Populate submodules
- bash: |
export INTERP=$(python -c 'from __future__ import print_function; import sys; info=sys.version_info; print("py{}{}".format(info.major, info.minor))')
export REPO_DIR="$(Build.Repository.LocalPath)"
export PKG_NAME=$(basename $(Build.Repository.Name))
export PKG_NAME_ALT=$(python -c "from __future__ import print_function; import os; print(os.environ['PKG_NAME'].split('-')[0])")
export PKG_NAME_DIR=$(python -c "from __future__ import print_function; import os; print(os.environ['PKG_NAME'] if os.path.isdir(os.path.join(os.environ['REPO_DIR'], os.environ['PKG_NAME'])) else os.environ['PKG_NAME_ALT'])")
export PKG_VERSION=$(python -c "import os, sys; sys.path.append(os.path.realpath('./'+os.environ['PKG_NAME_DIR']));import pkgdata; print(pkgdata.__version__)")
export PYTHON_SITE_PACKAGES=$(python -c "from __future__ import print_function;import os,pip;y=os.path.abspath(pip.__file__).split(os.sep);print('/'.join(y[:y.index('pip')]))")
export EXTRA_DIR=$(python -c "from __future__ import print_function; import os,sys; print(sys.prefix.replace(os.sep, '/'))")/share/${PKG_NAME}
export SBIN_DIR=${EXTRA_DIR}/pypkg
export RESULTS_DIR=${REPO_DIR}/results
export SOURCE_DIR=${PYTHON_SITE_PACKAGES}/${PKG_NAME_DIR}
export TRACER_DIR=${EXTRA_DIR}/docs/support
export ARTIFACTS_DIR=${EXTRA_DIR}/artifacts
export PYTHONPATH=${PYTHONPATH}:${SBIN_DIR}:${EXTRA_DIR}:${EXTRA_DIR}/tests:${EXTRA_DIR}/docs:${EXTRA_DIR}/docs/support
export COV_FILE=${SOURCE_DIR}/.coveragerc_ci_${INTERP}
export LANG="en_US.UTF-8"
export NUM_CPUS=$(python -c "from __future__ import print_function; import multiprocessing; print(multiprocessing.cpu_count())")
echo "##vso[task.setvariable variable=INTERP]$INTERP"
echo "##vso[task.setvariable variable=PKG_NAME]$PKG_NAME"
echo "##vso[task.setvariable variable=PKG_NAME_ALT]$PKG_NAME_ALT"
echo "##vso[task.setvariable variable=PKG_NAME_DIR]$PKG_NAME_DIR"
echo "##vso[task.setvariable variable=PKG_VERSION]$PKG_VERSION"
echo "##vso[task.setvariable variable=PYTHON_SITE_PACKAGES]$PYTHON_SITE_PACKAGES"
echo "##vso[task.setvariable variable=REPO_DIR]$REPO_DIR"
echo "##vso[task.setvariable variable=EXTRA_DIR]$EXTRA_DIR"
echo "##vso[task.setvariable variable=SBIN_DIR]$SBIN_DIR"
echo "##vso[task.setvariable variable=RESULTS_DIR]$RESULTS_DIR"
echo "##vso[task.setvariable variable=SOURCE_DIR]$SOURCE_DIR"
echo "##vso[task.setvariable variable=TRACER_DIR]$TRACER_DIR"
echo "##vso[task.setvariable variable=ARTIFACTS_DIR]$ARTIFACTS_DIR"
echo "##vso[task.setvariable variable=PYTHONPATH]$PYTHONPATH"
echo "##vso[task.setvariable variable=COV_FILE]$COV_FILE"
echo "##vso[task.setvariable variable=LANG]$LANG"
echo "##vso[task.setvariable variable=NUM_CPUS]$NUM_CPUS"
displayName: Set up environment
- bash: |
echo "INTERP=${INTERP}"
echo "PKG_NAME=${PKG_NAME}"
echo "PKG_NAME_ALT=${PKG_NAME_ALT}"
echo "PKG_NAME_DIR=${PKG_NAME_DIR}"
echo "PKG_VERSION=${PKG_VERSION}"
echo "PYTHON_SITE_PACKAGES=${PYTHON_SITE_PACKAGES}"
echo "REPO_DIR=${REPO_DIR}"
echo "EXTRA_DIR=${EXTRA_DIR}"
echo "SBIN_DIR=${SBIN_DIR}"
echo "RESULTS_DIR=${RESULTS_DIR}"
echo "SOURCE_DIR=${SOURCE_DIR}"
echo "TRACER_DIR=${TRACER_DIR}"
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}"
echo "PYTHONPATH=${PYTHONPATH}"
echo "COV_FILE=${COV_FILE}"
echo "LANG=${LANG}"
echo "NUM_CPUS=${NUM_CPUS}"
displayName: Print environment
- bash: |
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
python get-pip.py
pip install --upgrade setuptools
pip --version
rm -rf get-pip.py
displayName: Upgrade pip
- bash: |
sudo apt-get install hunspell
curl https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz -o shellcheck.tar.xz
tar xf shellcheck.tar.xz
export PATH="${PWD}/shellcheck-stable:${PATH}"
pip install --upgrade cyhunspell
echo "##vso[task.setvariable variable=PATH]$PATH"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install hunspell and shellcheck (Linux)
- bash:
HOMEBREW_NO_AUTO_UPDATE=1 brew install hunspell shellcheck
condition: eq( variables['Agent.OS'], 'Darwin' )
displayName: Install hunspell and shellcheck (macOS)
- powershell: |
# hunspell
Invoke-WebRequest -OutFile .\hunspell.zip -Uri "https://cfhcable.dl.sourceforge.net/project/ezwinports/hunspell-1.3.2-3-w32-bin.zip" -Headers @{"Upgrade-Insecure-Requests"="1"; "DNT"="1"; "User-Agent"="Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"; "Accept"="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"; "Referer"="https://sourceforge.net/projects/ezwinports/files/hunspell-1.3.2-3-w32-bin.zip/download"; "Accept-Encoding"="gzip, deflate, br"; "Accept-Language"="en-US,en;q=0.9"; "Cookie"="_ga=GA1.2.960330120.1549548167; _gid=GA1.2.253466023.1549548167; _scp=1549548167671.1888783760; _scs=1549548167673.1210103215; __gads=ID=13f7ab9f14784468:T=1549548167:S=ALNI_MatktSjb46-gGUtEr9aCuvgPeSgLQ"}
7z e .\hunspell.zip -o~\hunspell
# shellcheck
iex (new-object net.webclient).downloadstring('https://get.scoop.sh')
scoop install shellcheck
# Update environment
Write-Host "##vso[task.setvariable variable=PATH]${env:PATH};~\hunspell;~\hunspell\bin;~\scoop\shims;~\scoop\apps\shellcheck\current"
condition: eq( variables['Agent.OS'], 'Windows_NT' )
displayName: Install hunspell and shellcheck (Windows)
- script: shellcheck --version
displayName: Verify shellcheck in path
- script: hunspell --version
displayName: Verify hunspell in path
- bash: |
mkdir -p ${RESULTS_DIR}/testresults
mkdir -p ${RESULTS_DIR}/codecoverage
mkdir -p ${ARTIFACTS_DIR}
displayName: Create directories
- bash: |
"${REPO_DIR}"/pypkg/fix_windows_symlinks.py
displayName: Fix git Windows symbolic links
condition: eq( variables['Agent.OS'], 'Windows_NT' )
- bash: |
python setup.py sdist --formats=zip
export PKG_FILE="${PWD}/dist/${PKG_NAME}-${PKG_VERSION}.zip"
cd "${PYTHON_SITE_PACKAGES}" || exit 1
MERGE_REQUIREMENTS=1 pip install --upgrade "${PKG_FILE}"
displayName: Install package
- bash: |
echo "REPO_DIR=${REPO_DIR}"
ls "${REPO_DIR}"
echo "SOURCE_DIR=${SOURCE_DIR}"
ls "${SOURCE_DIR}"
echo "EXTRA_DIR=${EXTRA_DIR}"
ls "${EXTRA_DIR}"
if [ -d "${SBIN_DIR}" ]; then
echo "SBIN_DIR=${SBIN_DIR}"
ls "${SBIN_DIR}"
fi
if [ -d "${EXTRA_DIR}/data" ]; then
echo "EXTRA_DIR/data=${EXTRA_DIR}/data"
ls "${EXTRA_DIR}/data"
fi
if [ -d "${EXTRA_DIR}/tests" ]; then
echo "EXTRA_DIR/tests=${EXTRA_DIR}/tests"
ls "${EXTRA_DIR}/tests"
fi
if [ -d "${EXTRA_DIR}/tests/support" ]; then
echo "EXTRA_DIR/tests=${EXTRA_DIR}/tests/support"
ls "${EXTRA_DIR}/tests/support"
fi
if [ -d "${EXTRA_DIR}/docs" ]; then
echo "EXTRA_DIR/docs=${EXTRA_DIR}/docs"
ls "${EXTRA_DIR}/docs"
fi
if [ -d "${EXTRA_DIR}/docs/support" ]; then
echo "EXTRA_DIR/docs=${EXTRA_DIR}/docs/support"
ls "${EXTRA_DIR}/docs/support"
fi
displayName: List relevant directories
- bash: |
python "${SBIN_DIR}"/coveragerc_manager.py 'ci' 1 ${INTERP} ${PYTHON_SITE_PACKAGES}
cat "${COV_FILE}"
displayName: Create coverage config file
- bash: |
if [ -f "${EXTRA_DIR}"/data/linux-tasks.sh ]; then
"${EXTRA_DIR}"/data/linux-tasks.sh
fi
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Project-specific OS-specific tasks (Linux)
- bash: |
if [ -f "${EXTRA_DIR}"/data/macos-tasks.sh ]; then
"${EXTRA_DIR}"/data/macos-tasks.sh
fi
condition: eq( variables['Agent.OS'], 'Darwin' )
displayName: Project-specific OS-specific tasks (macOS)
- bash: |
if [ -f "${EXTRA_DIR}"/data/windows-tasks.sh ]; then
"${EXTRA_DIR}"/data/windows-tasks.sh
fi
condition: eq( variables['Agent.OS'], 'Windows_NT' )
displayName: Project-specific OS-specific tasks (Windows)
- bash: |
# Auto-detect installed plugins
export PYLINT_PLUGINS_DIR=$(python -c "from __future__ import print_function;import os;plugin_dir = os.path.join(os.environ.get('EXTRA_DIR', ''), 'pylint_plugins').replace(os.sep,'/');print(plugin_dir if os.path.isdir(plugin_dir) else '')")
export PYLINT_PLUGINS_LIST=$(python -c "from __future__ import print_function;import glob; import os; sdir = os.environ.get('PYLINT_PLUGINS_DIR', ''); print(','.join([os.path.basename(fname).replace('.py', '') for fname in glob.glob(os.path.join(sdir, '*.py')) if not os.path.basename(fname).startswith('common')]) if sdir else '')")
export PYLINT_CLI_APPEND=$(python -c "from __future__ import print_function; import os; svar=os.environ.get('PYLINT_PLUGINS_LIST', ''); print('--load-plugins='+svar if svar else '')")
export PYTHONPATH="${PYTHONPATH}:${PYLINT_PLUGINS_DIR}"
export PYTHONPATH="$("${SBIN_DIR}"/winnorm_path.py "${PYTHONPATH}")"
export DICPATH="${PYLINT_PLUGINS_DIR}/dicts"
echo "PYLINT_PLUGINS_DIR=${PYLINT_PLUGINS_DIR}"
echo "PYLINT_PLUGINS_LIST=${PYLINT_PLUGINS_LIST}"
echo "PYLINT_CLI_APPEND=${PYLINT_CLI_APPEND}"
echo "LANG=${LANG}"
echo "DICPATH=${DICPATH}"
if [ -f "${EXTRA_DIR}/data/exclude-linting" ]; then
export PKG_LINT_FILES="$("${SBIN_DIR}"/get_pylint_files.py -r "${REPO_DIR}" -s "${SOURCE_DIR}" -e "${EXTRA_DIR}" -x "${EXTRA_DIR}"/data/exclude-linting)"
else
export PKG_LINT_FILES="$("${SBIN_DIR}"/get_pylint_files.py -r "${REPO_DIR}" -s "${SOURCE_DIR}" -e "${EXTRA_DIR}")"
fi
echo "Files to lint"
for fname in ${PKG_LINT_FILES}; do
echo "${fname}"
done
pylint --rcfile="${EXTRA_DIR}/.pylintrc" -j"${NUM_CPUS}" "${PYLINT_CLI_APPEND}" --output-format=colorized --reports=no --score=no ${PKG_LINT_FILES} || exit 1
pydocstyle --config="${EXTRA_DIR}/.pydocstyle" ${PKG_LINT_FILES} || exit 1
displayName: Check code standards compliance
- bash: |
cd "${EXTRA_DIR}"/tests || exit 1
export PYTHONPATH="$("${SBIN_DIR}"/winnorm_path.py "${PYTHONPATH}")"
export ARTIFACTS_DIR="$("${SBIN_DIR}"/winnorm_path.py "${ARTIFACTS_DIR}")"
if pytest --collect-only --doctest-glob="*.rst" "${EXTRA_DIR}"/docs &> /dev/null; then
echo "Testing reStructuredText files"
pytest -n "${NUM_CPUS}" --doctest-glob="*.rst" "${EXTRA_DIR}"/docs || exit 1
fi
if pytest --collect-only --doctest-modules "${SOURCE_DIR}" &> /dev/null; then
echo "Testing docstrings"
pytest -n "${NUM_CPUS}" --doctest-modules "${SOURCE_DIR}" || exit 1
fi
pytest -n "${NUM_CPUS}" --junitxml="${RESULTS_DIR}"/testresults/test-results.xml --cov-config="${COV_FILE}" --cov="${SOURCE_DIR}" --cov-report=xml --cov-report=html
displayName: Unit tests
- bash: |
export PYTHONPATH="$("${SBIN_DIR}"/winnorm_path.py "${PYTHONPATH}")"
if [ -f "${SBIN_DIR}"/build_docs.py ]; then
"${SBIN_DIR}"/build_docs.py -r -t -n "${NUM_CPUS}" -d "$("${SBIN_DIR}"//winnorm_path.py "${SOURCE_DIR}")" || exit 1
fi
displayName: Documentation rebuild test
- task: ArchiveFiles@2
condition: failed()
inputs:
rootFolderOrFile: '$(ARTIFACTS_DIR)'
includeRootFolder: true
archiveType: 'zip'
archiveFile: '$(Build.ArtifactStagingDirectory)/$(aname)-$(pyname).zip'
replaceExistingArchive: true
- task: PublishBuildArtifacts@1
condition: failed()
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: build_artifacts
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testResultsFiles: '$(RESULTS_DIR)/testresults/**/test-*.xml'
testRunTitle: 'Publish test results for Python $(python.version)'
- task: PublishCodeCoverageResults@1
condition: succeededOrFailed()
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(RESULTS_DIR)/codecoverage/**/coverage.xml'
reportDirectory: '$(RESULTS_DIR)/codecoverage/**/htmlcov'