From 5049d07e5929e641e3fef2ef94ae4a97aa89f3fe Mon Sep 17 00:00:00 2001 From: David Staheli Date: Tue, 4 Sep 2018 17:31:57 -0400 Subject: [PATCH 1/9] Use new YAML syntax and agent pools for VSTS builds --- .vsts/docs-release.yml | 4 ++-- .vsts/docs.yml | 4 ++-- .vsts/linux-buildbot.yml | 5 ++--- .vsts/linux-coverage.yml | 4 ++-- .vsts/linux-pr.yml | 4 ++-- .vsts/macos-buildbot.yml | 4 ++-- .vsts/macos-pr.yml | 4 ++-- .vsts/windows-buildbot.yml | 7 ++++--- .vsts/windows-pr.yml | 7 ++++--- 9 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.vsts/docs-release.yml b/.vsts/docs-release.yml index e90428a42494e0a..5c4a9089f073b33 100644 --- a/.vsts/docs-release.yml +++ b/.vsts/docs-release.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted Linux Preview +pool: + vmImage: Ubuntu 16.04 #variables: diff --git a/.vsts/docs.yml b/.vsts/docs.yml index 93a7282f770ab4e..8c1217f116804cf 100644 --- a/.vsts/docs.yml +++ b/.vsts/docs.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted Linux Preview +pool: + vmImage: Ubuntu 16.04 trigger: branches: diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index fc2c8ca2486e1b4..3277c2981c13bf3 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted Linux Preview +pool: + vmImage: Ubuntu 16.04 trigger: branches: @@ -22,7 +22,6 @@ variables: OPENSSL: 1.1.0g OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - steps: - checkout: self clean: true diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index 1112555ab93f9b5..d64b039c42415e1 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted Linux Preview +pool: + vmImage: Ubuntu 16.04 trigger: branches: diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index 145ebb38016a612..f69a2bd52c252bf 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted Linux Preview +pool: + vmImage: Ubuntu 16.04 trigger: branches: diff --git a/.vsts/macos-buildbot.yml b/.vsts/macos-buildbot.yml index d9b2297283b0a86..fbbc63daa2ec3b4 100644 --- a/.vsts/macos-buildbot.yml +++ b/.vsts/macos-buildbot.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted macOS Preview +pool: + vmImage: macOS 10.13 trigger: branches: diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml index a3fd4487ed3a703..a25ee40a1983dbc 100644 --- a/.vsts/macos-pr.yml +++ b/.vsts/macos-pr.yml @@ -3,8 +3,8 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted macOS Preview +pool: + vmImage: macOS 10.13 trigger: branches: diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml index 5ec4522796cea90..230f10a7979fec8 100644 --- a/.vsts/windows-buildbot.yml +++ b/.vsts/windows-buildbot.yml @@ -3,9 +3,10 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted VS2017 - parallel: 2 +pool: + vmImage: 'VS2017-Win2016' +strategy: + maxParallel: 2 matrix: amd64: buildOpt: -p x64 diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml index 3dd5609a32e514d..793fbdd4e58e66e 100644 --- a/.vsts/windows-pr.yml +++ b/.vsts/windows-pr.yml @@ -3,9 +3,10 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -queue: - name: Hosted VS2017 - parallel: 2 +pool: + vmImage: 'VS2017-Win2016' +strategy: + maxParallel: 2 matrix: amd64: buildOpt: -p x64 From 11ce1e5c68c4a04a49fc961a4807bed90fdab979 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Tue, 4 Sep 2018 17:55:18 -0400 Subject: [PATCH 2/9] Add named jobs in YAML files for easier correlation in build logs --- .vsts/docs-release.yml | 79 ++++++++-------- .vsts/docs.yml | 109 ++++++++++----------- .vsts/linux-buildbot.yml | 115 +++++++++++----------- .vsts/linux-coverage.yml | 189 +++++++++++++++++++------------------ .vsts/linux-pr.yml | 189 +++++++++++++++++++------------------ .vsts/macos-buildbot.yml | 67 ++++++------- .vsts/macos-pr.yml | 103 ++++++++++---------- .vsts/windows-buildbot.yml | 93 +++++++++--------- .vsts/windows-pr.yml | 117 ++++++++++++----------- 9 files changed, 544 insertions(+), 517 deletions(-) diff --git a/.vsts/docs-release.yml b/.vsts/docs-release.yml index 5c4a9089f073b33..87441d06c625c89 100644 --- a/.vsts/docs-release.yml +++ b/.vsts/docs-release.yml @@ -3,41 +3,44 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: Ubuntu 16.04 - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full - displayName: 'Install LaTeX' - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - -- script: python -m pip install sphinx blurb python-docs-theme - displayName: 'Install build dependencies' - -- script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - -- task: PublishBuildArtifacts@1 - displayName: 'Publish build' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: build - publishLocation: Container - -- task: PublishBuildArtifacts@1 - displayName: 'Publish dist' - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/dist' - ArtifactName: dist - publishLocation: Container +jobs: + +- job: docsRelease + pool: + vmImage: 'Ubuntu 16.04' + + #variables: + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: sudo apt-get update && sudo apt-get install -qy --force-yes texlive-full + displayName: 'Install LaTeX' + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + + - script: python -m pip install sphinx blurb python-docs-theme + displayName: 'Install build dependencies' + + - script: make dist PYTHON=python SPHINXBUILD='python -m sphinx' BLURB='python -m blurb' + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish build' + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/build' + ArtifactName: build + publishLocation: Container + + - task: PublishBuildArtifacts@1 + displayName: 'Publish dist' + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/dist' + ArtifactName: dist + publishLocation: Container diff --git a/.vsts/docs.yml b/.vsts/docs.yml index 8c1217f116804cf..7c96853ad49eaa2 100644 --- a/.vsts/docs.yml +++ b/.vsts/docs.yml @@ -3,56 +3,59 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: Ubuntu 16.04 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - include: - - Doc/* - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qE '(\.rst$|^Doc|^Misc)' - then - echo "No docs were updated, stopping build process." - echo "##vso[task.setvariable variable=NoDocs]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -- task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme - displayName: 'Install build dependencies' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- script: make check suspicious html PYTHON=python - workingDirectory: '$(build.sourcesDirectory)/Doc' - displayName: 'Build documentation' - condition: and(succeeded(), ne(variables['NoDocs'], 'true')) - -- task: PublishBuildArtifacts@1 - displayName: 'Publish build' - condition: and(and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')), ne(variables['NoDocs'], 'true')) - inputs: - PathToPublish: '$(build.sourcesDirectory)/Doc/build' - ArtifactName: build - publishLocation: Container +jobs: + +- job: docs + pool: + vmImage: 'Ubuntu 16.04' + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + include: + - Doc/* + + #variables: + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: | + git fetch -q origin $(system.pullRequest.targetBranch) + if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qE '(\.rst$|^Doc|^Misc)' + then + echo "No docs were updated, stopping build process." + echo "##vso[task.setvariable variable=NoDocs]true" + exit + fi + displayName: Detect doc-only changes + condition: and(succeeded(), variables['system.pullRequest.targetBranch']) + + - task: UsePythonVersion@0 + displayName: 'Use Python 3.6 or later' + inputs: + versionSpec: '>=3.6' + condition: and(succeeded(), ne(variables['NoDocs'], 'true')) + + - script: python -m pip install sphinx~=1.6.1 blurb python-docs-theme + displayName: 'Install build dependencies' + condition: and(succeeded(), ne(variables['NoDocs'], 'true')) + + - script: make check suspicious html PYTHON=python + workingDirectory: '$(build.sourcesDirectory)/Doc' + displayName: 'Build documentation' + condition: and(succeeded(), ne(variables['NoDocs'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: 'Publish build' + condition: and(and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')), ne(variables['NoDocs'], 'true')) + inputs: + PathToPublish: '$(build.sourcesDirectory)/Doc/build' + ArtifactName: build + publishLocation: Container diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index 3277c2981c13bf3..c7c4b87dd1b1960 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -3,69 +3,72 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: Ubuntu 16.04 +jobs: -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* +- job: linuxBuildBot + pool: + vmImage: 'Ubuntu 16.04' -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* -steps: -- checkout: self - clean: true - fetchDepth: 5 + variables: + # Copy-pasted from linux-deps.yml until template support arrives + OPENSSL: 1.1.0g + OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" -#- template: linux-deps.yml + steps: + - checkout: self + clean: true + fetchDepth: 5 -# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md -# For now, we copy/paste the steps -- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update - displayName: 'Update apt-get lists' + #- template: linux-deps.yml -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' -- script: > - sudo apt-get -yq install - build-essential - zlib1g-dev - libbz2-dev - liblzma-dev - libncurses5-dev - libreadline6-dev - libsqlite3-dev - libssl-dev - libgdbm-dev - tk-dev - lzma - lzma-dev - liblzma-dev - libffi-dev - uuid-dev - xvfb - displayName: 'Install dependencies' -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' + # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md + # For now, we copy/paste the steps + - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + displayName: 'Update apt-get lists' -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' + - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) + displayName: 'Add $(OPENSSL_DIR) to PATH' + - script: > + sudo apt-get -yq install + build-essential + zlib1g-dev + libbz2-dev + liblzma-dev + libncurses5-dev + libreadline6-dev + libsqlite3-dev + libssl-dev + libgdbm-dev + tk-dev + lzma + lzma-dev + liblzma-dev + libffi-dev + uuid-dev + xvfb + displayName: 'Install dependencies' + - script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux + displayName: 'python multissltests.py' -- script: make -s -j4 - displayName: 'Build CPython' + - script: ./configure --with-pydebug + displayName: 'Configure CPython (debug)' -- script: make pythoninfo - displayName: 'Display build info' + - script: make -s -j4 + displayName: 'Build CPython' -- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" - displayName: 'Tests' + - script: make pythoninfo + displayName: 'Display build info' + + - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + displayName: 'Tests' diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index d64b039c42415e1..bacc649383f9ae4 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -3,96 +3,99 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: Ubuntu 16.04 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -#- template: linux-deps.yml - -# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md -# For now, we copy/paste the steps -- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update - displayName: 'Update apt-get lists' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' -- script: > - sudo apt-get -yq install - build-essential - zlib1g-dev - libbz2-dev - liblzma-dev - libncurses5-dev - libreadline6-dev - libsqlite3-dev - libssl-dev - libgdbm-dev - tk-dev - lzma - lzma-dev - liblzma-dev - libffi-dev - uuid-dev - xvfb - displayName: 'Install dependencies' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage - displayName: 'Set up virtual environment' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: ./venv/bin/python -m test.pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures - displayName: 'Tests with coverage' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) - displayName: 'Publish code coverage results' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) +jobs: + +- job: linuxCoverage + pool: + vmImage: 'Ubuntu 16.04' + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + variables: + # Copy-pasted from linux-deps.yml until template support arrives + OPENSSL: 1.1.0g + OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: | + git fetch -q origin $(system.pullRequest.targetBranch) + if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated, stopping build process." + echo "##vso[task.setvariable variable=DocOnly]true" + exit + fi + displayName: Detect doc-only changes + condition: and(succeeded(), variables['system.pullRequest.targetBranch']) + + #- template: linux-deps.yml + + # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md + # For now, we copy/paste the steps + - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + displayName: 'Update apt-get lists' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) + displayName: 'Add $(OPENSSL_DIR) to PATH' + - script: > + sudo apt-get -yq install + build-essential + zlib1g-dev + libbz2-dev + liblzma-dev + libncurses5-dev + libreadline6-dev + libsqlite3-dev + libssl-dev + libgdbm-dev + tk-dev + lzma + lzma-dev + liblzma-dev + libffi-dev + uuid-dev + xvfb + displayName: 'Install dependencies' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + - script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux + displayName: 'python multissltests.py' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + + - script: ./configure --with-pydebug + displayName: 'Configure CPython (debug)' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make -s -j4 + displayName: 'Build CPython' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage + displayName: 'Set up virtual environment' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: ./venv/bin/python -m test.pythoninfo + displayName: 'Display build info' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: xvfb-run ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures + displayName: 'Tests with coverage' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) + displayName: 'Publish code coverage results' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index f69a2bd52c252bf..f6393f573a3565f 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -3,96 +3,99 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: Ubuntu 16.04 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Copy-pasted from linux-deps.yml until template support arrives - OPENSSL: 1.1.0g - OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -#- template: linux-deps.yml - -# See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md -# For now, we copy/paste the steps -- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update - displayName: 'Update apt-get lists' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: echo ##vso[task.prependpath]$(OPENSSL_DIR) - displayName: 'Add $(OPENSSL_DIR) to PATH' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: > - sudo apt-get -yq install - build-essential - zlib1g-dev - libbz2-dev - liblzma-dev - libncurses5-dev - libreadline6-dev - libsqlite3-dev - libssl-dev - libgdbm-dev - tk-dev - lzma - lzma-dev - liblzma-dev - libffi-dev - uuid-dev - xvfb - displayName: 'Install dependencies' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux - displayName: 'python multissltests.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -# Run patchcheck and fail if anything is discovered -- script: ./python Tools/scripts/patchcheck.py --travis true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" - displayName: 'Tests' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) +jobs: + +- job: linuxPR + pool: + vmImage: 'Ubuntu 16.04' + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + variables: + # Copy-pasted from linux-deps.yml until template support arrives + OPENSSL: 1.1.0g + OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: | + git fetch -q origin $(system.pullRequest.targetBranch) + if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' + then + echo "Only docs were updated, stopping build process." + echo "##vso[task.setvariable variable=DocOnly]true" + exit + fi + displayName: Detect doc-only changes + condition: and(succeeded(), variables['system.pullRequest.targetBranch']) + + #- template: linux-deps.yml + + # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md + # For now, we copy/paste the steps + - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + displayName: 'Update apt-get lists' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) + displayName: 'Add $(OPENSSL_DIR) to PATH' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: > + sudo apt-get -yq install + build-essential + zlib1g-dev + libbz2-dev + liblzma-dev + libncurses5-dev + libreadline6-dev + libsqlite3-dev + libssl-dev + libgdbm-dev + tk-dev + lzma + lzma-dev + liblzma-dev + libffi-dev + uuid-dev + xvfb + displayName: 'Install dependencies' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: python3 Tools/ssl/multissltests.py --steps=library --base-directory $(build.sourcesDirectory)/multissl --openssl $(OPENSSL) --system Linux + displayName: 'python multissltests.py' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + + - script: ./configure --with-pydebug + displayName: 'Configure CPython (debug)' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make -s -j4 + displayName: 'Build CPython' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make pythoninfo + displayName: 'Display build info' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + # Run patchcheck and fail if anything is discovered + - script: ./python Tools/scripts/patchcheck.py --travis true + displayName: 'Run patchcheck.py' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + displayName: 'Tests' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/macos-buildbot.yml b/.vsts/macos-buildbot.yml index fbbc63daa2ec3b4..d6f7c242db20a0c 100644 --- a/.vsts/macos-buildbot.yml +++ b/.vsts/macos-buildbot.yml @@ -3,35 +3,38 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: macOS 10.13 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts - displayName: 'Configure CPython (debug)' - -- script: make -s -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu" - displayName: 'Tests' +jobs: + +- job: macOSBuildBot + pool: + vmImage: 'macOS 10.13' + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + #variables: + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts + displayName: 'Configure CPython (debug)' + + - script: make -s -j4 + displayName: 'Build CPython' + + - script: make pythoninfo + displayName: 'Display build info' + + - script: make buildbottest TESTOPTS="-j4 -uall,-cpu" + displayName: 'Tests' diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml index a25ee40a1983dbc..cceaa442fe66436 100644 --- a/.vsts/macos-pr.yml +++ b/.vsts/macos-pr.yml @@ -3,53 +3,56 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: macOS 10.13 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -#variables: - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: | - git fetch -q origin $(system.pullRequest.targetBranch) - changes = $(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) - echo "Files changed:" - echo "$changes" - if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - condition: and(succeeded(), variables['system.pullRequest.targetBranch']) - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts - displayName: 'Configure CPython (debug)' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make -s -j4 - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu" - displayName: 'Tests' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) +jobs: + +- job: macOSPR + pool: + vmImage: 'macOS 10.13' + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + #variables: + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: | + git fetch -q origin $(system.pullRequest.targetBranch) + changes = $(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) + echo "Files changed:" + echo "$changes" + if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + echo "##vso[task.setvariable variable=DocOnly]true" + exit + fi + displayName: Detect doc-only changes + condition: and(succeeded(), variables['system.pullRequest.targetBranch']) + + - script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-vsts + displayName: 'Configure CPython (debug)' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make -s -j4 + displayName: 'Build CPython' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make pythoninfo + displayName: 'Display build info' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: make buildbottest TESTOPTS="-j4 -uall,-cpu" + displayName: 'Tests' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml index 230f10a7979fec8..3fb38dceec69dba 100644 --- a/.vsts/windows-buildbot.yml +++ b/.vsts/windows-buildbot.yml @@ -3,48 +3,51 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: 'VS2017-Win2016' -strategy: - maxParallel: 2 - matrix: - amd64: - buildOpt: -p x64 - outDirSuffix: amd64 - win32: - buildOpt: - outDirSuffix: win32 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Relocate build outputs outside of source directory to make cleaning faster - Py_IntDir: $(Build.BinariesDirectory)\obj - # UNDONE: Do not build to a different directory because of broken tests - Py_OutDir: $(Build.SourcesDirectory)\PCbuild - EXTERNAL_DIR: $(Build.BinariesDirectory)\externals - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 - displayName: 'Tests' - env: - PREFIX: $(Py_OutDir)\$(outDirSuffix) +jobs: + +- job: windowsBuildBot + pool: + vmImage: 'vs2017-win2016' + strategy: + maxParallel: 2 + matrix: + amd64: + buildOpt: -p x64 + outDirSuffix: amd64 + win32: + buildOpt: + outDirSuffix: win32 + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + variables: + # Relocate build outputs outside of source directory to make cleaning faster + Py_IntDir: $(Build.BinariesDirectory)\obj + # UNDONE: Do not build to a different directory because of broken tests + Py_OutDir: $(Build.SourcesDirectory)\PCbuild + EXTERNAL_DIR: $(Build.BinariesDirectory)\externals + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - script: PCbuild\build.bat -e $(buildOpt) + displayName: 'Build CPython' + + - script: python.bat -m test.pythoninfo + displayName: 'Display build info' + + - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + displayName: 'Tests' + env: + PREFIX: $(Py_OutDir)\$(outDirSuffix) diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml index 793fbdd4e58e66e..6a95f213c8fdfff 100644 --- a/.vsts/windows-pr.yml +++ b/.vsts/windows-pr.yml @@ -3,60 +3,63 @@ name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) -pool: - vmImage: 'VS2017-Win2016' -strategy: - maxParallel: 2 - matrix: - amd64: - buildOpt: -p x64 - outDirSuffix: amd64 - win32: - buildOpt: - outDirSuffix: win32 - -trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - -variables: - # Relocate build outputs outside of source directory to make cleaning faster - Py_IntDir: $(Build.BinariesDirectory)\obj - # UNDONE: Do not build to a different directory because of broken tests - Py_OutDir: $(Build.SourcesDirectory)\PCbuild - EXTERNAL_DIR: $(Build.BinariesDirectory)\externals - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- powershell: | - git fetch -q origin $(System.PullRequest.TargetBranch) - if (-not (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD) | sls -NotMatch '(\.rst$)|(^Doc)|(^Misc)')) { - Write-Host 'Only docs were updated. Skipping build' - Write-Host '##vso[task.setvariable variable=DocOnly]true' - } - displayName: Detect doc-only changes - condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 - displayName: 'Tests' - env: - PREFIX: $(Py_OutDir)\$(outDirSuffix) - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) +jobs: + +- job: windowsPR + pool: + vmImage: 'vs2017-win2016' + strategy: + maxParallel: 2 + matrix: + amd64: + buildOpt: -p x64 + outDirSuffix: amd64 + win32: + buildOpt: + outDirSuffix: win32 + + trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* + + variables: + # Relocate build outputs outside of source directory to make cleaning faster + Py_IntDir: $(Build.BinariesDirectory)\obj + # UNDONE: Do not build to a different directory because of broken tests + Py_OutDir: $(Build.SourcesDirectory)\PCbuild + EXTERNAL_DIR: $(Build.BinariesDirectory)\externals + + steps: + - checkout: self + clean: true + fetchDepth: 5 + + - powershell: | + git fetch -q origin $(System.PullRequest.TargetBranch) + if (-not (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD) | sls -NotMatch '(\.rst$)|(^Doc)|(^Misc)')) { + Write-Host 'Only docs were updated. Skipping build' + Write-Host '##vso[task.setvariable variable=DocOnly]true' + } + displayName: Detect doc-only changes + condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) + + - script: PCbuild\build.bat -e $(buildOpt) + displayName: 'Build CPython' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: python.bat -m test.pythoninfo + displayName: 'Display build info' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + + - script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + displayName: 'Tests' + env: + PREFIX: $(Py_OutDir)\$(outDirSuffix) + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) From 50f0cee869fc527ee5e38dd93fee7dbe2bb6e955 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 19:58:49 -0400 Subject: [PATCH 3/9] Added a news.d blurb --- Misc/NEWS.d/next/Build/2018-09-05-19-58-20.bpo-34582.8Shyl4.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2018-09-05-19-58-20.bpo-34582.8Shyl4.rst diff --git a/Misc/NEWS.d/next/Build/2018-09-05-19-58-20.bpo-34582.8Shyl4.rst b/Misc/NEWS.d/next/Build/2018-09-05-19-58-20.bpo-34582.8Shyl4.rst new file mode 100644 index 000000000000000..c60d182bb971df1 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2018-09-05-19-58-20.bpo-34582.8Shyl4.rst @@ -0,0 +1,2 @@ +The latest YAML syntax is adopted for VSTS build pipelines. Also, new hosted +agent pools are used for better performance. From f232e59b87324474f7c89e8eff6e98541755a1ba Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 20:08:42 -0400 Subject: [PATCH 4/9] Fix location of triggers in YAML files --- .vsts/docs.yml | 19 +++++++++---------- .vsts/linux-buildbot.yml | 21 ++++++++++----------- .vsts/linux-coverage.yml | 21 ++++++++++----------- .vsts/linux-pr.yml | 21 ++++++++++----------- .vsts/macos-buildbot.yml | 21 ++++++++++----------- .vsts/macos-pr.yml | 21 ++++++++++----------- .vsts/windows-buildbot.yml | 21 ++++++++++----------- .vsts/windows-pr.yml | 21 ++++++++++----------- 8 files changed, 79 insertions(+), 87 deletions(-) diff --git a/.vsts/docs.yml b/.vsts/docs.yml index 7c96853ad49eaa2..9b84545415470cd 100644 --- a/.vsts/docs.yml +++ b/.vsts/docs.yml @@ -2,6 +2,15 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + include: + - Doc/* jobs: @@ -9,16 +18,6 @@ jobs: pool: vmImage: 'Ubuntu 16.04' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - include: - - Doc/* - #variables: steps: diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index c7c4b87dd1b1960..2475a3b1bc0d5b3 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -9,17 +19,6 @@ jobs: pool: vmImage: 'Ubuntu 16.04' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - variables: # Copy-pasted from linux-deps.yml until template support arrives OPENSSL: 1.1.0g diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index bacc649383f9ae4..cb87fe3bd79bc87 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -9,17 +19,6 @@ jobs: pool: vmImage: 'Ubuntu 16.04' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - variables: # Copy-pasted from linux-deps.yml until template support arrives OPENSSL: 1.1.0g diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index f6393f573a3565f..c589de0db68547f 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -9,17 +19,6 @@ jobs: pool: vmImage: 'Ubuntu 16.04' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - variables: # Copy-pasted from linux-deps.yml until template support arrives OPENSSL: 1.1.0g diff --git a/.vsts/macos-buildbot.yml b/.vsts/macos-buildbot.yml index d6f7c242db20a0c..576ea332363ab39 100644 --- a/.vsts/macos-buildbot.yml +++ b/.vsts/macos-buildbot.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -9,17 +19,6 @@ jobs: pool: vmImage: 'macOS 10.13' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - #variables: steps: diff --git a/.vsts/macos-pr.yml b/.vsts/macos-pr.yml index cceaa442fe66436..c0fe111e08443d5 100644 --- a/.vsts/macos-pr.yml +++ b/.vsts/macos-pr.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -9,17 +19,6 @@ jobs: pool: vmImage: 'macOS 10.13' - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - #variables: steps: diff --git a/.vsts/windows-buildbot.yml b/.vsts/windows-buildbot.yml index 3fb38dceec69dba..7dff20b42019141 100644 --- a/.vsts/windows-buildbot.yml +++ b/.vsts/windows-buildbot.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -18,17 +28,6 @@ jobs: buildOpt: outDirSuffix: win32 - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - variables: # Relocate build outputs outside of source directory to make cleaning faster Py_IntDir: $(Build.BinariesDirectory)\obj diff --git a/.vsts/windows-pr.yml b/.vsts/windows-pr.yml index 6a95f213c8fdfff..1b933936ed7c262 100644 --- a/.vsts/windows-pr.yml +++ b/.vsts/windows-pr.yml @@ -2,6 +2,16 @@ # https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted.md name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.rr) +trigger: + branches: + include: + - master + - 3.7 + - 3.6 + paths: + exclude: + - Doc/* + - Tools/* jobs: @@ -18,17 +28,6 @@ jobs: buildOpt: outDirSuffix: win32 - trigger: - branches: - include: - - master - - 3.7 - - 3.6 - paths: - exclude: - - Doc/* - - Tools/* - variables: # Relocate build outputs outside of source directory to make cleaning faster Py_IntDir: $(Build.BinariesDirectory)\obj From 04a412d1dc76724e6c56e9c9c1b62a18130d4990 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 20:22:46 -0400 Subject: [PATCH 5/9] Add sudo when writing to /etc/* --- .vsts/linux-buildbot.yml | 2 +- .vsts/linux-coverage.yml | 2 +- .vsts/linux-deps.yml | 2 +- .vsts/linux-pr.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index 2475a3b1bc0d5b3..c78b41221ff69ba 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -33,7 +33,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index cb87fe3bd79bc87..7e23142c8faa677 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -44,7 +44,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/linux-deps.yml b/.vsts/linux-deps.yml index 83b0b5961721d0c..e806ee317521e02 100644 --- a/.vsts/linux-deps.yml +++ b/.vsts/linux-deps.yml @@ -9,7 +9,7 @@ parameters: OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" steps: -- script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update +- script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index c589de0db68547f..4192be6489e7ff2 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -44,7 +44,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) From e3fbcc3104b62ed6f2cec9283ed8a9451c91824d Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 20:36:08 -0400 Subject: [PATCH 6/9] Reattempt use of sudo for writing to /etc/* --- .vsts/linux-buildbot.yml | 2 +- .vsts/linux-coverage.yml | 2 +- .vsts/linux-deps.yml | 2 +- .vsts/linux-pr.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index c78b41221ff69ba..4f350c97c55e3b2 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -33,7 +33,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index 7e23142c8faa677..cedb2630e6f23f5 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -44,7 +44,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/linux-deps.yml b/.vsts/linux-deps.yml index e806ee317521e02..ff199f93fb3a6fa 100644 --- a/.vsts/linux-deps.yml +++ b/.vsts/linux-deps.yml @@ -9,7 +9,7 @@ parameters: OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" steps: -- script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update +- script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index 4192be6489e7ff2..8b0d5eef1cbc0cd 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -44,7 +44,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) From 04a6113fb0420dd8e6830a3c50501f24434175b7 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 20:55:33 -0400 Subject: [PATCH 7/9] Test lsattr --- .vsts/linux-pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index 8b0d5eef1cbc0cd..d9c4dd33f1975dd 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -44,7 +44,9 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: | + sudo lsattr /etc/apt/sources.list.d/python.list + echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) From 0f85041cab43a9ab87ae7066c2f2880015cd8bdd Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 21:02:16 -0400 Subject: [PATCH 8/9] Try not updating apt-get lists --- .vsts/linux-pr.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index d9c4dd33f1975dd..8a16565b4fc3851 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -44,11 +44,9 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: | - sudo lsattr /etc/apt/sources.list.d/python.list - echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update - displayName: 'Update apt-get lists' - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) +# - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update +# displayName: 'Update apt-get lists' +# condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) displayName: 'Add $(OPENSSL_DIR) to PATH' From 649c6ca8d9fc072dfb391bf4998c7f71918c29c3 Mon Sep 17 00:00:00 2001 From: David Staheli Date: Wed, 5 Sep 2018 21:12:26 -0400 Subject: [PATCH 9/9] Simply do an apt-get update --- .vsts/linux-buildbot.yml | 2 +- .vsts/linux-coverage.yml | 2 +- .vsts/linux-deps.yml | 2 +- .vsts/linux-pr.yml | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.vsts/linux-buildbot.yml b/.vsts/linux-buildbot.yml index 4f350c97c55e3b2..976f0e7a365686b 100644 --- a/.vsts/linux-buildbot.yml +++ b/.vsts/linux-buildbot.yml @@ -33,7 +33,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-coverage.yml b/.vsts/linux-coverage.yml index cedb2630e6f23f5..67400554eef4077 100644 --- a/.vsts/linux-coverage.yml +++ b/.vsts/linux-coverage.yml @@ -44,7 +44,7 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps - - script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update + - script: sudo apt-get update displayName: 'Update apt-get lists' condition: and(succeeded(), ne(variables['DocOnly'], 'true')) diff --git a/.vsts/linux-deps.yml b/.vsts/linux-deps.yml index ff199f93fb3a6fa..20e638d44ff9f82 100644 --- a/.vsts/linux-deps.yml +++ b/.vsts/linux-deps.yml @@ -9,7 +9,7 @@ parameters: OPENSSL_DIR: "$(build.sourcesDirectory)/multissl/openssl/$(OPENSSL)" steps: -- script: echo "sudo deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update +- script: sudo apt-get update displayName: 'Update apt-get lists' - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) diff --git a/.vsts/linux-pr.yml b/.vsts/linux-pr.yml index 8a16565b4fc3851..6d90b1574c4e106 100644 --- a/.vsts/linux-pr.yml +++ b/.vsts/linux-pr.yml @@ -44,9 +44,9 @@ jobs: # See https://github.com/Microsoft/vsts-agent/blob/master/docs/preview/yamlgettingstarted-templates.md # For now, we copy/paste the steps -# - script: sudo echo "deb-src http://archive.ubuntu.com/ubuntu/ xenial main" > /etc/apt/sources.list.d/python.list && sudo apt-get update -# displayName: 'Update apt-get lists' -# condition: and(succeeded(), ne(variables['DocOnly'], 'true')) + - script: sudo apt-get update + displayName: 'Update apt-get lists' + condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - script: echo ##vso[task.prependpath]$(OPENSSL_DIR) displayName: 'Add $(OPENSSL_DIR) to PATH'