From 699376ada01dc6223e5365e79ae11c79502fd2ef Mon Sep 17 00:00:00 2001 From: John Erickson Date: Mon, 6 May 2019 11:15:52 -0700 Subject: [PATCH 01/57] Add Azure Pipelines --- .azure-pipelines/auto.fanout.yml | 521 ++++++++++++++++++ .azure-pipelines/auto.yml | 361 ++++++++++++ .azure-pipelines/master.yml | 22 + .azure-pipelines/pr.yml | 26 + .azure-pipelines/steps/linux.yml | 42 ++ .azure-pipelines/steps/macos.yml | 47 ++ .azure-pipelines/steps/run-script.yml | 24 + .azure-pipelines/steps/show-disk-usage.yml | 5 + .../steps/show-environment-variables.yml | 3 + .../steps/verify-publish-toolstate.yml | 9 + .azure-pipelines/steps/windows.yml | 160 ++++++ .azure-pipelines/test.yml | 131 +++++ .azure-pipelines/try.yml | 29 + README.md | 2 + appveyor.yml | 8 +- src/bootstrap/README.md | 3 +- src/bootstrap/mk/Makefile.in | 74 ++- src/bootstrap/native.rs | 9 +- src/bootstrap/util.rs | 4 + .../{i686-gnu => i686-gnu-1}/Dockerfile | 9 +- src/ci/docker/i686-gnu-2/Dockerfile | 21 + src/ci/docker/run.sh | 39 +- src/ci/docker/scripts/sccache.sh | 3 +- src/ci/docker/x86_64-gnu-tools/checktools.sh | 2 +- src/ci/run.sh | 46 +- src/ci/shared.sh | 16 + src/test/run-pass/fds-are-cloexec.rs | 1 + 27 files changed, 1568 insertions(+), 49 deletions(-) create mode 100644 .azure-pipelines/auto.fanout.yml create mode 100644 .azure-pipelines/auto.yml create mode 100644 .azure-pipelines/master.yml create mode 100644 .azure-pipelines/pr.yml create mode 100644 .azure-pipelines/steps/linux.yml create mode 100644 .azure-pipelines/steps/macos.yml create mode 100644 .azure-pipelines/steps/run-script.yml create mode 100644 .azure-pipelines/steps/show-disk-usage.yml create mode 100644 .azure-pipelines/steps/show-environment-variables.yml create mode 100644 .azure-pipelines/steps/verify-publish-toolstate.yml create mode 100644 .azure-pipelines/steps/windows.yml create mode 100644 .azure-pipelines/test.yml create mode 100644 .azure-pipelines/try.yml rename src/ci/docker/{i686-gnu => i686-gnu-1}/Dockerfile (54%) create mode 100644 src/ci/docker/i686-gnu-2/Dockerfile diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml new file mode 100644 index 0000000000000..08cba3c47ca8a --- /dev/null +++ b/.azure-pipelines/auto.fanout.yml @@ -0,0 +1,521 @@ +# +# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. +# +# Notes: +# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled +# and with openssl statically-linked for Linux. +# Here's the build (with a backpointer to source) of where the bits came from: +# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275 + +name: fanout +pr: none +trigger: +- auto + +variables: +- group: caching + +jobs: +# WINDOWS JOBS +- job: Windows + timeoutInMinutes: 600 + variables: + TAR_COMMAND: tar + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/windows.yml + strategy: + matrix: + # 32/64 bit MSVC tests + x86_64-msvc-1: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + x86_64-msvc-2: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + +- job: Windows_build + timeoutInMinutes: 600 + variables: + TAR_COMMAND: tar + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/windows.yml + - script: | + echo on + echo "##vso[task.setvariable variable=TAR_PATH]%TEMP%\drop.tar" + - bash: | + set -x + $TAR_COMMAND --dereference --exclude=.git --exclude=citools $SUBMODULES_EXCLUDES -cf $TEMP/drop.tar . + $TAR_COMMAND --append --file=$TEMP/drop.tar citools + - task: PublishPipelineArtifact@0 + inputs: + artifactName: $(System.JobDisplayName) + targetPath: $(TAR_PATH) + strategy: + matrix: + # 32/64 bit MSVC tests + x86_64-msvc-build: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-test-prepare + +- job: Windows_test + dependsOn: Windows_build + timeoutInMinutes: 600 + variables: + TAR_COMMAND: tar + SKIP_PREP: 1 + KEEP_STAGE: '--keep-stage 0 --keep-stage 1' + SKIP_LLVM_BUILD: 1 + pool: + vmImage: 'vs2017-win2016' + steps: + - checkout: self + fetchDepth: 2 + - script: | + set MSYS_PATH=%CD%\citools\msys64 + set PATH=%MSYS_PATH%\usr\bin;%PATH% + where rev + rev --help + where make + echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin + displayName: Install msys2 + + # # If we need to download a custom MinGW, do so here and set the path + # # appropriately. + # # + # # Note that this *also* means that we're not using what is typically + # # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where + # # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we + # # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe` + # # file exists in there (which it doesn't by default). + # - script: | + # powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" + # 7z x -y %MINGW_ARCHIVE% > nul + # echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin + # condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + # displayName: Download custom MinGW + + # If we're compiling for MSVC then we, like most other distribution builders, + # switch to clang as the compiler. This'll allow us eventually to enable LTO + # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think + # clang has an output mode compatible with MinGW that we need. If it does we + # should switch to clang for MinGW as well! + # + # Note that the LLVM installer is an NSIS installer + # + # Original downloaded here came from + # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe + - script: | + # powershell -Command "iwr -outf LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" + # .\LLVM-7.0.0-win64.exe /S /NCRC /D=C:\clang-rust + set CLANG_DIR=%CD%\citools\clang-rust + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + condition: and(succeeded(), eq(variables['MINGW_URL'],'')) + displayName: Download clang + + # # Here we do a pretty heinous thing which is to mangle the MinGW installation + # # we just had above. Currently, as of this writing, we're using MinGW-w64 + # # builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to + # # be the first version which contains a fix for #40546, builds randomly + # # failing during LLVM due to ar.exe/ranlib.exe failures. + # # + # # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds + # # to contain a regression in gdb (#40184). As a result if we were to use the + # # gdb provided (7.11.1) then we would fail all debuginfo tests. + # # + # # In order to fix spurious failures (pretty high priority) we use 6.3.0. To + # # avoid disabling gdb tests we download an *old* version of gdb, specifically + # # that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb + # # with the 6.2.0 gdb to get tests passing. + # # + # # Note that we don't literally overwrite the gdb.exe binary because it appears + # # to just use gdborig.exe, so that's the binary we deal with instead. + # - script: | + # echo ON + # powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" + # mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe + # condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + # displayName: Override with 6.3.0 gdb with 6.2.0 gdb + + # Otherwise pull in the MinGW installed on appveyor + - script: | + echo Find mingw + set PATH | findstr /i msys + set PATH | findstr /i mingw + echo ##vso[task.prependpath]C:\msys64\mingw%MSYS_BITS%\bin + condition: and(succeeded(), eq(variables['MINGW_URL'],'')) + displayName: Add MinGW to path + + - script: | + copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe + echo ##vso[task.prependpath]C:\Python27amd64 + displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes + + - script: | + REM md sccache + REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + echo ##vso[task.prependpath]%CD%\sccache + displayName: Download and install sccache + + # Note that this is originally from the github releases patch of Ninja + - script: | + REM md ninja + REM powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip" + REM 7z x -oninja 2017-03-15-ninja-win.zip + REM del 2017-03-15-ninja-win.zip + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + echo ##vso[task.prependpath]%CD%\ninja + displayName: Download and install ninja + + - script: | + IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( + IF NOT EXIST build ( + mkdir c:\MORE_SPACE + mklink /J build c:\MORE_SPACE + ) + ) + - script: | + echo on + echo ##vso[task.setvariable variable=TAR_FOLDER_PATH]%TEMP% + - template: steps/show-environment-variables.yml + - task: DownloadPipelineArtifact@1 + inputs: + artifactName: $(ARTIFACT_NAME) + downloadPath: $(TAR_FOLDER_PATH) + # - task: DownloadPipelineArtifact@1 + # inputs: + # artifactName: $(ARTIFACT_NAME) + # downloadPath: $(TAR_FOLDER_PATH) + # buildType: specific + # project: rust + # pipeline: 8 + # buildVersionToDownload: specific + # buildId: 200 + # # https://dev.azure.com/rust-lang-azure/rust/_build/results?buildId=145 + + # - template: steps/macos.yml + - bash: $TAR_COMMAND -tvf $TEMP/drop.tar + - bash: $TAR_COMMAND --touch -xf $TEMP/drop.tar + - bash: rm $TEMP/drop.tar + - script: | + REM mkdir handle + REM powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip" + REM 7z x -ohandle 2017-05-15-Handle.zip + REM del 2017-05-15-Handle.zip + set PATH=%PATH%;%CD%\handle + handle.exe -accepteula -help + echo ##vso[task.setvariable variable=PATH]%PATH% + displayName: Help debug handle issues + - script: | + REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8 + IF "%VCVARS_BAT%" NEQ "" ( + CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" + ) + + where sccache + where rev + set | findstr /v SCCACHE_AZURE_CONNECTION_STRING + + if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc + + sh src/ci/run.sh + env: + CI: true + CI_JOB_NAME: $(System.JobDisplayName) + SRC: . + NO_CCACHE: 1 + + # explicitly decrypt secret variables + # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch + SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) + DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + displayName: Run script + + strategy: + matrix: + # 32/64 bit MSVC tests + x86_64-msvc-A: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-A + x86_64-msvc-B: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-B + x86_64-msvc-C: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-C + x86_64-msvc-D: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-D + x86_64-msvc-E: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-E + x86_64-msvc-F: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-F + x86_64-msvc-G: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-G + x86_64-msvc-H: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + ARTIFACT_NAME: x86_64-msvc-build + SCRIPT: make ci-resume-subset-H + +# macOS JOBS +- job: macOS + timeoutInMinutes: 600 + variables: + TAR_COMMAND: gtar + pool: + vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - template: steps/macos.yml + strategy: + matrix: + x86_64-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + +- job: macOS_build + timeoutInMinutes: 600 + variables: + TAR_COMMAND: gtar + pool: + vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - template: steps/macos.yml + - script: | + set -x + + git submodule + export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') + echo $SUBMODULES_EXCLUDES + + $TAR_COMMAND --exclude=.git $SUBMODULES_EXCLUDES -cf /tmp/drop.tar . + - task: PublishPipelineArtifact@0 + inputs: + artifactName: $(System.JobDisplayName) + targetPath: '/tmp/drop.tar' + + strategy: + matrix: + # macOS builders. These are placed near the beginning because they are very + # slow to run. + + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. + x86_64-apple-build: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + SCRIPT: make ci-test-prepare + +- job: macOS_test + dependsOn: macOS_build + timeoutInMinutes: 600 + variables: + TAR_COMMAND: gtar + SKIP_PREP: 1 + KEEP_STAGE: '--keep-stage 0 --keep-stage 1' + SKIP_LLVM_BUILD: 1 + pool: + vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - bash: brew install gnu-tar + displayName: install a tar that works well + + - template: steps/show-disk-usage.yml + + - task: DownloadPipelineArtifact@1 + inputs: + artifactName: $(ARTIFACT_NAME) + downloadPath: $(System.DefaultWorkingDirectory) + # - task: DownloadPipelineArtifact@1 + # inputs: + # artifactName: $(ARTIFACT_NAME) + # downloadPath: $(System.DefaultWorkingDirectory) + # buildType: specific + # project: rust + # pipeline: 8 + # buildVersionToDownload: specific + # buildId: 200 + + # - template: steps/macos.yml + - template: steps/show-disk-usage.yml + - bash: $TAR_COMMAND -tvf ./drop.tar + - bash: $TAR_COMMAND -xf ./drop.tar + - bash: rm ./drop.tar + - template: steps/show-disk-usage.yml + - bash: | + export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ + mkdir -p $HOME/rustsrc + echo "##vso[task.setvariable variable=PATH;]$PATH" + + curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + chmod +x /usr/local/bin/sccache + + curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin + chmod +x /usr/local/bin/stamp + + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang + echo "##vso[task.setvariable variable=CC]$CC" + + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ + echo "##vso[task.setvariable variable=CXX]$CXX" + + echo "##vso[task.setvariable variable=AR]ar" + displayName: Prep + + - bash: brew install gnu-tar + displayName: install a tar that works well + + - bash: | + brew update + brew install xz + brew install swig + condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist')) + displayName: Install xz and swigw + + - bash: | + export RUN_SCRIPT="src/ci/run.sh" + echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" + displayName: Prepare run script (init and run) + + - template: steps/show-environment-variables.yml + + - template: steps/verify-publish-toolstate.yml + + - template: steps/run-script.yml + + strategy: + matrix: + # macOS builders. These are placed near the beginning because they are very + # slow to run. + + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. + x86_64-apple-A: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-A + x86_64-apple-B: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-B + x86_64-apple-C: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-C + x86_64-apple-D: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-D + x86_64-apple-E: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-E + x86_64-apple-F: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-F + x86_64-apple-G: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-G + x86_64-apple-H: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + ARTIFACT_NAME: x86_64-apple-build + SCRIPT: make ci-resume-subset-H diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml new file mode 100644 index 0000000000000..818b613740e6e --- /dev/null +++ b/.azure-pipelines/auto.yml @@ -0,0 +1,361 @@ +# +# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. +# +# Notes: +# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled +# and with openssl statically-linked for Linux. +# Here's the build (with a backpointer to source) of where the bits came from: +# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275 + +pr: none +# trigger: +# - auto + +variables: +- group: caching + +jobs: +- job: Linux + timeoutInMinutes: 600 + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/linux.yml + strategy: + matrix: + x86_64-gnu-llvm-6.0: + IMAGE: x86_64-gnu-llvm-6.0 + RUST_BACKTRACE: 1 + + dist-x86_64-linux: + IMAGE: dist-x86_64-linux + DEPLOY: 1 + + # "alternate" deployments, these are "nightlies" but have LLVM assertions + # turned on, they're deployed to a different location primarily for + # additional testing. + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 + + # Linux builders, remaining docker images + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu-1: + IMAGE: i686-gnu-1 + i686-gnu-2: + IMAGE: i686-gnu-2 + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various + x86_64-gnu: + IMAGE: x86_64-gnu + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck + mingw-check: + IMAGE: mingw-check + +- job: macOS + timeoutInMinutes: 600 + pool: + vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - template: steps/macos.yml + strategy: + matrix: + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + # macOS builders. These are placed near the beginning because they are very + # slow to run. + + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. + x86_64-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + + dist-x86_64-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + +- job: Windows + timeoutInMinutes: 600 + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/windows.yml + strategy: + matrix: + # 32/64 bit MSVC tests + x86_64-msvc-1: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-msvc-2: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + + # 32/64 bit MSVC and GNU deployment + dist-x86_64-msvc: + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-msvc: + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 diff --git a/.azure-pipelines/master.yml b/.azure-pipelines/master.yml new file mode 100644 index 0000000000000..e30c73c5e9241 --- /dev/null +++ b/.azure-pipelines/master.yml @@ -0,0 +1,22 @@ +# +# Azure Pipelines job to publish toolstate. Only triggers on pushes to master. +# + +pr: none +trigger: +- master + +pool: + vmImage: ubuntu-16.04 + +steps: +- checkout: self + fetchDepth: 2 + +- script: | + export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) + . src/ci/docker/x86_64-gnu-tools/repo.sh + commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" + displayName: Publish toolstate + env: + TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN_SECRET) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml new file mode 100644 index 0000000000000..a2a4fc2516683 --- /dev/null +++ b/.azure-pipelines/pr.yml @@ -0,0 +1,26 @@ +# +# Azure Pipelines pull request build for Rust +# + +trigger: none +pr: +- master # FIXME: really just want any branch, but want an explicit "pr" property set so it's clear + +variables: +- group: caching-pr + +jobs: +- job: Linux + timeoutInMinutes: 600 + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/linux.yml + strategy: + matrix: + x86_64-gnu-llvm-6.0: + RUST_BACKTRACE: 1 + + x86_64-gnu-tools: {} + # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) + mingw-check: {} diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml new file mode 100644 index 0000000000000..288c4c9854e5a --- /dev/null +++ b/.azure-pipelines/steps/linux.yml @@ -0,0 +1,42 @@ +steps: +- checkout: self + fetchDepth: 2 + +- template: show-environment-variables.yml +- template: show-disk-usage.yml + +- bash: | + sudo apt install gdb + + curl -fo $HOME/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-unknown-linux-musl + chmod +x $HOME/stamp + + export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME + echo "##vso[task.setvariable variable=PATH;]$PATH" + + mkdir -p $HOME/rustsrc + displayName: Prep + +# FIXME(#46924): these two commands are required to enable IPv6, +# they shouldn't exist, please revert once more official solutions appeared. +# see https://github.com/travis-ci/travis-ci/issues/8891#issuecomment-353403729 +- bash: | + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: Enable IPv6 + +# FIXME: using the job display name feels brittle. Is there a better variable that represents the current matrix key? +- bash: | + export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/docker/run.sh $IMAGE" + echo "##vso[task.setvariable variable=IMAGE]$IMAGE" + echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" + displayName: Prepare run script + +- template: show-environment-variables.yml + +- bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern' + displayName: Enable core dump + +- template: verify-publish-toolstate.yml + +- template: run-script.yml diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml new file mode 100644 index 0000000000000..a217df6f3081e --- /dev/null +++ b/.azure-pipelines/steps/macos.yml @@ -0,0 +1,47 @@ +steps: +- template: show-disk-usage.yml + +- bash: | + export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ + mkdir -p $HOME/rustsrc + echo "##vso[task.setvariable variable=PATH;]$PATH" + + curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + chmod +x /usr/local/bin/sccache + + curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin + chmod +x /usr/local/bin/stamp + + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang + echo "##vso[task.setvariable variable=CC]$CC" + + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ + echo "##vso[task.setvariable variable=CXX]$CXX" + + echo "##vso[task.setvariable variable=AR]ar" + displayName: Prep + +- bash: brew install gnu-tar + displayName: install a tar that works well + +- bash: | + curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + displayName: Download clang + +- bash: | + brew update + brew install xz + brew install swig + condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist')) + displayName: Install xz and swigw + +- bash: | + export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/run.sh" + echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" + displayName: Prepare run script (init and run) + +- template: show-environment-variables.yml + +- template: verify-publish-toolstate.yml + +- template: run-script.yml diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml new file mode 100644 index 0000000000000..4ab3e2e06d36a --- /dev/null +++ b/.azure-pipelines/steps/run-script.yml @@ -0,0 +1,24 @@ +steps: +# Log time information from this machine and an external machine for insight into possible +# clock drift. Timezones don't matter since relative deltas give all the necessary info. +- bash: | + date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) + displayName: Log time information (before) + +- bash: | + which sccache + stamp sh -x -c "$RUN_SCRIPT" + env: + CI: true + CI_JOB_NAME: $(IMAGE) + SRC: . + + # Explicitly decrypt secret variables + # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables + SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) + DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + displayName: Run script + +- bash: | + date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) + displayName: Log time information (after) diff --git a/.azure-pipelines/steps/show-disk-usage.yml b/.azure-pipelines/steps/show-disk-usage.yml new file mode 100644 index 0000000000000..902d803647174 --- /dev/null +++ b/.azure-pipelines/steps/show-disk-usage.yml @@ -0,0 +1,5 @@ +steps: +- bash: | + df -h + du . | sort -nr | head -n100 + displayName: Show disk usage diff --git a/.azure-pipelines/steps/show-environment-variables.yml b/.azure-pipelines/steps/show-environment-variables.yml new file mode 100644 index 0000000000000..f6ed063ec6be5 --- /dev/null +++ b/.azure-pipelines/steps/show-environment-variables.yml @@ -0,0 +1,3 @@ +steps: +- bash: printenv | sort + displayName: Show environment variables \ No newline at end of file diff --git a/.azure-pipelines/steps/verify-publish-toolstate.yml b/.azure-pipelines/steps/verify-publish-toolstate.yml new file mode 100644 index 0000000000000..5531c90e090a7 --- /dev/null +++ b/.azure-pipelines/steps/verify-publish-toolstate.yml @@ -0,0 +1,9 @@ +steps: +- bash: | + git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git + cd rust-toolstate + python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" + cd .. + rm -rf rust-toolstate + condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check')) + displayName: Verify the publish_toolstate script works diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml new file mode 100644 index 0000000000000..5dcba664bee87 --- /dev/null +++ b/.azure-pipelines/steps/windows.yml @@ -0,0 +1,160 @@ +steps: +- checkout: self + fetchDepth: 2 + +- bash: | + set -x + git submodule + export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') + echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" + +- script: | + REM echo hack as drive D is too small + IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( + mkdir c:\MORE_SPACE + mklink /J build c:\MORE_SPACE + ) + +- script: | + set MSYS_PATH=%CD%\citools\msys64 + choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y + set PATH=%MSYS_PATH%\usr\bin;%PATH% + pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar + IF "%MINGW_URL%"=="" ( + IF "%MSYS_BITS%"=="32" pacman -S --noconfirm --needed mingw-w64-i686-toolchain mingw-w64-i686-cmake mingw-w64-i686-gcc mingw-w64-i686-python2 + IF "%MSYS_BITS%"=="64" pacman -S --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-python2 + ) + where rev + rev --help + where make + + echo ##vso[task.setvariable variable=MSYS_PATH]%MSYS_PATH% + echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin + displayName: Install msys2 + +# If we need to download a custom MinGW, do so here and set the path +# appropriately. +# +# Note that this *also* means that we're not using what is typically +# /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where +# /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we +# move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe` +# file exists in there (which it doesn't by default). +- script: | + powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" + 7z x -y %MINGW_ARCHIVE% > nul + echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin + condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + displayName: Download custom MinGW + +# If we're compiling for MSVC then we, like most other distribution builders, +# switch to clang as the compiler. This'll allow us eventually to enable LTO +# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think +# clang has an output mode compatible with MinGW that we need. If it does we +# should switch to clang for MinGW as well! +# +# Note that the LLVM installer is an NSIS installer +# +# Original downloaded here came from +# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe +- script: | + powershell -Command "iwr -outf %TEMP%\LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" + set CLANG_DIR=%CD%\citools\clang-rust + %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + condition: and(succeeded(), eq(variables['MINGW_URL'],'')) + displayName: Download clang + +# Here we do a pretty heinous thing which is to mangle the MinGW installation +# we just had above. Currently, as of this writing, we're using MinGW-w64 +# builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to +# be the first version which contains a fix for #40546, builds randomly +# failing during LLVM due to ar.exe/ranlib.exe failures. +# +# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds +# to contain a regression in gdb (#40184). As a result if we were to use the +# gdb provided (7.11.1) then we would fail all debuginfo tests. +# +# In order to fix spurious failures (pretty high priority) we use 6.3.0. To +# avoid disabling gdb tests we download an *old* version of gdb, specifically +# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb +# with the 6.2.0 gdb to get tests passing. +# +# Note that we don't literally overwrite the gdb.exe binary because it appears +# to just use gdborig.exe, so that's the binary we deal with instead. +- script: | + echo ON + powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" + mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe + condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + displayName: Override with 6.3.0 gdb with 6.2.0 gdb + +# Otherwise pull in the MinGW installed on appveyor +- script: | + echo Find mingw + set PATH | findstr /i msys + set PATH | findstr /i mingw + echo ##vso[task.prependpath]%MSYS_PATH%\mingw%MSYS_BITS%\bin + condition: and(succeeded(), eq(variables['MINGW_URL'],'')) + displayName: Add MinGW to path + +- script: | + copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe + echo ##vso[task.prependpath]C:\Python27amd64 + displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes + +- script: | + md sccache + powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + echo ##vso[task.prependpath]%CD%\sccache + displayName: Download and install sccache + +# Note that this is originally from the github releases patch of Ninja +- script: | + md ninja + powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip" + 7z x -oninja 2017-03-15-ninja-win.zip + del 2017-03-15-ninja-win.zip + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + echo ##vso[task.prependpath]%CD%\ninja + displayName: Download and install ninja + +- script: | + mkdir handle + powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip" + 7z x -ohandle 2017-05-15-Handle.zip + del 2017-05-15-Handle.zip + set PATH=%PATH%;%CD%\handle + handle.exe -accepteula -help + echo ##vso[task.setvariable variable=PATH]%PATH% + displayName: Help debug handle issues + +- template: show-environment-variables.yml + +- script: | + REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8 + IF "%VCVARS_BAT%" NEQ "" ( + CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" + ) + + where sccache + where rev + set | findstr /v SCCACHE_AZURE_CONNECTION_STRING + + if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc + + sh src/ci/init_repo.sh . /d/cache/rustsrc + sh src/ci/run.sh + env: + CI: true + CI_JOB_NAME: $(System.JobDisplayName) + SRC: . + NO_CCACHE: 1 + + # explicitly decrypt secret variables + # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch + SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) + DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + displayName: Run script diff --git a/.azure-pipelines/test.yml b/.azure-pipelines/test.yml new file mode 100644 index 0000000000000..c4004aa46a97a --- /dev/null +++ b/.azure-pipelines/test.yml @@ -0,0 +1,131 @@ +# +# Azure Pipelines test build for Rust on Linux, macOS, and Windows. +# + +pr: none +trigger: none + +variables: +- group: caching + +jobs: +- job: Linux + timeoutInMinutes: 600 + pool: + vmImage: ubuntu-16.04 + steps: + - template: steps/linux.yml + strategy: + matrix: + x86_64-gnu-llvm-6.0: + RUST_BACKTRACE: 1 + + dist-x86_64-linux: + DEPLOY: 1 + + # "alternate" deployments, these are "nightlies" but have LLVM assertions + # turned on, they're deployed to a different location primarily for + # additional testing. + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 + + dist-various-1: + DEPLOY: 1 + + dist-android: + DEPLOY: 1 + + dist-x86_64-freebsd: + DEPLOY: 1 + + i686-gnu-nopt: {} + test-various: {} + x86_64-gnu-tools: {} + # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) + x86_64-gnu-nopt: {} + x86_64-gnu-distcheck: {} + mingw-check: {} + +- job: macOS + timeoutInMinutes: 600 + pool: + vmImage: macos-10.13 + steps: + - template: steps/macos.yml + strategy: + matrix: + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + # macOS builders. These are placed near the beginning because they are very + # slow to run. + + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. + x86_64-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + +- job: Windows + timeoutInMinutes: 600 + pool: + vmImage: 'vs2017-win2016' + steps: + - template: steps/windows.yml + strategy: + matrix: + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py test + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml new file mode 100644 index 0000000000000..0b7233fb2e72c --- /dev/null +++ b/.azure-pipelines/try.yml @@ -0,0 +1,29 @@ +# +# Azure Pipelines "try" branch build for Rust +# + +pr: none +trigger: +- try + +variables: +- group: caching + +jobs: +- job: Linux + timeoutInMinutes: 300 + pool: + vmImage: ubuntu-16.04 + strategy: + matrix: + dist-x86_64-linux: + DEPLOY: 1 + + # "alternate" deployments, these are "nightlies" but have LLVM assertions + # turned on, they're deployed to a different location primarily for + # additional testing. + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 + steps: + - template: steps/linux.yml diff --git a/README.md b/README.md index f6f796911b8fa..8fa12a6571b8c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Build Status](https://dev.azure.com/rust-lang-azure/rust/_apis/build/status/johnterickson.rust?branchName=master)](https://dev.azure.com/rust-lang-azure/rust/_build/latest?definitionId=6&branchName=master) + # The Rust Programming Language This is the main source code repository for [Rust]. It contains the compiler, diff --git a/appveyor.yml b/appveyor.yml index dffd79c56e48a..c35da8927ce93 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -20,11 +20,11 @@ environment: - CI_JOB_NAME: i686-msvc-1 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-1 + SCRIPT: make ci-subset-1 - CI_JOB_NAME: i686-msvc-2 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make appveyor-subset-2 + SCRIPT: make ci-subset-2 # MSVC aux tests - CI_JOB_NAME: x86_64-msvc-aux @@ -58,7 +58,7 @@ environment: - CI_JOB_NAME: i686-mingw-1 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-1 + SCRIPT: make ci-subset-1 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 @@ -68,7 +68,7 @@ environment: - CI_JOB_NAME: i686-mingw-2 MSYS_BITS: 32 RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make appveyor-subset-2 + SCRIPT: make ci-subset-2 MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z MINGW_DIR: mingw32 diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 1e01d68fb3a61..c47c0f765f0fe 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -39,8 +39,7 @@ The script accepts commands, flags, and arguments to determine what to do: ``` If files are dirty that would normally be rebuilt from stage 0, that can be - overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps - that belong to stage n or earlier: + overridden using `--keep-stage 0`. ``` # keep old build products for stage 0 and build stage 1 diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 07be27c2f5a02..29b0cdaef63fe 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -14,9 +14,10 @@ endif BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py -all: +build: $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) - $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) + +all: build doc help: $(Q)echo 'Welcome to the rustbuild build system!' @@ -80,10 +81,75 @@ TESTS_IN_2 := \ src/test/run-pass-fulldeps \ src/tools/linkchecker -appveyor-subset-1: +MIN_TEST := \ + src/test/debuginfo + +ci-subset-1: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) -appveyor-subset-2: +ci-subset-2: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) +ci-test-prepare: + $(Q)$(BOOTSTRAP) test $(MIN_TEST) + +ci-resume-subset-1: + $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) +ci-resume-subset-2: + $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) + +TESTS_IN_B := \ + src/tools/linkchecker + +TESTS_IN_C := \ + src/test/run-pass \ + src/test/run-pass-fulldeps + +TESTS_IN_D := \ + src/test/compile-fail \ + src/test/rustdoc \ + src/test/pretty + +TESTS_IN_E := \ + src/test/ui + +TESTS_IN_F := \ + src/test/run-fail \ + src/liballoc \ + src/libcore + +TESTS_IN_G := \ + src/tools/rustdoc \ + src/test/rustdoc-js-std \ + src/test/run-make-fulldeps \ + src/libstd + +TESTS_IN_H := \ + src/librustc_driver + +ci-resume-subset-A: + $(Q)$(BOOTSTRAP) test \ + $(KEEP_STAGE) \ + $(TESTS_IN_B:%=--exclude %) \ + $(TESTS_IN_C:%=--exclude %) \ + $(TESTS_IN_D:%=--exclude %) \ + $(TESTS_IN_E:%=--exclude %) \ + $(TESTS_IN_F:%=--exclude %) \ + $(TESTS_IN_G:%=--exclude %) \ + $(TESTS_IN_H:%=--exclude %) \ + $(MIN_TEST:%=--exclude %) +ci-resume-subset-B: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_B) +ci-resume-subset-C: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_C) +ci-resume-subset-D: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_D) +ci-resume-subset-E: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_E) +ci-resume-subset-F: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_F) +ci-resume-subset-G: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_G) +ci-resume-subset-H: + $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_H) .PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 5777331b9bfd5..62a8b7aad031d 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -81,6 +81,13 @@ impl Step for Llvm { (info, "src/llvm-project/llvm", builder.llvm_out(target), dir.join("bin")) }; + let build_llvm_config = llvm_config_ret_dir + .join(exe("llvm-config", &*builder.config.build)); + + if env::var_os("SKIP_LLVM_BUILD").is_some() { + return build_llvm_config + } + if !llvm_info.is_git() { println!( "git could not determine the LLVM submodule commit hash. \ @@ -88,8 +95,6 @@ impl Step for Llvm { ); } - let build_llvm_config = llvm_config_ret_dir - .join(exe("llvm-config", &*builder.config.build)); let done_stamp = out_dir.join("llvm-finished-building"); if let Some(llvm_commit) = llvm_info.sha() { diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index a162c65672f82..e79700d92d89c 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -326,6 +326,8 @@ pub enum CiEnv { Travis, /// The AppVeyor environment, for Windows builds. AppVeyor, + /// The Azure Pipelines environment, for Linux (including Docker), Window, and macOS builds. + AzurePipelines, } impl CiEnv { @@ -335,6 +337,8 @@ impl CiEnv { CiEnv::Travis } else if env::var("APPVEYOR").ok().map_or(false, |e| &*e == "True") { CiEnv::AppVeyor + } else if env::var("TF_BUILD").ok().map_or(false, |e| &*e == "True") { + CiEnv::AzurePipelines } else { CiEnv::None } diff --git a/src/ci/docker/i686-gnu/Dockerfile b/src/ci/docker/i686-gnu-1/Dockerfile similarity index 54% rename from src/ci/docker/i686-gnu/Dockerfile rename to src/ci/docker/i686-gnu-1/Dockerfile index 17441ddb4546b..1a04b8af4c17b 100644 --- a/src/ci/docker/i686-gnu/Dockerfile +++ b/src/ci/docker/i686-gnu-1/Dockerfile @@ -18,10 +18,5 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu -# Exclude some tests that are unlikely to be platform specific, to speed up -# this slow job. -ENV SCRIPT python2.7 ../x.py test \ - --exclude src/bootstrap \ - --exclude src/test/rustdoc-js \ - --exclude src/tools/error_index_generator \ - --exclude src/tools/linkchecker +ENV SCRIPT make ci-subset-1 + diff --git a/src/ci/docker/i686-gnu-2/Dockerfile b/src/ci/docker/i686-gnu-2/Dockerfile new file mode 100644 index 0000000000000..b1a0ea6f0e99e --- /dev/null +++ b/src/ci/docker/i686-gnu-2/Dockerfile @@ -0,0 +1,21 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++-multilib \ + make \ + file \ + curl \ + ca-certificates \ + python2.7 \ + git \ + cmake \ + sudo \ + gdb \ + xz-utils + + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu +ENV SCRIPT make ci-subset-2 diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 98a765e1cec95..cd913e30d1129 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# ignore-tidy-linelength + set -e export MSYS_NO_PATHCONV=1 @@ -40,9 +42,24 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then docker --version >> $hash_key cksum=$(sha512sum $hash_key | \ awk '{print $1}') - s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" - echo "Attempting to download $s3url" + + if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then + # install azcopy + echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list + sudo cp ./azure.list /etc/apt/sources.list.d/ + sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF + sudo apt-get update + sudo apt-get install azcopy + + url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum" + upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY" + else + s3url="s3://$SCCACHE_BUCKET/docker/$cksum" + url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" + upload="aws s3 cp - $s3url" + fi + + echo "Attempting to download $url" rm -f /tmp/rustci_docker_cache set +e retry curl -y 30 -Y 10 --connect-timeout 30 -f -L -C - -o /tmp/rustci_docker_cache "$url" @@ -65,17 +82,17 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then -f "$dockerfile" \ "$context" - if [ "$s3url" != "" ]; then + if [ "$upload" != "" ]; then digest=$(docker inspect rust-ci --format '{{.Id}}') echo "Built container $digest" if ! grep -q "$digest" <(echo "$loaded_images"); then - echo "Uploading finished image to $s3url" + echo "Uploading finished image to $url" set +e docker history -q rust-ci | \ grep -v missing | \ xargs docker save | \ gzip | \ - aws s3 cp - $s3url + $upload set -e else echo "Looks like docker image is the same as before, not uploading" @@ -87,8 +104,8 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then echo "$digest" >>"$info" fi elif [ -f "$docker_dir/disabled/$image/Dockerfile" ]; then - if [ -n "$TRAVIS_OS_NAME" ]; then - echo Cannot run disabled images on travis! + if isCI; then + echo Cannot run disabled images on CI! exit 1 fi # retry messes with the pipe from tar to docker. Not needed on non-travis @@ -117,6 +134,9 @@ if [ "$SCCACHE_BUCKET" != "" ]; then args="$args --env SCCACHE_REGION" args="$args --env AWS_ACCESS_KEY_ID" args="$args --env AWS_SECRET_ACCESS_KEY" +elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then + args="$args --env SCCACHE_AZURE_CONNECTION_STRING" + args="$args --env SCCACHE_AZURE_BLOB_CONTAINER" else mkdir -p $HOME/.cache/sccache args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache" @@ -140,8 +160,11 @@ exec docker \ --env DEPLOY \ --env DEPLOY_ALT \ --env LOCAL_USER_ID=`id -u` \ + --env CI \ --env TRAVIS \ --env TRAVIS_BRANCH \ + --env TF_BUILD \ + --env BUILD_SOURCEBRANCHNAME \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \ --volume "$HOME/.cargo:/cargo" \ diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index 4c03419894e7c..d6f2f4fadb2c6 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,6 +1,7 @@ set -ex +echo Use sccache built with Azure Storage support curl -fo /usr/local/bin/sccache \ - https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl + https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache chmod +x /usr/local/bin/sccache diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index af0198705a2fe..c7e64cd8c3e79 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -6,7 +6,7 @@ X_PY="$1" TOOLSTATE_FILE="$(realpath $2)" OS="$3" COMMIT="$(git rev-parse HEAD)" -CHANGED_FILES="$(git diff --name-status HEAD HEAD^)" +CHANGED_FILES="$(git diff --name-status HEAD HEAD^ || echo)" SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))" # ^ Number of days after the last promotion of beta. # Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. diff --git a/src/ci/run.sh b/src/ci/run.sh index 42d0d7db5964c..08ede122d65c0 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -23,7 +23,9 @@ fi ci_dir=`cd $(dirname $0) && pwd` source "$ci_dir/shared.sh" -if [ "$TRAVIS" != "true" ] || [ "$TRAVIS_BRANCH" == "auto" ]; then +branch_name=$(getCIBranch) + +if [ ! isCI ] || [ "$branch_name" = "auto" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" fi @@ -44,7 +46,7 @@ fi # FIXME: need a scheme for changing this `nightly` value to `beta` and `stable` # either automatically or manually. export RUST_RELEASE_CHANNEL=nightly -if [ "$DEPLOY$DEPLOY_ALT" != "" ]; then +if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --release-channel=$RUST_RELEASE_CHANNEL" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-static-stdcpp" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.remap-debuginfo" @@ -88,28 +90,30 @@ if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then rm -rf build fi -travis_fold start configure -travis_time_start -$SRC/configure $RUST_CONFIGURE_ARGS -travis_fold end configure -travis_time_finish - -travis_fold start make-prepare -travis_time_start -retry make prepare -travis_fold end make-prepare -travis_time_finish - -travis_fold start check-bootstrap -travis_time_start -make check-bootstrap -travis_fold end check-bootstrap -travis_time_finish +if [ "$SKIP_PREP" != "1" ]; then + travis_fold start configure + travis_time_start + $SRC/configure $RUST_CONFIGURE_ARGS + travis_fold end configure + travis_time_finish + + travis_fold start make-prepare + travis_time_start + retry make prepare + travis_fold end make-prepare + travis_time_finish + + travis_fold start check-bootstrap + travis_time_start + make check-bootstrap + travis_fold end check-bootstrap + travis_time_finish +fi # Display the CPU and memory information. This helps us know why the CI timing # is fluctuating. travis_fold start log-system-info -if [ "$TRAVIS_OS_NAME" = "osx" ]; then +if isOSX; then system_profiler SPHardwareDataType || true sysctl hw || true ncpus=$(sysctl -n hw.ncpu) @@ -138,3 +142,5 @@ else do_make all do_make "$RUST_CHECK_TARGET" fi + +sccache --show-stats diff --git a/src/ci/shared.sh b/src/ci/shared.sh index 3ba64ad412064..1e667fca5875a 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -24,6 +24,22 @@ function retry { done } +function isCI { + [ "$CI" = "true" ] || [ "$TRAVIS" = "true" ] || [ "$TF_BUILD" = "True" ] +} + +function isOSX { + [ "$TRAVIS_OS_NAME" = "osx" ] || [ "$AGENT_OS" = "Darwin" ] +} + +function getCIBranch { + if [ "$TRAVIS" = "true" ]; then + echo "$TRAVIS_BRANCH" + else + echo "$BUILD_SOURCEBRANCHNAME" + fi; +} + if ! declare -F travis_fold; then if [ "${TRAVIS-false}" = 'true' ]; then # This is a trimmed down copy of diff --git a/src/test/run-pass/fds-are-cloexec.rs b/src/test/run-pass/fds-are-cloexec.rs index 3bc0ceb5cf2fa..bcfc3245db30e 100644 --- a/src/test/run-pass/fds-are-cloexec.rs +++ b/src/test/run-pass/fds-are-cloexec.rs @@ -3,6 +3,7 @@ // ignore-cloudabi no processes // ignore-emscripten no processes // ignore-haiku +// ignore-macos #![feature(rustc_private)] From e234a901b6594ee90654f0b74cd1cf7948fab6e0 Mon Sep 17 00:00:00 2001 From: John Erickson Date: Mon, 6 May 2019 09:53:20 -0700 Subject: [PATCH 02/57] Disable 32-bit OSX --- .azure-pipelines/auto.yml | 52 +++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 818b613740e6e..2cefa5993a817 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -167,15 +167,6 @@ jobs: - template: steps/macos.yml strategy: matrix: - dist-x86_64-apple-alt: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY_ALT: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - # macOS builders. These are placed near the beginning because they are very # slow to run. @@ -194,18 +185,9 @@ jobs: NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 - i686-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - dist-i686-apple: + dist-x86_64-apple: RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc DEPLOY: 1 RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 @@ -213,15 +195,37 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - dist-x86_64-apple: + dist-x86_64-apple-alt: RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY: 1 + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 MACOSX_DEPLOYMENT_TARGET: 10.7 NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 + + # # temp disabled + # # https://github.com/johnterickson/rust/issues/18 + # i686-apple: + # RUST_CHECK_TARGET: check + # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # MACOSX_DEPLOYMENT_TARGET: 10.8 + # MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + # NO_LLVM_ASSERTIONS: 1 + # NO_DEBUG_ASSERTIONS: 1 + + # dist-i686-apple: + # RUST_CHECK_TARGET: dist + # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + # DEPLOY: 1 + # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # MACOSX_DEPLOYMENT_TARGET: 10.7 + # NO_LLVM_ASSERTIONS: 1 + # NO_DEBUG_ASSERTIONS: 1 + # DIST_REQUIRE_ALL_TOOLS: 1 + + - job: Windows timeoutInMinutes: 600 From c50517ae85960a9f16de0fa1f02e93413a8f1578 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 10 May 2019 09:56:46 +0200 Subject: [PATCH 03/57] ci: reduce all azure timeouts to 3 hours As the infra team we decided to have an hard timeout of 3 hours on all the jobs: while this sometimes causes spurious timeout errors it's a great limit to prevent CI time to grow even more. --- .azure-pipelines/auto.fanout.yml | 12 ++++++------ .azure-pipelines/auto.yml | 6 +++--- .azure-pipelines/pr.yml | 2 +- .azure-pipelines/test.yml | 6 +++--- .azure-pipelines/try.yml | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml index 08cba3c47ca8a..e12836257d933 100644 --- a/.azure-pipelines/auto.fanout.yml +++ b/.azure-pipelines/auto.fanout.yml @@ -18,7 +18,7 @@ variables: jobs: # WINDOWS JOBS - job: Windows - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: tar pool: @@ -38,7 +38,7 @@ jobs: SCRIPT: make ci-subset-2 - job: Windows_build - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: tar pool: @@ -66,7 +66,7 @@ jobs: - job: Windows_test dependsOn: Windows_build - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: tar SKIP_PREP: 1 @@ -285,7 +285,7 @@ jobs: # macOS JOBS - job: macOS - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: gtar pool: @@ -306,7 +306,7 @@ jobs: NO_DEBUG_ASSERTIONS: 1 - job: macOS_build - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: gtar pool: @@ -351,7 +351,7 @@ jobs: - job: macOS_test dependsOn: macOS_build - timeoutInMinutes: 600 + timeoutInMinutes: 180 variables: TAR_COMMAND: gtar SKIP_PREP: 1 diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 2cefa5993a817..5f4c155ce98b4 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -16,7 +16,7 @@ variables: jobs: - job: Linux - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 steps: @@ -158,7 +158,7 @@ jobs: IMAGE: mingw-check - job: macOS - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: macos-10.13 steps: @@ -228,7 +228,7 @@ jobs: - job: Windows - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: 'vs2017-win2016' steps: diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index a2a4fc2516683..ffdb4301e9350 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -11,7 +11,7 @@ variables: jobs: - job: Linux - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 steps: diff --git a/.azure-pipelines/test.yml b/.azure-pipelines/test.yml index c4004aa46a97a..358806fe14c9f 100644 --- a/.azure-pipelines/test.yml +++ b/.azure-pipelines/test.yml @@ -10,7 +10,7 @@ variables: jobs: - job: Linux - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 steps: @@ -48,7 +48,7 @@ jobs: mingw-check: {} - job: macOS - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: macos-10.13 steps: @@ -83,7 +83,7 @@ jobs: NO_DEBUG_ASSERTIONS: 1 - job: Windows - timeoutInMinutes: 600 + timeoutInMinutes: 180 pool: vmImage: 'vs2017-win2016' steps: diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 0b7233fb2e72c..eaa50729b0d27 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -11,7 +11,7 @@ variables: jobs: - job: Linux - timeoutInMinutes: 300 + timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 strategy: From abd4b3a83cdc6b2a8e36a238d096afa9c83d3eb5 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 10 May 2019 10:04:34 +0200 Subject: [PATCH 04/57] ci: remove travis ipv6 docker hack on azure --- .azure-pipelines/steps/linux.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml index 288c4c9854e5a..b347549ccb7d5 100644 --- a/.azure-pipelines/steps/linux.yml +++ b/.azure-pipelines/steps/linux.yml @@ -17,14 +17,6 @@ steps: mkdir -p $HOME/rustsrc displayName: Prep -# FIXME(#46924): these two commands are required to enable IPv6, -# they shouldn't exist, please revert once more official solutions appeared. -# see https://github.com/travis-ci/travis-ci/issues/8891#issuecomment-353403729 -- bash: | - echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - displayName: Enable IPv6 - # FIXME: using the job display name feels brittle. Is there a better variable that represents the current matrix key? - bash: | export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/docker/run.sh $IMAGE" From 869ddd8d0019683c007956bae17fc07cd30f39f7 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 10 May 2019 10:17:52 +0200 Subject: [PATCH 05/57] ci: still use aws for caches --- .azure-pipelines/auto.fanout.yml | 4 ++-- .azure-pipelines/auto.yml | 5 ----- .azure-pipelines/steps/macos.yml | 10 +++++----- .azure-pipelines/steps/run-script.yml | 6 +++--- .azure-pipelines/steps/windows.yml | 6 +++--- src/ci/docker/run.sh | 23 +++-------------------- src/ci/docker/scripts/sccache.sh | 3 +-- 7 files changed, 17 insertions(+), 40 deletions(-) diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml index e12836257d933..def19a87076d8 100644 --- a/.azure-pipelines/auto.fanout.yml +++ b/.azure-pipelines/auto.fanout.yml @@ -160,7 +160,7 @@ jobs: - script: | REM md sccache - REM powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Download and install sccache @@ -392,7 +392,7 @@ jobs: mkdir -p $HOME/rustsrc echo "##vso[task.setvariable variable=PATH;]$PATH" - curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 5f4c155ce98b4..189536cff5f4a 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -1,11 +1,6 @@ # # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. # -# Notes: -# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled -# and with openssl statically-linked for Linux. -# Here's the build (with a backpointer to source) of where the bits came from: -# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275 pr: none # trigger: diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml index a217df6f3081e..e88b5613e46d4 100644 --- a/.azure-pipelines/steps/macos.yml +++ b/.azure-pipelines/steps/macos.yml @@ -3,21 +3,21 @@ steps: - bash: | export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ - mkdir -p $HOME/rustsrc + mkdir -p $HOME/rustsrc echo "##vso[task.setvariable variable=PATH;]$PATH" - curl -fo /usr/local/bin/sccache https://rustlangtools.blob.core.windows.net/public/stable-x86_64-apple-darwin.sccache + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin chmod +x /usr/local/bin/stamp - + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang echo "##vso[task.setvariable variable=CC]$CC" - + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ echo "##vso[task.setvariable variable=CXX]$CXX" - + echo "##vso[task.setvariable variable=AR]ar" displayName: Prep diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml index 4ab3e2e06d36a..cb8e8a629a7e6 100644 --- a/.azure-pipelines/steps/run-script.yml +++ b/.azure-pipelines/steps/run-script.yml @@ -5,7 +5,7 @@ steps: date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) displayName: Log time information (before) -- bash: | +- bash: | which sccache stamp sh -x -c "$RUN_SCRIPT" env: @@ -15,8 +15,8 @@ steps: # Explicitly decrypt secret variables # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables - SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) - DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) displayName: Run script - bash: | diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml index 5dcba664bee87..e3c06d543a972 100644 --- a/.azure-pipelines/steps/windows.yml +++ b/.azure-pipelines/steps/windows.yml @@ -106,7 +106,7 @@ steps: - script: | md sccache - powershell -Command "iwr -outf sccache\sccache.exe https://rustlangtools.blob.core.windows.net/public/stable-x86_64-pc-windows-msvc.sccache.exe" + powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Download and install sccache @@ -155,6 +155,6 @@ steps: # explicitly decrypt secret variables # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) - DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) + AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) displayName: Run script diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index cd913e30d1129..0e1485601e96e 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# ignore-tidy-linelength - set -e export MSYS_NO_PATHCONV=1 @@ -43,21 +41,9 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then cksum=$(sha512sum $hash_key | \ awk '{print $1}') - if [ "$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT" != "" ]; then - # install azcopy - echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod/ xenial main" > azure.list - sudo cp ./azure.list /etc/apt/sources.list.d/ - sudo apt-key adv --keyserver packages.microsoft.com --recv-keys EB3E94ADBE1229CF - sudo apt-get update - sudo apt-get install azcopy - - url="https://$DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT.blob.core.windows.net/$DOCKER_LAYER_CACHE_AZURE_STORAGE_CONTAINER/$cksum" - upload="azcopy --quiet --destination $url --dest-key $DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY" - else - s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" - upload="aws s3 cp - $s3url" - fi + s3url="s3://$SCCACHE_BUCKET/docker/$cksum" + url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" + upload="aws s3 cp - $s3url" echo "Attempting to download $url" rm -f /tmp/rustci_docker_cache @@ -134,9 +120,6 @@ if [ "$SCCACHE_BUCKET" != "" ]; then args="$args --env SCCACHE_REGION" args="$args --env AWS_ACCESS_KEY_ID" args="$args --env AWS_SECRET_ACCESS_KEY" -elif [ "$SCCACHE_AZURE_CONNECTION_STRING" != "" ]; then - args="$args --env SCCACHE_AZURE_CONNECTION_STRING" - args="$args --env SCCACHE_AZURE_BLOB_CONTAINER" else mkdir -p $HOME/.cache/sccache args="$args --env SCCACHE_DIR=/sccache --volume $HOME/.cache/sccache:/sccache" diff --git a/src/ci/docker/scripts/sccache.sh b/src/ci/docker/scripts/sccache.sh index d6f2f4fadb2c6..4c03419894e7c 100644 --- a/src/ci/docker/scripts/sccache.sh +++ b/src/ci/docker/scripts/sccache.sh @@ -1,7 +1,6 @@ set -ex -echo Use sccache built with Azure Storage support curl -fo /usr/local/bin/sccache \ - https://rustlangtools.blob.core.windows.net/public/stable-x86_64-unknown-linux-musl.sccache + https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-unknown-linux-musl chmod +x /usr/local/bin/sccache From 7dfd8ceb4fe5fceb426ab6868312cc0890e28837 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 12 May 2019 17:55:48 +0200 Subject: [PATCH 06/57] ci: remove fanout from the azure pipelines config --- .azure-pipelines/auto.fanout.yml | 521 ------------------------------- .azure-pipelines/auto.yml | 4 +- src/bootstrap/README.md | 3 +- src/bootstrap/mk/Makefile.in | 70 +---- src/bootstrap/native.rs | 9 +- src/ci/run.sh | 36 +-- 6 files changed, 25 insertions(+), 618 deletions(-) delete mode 100644 .azure-pipelines/auto.fanout.yml diff --git a/.azure-pipelines/auto.fanout.yml b/.azure-pipelines/auto.fanout.yml deleted file mode 100644 index def19a87076d8..0000000000000 --- a/.azure-pipelines/auto.fanout.yml +++ /dev/null @@ -1,521 +0,0 @@ -# -# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. -# -# Notes: -# To get sccache working, I (John Erickson) had to build it with the Azure feature enabled -# and with openssl statically-linked for Linux. -# Here's the build (with a backpointer to source) of where the bits came from: -# https://dev.azure.com/johnterickson/rust-lang/_build/results?buildId=275 - -name: fanout -pr: none -trigger: -- auto - -variables: -- group: caching - -jobs: -# WINDOWS JOBS -- job: Windows - timeoutInMinutes: 180 - variables: - TAR_COMMAND: tar - pool: - vmImage: 'vs2017-win2016' - steps: - - template: steps/windows.yml - strategy: - matrix: - # 32/64 bit MSVC tests - x86_64-msvc-1: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-1 - x86_64-msvc-2: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-2 - -- job: Windows_build - timeoutInMinutes: 180 - variables: - TAR_COMMAND: tar - pool: - vmImage: 'vs2017-win2016' - steps: - - template: steps/windows.yml - - script: | - echo on - echo "##vso[task.setvariable variable=TAR_PATH]%TEMP%\drop.tar" - - bash: | - set -x - $TAR_COMMAND --dereference --exclude=.git --exclude=citools $SUBMODULES_EXCLUDES -cf $TEMP/drop.tar . - $TAR_COMMAND --append --file=$TEMP/drop.tar citools - - task: PublishPipelineArtifact@0 - inputs: - artifactName: $(System.JobDisplayName) - targetPath: $(TAR_PATH) - strategy: - matrix: - # 32/64 bit MSVC tests - x86_64-msvc-build: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-test-prepare - -- job: Windows_test - dependsOn: Windows_build - timeoutInMinutes: 180 - variables: - TAR_COMMAND: tar - SKIP_PREP: 1 - KEEP_STAGE: '--keep-stage 0 --keep-stage 1' - SKIP_LLVM_BUILD: 1 - pool: - vmImage: 'vs2017-win2016' - steps: - - checkout: self - fetchDepth: 2 - - script: | - set MSYS_PATH=%CD%\citools\msys64 - set PATH=%MSYS_PATH%\usr\bin;%PATH% - where rev - rev --help - where make - echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin - displayName: Install msys2 - - # # If we need to download a custom MinGW, do so here and set the path - # # appropriately. - # # - # # Note that this *also* means that we're not using what is typically - # # /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where - # # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we - # # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe` - # # file exists in there (which it doesn't by default). - # - script: | - # powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" - # 7z x -y %MINGW_ARCHIVE% > nul - # echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin - # condition: and(succeeded(), ne(variables['MINGW_URL'],'')) - # displayName: Download custom MinGW - - # If we're compiling for MSVC then we, like most other distribution builders, - # switch to clang as the compiler. This'll allow us eventually to enable LTO - # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think - # clang has an output mode compatible with MinGW that we need. If it does we - # should switch to clang for MinGW as well! - # - # Note that the LLVM installer is an NSIS installer - # - # Original downloaded here came from - # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe - - script: | - # powershell -Command "iwr -outf LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" - # .\LLVM-7.0.0-win64.exe /S /NCRC /D=C:\clang-rust - set CLANG_DIR=%CD%\citools\clang-rust - set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe - echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% - condition: and(succeeded(), eq(variables['MINGW_URL'],'')) - displayName: Download clang - - # # Here we do a pretty heinous thing which is to mangle the MinGW installation - # # we just had above. Currently, as of this writing, we're using MinGW-w64 - # # builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to - # # be the first version which contains a fix for #40546, builds randomly - # # failing during LLVM due to ar.exe/ranlib.exe failures. - # # - # # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds - # # to contain a regression in gdb (#40184). As a result if we were to use the - # # gdb provided (7.11.1) then we would fail all debuginfo tests. - # # - # # In order to fix spurious failures (pretty high priority) we use 6.3.0. To - # # avoid disabling gdb tests we download an *old* version of gdb, specifically - # # that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb - # # with the 6.2.0 gdb to get tests passing. - # # - # # Note that we don't literally overwrite the gdb.exe binary because it appears - # # to just use gdborig.exe, so that's the binary we deal with instead. - # - script: | - # echo ON - # powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" - # mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe - # condition: and(succeeded(), ne(variables['MINGW_URL'],'')) - # displayName: Override with 6.3.0 gdb with 6.2.0 gdb - - # Otherwise pull in the MinGW installed on appveyor - - script: | - echo Find mingw - set PATH | findstr /i msys - set PATH | findstr /i mingw - echo ##vso[task.prependpath]C:\msys64\mingw%MSYS_BITS%\bin - condition: and(succeeded(), eq(variables['MINGW_URL'],'')) - displayName: Add MinGW to path - - - script: | - copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe - echo ##vso[task.prependpath]C:\Python27amd64 - displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes - - - script: | - REM md sccache - REM powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" - echo ##vso[task.prependpath]%CD%\sccache - displayName: Download and install sccache - - # Note that this is originally from the github releases patch of Ninja - - script: | - REM md ninja - REM powershell -Command "iwr -outf 2017-03-15-ninja-win.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-15-ninja-win.zip" - REM 7z x -oninja 2017-03-15-ninja-win.zip - REM del 2017-03-15-ninja-win.zip - set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja - echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% - echo ##vso[task.prependpath]%CD%\ninja - displayName: Download and install ninja - - - script: | - IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( - IF NOT EXIST build ( - mkdir c:\MORE_SPACE - mklink /J build c:\MORE_SPACE - ) - ) - - script: | - echo on - echo ##vso[task.setvariable variable=TAR_FOLDER_PATH]%TEMP% - - template: steps/show-environment-variables.yml - - task: DownloadPipelineArtifact@1 - inputs: - artifactName: $(ARTIFACT_NAME) - downloadPath: $(TAR_FOLDER_PATH) - # - task: DownloadPipelineArtifact@1 - # inputs: - # artifactName: $(ARTIFACT_NAME) - # downloadPath: $(TAR_FOLDER_PATH) - # buildType: specific - # project: rust - # pipeline: 8 - # buildVersionToDownload: specific - # buildId: 200 - # # https://dev.azure.com/rust-lang-azure/rust/_build/results?buildId=145 - - # - template: steps/macos.yml - - bash: $TAR_COMMAND -tvf $TEMP/drop.tar - - bash: $TAR_COMMAND --touch -xf $TEMP/drop.tar - - bash: rm $TEMP/drop.tar - - script: | - REM mkdir handle - REM powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip" - REM 7z x -ohandle 2017-05-15-Handle.zip - REM del 2017-05-15-Handle.zip - set PATH=%PATH%;%CD%\handle - handle.exe -accepteula -help - echo ##vso[task.setvariable variable=PATH]%PATH% - displayName: Help debug handle issues - - script: | - REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8 - IF "%VCVARS_BAT%" NEQ "" ( - CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" - ) - - where sccache - where rev - set | findstr /v SCCACHE_AZURE_CONNECTION_STRING - - if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc - - sh src/ci/run.sh - env: - CI: true - CI_JOB_NAME: $(System.JobDisplayName) - SRC: . - NO_CCACHE: 1 - - # explicitly decrypt secret variables - # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - SCCACHE_AZURE_CONNECTION_STRING: $(SCCACHE_AZURE_CONNECTION_STRING_SECRET) - DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY: $(DOCKER_LAYER_CACHE_AZURE_STORAGE_ACCOUNT_KEY_SECRET) - displayName: Run script - - strategy: - matrix: - # 32/64 bit MSVC tests - x86_64-msvc-A: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-A - x86_64-msvc-B: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-B - x86_64-msvc-C: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-C - x86_64-msvc-D: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-D - x86_64-msvc-E: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-E - x86_64-msvc-F: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-F - x86_64-msvc-G: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-G - x86_64-msvc-H: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - ARTIFACT_NAME: x86_64-msvc-build - SCRIPT: make ci-resume-subset-H - -# macOS JOBS -- job: macOS - timeoutInMinutes: 180 - variables: - TAR_COMMAND: gtar - pool: - vmImage: macos-10.13 - steps: - - checkout: self - fetchDepth: 2 - - template: steps/macos.yml - strategy: - matrix: - x86_64-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - -- job: macOS_build - timeoutInMinutes: 180 - variables: - TAR_COMMAND: gtar - pool: - vmImage: macos-10.13 - steps: - - checkout: self - fetchDepth: 2 - - template: steps/macos.yml - - script: | - set -x - - git submodule - export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') - echo $SUBMODULES_EXCLUDES - - $TAR_COMMAND --exclude=.git $SUBMODULES_EXCLUDES -cf /tmp/drop.tar . - - task: PublishPipelineArtifact@0 - inputs: - artifactName: $(System.JobDisplayName) - targetPath: '/tmp/drop.tar' - - strategy: - matrix: - # macOS builders. These are placed near the beginning because they are very - # slow to run. - - # OSX builders running tests, these run the full test suite. - # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some - # runners that run `//ignore-debug` tests. - # - # Note that the compiler is compiled to target 10.8 here because the Xcode - # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - x86_64-apple-build: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - SCRIPT: make ci-test-prepare - -- job: macOS_test - dependsOn: macOS_build - timeoutInMinutes: 180 - variables: - TAR_COMMAND: gtar - SKIP_PREP: 1 - KEEP_STAGE: '--keep-stage 0 --keep-stage 1' - SKIP_LLVM_BUILD: 1 - pool: - vmImage: macos-10.13 - steps: - - checkout: self - fetchDepth: 2 - - bash: brew install gnu-tar - displayName: install a tar that works well - - - template: steps/show-disk-usage.yml - - - task: DownloadPipelineArtifact@1 - inputs: - artifactName: $(ARTIFACT_NAME) - downloadPath: $(System.DefaultWorkingDirectory) - # - task: DownloadPipelineArtifact@1 - # inputs: - # artifactName: $(ARTIFACT_NAME) - # downloadPath: $(System.DefaultWorkingDirectory) - # buildType: specific - # project: rust - # pipeline: 8 - # buildVersionToDownload: specific - # buildId: 200 - - # - template: steps/macos.yml - - template: steps/show-disk-usage.yml - - bash: $TAR_COMMAND -tvf ./drop.tar - - bash: $TAR_COMMAND -xf ./drop.tar - - bash: rm ./drop.tar - - template: steps/show-disk-usage.yml - - bash: | - export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ - mkdir -p $HOME/rustsrc - echo "##vso[task.setvariable variable=PATH;]$PATH" - - curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin - chmod +x /usr/local/bin/sccache - - curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin - chmod +x /usr/local/bin/stamp - - export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang - echo "##vso[task.setvariable variable=CC]$CC" - - export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ - echo "##vso[task.setvariable variable=CXX]$CXX" - - echo "##vso[task.setvariable variable=AR]ar" - displayName: Prep - - - bash: brew install gnu-tar - displayName: install a tar that works well - - - bash: | - brew update - brew install xz - brew install swig - condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist')) - displayName: Install xz and swigw - - - bash: | - export RUN_SCRIPT="src/ci/run.sh" - echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" - displayName: Prepare run script (init and run) - - - template: steps/show-environment-variables.yml - - - template: steps/verify-publish-toolstate.yml - - - template: steps/run-script.yml - - strategy: - matrix: - # macOS builders. These are placed near the beginning because they are very - # slow to run. - - # OSX builders running tests, these run the full test suite. - # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some - # runners that run `//ignore-debug` tests. - # - # Note that the compiler is compiled to target 10.8 here because the Xcode - # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - x86_64-apple-A: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-A - x86_64-apple-B: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-B - x86_64-apple-C: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-C - x86_64-apple-D: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-D - x86_64-apple-E: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-E - x86_64-apple-F: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-F - x86_64-apple-G: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-G - x86_64-apple-H: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - ARTIFACT_NAME: x86_64-apple-build - SCRIPT: make ci-resume-subset-H diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 189536cff5f4a..facc592422b8e 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -3,8 +3,8 @@ # pr: none -# trigger: -# - auto + trigger: + - auto variables: - group: caching diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index c47c0f765f0fe..1e01d68fb3a61 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -39,7 +39,8 @@ The script accepts commands, flags, and arguments to determine what to do: ``` If files are dirty that would normally be rebuilt from stage 0, that can be - overridden using `--keep-stage 0`. + overridden using `--keep-stage 0`. Using `--keep-stage n` will skip all steps + that belong to stage n or earlier: ``` # keep old build products for stage 0 and build stage 1 diff --git a/src/bootstrap/mk/Makefile.in b/src/bootstrap/mk/Makefile.in index 29b0cdaef63fe..ea05b30eceff9 100644 --- a/src/bootstrap/mk/Makefile.in +++ b/src/bootstrap/mk/Makefile.in @@ -14,10 +14,9 @@ endif BOOTSTRAP := $(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap.py -build: +all: $(Q)$(BOOTSTRAP) build $(BOOTSTRAP_ARGS) - -all: build doc + $(Q)$(BOOTSTRAP) doc $(BOOTSTRAP_ARGS) help: $(Q)echo 'Welcome to the rustbuild build system!' @@ -81,75 +80,10 @@ TESTS_IN_2 := \ src/test/run-pass-fulldeps \ src/tools/linkchecker -MIN_TEST := \ - src/test/debuginfo - ci-subset-1: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) ci-subset-2: $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) -ci-test-prepare: - $(Q)$(BOOTSTRAP) test $(MIN_TEST) - -ci-resume-subset-1: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_2:%=--exclude %) -ci-resume-subset-2: - $(Q)$(BOOTSTRAP) test $(TESTS_IN_2) - -TESTS_IN_B := \ - src/tools/linkchecker - -TESTS_IN_C := \ - src/test/run-pass \ - src/test/run-pass-fulldeps - -TESTS_IN_D := \ - src/test/compile-fail \ - src/test/rustdoc \ - src/test/pretty - -TESTS_IN_E := \ - src/test/ui - -TESTS_IN_F := \ - src/test/run-fail \ - src/liballoc \ - src/libcore - -TESTS_IN_G := \ - src/tools/rustdoc \ - src/test/rustdoc-js-std \ - src/test/run-make-fulldeps \ - src/libstd - -TESTS_IN_H := \ - src/librustc_driver - -ci-resume-subset-A: - $(Q)$(BOOTSTRAP) test \ - $(KEEP_STAGE) \ - $(TESTS_IN_B:%=--exclude %) \ - $(TESTS_IN_C:%=--exclude %) \ - $(TESTS_IN_D:%=--exclude %) \ - $(TESTS_IN_E:%=--exclude %) \ - $(TESTS_IN_F:%=--exclude %) \ - $(TESTS_IN_G:%=--exclude %) \ - $(TESTS_IN_H:%=--exclude %) \ - $(MIN_TEST:%=--exclude %) -ci-resume-subset-B: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_B) -ci-resume-subset-C: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_C) -ci-resume-subset-D: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_D) -ci-resume-subset-E: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_E) -ci-resume-subset-F: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_F) -ci-resume-subset-G: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_G) -ci-resume-subset-H: - $(Q)$(BOOTSTRAP) test $(KEEP_STAGE) $(TESTS_IN_H) .PHONY: dist diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 62a8b7aad031d..5777331b9bfd5 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -81,13 +81,6 @@ impl Step for Llvm { (info, "src/llvm-project/llvm", builder.llvm_out(target), dir.join("bin")) }; - let build_llvm_config = llvm_config_ret_dir - .join(exe("llvm-config", &*builder.config.build)); - - if env::var_os("SKIP_LLVM_BUILD").is_some() { - return build_llvm_config - } - if !llvm_info.is_git() { println!( "git could not determine the LLVM submodule commit hash. \ @@ -95,6 +88,8 @@ impl Step for Llvm { ); } + let build_llvm_config = llvm_config_ret_dir + .join(exe("llvm-config", &*builder.config.build)); let done_stamp = out_dir.join("llvm-finished-building"); if let Some(llvm_commit) = llvm_info.sha() { diff --git a/src/ci/run.sh b/src/ci/run.sh index 08ede122d65c0..a51c2da3cbdca 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -90,25 +90,23 @@ if [ "$RUN_CHECK_WITH_PARALLEL_QUERIES" != "" ]; then rm -rf build fi -if [ "$SKIP_PREP" != "1" ]; then - travis_fold start configure - travis_time_start - $SRC/configure $RUST_CONFIGURE_ARGS - travis_fold end configure - travis_time_finish - - travis_fold start make-prepare - travis_time_start - retry make prepare - travis_fold end make-prepare - travis_time_finish - - travis_fold start check-bootstrap - travis_time_start - make check-bootstrap - travis_fold end check-bootstrap - travis_time_finish -fi +travis_fold start configure +travis_time_start +$SRC/configure $RUST_CONFIGURE_ARGS +travis_fold end configure +travis_time_finish + +travis_fold start make-prepare +travis_time_start +retry make prepare +travis_fold end make-prepare +travis_time_finish + +travis_fold start check-bootstrap +travis_time_start +make check-bootstrap +travis_fold end check-bootstrap +travis_time_finish # Display the CPU and memory information. This helps us know why the CI timing # is fluctuating. From aab0bb47fa102b57a7e7c35af78c3f0cb79d05eb Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 12 May 2019 18:00:38 +0200 Subject: [PATCH 07/57] ci: merge the i1686-gnu images --- .azure-pipelines/auto.yml | 6 ++---- src/ci/docker/i686-gnu-2/Dockerfile | 21 ------------------- .../{i686-gnu-1 => i686-gnu}/Dockerfile | 9 ++++++-- 3 files changed, 9 insertions(+), 27 deletions(-) delete mode 100644 src/ci/docker/i686-gnu-2/Dockerfile rename src/ci/docker/{i686-gnu-1 => i686-gnu}/Dockerfile (54%) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index facc592422b8e..e8959e3f17c17 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -126,10 +126,8 @@ jobs: asmjs: IMAGE: asmjs - i686-gnu-1: - IMAGE: i686-gnu-1 - i686-gnu-2: - IMAGE: i686-gnu-2 + i686-gnu: + IMAGE: i686-gnu i686-gnu-nopt: IMAGE: i686-gnu-nopt test-various: diff --git a/src/ci/docker/i686-gnu-2/Dockerfile b/src/ci/docker/i686-gnu-2/Dockerfile deleted file mode 100644 index b1a0ea6f0e99e..0000000000000 --- a/src/ci/docker/i686-gnu-2/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++-multilib \ - make \ - file \ - curl \ - ca-certificates \ - python2.7 \ - git \ - cmake \ - sudo \ - gdb \ - xz-utils - - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu -ENV SCRIPT make ci-subset-2 diff --git a/src/ci/docker/i686-gnu-1/Dockerfile b/src/ci/docker/i686-gnu/Dockerfile similarity index 54% rename from src/ci/docker/i686-gnu-1/Dockerfile rename to src/ci/docker/i686-gnu/Dockerfile index 1a04b8af4c17b..17441ddb4546b 100644 --- a/src/ci/docker/i686-gnu-1/Dockerfile +++ b/src/ci/docker/i686-gnu/Dockerfile @@ -18,5 +18,10 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu -ENV SCRIPT make ci-subset-1 - +# Exclude some tests that are unlikely to be platform specific, to speed up +# this slow job. +ENV SCRIPT python2.7 ../x.py test \ + --exclude src/bootstrap \ + --exclude src/test/rustdoc-js \ + --exclude src/tools/error_index_generator \ + --exclude src/tools/linkchecker From 94f7660ab376c91c19eee67af7a9a41b268679b6 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 12 May 2019 18:16:42 +0200 Subject: [PATCH 08/57] ci: re-enable fds-are-cloexec test on macOS --- src/test/run-pass/fds-are-cloexec.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/run-pass/fds-are-cloexec.rs b/src/test/run-pass/fds-are-cloexec.rs index bcfc3245db30e..3bc0ceb5cf2fa 100644 --- a/src/test/run-pass/fds-are-cloexec.rs +++ b/src/test/run-pass/fds-are-cloexec.rs @@ -3,7 +3,6 @@ // ignore-cloudabi no processes // ignore-emscripten no processes // ignore-haiku -// ignore-macos #![feature(rustc_private)] From 06bc0da65c1ae7ef97945e05e19e7c6ca281e146 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 12 May 2019 18:17:42 +0200 Subject: [PATCH 09/57] ci: remove test configuration --- .azure-pipelines/test.yml | 131 -------------------------------------- 1 file changed, 131 deletions(-) delete mode 100644 .azure-pipelines/test.yml diff --git a/.azure-pipelines/test.yml b/.azure-pipelines/test.yml deleted file mode 100644 index 358806fe14c9f..0000000000000 --- a/.azure-pipelines/test.yml +++ /dev/null @@ -1,131 +0,0 @@ -# -# Azure Pipelines test build for Rust on Linux, macOS, and Windows. -# - -pr: none -trigger: none - -variables: -- group: caching - -jobs: -- job: Linux - timeoutInMinutes: 180 - pool: - vmImage: ubuntu-16.04 - steps: - - template: steps/linux.yml - strategy: - matrix: - x86_64-gnu-llvm-6.0: - RUST_BACKTRACE: 1 - - dist-x86_64-linux: - DEPLOY: 1 - - # "alternate" deployments, these are "nightlies" but have LLVM assertions - # turned on, they're deployed to a different location primarily for - # additional testing. - dist-x86_64-linux-alt: - IMAGE: dist-x86_64-linux - DEPLOY_ALT: 1 - - dist-various-1: - DEPLOY: 1 - - dist-android: - DEPLOY: 1 - - dist-x86_64-freebsd: - DEPLOY: 1 - - i686-gnu-nopt: {} - test-various: {} - x86_64-gnu-tools: {} - # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) - x86_64-gnu-nopt: {} - x86_64-gnu-distcheck: {} - mingw-check: {} - -- job: macOS - timeoutInMinutes: 180 - pool: - vmImage: macos-10.13 - steps: - - template: steps/macos.yml - strategy: - matrix: - dist-x86_64-apple-alt: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY_ALT: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - # macOS builders. These are placed near the beginning because they are very - # slow to run. - - # OSX builders running tests, these run the full test suite. - # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some - # runners that run `//ignore-debug` tests. - # - # Note that the compiler is compiled to target 10.8 here because the Xcode - # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - x86_64-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - -- job: Windows - timeoutInMinutes: 180 - pool: - vmImage: 'vs2017-win2016' - steps: - - template: steps/windows.yml - strategy: - matrix: - x86_64-msvc-tools: - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - - # 32/64-bit MinGW builds. - # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - i686-mingw-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - x86_64-mingw: - MSYS_BITS: 64 - SCRIPT: python x.py test - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - - # "alternate" deployment, see .travis.yml for more info - dist-x86_64-msvc-alt: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist - DEPLOY_ALT: 1 From 5db55e6fe0a82d56819b46d58d6afb6d4686ef4f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Sun, 12 May 2019 18:19:10 +0200 Subject: [PATCH 10/57] ci: disable toolstate commits on azure --- .azure-pipelines/master.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/master.yml b/.azure-pipelines/master.yml index e30c73c5e9241..5e8f87759d8cc 100644 --- a/.azure-pipelines/master.yml +++ b/.azure-pipelines/master.yml @@ -16,7 +16,9 @@ steps: - script: | export MESSAGE_FILE=$(mktemp -t msg.XXXXXX) . src/ci/docker/x86_64-gnu-tools/repo.sh - commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" + # FIXME(pietro): committing is disabled until we switch to Azure Pipelines + # as the source of truth, or until we setup a separate test repo. + #commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" displayName: Publish toolstate env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN_SECRET) From da949780f099f48066e77235d5ccff7f430b3323 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 13 May 2019 09:55:12 +0200 Subject: [PATCH 11/57] ci: remove trailing whitespaces from azure config --- .azure-pipelines/auto.yml | 8 ++++---- .azure-pipelines/master.yml | 4 ++-- .azure-pipelines/pr.yml | 4 ++-- .azure-pipelines/steps/linux.yml | 4 ++-- .azure-pipelines/steps/macos.yml | 4 ++-- .azure-pipelines/steps/windows.yml | 10 +++++----- .azure-pipelines/try.yml | 2 +- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index e8959e3f17c17..3ee52050e2c59 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -1,6 +1,6 @@ # # Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. -# +# pr: none trigger: @@ -33,7 +33,7 @@ jobs: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 - # Linux builders, remaining docker images + # Linux builders, remaining docker images arm-android: IMAGE: arm-android @@ -157,7 +157,7 @@ jobs: steps: - checkout: self fetchDepth: 2 - - template: steps/macos.yml + - template: steps/macos.yml strategy: matrix: # macOS builders. These are placed near the beginning because they are very @@ -200,7 +200,7 @@ jobs: # # temp disabled # # https://github.com/johnterickson/rust/issues/18 # i686-apple: - # RUST_CHECK_TARGET: check + # RUST_CHECK_TARGET: check # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 # MACOSX_DEPLOYMENT_TARGET: 10.8 diff --git a/.azure-pipelines/master.yml b/.azure-pipelines/master.yml index 5e8f87759d8cc..624b5e8fab039 100644 --- a/.azure-pipelines/master.yml +++ b/.azure-pipelines/master.yml @@ -9,7 +9,7 @@ trigger: pool: vmImage: ubuntu-16.04 -steps: +steps: - checkout: self fetchDepth: 2 @@ -19,6 +19,6 @@ steps: # FIXME(pietro): committing is disabled until we switch to Azure Pipelines # as the source of truth, or until we setup a separate test repo. #commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN" - displayName: Publish toolstate + displayName: Publish toolstate env: TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN_SECRET) diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index ffdb4301e9350..5161b8aafbf86 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,6 +1,6 @@ # # Azure Pipelines pull request build for Rust -# +# trigger: none pr: @@ -20,7 +20,7 @@ jobs: matrix: x86_64-gnu-llvm-6.0: RUST_BACKTRACE: 1 - + x86_64-gnu-tools: {} # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) mingw-check: {} diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml index b347549ccb7d5..3e83f0dcaaa83 100644 --- a/.azure-pipelines/steps/linux.yml +++ b/.azure-pipelines/steps/linux.yml @@ -7,10 +7,10 @@ steps: - bash: | sudo apt install gdb - + curl -fo $HOME/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-unknown-linux-musl chmod +x $HOME/stamp - + export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME echo "##vso[task.setvariable variable=PATH;]$PATH" diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml index e88b5613e46d4..d1adc34039212 100644 --- a/.azure-pipelines/steps/macos.yml +++ b/.azure-pipelines/steps/macos.yml @@ -25,9 +25,9 @@ steps: displayName: install a tar that works well - bash: | - curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - displayName: Download clang - + - bash: | brew update brew install xz diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml index e3c06d543a972..cc11e7a08369f 100644 --- a/.azure-pipelines/steps/windows.yml +++ b/.azure-pipelines/steps/windows.yml @@ -7,7 +7,7 @@ steps: git submodule export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" - + - script: | REM echo hack as drive D is too small IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( @@ -19,7 +19,7 @@ steps: set MSYS_PATH=%CD%\citools\msys64 choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y set PATH=%MSYS_PATH%\usr\bin;%PATH% - pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar + pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar IF "%MINGW_URL%"=="" ( IF "%MSYS_BITS%"=="32" pacman -S --noconfirm --needed mingw-w64-i686-toolchain mingw-w64-i686-cmake mingw-w64-i686-gcc mingw-w64-i686-python2 IF "%MSYS_BITS%"=="64" pacman -S --noconfirm --needed mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-python2 @@ -135,7 +135,7 @@ steps: - script: | REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8 - IF "%VCVARS_BAT%" NEQ "" ( + IF "%VCVARS_BAT%" NEQ "" ( CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" ) @@ -144,10 +144,10 @@ steps: set | findstr /v SCCACHE_AZURE_CONNECTION_STRING if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc - + sh src/ci/init_repo.sh . /d/cache/rustsrc sh src/ci/run.sh - env: + env: CI: true CI_JOB_NAME: $(System.JobDisplayName) SRC: . diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index eaa50729b0d27..229927f4d34ee 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -1,6 +1,6 @@ # # Azure Pipelines "try" branch build for Rust -# +# pr: none trigger: From 694ea7646af0003a8fc38cfb11cbeb9c3ffb9d38 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 13 May 2019 10:09:51 +0200 Subject: [PATCH 12/57] ci: remove links to johnterickson repo --- .azure-pipelines/auto.yml | 38 ++++++++++++++---------------- .azure-pipelines/steps/windows.yml | 2 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 3ee52050e2c59..748a6a9e74863 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -197,26 +197,24 @@ jobs: NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 - # # temp disabled - # # https://github.com/johnterickson/rust/issues/18 - # i686-apple: - # RUST_CHECK_TARGET: check - # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - # MACOSX_DEPLOYMENT_TARGET: 10.8 - # MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - # NO_LLVM_ASSERTIONS: 1 - # NO_DEBUG_ASSERTIONS: 1 - - # dist-i686-apple: - # RUST_CHECK_TARGET: dist - # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - # DEPLOY: 1 - # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - # MACOSX_DEPLOYMENT_TARGET: 10.7 - # NO_LLVM_ASSERTIONS: 1 - # NO_DEBUG_ASSERTIONS: 1 - # DIST_REQUIRE_ALL_TOOLS: 1 + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/windows.yml index cc11e7a08369f..5fb1da2083f70 100644 --- a/.azure-pipelines/steps/windows.yml +++ b/.azure-pipelines/steps/windows.yml @@ -134,7 +134,7 @@ steps: - template: show-environment-variables.yml - script: | - REM echo force the specific VS version https://github.com/johnterickson/rust/issues/8 + REM echo force the specific VS version IF "%VCVARS_BAT%" NEQ "" ( CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" ) From 807b7dd7863c4f662dbbd2091c34707664dc3d66 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 13 May 2019 10:13:23 +0200 Subject: [PATCH 13/57] ci: disable builders not useful during the evaluation --- .azure-pipelines/auto.yml | 516 ++++++++++++++++++------------------ .azure-pipelines/master.yml | 2 +- .azure-pipelines/pr.yml | 6 +- 3 files changed, 262 insertions(+), 262 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 748a6a9e74863..8202578b6701b 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -33,120 +33,120 @@ jobs: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 - # Linux builders, remaining docker images - arm-android: - IMAGE: arm-android - - armhf-gnu: - IMAGE: armhf-gnu - - dist-various-1: - IMAGE: dist-various-1 - DEPLOY: 1 - - dist-various-2: - IMAGE: dist-various-2 - DEPLOY: 1 - - dist-aarch64-linux: - IMAGE: dist-aarch64-linux - DEPLOY: 1 - - dist-android: - IMAGE: dist-android - DEPLOY: 1 - - dist-arm-linux: - IMAGE: dist-arm-linux - DEPLOY: 1 - - dist-armhf-linux: - IMAGE: dist-armhf-linux - DEPLOY: 1 - - dist-armv7-linux: - IMAGE: dist-armv7-linux - DEPLOY: 1 - - dist-i586-gnu-i586-i686-musl: - IMAGE: dist-i586-gnu-i586-i686-musl - DEPLOY: 1 - - dist-i686-freebsd: - IMAGE: dist-i686-freebsd - DEPLOY: 1 - - dist-i686-linux: - IMAGE: dist-i686-linux - DEPLOY: 1 - - dist-mips-linux: - IMAGE: dist-mips-linux - DEPLOY: 1 - - dist-mips64-linux: - IMAGE: dist-mips64-linux - DEPLOY: 1 - - dist-mips64el-linux: - IMAGE: dist-mips64el-linux - DEPLOY: 1 - - dist-mipsel-linux: - IMAGE: dist-mipsel-linux - DEPLOY: 1 - - dist-powerpc-linux: - IMAGE: dist-powerpc-linux - DEPLOY: 1 - - dist-powerpc64-linux: - IMAGE: dist-powerpc64-linux - DEPLOY: 1 - - dist-powerpc64le-linux: - IMAGE: dist-powerpc64le-linux - DEPLOY: 1 - - dist-s390x-linux: - IMAGE: dist-s390x-linux - DEPLOY: 1 - - dist-x86_64-freebsd: - IMAGE: dist-x86_64-freebsd - DEPLOY: 1 - - dist-x86_64-musl: - IMAGE: dist-x86_64-musl - DEPLOY: 1 - - dist-x86_64-netbsd: - IMAGE: dist-x86_64-netbsd - DEPLOY: 1 - - asmjs: - IMAGE: asmjs - i686-gnu: - IMAGE: i686-gnu - i686-gnu-nopt: - IMAGE: i686-gnu-nopt - test-various: - IMAGE: test-various - x86_64-gnu: - IMAGE: x86_64-gnu - x86_64-gnu-full-bootstrap: - IMAGE: x86_64-gnu-full-bootstrap - x86_64-gnu-aux: - IMAGE: x86_64-gnu-aux - x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools - # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) - x86_64-gnu-debug: - IMAGE: x86_64-gnu-debug - x86_64-gnu-nopt: - IMAGE: x86_64-gnu-nopt - x86_64-gnu-distcheck: - IMAGE: x86_64-gnu-distcheck +# # Linux builders, remaining docker images +# arm-android: +# IMAGE: arm-android +# +# armhf-gnu: +# IMAGE: armhf-gnu +# +# dist-various-1: +# IMAGE: dist-various-1 +# DEPLOY: 1 +# +# dist-various-2: +# IMAGE: dist-various-2 +# DEPLOY: 1 +# +# dist-aarch64-linux: +# IMAGE: dist-aarch64-linux +# DEPLOY: 1 +# +# dist-android: +# IMAGE: dist-android +# DEPLOY: 1 +# +# dist-arm-linux: +# IMAGE: dist-arm-linux +# DEPLOY: 1 +# +# dist-armhf-linux: +# IMAGE: dist-armhf-linux +# DEPLOY: 1 +# +# dist-armv7-linux: +# IMAGE: dist-armv7-linux +# DEPLOY: 1 +# +# dist-i586-gnu-i586-i686-musl: +# IMAGE: dist-i586-gnu-i586-i686-musl +# DEPLOY: 1 +# +# dist-i686-freebsd: +# IMAGE: dist-i686-freebsd +# DEPLOY: 1 +# +# dist-i686-linux: +# IMAGE: dist-i686-linux +# DEPLOY: 1 +# +# dist-mips-linux: +# IMAGE: dist-mips-linux +# DEPLOY: 1 +# +# dist-mips64-linux: +# IMAGE: dist-mips64-linux +# DEPLOY: 1 +# +# dist-mips64el-linux: +# IMAGE: dist-mips64el-linux +# DEPLOY: 1 +# +# dist-mipsel-linux: +# IMAGE: dist-mipsel-linux +# DEPLOY: 1 +# +# dist-powerpc-linux: +# IMAGE: dist-powerpc-linux +# DEPLOY: 1 +# +# dist-powerpc64-linux: +# IMAGE: dist-powerpc64-linux +# DEPLOY: 1 +# +# dist-powerpc64le-linux: +# IMAGE: dist-powerpc64le-linux +# DEPLOY: 1 +# +# dist-s390x-linux: +# IMAGE: dist-s390x-linux +# DEPLOY: 1 +# +# dist-x86_64-freebsd: +# IMAGE: dist-x86_64-freebsd +# DEPLOY: 1 +# +# dist-x86_64-musl: +# IMAGE: dist-x86_64-musl +# DEPLOY: 1 +# +# dist-x86_64-netbsd: +# IMAGE: dist-x86_64-netbsd +# DEPLOY: 1 +# +# asmjs: +# IMAGE: asmjs +# i686-gnu: +# IMAGE: i686-gnu +# i686-gnu-nopt: +# IMAGE: i686-gnu-nopt +# test-various: +# IMAGE: test-various +# x86_64-gnu: +# IMAGE: x86_64-gnu +# x86_64-gnu-full-bootstrap: +# IMAGE: x86_64-gnu-full-bootstrap +# x86_64-gnu-aux: +# IMAGE: x86_64-gnu-aux +# x86_64-gnu-tools: +# IMAGE: x86_64-gnu-tools +# # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) +# x86_64-gnu-debug: +# IMAGE: x86_64-gnu-debug +# x86_64-gnu-nopt: +# IMAGE: x86_64-gnu-nopt +# x86_64-gnu-distcheck: +# IMAGE: x86_64-gnu-distcheck mingw-check: IMAGE: mingw-check @@ -188,33 +188,33 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - dist-x86_64-apple-alt: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY_ALT: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - i686-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - dist-i686-apple: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 +# dist-x86_64-apple-alt: +# RUST_CHECK_TARGET: dist +# RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc +# DEPLOY_ALT: 1 +# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 +# MACOSX_DEPLOYMENT_TARGET: 10.7 +# NO_LLVM_ASSERTIONS: 1 +# NO_DEBUG_ASSERTIONS: 1 +# +# i686-apple: +# RUST_CHECK_TARGET: check +# RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc +# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 +# MACOSX_DEPLOYMENT_TARGET: 10.8 +# MACOSX_STD_DEPLOYMENT_TARGET: 10.7 +# NO_LLVM_ASSERTIONS: 1 +# NO_DEBUG_ASSERTIONS: 1 +# +# dist-i686-apple: +# RUST_CHECK_TARGET: dist +# RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc +# DEPLOY: 1 +# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 +# MACOSX_DEPLOYMENT_TARGET: 10.7 +# NO_LLVM_ASSERTIONS: 1 +# NO_DEBUG_ASSERTIONS: 1 +# DIST_REQUIRE_ALL_TOOLS: 1 @@ -226,89 +226,89 @@ jobs: - template: steps/windows.yml strategy: matrix: - # 32/64 bit MSVC tests - x86_64-msvc-1: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-1 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-msvc-2: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-2 - i686-msvc-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-1 - i686-msvc-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-2 - # MSVC aux tests - x86_64-msvc-aux: - MSYS_BITS: 64 - RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - x86_64-msvc-cargo: - MSYS_BITS: 64 - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - VCVARS_BAT: vcvars64.bat - # MSVC tools tests - x86_64-msvc-tools: - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - - # 32/64-bit MinGW builds. - # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - i686-mingw-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - i686-mingw-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-2 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - x86_64-mingw-1: - MSYS_BITS: 64 - SCRIPT: make ci-subset-1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-mingw-2: - MSYS_BITS: 64 - SCRIPT: make ci-subset-2 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 +# # 32/64 bit MSVC tests +# x86_64-msvc-1: +# MSYS_BITS: 64 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler +# SCRIPT: make ci-subset-1 +# # FIXME(#59637) +# NO_DEBUG_ASSERTIONS: 1 +# NO_LLVM_ASSERTIONS: 1 +# x86_64-msvc-2: +# MSYS_BITS: 64 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler +# SCRIPT: make ci-subset-2 +# i686-msvc-1: +# MSYS_BITS: 32 +# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc +# SCRIPT: make ci-subset-1 +# i686-msvc-2: +# MSYS_BITS: 32 +# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc +# SCRIPT: make ci-subset-2 +# # MSVC aux tests +# x86_64-msvc-aux: +# MSYS_BITS: 64 +# RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc +# x86_64-msvc-cargo: +# MSYS_BITS: 64 +# SCRIPT: python x.py test src/tools/cargotest src/tools/cargo +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc +# VCVARS_BAT: vcvars64.bat +# # MSVC tools tests +# x86_64-msvc-tools: +# MSYS_BITS: 64 +# SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri +# +# # 32/64-bit MinGW builds. +# # +# # We are using MinGW with posix threads since LLVM does not compile with +# # the win32 threads version due to missing support for C++'s std::thread. +# # +# # Instead of relying on the MinGW version installed on appveryor we download +# # and install one ourselves so we won't be surprised by changes to appveyor's +# # build image. +# # +# # Finally, note that the downloads below are all in the `rust-lang-ci` S3 +# # bucket, but they cleraly didn't originate there! The downloads originally +# # came from the mingw-w64 SourceForge download site. Unfortunately +# # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. +# i686-mingw-1: +# MSYS_BITS: 32 +# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu +# SCRIPT: make ci-subset-1 +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z +# MINGW_DIR: mingw32 +# # FIXME(#59637) +# NO_DEBUG_ASSERTIONS: 1 +# NO_LLVM_ASSERTIONS: 1 +# i686-mingw-2: +# MSYS_BITS: 32 +# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu +# SCRIPT: make ci-subset-2 +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z +# MINGW_DIR: mingw32 +# x86_64-mingw-1: +# MSYS_BITS: 64 +# SCRIPT: make ci-subset-1 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z +# MINGW_DIR: mingw64 +# # FIXME(#59637) +# NO_DEBUG_ASSERTIONS: 1 +# NO_LLVM_ASSERTIONS: 1 +# x86_64-mingw-2: +# MSYS_BITS: 64 +# SCRIPT: make ci-subset-2 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z +# MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -320,37 +320,37 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - dist-i686-msvc: - RUST_CONFIGURE_ARGS: > - --build=i686-pc-windows-msvc - --target=i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-i686-mingw: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - SCRIPT: python x.py dist - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-x86_64-mingw: - MSYS_BITS: 64 - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - # "alternate" deployment, see .travis.yml for more info - dist-x86_64-msvc-alt: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist - DEPLOY_ALT: 1 +# dist-i686-msvc: +# RUST_CONFIGURE_ARGS: > +# --build=i686-pc-windows-msvc +# --target=i586-pc-windows-msvc +# --enable-full-tools +# --enable-profiler +# SCRIPT: python x.py dist +# DIST_REQUIRE_ALL_TOOLS: 1 +# DEPLOY: 1 +# dist-i686-mingw: +# MSYS_BITS: 32 +# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools +# SCRIPT: python x.py dist +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z +# MINGW_DIR: mingw32 +# DIST_REQUIRE_ALL_TOOLS: 1 +# DEPLOY: 1 +# dist-x86_64-mingw: +# MSYS_BITS: 64 +# SCRIPT: python x.py dist +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools +# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror +# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z +# MINGW_DIR: mingw64 +# DIST_REQUIRE_ALL_TOOLS: 1 +# DEPLOY: 1 +# +# # "alternate" deployment, see .travis.yml for more info +# dist-x86_64-msvc-alt: +# MSYS_BITS: 64 +# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler +# SCRIPT: python x.py dist +# DEPLOY_ALT: 1 diff --git a/.azure-pipelines/master.yml b/.azure-pipelines/master.yml index 624b5e8fab039..3f3025ad65b66 100644 --- a/.azure-pipelines/master.yml +++ b/.azure-pipelines/master.yml @@ -4,7 +4,7 @@ pr: none trigger: -- master + - master pool: vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 5161b8aafbf86..ec9396e61e558 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -21,6 +21,6 @@ jobs: x86_64-gnu-llvm-6.0: RUST_BACKTRACE: 1 - x86_64-gnu-tools: {} - # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) - mingw-check: {} +# x86_64-gnu-tools: {} +# # if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) +# mingw-check: {} From 4965ffdb7a0dd825e251bcd1cc83c20ee6c8e45d Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 13 May 2019 15:19:29 +0200 Subject: [PATCH 14/57] Update src/bootstrap/util.rs Co-Authored-By: Jake Goulding --- src/bootstrap/util.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs index e79700d92d89c..f22f0559265b1 100644 --- a/src/bootstrap/util.rs +++ b/src/bootstrap/util.rs @@ -326,7 +326,7 @@ pub enum CiEnv { Travis, /// The AppVeyor environment, for Windows builds. AppVeyor, - /// The Azure Pipelines environment, for Linux (including Docker), Window, and macOS builds. + /// The Azure Pipelines environment, for Linux (including Docker), Windows, and macOS builds. AzurePipelines, } From 92bf1e6661b76e2431132f370f5452fc0ad3448a Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 15 May 2019 14:44:25 +0200 Subject: [PATCH 15/57] ci: update azure variable groups --- .azure-pipelines/auto.yml | 2 +- .azure-pipelines/pr.yml | 3 --- .azure-pipelines/try.yml | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 8202578b6701b..2d240f8191a03 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -7,7 +7,7 @@ pr: none - auto variables: -- group: caching +- group: prod-credentials jobs: - job: Linux diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index ec9396e61e558..fe20e35e344ab 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -6,9 +6,6 @@ trigger: none pr: - master # FIXME: really just want any branch, but want an explicit "pr" property set so it's clear -variables: -- group: caching-pr - jobs: - job: Linux timeoutInMinutes: 180 diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 229927f4d34ee..8a4f488302968 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -7,7 +7,7 @@ trigger: - try variables: -- group: caching +- group: prod-credentials jobs: - job: Linux From 30b008f5337332e0a1f941113d14f90fa876315e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 15 May 2019 19:37:02 +0200 Subject: [PATCH 16/57] ci: remove stray echo --- src/ci/docker/x86_64-gnu-tools/checktools.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh index c7e64cd8c3e79..af0198705a2fe 100755 --- a/src/ci/docker/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh @@ -6,7 +6,7 @@ X_PY="$1" TOOLSTATE_FILE="$(realpath $2)" OS="$3" COMMIT="$(git rev-parse HEAD)" -CHANGED_FILES="$(git diff --name-status HEAD HEAD^ || echo)" +CHANGED_FILES="$(git diff --name-status HEAD HEAD^)" SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))" # ^ Number of days after the last promotion of beta. # Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. From bf639a1c34f6cee0cfb3b2ae5264b5f1261eb9c9 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 15 May 2019 19:38:26 +0200 Subject: [PATCH 17/57] ci: remove outdated comment --- .azure-pipelines/steps/linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml index 3e83f0dcaaa83..238fd1d5ea785 100644 --- a/.azure-pipelines/steps/linux.yml +++ b/.azure-pipelines/steps/linux.yml @@ -17,7 +17,6 @@ steps: mkdir -p $HOME/rustsrc displayName: Prep -# FIXME: using the job display name feels brittle. Is there a better variable that represents the current matrix key? - bash: | export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/docker/run.sh $IMAGE" echo "##vso[task.setvariable variable=IMAGE]$IMAGE" From a89e8d2a66bd6016b3a516b5483cb89580b79fcb Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 20 May 2019 09:21:26 +0200 Subject: [PATCH 18/57] ci: remove yet another outdated comment --- .azure-pipelines/auto.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 2d240f8191a03..05d33e2f9d59f 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -160,9 +160,6 @@ jobs: - template: steps/macos.yml strategy: matrix: - # macOS builders. These are placed near the beginning because they are very - # slow to run. - # OSX builders running tests, these run the full test suite. # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some # runners that run `//ignore-debug` tests. From f23c1ba74ba22b076536885d98bec63a9da6572c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 20 May 2019 09:22:47 +0200 Subject: [PATCH 19/57] ci: use ##vso[task.prependpath] on azure --- .azure-pipelines/steps/linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml index 238fd1d5ea785..36f980cf992db 100644 --- a/.azure-pipelines/steps/linux.yml +++ b/.azure-pipelines/steps/linux.yml @@ -12,7 +12,9 @@ steps: chmod +x $HOME/stamp export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME - echo "##vso[task.setvariable variable=PATH;]$PATH" + echo "##vso[task.prependpath]$HOME/.local/bin" + echo "##vso[task.prependpath]$HOME/Library/Python/2.7/bin" + echo "##vso[task.prependpath]$HOME" mkdir -p $HOME/rustsrc displayName: Prep From d27cb827c60e70eb5415a45e882c8f0a1fa72f3e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 20 May 2019 09:38:11 +0200 Subject: [PATCH 20/57] ci: add an image for dist-x86_64-linux try on azure --- .azure-pipelines/try.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 8a4f488302968..bdd36e8555d7d 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -17,6 +17,7 @@ jobs: strategy: matrix: dist-x86_64-linux: + IMAGE: dist-x86_64-linux DEPLOY: 1 # "alternate" deployments, these are "nightlies" but have LLVM assertions From 9f37b3a4c251d7caaa2870952e217be369e8e9c1 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 20 May 2019 10:07:17 +0200 Subject: [PATCH 21/57] ci: deploy artifacts on azure --- .azure-pipelines/steps/run-script.yml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml index cb8e8a629a7e6..bf18518e6b1ca 100644 --- a/.azure-pipelines/steps/run-script.yml +++ b/.azure-pipelines/steps/run-script.yml @@ -1,13 +1,14 @@ steps: -# Log time information from this machine and an external machine for insight into possible -# clock drift. Timezones don't matter since relative deltas give all the necessary info. + - bash: | + # Log time information from this machine and an external machine for insight into possible + # clock drift. Timezones don't matter since relative deltas give all the necessary info. date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) - displayName: Log time information (before) -- bash: | which sccache stamp sh -x -c "$RUN_SCRIPT" + + date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) env: CI: true CI_JOB_NAME: $(IMAGE) @@ -20,5 +21,15 @@ steps: displayName: Run script - bash: | - date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) - displayName: Log time information (after) + deploy_dir=rustc-builds + if [ "$DEPLOY_ALT" == "1" ]; then + deploy_dir=rustc-builds-alt + fi + aws s3 cp --no-progress --recursive --acl public-read ./deploy s3://$DEPLOY_BUCKET/$deploy_dir + env: + # Explicitly decrypt secret variables + # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables + AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) + condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) + displayName: Upload artifacts From 79d28c203fbf46739d2e3f1f2f3e62bdf1ac5563 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 11:27:18 -0700 Subject: [PATCH 22/57] Don't bother with `stamp` utility That was just used because Travis doesn't have time stamps on all log lines, but Azure does, so no need to add our own. --- .azure-pipelines/steps/linux.yml | 3 --- .azure-pipelines/steps/macos.yml | 3 --- .azure-pipelines/steps/run-script.yml | 2 +- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml index 36f980cf992db..70327de92e812 100644 --- a/.azure-pipelines/steps/linux.yml +++ b/.azure-pipelines/steps/linux.yml @@ -8,9 +8,6 @@ steps: - bash: | sudo apt install gdb - curl -fo $HOME/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-unknown-linux-musl - chmod +x $HOME/stamp - export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME echo "##vso[task.prependpath]$HOME/.local/bin" echo "##vso[task.prependpath]$HOME/Library/Python/2.7/bin" diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml index d1adc34039212..5976f31502f7a 100644 --- a/.azure-pipelines/steps/macos.yml +++ b/.azure-pipelines/steps/macos.yml @@ -9,9 +9,6 @@ steps: curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache - curl -fo /usr/local/bin/stamp https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin - chmod +x /usr/local/bin/stamp - export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang echo "##vso[task.setvariable variable=CC]$CC" diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml index bf18518e6b1ca..0e6af4d6d12bd 100644 --- a/.azure-pipelines/steps/run-script.yml +++ b/.azure-pipelines/steps/run-script.yml @@ -6,7 +6,7 @@ steps: date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) which sccache - stamp sh -x -c "$RUN_SCRIPT" + "$RUN_SCRIPT" date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) env: From 1be9fe6a449148d2f31b3e82f4d983630442d981 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:21:14 -0700 Subject: [PATCH 23/57] Refactor azure pipelines configuration This commit is intended to go through and review/refactor the azure pipelines configuration we have. The major changes are: * The separate `{windows,macos,linux}.yml` files are now all merged into one `run.yml`. This allows a shared "master flow" for all platforms with divergence only where necessary. * Some install steps have been separated as `install-*.yml` scripts, where each script internally matches on the appropriate OS and then delegates accordingly. * Some various bits and pieces of cruft have been removed which were artifacts of Travis's setup or similar. --- .azure-pipelines/auto.yml | 9 +- .azure-pipelines/pr.yml | 3 +- .azure-pipelines/steps/install-clang.yml | 40 +++++++ .azure-pipelines/steps/install-sccache.yml | 21 ++++ ...ows.yml => install-windows-build-deps.yml} | 103 ++++-------------- .azure-pipelines/steps/linux.yml | 32 ------ .azure-pipelines/steps/macos.yml | 44 -------- .azure-pipelines/steps/run-script.yml | 35 ------ .azure-pipelines/steps/run.yml | 89 +++++++++++++++ .azure-pipelines/steps/show-disk-usage.yml | 5 - .../steps/show-environment-variables.yml | 3 - .../steps/verify-publish-toolstate.yml | 9 -- .azure-pipelines/try.yml | 3 +- 13 files changed, 174 insertions(+), 222 deletions(-) create mode 100644 .azure-pipelines/steps/install-clang.yml create mode 100644 .azure-pipelines/steps/install-sccache.yml rename .azure-pipelines/steps/{windows.yml => install-windows-build-deps.yml} (52%) delete mode 100644 .azure-pipelines/steps/linux.yml delete mode 100644 .azure-pipelines/steps/macos.yml delete mode 100644 .azure-pipelines/steps/run-script.yml create mode 100644 .azure-pipelines/steps/run.yml delete mode 100644 .azure-pipelines/steps/show-disk-usage.yml delete mode 100644 .azure-pipelines/steps/show-environment-variables.yml diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 05d33e2f9d59f..3b6cfed70719a 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -11,11 +11,10 @@ variables: jobs: - job: Linux - timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 steps: - - template: steps/linux.yml + - template: steps/run.yml strategy: matrix: x86_64-gnu-llvm-6.0: @@ -151,13 +150,12 @@ jobs: IMAGE: mingw-check - job: macOS - timeoutInMinutes: 180 pool: vmImage: macos-10.13 steps: - checkout: self fetchDepth: 2 - - template: steps/macos.yml + - template: steps/run.yml strategy: matrix: # OSX builders running tests, these run the full test suite. @@ -216,11 +214,10 @@ jobs: - job: Windows - timeoutInMinutes: 180 pool: vmImage: 'vs2017-win2016' steps: - - template: steps/windows.yml + - template: steps/run.yml strategy: matrix: # # 32/64 bit MSVC tests diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index fe20e35e344ab..84c9454fee57f 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -8,11 +8,10 @@ pr: jobs: - job: Linux - timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 steps: - - template: steps/linux.yml + - template: steps/run.yml strategy: matrix: x86_64-gnu-llvm-6.0: diff --git a/.azure-pipelines/steps/install-clang.yml b/.azure-pipelines/steps/install-clang.yml new file mode 100644 index 0000000000000..9e3545ea93cba --- /dev/null +++ b/.azure-pipelines/steps/install-clang.yml @@ -0,0 +1,40 @@ +steps: + +- bash: | + set -e + curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - + + export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang + echo "##vso[task.setvariable variable=CC]$CC" + + export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ + echo "##vso[task.setvariable variable=CXX]$CXX" + + # Configure `AR` specifically so rustbuild doesn't try to infer it as + # `clang-ar` by accident. + echo "##vso[task.setvariable variable=AR]ar" + displayName: Install clang (OSX) + condition: eq(variables['Agent.OS'], 'Darwin') + +# If we're compiling for MSVC then we, like most other distribution builders, +# switch to clang as the compiler. This'll allow us eventually to enable LTO +# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think +# clang has an output mode compatible with MinGW that we need. If it does we +# should switch to clang for MinGW as well! +# +# Note that the LLVM installer is an NSIS installer +# +# Original downloaded here came from +# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe +- script: | + powershell -Command "iwr -outf %TEMP%\LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" + set CLANG_DIR=%CD%\citools\clang-rust + %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% + set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe + echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% + condition: and(eq(variables['Agent.OS'], 'Darwin'), eq(variables['MINGW_URL'],'')) + displayName: Install clang (Windows) + +# Note that we don't install clang on Linux since its compiler story is just so +# different. Each container has its own toolchain configured appropriately +# already. diff --git a/.azure-pipelines/steps/install-sccache.yml b/.azure-pipelines/steps/install-sccache.yml new file mode 100644 index 0000000000000..6933f4e9f2794 --- /dev/null +++ b/.azure-pipelines/steps/install-sccache.yml @@ -0,0 +1,21 @@ +steps: + +- bash: | + set -e + curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin + chmod +x /usr/local/bin/sccache + displayName: Install sccache (OSX) + condition: eq(variables['Agent.OS'], 'Darwin') + +- script: | + md sccache + powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" + echo ##vso[task.prependpath]%CD%\sccache + displayName: Install sccache (Windows) + condition: eq(variables['Agent.OS'], 'Windows_NT') + +# Note that we don't install sccache on Linux since it's installed elsewhere +# through all the containers. +# +# FIXME: we should probably install sccache outside the containers and then +# mount it inside the containers so we can centralize all installation here. diff --git a/.azure-pipelines/steps/windows.yml b/.azure-pipelines/steps/install-windows-build-deps.yml similarity index 52% rename from .azure-pipelines/steps/windows.yml rename to .azure-pipelines/steps/install-windows-build-deps.yml index 5fb1da2083f70..4eab460543bbc 100644 --- a/.azure-pipelines/steps/windows.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -1,19 +1,17 @@ steps: -- checkout: self - fetchDepth: 2 - -- bash: | - set -x - git submodule - export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') - echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" - -- script: | - REM echo hack as drive D is too small - IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( - mkdir c:\MORE_SPACE - mklink /J build c:\MORE_SPACE - ) +# FIXME: are these still needed? +# - bash: | +# set -x +# git submodule +# export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') +# echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" +# +# - script: | +# REM echo hack as drive D is too small +# IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( +# mkdir c:\MORE_SPACE +# mklink /J build c:\MORE_SPACE +# ) - script: | set MSYS_PATH=%CD%\citools\msys64 @@ -31,6 +29,7 @@ steps: echo ##vso[task.setvariable variable=MSYS_PATH]%MSYS_PATH% echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin displayName: Install msys2 + condition: eq(variables['Agent.OS'], 'Windows_NT') # If we need to download a custom MinGW, do so here and set the path # appropriately. @@ -44,28 +43,9 @@ steps: powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" 7z x -y %MINGW_ARCHIVE% > nul echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin - condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) displayName: Download custom MinGW -# If we're compiling for MSVC then we, like most other distribution builders, -# switch to clang as the compiler. This'll allow us eventually to enable LTO -# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think -# clang has an output mode compatible with MinGW that we need. If it does we -# should switch to clang for MinGW as well! -# -# Note that the LLVM installer is an NSIS installer -# -# Original downloaded here came from -# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe -- script: | - powershell -Command "iwr -outf %TEMP%\LLVM-7.0.0-win64.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/LLVM-7.0.0-win64.exe" - set CLANG_DIR=%CD%\citools\clang-rust - %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% - set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe - echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% - condition: and(succeeded(), eq(variables['MINGW_URL'],'')) - displayName: Download clang - # Here we do a pretty heinous thing which is to mangle the MinGW installation # we just had above. Currently, as of this writing, we're using MinGW-w64 # builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to @@ -87,28 +67,20 @@ steps: echo ON powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe - condition: and(succeeded(), ne(variables['MINGW_URL'],'')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) displayName: Override with 6.3.0 gdb with 6.2.0 gdb # Otherwise pull in the MinGW installed on appveyor - script: | - echo Find mingw - set PATH | findstr /i msys - set PATH | findstr /i mingw echo ##vso[task.prependpath]%MSYS_PATH%\mingw%MSYS_BITS%\bin - condition: and(succeeded(), eq(variables['MINGW_URL'],'')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Add MinGW to path - script: | copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe echo ##vso[task.prependpath]C:\Python27amd64 displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes - -- script: | - md sccache - powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" - echo ##vso[task.prependpath]%CD%\sccache - displayName: Download and install sccache + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) # Note that this is originally from the github releases patch of Ninja - script: | @@ -120,41 +92,4 @@ steps: echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% echo ##vso[task.prependpath]%CD%\ninja displayName: Download and install ninja - -- script: | - mkdir handle - powershell -Command "iwr -outf 2017-05-15-Handle.zip https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2017-05-15-Handle.zip" - 7z x -ohandle 2017-05-15-Handle.zip - del 2017-05-15-Handle.zip - set PATH=%PATH%;%CD%\handle - handle.exe -accepteula -help - echo ##vso[task.setvariable variable=PATH]%PATH% - displayName: Help debug handle issues - -- template: show-environment-variables.yml - -- script: | - REM echo force the specific VS version - IF "%VCVARS_BAT%" NEQ "" ( - CALL "C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\VC\Auxiliary\Build\%VCVARS_BAT%" - ) - - where sccache - where rev - set | findstr /v SCCACHE_AZURE_CONNECTION_STRING - - if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc - - sh src/ci/init_repo.sh . /d/cache/rustsrc - sh src/ci/run.sh - env: - CI: true - CI_JOB_NAME: $(System.JobDisplayName) - SRC: . - NO_CCACHE: 1 - - # explicitly decrypt secret variables - # see https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch - AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) - displayName: Run script + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/.azure-pipelines/steps/linux.yml b/.azure-pipelines/steps/linux.yml deleted file mode 100644 index 70327de92e812..0000000000000 --- a/.azure-pipelines/steps/linux.yml +++ /dev/null @@ -1,32 +0,0 @@ -steps: -- checkout: self - fetchDepth: 2 - -- template: show-environment-variables.yml -- template: show-disk-usage.yml - -- bash: | - sudo apt install gdb - - export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/:$HOME - echo "##vso[task.prependpath]$HOME/.local/bin" - echo "##vso[task.prependpath]$HOME/Library/Python/2.7/bin" - echo "##vso[task.prependpath]$HOME" - - mkdir -p $HOME/rustsrc - displayName: Prep - -- bash: | - export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/docker/run.sh $IMAGE" - echo "##vso[task.setvariable variable=IMAGE]$IMAGE" - echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" - displayName: Prepare run script - -- template: show-environment-variables.yml - -- bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern' - displayName: Enable core dump - -- template: verify-publish-toolstate.yml - -- template: run-script.yml diff --git a/.azure-pipelines/steps/macos.yml b/.azure-pipelines/steps/macos.yml deleted file mode 100644 index 5976f31502f7a..0000000000000 --- a/.azure-pipelines/steps/macos.yml +++ /dev/null @@ -1,44 +0,0 @@ -steps: -- template: show-disk-usage.yml - -- bash: | - export PATH=$PATH:$HOME/.local/bin:$HOME/Library/Python/2.7/bin/ - mkdir -p $HOME/rustsrc - echo "##vso[task.setvariable variable=PATH;]$PATH" - - curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin - chmod +x /usr/local/bin/sccache - - export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang - echo "##vso[task.setvariable variable=CC]$CC" - - export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ - echo "##vso[task.setvariable variable=CXX]$CXX" - - echo "##vso[task.setvariable variable=AR]ar" - displayName: Prep - -- bash: brew install gnu-tar - displayName: install a tar that works well - -- bash: | - curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - - displayName: Download clang - -- bash: | - brew update - brew install xz - brew install swig - condition: and(succeeded(), eq(variables['RUST_CHECK_TARGET'],'dist')) - displayName: Install xz and swigw - -- bash: | - export RUN_SCRIPT="$BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc && src/ci/run.sh" - echo "##vso[task.setvariable variable=RUN_SCRIPT]$RUN_SCRIPT" - displayName: Prepare run script (init and run) - -- template: show-environment-variables.yml - -- template: verify-publish-toolstate.yml - -- template: run-script.yml diff --git a/.azure-pipelines/steps/run-script.yml b/.azure-pipelines/steps/run-script.yml deleted file mode 100644 index 0e6af4d6d12bd..0000000000000 --- a/.azure-pipelines/steps/run-script.yml +++ /dev/null @@ -1,35 +0,0 @@ -steps: - -- bash: | - # Log time information from this machine and an external machine for insight into possible - # clock drift. Timezones don't matter since relative deltas give all the necessary info. - date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) - - which sccache - "$RUN_SCRIPT" - - date && (curl -fs --head https://google.com | grep ^Date: | sed 's/Date: //g' || true) - env: - CI: true - CI_JOB_NAME: $(IMAGE) - SRC: . - - # Explicitly decrypt secret variables - # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables - AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) - displayName: Run script - -- bash: | - deploy_dir=rustc-builds - if [ "$DEPLOY_ALT" == "1" ]; then - deploy_dir=rustc-builds-alt - fi - aws s3 cp --no-progress --recursive --acl public-read ./deploy s3://$DEPLOY_BUCKET/$deploy_dir - env: - # Explicitly decrypt secret variables - # See https://docs.microsoft.com/azure/devops/pipelines/process/variables?view=azure-devops&tabs=yaml%2Cbatch#secret-variables - AWS_ACCESS_KEY_ID: $(SCCACHE_AWS_ACCESS_KEY_ID) - AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) - condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) - displayName: Upload artifacts diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml new file mode 100644 index 0000000000000..06bb8eb4dc8e9 --- /dev/null +++ b/.azure-pipelines/steps/run.yml @@ -0,0 +1,89 @@ +# FIXME(linux): need to configure core dumps, enable them, and then dump +# backtraces on failure from all core dumps: +# +# - bash: sudo apt install gdb +# - bash: sudo sh -c 'echo "/checkout/obj/cores/core.%p.%E" > /proc/sys/kernel/core_pattern' +# +# Check travis config for `gdb --batch` command to print all crash logs + +steps: +- checkout: self + fetchDepth: 2 + +- bash: printenv | sort + displayName: Show environment variables + +- bash: | + set -e + df -h + du . | sort -nr | head -n100 + displayName: Show disk usage + # FIXME: this hasn't been tested, but maybe it works on Windows? Should test! + condition: ne(variables['Agent.OS'], 'Windows_NT') + +- template: install-sccache.yml +- template: install-clang.yml + +# Install some dependencies needed to build LLDB/Clang, currently only needed +# during the `dist` target +- bash: | + set -e + brew update + brew install xz + brew install swig + displayName: Install build dependencies (OSX) + condition: and(eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) + +- template: install-windows-build-deps.yml + +# Check out all our submodules, but more quickly than using git by using one of +# our custom scripts +- bash: | + set -e + mkdir -p $HOME/rustsrc + $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc + condition: ne(variables['Agent.OS'], 'Windows_NT') + displayName: Check out submodules (Unix) +- script: | + if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc + sh src/ci/init_repo.sh . /d/cache/rustsrc + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: Check out submodules (Windows) + +# Configure our CI_JOB_NAME variable which log analyzers can use for the main +# step to see what's going on. +- bash: echo "##vso[task.setvariable variable=CI_JOB_NAME]$SYSTEM_JOBNAME" + condition: eq(variables['Agent.OS'], 'Windows_NT') + displayName: Configure Job Name (Windows) + +# As a quick smoke check on the otherwise very fast mingw-check linux builder +# check our own internal scripts. +- bash: | + set -e + git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git + cd rust-toolstate + python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" + cd .. + rm -rf rust-toolstate + condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check')) + displayName: Verify the publish_toolstate script works + +- script: sh src/ci/run.sh + timeoutInMinutes: 180 + env: + CI: true + SRC: . + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) + displayName: Run build + +- bash: | + set -e + deploy_dir=rustc-builds + if [ "$DEPLOY_ALT" == "1" ]; then + deploy_dir=rustc-builds-alt + fi + aws s3 cp --no-progress --recursive --acl public-read ./deploy s3://$DEPLOY_BUCKET/$deploy_dir + env: + AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) + condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) + displayName: Upload artifacts diff --git a/.azure-pipelines/steps/show-disk-usage.yml b/.azure-pipelines/steps/show-disk-usage.yml deleted file mode 100644 index 902d803647174..0000000000000 --- a/.azure-pipelines/steps/show-disk-usage.yml +++ /dev/null @@ -1,5 +0,0 @@ -steps: -- bash: | - df -h - du . | sort -nr | head -n100 - displayName: Show disk usage diff --git a/.azure-pipelines/steps/show-environment-variables.yml b/.azure-pipelines/steps/show-environment-variables.yml deleted file mode 100644 index f6ed063ec6be5..0000000000000 --- a/.azure-pipelines/steps/show-environment-variables.yml +++ /dev/null @@ -1,3 +0,0 @@ -steps: -- bash: printenv | sort - displayName: Show environment variables \ No newline at end of file diff --git a/.azure-pipelines/steps/verify-publish-toolstate.yml b/.azure-pipelines/steps/verify-publish-toolstate.yml index 5531c90e090a7..e69de29bb2d1d 100644 --- a/.azure-pipelines/steps/verify-publish-toolstate.yml +++ b/.azure-pipelines/steps/verify-publish-toolstate.yml @@ -1,9 +0,0 @@ -steps: -- bash: | - git clone --depth=1 https://github.com/rust-lang-nursery/rust-toolstate.git - cd rust-toolstate - python2.7 "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "" "" - cd .. - rm -rf rust-toolstate - condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check')) - displayName: Verify the publish_toolstate script works diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index bdd36e8555d7d..e04804d2deedb 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -11,7 +11,6 @@ variables: jobs: - job: Linux - timeoutInMinutes: 180 pool: vmImage: ubuntu-16.04 strategy: @@ -27,4 +26,4 @@ jobs: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 steps: - - template: steps/linux.yml + - template: steps/run.yml From 4183f3c41b5a6b7d046b66a270f8d9a8ffd671c8 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:29:30 -0700 Subject: [PATCH 24/57] Configure a few more builders on `try` temporarily Make sure there's one dist and one test builder for each of the three main platforms --- .azure-pipelines/try.yml | 58 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index e04804d2deedb..47470a2aa348a 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -18,12 +18,62 @@ jobs: dist-x86_64-linux: IMAGE: dist-x86_64-linux DEPLOY: 1 - - # "alternate" deployments, these are "nightlies" but have LLVM assertions - # turned on, they're deployed to a different location primarily for - # additional testing. dist-x86_64-linux-alt: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 + test-various: + IMAGE: test-various + steps: + - template: steps/run.yml + +- job: macOS + pool: + vmImage: macos-10.13 + strategy: + matrix: + x86_64-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-x86_64-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + steps: + - template: steps/run.yml + +- job: Windows + pool: + vmImage: 'vs2017-win2016' + strategy: + matrix: + x86_64-msvc-1: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-msvc-2: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + + dist-x86_64-msvc: + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 steps: - template: steps/run.yml From 91b1655528bf47224ae325ef7b4973afb6cd9cda Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:35:21 -0700 Subject: [PATCH 25/57] Remove a now stray file --- .azure-pipelines/steps/verify-publish-toolstate.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .azure-pipelines/steps/verify-publish-toolstate.yml diff --git a/.azure-pipelines/steps/verify-publish-toolstate.yml b/.azure-pipelines/steps/verify-publish-toolstate.yml deleted file mode 100644 index e69de29bb2d1d..0000000000000 From 528cce96cf1d40f50b9b9768fa763d7702e67da9 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:37:18 -0700 Subject: [PATCH 26/57] Job name config works for all platforms --- .azure-pipelines/steps/run.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 06bb8eb4dc8e9..4e7889a1d22cb 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -53,8 +53,7 @@ steps: # Configure our CI_JOB_NAME variable which log analyzers can use for the main # step to see what's going on. - bash: echo "##vso[task.setvariable variable=CI_JOB_NAME]$SYSTEM_JOBNAME" - condition: eq(variables['Agent.OS'], 'Windows_NT') - displayName: Configure Job Name (Windows) + displayName: Configure Job Name # As a quick smoke check on the otherwise very fast mingw-check linux builder # check our own internal scripts. From 7b266ff1812cc1576bcd61906c592c2e7b5642d4 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:38:17 -0700 Subject: [PATCH 27/57] Fix a typo in clang install --- .azure-pipelines/steps/install-clang.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/install-clang.yml b/.azure-pipelines/steps/install-clang.yml index 9e3545ea93cba..edb7679d4c583 100644 --- a/.azure-pipelines/steps/install-clang.yml +++ b/.azure-pipelines/steps/install-clang.yml @@ -32,7 +32,7 @@ steps: %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% - condition: and(eq(variables['Agent.OS'], 'Darwin'), eq(variables['MINGW_URL'],'')) + condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Install clang (Windows) # Note that we don't install clang on Linux since its compiler story is just so From 804ec5faab1bbe487b42fbd4147d9c8f4c1352b1 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:39:25 -0700 Subject: [PATCH 28/57] Execute `docker/run.sh` on Linux --- .azure-pipelines/steps/run.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 4e7889a1d22cb..a0d92cc4d4e85 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -67,7 +67,13 @@ steps: condition: and(succeeded(), eq(variables['IMAGE'], 'mingw-check')) displayName: Verify the publish_toolstate script works -- script: sh src/ci/run.sh +- bash: | + set -e + if [ "$IMAGE" = ""]; then + src/ci/run.sh + else + src/ci/docker/run.sh $IMAGE + fi timeoutInMinutes: 180 env: CI: true From fa8d3b59bd1b547ea34fee1fe76d66b1dbd86695 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:46:10 -0700 Subject: [PATCH 29/57] Fix a typo in the run script --- .azure-pipelines/steps/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index a0d92cc4d4e85..05ceb66cdbad6 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -69,7 +69,7 @@ steps: - bash: | set -e - if [ "$IMAGE" = ""]; then + if [ "$IMAGE" = "" ]; then src/ci/run.sh else src/ci/docker/run.sh $IMAGE From 0843207d1b8e079aebbb06527abb26bbd083e54d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:55:01 -0700 Subject: [PATCH 30/57] Make sure sccache not present doesn't kill the build --- src/ci/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/run.sh b/src/ci/run.sh index a51c2da3cbdca..c996dcb14af38 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -141,4 +141,4 @@ else do_make "$RUST_CHECK_TARGET" fi -sccache --show-stats +sccache --show-stats || true From 751597f0c79a7602db1481b44ebfd5c9a554b591 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 12:58:53 -0700 Subject: [PATCH 31/57] Check out rustfmt submodule through tarballs This takes 30+ seconds to check out on Windows, so let's speed it up through a tarball like we do other big git repositories. --- src/ci/init_repo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh index 3dfd338157617..b1e4f931c4413 100755 --- a/src/ci/init_repo.sh +++ b/src/ci/init_repo.sh @@ -46,7 +46,7 @@ function fetch_github_commit_archive { rm $cached } -included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example" +included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example src/tools/rustfmt" modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)" modules=($modules) use_git="" From 9b8af0608f0817caabbe15ec25c9ebedbb670245 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 13:16:16 -0700 Subject: [PATCH 32/57] Manually install AWS CLI --- .azure-pipelines/steps/run.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 05ceb66cdbad6..fa1afd2d634f8 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -7,6 +7,17 @@ # Check travis config for `gdb --batch` command to print all crash logs steps: + +# Ensure the `aws` CLI is installed so we can deploy later on, cache docker +# images, etc. +- bash: | + set -e + pip install setuptools + pip install awscli + displayName: Install awscli + +- bash: aws s3 help +- bash: exit 1 - checkout: self fetchDepth: 2 From 12f370156ddcdff567e7b0acf83cdf871a997a0a Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 14:21:12 -0700 Subject: [PATCH 33/57] Only execute conditional steps on success Make sure `succeeded()` is in all the conditionals --- .azure-pipelines/steps/install-clang.yml | 4 ++-- .azure-pipelines/steps/install-sccache.yml | 4 ++-- .../steps/install-windows-build-deps.yml | 2 +- .azure-pipelines/steps/run.yml | 20 ++++++++++++------- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/.azure-pipelines/steps/install-clang.yml b/.azure-pipelines/steps/install-clang.yml index edb7679d4c583..26a223282cd87 100644 --- a/.azure-pipelines/steps/install-clang.yml +++ b/.azure-pipelines/steps/install-clang.yml @@ -14,7 +14,7 @@ steps: # `clang-ar` by accident. echo "##vso[task.setvariable variable=AR]ar" displayName: Install clang (OSX) - condition: eq(variables['Agent.OS'], 'Darwin') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) # If we're compiling for MSVC then we, like most other distribution builders, # switch to clang as the compiler. This'll allow us eventually to enable LTO @@ -32,7 +32,7 @@ steps: %TEMP%\LLVM-7.0.0-win64.exe /S /NCRC /D=%CLANG_DIR% set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --set llvm.clang-cl=%CLANG_DIR%\bin\clang-cl.exe echo ##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]%RUST_CONFIGURE_ARGS% - condition: and(eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Install clang (Windows) # Note that we don't install clang on Linux since its compiler story is just so diff --git a/.azure-pipelines/steps/install-sccache.yml b/.azure-pipelines/steps/install-sccache.yml index 6933f4e9f2794..39f58002a7358 100644 --- a/.azure-pipelines/steps/install-sccache.yml +++ b/.azure-pipelines/steps/install-sccache.yml @@ -5,14 +5,14 @@ steps: curl -fo /usr/local/bin/sccache https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-02-sccache-x86_64-apple-darwin chmod +x /usr/local/bin/sccache displayName: Install sccache (OSX) - condition: eq(variables['Agent.OS'], 'Darwin') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - script: | md sccache powershell -Command "iwr -outf sccache\sccache.exe https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror/2018-04-26-sccache-x86_64-pc-windows-msvc" echo ##vso[task.prependpath]%CD%\sccache displayName: Install sccache (Windows) - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) # Note that we don't install sccache on Linux since it's installed elsewhere # through all the containers. diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml index 4eab460543bbc..45ce01fee758e 100644 --- a/.azure-pipelines/steps/install-windows-build-deps.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -29,7 +29,7 @@ steps: echo ##vso[task.setvariable variable=MSYS_PATH]%MSYS_PATH% echo ##vso[task.prependpath]%MSYS_PATH%\usr\bin displayName: Install msys2 - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) # If we need to download a custom MinGW, do so here and set the path # appropriately. diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index fa1afd2d634f8..2d98efd586e77 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -12,12 +12,18 @@ steps: # images, etc. - bash: | set -e - pip install setuptools - pip install awscli - displayName: Install awscli + sudo apt-get install -y python3-setuptools + pip3 install awscli --upgrade --user + echo "##vso[task.prependpath]$HOME/.local/bin" + displayName: Install awscli (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) +- script: pip install awscli + displayName: Install awscli (non-Linux) + condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - bash: aws s3 help - bash: exit 1 + - checkout: self fetchDepth: 2 @@ -30,7 +36,7 @@ steps: du . | sort -nr | head -n100 displayName: Show disk usage # FIXME: this hasn't been tested, but maybe it works on Windows? Should test! - condition: ne(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - template: install-sccache.yml - template: install-clang.yml @@ -43,7 +49,7 @@ steps: brew install xz brew install swig displayName: Install build dependencies (OSX) - condition: and(eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) - template: install-windows-build-deps.yml @@ -53,12 +59,12 @@ steps: set -e mkdir -p $HOME/rustsrc $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc - condition: ne(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) displayName: Check out submodules (Unix) - script: | if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc sh src/ci/init_repo.sh . /d/cache/rustsrc - condition: eq(variables['Agent.OS'], 'Windows_NT') + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) displayName: Check out submodules (Windows) # Configure our CI_JOB_NAME variable which log analyzers can use for the main From 9843a79496a4943b882d8046abfbff52caf51c3d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 20 May 2019 14:37:24 -0700 Subject: [PATCH 34/57] Finalize AWS install --- .azure-pipelines/steps/run.yml | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 2d98efd586e77..dfe54e65e4ecc 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -8,22 +8,6 @@ steps: -# Ensure the `aws` CLI is installed so we can deploy later on, cache docker -# images, etc. -- bash: | - set -e - sudo apt-get install -y python3-setuptools - pip3 install awscli --upgrade --user - echo "##vso[task.prependpath]$HOME/.local/bin" - displayName: Install awscli (Linux) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) -- script: pip install awscli - displayName: Install awscli (non-Linux) - condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) - -- bash: aws s3 help -- bash: exit 1 - - checkout: self fetchDepth: 2 @@ -67,6 +51,19 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) displayName: Check out submodules (Windows) +# Ensure the `aws` CLI is installed so we can deploy later on, cache docker +# images, etc. +- bash: | + set -e + sudo apt-get install -y python3-setuptools + pip3 install awscli --upgrade --user + echo "##vso[task.prependpath]$HOME/.local/bin" + displayName: Install awscli (Linux) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) +- script: pip install awscli + displayName: Install awscli (non-Linux) + condition: and(succeeded(), ne(variables['Agent.OS'], 'Linux')) + # Configure our CI_JOB_NAME variable which log analyzers can use for the main # step to see what's going on. - bash: echo "##vso[task.setvariable variable=CI_JOB_NAME]$SYSTEM_JOBNAME" From 1daab471ce0d71fd0bf071a4bce0e19dd171f546 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 21 May 2019 10:27:41 +0200 Subject: [PATCH 35/57] ci: temp increase of the timeout to 10 hours Let's see how long a full build takes. --- .azure-pipelines/steps/run.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index dfe54e65e4ecc..8882278e3fe0b 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -88,7 +88,8 @@ steps: else src/ci/docker/run.sh $IMAGE fi - timeoutInMinutes: 180 + #timeoutInMinutes: 180 + timeoutInMinutes: 600 env: CI: true SRC: . From 67db2304771d3c7065a68f6a486048b770b8e63e Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 21 May 2019 10:51:37 +0200 Subject: [PATCH 36/57] ci: fix tidy --- src/ci/init_repo.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh index b1e4f931c4413..10215d863be0c 100755 --- a/src/ci/init_repo.sh +++ b/src/ci/init_repo.sh @@ -46,7 +46,8 @@ function fetch_github_commit_archive { rm $cached } -included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example src/tools/rustfmt" +included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example" +included="${included} src/tools/rustfmt" modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)" modules=($modules) use_git="" From d2304e1e26aec2ffd4ae3eefcd0745b2b5065233 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 21 May 2019 12:05:57 +0200 Subject: [PATCH 37/57] ci: increase timeout, take 2 --- .azure-pipelines/try.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 47470a2aa348a..8060dc0a2f329 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -11,6 +11,7 @@ variables: jobs: - job: Linux + timeoutInMinutes: 600 pool: vmImage: ubuntu-16.04 strategy: @@ -27,6 +28,7 @@ jobs: - template: steps/run.yml - job: macOS + timeoutInMinutes: 600 pool: vmImage: macos-10.13 strategy: @@ -53,6 +55,7 @@ jobs: - template: steps/run.yml - job: Windows + timeoutInMinutes: 600 pool: vmImage: 'vs2017-win2016' strategy: From 0e5f02f9025ce2f5b9ff5035c50bcdde051b6ade Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:28:56 -0700 Subject: [PATCH 38/57] Fix typo in key configuration --- .azure-pipelines/steps/run.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 8882278e3fe0b..e697eb1c064eb 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -93,7 +93,7 @@ steps: env: CI: true SRC: . - AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) displayName: Run build - bash: | @@ -104,6 +104,6 @@ steps: fi aws s3 cp --no-progress --recursive --acl public-read ./deploy s3://$DEPLOY_BUCKET/$deploy_dir env: - AWS_SECRET_ACCESS_KEY: $(SCCACHE_AWS_SECRET_ACCESS_KEY) + AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) displayName: Upload artifacts From 571366f71bd1d76934959ba33ce7ef2ee07649a3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:33:19 -0700 Subject: [PATCH 39/57] Attempt to enable IPv6 for Linux --- .azure-pipelines/steps/run.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index e697eb1c064eb..2d38920da88ed 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -37,6 +37,15 @@ steps: - template: install-windows-build-deps.yml +# Looks like docker containers have IPv6 disabled by default, so let's turn it +# on since libstd tests require it +- bash: | + set -e + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: Enable IPv6 + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + # Check out all our submodules, but more quickly than using git by using one of # our custom scripts - bash: | From a83250dcfa5d0751f7d896cb8999ef76ea074771 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:38:05 -0700 Subject: [PATCH 40/57] Update upload logic to upload right directory --- .azure-pipelines/steps/run.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 2d38920da88ed..3516ca5980757 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -107,11 +107,19 @@ steps: - bash: | set -e + if [ "$AGENT_OS" = "Linux" ]; then + rm -rf obj/build/dist/doc + upload_dir=obj/build/dist deploy/$BUILD_SOURCEVERSION + else + rm -rf build/dist/doc + upload_dir=build/dist + fi + ls -la $upload_dir deploy_dir=rustc-builds if [ "$DEPLOY_ALT" == "1" ]; then deploy_dir=rustc-builds-alt fi - aws s3 cp --no-progress --recursive --acl public-read ./deploy s3://$DEPLOY_BUCKET/$deploy_dir + aws s3 cp --no-progress --recursive --acl public-read ./$upload_dir s3://$DEPLOY_BUCKET/$deploy_dir/$BUILD_SOURCEVERSION env: AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) condition: and(succeeded(), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1'))) From ebdc36eb8e4b0fa8d8954ded3cb282951bab95d3 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:39:37 -0700 Subject: [PATCH 41/57] Re-enable hack for Windows builds to see if it works --- .../steps/install-windows-build-deps.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml index 45ce01fee758e..8bda290cd9853 100644 --- a/.azure-pipelines/steps/install-windows-build-deps.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -5,13 +5,14 @@ steps: # git submodule # export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') # echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" -# -# - script: | -# REM echo hack as drive D is too small -# IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( -# mkdir c:\MORE_SPACE -# mklink /J build c:\MORE_SPACE -# ) + +# FIXME: needs a comment to justify its existence +- script: | + REM echo hack as drive D is too small + IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( + mkdir c:\MORE_SPACE + mklink /J build c:\MORE_SPACE + ) - script: | set MSYS_PATH=%CD%\citools\msys64 From 55108b379ebc1e5de5a7a40ab8f3abbfdbb20be2 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:45:13 -0700 Subject: [PATCH 42/57] Run a full build on `try` temporarily --- .azure-pipelines/try.yml | 296 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 279 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 8060dc0a2f329..603516370d9f0 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -1,38 +1,163 @@ # -# Azure Pipelines "try" branch build for Rust +# Azure Pipelines "auto" branch build for Rust on Linux, macOS, and Windows. # pr: none -trigger: -- try + trigger: + - auto variables: - group: prod-credentials jobs: - job: Linux - timeoutInMinutes: 600 pool: vmImage: ubuntu-16.04 + steps: + - template: steps/run.yml strategy: matrix: + x86_64-gnu-llvm-6.0: + IMAGE: x86_64-gnu-llvm-6.0 + RUST_BACKTRACE: 1 + dist-x86_64-linux: IMAGE: dist-x86_64-linux DEPLOY: 1 dist-x86_64-linux-alt: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 - test-various: - IMAGE: test-various - steps: - - template: steps/run.yml + + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu: + IMAGE: i686-gnu + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various + x86_64-gnu: + IMAGE: x86_64-gnu + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck + mingw-check: + IMAGE: mingw-check - job: macOS - timeoutInMinutes: 600 pool: vmImage: macos-10.13 + steps: + - checkout: self + fetchDepth: 2 + - template: steps/run.yml strategy: matrix: + # OSX builders running tests, these run the full test suite. + # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some + # runners that run `//ignore-debug` tests. + # + # Note that the compiler is compiled to target 10.8 here because the Xcode + # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. x86_64-apple: RUST_CHECK_TARGET: check RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc @@ -51,32 +176,169 @@ jobs: NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - steps: - - template: steps/run.yml + + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 + + - job: Windows - timeoutInMinutes: 600 pool: vmImage: 'vs2017-win2016' + steps: + - template: steps/run.yml strategy: matrix: + # 32/64 bit MSVC tests x86_64-msvc-1: + MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-1 + # FIXME(#59637) NO_DEBUG_ASSERTIONS: 1 NO_LLVM_ASSERTIONS: 1 x86_64-msvc-2: + MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 - dist-x86_64-msvc: + # 32/64 bit MSVC and GNU deployment + dist-x86_64-msvc: + RUST_CONFIGURE_ARGS: > + --build=x86_64-pc-windows-msvc + --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-msvc: RUST_CONFIGURE_ARGS: > - --build=x86_64-pc-windows-msvc - --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc --enable-full-tools --enable-profiler SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - steps: - - template: steps/run.yml + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 From e8f0e70b004f5bbce00249e95768ba9fdc48838f Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 10:46:41 -0700 Subject: [PATCH 43/57] Typos --- .azure-pipelines/try.yml | 516 +++++++++++++++++++-------------------- 1 file changed, 256 insertions(+), 260 deletions(-) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 603516370d9f0..3ef24d2de9739 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -3,8 +3,8 @@ # pr: none - trigger: - - auto +trigger: +- try variables: - group: prod-credentials @@ -28,118 +28,118 @@ jobs: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 - arm-android: - IMAGE: arm-android - - armhf-gnu: - IMAGE: armhf-gnu - - dist-various-1: - IMAGE: dist-various-1 - DEPLOY: 1 - - dist-various-2: - IMAGE: dist-various-2 - DEPLOY: 1 - - dist-aarch64-linux: - IMAGE: dist-aarch64-linux - DEPLOY: 1 - - dist-android: - IMAGE: dist-android - DEPLOY: 1 - - dist-arm-linux: - IMAGE: dist-arm-linux - DEPLOY: 1 - - dist-armhf-linux: - IMAGE: dist-armhf-linux - DEPLOY: 1 - - dist-armv7-linux: - IMAGE: dist-armv7-linux - DEPLOY: 1 - - dist-i586-gnu-i586-i686-musl: - IMAGE: dist-i586-gnu-i586-i686-musl - DEPLOY: 1 - - dist-i686-freebsd: - IMAGE: dist-i686-freebsd - DEPLOY: 1 - - dist-i686-linux: - IMAGE: dist-i686-linux - DEPLOY: 1 - - dist-mips-linux: - IMAGE: dist-mips-linux - DEPLOY: 1 - - dist-mips64-linux: - IMAGE: dist-mips64-linux - DEPLOY: 1 - - dist-mips64el-linux: - IMAGE: dist-mips64el-linux - DEPLOY: 1 - - dist-mipsel-linux: - IMAGE: dist-mipsel-linux - DEPLOY: 1 - - dist-powerpc-linux: - IMAGE: dist-powerpc-linux - DEPLOY: 1 - - dist-powerpc64-linux: - IMAGE: dist-powerpc64-linux - DEPLOY: 1 - - dist-powerpc64le-linux: - IMAGE: dist-powerpc64le-linux - DEPLOY: 1 - - dist-s390x-linux: - IMAGE: dist-s390x-linux - DEPLOY: 1 - - dist-x86_64-freebsd: - IMAGE: dist-x86_64-freebsd - DEPLOY: 1 - - dist-x86_64-musl: - IMAGE: dist-x86_64-musl - DEPLOY: 1 - - dist-x86_64-netbsd: - IMAGE: dist-x86_64-netbsd - DEPLOY: 1 - - asmjs: - IMAGE: asmjs - i686-gnu: - IMAGE: i686-gnu - i686-gnu-nopt: - IMAGE: i686-gnu-nopt - test-various: - IMAGE: test-various - x86_64-gnu: - IMAGE: x86_64-gnu - x86_64-gnu-full-bootstrap: - IMAGE: x86_64-gnu-full-bootstrap - x86_64-gnu-aux: - IMAGE: x86_64-gnu-aux - x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools - x86_64-gnu-debug: - IMAGE: x86_64-gnu-debug - x86_64-gnu-nopt: - IMAGE: x86_64-gnu-nopt - x86_64-gnu-distcheck: - IMAGE: x86_64-gnu-distcheck + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu: + IMAGE: i686-gnu + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various + x86_64-gnu: + IMAGE: x86_64-gnu + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck mingw-check: IMAGE: mingw-check @@ -147,8 +147,6 @@ jobs: pool: vmImage: macos-10.13 steps: - - checkout: self - fetchDepth: 2 - template: steps/run.yml strategy: matrix: @@ -177,35 +175,33 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - dist-x86_64-apple-alt: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY_ALT: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - i686-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - dist-i686-apple: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 - job: Windows pool: @@ -214,89 +210,89 @@ jobs: - template: steps/run.yml strategy: matrix: - # 32/64 bit MSVC tests - x86_64-msvc-1: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-1 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-msvc-2: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-2 - i686-msvc-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-1 - i686-msvc-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-2 - # MSVC aux tests - x86_64-msvc-aux: - MSYS_BITS: 64 - RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - x86_64-msvc-cargo: - MSYS_BITS: 64 - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - VCVARS_BAT: vcvars64.bat - # MSVC tools tests - x86_64-msvc-tools: - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - - # 32/64-bit MinGW builds. - # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - i686-mingw-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - i686-mingw-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-2 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - x86_64-mingw-1: - MSYS_BITS: 64 - SCRIPT: make ci-subset-1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-mingw-2: - MSYS_BITS: 64 - SCRIPT: make ci-subset-2 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 + # 32/64 bit MSVC tests + x86_64-msvc-1: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-msvc-2: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -308,37 +304,37 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - dist-i686-msvc: - RUST_CONFIGURE_ARGS: > - --build=i686-pc-windows-msvc - --target=i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-i686-mingw: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - SCRIPT: python x.py dist - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-x86_64-mingw: - MSYS_BITS: 64 - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - # "alternate" deployment, see .travis.yml for more info - dist-x86_64-msvc-alt: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist - DEPLOY_ALT: 1 + dist-i686-msvc: + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 From a2255518c76f67d8993bebffb2b62903ef707d15 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 11:01:39 -0700 Subject: [PATCH 44/57] Only run Window script on Windows --- .azure-pipelines/steps/install-windows-build-deps.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml index 8bda290cd9853..e54f7b6eb6724 100644 --- a/.azure-pipelines/steps/install-windows-build-deps.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -13,6 +13,8 @@ steps: mkdir c:\MORE_SPACE mklink /J build c:\MORE_SPACE ) + displayName: "Ensure build happens on C:/ instead of D:/" + condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - script: | set MSYS_PATH=%CD%\citools\msys64 From 7b7e172f6d5d603c9d266e0c660a5649eb845126 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 12:06:47 -0700 Subject: [PATCH 45/57] Place timeouts at the job level --- .azure-pipelines/try.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 3ef24d2de9739..5c76d20533452 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -11,6 +11,7 @@ variables: jobs: - job: Linux + timeoutInMinutes: 600 pool: vmImage: ubuntu-16.04 steps: @@ -144,6 +145,7 @@ jobs: IMAGE: mingw-check - job: macOS + timeoutInMinutes: 600 pool: vmImage: macos-10.13 steps: @@ -204,6 +206,7 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 - job: Windows + timeoutInMinutes: 600 pool: vmImage: 'vs2017-win2016' steps: From 0f4b546af3c6ce6fc201ee4b283a6600b49ea1d2 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 12:09:44 -0700 Subject: [PATCH 46/57] Go back to fewer builders to test on `try` --- .azure-pipelines/try.yml | 498 +++++++++++++++++++-------------------- 1 file changed, 249 insertions(+), 249 deletions(-) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 5c76d20533452..135fec5f83273 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -18,131 +18,131 @@ jobs: - template: steps/run.yml strategy: matrix: - x86_64-gnu-llvm-6.0: - IMAGE: x86_64-gnu-llvm-6.0 - RUST_BACKTRACE: 1 + # x86_64-gnu-llvm-6.0: + # IMAGE: x86_64-gnu-llvm-6.0 + # RUST_BACKTRACE: 1 dist-x86_64-linux: IMAGE: dist-x86_64-linux DEPLOY: 1 - dist-x86_64-linux-alt: - IMAGE: dist-x86_64-linux - DEPLOY_ALT: 1 - - arm-android: - IMAGE: arm-android - - armhf-gnu: - IMAGE: armhf-gnu - - dist-various-1: - IMAGE: dist-various-1 - DEPLOY: 1 - - dist-various-2: - IMAGE: dist-various-2 - DEPLOY: 1 - - dist-aarch64-linux: - IMAGE: dist-aarch64-linux - DEPLOY: 1 - - dist-android: - IMAGE: dist-android - DEPLOY: 1 - - dist-arm-linux: - IMAGE: dist-arm-linux - DEPLOY: 1 - - dist-armhf-linux: - IMAGE: dist-armhf-linux - DEPLOY: 1 - - dist-armv7-linux: - IMAGE: dist-armv7-linux - DEPLOY: 1 - - dist-i586-gnu-i586-i686-musl: - IMAGE: dist-i586-gnu-i586-i686-musl - DEPLOY: 1 - - dist-i686-freebsd: - IMAGE: dist-i686-freebsd - DEPLOY: 1 - - dist-i686-linux: - IMAGE: dist-i686-linux - DEPLOY: 1 - - dist-mips-linux: - IMAGE: dist-mips-linux - DEPLOY: 1 - - dist-mips64-linux: - IMAGE: dist-mips64-linux - DEPLOY: 1 - - dist-mips64el-linux: - IMAGE: dist-mips64el-linux - DEPLOY: 1 - - dist-mipsel-linux: - IMAGE: dist-mipsel-linux - DEPLOY: 1 - - dist-powerpc-linux: - IMAGE: dist-powerpc-linux - DEPLOY: 1 - - dist-powerpc64-linux: - IMAGE: dist-powerpc64-linux - DEPLOY: 1 - - dist-powerpc64le-linux: - IMAGE: dist-powerpc64le-linux - DEPLOY: 1 - - dist-s390x-linux: - IMAGE: dist-s390x-linux - DEPLOY: 1 - - dist-x86_64-freebsd: - IMAGE: dist-x86_64-freebsd - DEPLOY: 1 - - dist-x86_64-musl: - IMAGE: dist-x86_64-musl - DEPLOY: 1 - - dist-x86_64-netbsd: - IMAGE: dist-x86_64-netbsd - DEPLOY: 1 + # dist-x86_64-linux-alt: + # IMAGE: dist-x86_64-linux + # DEPLOY_ALT: 1 - asmjs: - IMAGE: asmjs - i686-gnu: - IMAGE: i686-gnu - i686-gnu-nopt: - IMAGE: i686-gnu-nopt - test-various: - IMAGE: test-various + # arm-android: + # IMAGE: arm-android + # + # armhf-gnu: + # IMAGE: armhf-gnu + # + # dist-various-1: + # IMAGE: dist-various-1 + # DEPLOY: 1 + # + # dist-various-2: + # IMAGE: dist-various-2 + # DEPLOY: 1 + # + # dist-aarch64-linux: + # IMAGE: dist-aarch64-linux + # DEPLOY: 1 + # + # dist-android: + # IMAGE: dist-android + # DEPLOY: 1 + # + # dist-arm-linux: + # IMAGE: dist-arm-linux + # DEPLOY: 1 + # + # dist-armhf-linux: + # IMAGE: dist-armhf-linux + # DEPLOY: 1 + # + # dist-armv7-linux: + # IMAGE: dist-armv7-linux + # DEPLOY: 1 + # + # dist-i586-gnu-i586-i686-musl: + # IMAGE: dist-i586-gnu-i586-i686-musl + # DEPLOY: 1 + # + # dist-i686-freebsd: + # IMAGE: dist-i686-freebsd + # DEPLOY: 1 + # + # dist-i686-linux: + # IMAGE: dist-i686-linux + # DEPLOY: 1 + # + # dist-mips-linux: + # IMAGE: dist-mips-linux + # DEPLOY: 1 + # + # dist-mips64-linux: + # IMAGE: dist-mips64-linux + # DEPLOY: 1 + # + # dist-mips64el-linux: + # IMAGE: dist-mips64el-linux + # DEPLOY: 1 + # + # dist-mipsel-linux: + # IMAGE: dist-mipsel-linux + # DEPLOY: 1 + # + # dist-powerpc-linux: + # IMAGE: dist-powerpc-linux + # DEPLOY: 1 + # + # dist-powerpc64-linux: + # IMAGE: dist-powerpc64-linux + # DEPLOY: 1 + # + # dist-powerpc64le-linux: + # IMAGE: dist-powerpc64le-linux + # DEPLOY: 1 + # + # dist-s390x-linux: + # IMAGE: dist-s390x-linux + # DEPLOY: 1 + # + # dist-x86_64-freebsd: + # IMAGE: dist-x86_64-freebsd + # DEPLOY: 1 + # + # dist-x86_64-musl: + # IMAGE: dist-x86_64-musl + # DEPLOY: 1 + # + # dist-x86_64-netbsd: + # IMAGE: dist-x86_64-netbsd + # DEPLOY: 1 + # + # asmjs: + # IMAGE: asmjs + # i686-gnu: + # IMAGE: i686-gnu + # i686-gnu-nopt: + # IMAGE: i686-gnu-nopt + # test-various: + # IMAGE: test-various x86_64-gnu: IMAGE: x86_64-gnu - x86_64-gnu-full-bootstrap: - IMAGE: x86_64-gnu-full-bootstrap - x86_64-gnu-aux: - IMAGE: x86_64-gnu-aux - x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools - x86_64-gnu-debug: - IMAGE: x86_64-gnu-debug - x86_64-gnu-nopt: - IMAGE: x86_64-gnu-nopt - x86_64-gnu-distcheck: - IMAGE: x86_64-gnu-distcheck - mingw-check: - IMAGE: mingw-check + # x86_64-gnu-full-bootstrap: + # IMAGE: x86_64-gnu-full-bootstrap + # x86_64-gnu-aux: + # IMAGE: x86_64-gnu-aux + # x86_64-gnu-tools: + # IMAGE: x86_64-gnu-tools + # x86_64-gnu-debug: + # IMAGE: x86_64-gnu-debug + # x86_64-gnu-nopt: + # IMAGE: x86_64-gnu-nopt + # x86_64-gnu-distcheck: + # IMAGE: x86_64-gnu-distcheck + # mingw-check: + # IMAGE: mingw-check - job: macOS timeoutInMinutes: 600 @@ -177,33 +177,33 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - dist-x86_64-apple-alt: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY_ALT: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - i686-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - dist-i686-apple: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 + # dist-x86_64-apple-alt: + # RUST_CHECK_TARGET: dist + # RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + # DEPLOY_ALT: 1 + # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # MACOSX_DEPLOYMENT_TARGET: 10.7 + # NO_LLVM_ASSERTIONS: 1 + # NO_DEBUG_ASSERTIONS: 1 + # + # i686-apple: + # RUST_CHECK_TARGET: check + # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # MACOSX_DEPLOYMENT_TARGET: 10.8 + # MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + # NO_LLVM_ASSERTIONS: 1 + # NO_DEBUG_ASSERTIONS: 1 + # + # dist-i686-apple: + # RUST_CHECK_TARGET: dist + # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + # DEPLOY: 1 + # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + # MACOSX_DEPLOYMENT_TARGET: 10.7 + # NO_LLVM_ASSERTIONS: 1 + # NO_DEBUG_ASSERTIONS: 1 + # DIST_REQUIRE_ALL_TOOLS: 1 - job: Windows timeoutInMinutes: 600 @@ -225,77 +225,77 @@ jobs: MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 - i686-msvc-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-1 - i686-msvc-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-2 - # MSVC aux tests - x86_64-msvc-aux: - MSYS_BITS: 64 - RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - x86_64-msvc-cargo: - MSYS_BITS: 64 - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - VCVARS_BAT: vcvars64.bat - # MSVC tools tests - x86_64-msvc-tools: - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - - # 32/64-bit MinGW builds. + # i686-msvc-1: + # MSYS_BITS: 32 + # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + # SCRIPT: make ci-subset-1 + # i686-msvc-2: + # MSYS_BITS: 32 + # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + # SCRIPT: make ci-subset-2 + # # MSVC aux tests + # x86_64-msvc-aux: + # MSYS_BITS: 64 + # RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + # x86_64-msvc-cargo: + # MSYS_BITS: 64 + # SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + # VCVARS_BAT: vcvars64.bat + # # MSVC tools tests + # x86_64-msvc-tools: + # MSYS_BITS: 64 + # SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - i686-mingw-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - i686-mingw-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-2 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - x86_64-mingw-1: - MSYS_BITS: 64 - SCRIPT: make ci-subset-1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-mingw-2: - MSYS_BITS: 64 - SCRIPT: make ci-subset-2 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 + # # 32/64-bit MinGW builds. + # # + # # We are using MinGW with posix threads since LLVM does not compile with + # # the win32 threads version due to missing support for C++'s std::thread. + # # + # # Instead of relying on the MinGW version installed on appveryor we download + # # and install one ourselves so we won't be surprised by changes to appveyor's + # # build image. + # # + # # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # # bucket, but they cleraly didn't originate there! The downloads originally + # # came from the mingw-w64 SourceForge download site. Unfortunately + # # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + # i686-mingw-1: + # MSYS_BITS: 32 + # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + # SCRIPT: make ci-subset-1 + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + # MINGW_DIR: mingw32 + # # FIXME(#59637) + # NO_DEBUG_ASSERTIONS: 1 + # NO_LLVM_ASSERTIONS: 1 + # i686-mingw-2: + # MSYS_BITS: 32 + # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + # SCRIPT: make ci-subset-2 + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + # MINGW_DIR: mingw32 + # x86_64-mingw-1: + # MSYS_BITS: 64 + # SCRIPT: make ci-subset-1 + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + # MINGW_DIR: mingw64 + # # FIXME(#59637) + # NO_DEBUG_ASSERTIONS: 1 + # NO_LLVM_ASSERTIONS: 1 + # x86_64-mingw-2: + # MSYS_BITS: 64 + # SCRIPT: make ci-subset-2 + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + # MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -307,37 +307,37 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - dist-i686-msvc: - RUST_CONFIGURE_ARGS: > - --build=i686-pc-windows-msvc - --target=i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-i686-mingw: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - SCRIPT: python x.py dist - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-x86_64-mingw: - MSYS_BITS: 64 - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - - # "alternate" deployment, see .travis.yml for more info - dist-x86_64-msvc-alt: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - SCRIPT: python x.py dist - DEPLOY_ALT: 1 + # dist-i686-msvc: + # RUST_CONFIGURE_ARGS: > + # --build=i686-pc-windows-msvc + # --target=i586-pc-windows-msvc + # --enable-full-tools + # --enable-profiler + # SCRIPT: python x.py dist + # DIST_REQUIRE_ALL_TOOLS: 1 + # DEPLOY: 1 + # dist-i686-mingw: + # MSYS_BITS: 32 + # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + # SCRIPT: python x.py dist + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + # MINGW_DIR: mingw32 + # DIST_REQUIRE_ALL_TOOLS: 1 + # DEPLOY: 1 + # dist-x86_64-mingw: + # MSYS_BITS: 64 + # SCRIPT: python x.py dist + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + # MINGW_DIR: mingw64 + # DIST_REQUIRE_ALL_TOOLS: 1 + # DEPLOY: 1 + # + # # "alternate" deployment, see .travis.yml for more info + # dist-x86_64-msvc-alt: + # MSYS_BITS: 64 + # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + # SCRIPT: python x.py dist + # DEPLOY_ALT: 1 From 8d42b87ac041dc05c9ae8c060aaa8864826639d4 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 12:20:50 -0700 Subject: [PATCH 47/57] Use new-style s3 urls in docker download script --- src/ci/docker/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 0e1485601e96e..c9642dbf60c66 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -42,7 +42,7 @@ if [ -f "$docker_dir/$image/Dockerfile" ]; then awk '{print $1}') s3url="s3://$SCCACHE_BUCKET/docker/$cksum" - url="https://s3-us-west-1.amazonaws.com/$SCCACHE_BUCKET/docker/$cksum" + url="https://$SCCACHE_BUCKET.s3.amazonaws.com/docker/$cksum" upload="aws s3 cp - $s3url" echo "Attempting to download $url" From 190d1988e010db8e09e07324f7a41dd927d0c580 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 21 May 2019 21:24:34 +0200 Subject: [PATCH 48/57] ci: switch to xcode 9.3 on azure --- .azure-pipelines/steps/run.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 3516ca5980757..22ffa1010e49a 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -35,6 +35,12 @@ steps: displayName: Install build dependencies (OSX) condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) +- bash: | + set -e + sudo xcode-select --switch /Applications/Xcode_9.3.app + displayName: Switch to Xcode 9.3 (OSX) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) + - template: install-windows-build-deps.yml # Looks like docker containers have IPv6 disabled by default, so let's turn it From beea4f5e4a95126ab5837c76eea06f9fcd15fb45 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 13:32:53 -0700 Subject: [PATCH 49/57] Execute everything! --- .azure-pipelines/try.yml | 498 +++++++++++++++++++-------------------- 1 file changed, 249 insertions(+), 249 deletions(-) diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 135fec5f83273..5c76d20533452 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -18,131 +18,131 @@ jobs: - template: steps/run.yml strategy: matrix: - # x86_64-gnu-llvm-6.0: - # IMAGE: x86_64-gnu-llvm-6.0 - # RUST_BACKTRACE: 1 + x86_64-gnu-llvm-6.0: + IMAGE: x86_64-gnu-llvm-6.0 + RUST_BACKTRACE: 1 dist-x86_64-linux: IMAGE: dist-x86_64-linux DEPLOY: 1 - # dist-x86_64-linux-alt: - # IMAGE: dist-x86_64-linux - # DEPLOY_ALT: 1 + dist-x86_64-linux-alt: + IMAGE: dist-x86_64-linux + DEPLOY_ALT: 1 - # arm-android: - # IMAGE: arm-android - # - # armhf-gnu: - # IMAGE: armhf-gnu - # - # dist-various-1: - # IMAGE: dist-various-1 - # DEPLOY: 1 - # - # dist-various-2: - # IMAGE: dist-various-2 - # DEPLOY: 1 - # - # dist-aarch64-linux: - # IMAGE: dist-aarch64-linux - # DEPLOY: 1 - # - # dist-android: - # IMAGE: dist-android - # DEPLOY: 1 - # - # dist-arm-linux: - # IMAGE: dist-arm-linux - # DEPLOY: 1 - # - # dist-armhf-linux: - # IMAGE: dist-armhf-linux - # DEPLOY: 1 - # - # dist-armv7-linux: - # IMAGE: dist-armv7-linux - # DEPLOY: 1 - # - # dist-i586-gnu-i586-i686-musl: - # IMAGE: dist-i586-gnu-i586-i686-musl - # DEPLOY: 1 - # - # dist-i686-freebsd: - # IMAGE: dist-i686-freebsd - # DEPLOY: 1 - # - # dist-i686-linux: - # IMAGE: dist-i686-linux - # DEPLOY: 1 - # - # dist-mips-linux: - # IMAGE: dist-mips-linux - # DEPLOY: 1 - # - # dist-mips64-linux: - # IMAGE: dist-mips64-linux - # DEPLOY: 1 - # - # dist-mips64el-linux: - # IMAGE: dist-mips64el-linux - # DEPLOY: 1 - # - # dist-mipsel-linux: - # IMAGE: dist-mipsel-linux - # DEPLOY: 1 - # - # dist-powerpc-linux: - # IMAGE: dist-powerpc-linux - # DEPLOY: 1 - # - # dist-powerpc64-linux: - # IMAGE: dist-powerpc64-linux - # DEPLOY: 1 - # - # dist-powerpc64le-linux: - # IMAGE: dist-powerpc64le-linux - # DEPLOY: 1 - # - # dist-s390x-linux: - # IMAGE: dist-s390x-linux - # DEPLOY: 1 - # - # dist-x86_64-freebsd: - # IMAGE: dist-x86_64-freebsd - # DEPLOY: 1 - # - # dist-x86_64-musl: - # IMAGE: dist-x86_64-musl - # DEPLOY: 1 - # - # dist-x86_64-netbsd: - # IMAGE: dist-x86_64-netbsd - # DEPLOY: 1 - # - # asmjs: - # IMAGE: asmjs - # i686-gnu: - # IMAGE: i686-gnu - # i686-gnu-nopt: - # IMAGE: i686-gnu-nopt - # test-various: - # IMAGE: test-various + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu: + IMAGE: i686-gnu + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various x86_64-gnu: IMAGE: x86_64-gnu - # x86_64-gnu-full-bootstrap: - # IMAGE: x86_64-gnu-full-bootstrap - # x86_64-gnu-aux: - # IMAGE: x86_64-gnu-aux - # x86_64-gnu-tools: - # IMAGE: x86_64-gnu-tools - # x86_64-gnu-debug: - # IMAGE: x86_64-gnu-debug - # x86_64-gnu-nopt: - # IMAGE: x86_64-gnu-nopt - # x86_64-gnu-distcheck: - # IMAGE: x86_64-gnu-distcheck - # mingw-check: - # IMAGE: mingw-check + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck + mingw-check: + IMAGE: mingw-check - job: macOS timeoutInMinutes: 600 @@ -177,33 +177,33 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - # dist-x86_64-apple-alt: - # RUST_CHECK_TARGET: dist - # RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc - # DEPLOY_ALT: 1 - # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - # MACOSX_DEPLOYMENT_TARGET: 10.7 - # NO_LLVM_ASSERTIONS: 1 - # NO_DEBUG_ASSERTIONS: 1 - # - # i686-apple: - # RUST_CHECK_TARGET: check - # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - # MACOSX_DEPLOYMENT_TARGET: 10.8 - # MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - # NO_LLVM_ASSERTIONS: 1 - # NO_DEBUG_ASSERTIONS: 1 - # - # dist-i686-apple: - # RUST_CHECK_TARGET: dist - # RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - # DEPLOY: 1 - # RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - # MACOSX_DEPLOYMENT_TARGET: 10.7 - # NO_LLVM_ASSERTIONS: 1 - # NO_DEBUG_ASSERTIONS: 1 - # DIST_REQUIRE_ALL_TOOLS: 1 + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 - job: Windows timeoutInMinutes: 600 @@ -225,77 +225,77 @@ jobs: MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler SCRIPT: make ci-subset-2 - # i686-msvc-1: - # MSYS_BITS: 32 - # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - # SCRIPT: make ci-subset-1 - # i686-msvc-2: - # MSYS_BITS: 32 - # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - # SCRIPT: make ci-subset-2 - # # MSVC aux tests - # x86_64-msvc-aux: - # MSYS_BITS: 64 - # RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - # x86_64-msvc-cargo: - # MSYS_BITS: 64 - # SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - # VCVARS_BAT: vcvars64.bat - # # MSVC tools tests - # x86_64-msvc-tools: - # MSYS_BITS: 64 - # SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. # - # # 32/64-bit MinGW builds. - # # - # # We are using MinGW with posix threads since LLVM does not compile with - # # the win32 threads version due to missing support for C++'s std::thread. - # # - # # Instead of relying on the MinGW version installed on appveryor we download - # # and install one ourselves so we won't be surprised by changes to appveyor's - # # build image. - # # - # # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # # bucket, but they cleraly didn't originate there! The downloads originally - # # came from the mingw-w64 SourceForge download site. Unfortunately - # # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - # i686-mingw-1: - # MSYS_BITS: 32 - # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - # SCRIPT: make ci-subset-1 - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - # MINGW_DIR: mingw32 - # # FIXME(#59637) - # NO_DEBUG_ASSERTIONS: 1 - # NO_LLVM_ASSERTIONS: 1 - # i686-mingw-2: - # MSYS_BITS: 32 - # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - # SCRIPT: make ci-subset-2 - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - # MINGW_DIR: mingw32 - # x86_64-mingw-1: - # MSYS_BITS: 64 - # SCRIPT: make ci-subset-1 - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - # MINGW_DIR: mingw64 - # # FIXME(#59637) - # NO_DEBUG_ASSERTIONS: 1 - # NO_LLVM_ASSERTIONS: 1 - # x86_64-mingw-2: - # MSYS_BITS: 64 - # SCRIPT: make ci-subset-2 - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - # MINGW_DIR: mingw64 + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -307,37 +307,37 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - # dist-i686-msvc: - # RUST_CONFIGURE_ARGS: > - # --build=i686-pc-windows-msvc - # --target=i586-pc-windows-msvc - # --enable-full-tools - # --enable-profiler - # SCRIPT: python x.py dist - # DIST_REQUIRE_ALL_TOOLS: 1 - # DEPLOY: 1 - # dist-i686-mingw: - # MSYS_BITS: 32 - # RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - # SCRIPT: python x.py dist - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - # MINGW_DIR: mingw32 - # DIST_REQUIRE_ALL_TOOLS: 1 - # DEPLOY: 1 - # dist-x86_64-mingw: - # MSYS_BITS: 64 - # SCRIPT: python x.py dist - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - # MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - # MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - # MINGW_DIR: mingw64 - # DIST_REQUIRE_ALL_TOOLS: 1 - # DEPLOY: 1 - # - # # "alternate" deployment, see .travis.yml for more info - # dist-x86_64-msvc-alt: - # MSYS_BITS: 64 - # RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler - # SCRIPT: python x.py dist - # DEPLOY_ALT: 1 + dist-i686-msvc: + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 From 78965f49c8f730327240f59ed0eae4e007f0a0ea Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 15:14:20 -0700 Subject: [PATCH 50/57] Use xcode 9.3 on all osx builders --- .azure-pipelines/steps/run.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 22ffa1010e49a..3d07df4d936e4 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -35,11 +35,14 @@ steps: displayName: Install build dependencies (OSX) condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) +# Switch to XCode 9.3 on OSX since it seems to be the last version that supports +# i686-apple-darwin. We'll eventually want to upgrade this and it will probably +# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. - bash: | set -e sudo xcode-select --switch /Applications/Xcode_9.3.app displayName: Switch to Xcode 9.3 (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin'), eq(variables['RUST_CHECK_TARGET'],'dist')) + condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - template: install-windows-build-deps.yml From e764f475ca7fffd6167ea991afc7d1b2b3f642dc Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 15:34:56 -0700 Subject: [PATCH 51/57] Disable solaris target since toolchain no longer builds --- src/ci/docker/dist-various-2/Dockerfile | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/ci/docker/dist-various-2/Dockerfile b/src/ci/docker/dist-various-2/Dockerfile index 1d6a0269c5b5b..04db5e9c05724 100644 --- a/src/ci/docker/dist-various-2/Dockerfile +++ b/src/ci/docker/dist-various-2/Dockerfile @@ -32,9 +32,10 @@ COPY dist-various-2/build-cloudabi-toolchain.sh /tmp/ RUN /tmp/build-cloudabi-toolchain.sh x86_64-unknown-cloudabi COPY dist-various-2/build-fuchsia-toolchain.sh /tmp/ RUN /tmp/build-fuchsia-toolchain.sh -COPY dist-various-2/build-solaris-toolchain.sh /tmp/ -RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 -RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc +# FIXME(#61022) - reenable solaris +# COPY dist-various-2/build-solaris-toolchain.sh /tmp/ +# RUN /tmp/build-solaris-toolchain.sh x86_64 amd64 solaris-i386 +# RUN /tmp/build-solaris-toolchain.sh sparcv9 sparcv9 solaris-sparc COPY dist-various-2/build-x86_64-fortanix-unknown-sgx-toolchain.sh /tmp/ # We pass the commit id of the port of LLVM's libunwind to the build script. # Any update to the commit id here, should cause the container image to be re-built from this point on. @@ -73,10 +74,11 @@ ENV CARGO_TARGET_AARCH64_FUCHSIA_RUSTFLAGS \ ENV TARGETS=x86_64-fuchsia ENV TARGETS=$TARGETS,aarch64-fuchsia -ENV TARGETS=$TARGETS,sparcv9-sun-solaris ENV TARGETS=$TARGETS,wasm32-unknown-unknown ENV TARGETS=$TARGETS,wasm32-unknown-wasi -ENV TARGETS=$TARGETS,x86_64-sun-solaris +# FIXME(#61022) - reenable solaris +# ENV TARGETS=$TARGETS,sparcv9-sun-solaris +# ENV TARGETS=$TARGETS,x86_64-sun-solaris ENV TARGETS=$TARGETS,x86_64-unknown-linux-gnux32 ENV TARGETS=$TARGETS,x86_64-unknown-cloudabi ENV TARGETS=$TARGETS,x86_64-fortanix-unknown-sgx From e0ed2a3f60f3e477c372ec0197e0cd676ada36b6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 15:55:26 -0700 Subject: [PATCH 52/57] Correct the upload_dir in script --- .azure-pipelines/steps/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index 3d07df4d936e4..f7ee490b1fdca 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -118,7 +118,7 @@ steps: set -e if [ "$AGENT_OS" = "Linux" ]; then rm -rf obj/build/dist/doc - upload_dir=obj/build/dist deploy/$BUILD_SOURCEVERSION + upload_dir=obj/build/dist else rm -rf build/dist/doc upload_dir=build/dist From e5418f61deb65a2d5a2bd48320e05d9cf689cff6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 21 May 2019 15:57:47 -0700 Subject: [PATCH 53/57] Update webrender we're testing The one locked here has a dependency on an old enough `cmake` crate that it's not detecting visual studio correctly. Let's update webrender which updates `cmake` which should be able to detect Visual Studio correctly. --- src/tools/cargotest/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/cargotest/main.rs b/src/tools/cargotest/main.rs index 3126b44f0d618..14035eedbb44b 100644 --- a/src/tools/cargotest/main.rs +++ b/src/tools/cargotest/main.rs @@ -61,7 +61,7 @@ const TEST_REPOS: &'static [Test] = &[ Test { name: "webrender", repo: "https://github.com/servo/webrender", - sha: "57250b2b8fa63934f80e5376a29f7dcb3f759ad6", + sha: "cdadd068f4c7218bd983d856981d561e605270ab", lock: None, packages: &[], }, From 36ee777cebd865ffd7e8223b4ae4bf533f7b476b Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 22 May 2019 13:28:20 +0200 Subject: [PATCH 54/57] remove azure badge from the readme --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 4a130d257770a..75d7823490a20 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -[![Build Status](https://dev.azure.com/rust-lang-azure/rust/_apis/build/status/johnterickson.rust?branchName=master)](https://dev.azure.com/rust-lang-azure/rust/_build/latest?definitionId=6&branchName=master) - # The Rust Programming Language This is the main source code repository for [Rust]. It contains the compiler, From a8ac80b7bcb46d199c196ec0313ca7fd70aee857 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 22 May 2019 07:09:25 -0700 Subject: [PATCH 55/57] Final cleanups * Clean up the `install-windows-build-deps.yml` file and add some more comments where appropriate. * Add some comments to `run.yml` * Don't fast path the `rustfmt` submodule, but we'll take care of that later if necessary. --- .../steps/install-windows-build-deps.yml | 57 ++++++++----------- .azure-pipelines/steps/run.yml | 2 + src/ci/init_repo.sh | 1 - 3 files changed, 27 insertions(+), 33 deletions(-) diff --git a/.azure-pipelines/steps/install-windows-build-deps.yml b/.azure-pipelines/steps/install-windows-build-deps.yml index e54f7b6eb6724..037c8daa2a80a 100644 --- a/.azure-pipelines/steps/install-windows-build-deps.yml +++ b/.azure-pipelines/steps/install-windows-build-deps.yml @@ -1,21 +1,20 @@ steps: -# FIXME: are these still needed? -# - bash: | -# set -x -# git submodule -# export SUBMODULES_EXCLUDES=$(git submodule | grep -Eow 'src/[^ ]+' | sed 's/\(.*\)/--exclude=\1\/\.git/') -# echo "##vso[task.setvariable variable=SUBMODULES_EXCLUDES;]$SUBMODULES_EXCLUDES" - -# FIXME: needs a comment to justify its existence +# We've had issues with the default drive in use running out of space during a +# build, and it looks like the `C:` drive has more space than the default `D:` +# drive. We should probably confirm this with the azure pipelines team at some +# point, but this seems to fix our "disk space full" problems. - script: | - REM echo hack as drive D is too small - IF NOT "%DISABLE_DISK_SPACE_HACK%"=="1" ( - mkdir c:\MORE_SPACE - mklink /J build c:\MORE_SPACE - ) + mkdir c:\MORE_SPACE + mklink /J build c:\MORE_SPACE displayName: "Ensure build happens on C:/ instead of D:/" condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) +# Download and install MSYS2, needed primarily for the test suite (run-make) but +# also used by the MinGW toolchain for assembling things. +# +# FIXME: we should probe the default azure image and see if we can use the MSYS2 +# toolchain there. (if there's even one there). For now though this gets the job +# done. - script: | set MSYS_PATH=%CD%\citools\msys64 choco install msys2 --params="/InstallDir:%MSYS_PATH% /NoPath" -y @@ -37,23 +36,11 @@ steps: # If we need to download a custom MinGW, do so here and set the path # appropriately. # -# Note that this *also* means that we're not using what is typically -# /mingw32/bin/python2.7.exe, which is a "correct" python interpreter where -# /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we -# move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe` -# file exists in there (which it doesn't by default). -- script: | - powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" - 7z x -y %MINGW_ARCHIVE% > nul - echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) - displayName: Download custom MinGW - -# Here we do a pretty heinous thing which is to mangle the MinGW installation -# we just had above. Currently, as of this writing, we're using MinGW-w64 -# builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it appears to -# be the first version which contains a fix for #40546, builds randomly -# failing during LLVM due to ar.exe/ranlib.exe failures. +# Here we also do a pretty heinous thing which is to mangle the MinGW +# installation we just downloaded. Currently, as of this writing, we're using +# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it +# appears to be the first version which contains a fix for #40546, builds +# randomly failing during LLVM due to ar.exe/ranlib.exe failures. # # Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds # to contain a regression in gdb (#40184). As a result if we were to use the @@ -67,11 +54,13 @@ steps: # Note that we don't literally overwrite the gdb.exe binary because it appears # to just use gdborig.exe, so that's the binary we deal with instead. - script: | - echo ON + powershell -Command "iwr -outf %MINGW_ARCHIVE% %MINGW_URL%/%MINGW_ARCHIVE%" + 7z x -y %MINGW_ARCHIVE% > nul powershell -Command "iwr -outf 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_URL%/2017-04-20-%MSYS_BITS%bit-gdborig.exe" mv 2017-04-20-%MSYS_BITS%bit-gdborig.exe %MINGW_DIR%\bin\gdborig.exe + echo ##vso[task.prependpath]%CD%\%MINGW_DIR%\bin condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) - displayName: Override with 6.3.0 gdb with 6.2.0 gdb + displayName: Download custom MinGW # Otherwise pull in the MinGW installed on appveyor - script: | @@ -79,6 +68,10 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) displayName: Add MinGW to path +# Make sure we use the native python interpreter instead of some msys equivalent +# one way or another. The msys interpreters seem to have weird path conversions +# baked in which break LLVM's build system one way or another, so let's use the +# native version which keeps everything as native as possible. - script: | copy C:\Python27amd64\python.exe C:\Python27amd64\python2.7.exe echo ##vso[task.prependpath]C:\Python27amd64 diff --git a/.azure-pipelines/steps/run.yml b/.azure-pipelines/steps/run.yml index f7ee490b1fdca..47a4c18308512 100644 --- a/.azure-pipelines/steps/run.yml +++ b/.azure-pipelines/steps/run.yml @@ -114,6 +114,8 @@ steps: AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY) displayName: Run build +# If we're a deploy builder, use the `aws` command to publish everything to our +# bucket. - bash: | set -e if [ "$AGENT_OS" = "Linux" ]; then diff --git a/src/ci/init_repo.sh b/src/ci/init_repo.sh index 10215d863be0c..3dfd338157617 100755 --- a/src/ci/init_repo.sh +++ b/src/ci/init_repo.sh @@ -47,7 +47,6 @@ function fetch_github_commit_archive { } included="src/llvm-project src/llvm-emscripten src/doc/book src/doc/rust-by-example" -included="${included} src/tools/rustfmt" modules="$(git config --file .gitmodules --get-regexp '\.path$' | cut -d' ' -f2)" modules=($modules) use_git="" From 24f262f35e5a36a29f7f2f7ee3e128b489666871 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Wed, 22 May 2019 18:10:05 +0200 Subject: [PATCH 56/57] ci: toggle builders --- .azure-pipelines/auto.yml | 516 +++++++++++++++++++------------------- .azure-pipelines/try.yml | 267 +------------------- 2 files changed, 259 insertions(+), 524 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 3b6cfed70719a..800cbb5a3aee1 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -32,120 +32,120 @@ jobs: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 -# # Linux builders, remaining docker images -# arm-android: -# IMAGE: arm-android -# -# armhf-gnu: -# IMAGE: armhf-gnu -# -# dist-various-1: -# IMAGE: dist-various-1 -# DEPLOY: 1 -# -# dist-various-2: -# IMAGE: dist-various-2 -# DEPLOY: 1 -# -# dist-aarch64-linux: -# IMAGE: dist-aarch64-linux -# DEPLOY: 1 -# -# dist-android: -# IMAGE: dist-android -# DEPLOY: 1 -# -# dist-arm-linux: -# IMAGE: dist-arm-linux -# DEPLOY: 1 -# -# dist-armhf-linux: -# IMAGE: dist-armhf-linux -# DEPLOY: 1 -# -# dist-armv7-linux: -# IMAGE: dist-armv7-linux -# DEPLOY: 1 -# -# dist-i586-gnu-i586-i686-musl: -# IMAGE: dist-i586-gnu-i586-i686-musl -# DEPLOY: 1 -# -# dist-i686-freebsd: -# IMAGE: dist-i686-freebsd -# DEPLOY: 1 -# -# dist-i686-linux: -# IMAGE: dist-i686-linux -# DEPLOY: 1 -# -# dist-mips-linux: -# IMAGE: dist-mips-linux -# DEPLOY: 1 -# -# dist-mips64-linux: -# IMAGE: dist-mips64-linux -# DEPLOY: 1 -# -# dist-mips64el-linux: -# IMAGE: dist-mips64el-linux -# DEPLOY: 1 -# -# dist-mipsel-linux: -# IMAGE: dist-mipsel-linux -# DEPLOY: 1 -# -# dist-powerpc-linux: -# IMAGE: dist-powerpc-linux -# DEPLOY: 1 -# -# dist-powerpc64-linux: -# IMAGE: dist-powerpc64-linux -# DEPLOY: 1 -# -# dist-powerpc64le-linux: -# IMAGE: dist-powerpc64le-linux -# DEPLOY: 1 -# -# dist-s390x-linux: -# IMAGE: dist-s390x-linux -# DEPLOY: 1 -# -# dist-x86_64-freebsd: -# IMAGE: dist-x86_64-freebsd -# DEPLOY: 1 -# -# dist-x86_64-musl: -# IMAGE: dist-x86_64-musl -# DEPLOY: 1 -# -# dist-x86_64-netbsd: -# IMAGE: dist-x86_64-netbsd -# DEPLOY: 1 -# -# asmjs: -# IMAGE: asmjs -# i686-gnu: -# IMAGE: i686-gnu -# i686-gnu-nopt: -# IMAGE: i686-gnu-nopt -# test-various: -# IMAGE: test-various -# x86_64-gnu: -# IMAGE: x86_64-gnu -# x86_64-gnu-full-bootstrap: -# IMAGE: x86_64-gnu-full-bootstrap -# x86_64-gnu-aux: -# IMAGE: x86_64-gnu-aux -# x86_64-gnu-tools: -# IMAGE: x86_64-gnu-tools -# # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) -# x86_64-gnu-debug: -# IMAGE: x86_64-gnu-debug -# x86_64-gnu-nopt: -# IMAGE: x86_64-gnu-nopt -# x86_64-gnu-distcheck: -# IMAGE: x86_64-gnu-distcheck + # Linux builders, remaining docker images + arm-android: + IMAGE: arm-android + + armhf-gnu: + IMAGE: armhf-gnu + + dist-various-1: + IMAGE: dist-various-1 + DEPLOY: 1 + + dist-various-2: + IMAGE: dist-various-2 + DEPLOY: 1 + + dist-aarch64-linux: + IMAGE: dist-aarch64-linux + DEPLOY: 1 + + dist-android: + IMAGE: dist-android + DEPLOY: 1 + + dist-arm-linux: + IMAGE: dist-arm-linux + DEPLOY: 1 + + dist-armhf-linux: + IMAGE: dist-armhf-linux + DEPLOY: 1 + + dist-armv7-linux: + IMAGE: dist-armv7-linux + DEPLOY: 1 + + dist-i586-gnu-i586-i686-musl: + IMAGE: dist-i586-gnu-i586-i686-musl + DEPLOY: 1 + + dist-i686-freebsd: + IMAGE: dist-i686-freebsd + DEPLOY: 1 + + dist-i686-linux: + IMAGE: dist-i686-linux + DEPLOY: 1 + + dist-mips-linux: + IMAGE: dist-mips-linux + DEPLOY: 1 + + dist-mips64-linux: + IMAGE: dist-mips64-linux + DEPLOY: 1 + + dist-mips64el-linux: + IMAGE: dist-mips64el-linux + DEPLOY: 1 + + dist-mipsel-linux: + IMAGE: dist-mipsel-linux + DEPLOY: 1 + + dist-powerpc-linux: + IMAGE: dist-powerpc-linux + DEPLOY: 1 + + dist-powerpc64-linux: + IMAGE: dist-powerpc64-linux + DEPLOY: 1 + + dist-powerpc64le-linux: + IMAGE: dist-powerpc64le-linux + DEPLOY: 1 + + dist-s390x-linux: + IMAGE: dist-s390x-linux + DEPLOY: 1 + + dist-x86_64-freebsd: + IMAGE: dist-x86_64-freebsd + DEPLOY: 1 + + dist-x86_64-musl: + IMAGE: dist-x86_64-musl + DEPLOY: 1 + + dist-x86_64-netbsd: + IMAGE: dist-x86_64-netbsd + DEPLOY: 1 + + asmjs: + IMAGE: asmjs + i686-gnu: + IMAGE: i686-gnu + i686-gnu-nopt: + IMAGE: i686-gnu-nopt + test-various: + IMAGE: test-various + x86_64-gnu: + IMAGE: x86_64-gnu + x86_64-gnu-full-bootstrap: + IMAGE: x86_64-gnu-full-bootstrap + x86_64-gnu-aux: + IMAGE: x86_64-gnu-aux + x86_64-gnu-tools: + IMAGE: x86_64-gnu-tools + # FIXME if: branch = auto OR (type = pull_request AND commit_message =~ /(?i:^update.*\b(rls|rustfmt|clippy|miri|cargo)\b)/) + x86_64-gnu-debug: + IMAGE: x86_64-gnu-debug + x86_64-gnu-nopt: + IMAGE: x86_64-gnu-nopt + x86_64-gnu-distcheck: + IMAGE: x86_64-gnu-distcheck mingw-check: IMAGE: mingw-check @@ -183,33 +183,33 @@ jobs: NO_DEBUG_ASSERTIONS: 1 DIST_REQUIRE_ALL_TOOLS: 1 -# dist-x86_64-apple-alt: -# RUST_CHECK_TARGET: dist -# RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc -# DEPLOY_ALT: 1 -# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 -# MACOSX_DEPLOYMENT_TARGET: 10.7 -# NO_LLVM_ASSERTIONS: 1 -# NO_DEBUG_ASSERTIONS: 1 -# -# i686-apple: -# RUST_CHECK_TARGET: check -# RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc -# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 -# MACOSX_DEPLOYMENT_TARGET: 10.8 -# MACOSX_STD_DEPLOYMENT_TARGET: 10.7 -# NO_LLVM_ASSERTIONS: 1 -# NO_DEBUG_ASSERTIONS: 1 -# -# dist-i686-apple: -# RUST_CHECK_TARGET: dist -# RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc -# DEPLOY: 1 -# RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 -# MACOSX_DEPLOYMENT_TARGET: 10.7 -# NO_LLVM_ASSERTIONS: 1 -# NO_DEBUG_ASSERTIONS: 1 -# DIST_REQUIRE_ALL_TOOLS: 1 + dist-x86_64-apple-alt: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --enable-extended --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY_ALT: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + i686-apple: + RUST_CHECK_TARGET: check + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.8 + MACOSX_STD_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + + dist-i686-apple: + RUST_CHECK_TARGET: dist + RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc + DEPLOY: 1 + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + MACOSX_DEPLOYMENT_TARGET: 10.7 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + DIST_REQUIRE_ALL_TOOLS: 1 @@ -220,89 +220,89 @@ jobs: - template: steps/run.yml strategy: matrix: -# # 32/64 bit MSVC tests -# x86_64-msvc-1: -# MSYS_BITS: 64 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler -# SCRIPT: make ci-subset-1 -# # FIXME(#59637) -# NO_DEBUG_ASSERTIONS: 1 -# NO_LLVM_ASSERTIONS: 1 -# x86_64-msvc-2: -# MSYS_BITS: 64 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler -# SCRIPT: make ci-subset-2 -# i686-msvc-1: -# MSYS_BITS: 32 -# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc -# SCRIPT: make ci-subset-1 -# i686-msvc-2: -# MSYS_BITS: 32 -# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc -# SCRIPT: make ci-subset-2 -# # MSVC aux tests -# x86_64-msvc-aux: -# MSYS_BITS: 64 -# RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc -# x86_64-msvc-cargo: -# MSYS_BITS: 64 -# SCRIPT: python x.py test src/tools/cargotest src/tools/cargo -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc -# VCVARS_BAT: vcvars64.bat -# # MSVC tools tests -# x86_64-msvc-tools: -# MSYS_BITS: 64 -# SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri -# -# # 32/64-bit MinGW builds. -# # -# # We are using MinGW with posix threads since LLVM does not compile with -# # the win32 threads version due to missing support for C++'s std::thread. -# # -# # Instead of relying on the MinGW version installed on appveryor we download -# # and install one ourselves so we won't be surprised by changes to appveyor's -# # build image. -# # -# # Finally, note that the downloads below are all in the `rust-lang-ci` S3 -# # bucket, but they cleraly didn't originate there! The downloads originally -# # came from the mingw-w64 SourceForge download site. Unfortunately -# # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. -# i686-mingw-1: -# MSYS_BITS: 32 -# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu -# SCRIPT: make ci-subset-1 -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z -# MINGW_DIR: mingw32 -# # FIXME(#59637) -# NO_DEBUG_ASSERTIONS: 1 -# NO_LLVM_ASSERTIONS: 1 -# i686-mingw-2: -# MSYS_BITS: 32 -# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu -# SCRIPT: make ci-subset-2 -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z -# MINGW_DIR: mingw32 -# x86_64-mingw-1: -# MSYS_BITS: 64 -# SCRIPT: make ci-subset-1 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z -# MINGW_DIR: mingw64 -# # FIXME(#59637) -# NO_DEBUG_ASSERTIONS: 1 -# NO_LLVM_ASSERTIONS: 1 -# x86_64-mingw-2: -# MSYS_BITS: 64 -# SCRIPT: make ci-subset-2 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z -# MINGW_DIR: mingw64 + # 32/64 bit MSVC tests + x86_64-msvc-1: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-msvc-2: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-2 + i686-msvc-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-1 + i686-msvc-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc + SCRIPT: make ci-subset-2 + # MSVC aux tests + x86_64-msvc-aux: + MSYS_BITS: 64 + RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + x86_64-msvc-cargo: + MSYS_BITS: 64 + SCRIPT: python x.py test src/tools/cargotest src/tools/cargo + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc + VCVARS_BAT: vcvars64.bat + # MSVC tools tests + x86_64-msvc-tools: + MSYS_BITS: 64 + SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri + + # 32/64-bit MinGW builds. + # + # We are using MinGW with posix threads since LLVM does not compile with + # the win32 threads version due to missing support for C++'s std::thread. + # + # Instead of relying on the MinGW version installed on appveryor we download + # and install one ourselves so we won't be surprised by changes to appveyor's + # build image. + # + # Finally, note that the downloads below are all in the `rust-lang-ci` S3 + # bucket, but they cleraly didn't originate there! The downloads originally + # came from the mingw-w64 SourceForge download site. Unfortunately + # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. + i686-mingw-1: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-1 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + i686-mingw-2: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu + SCRIPT: make ci-subset-2 + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + x86_64-mingw-1: + MSYS_BITS: 64 + SCRIPT: make ci-subset-1 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + # FIXME(#59637) + NO_DEBUG_ASSERTIONS: 1 + NO_LLVM_ASSERTIONS: 1 + x86_64-mingw-2: + MSYS_BITS: 64 + SCRIPT: make ci-subset-2 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: @@ -314,37 +314,37 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 -# dist-i686-msvc: -# RUST_CONFIGURE_ARGS: > -# --build=i686-pc-windows-msvc -# --target=i586-pc-windows-msvc -# --enable-full-tools -# --enable-profiler -# SCRIPT: python x.py dist -# DIST_REQUIRE_ALL_TOOLS: 1 -# DEPLOY: 1 -# dist-i686-mingw: -# MSYS_BITS: 32 -# RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools -# SCRIPT: python x.py dist -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z -# MINGW_DIR: mingw32 -# DIST_REQUIRE_ALL_TOOLS: 1 -# DEPLOY: 1 -# dist-x86_64-mingw: -# MSYS_BITS: 64 -# SCRIPT: python x.py dist -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools -# MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror -# MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z -# MINGW_DIR: mingw64 -# DIST_REQUIRE_ALL_TOOLS: 1 -# DEPLOY: 1 -# -# # "alternate" deployment, see .travis.yml for more info -# dist-x86_64-msvc-alt: -# MSYS_BITS: 64 -# RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler -# SCRIPT: python x.py dist -# DEPLOY_ALT: 1 + dist-i686-msvc: + RUST_CONFIGURE_ARGS: > + --build=i686-pc-windows-msvc + --target=i586-pc-windows-msvc + --enable-full-tools + --enable-profiler + SCRIPT: python x.py dist + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-i686-mingw: + MSYS_BITS: 32 + RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools + SCRIPT: python x.py dist + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z + MINGW_DIR: mingw32 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + dist-x86_64-mingw: + MSYS_BITS: 64 + SCRIPT: python x.py dist + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools + MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror + MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z + MINGW_DIR: mingw64 + DIST_REQUIRE_ALL_TOOLS: 1 + DEPLOY: 1 + + # "alternate" deployment, see .travis.yml for more info + dist-x86_64-msvc-alt: + MSYS_BITS: 64 + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler + SCRIPT: python x.py dist + DEPLOY_ALT: 1 diff --git a/.azure-pipelines/try.yml b/.azure-pipelines/try.yml index 5c76d20533452..f37bb7d26ed6d 100644 --- a/.azure-pipelines/try.yml +++ b/.azure-pipelines/try.yml @@ -18,132 +18,14 @@ jobs: - template: steps/run.yml strategy: matrix: - x86_64-gnu-llvm-6.0: - IMAGE: x86_64-gnu-llvm-6.0 - RUST_BACKTRACE: 1 - dist-x86_64-linux: IMAGE: dist-x86_64-linux DEPLOY: 1 + dist-x86_64-linux-alt: IMAGE: dist-x86_64-linux DEPLOY_ALT: 1 - arm-android: - IMAGE: arm-android - - armhf-gnu: - IMAGE: armhf-gnu - - dist-various-1: - IMAGE: dist-various-1 - DEPLOY: 1 - - dist-various-2: - IMAGE: dist-various-2 - DEPLOY: 1 - - dist-aarch64-linux: - IMAGE: dist-aarch64-linux - DEPLOY: 1 - - dist-android: - IMAGE: dist-android - DEPLOY: 1 - - dist-arm-linux: - IMAGE: dist-arm-linux - DEPLOY: 1 - - dist-armhf-linux: - IMAGE: dist-armhf-linux - DEPLOY: 1 - - dist-armv7-linux: - IMAGE: dist-armv7-linux - DEPLOY: 1 - - dist-i586-gnu-i586-i686-musl: - IMAGE: dist-i586-gnu-i586-i686-musl - DEPLOY: 1 - - dist-i686-freebsd: - IMAGE: dist-i686-freebsd - DEPLOY: 1 - - dist-i686-linux: - IMAGE: dist-i686-linux - DEPLOY: 1 - - dist-mips-linux: - IMAGE: dist-mips-linux - DEPLOY: 1 - - dist-mips64-linux: - IMAGE: dist-mips64-linux - DEPLOY: 1 - - dist-mips64el-linux: - IMAGE: dist-mips64el-linux - DEPLOY: 1 - - dist-mipsel-linux: - IMAGE: dist-mipsel-linux - DEPLOY: 1 - - dist-powerpc-linux: - IMAGE: dist-powerpc-linux - DEPLOY: 1 - - dist-powerpc64-linux: - IMAGE: dist-powerpc64-linux - DEPLOY: 1 - - dist-powerpc64le-linux: - IMAGE: dist-powerpc64le-linux - DEPLOY: 1 - - dist-s390x-linux: - IMAGE: dist-s390x-linux - DEPLOY: 1 - - dist-x86_64-freebsd: - IMAGE: dist-x86_64-freebsd - DEPLOY: 1 - - dist-x86_64-musl: - IMAGE: dist-x86_64-musl - DEPLOY: 1 - - dist-x86_64-netbsd: - IMAGE: dist-x86_64-netbsd - DEPLOY: 1 - - asmjs: - IMAGE: asmjs - i686-gnu: - IMAGE: i686-gnu - i686-gnu-nopt: - IMAGE: i686-gnu-nopt - test-various: - IMAGE: test-various - x86_64-gnu: - IMAGE: x86_64-gnu - x86_64-gnu-full-bootstrap: - IMAGE: x86_64-gnu-full-bootstrap - x86_64-gnu-aux: - IMAGE: x86_64-gnu-aux - x86_64-gnu-tools: - IMAGE: x86_64-gnu-tools - x86_64-gnu-debug: - IMAGE: x86_64-gnu-debug - x86_64-gnu-nopt: - IMAGE: x86_64-gnu-nopt - x86_64-gnu-distcheck: - IMAGE: x86_64-gnu-distcheck - mingw-check: - IMAGE: mingw-check - - job: macOS timeoutInMinutes: 600 pool: @@ -152,21 +34,6 @@ jobs: - template: steps/run.yml strategy: matrix: - # OSX builders running tests, these run the full test suite. - # NO_DEBUG_ASSERTIONS=1 to make them go faster, but also do have some - # runners that run `//ignore-debug` tests. - # - # Note that the compiler is compiled to target 10.8 here because the Xcode - # version that we're using, 8.2, cannot compile LLVM for OSX 10.7. - x86_64-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - dist-x86_64-apple: RUST_CHECK_TARGET: dist RUST_CONFIGURE_ARGS: --target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc @@ -186,25 +53,6 @@ jobs: NO_LLVM_ASSERTIONS: 1 NO_DEBUG_ASSERTIONS: 1 - i686-apple: - RUST_CHECK_TARGET: check - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --set rust.jemalloc - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - - dist-i686-apple: - RUST_CHECK_TARGET: dist - RUST_CONFIGURE_ARGS: --build=i686-apple-darwin --enable-full-tools --enable-profiler --enable-lldb --set rust.jemalloc - DEPLOY: 1 - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - - job: Windows timeoutInMinutes: 600 pool: @@ -213,91 +61,6 @@ jobs: - template: steps/run.yml strategy: matrix: - # 32/64 bit MSVC tests - x86_64-msvc-1: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-1 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-msvc-2: - MSYS_BITS: 64 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-2 - i686-msvc-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-1 - i686-msvc-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc - SCRIPT: make ci-subset-2 - # MSVC aux tests - x86_64-msvc-aux: - MSYS_BITS: 64 - RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - x86_64-msvc-cargo: - MSYS_BITS: 64 - SCRIPT: python x.py test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc - VCVARS_BAT: vcvars64.bat - # MSVC tools tests - x86_64-msvc-tools: - MSYS_BITS: 64 - SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri - - # 32/64-bit MinGW builds. - # - # We are using MinGW with posix threads since LLVM does not compile with - # the win32 threads version due to missing support for C++'s std::thread. - # - # Instead of relying on the MinGW version installed on appveryor we download - # and install one ourselves so we won't be surprised by changes to appveyor's - # build image. - # - # Finally, note that the downloads below are all in the `rust-lang-ci` S3 - # bucket, but they cleraly didn't originate there! The downloads originally - # came from the mingw-w64 SourceForge download site. Unfortunately - # SourceForge is notoriously flaky, so we mirror it on our own infrastructure. - i686-mingw-1: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-1 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - i686-mingw-2: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu - SCRIPT: make ci-subset-2 - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - x86_64-mingw-1: - MSYS_BITS: 64 - SCRIPT: make ci-subset-1 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - # FIXME(#59637) - NO_DEBUG_ASSERTIONS: 1 - NO_LLVM_ASSERTIONS: 1 - x86_64-mingw-2: - MSYS_BITS: 64 - SCRIPT: make ci-subset-2 - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - - # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: RUST_CONFIGURE_ARGS: > --build=x86_64-pc-windows-msvc @@ -307,35 +70,7 @@ jobs: SCRIPT: python x.py dist DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 - dist-i686-msvc: - RUST_CONFIGURE_ARGS: > - --build=i686-pc-windows-msvc - --target=i586-pc-windows-msvc - --enable-full-tools - --enable-profiler - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-i686-mingw: - MSYS_BITS: 32 - RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools - SCRIPT: python x.py dist - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z - MINGW_DIR: mingw32 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - dist-x86_64-mingw: - MSYS_BITS: 64 - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools - MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror - MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z - MINGW_DIR: mingw64 - DIST_REQUIRE_ALL_TOOLS: 1 - DEPLOY: 1 - # "alternate" deployment, see .travis.yml for more info dist-x86_64-msvc-alt: MSYS_BITS: 64 RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler From 2244ca3973fa4a43ef53a16826086e43eae86539 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 24 May 2019 10:33:14 +0200 Subject: [PATCH 57/57] ci: fix invalid syntax in the azure auto.yml --- .azure-pipelines/auto.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/auto.yml b/.azure-pipelines/auto.yml index 800cbb5a3aee1..7bb38a4e8eb9e 100644 --- a/.azure-pipelines/auto.yml +++ b/.azure-pipelines/auto.yml @@ -3,8 +3,8 @@ # pr: none - trigger: - - auto +trigger: + - auto variables: - group: prod-credentials