ci: Conditionally build parallel compiler on try

This commit configures Travis/AppVeyor to conditionally compile parallel compilers on `@bors: try`. This is an experiment currently to see how this plays out, but the intention is that if the commit message contains the term "parallel-compiler" then when `@bors: try` is issued it will perform differently than the try branch does today, building three compilers: Linux, macOS, and Windows. We currently have no `try` builders for macOS or Windows due to typical capacity issues, so it's intended that this is only very sparingly used from time to time when necessary. [parallel-compiler]
rust-lang · Mar 25, 2019 · f21a904 · f21a904
1 parent d91b32b
commit f21a904
Show file tree

Hide file tree

Showing 4 changed files with 165 additions and 113 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -26,14 +26,14 @@ matrix:
 
     - env: IMAGE=dist-x86_64-linux DEPLOY=1
       name: dist-x86_64-linux
-      if: branch = try OR branch = auto
+      if: (branch = try AND NOT commit_message ~= parallel-compiler) OR branch = auto
 
     # "alternate" deployments, these are "nightlies" but have LLVM assertions
     # turned on, they're deployed to a different location primarily for
     # additional testing.
     - env: IMAGE=dist-x86_64-linux DEPLOY_ALT=1
       name: dist-x86_64-linux-alt
-      if: branch = try OR branch = auto
+      if: (branch = try AND NOT commit_message ~= parallel-compiler) OR branch = auto
 
     - env: >
         RUST_CHECK_TARGET=dist
@@ -239,6 +239,25 @@ matrix:
         . src/ci/docker/x86_64-gnu-tools/repo.sh;
         commit_toolstate_change "$MESSAGE_FILE" "$TRAVIS_BUILD_DIR/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN";
 
+    - env: IMAGE=dist-x86_64-linux DEPLOY=1 DIST_PARALLEL_COMPILER=1
+      name: dist-x86_64-linux-parallel-compiler
+      if: branch = try AND commit_message ~= parallel-compiler
+    - env: >
+        RUST_CHECK_TARGET=dist
+        RUST_CONFIGURE_ARGS="--enable-full-tools --enable-sanitizers --enable-profiler --enable-lldb --set rust.jemalloc"
+        SRC=.
+        DEPLOY=1
+        RUSTC_RETRY_LINKER_ON_SEGFAULT=1
+        MACOSX_DEPLOYMENT_TARGET=10.7
+        NO_LLVM_ASSERTIONS=1
+        NO_DEBUG_ASSERTIONS=1
+        DIST_REQUIRE_ALL_TOOLS=1
+        DIST_PARALLEL_COMPILER=1
+      os: osx
+      osx_image: xcode9.3-moar
+      name: dist-x86_64-apple-parallel-compiler
+      if: branch = try AND commit_message ~= parallel-compiler
+
 before_install:
   # We'll use the AWS cli to download/upload cached docker layers as well as
   # push our deployments, so download that here.

diff --git a/appveyor.yml b/appveyor.yml
@@ -8,116 +8,150 @@ environment:
   # server goes down presumably. See #43333 for more info
   CARGO_HTTP_CHECK_REVOKE: false
 
-  matrix:
-  # 32/64 bit MSVC tests
-  - CI_JOB_NAME: x86_64-msvc
-    MSYS_BITS: 64
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
-    SCRIPT: python x.py test
-  - CI_JOB_NAME: i686-msvc-1
-    MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    SCRIPT: make appveyor-subset-1
-  - CI_JOB_NAME: i686-msvc-2
-    MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
-    SCRIPT: make appveyor-subset-2
 
-  # MSVC aux tests
-  - CI_JOB_NAME: x86_64-msvc-aux
-    MSYS_BITS: 64
-    RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
-  - CI_JOB_NAME: x86_64-msvc-cargo
-    MSYS_BITS: 64
-    SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
 
-  # MSVC tools tests
-  - CI_JOB_NAME: x86_64-msvc-tools
-    MSYS_BITS: 64
-    SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
+for:
+-
+  branches:
+    only:
+      - auto
+  environment:
+    matrix:
+    # 32/64 bit MSVC tests
+    - CI_JOB_NAME: x86_64-msvc
+      MSYS_BITS: 64
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler
+      SCRIPT: python x.py test
+    - CI_JOB_NAME: i686-msvc-1
+      MSYS_BITS: 32
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+      SCRIPT: make appveyor-subset-1
+    - CI_JOB_NAME: i686-msvc-2
+      MSYS_BITS: 32
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-msvc
+      SCRIPT: make appveyor-subset-2
 
-  # 32/64-bit MinGW builds.
-  #
-  # We are using MinGW with posix threads since LLVM does not compile with
-  # the win32 threads version due to missing support for C++'s std::thread.
-  #
-  # Instead of relying on the MinGW version installed on appveryor we download
-  # and install one ourselves so we won't be surprised by changes to appveyor's
-  # build image.
-  #
-  # Finally, note that the downloads below are all in the `rust-lang-ci` S3
-  # bucket, but they cleraly didn't originate there! The downloads originally
-  # came from the mingw-w64 SourceForge download site. Unfortunately
-  # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
-  - CI_JOB_NAME: i686-mingw-1
-    MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    SCRIPT: make appveyor-subset-1
-    MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
-    MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
-    MINGW_DIR: mingw32
-  - CI_JOB_NAME: i686-mingw-2
-    MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
-    SCRIPT: make appveyor-subset-2
-    MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
-    MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
-    MINGW_DIR: mingw32
-  - CI_JOB_NAME: x86_64-mingw
-    MSYS_BITS: 64
-    SCRIPT: python x.py test
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
-    MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
-    MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
-    MINGW_DIR: mingw64
+    # MSVC aux tests
+    - CI_JOB_NAME: x86_64-msvc-aux
+      MSYS_BITS: 64
+      RUST_CHECK_TARGET: check-aux EXCLUDE_CARGO=1
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+    - CI_JOB_NAME: x86_64-msvc-cargo
+      MSYS_BITS: 64
+      SCRIPT: python x.py test src/tools/cargotest src/tools/cargo
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc
+
+    # MSVC tools tests
+    - CI_JOB_NAME: x86_64-msvc-tools
+      MSYS_BITS: 64
+      SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstates.json windows
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstates.json --enable-test-miri
+
+    # 32/64-bit MinGW builds.
+    #
+    # We are using MinGW with posix threads since LLVM does not compile with
+    # the win32 threads version due to missing support for C++'s std::thread.
+    #
+    # Instead of relying on the MinGW version installed on appveryor we download
+    # and install one ourselves so we won't be surprised by changes to appveyor's
+    # build image.
+    #
+    # Finally, note that the downloads below are all in the `rust-lang-ci` S3
+    # bucket, but they cleraly didn't originate there! The downloads originally
+    # came from the mingw-w64 SourceForge download site. Unfortunately
+    # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
+    - CI_JOB_NAME: i686-mingw-1
+      MSYS_BITS: 32
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+      SCRIPT: make appveyor-subset-1
+      MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+      MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+      MINGW_DIR: mingw32
+    - CI_JOB_NAME: i686-mingw-2
+      MSYS_BITS: 32
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+      SCRIPT: make appveyor-subset-2
+      MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+      MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+      MINGW_DIR: mingw32
+    - CI_JOB_NAME: x86_64-mingw
+      MSYS_BITS: 64
+      SCRIPT: python x.py test
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+      MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+      MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+      MINGW_DIR: mingw64
+
+    # 32/64 bit MSVC and GNU deployment
+    - CI_JOB_NAME: dist-x86_64-msvc
+      RUST_CONFIGURE_ARGS: >
+        --build=x86_64-pc-windows-msvc
+        --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist
+      DIST_REQUIRE_ALL_TOOLS: 1
+      DEPLOY: 1
+    - CI_JOB_NAME: dist-i686-msvc
+      RUST_CONFIGURE_ARGS: >
+        --build=i686-pc-windows-msvc
+        --target=i586-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist
+      DIST_REQUIRE_ALL_TOOLS: 1
+      DEPLOY: 1
+    - CI_JOB_NAME: dist-i686-mingw
+      MSYS_BITS: 32
+      RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools
+      SCRIPT: python x.py dist
+      MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+      MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
+      MINGW_DIR: mingw32
+      DIST_REQUIRE_ALL_TOOLS: 1
+      DEPLOY: 1
+    - CI_JOB_NAME: dist-x86_64-mingw
+      MSYS_BITS: 64
+      SCRIPT: python x.py dist
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools
+      MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
+      MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
+      MINGW_DIR: mingw64
+      DIST_REQUIRE_ALL_TOOLS: 1
+      DEPLOY: 1
 
-  # 32/64 bit MSVC and GNU deployment
-  - CI_JOB_NAME: dist-x86_64-msvc
-    RUST_CONFIGURE_ARGS: >
-      --build=x86_64-pc-windows-msvc
-      --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
-      --enable-full-tools
-      --enable-profiler
-    SCRIPT: python x.py dist
-    DIST_REQUIRE_ALL_TOOLS: 1
-    DEPLOY: 1
-  - CI_JOB_NAME: dist-i686-msvc
-    RUST_CONFIGURE_ARGS: >
-      --build=i686-pc-windows-msvc
-      --target=i586-pc-windows-msvc
-      --enable-full-tools
-      --enable-profiler
-    SCRIPT: python x.py dist
-    DIST_REQUIRE_ALL_TOOLS: 1
-    DEPLOY: 1
-  - CI_JOB_NAME: dist-i686-mingw
-    MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools
-    SCRIPT: python x.py dist
-    MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
-    MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
-    MINGW_DIR: mingw32
-    DIST_REQUIRE_ALL_TOOLS: 1
-    DEPLOY: 1
-  - CI_JOB_NAME: dist-x86_64-mingw
-    MSYS_BITS: 64
-    SCRIPT: python x.py dist
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-full-tools
-    MINGW_URL: https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rust-ci-mirror
-    MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
-    MINGW_DIR: mingw64
-    DIST_REQUIRE_ALL_TOOLS: 1
-    DEPLOY: 1
+    # "alternate" deployment, see .travis.yml for more info
+    - CI_JOB_NAME: dist-x86_64-msvc-alt
+      MSYS_BITS: 64
+      RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
+      SCRIPT: python x.py dist
+      DEPLOY_ALT: 1
 
-  # "alternate" deployment, see .travis.yml for more info
-  - CI_JOB_NAME: dist-x86_64-msvc-alt
-    MSYS_BITS: 64
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-    SCRIPT: python x.py dist
-    DEPLOY_ALT: 1
+# Try builder
+-
+  branches:
+    only:
+      - try
+  environment:
+    matrix:
+    - CI_JOB_NAME: dist-x86_64-msvc-parallel
+      RUST_CONFIGURE_ARGS: >
+        --build=x86_64-pc-windows-msvc
+        --enable-full-tools
+        --enable-profiler
+      SCRIPT: python x.py dist
+      DIST_REQUIRE_ALL_TOOLS: 1
+      DIST_PARALLEL_COMPILER: 1
+      DEPLOY: 1
+
+# Skip builds on all other branches
+-
+  branches:
+    except:
+      - try
+      - auto
+  only_commits:
+    message: /nonexistent-message-to-fool-appveyor-to-not-run-a-build/
 
 matrix:
   fast_finish: true
@@ -216,10 +250,6 @@ on_failure:
   - set PATH=%PATH%;"C:\Program Files (x86)\Windows Kits\10\Debuggers\X64"
   - if exist %LOCALAPPDATA%\CrashDumps for %%f in (%LOCALAPPDATA%\CrashDumps\*) do cdb -c "k;q" -G -z "%%f"
 
-branches:
-  only:
-    - auto
-
 before_deploy:
   - ps: |
         New-Item -Path deploy -ItemType directory
@@ -239,7 +269,6 @@ deploy:
     artifact: /.*/
     folder: rustc-builds
     on:
-      branch: auto
       DEPLOY: 1
     max_error_retry: 5
 
@@ -254,7 +283,6 @@ deploy:
     artifact: /.*/
     folder: rustc-builds-alt
     on:
-      branch: auto
       DEPLOY_ALT: 1
     max_error_retry: 5
 

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
@@ -133,6 +133,7 @@ exec docker \
   --env CARGO_HOME=/cargo \
   --env DEPLOY \
   --env DEPLOY_ALT \
+  --env DIST_PARALLEL_COMPILER \
   --env LOCAL_USER_ID=`id -u` \
   --env TRAVIS \
   --env TRAVIS_BRANCH \

diff --git a/src/ci/run.sh b/src/ci/run.sh
@@ -37,6 +37,10 @@ if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
 fi
 
+if [ "$DIST_PARALLEL_COMPILER" = "" ]; then
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler"
+fi
+
 # If we're deploying artifacts then we set the release channel, otherwise if
 # we're not deploying then we want to be sure to enable all assertions because
 # we'll be running tests