Move all Linux/OSX CI infastructure to Travis

This commit configures our `.travis.yml` to test the full suite of tests we have
on Buildbot right now. A whole mess of docker images are added to the `src/ci`
directory which represent all the build environments for each configuration.
Each of these environments is then configured in `.travis.yml` to run on the
auto branch.

Note that the full matrix of tests aren't intended to be run on all PRs.
Instead, we continue to run only one entry in the matrix, forcing all others to
finish quickly. Only the `auto` branch should run the full matrix of builds.

Also note that the infrastructure hasn't quite been allocated yet to the
rust-lang/rust repository, so everything is disabled for now except for the one
build that happens on PRs. Once that infrastructure is allocated though we can
enable this and let it fly!

Notable modifications from the current test suite today:

* Android tests are run in rustbuild instead of the makefiles, for whatever
  reason I couldn't get the makefiles to work on Travis.
* A debuginfo test was updated to work with the current version of the Android
  NDK.
* Some dependencies in `mk/tests.mk` were fixed to allow running tests in
  parallel.

.travis.yml

@@ -1,33 +1,82 @@
-language: generic
+language: rust
 sudo: required
+dist: trusty
 services:
   - docker
 
-# LLVM takes awhile to check out and otherwise we'll manage the submodules in
-# our configure script, so disable auto submodule management.
 git:
-  submodules: false
   depth: 1
+  submodules: false
 
-before_install:
-  - docker build -t rust -f src/etc/Dockerfile src/etc
+matrix:
+  include:
+    # Linux builders, all docker images
+    - env: IMAGE=arm-android
+    - env: IMAGE=cross
+    - env: IMAGE=i686-gnu
+    - env: IMAGE=i686-gnu-nopt
+    - env: IMAGE=x86_64-freebsd
+    - env: IMAGE=x86_64-gnu
+    - env: IMAGE=x86_64-gnu-cargotest
+    - env: IMAGE=x86_64-gnu-debug
+    - env: IMAGE=x86_64-gnu-nopt
+    - env: IMAGE=x86_64-gnu-rustbuild
+    - env: IMAGE=x86_64-gnu-llvm-3.7 ALLOW_PR=1
+    - env: IMAGE=x86_64-musl
+
+    # OSX builders
+    - env: >
+        RUST_CHECK_TARGET=check
+        RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin
+        SRC=.
+      os: osx
+      install: brew install ccache
+    - env: >
+        RUST_CHECK_TARGET=check
+        RUST_CONFIGURE_ARGS=--target=i686-apple-darwin
+        SRC=.
+      os: osx
+      install: brew install ccache
+    - env: >
+        RUST_CHECK_TARGET=check
+        RUST_CONFIGURE_ARGS=--target=x86_64-apple-darwin --enable-rustbuild
+        SRC=.
+      os: osx
+      install: brew install ccache
+    - env: >
+        RUST_CHECK_TARGET=
+        RUST_CONFIGURE_ARGS=--target=aarch64-apple-ios,armv7-apple-ios,armv7s-apple-ios,i386-apple-ios,x86_64-apple-ios
+        SRC=.
+      os: osx
+      install: brew install ccache
 
 script:
-  - docker run -v `pwd`:/build rust
-    sh -c "
-      ./configure --enable-vendor --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
-      make tidy &&
-      make check -j4
-    "
+  - if [ -z "$ALLOW_PR" ] && [ "$TRAVIS_BRANCH" != "auto" ]; then
+        echo skipping, not a full build;
+    elif [ -z "$ENABLE_AUTO" ] then
+        echo skipping, not quite ready yet
+    elif [ "$TRAVIS_OS_NAME" = "osx" ]; then
+        git submodule update --init;
+        src/ci/run.sh;
+    else
+        git submodule update --init;
+        src/ci/docker/run.sh $IMAGE;
+    fi
 
-# Real testing happens on http://buildbot.rust-lang.org/
-#
-# See https://github.com/rust-lang/rust-buildbot
-#     CONTRIBUTING.md#pull-requests
+# Save tagged docker images we created and load them if they're available
+before_cache:
+  - docker history -q rust-ci |
+    grep -v missing |
+    xargs docker save |
+    gzip -9 > $HOME/docker/rust-ci.tar.gz
+before_install:
+  - zcat $HOME/docker/rust-ci.tar.gz | docker load || true
 
 notifications:
   email: false
 
-branches:
-  only:
-    - master
+cache:
+  directories:
+    - $HOME/docker
+    - $HOME/.ccache
+    - $HOME/.cargo

mk/tests.mk

@@ -697,6 +697,8 @@ CTEST_DEPS_ui_$(1)-T-$(2)-H-$(3) = $$(UI_TESTS)
 CTEST_DEPS_mir-opt_$(1)-T-$(2)-H-$(3) = $$(MIR_OPT_TESTS)
 CTEST_DEPS_rustdocck_$(1)-T-$(2)-H-$(3) = $$(RUSTDOCCK_TESTS) \
 		$$(HBIN$(1)_H_$(3))/rustdoc$$(X_$(3)) \
+		$$(CSREQ$(1)_T_$(3)_H_$(3)) \
+		$$(SREQ$(1)_T_$(3)_H_$(3)) \
 		$(S)src/etc/htmldocck.py
 
 endef

src/bootstrap/mk/Makefile.in

@@ -57,6 +57,9 @@ else
 	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
 endif
 tidy:
-	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
+	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS) --stage 0
+
+check-stage2-android:
+	$(Q)$(BOOTSTRAP) --step check-target --target arm-linux-androideabi
 
 .PHONY: dist

src/ci/docker/arm-android/Dockerfile

@@ -0,0 +1,46 @@
+FROM ubuntu:16.04
+
+RUN dpkg --add-architecture i386 && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  python-minimal \
+  git \
+  cmake \
+  ccache \
+  unzip \
+  expect \
+  openjdk-9-jre \
+  sudo \
+  libstdc++6:i386
+
+WORKDIR /android/
+ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
+
+COPY install-ndk.sh install-sdk.sh accept-licenses.sh /android/
+RUN sh /android/install-ndk.sh
+RUN sh /android/install-sdk.sh
+
+COPY start-emulator.sh /android/
+ENTRYPOINT ["/android/start-emulator.sh"]
+
+ENV TARGETS=arm-linux-androideabi
+ENV TARGETS=$TARGETS,i686-linux-android
+ENV TARGETS=$TARGETS,aarch64-linux-android
+ENV TARGETS=$TARGETS,armv7-linux-androideabi
+
+ENV RUST_CONFIGURE_ARGS \
+      --target=$TARGETS \
+      --arm-linux-androideabi-ndk=/android/ndk-arm-9 \
+      --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
+      --i686-linux-android-ndk=/android/ndk-x86-9 \
+      --aarch64-linux-android-ndk=/android/ndk-aarch64 \
+      --enable-rustbuild
+ENV RUST_CHECK_TARGET check-stage2-android
+RUN mkdir /tmp/obj
+RUN chmod 777 /tmp/obj

src/ci/docker/arm-android/accept-licenses.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/expect -f
+# ignore-license
+
+set timeout 1800
+set cmd [lindex $argv 0]
+set licenses [lindex $argv 1]
+
+spawn {*}$cmd
+expect {
+  "Do you accept the license '*'*" {
+        exp_send "y\r"
+        exp_continue
+  }
+  eof
+}

src/ci/docker/arm-android/install-ndk.sh

@@ -0,0 +1,45 @@
+#!/bin/sh
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+cpgdb() {
+  cp android-ndk-r11c/prebuilt/linux-x86_64/bin/gdb /android/$1/bin/$2-gdb
+  cp android-ndk-r11c/prebuilt/linux-x86_64/bin/gdb-orig /android/$1/bin/gdb-orig
+  cp -r android-ndk-r11c/prebuilt/linux-x86_64/share /android/$1/share
+}
+
+# Prep the Android NDK
+#
+# See https://github.com/servo/servo/wiki/Building-for-Android
+curl -O https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip
+unzip -q android-ndk-r11c-linux-x86_64.zip
+bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
+        --platform=android-9 \
+        --toolchain=arm-linux-androideabi-4.9 \
+        --install-dir=/android/ndk-arm-9 \
+        --ndk-dir=/android/android-ndk-r11c \
+        --arch=arm
+cpgdb ndk-arm-9 arm-linux-androideabi
+bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
+        --platform=android-21 \
+        --toolchain=aarch64-linux-android-4.9 \
+        --install-dir=/android/ndk-aarch64 \
+        --ndk-dir=/android/android-ndk-r11c \
+        --arch=arm64
+bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
+        --platform=android-9 \
+        --toolchain=x86-4.9 \
+        --install-dir=/android/ndk-x86-9 \
+        --ndk-dir=/android/android-ndk-r11c \
+        --arch=x86
+
+rm -rf ./android-ndk-r11c-linux-x86_64.zip ./android-ndk-r11c

src/ci/docker/arm-android/install-sdk.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+
+# Prep the SDK and emulator
+#
+# Note that the update process requires that we accept a bunch of licenses, and
+# we can't just pipe `yes` into it for some reason, so we take the same strategy
+# located in https://github.com/appunite/docker by just wrapping it in a script
+# which apparently magically accepts the licenses.
+
+mkdir sdk
+curl https://dl.google.com/android/android-sdk_r24.4-linux.tgz | \
+    tar xzf - -C sdk --strip-components=1
+
+filter="platform-tools,android-18"
+filter="$filter,sys-img-armeabi-v7a-android-18"
+
+./accept-licenses.sh "android - update sdk -a --no-ui --filter $filter"
+
+echo "no" | android create avd \
+                --name arm-18 \
+                --target android-18 \
+                --abi armeabi-v7a

src/ci/docker/arm-android/start-emulator.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+set -ex
+ANDROID_EMULATOR_FORCE_32BIT=true \
+  emulator @arm-18 -no-window -partition-size 2047 &
+exec "$@"

src/ci/docker/cross/Dockerfile

@@ -0,0 +1,66 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++ \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  python-minimal \
+  git \
+  cmake \
+  ccache \
+  sudo \
+  gcc-aarch64-linux-gnu libc6-dev-arm64-cross \
+  gcc-arm-linux-gnueabi libc6-dev-armel-cross \
+  gcc-arm-linux-gnueabihf libc6-dev-armhf-cross \
+  gcc-mips-linux-gnu libc6-dev-mips-cross \
+  gcc-mipsel-linux-gnu libc6-dev-mipsel-cross \
+  gcc-mips64-linux-gnuabi64 libc6-dev-mips64-cross \
+  gcc-mips64el-linux-gnuabi64 libc6-dev-mips64el-cross \
+  gcc-powerpc-linux-gnu libc6-dev-powerpc-cross \
+  gcc-powerpc64-linux-gnu libc6-dev-ppc64-cross \
+  gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross \
+  gcc-s390x-linux-gnu libc6-dev-s390x-cross
+
+ENV TARGETS=aarch64-unknown-linux-gnu
+ENV TARGETS=$TARGETS,arm-unknown-linux-gnueabi
+ENV TARGETS=$TARGETS,arm-unknown-linux-gnueabihf
+ENV TARGETS=$TARGETS,armv7-unknown-linux-gnueabihf
+ENV TARGETS=$TARGETS,asmjs-unknown-emscripten
+ENV TARGETS=$TARGETS,mips-unknown-linux-gnu
+ENV TARGETS=$TARGETS,mips64-unknown-linux-gnuabi64
+ENV TARGETS=$TARGETS,mips64el-unknown-linux-gnuabi64
+ENV TARGETS=$TARGETS,mipsel-unknown-linux-gnu
+ENV TARGETS=$TARGETS,powerpc-unknown-linux-gnu
+ENV TARGETS=$TARGETS,powerpc64-unknown-linux-gnu
+ENV TARGETS=$TARGETS,powerpc64le-unknown-linux-gnu
+ENV TARGETS=$TARGETS,s390x-unknown-linux-gnu
+ENV TARGETS=$TARGETS,wasm32-unknown-emscripten
+
+#ENV TARGETS=$TARGETS,mips-unknown-linux-musl
+#ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi
+#ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf
+#ENV TARGETS=$TARGETS,armv7-unknown-linux-musleabihf
+#ENV TARGETS=$TARGETS,x86_64-rumprun-netbsd
+
+ENV RUST_CONFIGURE_ARGS \
+      --target=$TARGETS \
+      --enable-rustbuild
+ENV RUST_CHECK_TARGET ""
+
+ENV AR_s390x_unknown_linux_gnu=s390x-linux-gnu-ar \
+    CC_s390x_unknown_linux_gnu=s390x-linux-gnu-gcc \
+    AR_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-ar \
+    CC_mips64_unknown_linux_gnuabi64=mips64-linux-gnuabi64-gcc \
+    AR_mips64el_unknown_linux_gnuabi64=mips64el-linux-gnuabi64-ar \
+    CC_mips64el_unknown_linux_gnuabi64=mips64el-linux-gnuabi64-gcc \
+    AR_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-ar \
+    CC_powerpc64_unknown_linux_gnu=powerpc64-linux-gnu-gcc
+
+# FIXME(rust-lang/rust#36150): powerpc unfortunately aborts right now
+ENV NO_LLVM_ASSERTIONS=1
+
+RUN mkdir /tmp/obj
+RUN chmod 777 /tmp/obj

src/ci/docker/i686-gnu-nopt/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  ccache \
+  sudo \
+  gdb
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu --disable-optimize-tests
+ENV RUST_CHECK_TARGET check
+RUN mkdir /tmp/obj
+RUN chmod 777 /tmp/obj

src/ci/docker/i686-gnu/Dockerfile

@@ -0,0 +1,19 @@
+FROM ubuntu:16.04
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  g++-multilib \
+  make \
+  file \
+  curl \
+  ca-certificates \
+  python2.7 \
+  git \
+  cmake \
+  ccache \
+  sudo \
+  gdb
+
+ENV RUST_CONFIGURE_ARGS --build=i686-unknown-linux-gnu
+ENV RUST_CHECK_TARGET check
+RUN mkdir /tmp/obj
+RUN chmod 777 /tmp/obj