Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
travis: Split all dist builders in two
Previously we would use one builder on Travis to produce two sets of host
compilers for two different targets. Unfortunately though we've recently
increased how much we're building for each target so this is starting to take
unnecessarily long (#40804). This commit splits the dist builders in two by
ensuring that we only dist one target on each builder, which should take a much
shorter amount of time. This should also unblock other work such as landing the
RLS (#40584).
  • Loading branch information
alexcrichton committed Apr 3, 2017
1 parent 5e122f5 commit 541512b
Show file tree
Hide file tree
Showing 50 changed files with 996 additions and 66 deletions.
18 changes: 13 additions & 5 deletions .travis.yml
Expand Up @@ -15,19 +15,27 @@ matrix:
- env: IMAGE=arm-android
- env: IMAGE=armhf-gnu
- env: IMAGE=cross DEPLOY=1
- env: IMAGE=dist-aarch64-linux DEPLOY=1
- env: IMAGE=dist-android DEPLOY=1
- env: IMAGE=dist-arm-linux DEPLOY=1
- env: IMAGE=dist-armv7-aarch64-linux DEPLOY=1
- env: IMAGE=dist-freebsd DEPLOY=1
- env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1
- env: IMAGE=dist-armhf-linux DEPLOY=1
- env: IMAGE=dist-armv7-linux DEPLOY=1
- env: IMAGE=dist-fuchsia DEPLOY=1
- env: IMAGE=dist-i586-gnu-i686-musl DEPLOY=1
- env: IMAGE=dist-i686-freebsd DEPLOY=1
- env: IMAGE=dist-i686-linux DEPLOY=1
- env: IMAGE=dist-mips-linux DEPLOY=1
- env: IMAGE=dist-mips64-linux DEPLOY=1
- env: IMAGE=dist-mips64el-linux DEPLOY=1
- env: IMAGE=dist-mipsel-linux DEPLOY=1
- env: IMAGE=dist-powerpc-linux DEPLOY=1
- env: IMAGE=dist-powerpc64-linux DEPLOY=1
- env: IMAGE=dist-s390x-linux-netbsd DEPLOY=1
- env: IMAGE=dist-x86-linux DEPLOY=1
- env: IMAGE=dist-powerpc64le-linux DEPLOY=1
- env: IMAGE=dist-s390x-linux DEPLOY=1
- env: IMAGE=dist-x86_64-freebsd DEPLOY=1
- env: IMAGE=dist-x86_64-linux DEPLOY=1
- env: IMAGE=dist-x86_64-musl DEPLOY=1
- env: IMAGE=dist-x86_64-netbsd DEPLOY=1
- env: IMAGE=emscripten
- env: IMAGE=i686-gnu
- env: IMAGE=i686-gnu-nopt
Expand Down
Expand Up @@ -56,8 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
USER rustbuild
WORKDIR /tmp

COPY armv7-linux-gnueabihf.config /tmp/
COPY armv7-linux-gnueabihf.config aarch64-linux-gnu.config build-toolchains.sh /tmp/
COPY aarch64-linux-gnu.config build-toolchains.sh /tmp/
RUN ./build-toolchains.sh

USER root
Expand All @@ -67,17 +66,12 @@ RUN curl -o /usr/local/bin/sccache \
chmod +x /usr/local/bin/sccache

ENV PATH=$PATH:/x-tools/aarch64-unknown-linux-gnueabi/bin
ENV PATH=$PATH:/x-tools/armv7-unknown-linux-gnueabihf/bin

ENV CC_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-gcc \
AR_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-ar \
CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-g++ \
CC_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-gcc \
AR_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-ar \
CXX_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-g++
CXX_aarch64_unknown_linux_gnu=aarch64-unknown-linux-gnueabi-g++

ENV HOSTS=armv7-unknown-linux-gnueabihf
ENV HOSTS=$HOSTS,aarch64-unknown-linux-gnu
ENV HOSTS=aarch64-unknown-linux-gnu

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
37 changes: 37 additions & 0 deletions src/ci/docker/dist-aarch64-linux/build-toolchains.sh
@@ -0,0 +1,37 @@
#!/bin/bash
# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

set -ex

hide_output() {
set +x
on_err="
echo ERROR: An error was encountered with the build.
cat /tmp/build.log
exit 1
"
trap "$on_err" ERR
bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
PING_LOOP_PID=$!
$@ &> /tmp/build.log
rm /tmp/build.log
trap - ERR
kill $PING_LOOP_PID
set -x
}

mkdir build
cd build
cp ../aarch64-linux-gnu.config .config
ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build
9 changes: 2 additions & 7 deletions src/ci/docker/dist-arm-linux/Dockerfile
Expand Up @@ -56,7 +56,7 @@ RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
USER rustbuild
WORKDIR /tmp

COPY arm-linux-gnueabihf.config arm-linux-gnueabi.config build-toolchains.sh /tmp/
COPY arm-linux-gnueabi.config build-toolchains.sh /tmp/
RUN ./build-toolchains.sh

USER root
Expand All @@ -66,17 +66,12 @@ RUN curl -o /usr/local/bin/sccache \
chmod +x /usr/local/bin/sccache

ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabi/bin
ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabihf/bin

ENV CC_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-gcc \
AR_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-ar \
CXX_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-g++ \
CC_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-gcc \
AR_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-ar \
CXX_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-g++
CXX_arm_unknown_linux_gnueabi=arm-unknown-linux-gnueabi-g++

ENV HOSTS=arm-unknown-linux-gnueabi
ENV HOSTS=$HOSTS,arm-unknown-linux-gnueabihf

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
8 changes: 0 additions & 8 deletions src/ci/docker/dist-arm-linux/build-toolchains.sh
Expand Up @@ -35,11 +35,3 @@ ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build

mkdir build
cd build
cp ../arm-linux-gnueabihf.config .config
ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build
77 changes: 77 additions & 0 deletions src/ci/docker/dist-armhf-linux/Dockerfile
@@ -0,0 +1,77 @@
FROM ubuntu:16.04

RUN apt-get update && apt-get install -y --no-install-recommends \
automake \
bison \
bzip2 \
ca-certificates \
cmake \
curl \
file \
flex \
g++ \
gawk \
gdb \
git \
gperf \
help2man \
libncurses-dev \
libtool-bin \
make \
patch \
python2.7 \
sudo \
texinfo \
wget \
xz-utils \
libssl-dev \
pkg-config

RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

# Ubuntu 16.04 (this contianer) ships with make 4, but something in the
# toolchains we build below chokes on that, so go back to make 3
RUN curl https://ftp.gnu.org/gnu/make/make-3.81.tar.gz | tar xzf - && \
cd make-3.81 && \
./configure --prefix=/usr && \
make && \
make install && \
cd .. && \
rm -rf make-3.81

RUN curl http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.22.0.tar.bz2 | \
tar xjf - && \
cd crosstool-ng && \
./configure --prefix=/usr/local && \
make -j$(nproc) && \
make install && \
cd .. && \
rm -rf crosstool-ng

RUN groupadd -r rustbuild && useradd -m -r -g rustbuild rustbuild
RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
USER rustbuild
WORKDIR /tmp

COPY arm-linux-gnueabihf.config build-toolchains.sh /tmp/
RUN ./build-toolchains.sh

USER root

RUN curl -o /usr/local/bin/sccache \
https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-24-sccache-x86_64-unknown-linux-musl && \
chmod +x /usr/local/bin/sccache

ENV PATH=$PATH:/x-tools/arm-unknown-linux-gnueabihf/bin

ENV CC_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-gcc \
AR_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-ar \
CXX_arm_unknown_linux_gnueabihf=arm-unknown-linux-gnueabihf-g++

ENV HOSTS=arm-unknown-linux-gnueabihf

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
37 changes: 37 additions & 0 deletions src/ci/docker/dist-armhf-linux/build-toolchains.sh
@@ -0,0 +1,37 @@
#!/bin/bash
# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.

set -ex

hide_output() {
set +x
on_err="
echo ERROR: An error was encountered with the build.
cat /tmp/build.log
exit 1
"
trap "$on_err" ERR
bash -c "while true; do sleep 30; echo \$(date) - building ...; done" &
PING_LOOP_PID=$!
$@ &> /tmp/build.log
rm /tmp/build.log
trap - ERR
kill $PING_LOOP_PID
set -x
}

mkdir build
cd build
cp ../arm-linux-gnueabihf.config .config
ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build
77 changes: 77 additions & 0 deletions src/ci/docker/dist-armv7-linux/Dockerfile
@@ -0,0 +1,77 @@
FROM ubuntu:16.04

RUN apt-get update && apt-get install -y --no-install-recommends \
automake \
bison \
bzip2 \
ca-certificates \
cmake \
curl \
file \
flex \
g++ \
gawk \
gdb \
git \
gperf \
help2man \
libncurses-dev \
libtool-bin \
make \
patch \
python2.7 \
sudo \
texinfo \
wget \
xz-utils \
libssl-dev \
pkg-config

RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
dpkg -i dumb-init_*.deb && \
rm dumb-init_*.deb
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

# Ubuntu 16.04 (this contianer) ships with make 4, but something in the
# toolchains we build below chokes on that, so go back to make 3
RUN curl https://ftp.gnu.org/gnu/make/make-3.81.tar.gz | tar xzf - && \
cd make-3.81 && \
./configure --prefix=/usr && \
make && \
make install && \
cd .. && \
rm -rf make-3.81

RUN curl http://crosstool-ng.org/download/crosstool-ng/crosstool-ng-1.22.0.tar.bz2 | \
tar xjf - && \
cd crosstool-ng && \
./configure --prefix=/usr/local && \
make -j$(nproc) && \
make install && \
cd .. && \
rm -rf crosstool-ng

RUN groupadd -r rustbuild && useradd -m -r -g rustbuild rustbuild
RUN mkdir /x-tools && chown rustbuild:rustbuild /x-tools
USER rustbuild
WORKDIR /tmp

COPY build-toolchains.sh armv7-linux-gnueabihf.config /tmp/
RUN ./build-toolchains.sh

USER root

RUN curl -o /usr/local/bin/sccache \
https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-24-sccache-x86_64-unknown-linux-musl && \
chmod +x /usr/local/bin/sccache

ENV PATH=$PATH:/x-tools/armv7-unknown-linux-gnueabihf/bin

ENV CC_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-gcc \
AR_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-ar \
CXX_armv7_unknown_linux_gnueabihf=armv7-unknown-linux-gnueabihf-g++

ENV HOSTS=armv7-unknown-linux-gnueabihf

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
Expand Up @@ -35,11 +35,3 @@ ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build

mkdir build
cd build
cp ../aarch64-linux-gnu.config .config
ct-ng oldconfig
hide_output ct-ng build
cd ..
rm -rf build
Expand Up @@ -17,7 +17,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
pkg-config

COPY build-toolchain.sh /tmp/
RUN /tmp/build-toolchain.sh x86_64
RUN /tmp/build-toolchain.sh i686

RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
Expand All @@ -30,15 +29,11 @@ RUN curl -o /usr/local/bin/sccache \
chmod +x /usr/local/bin/sccache

ENV \
AR_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-ar \
CC_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-gcc \
CXX_x86_64_unknown_freebsd=x86_64-unknown-freebsd10-g++ \
AR_i686_unknown_freebsd=i686-unknown-freebsd10-ar \
CC_i686_unknown_freebsd=i686-unknown-freebsd10-gcc \
CXX_i686_unknown_freebsd=i686-unknown-freebsd10-g++

ENV HOSTS=x86_64-unknown-freebsd
ENV HOSTS=$HOSTS,i686-unknown-freebsd
ENV HOSTS=i686-unknown-freebsd

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
Expand Up @@ -80,7 +80,6 @@ RUN curl -o /usr/local/bin/sccache \
chmod +x /usr/local/bin/sccache

ENV HOSTS=i686-unknown-linux-gnu
ENV HOSTS=$HOSTS,x86_64-unknown-linux-gnu

ENV RUST_CONFIGURE_ARGS \
--host=$HOSTS \
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
2 changes: 0 additions & 2 deletions src/ci/docker/dist-mips-linux/Dockerfile
Expand Up @@ -13,7 +13,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
gdb \
xz-utils \
g++-mips-linux-gnu \
g++-mipsel-linux-gnu \
libssl-dev \
pkg-config

Expand All @@ -27,7 +26,6 @@ RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-ini
ENTRYPOINT ["/usr/bin/dumb-init", "--"]

ENV HOSTS=mips-unknown-linux-gnu
ENV HOSTS=$HOSTS,mipsel-unknown-linux-gnu

ENV RUST_CONFIGURE_ARGS --host=$HOSTS --enable-extended
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS

0 comments on commit 541512b

Please sign in to comment.