Skip to content

Commit

Permalink
Introduce dockercore/buildbase x64 base image
Browse files Browse the repository at this point in the history
Split the build Dockerfile in two, and extra the most stable pieces of
the build environment into a `dockercore/buildbase` image.

The goal is to minimize build times for the most traditional use cases,
while minimizing the dependencies to external services in our build
process.

Signed-off-by: Arnaud Porterie <arnaud.porterie@docker.com>
  • Loading branch information
Arnaud Porterie committed Mar 2, 2016
1 parent 32d1a6c commit 8b9d27e
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 160 deletions.
162 changes: 2 additions & 160 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,97 +23,8 @@
# the case. Therefore, you don't have to disable it anymore.
#

FROM debian:jessie

# add zfs ppa
RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 \
|| apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61
RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list

# add llvm repo
RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 \
|| apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421
RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list

# Packaged dependencies
RUN apt-get update && apt-get install -y \
apparmor \
aufs-tools \
automake \
bash-completion \
btrfs-tools \
build-essential \
clang-3.8 \
createrepo \
curl \
dpkg-sig \
gcc-mingw-w64 \
git \
iptables \
jq \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libsqlite3-dev \
libsystemd-journal-dev \
libtool \
mercurial \
net-tools \
pkg-config \
python-dev \
python-mock \
python-pip \
python-websocket \
s3cmd=1.5.0* \
ubuntu-zfs \
xfsprogs \
libzfs-dev \
tar \
--no-install-recommends \
&& ln -snf /usr/bin/clang-3.8 /usr/local/bin/clang \
&& ln -snf /usr/bin/clang++-3.8 /usr/local/bin/clang++

# Get lvm2 source for compiling statically
ENV LVM2_VERSION 2.02.103
RUN mkdir -p /usr/local/lvm2 \
&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
| tar -xzC /usr/local/lvm2 --strip-components=1
# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags

# Compile and install lvm2
RUN cd /usr/local/lvm2 \
&& ./configure \
--build="$(gcc -print-multiarch)" \
--enable-static_link \
&& make device-mapper \
&& make install_device-mapper
# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL

# Configure the container for OSX cross compilation
ENV OSX_SDK MacOSX10.11.sdk
ENV OSX_CROSS_COMMIT 8aa9b71a394905e6c5f4b59e2b97b87a004658a4
RUN set -x \
&& export OSXCROSS_PATH="/osxcross" \
&& git clone https://github.com/tpoechtrager/osxcross.git $OSXCROSS_PATH \
&& ( cd $OSXCROSS_PATH && git checkout -q $OSX_CROSS_COMMIT) \
&& curl -sSL https://s3.dockerproject.org/darwin/${OSX_SDK}.tar.xz -o "${OSXCROSS_PATH}/tarballs/${OSX_SDK}.tar.xz" \
&& UNATTENDED=yes OSX_VERSION_MIN=10.6 ${OSXCROSS_PATH}/build.sh
ENV PATH /osxcross/target/bin:$PATH

# install seccomp: the version shipped in trusty is too old
ENV SECCOMP_VERSION 2.2.3
RUN set -x \
&& export SECCOMP_PATH="$(mktemp -d)" \
&& curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \
| tar -xzC "$SECCOMP_PATH" --strip-components=1 \
&& ( \
cd "$SECCOMP_PATH" \
&& ./configure --prefix=/usr/local \
&& make \
&& make install \
&& ldconfig \
) \
&& rm -rf "$SECCOMP_PATH"
# See `Dockerfile.base`
FROM dockercore/buildbase

# Install Go
# IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines
Expand All @@ -125,34 +36,6 @@ RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd6
ENV PATH /go/bin:/usr/local/go/bin:$PATH
ENV GOPATH /go:/go/src/github.com/docker/docker/vendor

# Compile Go for cross compilation
ENV DOCKER_CROSSPLATFORMS \
linux/386 linux/arm \
darwin/amd64 \
freebsd/amd64 freebsd/386 freebsd/arm \
windows/amd64 windows/386

# (set an explicit GOARM of 5 for maximum compatibility)
ENV GOARM 5

# This has been commented out and kept as reference because we don't support compiling with older Go anymore.
# ENV GOFMT_VERSION 1.3.3
# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt

ENV GO_TOOLS_COMMIT 823804e1ae08dbb14eb807afc7db9993bc9e3cc3
# Grab Go's cover tool for dead-simple code coverage testing
# Grab Go's vet tool for examining go code to find suspicious constructs
# and help prevent errors that the compiler might not catch
RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
&& (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \
&& go install -v golang.org/x/tools/cmd/cover \
&& go install -v golang.org/x/tools/cmd/vet
# Grab Go's lint tool
ENV GO_LINT_COMMIT 32a87160691b3c96046c0c678fe57c5bef761456
RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \
&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
&& go install -v github.com/golang/lint/golint

# Install two versions of the registry. The first is an older version that
# only supports schema1 manifests. The second is a newer version that supports
# both. This allows integration-cli tests to cover push/pull with both schema1
Expand Down Expand Up @@ -189,20 +72,6 @@ RUN git clone https://github.com/docker/docker-py.git /docker-py \
&& git checkout -q $DOCKER_PY_COMMIT \
&& pip install -r test-requirements.txt

# Setup s3cmd config
RUN { \
echo '[default]'; \
echo 'access_key=$AWS_ACCESS_KEY'; \
echo 'secret_key=$AWS_SECRET_KEY'; \
} > ~/.s3cfg

# Set user.email so crosbymichael's in-container merge commits go smoothly
RUN git config --global user.email 'docker-dummy@example.com'

# Add an unprivileged user to be used for tests which need it
RUN groupadd -r docker
RUN useradd --create-home --gid docker unprivilegeduser

VOLUME /var/lib/docker
WORKDIR /go/src/github.com/docker/docker
ENV DOCKER_BUILDTAGS apparmor seccomp selinux
Expand All @@ -222,33 +91,6 @@ RUN ./contrib/download-frozen-image-v2.sh /docker-frozen-images \
hello-world:latest@sha256:8be990ef2aeb16dbcb9271ddfe2610fa6658d13f6dfb8bc72074cc1ca36966a7
# see also "hack/make/.ensure-frozen-images" (which needs to be updated any time this list is)

# Download man page generator
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
&& go get -v -d github.com/cpuguy83/go-md2man \
&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
&& rm -rf "$GOPATH"

# Download toml validator
ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \
&& (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \
&& go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \
&& rm -rf "$GOPATH"

# Build/install the tool for embedding resources in Windows binaries
ENV RSRC_COMMIT ba14da1f827188454a4591717fff29999010887f
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
&& (cd "$GOPATH/src/github.com/akavel/rsrc" && git checkout -q "$RSRC_COMMIT") \
&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
&& rm -rf "$GOPATH"

# Wrap all commands in the "docker-in-docker" script to allow nested containers
ENTRYPOINT ["hack/dind"]

Expand Down
180 changes: 180 additions & 0 deletions Dockerfile.base
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
# This file describes a base build image to build Docker, using docker.
#
# It contains the more stable parts of the build process, and allows us to
# reduce Docker build time while keeping the dependencies to external services
# at build time to a minimum.
#
# This image is not useful on its own: see `Dockerfile` and its architecture
# specific variants for a complete Docker build environment.
#

FROM debian:jessie

# add zfs ppa
RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61 \
|| apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys E871F18B51E0147C77796AC81196BA81F6B0FC61
RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /etc/apt/sources.list.d/zfs.list

# add llvm repo
RUN apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421 \
|| apt-key adv --keyserver hkp://pgp.mit.edu:80 --recv-keys 6084F3CF814B57C1CF12EFD515CF4D18AF4F7421
RUN echo deb http://llvm.org/apt/trusty/ llvm-toolchain-trusty main > /etc/apt/sources.list.d/llvm.list

# Packaged dependencies
RUN apt-get update && apt-get install -y \
apparmor \
aufs-tools \
automake \
bash-completion \
btrfs-tools \
build-essential \
clang-3.8 \
createrepo \
curl \
dpkg-sig \
gcc-mingw-w64 \
git \
iptables \
jq \
libapparmor-dev \
libcap-dev \
libltdl-dev \
libsqlite3-dev \
libsystemd-journal-dev \
libtool \
mercurial \
net-tools \
pkg-config \
python-dev \
python-mock \
python-pip \
python-websocket \
s3cmd=1.5.0* \
ubuntu-zfs \
xfsprogs \
libzfs-dev \
tar \
--no-install-recommends \
&& ln -snf /usr/bin/clang-3.8 /usr/local/bin/clang \
&& ln -snf /usr/bin/clang++-3.8 /usr/local/bin/clang++

# Get lvm2 source for compiling statically
ENV LVM2_VERSION 2.02.103
RUN mkdir -p /usr/local/lvm2 \
&& curl -fsSL "https://mirrors.kernel.org/sourceware/lvm2/LVM2.${LVM2_VERSION}.tgz" \
| tar -xzC /usr/local/lvm2 --strip-components=1
# see https://git.fedorahosted.org/cgit/lvm2.git/refs/tags for release tags

# Compile and install lvm2
RUN cd /usr/local/lvm2 \
&& ./configure \
--build="$(gcc -print-multiarch)" \
--enable-static_link \
&& make device-mapper \
&& make install_device-mapper
# see https://git.fedorahosted.org/cgit/lvm2.git/tree/INSTALL

# Configure the container for OSX cross compilation
ENV OSX_SDK MacOSX10.11.sdk
ENV OSX_CROSS_COMMIT 8aa9b71a394905e6c5f4b59e2b97b87a004658a4
RUN set -x \
&& export OSXCROSS_PATH="/osxcross" \
&& git clone https://github.com/tpoechtrager/osxcross.git $OSXCROSS_PATH \
&& ( cd $OSXCROSS_PATH && git checkout -q $OSX_CROSS_COMMIT) \
&& curl -sSL https://s3.dockerproject.org/darwin/${OSX_SDK}.tar.xz -o "${OSXCROSS_PATH}/tarballs/${OSX_SDK}.tar.xz" \
&& UNATTENDED=yes OSX_VERSION_MIN=10.6 ${OSXCROSS_PATH}/build.sh
ENV PATH /osxcross/target/bin:$PATH

# install seccomp: the version shipped in trusty is too old
ENV SECCOMP_VERSION 2.2.3
RUN set -x \
&& export SECCOMP_PATH="$(mktemp -d)" \
&& curl -fsSL "https://github.com/seccomp/libseccomp/releases/download/v${SECCOMP_VERSION}/libseccomp-${SECCOMP_VERSION}.tar.gz" \
| tar -xzC "$SECCOMP_PATH" --strip-components=1 \
&& ( \
cd "$SECCOMP_PATH" \
&& ./configure --prefix=/usr/local \
&& make \
&& make install \
&& ldconfig \
) \
&& rm -rf "$SECCOMP_PATH"

# Install Go
# IMPORTANT: If the version of Go is updated, the Windows to Linux CI machines
# will need updating, to avoid errors. Ping #docker-maintainers on IRC
# with a heads-up.
ENV GO_VERSION 1.6
RUN curl -fsSL "https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz" \
| tar -xzC /usr/local
ENV PATH /go/bin:/usr/local/go/bin:$PATH
ENV GOPATH /go:/go/src/github.com/docker/docker/vendor

# Compile Go for cross compilation
ENV DOCKER_CROSSPLATFORMS \
linux/386 linux/arm \
darwin/amd64 \
freebsd/amd64 freebsd/386 freebsd/arm \
windows/amd64 windows/386

# (set an explicit GOARM of 5 for maximum compatibility)
ENV GOARM 5

# This has been commented out and kept as reference because we don't support compiling with older Go anymore.
# ENV GOFMT_VERSION 1.3.3
# RUN curl -sSL https://storage.googleapis.com/golang/go${GOFMT_VERSION}.$(go env GOOS)-$(go env GOARCH).tar.gz | tar -C /go/bin -xz --strip-components=2 go/bin/gofmt

ENV GO_TOOLS_COMMIT 823804e1ae08dbb14eb807afc7db9993bc9e3cc3
# Grab Go's cover tool for dead-simple code coverage testing
# Grab Go's vet tool for examining go code to find suspicious constructs
# and help prevent errors that the compiler might not catch
RUN git clone https://github.com/golang/tools.git /go/src/golang.org/x/tools \
&& (cd /go/src/golang.org/x/tools && git checkout -q $GO_TOOLS_COMMIT) \
&& go install -v golang.org/x/tools/cmd/cover \
&& go install -v golang.org/x/tools/cmd/vet
# Grab Go's lint tool
ENV GO_LINT_COMMIT 32a87160691b3c96046c0c678fe57c5bef761456
RUN git clone https://github.com/golang/lint.git /go/src/github.com/golang/lint \
&& (cd /go/src/github.com/golang/lint && git checkout -q $GO_LINT_COMMIT) \
&& go install -v github.com/golang/lint/golint

# Setup s3cmd config
RUN { \
echo '[default]'; \
echo 'access_key=$AWS_ACCESS_KEY'; \
echo 'secret_key=$AWS_SECRET_KEY'; \
} > ~/.s3cfg

# Set user.email so crosbymichael's in-container merge commits go smoothly
RUN git config --global user.email 'docker-dummy@example.com'

# Add an unprivileged user to be used for tests which need it
RUN groupadd -r docker
RUN useradd --create-home --gid docker unprivilegeduser

# Download man page generator
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone --depth 1 -b v1.0.4 https://github.com/cpuguy83/go-md2man.git "$GOPATH/src/github.com/cpuguy83/go-md2man" \
&& git clone --depth 1 -b v1.4 https://github.com/russross/blackfriday.git "$GOPATH/src/github.com/russross/blackfriday" \
&& go get -v -d github.com/cpuguy83/go-md2man \
&& go build -v -o /usr/local/bin/go-md2man github.com/cpuguy83/go-md2man \
&& rm -rf "$GOPATH"

# Download toml validator
ENV TOMLV_COMMIT 9baf8a8a9f2ed20a8e54160840c492f937eeaf9a
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/BurntSushi/toml.git "$GOPATH/src/github.com/BurntSushi/toml" \
&& (cd "$GOPATH/src/github.com/BurntSushi/toml" && git checkout -q "$TOMLV_COMMIT") \
&& go build -v -o /usr/local/bin/tomlv github.com/BurntSushi/toml/cmd/tomlv \
&& rm -rf "$GOPATH"

# Build/install the tool for embedding resources in Windows binaries
ENV RSRC_COMMIT ba14da1f827188454a4591717fff29999010887f
RUN set -x \
&& export GOPATH="$(mktemp -d)" \
&& git clone https://github.com/akavel/rsrc.git "$GOPATH/src/github.com/akavel/rsrc" \
&& (cd "$GOPATH/src/github.com/akavel/rsrc" && git checkout -q "$RSRC_COMMIT") \
&& go build -v -o /usr/local/bin/rsrc github.com/akavel/rsrc \
&& rm -rf "$GOPATH"
2 changes: 2 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ DOCKER_MOUNT := $(if $(BIND_DIR),-v "$(CURDIR)/$(BIND_DIR):/go/src/github.com/do
GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null)
DOCKER_IMAGE := docker-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
DOCKER_DOCS_IMAGE := docker-docs$(if $(GIT_BRANCH),:$(GIT_BRANCH))
DOCKER_BUILD_BASE := dockercore/buildbase

DOCKER_FLAGS := docker run --rm -i --privileged $(DOCKER_ENVS) $(DOCKER_MOUNT)

Expand Down Expand Up @@ -90,6 +91,7 @@ ifeq ($(DOCKER_OSARCH), linux/arm)
depmod
modprobe dummy
endif
docker pull ${DOCKER_BUILD_BASE}
docker build ${DOCKER_BUILD_ARGS} -t "$(DOCKER_IMAGE)" -f "$(DOCKERFILE)" .

bundles:
Expand Down

0 comments on commit 8b9d27e

Please sign in to comment.