forked from MaterializeInc/materialize
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Dockerfile
229 lines (184 loc) · 8.37 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
# Copyright Materialize, Inc. and contributors. All rights reserved.
#
# Use of this software is governed by the Business Source License
# included in the LICENSE file at the root of this repository.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0.
# Build a cross-compiling toolchain that targets the oldest version of Linux
# that we support.
FROM ubuntu:focal-20210921 AS crosstool
WORKDIR /scratch
# The environment variables are necessary to convince `tzdata`'s install scripts
# not to hang -- see https://serverfault.com/questions/949991/how-to-install-tzdata-on-a-ubuntu-docker-image
RUN apt-get update && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
autoconf \
automake \
bison \
ca-certificates \
curl \
file \
flex \
gawk \
g++ \
gcc \
help2man \
libc-dev \
libncurses-dev \
libtool-bin \
make \
patch \
texinfo \
unzip \
xz-utils
# TODO(benesch): switch back to an official crosstool-NG release when the
# official releases support GCC 9.0.
RUN mkdir crosstool \
&& curl -fsSL https://github.com/crosstool-ng/crosstool-ng/archive/5b0e0127e189c08e433ae25dbceead63da5ef2d7.tar.gz > crosstool.tar.gz \
&& tar -xf crosstool.tar.gz -C crosstool --strip-components=1 \
&& rm crosstool.tar.gz \
&& (cd crosstool && ./bootstrap && ./configure && make install) \
&& rm -rf crosstool
COPY crosstool.defconfig ./
# The sed command below works around
# https://github.com/crosstool-ng/crosstool-ng/issues/1609.
RUN DEFCONFIG=crosstool.defconfig ct-ng defconfig \
&& rm crosstool.defconfig \
&& sed -i 's,http://isl.gforge.inria.fr,https://mirror.sobukus.de/files/src/isl/,' .config \
&& ct-ng build
# Import the cross-compiling toolchain into a fresh image, omitting the
# dependencies that we needed to actually build the toolchain.
FROM ubuntu:focal-20210921
WORKDIR /workdir
# Import the cross toolchain.
COPY --from=crosstool /opt/x-tools /opt/x-tools
# Install dependencies needed by any CI job. Not all of these are available in
# the Ubuntu repositories.
RUN apt-get update && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
bsdmainutils \
ca-certificates \
cmake \
curl \
docker.io \
gcc \
g++ \
git \
gnupg2 \
jq \
make \
openssh-client \
pkg-config \
python3 \
python3-dev \
python3-pip \
python3-setuptools \
rsync \
ruby \
unzip \
xz-utils \
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.2.0/autouseradd-1.2.0-amd64.tar.gz \
| tar xz -C / --strip-components 1 \
&& curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.7.0/shellcheck-v0.7.0.linux.x86_64.tar.xz > shellcheck.tar.xz \
&& echo '39c501aaca6aae3f3c7fc125b3c3af779ddbe4e67e4ebdc44c2ae5cba76c847f shellcheck.tar.xz' | sha256sum --check \
&& tar -xJf shellcheck.tar.xz -C /usr/local/bin --strip-components 1 shellcheck-v0.7.0/shellcheck \
&& rm shellcheck.tar.xz \
&& curl -fsSL "https://github.com/docker/compose/releases/download/1.25.4/docker-compose-Linux-x86_64" > /usr/local/bin/docker-compose \
&& chmod +x /usr/local/bin/docker-compose
ENTRYPOINT ["autouseradd", "--user", "materialize"]
# Install Rust. This is done separately from above because we bump the Rust
# version frequently, and we want to reuse the cache up through aboe when
# possible.
COPY rust.asc .
RUN gpg --import rust.asc \
&& rm rust.asc \
&& echo "trusted-key 85AB96E6FA1BE5FE" >> ~/.gnupg/gpg.conf
ARG RUST_DATE
ARG RUST_VERSION
ARG RUST_COMPONENTS
RUN mkdir rust \
&& curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-x86_64-unknown-linux-gnu.tar.gz > rust.tar.gz \
&& curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-x86_64-unknown-linux-gnu.tar.gz.asc > rust.asc \
&& gpg --verify rust.asc rust.tar.gz \
&& tar -xzf rust.tar.gz -C rust --strip-components=1 \
&& rm -f rust.asc rust.tar.gz \
&& rust/install.sh --components=$RUST_COMPONENTS \
&& curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rustc-$RUST_VERSION-src.tar.gz > rust-src.tar.gz \
&& curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rustc-$RUST_VERSION-src.tar.gz.asc > rust-src.asc \
&& gpg --verify rust-src.asc rust-src.tar.gz \
&& mkdir -p /usr/local/lib/rustlib/src/rust \
&& tar -xzf rust-src.tar.gz -C /usr/local/lib/rustlib/src/rust --strip-components=1 \
&& rm -f rust-src.asc rust-src.tar.gz \
&& case $RUST_COMPONENTS in *miri*) \
PATH=$PATH:/root/.cargo/bin cargo miri setup \
&& mv /root/.cargo/bin/xargo /usr/local/bin/xargo \
&& mv /root/.cargo/bin/xargo-check /usr/local/bin/xargo-check \
;; \
esac \
&& rm -rf rust \
&& cargo install --root /usr/local --version "=1.28.0" cargo-deb \
&& cargo install --root /usr/local --version "=0.8.4" cargo-deny
# Link Rust's internal lld into the cross-compiling sysroot. This avoids
# bringing in an extra copy of LLVM via `apt install lld`.
RUN ln -s /usr/local/lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld /opt/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-ld.lld \
&& ln -s /usr/local/lib/rustlib/x86_64-unknown-linux-gnu/bin/rust-lld /opt/x-tools/x86_64-unknown-linux-gnu/bin/x86_64-unknown-linux-gnu-lld
# Install the Antithesis stub instrumentation library.
COPY libvoidstar.so /usr/local/lib/
# Set LC_CTYPE to allow UTF-8, for Python 3.6. This will become unnecessary when
# we upgrade to Python 3.7+. See: https://www.python.org/dev/peps/pep-0538/
ENV LC_CTYPE=C.UTF-8
# Install Python dependencies. These are so quick to install and change
# frequently enough that it makes sense to install them last.
COPY requirements.txt requirements-dev.txt /workdir/
RUN pip3 install \
-r /workdir/requirements.txt \
-r /workdir/requirements-dev.txt \
&& rm /workdir/requirements*.txt
# Install APT repo generator.
RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-0.11.3.gem > deb-s3.gem \
&& gem install ./deb-s3.gem \
&& rm deb-s3.gem
# Install the AWS CLI.
RUN curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" > awscli.zip \
&& unzip awscli.zip \
&& ./aws/install \
&& rm -rf aws
# Install docs site dependencies. These are towards the end for the same reason
# as the Python dependencies.
RUN curl -fsSL https://github.com/wjdp/htmltest/releases/download/v0.12.1/htmltest_0.12.1_linux_amd64.tar.gz > htmltest.tar.gz \
&& echo '04d4be5097b98cd28de469f8856b3fbe82669f57b482a4cf3092a55e9e8e9e0d htmltest.tar.gz' | sha256sum --check \
&& tar -xzf htmltest.tar.gz -C /usr/local/bin htmltest \
&& rm htmltest.tar.gz \
&& curl -fsSL https://github.com/gohugoio/hugo/releases/download/v0.68.3/hugo_extended_0.68.3_Linux-64bit.tar.gz > hugo.tar.gz \
&& echo 'd93d0deac782a4dd8afc2bbc5d96b30590ce47e8afb9810bbe7551eb3acf9189 hugo.tar.gz' | sha256sum --check \
&& tar -xzf hugo.tar.gz -C /usr/local/bin hugo \
&& rm hugo.tar.gz
# Hardcode some known SSH hosts, or else SSH will ask whether the host is
# trustworthy on the first connection.
COPY ssh_known_hosts /etc/ssh/ssh_known_hosts
# Set the necessary environment variables to point Cargo and C/C++ build systems
# at our cross-compiling toolchain.
ENV AR=x86_64-unknown-linux-gnu-ar
ENV LD=x86_64-unknown-linux-gnu-lld
ENV RANLIB=x86_64-unknown-linux-gnu-ranlib
ENV CPP=x86_64-unknown-linux-gnu-cpp
ENV CC=x86_64-unknown-linux-gnu-cc
ENV CXX=x86_64-unknown-linux-gnu-c++
ENV LDFLAGS=-fuse-ld=lld
ENV TARGET_CC=$CC
ENV TARGET_CXX=$CXX
ENV PATH=/opt/x-tools/x86_64-unknown-linux-gnu/bin:$PATH
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-cc
# https://github.com/rust-lang/cargo/issues/7529
ENV RUSTDOCFLAGS="-C linker=x86_64-unknown-linux-gnu-cc"
# Set a environment variable that tools can check to see if they're in the
# builder or not.
ENV MZ_DEV_CI_BUILDER=1
# Set up for a persistent volume to hold Cargo metadata, so that crate metadata
# does not need to be refetched on every compile.
ENV CARGO_HOME=/cargo
RUN mkdir /cargo && chmod 777 /cargo
VOLUME /cargo
# Make the image as small as possible.
RUN find /workdir /root -mindepth 1 -maxdepth 1 -exec rm -rf {} +