Merge pull request #4706 from planetscale/dk-docker-file-refactor

Docker File Refactor

Makefile

@@ -160,7 +160,7 @@ $(PROTO_GO_OUTS): install_protoc-gen-go proto/*.proto
 # Please read docker/README.md to understand the different available images.
 
 # This rule builds the bootstrap images for all flavors.
-DOCKER_IMAGES_FOR_TEST = mariadb mysql56 mysql57 percona percona57
+DOCKER_IMAGES_FOR_TEST = mariadb mariadb103 mysql56 mysql57 mysql80 percona percona57 percona80
 DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST)
 docker_bootstrap:
 	for i in $(DOCKER_IMAGES); do echo "building bootstrap image: $$i"; docker/bootstrap/build.sh $$i || exit 1; done
@@ -184,10 +184,18 @@ docker_base_mysql56:
 	chmod -R o=g *
 	docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql56 .
 
+docker_base_mysql80:
+	chmod -R o=g *
+	docker build -f docker/base/Dockerfile.mysql56 -t vitess/base:mysql80 .
+
 docker_base_mariadb:
 	chmod -R o=g *
 	docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb .
 
+docker_base_mariadb103:
+	chmod -R o=g *
+	docker build -f docker/base/Dockerfile.mariadb -t vitess/base:mariadb103 .
+
 docker_base_percona:
 	chmod -R o=g *
 	docker build -f docker/base/Dockerfile.percona -t vitess/base:percona .
@@ -196,6 +204,10 @@ docker_base_percona57:
 	chmod -R o=g *
 	docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona57 .
 
+docker_base_percona80:
+	chmod -R o=g *
+	docker build -f docker/base/Dockerfile.percona57 -t vitess/base:percona80 .
+
 # Run "make docker_lite PROMPT_NOTICE=false" to avoid that the script
 # prompts you to press ENTER and confirm that the vitess/base image is not
 # rebuild by this target as well.
@@ -205,15 +217,27 @@ docker_lite:
 docker_lite_mysql56:
 	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql56
 
+docker_lite_mysql57:
+	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql57
+
+docker_lite_mysql80:
+	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mysql80
+
 docker_lite_mariadb:
 	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mariadb
 
+docker_lite_mariadb103:
+	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) mariadb103
+
 docker_lite_percona:
 	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona
 
 docker_lite_percona57:
 	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona57
 
+docker_lite_percona80:
+	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) percona80
+
 docker_lite_alpine:
 	cd docker/lite && ./build.sh --prompt=$(PROMPT_NOTICE) alpine
 

docker/base/Dockerfile.mariadb103

@@ -0,0 +1,13 @@
+FROM vitess/bootstrap:mariadb103
+
+# Re-copy sources from working tree
+USER root
+COPY . /vt/src/vitess.io/vitess
+
+# Build Vitess
+RUN make build
+
+# Fix permissions
+RUN chown -R vitess:vitess /vt
+USER vitess
+

docker/base/Dockerfile.mysql80

@@ -0,0 +1,13 @@
+FROM vitess/bootstrap:mysql80
+
+# Re-copy sources from working tree
+USER root
+COPY . /vt/src/vitess.io/vitess
+
+# Build Vitess
+RUN make build
+
+# Fix permissions
+RUN chown -R vitess:vitess /vt
+USER vitess
+

docker/base/Dockerfile.percona80

@@ -0,0 +1,12 @@
+FROM vitess/bootstrap:percona80
+
+# Re-copy sources from working tree
+USER root
+COPY . /vt/src/vitess.io/vitess
+
+# Fix permissions
+RUN chown -R vitess:vitess /vt
+USER vitess
+
+# Build Vitess
+RUN make build

docker/bootstrap/Dockerfile.mariadb

@@ -1,10 +1,12 @@
 FROM vitess/bootstrap:common
 
-# Install MariaDB 10.
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-    mariadb-server \
-    libmariadbclient-dev \
-    && rm -rf /var/lib/apt/lists/*
+# Install MariaDB 10
+RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      bzip2 \
+      mariadb-server \
+      libmariadbclient-dev \
+ && rm -rf /var/lib/apt/lists/*
 
 # Bootstrap Vitess
 WORKDIR /vt/src/vitess.io/vitess

docker/bootstrap/Dockerfile.mariadb103

@@ -0,0 +1,16 @@
+FROM vitess/bootstrap:common
+
+# Install MariaDB 10.
+RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 &&\
+    add-apt-repository 'deb [arch=amd64] http://ftp.osuosl.org/pub/mariadb/repo/10.3/debian stretch main'
+    apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+    mariadb-server \
+    libmariadbclient-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Bootstrap Vitess
+WORKDIR /vt/src/vitess.io/vitess
+
+ENV MYSQL_FLAVOR MariaDB103
+USER vitess
+RUN ./bootstrap.sh

docker/bootstrap/Dockerfile.mysql80

@@ -0,0 +1,16 @@
+FROM vitess/bootstrap:common
+
+# Install MySQL 5.7
+RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.pool.sks-keyservers.net 8C718D3B5072E1F5 && break; done && \
+    add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-8.0' && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server libmysqlclient-dev && \
+    rm -rf /var/lib/apt/lists/*
+
+# Bootstrap Vitess
+WORKDIR /vt/src/vitess.io/vitess
+
+
+ENV MYSQL_FLAVOR MySQL56
+USER vitess
+RUN ./bootstrap.sh

docker/bootstrap/Dockerfile.percona

@@ -1,7 +1,7 @@
 FROM vitess/bootstrap:common
 
 # Install Percona 5.6
-RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 8507EFA5 && break; done && \
+RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
     add-apt-repository 'deb http://repo.percona.com/apt stretch main' && \
     { \
         echo debconf debconf/frontend select Noninteractive; \

docker/bootstrap/Dockerfile.percona57

@@ -1,7 +1,7 @@
 FROM vitess/bootstrap:common
 
 # Install Percona 5.7
-RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 8507EFA5 && break; done && \
+RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done && \
     add-apt-repository 'deb http://repo.percona.com/apt stretch main' && \
     { \
         echo debconf debconf/frontend select Noninteractive; \

docker/bootstrap/Dockerfile.percona80

@@ -0,0 +1,24 @@
+FROM vitess/bootstrap:common
+
+# Install Percona 8.0
+RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --recv-keys 9334A25F8507EFA5 && break; done &&
+    && echo 'deb http://repo.percona.com/ps-80/apt stretch main' > /etc/apt/sources.list.d/percona.list && \
+    { \
+        echo debconf debconf/frontend select Noninteractive; \
+        echo percona-server-server-8.0 percona-server-server/root_password password 'unused'; \
+        echo percona-server-server-8.0 percona-server-server/root_password_again password 'unused'; \
+    } | debconf-set-selections \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        percona-server-server \
+	libperconaserverclient21 \
+	percona-server-tokudb \
+	percona-server-rocksdb \
+	bzip2
+
+# Bootstrap Vitess
+WORKDIR /vt/src/vitess.io/vitess
+
+ENV MYSQL_FLAVOR MySQL56
+USER vitess
+RUN ./bootstrap.sh

docker/lite/Dockerfile.mariadb

@@ -1,14 +1,35 @@
-# This image is only meant to be built from within the build.sh script.
-FROM debian:jessie
+FROM vitess/base AS builder
+FROM debian:stretch-slim AS staging
+
+RUN mkdir -p /vt/vtdataroot/ \
+    && mkdir -p /vt/bin \
+    && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \
+    && groupadd -r vitess && useradd -r -g vitess vitess
+
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app
+COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config
+COPY --from=builder /vt/bin/mysqlctld /vt/bin/
+COPY --from=builder /vt/bin/vtctld /vt/bin/
+COPY --from=builder /vt/bin/vtctlclient /vt/bin/
+COPY --from=builder /vt/bin/vtgate /vt/bin/
+COPY --from=builder /vt/bin/vttablet /vt/bin/
+COPY --from=builder /vt/bin/vtworker /vt/bin/
+
+RUN chown -R vitess:vitess /vt
+
+FROM debian:stretch-slim
 
 # Install dependencies
-RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db \
- && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.0/debian jessie main' > /etc/apt/sources.list.d/mariadb.list \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \
+ && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \
+ && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.2/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
       bzip2 \
       mariadb-server \
- && rm -rf /var/lib/apt/lists/*
+ && rm -rf /var/lib/apt/lists/* \
+ && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
 ENV VTTOP /vt/src/vitess.io/vitess
@@ -20,14 +41,11 @@ ENV GOPATH $VTROOT
 ENV PATH $VTROOT/bin:$PATH
 ENV VT_MYSQL_ROOT /usr
 ENV PKG_CONFIG_PATH $VTROOT/lib
+ENV MYSQL_FLAVOR MariaDB
 
 # Copy binaries (placed by build.sh)
-COPY lite/vt /vt
-
-# Create vitess user
-RUN groupadd -r vitess && useradd -r -g vitess vitess && \
-    mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt
+COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot
-USER vitess
+USER vitess

docker/lite/Dockerfile.mariadb103

@@ -0,0 +1,50 @@
+FROM vitess/base AS builder
+FROM debian:stretch-slim AS staging
+
+RUN mkdir -p /vt/vtdataroot/ \
+    && mkdir -p /vt/bin \
+    && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \
+    && groupadd -r vitess && useradd -r -g vitess vitess
+
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app
+COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config
+COPY --from=builder /vt/bin/mysqlctld /vt/bin/
+COPY --from=builder /vt/bin/vtctld /vt/bin/
+COPY --from=builder /vt/bin/vtctlclient /vt/bin/
+COPY --from=builder /vt/bin/vtgate /vt/bin/
+COPY --from=builder /vt/bin/vttablet /vt/bin/
+COPY --from=builder /vt/bin/vtworker /vt/bin/
+RUN chown -R vitess:vitess /vt
+
+FROM debian:stretch-slim
+
+# Install dependencies
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates \
+ && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 \
+ && echo 'deb http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/debian stretch main' > /etc/apt/sources.list.d/mariadb.list \
+ && apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      bzip2 \
+      mariadb-server \
+ && rm -rf /var/lib/apt/lists/* \
+ && groupadd -r vitess && useradd -r -g vitess vitess
+
+# Set up Vitess environment (just enough to run pre-built Go binaries)
+ENV VTTOP /vt/src/vitess.io/vitess
+ENV VTROOT /vt
+ENV GOTOP $VTTOP/go
+ENV VTDATAROOT $VTROOT/vtdataroot
+ENV GOBIN $VTROOT/bin
+ENV GOPATH $VTROOT
+ENV PATH $VTROOT/bin:$PATH
+ENV VT_MYSQL_ROOT /usr
+ENV PKG_CONFIG_PATH $VTROOT/lib
+ENV MYSQL_FLAVOR MariaDB103
+
+# Copy binaries (placed by build.sh)
+COPY --from=staging /vt/ /vt/
+
+# Create mount point for actual data (e.g. MySQL data dir)
+VOLUME /vt/vtdataroot
+USER vitess

docker/lite/Dockerfile.mysql56

@@ -1,17 +1,37 @@
-# This image is only meant to be built from within the build.sh script.
-FROM debian:jessie
+FROM vitess/base AS builder
+FROM debian:stretch-slim AS staging
+
+RUN mkdir -p /vt/vtdataroot/ \
+    && mkdir -p /vt/bin \
+    && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \
+    && groupadd -r vitess && useradd -r -g vitess vitess
+
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld
+COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app
+COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config
+COPY --from=builder /vt/bin/mysqlctld /vt/bin/
+COPY --from=builder /vt/bin/vtctld /vt/bin/
+COPY --from=builder /vt/bin/vtctlclient /vt/bin/
+COPY --from=builder /vt/bin/vtgate /vt/bin/
+COPY --from=builder /vt/bin/vttablet /vt/bin/
+COPY --from=builder /vt/bin/vtworker /vt/bin/
+
+RUN chown -R vitess:vitess /vt
+
+FROM debian:stretch-slim
 
 # Install dependencies
-RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 5072E1F5 \
- && echo 'deb http://repo.mysql.com/apt/debian/ jessie mysql-5.6' > /etc/apt/sources.list.d/mysql.list \
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 \
+ && echo 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' > /etc/apt/sources.list.d/mysql.list \
  && apt-get update \
  && DEBIAN_FRONTEND=noninteractive \
     apt-get install -y --no-install-recommends \
       bzip2 \
       libmysqlclient18 \
       mysql-client \
       mysql-server \
- && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  && groupadd -r vitess && useradd -r -g vitess vitess
 
 # Set up Vitess environment (just enough to run pre-built Go binaries)
 ENV VTTOP /vt/src/vitess.io/vitess
@@ -23,13 +43,10 @@ ENV GOPATH $VTROOT
 ENV PATH $VTROOT/bin:$PATH
 ENV VT_MYSQL_ROOT /usr
 ENV PKG_CONFIG_PATH $VTROOT/lib
+ENV MYSQL_FLAVOR MySQL56
 
 # Copy binaries (placed by build.sh)
-COPY lite/vt /vt
-
-# Create vitess user
-RUN groupadd -r vitess && useradd -r -g vitess vitess && \
-    mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt
+COPY --from=staging /vt/ /vt/
 
 # Create mount point for actual data (e.g. MySQL data dir)
 VOLUME /vt/vtdataroot