devxp: Base the application container images off new Ruby base image. (…

…#19633) This "rebases" our application images off of the new Ruby base image added in #19632, and fixes numerous problems and quirks with how the images were built along the way. Notably: - Issues where layers attempted to delete files in prior layers have been resolved (this caused build failures on some Docker filesystem drivers, notably overlay2). - Bundler is no longer allowed to deviate from or modify the lockfile (`BUNDLE_FROZEN` is now `true`). - `git(1)` is no longer required to live inside the container and `.git` is no longer required to be copied into the Docker build context, as these were only used to calculate `FOREM_BUILD_SHA`, which is now passed in as a Build Argument to the container build context. - The entire source tree is no longer `chmod` in one giant swing, which ran so long on my system (as just one example) that I gave up after 15-20 minutes and issued it a `SIGTERM`. Instead, `COPY --chown` is used more heavily and ensures the `APP_USER` will have access to the requisite files. This new container image appears to build successfully for `linux/arm64`, which refs (but does not complete) #19626. Currently, such builds aren't automated , and must be built on a developer workstation. For example: ```sh docker buildx build --platform linux/amd64,linux/arm64 -f Containerfile . -t ghcr.io/forem/forem:klardotsh-test --push --build-arg VCS_REF=$(git rev-parse --short HEAD) ``` In the meantime, the existing `linux/amd64`-only BuildKite scripts have been updated to allow this PR to merge as a separate unit, and CI refactors to enable the multiarch builds of `linux/arm64,linux/amd64` can come later when more time is available. This is one of several blockers on the path to getting #19603 merged. The next step in that chronology will be rebasing that work on top of this work, which *should* be, on the containerization side, as straightforward as bumping `Containerfile.base` to reference the new upstream image, rebuilding the base container, and then bumping the reference in `Containerfile`.
forem · Jun 29, 2023 · e4d33b3 · e4d33b3
1 parent e52d53b
commit e4d33b3
Show file tree

Hide file tree

Showing 3 changed files with 187 additions and 91 deletions.
diff --git a/Containerfile b/Containerfile
@@ -1,76 +1,121 @@
-FROM quay.io/forem/ruby:3.0.2 as base
+FROM ghcr.io/forem/ruby:3.0.2@sha256:04c945460e5999c0483b119074597dba62b863f5f5ea3c98bff4169e0d2f990b as base
 
 FROM base as builder
 
+# This is provided by BuildKit
+ARG TARGETARCH
+
 USER root
 
-RUN curl -sL https://dl.yarnpkg.com/rpm/yarn.repo -o /etc/yum.repos.d/yarn.repo && \
-    dnf install --setopt install_weak_deps=false -y \
-    ImageMagick iproute jemalloc less libcurl libcurl-devel \
-    libffi-devel libxml2-devel libxslt-devel nodejs pcre-devel \
-    postgresql postgresql-devel tzdata yarn && \
-    dnf -y clean all && \
-    rm -rf /var/cache/yum
+# pkg-config,
+# libpixman-1-dev,
+# libcairo2-dev,
+# libpango1.0-dev
+#
+# are needed only on arm64: some nodejs dependency doesn't provide
+# pre-built binaries for that arch, and so falls back to building
+# from source, which then requires a few extra packages installed.
+#
+# Since we wipe out node_modules as part of this image after calling
+# the bundler, we don't need these headers (or their sofile counterparts)
+# in any of the other build stages.
+RUN apt update && \
+    apt install -y \
+        build-essential \
+        libcurl4-openssl-dev \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev \
+        libpcre3-dev \
+        libpq-dev \
+        pkg-config \
+        libpixman-1-dev \
+        libcairo2-dev \
+        libpango1.0-dev \
+        && \
+    apt clean
+
+ENV BUNDLER_VERSION=2.2.22 \
+    BUNDLE_SILENCE_ROOT_WARNING=true \
+    BUNDLE_SILENCE_DEPRECATIONS=true
 
-ENV BUNDLER_VERSION=2.2.22 BUNDLE_SILENCE_ROOT_WARNING=true BUNDLE_SILENCE_DEPRECATIONS=true
 RUN gem install -N bundler:"${BUNDLER_VERSION}"
 
 ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \
-    LD_PRELOAD=/usr/lib64/libjemalloc.so.2
+    LD_PRELOAD=libjemalloc.so.2
 RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \
     groupadd -g "${APP_GID}" "${APP_USER}" && \
-    adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}"
+    adduser --uid "${APP_UID}" --gid "${APP_GID}" --home "${APP_HOME}" "${APP_USER}"
 
-ENV DOCKERIZE_VERSION=v0.6.1
-RUN wget https://github.com/jwilder/dockerize/releases/download/"${DOCKERIZE_VERSION}"/dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
-    && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
-    && rm dockerize-linux-amd64-"${DOCKERIZE_VERSION}".tar.gz \
+ENV DOCKERIZE_VERSION=v0.7.0
+RUN curl -fsSLO https://github.com/jwilder/dockerize/releases/download/"${DOCKERIZE_VERSION}"/dockerize-linux-${TARGETARCH}-"${DOCKERIZE_VERSION}".tar.gz \
+    && tar -C /usr/local/bin -xzvf dockerize-linux-${TARGETARCH}-"${DOCKERIZE_VERSION}".tar.gz \
+    && rm dockerize-linux-${TARGETARCH}-"${DOCKERIZE_VERSION}".tar.gz \
     && chown root:root /usr/local/bin/dockerize
 
+USER "${APP_USER}"
 WORKDIR "${APP_HOME}"
 
-COPY ./.ruby-version "${APP_HOME}"/
-COPY ./Gemfile ./Gemfile.lock "${APP_HOME}"/
-COPY ./vendor/cache "${APP_HOME}"/vendor/cache
-
-RUN bundle config --local build.sassc --disable-march-tune-native && \
-    BUNDLE_WITHOUT="development:test" bundle install --deployment --jobs 4 --retry 5 && \
+COPY --chown=${APP_UID}:${APP_GID} ./.ruby-version "${APP_HOME}"/
+COPY --chown=${APP_UID}:${APP_GID} ./Gemfile ./Gemfile.lock "${APP_HOME}"/
+COPY --chown=${APP_UID}:${APP_GID} ./vendor/cache "${APP_HOME}"/vendor/cache
+
+# Have to reset APP_CONFIG, which appears to be set by upstream images, to
+# avoid permission errors in the development/test images (which run bundle
+# as a user and require write access to the config file for setting things
+# like BUNDLE_WITHOUT (a value that is cached by root here in this builder
+# layer, see https://michaelheap.com/bundler-ignoring-bundle-without/))
+ENV BUNDLE_APP_CONFIG="${APP_HOME}/.bundle"
+RUN mkdir -p "${BUNDLE_APP_CONFIG}" && \
+    touch "${BUNDLE_APP_CONFIG}/config" && \
+    chown -R "${APP_UID}:${APP_GID}" "${BUNDLE_APP_CONFIG}" && \
+    bundle config --local build.sassc --disable-march-tune-native && \
+    bundle config --local without development:test && \
+    BUNDLE_FROZEN=true bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 
-COPY . "${APP_HOME}"
+COPY --chown=${APP_UID}:${APP_GID} . "${APP_HOME}"
 
 RUN mkdir -p "${APP_HOME}"/public/{assets,images,packs,podcasts,uploads}
 
-RUN NODE_ENV=production yarn install
-
-RUN RAILS_ENV=production NODE_ENV=production bundle exec rake assets:precompile
+# While it's relatively rare for bare metal builds to hit the default
+# timeout, QEMU-based ones (as is the case with Docker BuildX for
+# cross-compiling) quite often can. This increased timeout should help
+# reduce false-negatives when building multiarch images.
+RUN echo 'network-timeout 300000' >> ~/.yarnrc
+
+# This is one giant step now because previously, removing node_modules to save
+# layer space was done in a later step, which is invalid in at least some
+# Docker storage drivers (resulting in Directory Not Empty errors).
+RUN NODE_ENV=production yarn install && \
+    RAILS_ENV=production NODE_ENV=production bundle exec rake assets:precompile && \
+    rm -rf node_modules
+
+# This used to be calculated within the container build, but we then tried
+# to rm -rf the .git that was copied in, which isn't valid (removing
+# directories created in lower layers of an image isn't a thing (at least
+# with the overlayfs drivers). Instead, we'll pass this in over CLI when
+# building images (eg. in CI), but leave a default value for callers who don't
+# override (perhaps docker-compose). This isn't perfect, but it'll do for now.
+ARG VCS_REF=unspecified
 
 RUN echo $(date -u +'%Y-%m-%dT%H:%M:%SZ') >> "${APP_HOME}"/FOREM_BUILD_DATE && \
-    echo $(git rev-parse --short HEAD) >> "${APP_HOME}"/FOREM_BUILD_SHA && \
-    rm -rf "${APP_HOME}"/.git/
-
-RUN rm -rf node_modules vendor/assets spec
+    echo "${VCS_REF}" >> "${APP_HOME}"/FOREM_BUILD_SHA
 
 ## Production
 FROM base as production
 
 USER root
 
-RUN dnf install --setopt install_weak_deps=false -y bash curl ImageMagick \
-                iproute jemalloc less libcurl \
-                postgresql tzdata nodejs libpq \
-                && dnf -y clean all \
-                && rm -rf /var/cache/yum
-
 ENV BUNDLER_VERSION=2.2.22 BUNDLE_SILENCE_ROOT_WARNING=1
 RUN gem install -N bundler:"${BUNDLER_VERSION}"
 
 ENV APP_USER=forem APP_UID=1000 APP_GID=1000 APP_HOME=/opt/apps/forem \
-    LD_PRELOAD=/usr/lib64/libjemalloc.so.2
+    LD_PRELOAD=libjemalloc.so.2
 RUN mkdir -p ${APP_HOME} && chown "${APP_UID}":"${APP_GID}" "${APP_HOME}" && \
     groupadd -g "${APP_GID}" "${APP_USER}" && \
-    adduser -u "${APP_UID}" -g "${APP_GID}" -d "${APP_HOME}" "${APP_USER}"
+    adduser --uid "${APP_UID}" --gid "${APP_GID}" --home "${APP_HOME}" "${APP_USER}"
 
 COPY --from=builder --chown="${APP_USER}":"${APP_USER}" ${APP_HOME} ${APP_HOME}
 
@@ -86,23 +131,14 @@ CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
 ## Testing
 FROM builder AS testing
 
-USER root
-
-RUN dnf install --setopt install_weak_deps=false -y \
-    chromium-headless chromedriver && \
-    yum clean all && \
-    rm -rf /var/cache/yum
+USER "${APP_USER}"
 
 COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
 COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
 
-RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
-
-USER "${APP_USER}"
-
 RUN bundle config --local build.sassc --disable-march-tune-native && \
     bundle config --delete without && \
-    bundle install --deployment --jobs 4 --retry 5 && \
+    BUNDLE_FROZEN=true bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 
@@ -113,16 +149,14 @@ CMD ["bundle", "exec", "rails", "server", "-b", "0.0.0.0", "-p", "3000"]
 ## Development
 FROM builder AS development
 
+USER "${APP_USER}"
+
 COPY --chown="${APP_USER}":"${APP_USER}" ./spec "${APP_HOME}"/spec
 COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/dockerize
 
-RUN chown "${APP_USER}":"${APP_USER}" -R "${APP_HOME}"
-
-USER "${APP_USER}"
-
 RUN bundle config --local build.sassc --disable-march-tune-native && \
     bundle config --delete without && \
-    bundle install --deployment --jobs 4 --retry 5 && \
+    BUNDLE_FROZEN=true bundle install --deployment --jobs 4 --retry 5 && \
     find "${APP_HOME}"/vendor/bundle -name "*.c" -delete && \
     find "${APP_HOME}"/vendor/bundle -name "*.o" -delete
 

diff --git a/scripts/build_containers.sh b/scripts/build_containers.sh
@@ -5,6 +5,8 @@ set -euo pipefail
 : ${CONTAINER_REPO:="quay.io/forem"}
 : ${CONTAINER_APP:=forem}
 
+export DOCKER_BUILDKIT=1
+
 function create_pr_containers {
 
   PULL_REQUEST=$1
@@ -22,6 +24,7 @@ function create_pr_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
                --label quay.expires-after=8w \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" .
 
   # Build the pull request image
@@ -30,6 +33,7 @@ function create_pr_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder-"${PULL_REQUEST}" \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \
                --label quay.expires-after=8w \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" .
 
   # Build the testing image
@@ -39,6 +43,7 @@ function create_pr_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":pr-"${PULL_REQUEST}" \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" \
                --label quay.expires-after=8w \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-"${PULL_REQUEST}" .
 
   # Push images to Quay
@@ -60,12 +65,14 @@ function create_production_containers {
   # Build the builder image
   docker build --target builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder .
 
   # Build the production image
   docker build --target production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":$(date +%Y%m%d) \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":latest .
@@ -74,13 +81,15 @@ function create_production_containers {
                --label quay.expires-after=8w \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7} .
 
   # Build the testing image
   docker build --target testing \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing .
 
   # Build the development image
@@ -89,6 +98,7 @@ function create_production_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":development \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":development .
 
   # Push images to Quay
@@ -114,12 +124,14 @@ function create_release_containers {
   # Build the builder image
   docker build --target builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":builder .
 
   # Build the production image
   docker build --target production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_COMMIT:0:7} \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BRANCH} .
 
@@ -128,13 +140,15 @@ function create_release_containers {
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":testing-${BRANCH} .
 
   # Build the development image
   docker build --target development \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
                --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":testing \
+               --build-arg "VCS_REF=${BUILDKITE_COMMIT}" \
                --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":development-${BRANCH} .
 
   # If the env var for the git tag doesn't exist or is an empty string, then we
@@ -144,6 +158,7 @@ function create_release_containers {
     docker build --target production \
                  --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":builder \
                  --cache-from="${CONTAINER_REPO}"/"${CONTAINER_APP}":production \
+                 --build-arg "VCS_REF=${BUILDKITE_TAG}" \
                  --tag "${CONTAINER_REPO}"/"${CONTAINER_APP}":${BUILDKITE_TAG} .
   fi