build: Simpler Docker build

.dockerignore

@@ -1,10 +1,21 @@
 # Prevent vendor directory from being copied to ensure we are not not pulling unexpected cruft from
 # a user's workspace, and are only building off of what is locked by dep.
+*.iml
+*.md
+*.yaml
+.github
 .idea
+.run
 assets
+community
 coverage.out
 dist
+docs
+examples
+manifests
 sdks
-vendor
+test/e2e
 ui/dist
-ui/node_modules
+ui/node_modules
+v3
+vendor

.github/workflows/ci-build.yaml

@@ -86,6 +86,10 @@ jobs:
           echo "- name: fake_token_user" >> ~/.kube/config
           echo "  user:" >> ~/.kube/config
           echo "    token: xxxxxx" >> ~/.kube/config
+      - name: Start logging
+        run: |
+          mkdir -p /tmp/log/argo-e2e
+          make logs > /tmp/log/argo-e2e/pods.log &
       - name: Start Argo
         env:
           GOPATH: /home/runner/go
@@ -94,7 +98,6 @@ jobs:
           echo '127.0.0.1 minio'    | sudo tee -a /etc/hosts
           echo '127.0.0.1 postgres' | sudo tee -a /etc/hosts
           echo '127.0.0.1 mysql'    | sudo tee -a /etc/hosts
-          mkdir -p /tmp/log/argo-e2e
           git fetch --tags
           KUBECONFIG=~/.kube/config make install PROFILE=mysql E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=true DEV_IMAGE=true STATIC_FILES=false
           KUBECONFIG=~/.kube/config make start   PROFILE=mysql E2E_EXECUTOR=${{matrix.containerRuntimeExecutor}} ALWAYS_OFFLOAD_NODE_STATUS=true DEV_IMAGE=true STATIC_FILES=false 2>&1 > /tmp/log/argo-e2e/argo.log &
@@ -112,6 +115,12 @@ jobs:
         with:
           name: ${{ matrix.test }}-${{matrix.containerRuntimeExecutor}}-${{ github.run_id }}-argo.log
           path: /tmp/log/argo-e2e/argo.log
+      - name: Upload pod logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ matrix.test }}-${{matrix.containerRuntimeExecutor}}-${{ github.run_id }}-pod.log
+          path: /tmp/log/argo-e2e/pod.log
 
   codegen:
     name: Codegen

.github/workflows/docker-image.yml

@@ -6,6 +6,7 @@ on:
       - v*
     branches:
       - master
+      - buildx
 
 defaults:
   run:

Dockerfile

@@ -1,11 +1,11 @@
-####################################################################################################
-# Builder image
-# Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image
-# Also used as the image in CI jobs so needs all dependencies
-####################################################################################################
-FROM golang:1.15.7 as builder
 
-ARG IMAGE_OS=linux
+ARG DOCKER_CHANNEL=stable
+ARG DOCKER_VERSION=18.09.1
+# NOTE: kubectl version should be one minor version less than https://storage.googleapis.com/kubernetes-release/release/stable.txt
+ARG KUBECTL_VERSION=1.19.6
+ARG JQ_VERSION=1.6
+
+FROM golang:1.15.7 as builder
 
 RUN apt-get update && apt-get --no-install-recommends install -y \
     git \
@@ -27,32 +27,24 @@ RUN apt-get update && apt-get --no-install-recommends install -y \
 
 WORKDIR /tmp
 
-# Install docker
-ENV DOCKER_CHANNEL stable
-ENV DOCKER_VERSION 18.09.1
-
-RUN if [ "${IMAGE_OS}" = "linux" ]; then \
-        export IMAGE_ARCH=`uname -m`; \
-        if [ "${IMAGE_ARCH}" = "ppc64le" ] ||[ "${IMAGE_ARCH}" = "s390x" ]; then \
-	        wget -O docker.tgz https://download.docker.com/${IMAGE_OS}/static/${DOCKER_CHANNEL}/${IMAGE_ARCH}/docker-18.06.3-ce.tgz; \
-        else \
-            wget -O docker.tgz https://download.docker.com/${IMAGE_OS}/static/${DOCKER_CHANNEL}/${IMAGE_ARCH}/docker-${DOCKER_VERSION}.tgz; \
-        fi \
-    fi && \
-    tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
-    rm docker.tgz
+# https://blog.container-solutions.com/faster-builds-in-docker-with-go-1-11
+WORKDIR /go/src/github.com/argoproj/argo
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+WORKDIR /go/src/github.com/argoproj/argo
+COPY . .
 
 ####################################################################################################
-# argoexec-base
-# Used as the base for both the release and development version of argoexec
-####################################################################################################
+
 FROM debian:10.7-slim as argoexec-base
 
-ARG IMAGE_OS=linux
+ARG DOCKER_CHANNEL
+ARG DOCKER_VERSION
+ARG KUBECTL_VERSION
+ARG JQ_VERSION
 
-# NOTE: kubectl version should be one minor version less than https://storage.googleapis.com/kubernetes-release/release/stable.txt
-ENV KUBECTL_VERSION=1.19.6
-ENV JQ_VERSION=1.6
 RUN apt-get update && \
     apt-get --no-install-recommends install -y curl procps git apt-utils apt-transport-https ca-certificates tar mime-support && \
     apt-get clean \
@@ -64,85 +56,89 @@ RUN apt-get update && \
         /usr/share/doc \
         /usr/share/doc-base
 
-ADD hack/recurl.sh .
-ADD hack/image_arch.sh .
-RUN . ./image_arch.sh && ./recurl.sh /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/${IMAGE_OS}/${IMAGE_ARCH}/kubectl
+COPY hack/recurl.sh .
+RUN if [ $(uname -m) = ppc64le ] || [ $(uname -m)  = s390x ]; then \
+        ./recurl.sh docker.tgz https://download.docker.com/$(uname -s|tr '[:upper:]' '[:lower:]')/static/${DOCKER_CHANNEL}/$(uname -m) docker-18.06.3-ce.tgz; \
+    else \
+        ./recurl.sh docker.tgz https://download.docker.com/$(uname -s|tr '[:upper:]' '[:lower:]')/static/${DOCKER_CHANNEL}/$(uname -m)/docker-${DOCKER_VERSION}.tgz; \
+    fi && \
+    tar --extract --file docker.tgz --strip-components 1 --directory /usr/local/bin/ && \
+    rm docker.tgz
+
+RUN ./recurl.sh /usr/local/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/$(uname -s|tr '[:upper:]' '[:lower:]')/$(uname -m)/kubectl
 RUN ./recurl.sh /usr/local/bin/jq https://github.com/stedolan/jq/releases/download/jq-${JQ_VERSION}/jq-linux64
 RUN rm recurl.sh
 
-COPY hack/ssh_known_hosts /etc/ssh/ssh_known_hosts
-COPY hack/nsswitch.conf /etc/nsswitch.conf
-COPY --from=builder /usr/local/bin/docker /usr/local/bin/
+COPY hack/ssh_known_hosts /etc/ssh/
+COPY hack/nsswitch.conf /etc/
+
 
 ####################################################################################################
 
 FROM node:14.0.0 as argo-ui
 
-ADD ["ui", "ui"]
-ADD ["api", "api"]
+COPY ui/package.json ui/yarn.lock ui/
 
 RUN JOBS=max yarn --cwd ui install --network-timeout 1000000
+
+COPY ui ui
+COPY api api
+
 RUN JOBS=max yarn --cwd ui build
 
 ####################################################################################################
-# Argo Build stage which performs the actual build of Argo binaries
+
+FROM builder as argoexec-build
+
+RUN --mount=type=cache,target=/root/.cache/go-build make dist/argoexec
+
 ####################################################################################################
-FROM builder as argo-build
 
-ARG IMAGE_OS=linux
+FROM builder as workflow-controller-build
 
-# Perform the build
-WORKDIR /go/src/github.com/argoproj/argo
-COPY . .
-# check we can use Git
-RUN git rev-parse HEAD
+RUN --mount=type=cache,target=/root/.cache/go-build make dist/workflow-controller
 
-# controller image
-RUN . hack/image_arch.sh && make dist/workflow-controller-${IMAGE_OS}-${IMAGE_ARCH}
-RUN . hack/image_arch.sh && ./dist/workflow-controller-${IMAGE_OS}-${IMAGE_ARCH} version | grep clean
+####################################################################################################
 
-# executor image
-RUN . hack/image_arch.sh && make dist/argoexec-${IMAGE_OS}-${IMAGE_ARCH}
-RUN . hack/image_arch.sh && ./dist/argoexec-${IMAGE_OS}-${IMAGE_ARCH} version | grep clean
+FROM builder as argocli-build
 
-# cli image
 RUN mkdir -p ui/dist
 COPY --from=argo-ui ui/dist/app ui/dist/app
 # stop make from trying to re-build this without yarn installed
 RUN touch ui/dist/node_modules.marker
 RUN touch ui/dist/app/index.html
-RUN . hack/image_arch.sh && make argo-server.crt argo-server.key dist/argo-${IMAGE_OS}-${IMAGE_ARCH}
-RUN . hack/image_arch.sh && ./dist/argo-${IMAGE_OS}-${IMAGE_ARCH} version 2>&1 | grep clean
+RUN --mount=type=cache,target=/root/.cache/go-build make dist/argo
 
 ####################################################################################################
-# argoexec
-####################################################################################################
+
 FROM argoexec-base as argoexec
-ARG IMAGE_OS=linux
-COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/argoexec-${IMAGE_OS}-* /usr/local/bin/argoexec
+
+COPY --from=argoexec-build /go/src/github.com/argoproj/argo/dist/argoexec /usr/local/bin/
+
 ENTRYPOINT [ "argoexec" ]
 
 ####################################################################################################
-# workflow-controller
-####################################################################################################
+
 FROM scratch as workflow-controller
+
 USER 8737
-ARG IMAGE_OS=linux
-# Add timezone data
-COPY --from=argo-build /usr/share/zoneinfo /usr/share/zoneinfo
-COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/workflow-controller-${IMAGE_OS}-* /bin/workflow-controller
+
+COPY --from=workflow-controller-build /usr/share/zoneinfo /usr/share/
+COPY --chown=8737 --from=workflow-controller-build /go/src/github.com/argoproj/argo/dist/workflow-controller /bin/
+
 ENTRYPOINT [ "workflow-controller" ]
 
 ####################################################################################################
-# argocli
-####################################################################################################
+
 FROM scratch as argocli
+
 USER 8737
-ARG IMAGE_OS=linux
-COPY --from=argoexec-base /etc/ssh/ssh_known_hosts /etc/ssh/ssh_known_hosts
-COPY --from=argoexec-base /etc/nsswitch.conf /etc/nsswitch.conf
-COPY --from=argoexec-base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=argo-build --chown=8737 /go/src/github.com/argoproj/argo/argo-server.crt argo-server.crt
-COPY --from=argo-build --chown=8737 /go/src/github.com/argoproj/argo/argo-server.key argo-server.key
-COPY --from=argo-build /go/src/github.com/argoproj/argo/dist/argo-${IMAGE_OS}-* /bin/argo
+
+COPY hack/ssh_known_hosts /etc/ssh/
+COPY hack/nsswitch.conf /etc/
+COPY --from=argocli-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=argocli-build --chown=8737 /go/src/github.com/argoproj/argo/argo-server.crt /
+COPY --from=argocli-build --chown=8737 /go/src/github.com/argoproj/argo/argo-server.key /
+COPY --from=argocli-build /go/src/github.com/argoproj/argo/dist/argo /bin/
+
 ENTRYPOINT [ "argo" ]