releng - improve docker build time via better layer cache utilization (…

…#7862)
cloud-custodian · Oct 16, 2022 · 24a9d73 · 24a9d73
1 parent 2f43ece
commit 24a9d73
Show file tree

Hide file tree

Showing 10 changed files with 473 additions and 135 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -30,3 +30,5 @@ results.xml
 pip-selfcheck.json
 .coverage
 *~
+**/tests/
+**/tests_azure/
diff --git a/docker/c7n b/docker/c7n
@@ -3,6 +3,7 @@
 FROM ubuntu:22.04 as build-env
 
 ARG POETRY_VERSION="1.2.1"
+SHELL ["/bin/bash", "-c"]
 
 # pre-requisite distro deps, and build env setup
 RUN adduser --disabled-login --gecos "" custodian
@@ -15,29 +16,64 @@ WORKDIR /src
 
 # Add core & aws packages
 ADD pyproject.toml poetry.lock README.md /src/
-ADD c7n /src/c7n/
 RUN . /usr/local/bin/activate && pip install -U pip
-RUN . /usr/local/bin/activate && poetry install --no-dev
+
+# Ignore root first pass so if source changes we don't have to invalidate
+# dependency install
+RUN . /usr/local/bin/activate && poetry install --without dev --no-root
 RUN . /usr/local/bin/activate && pip install -q wheel &&       pip install -U pip
 RUN . /usr/local/bin/activate && pip install -q aws-xray-sdk psutil jsonpatch
 
+ARG providers="gcp azure kube openstack tencentcloud"
 # Add provider packages
+# We include `pyproject.toml` and `poetry.lock` first to allow
+# cache of dependency installs.
+
+ADD tools/c7n_gcp/pyproject.toml tools/c7n_gcp/poetry.lock /src/tools/c7n_gcp/
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_azure/pyproject.toml tools/c7n_azure/poetry.lock /src/tools/c7n_azure/
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_kube/pyproject.toml tools/c7n_kube/poetry.lock /src/tools/c7n_kube/
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_openstack/pyproject.toml tools/c7n_openstack/poetry.lock /src/tools/c7n_openstack/
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_tencentcloud/pyproject.toml tools/c7n_tencentcloud/poetry.lock /src/tools/c7n_tencentcloud/
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --without dev --no-root; fi
+
+
+# Now install the root package
+ADD c7n /src/c7n/
+RUN . /usr/local/bin/activate && poetry install --only-root
+
+# Now install the root of each provider
+
 ADD tools/c7n_gcp /src/tools/c7n_gcp
-RUN rm -R tools/c7n_gcp/tests
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_azure /src/tools/c7n_azure
-RUN rm -R tools/c7n_azure/tests_azure
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_kube /src/tools/c7n_kube
-RUN rm -R tools/c7n_kube/tests
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_openstack /src/tools/c7n_openstack
-RUN rm -R tools/c7n_openstack/tests
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_tencentcloud /src/tools/c7n_tencentcloud
-RUN rm -R tools/c7n_tencentcloud/tests
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --only-root; fi
 
-# Install requested providers
-ARG providers="gcp kube openstack tencentcloud azure"
-RUN . /usr/local/bin/activate && \
-    for pkg in $providers; do cd tools/c7n_$pkg && \
-    poetry install && cd ../../; done
 
 RUN mkdir /output
 
@@ -46,10 +82,6 @@ FROM ubuntu:22.04
 LABEL name="cli" \
       repository="http://github.com/cloud-custodian/cloud-custodian"
 
-COPY --from=build-env /src /src
-COPY --from=build-env /usr/local /usr/local
-COPY --from=build-env /output /output
-
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get --yes update \
@@ -58,6 +90,13 @@ RUN apt-get --yes update \
         && rm -Rf /var/lib/apt/lists/* \
         && rm -Rf /var/log/*
 
+# These should remain below any other commands because they will invalidate
+# the layer cache
+COPY --from=build-env /src /src
+COPY --from=build-env /usr/local /usr/local
+COPY --from=build-env /output /output
+
+
 RUN adduser --disabled-login --gecos "" custodian
 USER custodian
 WORKDIR /home/custodian

diff --git a/docker/c7n-distroless b/docker/c7n-distroless
@@ -3,6 +3,7 @@
 FROM debian:10-slim as build-env
 
 ARG POETRY_VERSION="1.2.1"
+SHELL ["/bin/bash", "-c"]
 
 # pre-requisite distro deps, and build env setup
 RUN adduser --disabled-login --gecos "" custodian
@@ -15,29 +16,64 @@ WORKDIR /src
 
 # Add core & aws packages
 ADD pyproject.toml poetry.lock README.md /src/
-ADD c7n /src/c7n/
 RUN . /usr/local/bin/activate && pip install -U pip
-RUN . /usr/local/bin/activate && poetry install --no-dev
+
+# Ignore root first pass so if source changes we don't have to invalidate
+# dependency install
+RUN . /usr/local/bin/activate && poetry install --without dev --no-root
 RUN . /usr/local/bin/activate && pip install -q wheel &&       pip install -U pip
 RUN . /usr/local/bin/activate && pip install -q aws-xray-sdk psutil jsonpatch
 
+ARG providers="gcp azure kube openstack tencentcloud"
 # Add provider packages
+# We include `pyproject.toml` and `poetry.lock` first to allow
+# cache of dependency installs.
+
+ADD tools/c7n_gcp/pyproject.toml tools/c7n_gcp/poetry.lock /src/tools/c7n_gcp/
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_azure/pyproject.toml tools/c7n_azure/poetry.lock /src/tools/c7n_azure/
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_kube/pyproject.toml tools/c7n_kube/poetry.lock /src/tools/c7n_kube/
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_openstack/pyproject.toml tools/c7n_openstack/poetry.lock /src/tools/c7n_openstack/
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_tencentcloud/pyproject.toml tools/c7n_tencentcloud/poetry.lock /src/tools/c7n_tencentcloud/
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --without dev --no-root; fi
+
+
+# Now install the root package
+ADD c7n /src/c7n/
+RUN . /usr/local/bin/activate && poetry install --only-root
+
+# Now install the root of each provider
+
 ADD tools/c7n_gcp /src/tools/c7n_gcp
-RUN rm -R tools/c7n_gcp/tests
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_azure /src/tools/c7n_azure
-RUN rm -R tools/c7n_azure/tests_azure
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_kube /src/tools/c7n_kube
-RUN rm -R tools/c7n_kube/tests
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_openstack /src/tools/c7n_openstack
-RUN rm -R tools/c7n_openstack/tests
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_tencentcloud /src/tools/c7n_tencentcloud
-RUN rm -R tools/c7n_tencentcloud/tests
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --only-root; fi
 
-# Install requested providers
-ARG providers="gcp kube openstack tencentcloud azure"
-RUN . /usr/local/bin/activate && \
-    for pkg in $providers; do cd tools/c7n_$pkg && \
-    poetry install && cd ../../; done
 
 RUN mkdir /output
 

diff --git a/docker/c7n-org b/docker/c7n-org
@@ -3,6 +3,7 @@
 FROM ubuntu:22.04 as build-env
 
 ARG POETRY_VERSION="1.2.1"
+SHELL ["/bin/bash", "-c"]
 
 # pre-requisite distro deps, and build env setup
 RUN adduser --disabled-login --gecos "" custodian
@@ -15,29 +16,64 @@ WORKDIR /src
 
 # Add core & aws packages
 ADD pyproject.toml poetry.lock README.md /src/
-ADD c7n /src/c7n/
 RUN . /usr/local/bin/activate && pip install -U pip
-RUN . /usr/local/bin/activate && poetry install --no-dev
+
+# Ignore root first pass so if source changes we don't have to invalidate
+# dependency install
+RUN . /usr/local/bin/activate && poetry install --without dev --no-root
 RUN . /usr/local/bin/activate && pip install -q wheel &&       pip install -U pip
 RUN . /usr/local/bin/activate && pip install -q aws-xray-sdk psutil jsonpatch
 
+ARG providers="gcp azure kube openstack tencentcloud"
 # Add provider packages
+# We include `pyproject.toml` and `poetry.lock` first to allow
+# cache of dependency installs.
+
+ADD tools/c7n_gcp/pyproject.toml tools/c7n_gcp/poetry.lock /src/tools/c7n_gcp/
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_azure/pyproject.toml tools/c7n_azure/poetry.lock /src/tools/c7n_azure/
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_kube/pyproject.toml tools/c7n_kube/poetry.lock /src/tools/c7n_kube/
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_openstack/pyproject.toml tools/c7n_openstack/poetry.lock /src/tools/c7n_openstack/
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_tencentcloud/pyproject.toml tools/c7n_tencentcloud/poetry.lock /src/tools/c7n_tencentcloud/
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --without dev --no-root; fi
+
+
+# Now install the root package
+ADD c7n /src/c7n/
+RUN . /usr/local/bin/activate && poetry install --only-root
+
+# Now install the root of each provider
+
 ADD tools/c7n_gcp /src/tools/c7n_gcp
-RUN rm -R tools/c7n_gcp/tests
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_azure /src/tools/c7n_azure
-RUN rm -R tools/c7n_azure/tests_azure
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_kube /src/tools/c7n_kube
-RUN rm -R tools/c7n_kube/tests
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_openstack /src/tools/c7n_openstack
-RUN rm -R tools/c7n_openstack/tests
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_tencentcloud /src/tools/c7n_tencentcloud
-RUN rm -R tools/c7n_tencentcloud/tests
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --only-root; fi
 
-# Install requested providers
-ARG providers="gcp kube openstack tencentcloud azure"
-RUN . /usr/local/bin/activate && \
-    for pkg in $providers; do cd tools/c7n_$pkg && \
-    poetry install && cd ../../; done
 
 RUN mkdir /output
 
@@ -50,10 +86,6 @@ FROM ubuntu:22.04
 LABEL name="org" \
       repository="http://github.com/cloud-custodian/cloud-custodian"
 
-COPY --from=build-env /src /src
-COPY --from=build-env /usr/local /usr/local
-COPY --from=build-env /output /output
-
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get --yes update \
@@ -62,6 +94,13 @@ RUN apt-get --yes update \
         && rm -Rf /var/lib/apt/lists/* \
         && rm -Rf /var/log/*
 
+# These should remain below any other commands because they will invalidate
+# the layer cache
+COPY --from=build-env /src /src
+COPY --from=build-env /usr/local /usr/local
+COPY --from=build-env /output /output
+
+
 RUN adduser --disabled-login --gecos "" custodian
 USER custodian
 WORKDIR /home/custodian

diff --git a/docker/c7n-org-distroless b/docker/c7n-org-distroless
@@ -3,6 +3,7 @@
 FROM debian:10-slim as build-env
 
 ARG POETRY_VERSION="1.2.1"
+SHELL ["/bin/bash", "-c"]
 
 # pre-requisite distro deps, and build env setup
 RUN adduser --disabled-login --gecos "" custodian
@@ -15,29 +16,64 @@ WORKDIR /src
 
 # Add core & aws packages
 ADD pyproject.toml poetry.lock README.md /src/
-ADD c7n /src/c7n/
 RUN . /usr/local/bin/activate && pip install -U pip
-RUN . /usr/local/bin/activate && poetry install --no-dev
+
+# Ignore root first pass so if source changes we don't have to invalidate
+# dependency install
+RUN . /usr/local/bin/activate && poetry install --without dev --no-root
 RUN . /usr/local/bin/activate && pip install -q wheel &&       pip install -U pip
 RUN . /usr/local/bin/activate && pip install -q aws-xray-sdk psutil jsonpatch
 
+ARG providers="gcp azure kube openstack tencentcloud"
 # Add provider packages
+# We include `pyproject.toml` and `poetry.lock` first to allow
+# cache of dependency installs.
+
+ADD tools/c7n_gcp/pyproject.toml tools/c7n_gcp/poetry.lock /src/tools/c7n_gcp/
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_azure/pyproject.toml tools/c7n_azure/poetry.lock /src/tools/c7n_azure/
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_kube/pyproject.toml tools/c7n_kube/poetry.lock /src/tools/c7n_kube/
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_openstack/pyproject.toml tools/c7n_openstack/poetry.lock /src/tools/c7n_openstack/
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --without dev --no-root; fi
+
+
+ADD tools/c7n_tencentcloud/pyproject.toml tools/c7n_tencentcloud/poetry.lock /src/tools/c7n_tencentcloud/
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --without dev --no-root; fi
+
+
+# Now install the root package
+ADD c7n /src/c7n/
+RUN . /usr/local/bin/activate && poetry install --only-root
+
+# Now install the root of each provider
+
 ADD tools/c7n_gcp /src/tools/c7n_gcp
-RUN rm -R tools/c7n_gcp/tests
+RUN if [[ " ${providers[*]} " =~ "gcp" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_gcp &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_azure /src/tools/c7n_azure
-RUN rm -R tools/c7n_azure/tests_azure
+RUN if [[ " ${providers[*]} " =~ "azure" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_azure &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_kube /src/tools/c7n_kube
-RUN rm -R tools/c7n_kube/tests
+RUN if [[ " ${providers[*]} " =~ "kube" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_kube &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_openstack /src/tools/c7n_openstack
-RUN rm -R tools/c7n_openstack/tests
+RUN if [[ " ${providers[*]} " =~ "openstack" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_openstack &&     poetry install --only-root; fi
+
+
 ADD tools/c7n_tencentcloud /src/tools/c7n_tencentcloud
-RUN rm -R tools/c7n_tencentcloud/tests
+RUN if [[ " ${providers[*]} " =~ "tencentcloud" ]]; then     . /usr/local/bin/activate &&     cd tools/c7n_tencentcloud &&     poetry install --only-root; fi
 
-# Install requested providers
-ARG providers="gcp kube openstack tencentcloud azure"
-RUN . /usr/local/bin/activate && \
-    for pkg in $providers; do cd tools/c7n_$pkg && \
-    poetry install && cd ../../; done
 
 RUN mkdir /output