From 0ae1458b66b146103b1eeb50b4f5836a343f4ff1 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Wed, 15 Dec 2021 10:42:53 +0100 Subject: [PATCH] remove git-lfs=2.12.0 requirement use mamba instead of conda when possible remove libc symlink hack use apt instead of apt-get --- .github/workflows/acceptance_tests.yml | 63 ++++++++++++++++--- .../workflows/build_and_push_to_docker.yml | 32 ++++++---- docker/batch/Dockerfile | 5 +- docker/generic/Dockerfile | 4 +- docker/julia/Dockerfile | 20 +++--- docker/py/Dockerfile | 17 +++-- docker/r/Dockerfile | 1 - 7 files changed, 94 insertions(+), 48 deletions(-) diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml index d5767905..e3f2e40c 100644 --- a/.github/workflows/acceptance_tests.yml +++ b/.github/workflows/acceptance_tests.yml @@ -5,11 +5,21 @@ on: types: - synchronize +# NOTE: although docker platform specs are in the form of / +# since these variables are used for other purposes as well, we are +# using the _ form and will use a char-replace for the docker +# format env: - DOCKER_PLATFORMS: linux/amd64,linux/arm64 + DOCKER_PLATFORMS: linux_amd64,linux_arm64 + BUILDX_CACHE: /tmp/docker-cache jobs: build-base-python-image: + strategy: + matrix: + include: + - platform: linux_amd64 + - platform: linux_arm64 runs-on: ubuntu-latest steps: - name: Checkout @@ -21,13 +31,14 @@ jobs: uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build Image run: | - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/ - docker save py > /tmp/py.tar + PLATFORM=`echo ${{matrix.platform}} | tr "_" /` + docker buildx build --cache-to=type=local,dest=${{env.BUILDX_CACHE}}/$PLATFORM --platform $PLATFORM -t py docker/py/ + tar cf /tmp/py_${{matrix.platform}}.tar -C ${{env.BUILDX_CACHE}} $PLATFORM - name: Upload Image Artifact uses: actions/upload-artifact@v2 with: - name: py - path: /tmp/py.tar + name: py_${{matrix.platform}} + path: /tmp/py_${{matrix.platform}}.tar acceptance-tests: needs: build-base-python-image @@ -39,6 +50,7 @@ jobs: - image_name: cuda-tf build_arg: BASE_IMAGE=py test_user: jovyan + platform: linux/amd64 - image_name: generic build_arg: RENKU_BASE=py test_user: jovyan @@ -51,23 +63,54 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Download Base Image + - name: Download Base Image linux/amd64 + uses: actions/download-artifact@v2 + with: + name: py_linux_amd64 + path: /tmp + - name: Download Base Image linux/arm64 uses: actions/download-artifact@v2 with: - name: py + name: py_linux_arm64 path: /tmp - name: Set up QEMU # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 - name: Set up Docker Buildx uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - - name: Load Base Image in Docker + - name: Extract Docker caches run: | - docker load --input /tmp/py.tar + mkdir ${{env.BUILDX_CACHE}} + + PLATFORMS="${{env.DOCKER_PLATFORMS}}" + IFS=, + for p in $PLATFORMS; + do + tar xf /tmp/py_${p}.tar -C ${{env.BUILDX_CACHE}} + done - name: Build images if: ${{ matrix.image_name != 'py' }} run: | - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ + if [ "$DOCKER_PLATFORM" != "" ]; then + PLATFORMS=${DOCKER_PLATFORM} + else + PLATFORMS=`echo ${{env.DOCKER_PLATFORMS}} | tr "_" / | tr "," " "` + fi + + CACHE_FROM="" + for p in $PLATFORMS; + do + CACHE_FROM+="--cache-from=type=local,src=${{env.BUILDX_CACHE}}/${p} " + done + + docker buildx build \ + $CACHE_FROM \ + --platform ${PLATFORMS} \ + -t ${{ matrix.image_name }} \ + --build-arg ${{ matrix.build_arg }} \ + docker/${{ matrix.image_name }}/ + env: + DOCKER_PLATFORM: ${{ matrix.platform }} - name: Py Image Acceptance Tests uses: cypress-io/github-action@v2 env: diff --git a/.github/workflows/build_and_push_to_docker.yml b/.github/workflows/build_and_push_to_docker.yml index afe95d15..57ff370f 100644 --- a/.github/workflows/build_and_push_to_docker.yml +++ b/.github/workflows/build_and_push_to_docker.yml @@ -8,6 +8,11 @@ env: jobs: build-py: + strategy: + matrix: + include: + - platform: linux/amd64 + - platform: linux/arm64 runs-on: ubuntu-latest @@ -32,8 +37,7 @@ jobs: export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/py --tag $DOCKER_NAME-py:3.9-$LABEL - docker push $DOCKER_NAME-py:3.9-$LABEL + docker buildx build --platform ${{matrix.platform}} docker/py --tag $DOCKER_NAME-py:3.9-$LABEL --push # on master push latest image if [ "$REF" == "refs/heads/master" ] @@ -54,6 +58,8 @@ jobs: EXTENSIONS: - cuda-tf - vnc + PLATFORMS: + - linux/amd64 steps: - name: Docker Login @@ -76,10 +82,10 @@ jobs: export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/$EXTENSION \ + docker buildx build --platform ${{matrix.PLATFORMS}} docker/$EXTENSION \ --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \ - --tag $DOCKER_NAME-$EXTENSION:$LABEL - docker push $DOCKER_NAME-$EXTENSION:$LABEL + --tag $DOCKER_NAME-$EXTENSION:$LABEL \ + --push env: EXTENSION: ${{ matrix.EXTENSIONS }} DOCKER_NAME: "renku/renkulab" @@ -112,8 +118,8 @@ jobs: docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/batch \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE="python:3.9-slim-buster" \ - --tag $DOCKER_NAME-batch:$LABEL - docker push $DOCKER_NAME-batch:$LABEL + --tag $DOCKER_NAME-batch:$LABEL \ + --push env: DOCKER_NAME: "renku/renkulab" @@ -151,8 +157,8 @@ jobs: export DOCKER_TAG="$JULIAVERSION-$LABEL" docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/julia \ --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \ - --tag $DOCKER_NAME-julia:$DOCKER_TAG - docker push $DOCKER_NAME-julia:$DOCKER_TAG + --tag $DOCKER_NAME-julia:$DOCKER_TAG \ + --push env: DOCKER_NAME: "renku/renkulab" JULIAVERSION: ${{ matrix.JULIAVERSIONS }} @@ -195,8 +201,8 @@ jobs: docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE="rocker/verse:${RVERSION}" \ - --tag $DOCKER_NAME-r:$DOCKER_TAG - docker push $DOCKER_NAME-r:$DOCKER_TAG + --tag $DOCKER_NAME-r:$DOCKER_TAG \ + --push # on master push latest image if [ "$REF" == "refs/heads/master" ] && [ "$RVERSION" == "4.0.5" ] @@ -247,8 +253,8 @@ jobs: docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${RELEASE} \ - --tag $DOCKER_NAME-bioc:$DOCKER_TAG - docker push $DOCKER_NAME-bioc:$DOCKER_TAG + --tag $DOCKER_NAME-bioc:$DOCKER_TAG \ + --push # on master push latest image if [ "$REF" == "refs/heads/master" ] && [ "$RELEASE" == "devel" ] diff --git a/docker/batch/Dockerfile b/docker/batch/Dockerfile index e9ae80a1..2a47235f 100644 --- a/docker/batch/Dockerfile +++ b/docker/batch/Dockerfile @@ -11,10 +11,7 @@ LABEL maintainer="Swiss Data Science Center " # install dependencies RUN apt-get update -y && \ - apt-get install -y curl && \ - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ - apt-get update -y && \ - apt-get install -y gcc git git-lfs nodejs && \ + apt-get install -y curl gcc git-lfs nodejs && \ apt-get purge && \ apt-get clean && \ apt autoremove --yes && \ diff --git a/docker/generic/Dockerfile b/docker/generic/Dockerfile index 854d73b3..52620422 100644 --- a/docker/generic/Dockerfile +++ b/docker/generic/Dockerfile @@ -26,9 +26,7 @@ RUN apt-get update --fix-missing && \ bzip2 \ ca-certificates \ curl \ - gpg-agent && \ - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ - apt-get install -yq --no-install-recommends \ + gpg-agent \ git-lfs \ gnupg \ libglib2.0-0 \ diff --git a/docker/julia/Dockerfile b/docker/julia/Dockerfile index 4656870b..10a8a11d 100644 --- a/docker/julia/Dockerfile +++ b/docker/julia/Dockerfile @@ -12,13 +12,19 @@ ENV JULIA_PKGDIR=/opt/julia # If this changes, you also need to change the checksum below ENV JULIA_VERSION=1.6.1 -RUN mkdir /opt/julia-${JULIA_VERSION} && \ - cd /tmp && \ - wget -q https://julialang-s3.julialang.org/bin/linux/x64/`echo ${JULIA_VERSION} | cut -d. -f 1,2`/julia-${JULIA_VERSION}-linux-x86_64.tar.gz && \ - echo "7c888adec3ea42afbfed2ce756ce1164a570d50fa7506c3f2e1e2cbc49d52506 *julia-${JULIA_VERSION}-linux-x86_64.tar.gz" | sha256sum -c - && \ - tar xzf julia-${JULIA_VERSION}-linux-x86_64.tar.gz -C /opt/julia-${JULIA_VERSION} --strip-components=1 && \ - rm /tmp/julia-${JULIA_VERSION}-linux-x86_64.tar.gz -RUN ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia +RUN set -x && \ + julia_arch=$(uname -m) && \ + julia_short_arch="${julia_arch}" && \ + if [ "${julia_short_arch}" == "x86_64" ]; then \ + julia_short_arch="x64"; \ + fi; \ + julia_installer="julia-${JULIA_VERSION}-linux-${julia_arch}.tar.gz" && \ + julia_major_minor=$(echo "${JULIA_VERSION}" | cut -d. -f 1,2) && \ + mkdir "/opt/julia-${JULIA_VERSION}" && \ + wget -q "https://julialang-s3.julialang.org/bin/linux/${julia_short_arch}/${julia_major_minor}/${julia_installer}" && \ + tar xzf "${julia_installer}" -C "/opt/julia-${JULIA_VERSION}" --strip-components=1 && \ + rm "${julia_installer}" && \ + ln -fs /opt/julia-*/bin/julia /usr/local/bin/julia # Show Julia where conda libraries are \ RUN mkdir /etc/julia && \ diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 6876dff1..fbb95bb4 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -8,13 +8,11 @@ LABEL maintainer="Swiss Data Science Center " USER root # Install additional dependencies and nice-to-have packages -RUN apt-get update && apt-get install -yq --no-install-recommends \ +RUN apt update && apt install -yq --no-install-recommends \ build-essential \ - curl && \ - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ - apt-get update && apt-get install -yq --no-install-recommends \ + curl \ git \ - git-lfs=2.12.0 \ + git-lfs \ gnupg \ graphviz \ jq \ @@ -31,9 +29,8 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ python-dev \ unzip \ vim && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1 + apt clean && \ + rm -rf /var/lib/apt/lists/* # Add a new group with id 1000 and arbitrary name # Note: this is to standardize groups between the R and jupyter images @@ -52,10 +49,10 @@ RUN python3 -m pip install --no-cache -U pip && \ rm -rf /home/${NB_USER}/.cache # fix https://github.com/SwissDataScienceCenter/renku-jupyter/issues/14 -RUN conda install gxx_linux-64 && \ +RUN mamba install -y gxx_linux-64 && \ # jupyter sets channel priority to strict which often causes very long error messages conda config --system --set channel_priority flexible && \ - conda clean --all -f -y + mamba clean --all -f -y # install renku-python ENV RENKU_DISABLE_VERSION_CHECK 1 diff --git a/docker/r/Dockerfile b/docker/r/Dockerfile index 8a079d3a..a39d1d1f 100644 --- a/docker/r/Dockerfile +++ b/docker/r/Dockerfile @@ -47,7 +47,6 @@ RUN apt-get update --fix-missing && \ curl \ gpg-agent && \ curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ - curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ apt-get update && \ apt-get install -yq --no-install-recommends \ git-lfs \