From ab0bd6354a20b7d80d1e5977548a95e365ce4589 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Wed, 15 Dec 2021 09:13:02 +0100 Subject: [PATCH 01/26] support multi-platform build using buildx --- .github/workflows/acceptance_tests.yml | 78 +++++++++++++++++++ .../workflows/build-and-push-to-docker.yml | 25 ++++++ 2 files changed, 103 insertions(+) create mode 100644 .github/workflows/acceptance_tests.yml diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml new file mode 100644 index 00000000..d5767905 --- /dev/null +++ b/.github/workflows/acceptance_tests.yml @@ -0,0 +1,78 @@ +name: Acceptance tests + +on: + pull_request: + types: + - synchronize + +env: + DOCKER_PLATFORMS: linux/amd64,linux/arm64 + +jobs: + build-base-python-image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + - name: Build Image + run: | + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/ + docker save py > /tmp/py.tar + - name: Upload Image Artifact + uses: actions/upload-artifact@v2 + with: + name: py + path: /tmp/py.tar + + acceptance-tests: + needs: build-base-python-image + strategy: + matrix: + include: + - image_name: py + test_user: jovyan + - image_name: cuda-tf + build_arg: BASE_IMAGE=py + test_user: jovyan + - image_name: generic + build_arg: RENKU_BASE=py + test_user: jovyan + - image_name: julia + build_arg: BASE_IMAGE=py + test_user: jovyan + - image_name: r + build_arg: RENKU_BASE=py + test_user: rstudio + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Download Base Image + uses: actions/download-artifact@v2 + with: + name: py + path: /tmp + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + - name: Load Base Image in Docker + run: | + docker load --input /tmp/py.tar + - name: Build images + if: ${{ matrix.image_name != 'py' }} + run: | + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ + - name: Py Image Acceptance Tests + uses: cypress-io/github-action@v2 + env: + TEST_IMAGE_NAME: ${{ matrix.image_name }} + TEST_USER_NAME: ${{ matrix.test_user }} + with: + working-directory: tests + command: npx mocha test.js diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 0f534b10..1e516b1b 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -72,6 +72,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project python-based docker images id: build-image run: | @@ -219,6 +224,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project python-based docker image extensions env: EXTENSION: ${{ matrix.EXTENSIONS }} @@ -289,6 +299,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project julia docker image extensions id: build-image run: | @@ -345,6 +360,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project rocker docker images id: build-image run: | @@ -404,6 +424,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then From 8c5a31048a6a2ceaf6c75fd45344963e51f6b588 Mon Sep 17 00:00:00 2001 From: Viktor Gal Date: Wed, 15 Dec 2021 10:42:53 +0100 Subject: [PATCH 02/26] remove git-lfs=2.12.0 requirement use mamba instead of conda when possible remove libc symlink hack use apt instead of apt-get --- .github/workflows/acceptance_tests.yml | 20 +++----------------- docker/py/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml index d5767905..01b4ab75 100644 --- a/.github/workflows/acceptance_tests.yml +++ b/.github/workflows/acceptance_tests.yml @@ -5,23 +5,14 @@ on: types: - synchronize -env: - DOCKER_PLATFORMS: linux/amd64,linux/arm64 - jobs: build-base-python-image: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + - uses: actions/checkout@v2 - name: Build Image run: | - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/ + docker build -t py docker/py/ docker save py > /tmp/py.tar - name: Upload Image Artifact uses: actions/upload-artifact@v2 @@ -56,18 +47,13 @@ jobs: with: name: py path: /tmp - - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Load Base Image in Docker run: | docker load --input /tmp/py.tar - name: Build images if: ${{ matrix.image_name != 'py' }} run: | - docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ + docker build -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ - name: Py Image Acceptance Tests uses: cypress-io/github-action@v2 env: diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 8754a49a..6cb06058 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -7,7 +7,7 @@ USER root SHELL [ "/bin/bash", "-c", "-o", "pipefail" ] # Install additional dependencies and nice-to-have packages -RUN apt-get update && apt-get install -yq --no-install-recommends \ +RUN apt update && apt install -yq --no-install-recommends \ build-essential \ curl \ git \ From 6e192217909e03b795b891e5b58dc2546f86e269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Wed, 15 Jun 2022 00:29:47 +0200 Subject: [PATCH 03/26] chore: update action version --- .../workflows/build-and-push-to-docker.yml | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 1e516b1b..9936e58f 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -73,10 +73,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + uses: docker/setup-buildx-action@v3.0.0 - name: Build renku project python-based docker images id: build-image run: | @@ -225,10 +224,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + uses: docker/setup-buildx-action@v3.0.0 - name: Build renku project python-based docker image extensions env: EXTENSION: ${{ matrix.EXTENSIONS }} @@ -300,10 +298,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + uses: docker/setup-buildx-action@v3.0.0 - name: Build renku project julia docker image extensions id: build-image run: | @@ -361,10 +358,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + uses: docker/setup-buildx-action@v3.0.0 - name: Build renku project rocker docker images id: build-image run: | @@ -425,10 +421,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 - uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + uses: docker/setup-buildx-action@v3.0.0 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then From 7b36d9f34b694a57d0fc4900ca1bac0046e248bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Wed, 15 Jun 2022 00:34:39 +0200 Subject: [PATCH 04/26] chore: use v2 --- .../workflows/build-and-push-to-docker.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 9936e58f..9bcdebf8 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -73,9 +73,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project python-based docker images id: build-image run: | @@ -224,9 +224,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project python-based docker image extensions env: EXTENSION: ${{ matrix.EXTENSIONS }} @@ -298,9 +298,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project julia docker image extensions id: build-image run: | @@ -358,9 +358,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project rocker docker images id: build-image run: | @@ -421,9 +421,9 @@ jobs: - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v3.0.0 + uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.0.0 + uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then From 84de5014bffe2e090f44751f3b0a27ef080b8964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Fri, 22 Jul 2022 12:08:06 +0200 Subject: [PATCH 05/26] chore: update build steps for buildx --- .github/workflows/build-and-push-to-docker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 9bcdebf8..c875207e 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -112,7 +112,12 @@ jobs: - name: Push to docker run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + docker buildx build \ + --platform ${{env.DOCKER_PLATFORMS}} \ + --tag ${{ steps.build-image.outputs.IMAGE_NAME }} \ + --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}" \ + --push \ + docker/py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then From e165499f494c619a573d6b26f53b939468b3dc10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Fri, 17 Mar 2023 19:54:12 +0100 Subject: [PATCH 06/26] feat: build and push multiarch py image --- .../workflows/build-and-push-to-docker.yml | 13 ++++---- Makefile | 31 +++++++++++++------ docker/py/Dockerfile | 2 +- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index c875207e..2051450b 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -77,6 +77,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project python-based docker images + env: + # build multi-platform + MULTIARCH: true id: build-image run: | if [[ ${{ github.ref }} == refs/tags* ]]; then @@ -99,6 +102,8 @@ jobs: export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT + # load the linux image + BUILDX_EXTRA_FLAGS=--load PLATFORM=linux/amd64 make py - name: Image Acceptance Tests uses: cypress-io/github-action@v4 @@ -112,13 +117,7 @@ jobs: - name: Push to docker run: | - docker buildx build \ - --platform ${{env.DOCKER_PLATFORMS}} \ - --tag ${{ steps.build-image.outputs.IMAGE_NAME }} \ - --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}" \ - --push \ - docker/py - + BUILDX_EXTRA_FLAGS=--push make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest diff --git a/Makefile b/Makefile index 62861ba5..5d3642c5 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,16 @@ extensions = \ batch \ bioc -PLATFORM?=linux/amd64 +# set the build command to buildx if multi-arch +ifdef MULTIARCH + BUILDX=buildx + PLATFORM?=linux/amd64,linux/arm64 +else + PLATFORM?=linux/amd64 +endif + +# use BUILDX_EXTRA_FLAGS to set either --push or --load +BUILDX_EXTRA_FLAGS?= DOCKER_PREFIX?=renku/renkulab GIT_COMMIT_SHA?=$(shell git rev-parse --short=7 --verify HEAD) @@ -96,10 +105,11 @@ pull: # BASE_IMAGE_TAG: used to identify the jupyter base notebook to build from # RENKU_PYTHON_BASE_TAG: used to tag the resulting image py: - docker build docker/py \ + docker $(BUILDX) build docker/py \ --build-arg BASE_IMAGE=jupyter/base-notebook:$(BASE_IMAGE_TAG) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) r: py docker build docker/r \ @@ -123,17 +133,19 @@ cuda: py # this image is tagged with the julia version and the commit hash julia: py - docker build docker/julia \ + docker $(BUILDX) build docker/julia \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is just tagged with the commit hash vnc: py - docker build docker/vnc \ + docker $(BUILDX) build docker/vnc \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is built on the vnc image and tagged as matlab with the commit hash vnc-%: vnc @@ -143,10 +155,11 @@ vnc-%: vnc -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) batch: py - docker build docker/batch \ + docker $(BUILDX) build docker/batch \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="python:3.10-slim-buster" \ - -t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) bioc: py docker build docker/r \ diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 6cb06058..8754a49a 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -7,7 +7,7 @@ USER root SHELL [ "/bin/bash", "-c", "-o", "pipefail" ] # Install additional dependencies and nice-to-have packages -RUN apt update && apt install -yq --no-install-recommends \ +RUN apt-get update && apt-get install -yq --no-install-recommends \ build-essential \ curl \ git \ From 2d4bec6860185d7ab378c33b152e90700e34a7fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Fri, 17 Mar 2023 20:05:02 +0100 Subject: [PATCH 07/26] chore: remove unused acceptance tests --- .github/workflows/acceptance_tests.yml | 64 -------------------------- 1 file changed, 64 deletions(-) delete mode 100644 .github/workflows/acceptance_tests.yml diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml deleted file mode 100644 index 01b4ab75..00000000 --- a/.github/workflows/acceptance_tests.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Acceptance tests - -on: - pull_request: - types: - - synchronize - -jobs: - build-base-python-image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Build Image - run: | - docker build -t py docker/py/ - docker save py > /tmp/py.tar - - name: Upload Image Artifact - uses: actions/upload-artifact@v2 - with: - name: py - path: /tmp/py.tar - - acceptance-tests: - needs: build-base-python-image - strategy: - matrix: - include: - - image_name: py - test_user: jovyan - - image_name: cuda-tf - build_arg: BASE_IMAGE=py - test_user: jovyan - - image_name: generic - build_arg: RENKU_BASE=py - test_user: jovyan - - image_name: julia - build_arg: BASE_IMAGE=py - test_user: jovyan - - image_name: r - build_arg: RENKU_BASE=py - test_user: rstudio - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Download Base Image - uses: actions/download-artifact@v2 - with: - name: py - path: /tmp - - name: Load Base Image in Docker - run: | - docker load --input /tmp/py.tar - - name: Build images - if: ${{ matrix.image_name != 'py' }} - run: | - docker build -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ - - name: Py Image Acceptance Tests - uses: cypress-io/github-action@v2 - env: - TEST_IMAGE_NAME: ${{ matrix.image_name }} - TEST_USER_NAME: ${{ matrix.test_user }} - with: - working-directory: tests - command: npx mocha test.js From c8098948bc8c2024cca61fac836c30d1407c47da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Fri, 17 Mar 2023 20:11:02 +0100 Subject: [PATCH 08/26] chore: add multiarch push --- .github/workflows/build-and-push-to-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 2051450b..1fc4becb 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -117,7 +117,7 @@ jobs: - name: Push to docker run: | - BUILDX_EXTRA_FLAGS=--push make py + BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest From dc2a93fe4cb50ac6a639eb097142924df290ce5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Fri, 17 Mar 2023 20:34:41 +0100 Subject: [PATCH 09/26] chore: make multiarch when possible --- .../workflows/build-and-push-to-docker.yml | 22 ++++--------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 1fc4becb..2f51e4c7 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -77,9 +77,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project python-based docker images - env: - # build multi-platform - MULTIARCH: true id: build-image run: | if [[ ${{ github.ref }} == refs/tags* ]]; then @@ -102,9 +99,7 @@ jobs: export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - # load the linux image - BUILDX_EXTRA_FLAGS=--load PLATFORM=linux/amd64 make py - + - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: @@ -244,9 +239,8 @@ jobs: # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" - make $EXTENSION - docker push $DOCKER_PREFIX-$EXTENSION:$LABEL - + BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make $EXTENSION + build-vnc-ext: needs: build-py-ext runs-on: ubuntu-latest @@ -333,7 +327,7 @@ jobs: command: npx mocha -r ts-node/register index.ts - name: Push to docker run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make julia build-r-ubuntu: needs: build-py @@ -361,10 +355,6 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2.0.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project rocker docker images id: build-image run: | @@ -424,10 +414,6 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2.0.0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then From 4a86b2e9e372f12bfd91a0938cf54420f73be7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Sat, 18 Mar 2023 10:37:46 +0100 Subject: [PATCH 10/26] chore: julia is complicated - no multiarch --- .github/workflows/build-and-push-to-docker.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 2f51e4c7..a8aac901 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -327,7 +327,12 @@ jobs: command: npx mocha -r ts-node/register index.ts - name: Push to docker run: | - BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make julia + docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + # on master push latest image + if [ "$REF" == "refs/heads/master" ]; then + docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-julia:latest + docker push $DOCKER_PREFIX-julia:latest + fi build-r-ubuntu: needs: build-py From ad7c61a51f4d8f87249987a2dd102468c56e720c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Mon, 20 Mar 2023 09:39:36 +0100 Subject: [PATCH 11/26] chore: enable buildx gh actions caching --- .../workflows/build-and-push-to-docker.yml | 35 +++++++++++++++---- Makefile | 19 +++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index a8aac901..d4247edd 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -28,6 +28,8 @@ on: [push] env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" + USE_BUILDX: true + BUILDX_EXTRA_FLAGS: "--cache-to type=gha,mode=max --cache-from type=gha" jobs: lint: @@ -76,6 +78,8 @@ jobs: uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project python-based docker images id: build-image run: | @@ -97,7 +101,7 @@ jobs: export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - make py + BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -112,7 +116,7 @@ jobs: - name: Push to docker run: | - BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make py + BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest @@ -163,7 +167,11 @@ jobs: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project python-based docker image extensions id: build-image run: | @@ -226,6 +234,8 @@ jobs: uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project python-based docker image extensions env: EXTENSION: ${{ matrix.EXTENSIONS }} @@ -239,7 +249,7 @@ jobs: # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" - BUILDX_EXTRA_FLAGS=--push MULTIARCH=true make $EXTENSION + BUILDX_EXTRA_FLAGS="--push --cache-to type=gha --cache-from type=gha" make $EXTENSION build-vnc-ext: needs: build-py-ext @@ -258,8 +268,11 @@ jobs: with: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 - uses: actions/checkout@v3 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project python-based docker image extensions env: EXTENSION: ${{ matrix.VNC-EXTENSIONS }} @@ -299,6 +312,8 @@ jobs: uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project julia docker image extensions id: build-image run: | @@ -333,7 +348,7 @@ jobs: docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-julia:latest docker push $DOCKER_PREFIX-julia:latest fi - + build-r-ubuntu: needs: build-py runs-on: ubuntu-latest @@ -360,6 +375,10 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project rocker docker images id: build-image run: | @@ -419,6 +438,10 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then diff --git a/Makefile b/Makefile index 5d3642c5..97ac41c0 100644 --- a/Makefile +++ b/Makefile @@ -27,12 +27,11 @@ extensions = \ batch \ bioc -# set the build command to buildx if multi-arch -ifdef MULTIARCH - BUILDX=buildx - PLATFORM?=linux/amd64,linux/arm64 -else - PLATFORM?=linux/amd64 +PLATFORM?=linux/amd64 +BUILD_CMD=build +# set the build command +ifdef USE_BUILDX + BUILD_CMD=buildx build endif # use BUILDX_EXTRA_FLAGS to set either --push or --load @@ -105,7 +104,7 @@ pull: # BASE_IMAGE_TAG: used to identify the jupyter base notebook to build from # RENKU_PYTHON_BASE_TAG: used to tag the resulting image py: - docker $(BUILDX) build docker/py \ + docker $(BUILD_CMD) docker/py \ --build-arg BASE_IMAGE=jupyter/base-notebook:$(BASE_IMAGE_TAG) \ --platform=$(PLATFORM) \ -t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL) \ @@ -133,7 +132,7 @@ cuda: py # this image is tagged with the julia version and the commit hash julia: py - docker $(BUILDX) build docker/julia \ + docker $(BUILD_CMD) docker/julia \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ -t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL) \ @@ -141,7 +140,7 @@ julia: py # this image is just tagged with the commit hash vnc: py - docker $(BUILDX) build docker/vnc \ + docker $(BUILD_CMD) docker/vnc \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ -t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL) \ @@ -155,7 +154,7 @@ vnc-%: vnc -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) batch: py - docker $(BUILDX) build docker/batch \ + docker $(BUILD_CMD) docker/batch \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="python:3.10-slim-buster" \ -t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL) \ From 5e9f548181c47ac022170b142c02c5268c8dda08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Mon, 20 Mar 2023 15:00:36 +0100 Subject: [PATCH 12/26] chore: enable multiarch for py again --- .../workflows/build-and-push-to-docker.yml | 25 ++++++------------- Makefile | 8 +++--- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index d4247edd..c065f9b3 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -116,7 +116,7 @@ jobs: - name: Push to docker run: | - BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py + PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest @@ -180,7 +180,6 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} # these are set dynamically so not part of env export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" @@ -189,7 +188,7 @@ jobs: export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}" export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" - make cuda + BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make cuda echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -245,11 +244,10 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" - BUILDX_EXTRA_FLAGS="--push --cache-to type=gha --cache-from type=gha" make $EXTENSION + BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make $EXTENSION build-vnc-ext: needs: build-py-ext @@ -282,13 +280,10 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - docker pull ${DOCKER_PREFIX}-vnc:${LABEL} # needed by the Makefile export EXTRA_DOCKER_LABEL="${LABEL}" - make vnc-$EXTENSION - docker push $DOCKER_PREFIX-$EXTENSION:$LABEL + BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make vnc-$EXTENSION build-julia-ext: needs: build-py @@ -322,13 +317,12 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} # needed by the makefile export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" - make julia + BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make julia echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -342,7 +336,7 @@ jobs: command: npx mocha -r ts-node/register index.ts - name: Push to docker run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make julia # on master push latest image if [ "$REF" == "refs/heads/master" ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-julia:latest @@ -387,14 +381,13 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} # needed by the makefile export RVERSION="${{ matrix.RVERSION }}" export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL" export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" - make r + BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make r echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -449,13 +442,11 @@ jobs: else export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} # needed by the makefile export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" export BIOC_VERSION="${{ matrix.RELEASE }}" - make bioc - docker push $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL + BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make bioc # on master push latest image if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ] diff --git a/Makefile b/Makefile index 97ac41c0..a4dedb52 100644 --- a/Makefile +++ b/Makefile @@ -111,7 +111,7 @@ py: $(BUILDX_EXTRA_FLAGS) r: py - docker build docker/r \ + docker $(BUILD_CMD) docker/r \ --build-arg BASE_IMAGE=$(RSTUDIO_BASE_IMAGE) \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --build-arg RVERSION=$(RVERSION) \ @@ -120,7 +120,7 @@ r: py -t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL) cuda: py - docker build docker/cuda \ + docker $(BUILD_CMD) docker/cuda \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg CUDA_COMPAT_PACKAGE="$(CUDA_COMPAT_PACKAGE)" \ --build-arg CUDA_CUDART_PACKAGE="$(CUDA_CUDART_PACKAGE)" \ @@ -148,7 +148,7 @@ vnc: py # this image is built on the vnc image and tagged as matlab with the commit hash vnc-%: vnc - docker build docker/$* \ + docker $(BUILD_CMD) docker/$* \ --build-arg BASE_IMAGE=renku/renkulab-vnc:$(GIT_COMMIT_SHA) \ --platform=$(PLATFORM) \ -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) @@ -161,7 +161,7 @@ batch: py $(BUILDX_EXTRA_FLAGS) bioc: py - docker build docker/r \ + docker $(BUILD_CMD) docker/r \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="bioconductor/bioconductor_docker:$(BIOC_VERSION)" \ --platform=$(PLATFORM) \ From ba8726ec27c004478cdea0391407b87c2f2f6e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Mon, 20 Mar 2023 15:19:46 +0100 Subject: [PATCH 13/26] chore: only cache py images --- .github/workflows/build-and-push-to-docker.yml | 6 +++--- Makefile | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index c065f9b3..e9a07bbb 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -29,7 +29,7 @@ env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true - BUILDX_EXTRA_FLAGS: "--cache-to type=gha,mode=max --cache-from type=gha" + BUILDX_EXTRA_FLAGS: "--cache-from type=gha" jobs: lint: @@ -101,7 +101,7 @@ jobs: export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make py + BUILDX_EXTRA_FLAGS="--cache-to type=gha,mode=max --load $BUILDX_EXTRA_FLAGS" make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -116,7 +116,7 @@ jobs: - name: Push to docker run: | - PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py + PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--cache-to type=gha,mode=max --push $BUILDX_EXTRA_FLAGS" make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest diff --git a/Makefile b/Makefile index a4dedb52..ecf284f2 100644 --- a/Makefile +++ b/Makefile @@ -117,7 +117,8 @@ r: py --build-arg RVERSION=$(RVERSION) \ --build-arg RSTUDIO_VERSION_OVERRIDE=$(RSTUDIO_VERSION_OVERRIDE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) cuda: py docker $(BUILD_CMD) docker/cuda \ @@ -128,7 +129,8 @@ cuda: py --build-arg EXTRA_LIBRARIES="$(EXTRA_LIBRARIES)" \ --build-arg LIBCUDNN_PACKAGE="$(LIBCUDNN_PACKAGE)" \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-cuda:$(CUDA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-cuda:$(CUDA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is tagged with the julia version and the commit hash julia: py @@ -151,7 +153,8 @@ vnc-%: vnc docker $(BUILD_CMD) docker/$* \ --build-arg BASE_IMAGE=renku/renkulab-vnc:$(GIT_COMMIT_SHA) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) batch: py docker $(BUILD_CMD) docker/batch \ @@ -165,4 +168,5 @@ bioc: py --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="bioconductor/bioconductor_docker:$(BIOC_VERSION)" \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-bioc:$(BIOC_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-bioc:$(BIOC_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) From 99c94275bba8d9fe6299ada38b313d333461388a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Mon, 20 Mar 2023 17:22:19 +0100 Subject: [PATCH 14/26] chore: use inline caching --- .github/workflows/build-and-push-to-docker.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index e9a07bbb..8723d95e 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -30,7 +30,7 @@ env: DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true BUILDX_EXTRA_FLAGS: "--cache-from type=gha" - + BUILDKIT_INLINE_CACHE: 1 jobs: lint: strategy: @@ -188,6 +188,8 @@ jobs: export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}" export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" + + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make cuda echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT @@ -247,6 +249,7 @@ jobs: # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make $EXTENSION build-vnc-ext: @@ -283,6 +286,7 @@ jobs: # needed by the Makefile export EXTRA_DOCKER_LABEL="${LABEL}" + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make vnc-$EXTENSION build-julia-ext: @@ -322,6 +326,7 @@ jobs: export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make julia echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT @@ -387,6 +392,7 @@ jobs: export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL" export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make r echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT @@ -446,6 +452,7 @@ jobs: # needed by the makefile export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" export BIOC_VERSION="${{ matrix.RELEASE }}" + docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make bioc # on master push latest image From c5bbc67b92513c10e6f0d7efba88ac7014742dc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Mon, 20 Mar 2023 18:29:41 +0100 Subject: [PATCH 15/26] chore: try s3 cache --- .github/workflows/build-and-push-to-docker.yml | 9 ++++++--- Makefile | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 8723d95e..9c1f7777 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -29,8 +29,11 @@ env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true - BUILDX_EXTRA_FLAGS: "--cache-from type=gha" BUILDKIT_INLINE_CACHE: 1 + BUILDX_EXTRA_FLAGS: --cache-to type=s3,mode=max,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} + --cache-from type=s3,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} + --cache-from type=inline + jobs: lint: strategy: @@ -101,7 +104,7 @@ jobs: export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - BUILDX_EXTRA_FLAGS="--cache-to type=gha,mode=max --load $BUILDX_EXTRA_FLAGS" make py + BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -116,7 +119,7 @@ jobs: - name: Push to docker run: | - PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--cache-to type=gha,mode=max --push $BUILDX_EXTRA_FLAGS" make py + PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest diff --git a/Makefile b/Makefile index ecf284f2..9689eb2b 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ extensions = \ batch \ bioc +BUILDKIT_INLINE_CACHE?=1 PLATFORM?=linux/amd64 BUILD_CMD=build # set the build command From 2f8b5f731a6d1b7e28d43108b7a22e905cd9f979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 10:07:36 +0100 Subject: [PATCH 16/26] try to remove inline caching --- .github/workflows/build-and-push-to-docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 9c1f7777..b3ac158d 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -29,10 +29,9 @@ env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true - BUILDKIT_INLINE_CACHE: 1 + # BUILDKIT_INLINE_CACHE: 1 BUILDX_EXTRA_FLAGS: --cache-to type=s3,mode=max,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} --cache-from type=s3,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} - --cache-from type=inline jobs: lint: From 73b33b247a11bf7fb71db13becb4b1691db6f567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 10:14:40 +0100 Subject: [PATCH 17/26] chore: move ssh key generation to the end --- docker/py/Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 8754a49a..c8b9c7ce 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -36,20 +36,6 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ /tmp/git-lfs-3.3.0/install.sh && \ rm -rf /tmp/git-lfs* -# Setup ssh keys -RUN mkdir -p /opt/ssh/sshd_config.d /opt/ssh/ssh_host_keys /opt/ssh/pid && \ - ssh-keygen -q -N "" -t dsa -f /opt/ssh/ssh_host_keys/ssh_host_dsa_key && \ - ssh-keygen -q -N "" -t rsa -b 4096 -f /opt/ssh/ssh_host_keys/ssh_host_rsa_key && \ - ssh-keygen -q -N "" -t ecdsa -f /opt/ssh/ssh_host_keys/ssh_host_ecdsa_key && \ - ssh-keygen -q -N "" -t ed25519 -f /opt/ssh/ssh_host_keys/ssh_host_ed25519_key - - -COPY sshd_config /opt/ssh/sshd_config - -RUN chown -R 0:100 /opt/ssh/ && \ - chmod -R u=rwX,g=rX,o= /opt/ssh && \ - chmod -R u=rwX,g=rwX,o= /opt/ssh/pid - # switch to the notebook user USER $NB_USER @@ -96,6 +82,23 @@ COPY --chown=1000:100 powerline.config /home/${NB_USER}/.config/powerline-shell/ RUN cat /tmp/powerline.bashrc >> ~/.bashrc && rm /tmp/powerline.bashrc COPY entrypoint.sh /entrypoint.sh + +# Setup ssh keys +USER root +RUN mkdir -p /opt/ssh/sshd_config.d /opt/ssh/ssh_host_keys /opt/ssh/pid && \ + ssh-keygen -q -N "" -t dsa -f /opt/ssh/ssh_host_keys/ssh_host_dsa_key && \ + ssh-keygen -q -N "" -t rsa -b 4096 -f /opt/ssh/ssh_host_keys/ssh_host_rsa_key && \ + ssh-keygen -q -N "" -t ecdsa -f /opt/ssh/ssh_host_keys/ssh_host_ecdsa_key && \ + ssh-keygen -q -N "" -t ed25519 -f /opt/ssh/ssh_host_keys/ssh_host_ed25519_key + +COPY sshd_config /opt/ssh/sshd_config + +RUN chown -R 0:100 /opt/ssh/ && \ + chmod -R u=rwX,g=rX,o= /opt/ssh && \ + chmod -R u=rwX,g=rwX,o= /opt/ssh/pid + ENTRYPOINT [ "tini", "--", "/entrypoint.sh" ] CMD [ "/usr/local/bin/start-singleuser.sh" ] + +USER $NB_USER \ No newline at end of file From 904732b70be4ee13a39b80b633de5d014e2caa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 13:37:52 +0100 Subject: [PATCH 18/26] chore: make py image multi-stage --- .../workflows/build-and-push-to-docker.yml | 10 ++- docker/py/Dockerfile | 62 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index b3ac158d..531c12e4 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -29,10 +29,8 @@ env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true - # BUILDKIT_INLINE_CACHE: 1 - BUILDX_EXTRA_FLAGS: --cache-to type=s3,mode=max,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} - --cache-from type=s3,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} - + BUILDX_CACHE_TO: --cache-to type=s3,mode=max,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} + BUILDX_CACHE_FROM: --cache-from type=s3,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} jobs: lint: strategy: @@ -103,7 +101,7 @@ jobs: export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make py + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-to type=inline" make py echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -118,7 +116,7 @@ jobs: - name: Push to docker run: | - PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make py + PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make py # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index c8b9c7ce..a51cbb7e 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -1,5 +1,37 @@ ARG BASE_IMAGE=jupyter/base-notebook:lab-3.4.0 -FROM $BASE_IMAGE as base + +FROM ${BASE_IMAGE} as builder + +USER root +RUN apt-get update && apt-get install -yq --no-install-recommends \ + build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# switch to the notebook user +USER $NB_USER +# install jupyterlab, papermill, git extension and renku-jupyterlab-ts +COPY requirements.txt /tmp/requirements.txt +RUN python3 -m pip install --no-cache-dir -U pip && \ + python3 -m pip install --no-cache-dir -r /tmp/requirements.txt && \ + # jupyter lab build && \ + # jupyter labextension list && \ + # npm cache clean --force && \ + rm -rf "/home/${NB_USER}/.cache" + +# jupyter sets channel priority to strict which often causes very long error messages +RUN mamba config --system --set channel_priority flexible && \ + mamba clean --all -f -y + +COPY renku-requirements/requirements.txt /tmp/renku-requirements.txt + +RUN mkdir -p "$HOME/.renku/bin" && \ + virtualenv --no-periodic-update "$HOME/.renku/venv" && \ + source "$HOME/.renku/venv/bin/activate" && \ + pip install --no-cache-dir -r /tmp/renku-requirements.txt && \ + deactivate + +FROM $BASE_IMAGE LABEL maintainer="Swiss Data Science Center " @@ -36,22 +68,8 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ /tmp/git-lfs-3.3.0/install.sh && \ rm -rf /tmp/git-lfs* -# switch to the notebook user USER $NB_USER -# install jupyterlab, papermill, git extension and renku-jupyterlab-ts -COPY requirements.txt /tmp/requirements.txt -RUN python3 -m pip install --no-cache-dir -U pip && \ - python3 -m pip install --no-cache-dir -r /tmp/requirements.txt && \ - # jupyter lab build && \ - # jupyter labextension list && \ - # npm cache clean --force && \ - rm -rf "/home/${NB_USER}/.cache" - -# jupyter sets channel priority to strict which often causes very long error messages -RUN mamba config --system --set channel_priority flexible && \ - mamba clean --all -f -y - # setup sshd RUN mkdir -p "$HOME/.ssh" && \ touch "$HOME/.ssh/authorized_keys" && \ @@ -62,15 +80,6 @@ ENV RENKU_DISABLE_VERSION_CHECK 1 ENV PATH=$HOME/.local/bin:$PATH:$HOME/.renku/bin -COPY renku-requirements/requirements.txt /tmp/renku-requirements.txt - -RUN mkdir -p "$HOME/.renku/bin" && \ - virtualenv --no-periodic-update "$HOME/.renku/venv" && \ - source "$HOME/.renku/venv/bin/activate" && \ - pip install --no-cache-dir -r /tmp/renku-requirements.txt && \ - deactivate && \ - ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" - # configure git COPY git-config.bashrc /home/$NB_USER/ RUN cat "/home/$NB_USER/git-config.bashrc" >> "/home/$NB_USER/.bashrc" && rm "/home/$NB_USER/git-config.bashrc" @@ -101,4 +110,7 @@ ENTRYPOINT [ "tini", "--", "/entrypoint.sh" ] CMD [ "/usr/local/bin/start-singleuser.sh" ] -USER $NB_USER \ No newline at end of file +USER $NB_USER +COPY --from=builder /opt/conda /opt/conda +COPY --from=builder "$HOME/.renku" "$HOME/.renku" +RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" From aa2ddb6b866a28b1abb31aca1632a42719654b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 14:35:33 +0100 Subject: [PATCH 19/26] chore: adapt caching flags --- .github/workflows/build-and-push-to-docker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 531c12e4..83a82c3e 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -190,7 +190,7 @@ jobs: export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make cuda + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM $BUILDX_CACHE_TO" make cuda echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -250,7 +250,7 @@ jobs: # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make $EXTENSION + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make $EXTENSION build-vnc-ext: needs: build-py-ext @@ -287,7 +287,7 @@ jobs: # needed by the Makefile export EXTRA_DOCKER_LABEL="${LABEL}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make vnc-$EXTENSION + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make vnc-$EXTENSION build-julia-ext: needs: build-py @@ -327,7 +327,7 @@ jobs: export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make julia + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-to type=inline --cache-from type=inline" make julia echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -341,7 +341,7 @@ jobs: command: npx mocha -r ts-node/register index.ts - name: Push to docker run: | - BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make julia + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make julia # on master push latest image if [ "$REF" == "refs/heads/master" ]; then docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-julia:latest @@ -393,7 +393,7 @@ jobs: export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_EXTRA_FLAGS" make r + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-from type=inline --cache-to type=inline" make r echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -409,7 +409,7 @@ jobs: - name: Push to docker run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM --cache-from type=inline $BUILDX_CACHE_TO" make r # on master push latest image if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ] @@ -453,7 +453,7 @@ jobs: export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" export BIOC_VERSION="${{ matrix.RELEASE }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_EXTRA_FLAGS" make bioc + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM --cache-from type=inline $BUILDX_CACHE_TO" make bioc # on master push latest image if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ] From 0137af601b635c25cecd013e5f9601aff4303373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 15:47:05 +0100 Subject: [PATCH 20/26] chore: compare gha cache --- .github/workflows/build-and-push-to-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 83a82c3e..aa444575 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -29,8 +29,8 @@ env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true - BUILDX_CACHE_TO: --cache-to type=s3,mode=max,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} - BUILDX_CACHE_FROM: --cache-from type=s3,bucket=gha-cache,region=zh,endpoint_url=https://s3-zh.os.switch.ch,access_key_id=${{ secrets.GHA_S3_CACHE_ACCESS_KEY }},secret_access_key=${{ secrets.GHA_S3_CACHE_SECRET_ACCESS_KEY }} + BUILDX_CACHE_TO: --cache-to type=gha + BUILDX_CACHE_FROM: --cache-from type=gha jobs: lint: strategy: From de6dc48b40f372d9d26352bac29d7e9a52113875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 17:59:22 +0100 Subject: [PATCH 21/26] chore: move multiarch to second phase --- .../workflows/build-and-push-to-docker.yml | 62 +++++++++++++++++-- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index aa444575..1ed74c11 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -62,7 +62,6 @@ jobs: matrix: BASE_IMAGE_TAG: - lab-3.6.1 - - python-3.10.9 - python-3.9.13 - python-3.8.13 @@ -116,12 +115,63 @@ jobs: - name: Push to docker run: | - PLATFORM=linux/arm64,linux/amd64 BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make py + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make py + + build-py-multiarch: + needs: build-py + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + BASE_IMAGE_TAG: + - lab-3.6.1 + - python-3.9.13 + - python-3.8.13 + + steps: + - name: Docker Login + uses: Azure/docker-login@v1 + with: + username: ${{ secrets.RENKU_DOCKER_USERNAME }} + password: ${{ secrets.RENKU_DOCKER_PASSWORD }} + + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Expose GitHub Runtime + uses: crazy-max/ghaction-github-runtime@v2 + - name: Build renku project python-based docker images + id: build-image + run: | + if [[ ${{ github.ref }} == refs/tags* ]]; then + export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) + else + export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) + fi + + # This ensures the same image tags as before are built, in addition to new ones + if [[ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]]; then + export RENKU_PYTHON_BASE_IMAGE_TAG=$DEFAULT_PYTHON_VERSION + else + export RENKU_PYTHON_BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} + fi + + # needed by the makefile - these are generated dynamically + # and hence they are not part of the env setup above + export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} + export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" + export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) + export PLATFORM=linux/amd64,linux/arm64 + export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" + # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest - docker push $DOCKER_PREFIX-py:latest + export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-py:latest" fi + make py build-cuda: needs: build-py @@ -190,9 +240,9 @@ jobs: export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM $BUILDX_CACHE_TO" make cuda + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make cuda echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT - + - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: From ad4436e3f92dd04902655db358bdf56595e85125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 18:42:31 +0100 Subject: [PATCH 22/26] chore: variable cleanup --- .../workflows/build-and-push-to-docker.yml | 67 +++++++++---------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 1ed74c11..8a971e7c 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -30,7 +30,7 @@ env: DEFAULT_PYTHON_VERSION: "3.10" USE_BUILDX: true BUILDX_CACHE_TO: --cache-to type=gha - BUILDX_CACHE_FROM: --cache-from type=gha + BUILDX_CACHE_FROM: --cache-from type=gha --cache-from type=inline jobs: lint: strategy: @@ -64,7 +64,8 @@ jobs: - lab-3.6.1 - python-3.9.13 - python-3.8.13 - + env: + BASE_IMAGE_TAG: ${{ matrix.BASE_IMAGE_TAG }} steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -97,16 +98,14 @@ jobs: # needed by the makefile - these are generated dynamically # and hence they are not part of the env setup above - export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" - export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-to type=inline" make py - echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make py + echo "PY_DOCKER_LABEL=${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${DOCKER_PREFIX}-py:${{ steps.build-image.outputs.PY_DOCKER_LABEL }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: @@ -115,7 +114,8 @@ jobs: - name: Push to docker run: | - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make py + export PY_DOCKER_LABEL=${{ steps.build-image.outputs.PY_DOCKER_LABEL }} + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM" make py build-py-multiarch: needs: build-py @@ -163,9 +163,8 @@ jobs: # and hence they are not part of the env setup above export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" - export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) export PLATFORM=linux/amd64,linux/arm64 - export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" + export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM" # on master push latest image - the lab-* image is tagged latest if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then @@ -300,7 +299,7 @@ jobs: # needed by the makefile - for these images, we only tag with commit sha export EXTRA_DOCKER_LABEL="${LABEL}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make $EXTENSION + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" make $EXTENSION build-vnc-ext: needs: build-py-ext @@ -337,7 +336,7 @@ jobs: # needed by the Makefile export EXTRA_DOCKER_LABEL="${LABEL}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make vnc-$EXTENSION + BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" make vnc-$EXTENSION build-julia-ext: needs: build-py @@ -374,11 +373,10 @@ jobs: # needed by the makefile export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" - export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" + export BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-to type=inline --cache-from type=inline" make julia - echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT + make julia - name: Image Acceptance Tests uses: cypress-io/github-action@v4 @@ -391,13 +389,14 @@ jobs: command: npx mocha -r ts-node/register index.ts - name: Push to docker run: | - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM --cache-from type=inline" make julia - # on master push latest image - if [ "$REF" == "refs/heads/master" ]; then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-julia:latest - docker push $DOCKER_PREFIX-julia:latest + export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" + export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" + export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" + # on master push latest image - the lab-* image is tagged latest + if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then + export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-julia:latest" fi - + make julia build-r-ubuntu: needs: build-py runs-on: ubuntu-latest @@ -443,7 +442,7 @@ jobs: export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM --cache-from type=inline --cache-to type=inline" make r + BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make r echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT - name: Image Acceptance Tests @@ -459,14 +458,12 @@ jobs: - name: Push to docker run: | - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM --cache-from type=inline $BUILDX_CACHE_TO" make r - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ] - then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-r:latest - docker push $DOCKER_PREFIX-r:latest + export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" + # on master push latest image - the 4.2.0 image is tagged latest + if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ]; then + export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-r:latest" fi + make r build-bioc: needs: build-py @@ -503,12 +500,10 @@ jobs: export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" export BIOC_VERSION="${{ matrix.RELEASE }}" docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM --cache-from type=inline $BUILDX_CACHE_TO" make bioc - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ] - then - docker tag $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL $DOCKER_PREFIX-bioc:latest - docker push $DOCKER_PREFIX-bioc:latest + export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" + # on master push latest image - the devel image is tagged latest + if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ]; then + export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-bioc:latest" fi + make bioc From 8b325eb22454981cc2b682b6af22a7be485f680a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Tue, 21 Mar 2023 21:34:26 +0100 Subject: [PATCH 23/26] chore: use docker actions for building --- .../workflows/build-and-push-to-docker.yml | 519 +++++++++--------- 1 file changed, 266 insertions(+), 253 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index 8a971e7c..27bbf105 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -31,6 +31,8 @@ env: USE_BUILDX: true BUILDX_CACHE_TO: --cache-to type=gha BUILDX_CACHE_FROM: --cache-from type=gha --cache-from type=inline + BASE_IMAGE_FILENAME: renkulab-py.tar + BUILDKIT_INLINE_CACHE: 1 jobs: lint: strategy: @@ -60,12 +62,12 @@ jobs: strategy: fail-fast: true matrix: - BASE_IMAGE_TAG: - - lab-3.6.1 - - python-3.9.13 - - python-3.8.13 + BASE_PYTHON_VERSION: + - "3.10" + - "3.9" + - "3.8" env: - BASE_IMAGE_TAG: ${{ matrix.BASE_IMAGE_TAG }} + BASE_IMAGE_TAG: python-${{ matrix.BASE_PYTHON_VERSION }} steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -78,45 +80,46 @@ jobs: uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project python-based docker images - id: build-image - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # This ensures the same image tags as before are built, in addition to new ones - if [[ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]]; then - export RENKU_PYTHON_BASE_IMAGE_TAG=$DEFAULT_PYTHON_VERSION - else - export RENKU_PYTHON_BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} - fi - - # needed by the makefile - these are generated dynamically - # and hence they are not part of the env setup above - export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make py - echo "PY_DOCKER_LABEL=${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-py + tags: | + type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + cache-from: type=gha + cache-to: type=gha,mode=max + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${DOCKER_PREFIX}-py:${{ steps.build-image.outputs.PY_DOCKER_LABEL }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-py:${{ steps.meta.outputs.version }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - - name: Push to docker - run: | - export PY_DOCKER_LABEL=${{ steps.build-image.outputs.PY_DOCKER_LABEL }} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM" make py - + # export the default base image for other jobs to use + - name: Build and export + if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + cache-from: type=gha + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-py-multiarch: needs: build-py runs-on: ubuntu-latest @@ -124,53 +127,44 @@ jobs: strategy: fail-fast: true matrix: - BASE_IMAGE_TAG: - - lab-3.6.1 - - python-3.9.13 - - python-3.8.13 - + BASE_PYTHON_VERSION: + - "3.10" + - "3.9" + - "3.8" + env: + BASE_IMAGE_TAG: python-${{ matrix.BASE_IMAGE_TAG }} steps: - name: Docker Login uses: Azure/docker-login@v1 with: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project python-based docker images - id: build-image - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # This ensures the same image tags as before are built, in addition to new ones - if [[ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]]; then - export RENKU_PYTHON_BASE_IMAGE_TAG=$DEFAULT_PYTHON_VERSION - else - export RENKU_PYTHON_BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} - fi - - # needed by the makefile - these are generated dynamically - # and hence they are not part of the env setup above - export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} - export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" - export PLATFORM=linux/amd64,linux/arm64 - export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_TO $BUILDX_CACHE_FROM" - - # on master push latest image - the lab-* image is tagged latest - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then - export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-py:latest" - fi - make py + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-py + tags: | + type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.BASE_PYTHON_VERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: docker/py + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-to: type=gha,mode=max + cache-from: type=gha + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + platforms: linux/amd64,linux/arm64 build-cuda: needs: build-py @@ -208,59 +202,71 @@ jobs: LIBCUDNN_PACKAGE: "libcudnn8=8.5.0.96-1+cuda11.7" # note cuda 11.8 is available now, but there I could not # find a cuda-compat package in the ubuntu repos at this time - steps: - name: Docker Login uses: Azure/docker-login@v1 with: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - uses: actions/checkout@v3 + - name: Set outputs + id: vars + run: | + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project python-based docker image extensions - id: build-image - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # these are set dynamically so not part of env - export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" - export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}" - export CUDA_VERSION="${{ matrix.CUDA_VERSION }}" - export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}" - export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" - export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" - - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make cuda - echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT - + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-cuda + tags: | + type=sha,prefix=${{ matrix.CUDA_VERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.CUDA_VERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.CUDA_VERSION == '11.7' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + CUDA_COMPAT_PACKAGE=${{ matrix.CUDA_COMPAT_PACKAGE }} + CUDA_CUDART_PACKAGE=${{ matrix.CUDA_CUDART_PACKAGE }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }} + EXTRA_LIBRARIES=${{ matrix.EXTRA_LIBRARIES }} + LIBCUDNN_PACKAGE=${{ matrix.LIBCUDNN_PACKAGE }} + context: docker/cuda + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: | + type=inline + type=registry,ref=${{ steps.vars.outputs.renku_base }} - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-cuda:${{ steps.meta.outputs.version }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - name: Push to docker - run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.7" ]; then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-cuda:latest - docker push $DOCKER_PREFIX-cuda:latest - fi - + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + CUDA_COMPAT_PACKAGE=${{ matrix.CUDA_COMPAT_PACKAGE }} + CUDA_CUDART_PACKAGE=${{ matrix.CUDA_CUDART_PACKAGE }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }} + EXTRA_LIBRARIES=${{ matrix.EXTRA_LIBRARIES }} + LIBCUDNN_PACKAGE=${{ matrix.LIBCUDNN_PACKAGE }} + context: docker/cuda + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-py-ext: needs: build-py runs-on: ubuntu-latest @@ -268,7 +274,7 @@ jobs: strategy: fail-fast: true matrix: - EXTENSIONS: + EXTENSION: - vnc - batch @@ -280,38 +286,45 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 + - name: Set outputs + id: vars + run: | + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT - name: Set up QEMU uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project python-based docker image extensions - env: - EXTENSION: ${{ matrix.EXTENSIONS }} - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # needed by the makefile - for these images, we only tag with commit sha - export EXTRA_DOCKER_LABEL="${LABEL}" - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" make $EXTENSION + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }} + tags: | + type=sha,prefix= + type=semver,pattern={{version}},prefix= + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }},prefix= + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/${{ matrix.EXTENSION }} + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha build-vnc-ext: needs: build-py-ext runs-on: ubuntu-latest - strategy: fail-fast: true matrix: - VNC-EXTENSIONS: + VNC-EXTENSION: - matlab - qgis - steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -321,23 +334,32 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - uses: actions/checkout@v3 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project python-based docker image extensions - env: - EXTENSION: ${{ matrix.VNC-EXTENSIONS }} + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # needed by the Makefile - export EXTRA_DOCKER_LABEL="${LABEL}" - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" make vnc-$EXTENSION - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.VNC-EXTENSION }} + tags: | + type=sha,prefix= + type=semver,pattern={{version}},prefix= + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }},prefix= + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=${{ env.DOCKER_PREFIX }}-vnc:${{ steps.vars.outputs.sha_short }} + context: docker/${{ matrix.VNC-EXTENSION }} + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + build-julia-ext: needs: build-py runs-on: ubuntu-latest @@ -345,9 +367,8 @@ jobs: strategy: fail-fast: true matrix: - JULIAVERSIONS: - - 1.7.1 - + JULIAVERSION: + - "1.7.1" steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -360,61 +381,85 @@ jobs: uses: docker/setup-qemu-action@v2.0.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project julia docker image extensions - id: build-image + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # needed by the makefile - export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" - export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" - export BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - make julia - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-julia + tags: | + type=sha,prefix=${{ matrix.JULIAVERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.JULIAVERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.JULIAVERSION == '1.7.1' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/julia + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${DOCKER_PREFIX}-julia:${{ matrix.JULIAVERSION }}-${{ steps.vars.outputs.sha_short }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - name: Push to docker - run: | - export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" - export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" - export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" - # on master push latest image - the lab-* image is tagged latest - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then - export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-julia:latest" - fi - make julia + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/julia + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + build-r-ubuntu: needs: build-py runs-on: ubuntu-latest - + env: + RSTUDIO_VERSION: 2022.02.3-492 strategy: - fail-fast: true + fail-fast: false matrix: include: - RVERSION: 4.1.0 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.1.1 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.1.2 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.2.0 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r + - RVERSION: devel + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r + - RVERSION: RELEASE_3_14 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc + - RVERSION: RELEASE_3_15 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc - RVERSION: devel - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -425,85 +470,53 @@ jobs: - uses: actions/checkout@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project rocker docker images - id: build-image + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # needed by the makefile - export RVERSION="${{ matrix.RVERSION }}" - export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL" - export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" - export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - BUILDX_EXTRA_FLAGS="--load $BUILDX_CACHE_FROM" make r - echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }} + tags: | + type=sha,prefix=${{ matrix.RVERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.RVERSION == 'devel' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + BASE_IMAGE=${{ matrix.BASE_IMAGE_NAME }}:${{ matrix.RVERSION }} + RSTUDIO_VERSION_OVERRIDE=${{ env.RSTUDIO_VERSION }} + context: docker/r + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }}:${{ matrix.RVERSION }}-${{ steps.vars.outputs.sha_short }} TEST_ENV: rstudio TEST_USER_NAME: rstudio TEST_SPEC: rstudio.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - - name: Push to docker - run: | - export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" - # on master push latest image - the 4.2.0 image is tagged latest - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ]; then - export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-r:latest" - fi - make r - - build-bioc: - needs: build-py - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - RELEASE: - - RELEASE_3_14 - - RELEASE_3_15 - - devel - steps: - - name: Docker Login - uses: Azure/docker-login@v1 + - name: Build and push + uses: docker/build-push-action@v4 with: - username: ${{ secrets.RENKU_DOCKER_USERNAME }} - password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - - uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.0.0 - - name: Expose GitHub Runtime - uses: crazy-max/ghaction-github-runtime@v2 - - name: Build renku project bioconductor docker images - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # needed by the makefile - export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" - export BIOC_VERSION="${{ matrix.RELEASE }}" - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - export BUILDX_EXTRA_FLAGS="--push $BUILDX_CACHE_FROM" - # on master push latest image - the devel image is tagged latest - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ]; then - export BUILDX_EXTRA_FLAGS="$BUILDX_EXTRA_FLAGS --tag $DOCKER_PREFIX-bioc:latest" - fi - make bioc - + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + RSTUDIO_BASE_IMAGE=rocker/verse:${{ matrix.RVERSION }} + RSTUDIO_VERSION_OVERRIDE=${{ matrix.RSTUDIO_VERSION }} + context: docker/r + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + \ No newline at end of file From 6aae3db0e58e56195fe8834e29c9e14deee6b698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Thu, 23 Mar 2023 00:19:48 +0100 Subject: [PATCH 24/26] chore: remove unused env vars --- .github/workflows/build-and-push-to-docker.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index a103fa06..bd8a3515 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -28,11 +28,6 @@ on: [push] env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" - USE_BUILDX: true - BUILDX_CACHE_TO: --cache-to type=gha - BUILDX_CACHE_FROM: --cache-from type=gha --cache-from type=inline - BASE_IMAGE_FILENAME: renkulab-py.tar - BUILDKIT_INLINE_CACHE: 1 jobs: lint: strategy: @@ -66,8 +61,6 @@ jobs: - "3.10" - "3.9" - "3.8" - env: - BASE_IMAGE_TAG: python-${{ matrix.BASE_PYTHON_VERSION }} steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -131,8 +124,6 @@ jobs: - "3.10" - "3.9" - "3.8" - env: - BASE_IMAGE_TAG: python-${{ matrix.BASE_IMAGE_TAG }} steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -433,7 +424,7 @@ jobs: env: RSTUDIO_VERSION: 2022.02.3-492 strategy: - fail-fast: false + fail-fast: true matrix: include: - RVERSION: 4.1.0 @@ -518,8 +509,8 @@ jobs: with: build-args: | RENKU_BASE=${{ steps.vars.outputs.renku_base }} - RSTUDIO_BASE_IMAGE=rocker/verse:${{ matrix.RVERSION }} - RSTUDIO_VERSION_OVERRIDE=${{ matrix.RSTUDIO_VERSION }} + BASE_IMAGE=${{ matrix.BASE_IMAGE_NAME }}:${{ matrix.RVERSION }} + RSTUDIO_VERSION_OVERRIDE=${{ env.RSTUDIO_VERSION }} context: docker/r labels: ${{ steps.meta.outputs.labels }} push: true From 3f71e16e7d1ffa309e16563d0f5828545b60496f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Thu, 23 Mar 2023 00:59:18 +0100 Subject: [PATCH 25/26] update readme --- .../workflows/build-and-push-to-docker.yml | 3 +- README.md | 32 +++++++++++++++---- docker/py/Dockerfile | 2 +- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index bd8a3515..f1bcf2bc 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -18,8 +18,7 @@ # # It's probbaly worth noting here that the same action is used multiple # times with multiple input parameters in the `matrix` variable to -# build different versions of the container; these parameters are then -# passed on to the makefile. +# build different versions of the image. name: Renku Docker Image CI diff --git a/README.md b/README.md index 6d02baa5..b0f12e73 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ folder for more). Images are updated from time to time; we try to keep them reasonably current with modern python versions, cuda/torch versions and R versions. Typically, this -involves updating our github actions to include new versions and occasionally +involves updating our github actions to include new versions and updating the `Makefile` to modify what it builds by default. ## Usage @@ -64,14 +64,14 @@ the image that's right for you: `renku/renkulab-[image flavor]:[image flavor version]-[tag|hash]` For example: -`renku/renkulab-py:python-3.10-0.15.0` +`renku/renkulab-py:3.10-0.15.0` * `renku/renkulab`: indicates this is an image you can use to spawn an environment from your project on RenkuLab. * `-py`: indicates this is a python image flavor; either the programming language installed in the environment, or the base image that extra dependencies are added to. See below for details about the available flavors. -* `python-3.10`: indicates the version of python is 3.10 +* `3.10`: indicates the version of python is 3.10 * `0.15.0` (or `d572e9a`): the tag is a value given to a commit of the repository and indicates that the version is part of a release. If the version is not part of a release, this value is the first few chars of the git commit SHA from which the @@ -84,9 +84,9 @@ This can easily be overridden modifying the renku version in the project's Docke | Image | Description | Base image | |--------------------------------------------------------------------------------|-------------------------------------------------|----------------------------------------------------------------------------------------------------| -| [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | Basic Jupyter image with minimal dependencies | [jupyter/base-notebook](https://hub.docker.com/r/jupyter/base-notebook/tags) | -| [renku/renkulab-r](https://hub.docker.com/r/renku/renkulab-r/tags) | Basic Rstudio image | [rocker/verse](https://hub.docker.com/r/rocker/verse/tags) | -| [renku/renkulab-julia](https://hub.docker.com/r/renku/renkulab-julia/tags) | Basic Julia image | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | +| [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | Jupyter image with minimal dependencies | [jupyter/base-notebook](https://hub.docker.com/r/jupyter/base-notebook/tags) | +| [renku/renkulab-r](https://hub.docker.com/r/renku/renkulab-r/tags) | Rstudio image | [rocker/verse](https://hub.docker.com/r/rocker/verse/tags) | +| [renku/renkulab-julia](https://hub.docker.com/r/renku/renkulab-julia/tags) | Julia image | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | | [renku/renkulab-cuda](https://hub.docker.com/r/renku/renkulab-cuda/tags) | Cuda image with Python and minimal dependencies | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | | [renku/renkulab-cuda-tf](https://hub.docker.com/r/renku/renkulab-cuda-tf/tags) | Cuda image with Python and Tensorflow | [renku/renkulab-cuda](https://hub.docker.com/r/renku/renkulab-cuda/tags) | | [renku/renkulab-vnc](https://hub.docker.com/r/renku/renkulab-vnc/tags) | VNC Image with Python | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | @@ -166,6 +166,26 @@ image you would like to build. Note that on arm-based systems (e.g. Apple M1/M2) you may need to use the flag `--platform=linux/amd64` for the build because not all base images are available for ARM architecture. +### M1/M2 (arm64) support +Starting with 0.16.0, the python base images are built to support both `X86_64` +and `arm64` architectures. If you need to build a multi-arch image locally, you can +do it with `make`: + +``` +$ PLATFORM=linux/amd64,linux/arm64 USE_BUILDX=1 make py +``` + +Alternatively you can use the `buildx` command directly: + +``` +$ docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --tag myimage:tag \ + --push \ + docker/py + +Note that for the time being we can only provide the python images with `arm64` support. + ## Adding renku to your own images If you already have a Docker image with complicated dependencies that are needed diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index a51cbb7e..2a071ce1 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=jupyter/base-notebook:lab-3.4.0 +ARG BASE_IMAGE=jupyter/base-notebook:python-3.10 FROM ${BASE_IMAGE} as builder From f86165e6f6f4762240a07b33310475a9d74bd01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rok=20Ro=C5=A1kar?= Date: Thu, 23 Mar 2023 11:19:29 +0100 Subject: [PATCH 26/26] chore: ensure proper ownership of .renku --- Makefile | 2 +- docker/py/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9689eb2b..c20f3496 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,8 @@ DOCKER_PREFIX?=renku/renkulab GIT_COMMIT_SHA?=$(shell git rev-parse --short=7 --verify HEAD) # for building the base image -BASE_IMAGE_TAG?=lab-3.6.1 DEFAULT_PYTHON_VERSION?=3.10 +BASE_IMAGE_TAG?=python-$(DEFAULT_PYTHON_VERSION) PY_DOCKER_LABEL?=$(DEFAULT_PYTHON_VERSION)-$(GIT_COMMIT_SHA) RENKU_BASE?=$(DOCKER_PREFIX)-py:$(PY_DOCKER_LABEL) diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 2a071ce1..d20b91c9 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -111,6 +111,6 @@ ENTRYPOINT [ "tini", "--", "/entrypoint.sh" ] CMD [ "/usr/local/bin/start-singleuser.sh" ] USER $NB_USER -COPY --from=builder /opt/conda /opt/conda -COPY --from=builder "$HOME/.renku" "$HOME/.renku" +COPY --chown=1000:100 --from=builder /opt/conda /opt/conda +COPY --chown=1000:100 --from=builder "$HOME/.renku" "$HOME/.renku" RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku"