diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml index c7a2f745..f1bcf2bc 100644 --- a/.github/workflows/build-and-push-to-docker.yml +++ b/.github/workflows/build-and-push-to-docker.yml @@ -18,8 +18,7 @@ # # It's probbaly worth noting here that the same action is used multiple # times with multiple input parameters in the `matrix` variable to -# build different versions of the container; these parameters are then -# passed on to the makefile. +# build different versions of the image. name: Renku Docker Image CI @@ -28,7 +27,6 @@ on: [push] env: DOCKER_PREFIX: renku/renkulab DEFAULT_PYTHON_VERSION: "3.10" - jobs: lint: strategy: @@ -58,12 +56,10 @@ jobs: strategy: fail-fast: true matrix: - BASE_IMAGE_TAG: - - lab-3.6.1 - - python-3.10.9 - - python-3.9.13 - - python-3.8.13 - + BASE_PYTHON_VERSION: + - "3.10" + - "3.9" + - "3.8" steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -72,49 +68,93 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Build renku project python-based docker images - id: build-image - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - - # This ensures the same image tags as before are built, in addition to new ones - if [[ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]]; then - export RENKU_PYTHON_BASE_IMAGE_TAG=$DEFAULT_PYTHON_VERSION - else - export RENKU_PYTHON_BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} - fi - - # needed by the makefile - these are generated dynamically - # and hence they are not part of the env setup above - export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }} - export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}" - export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - make py - echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT - + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-py + tags: | + type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + cache-from: type=gha + cache-to: type=gha,mode=max + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-py:${{ steps.meta.outputs.version }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts + # export the default base image for other jobs to use + - name: Build and export + if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + cache-from: type=gha + context: docker/py + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + + build-py-multiarch: + needs: build-py + runs-on: ubuntu-latest - - name: Push to docker - run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} - - # on master push latest image - the lab-* image is tagged latest - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest - docker push $DOCKER_PREFIX-py:latest - fi + strategy: + fail-fast: true + matrix: + BASE_PYTHON_VERSION: + - "3.10" + - "3.9" + - "3.8" + steps: + - name: Docker Login + uses: Azure/docker-login@v1 + with: + username: ${{ secrets.RENKU_DOCKER_USERNAME }} + password: ${{ secrets.RENKU_DOCKER_PASSWORD }} + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-py + tags: | + type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.BASE_PYTHON_VERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: docker/py + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-to: type=gha,mode=max + cache-from: type=gha + build-args: | + BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }} + platforms: linux/amd64,linux/arm64 build-cuda: needs: build-py @@ -152,54 +192,71 @@ jobs: LIBCUDNN_PACKAGE: "libcudnn8=8.5.0.96-1+cuda11.7" # note cuda 11.8 is available now, but there I could not # find a cuda-compat package in the ubuntu repos at this time - steps: - name: Docker Login uses: Azure/docker-login@v1 with: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - - uses: actions/checkout@v3 - - name: Build renku project python-based docker image extensions - id: build-image + - uses: actions/checkout@v3 + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - - # these are set dynamically so not part of env - export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" - export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}" - export CUDA_VERSION="${{ matrix.CUDA_VERSION }}" - export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}" - export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" - export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" - make cuda - echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-cuda + tags: | + type=sha,prefix=${{ matrix.CUDA_VERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.CUDA_VERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.CUDA_VERSION == '11.7' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + CUDA_COMPAT_PACKAGE=${{ matrix.CUDA_COMPAT_PACKAGE }} + CUDA_CUDART_PACKAGE=${{ matrix.CUDA_CUDART_PACKAGE }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }} + EXTRA_LIBRARIES=${{ matrix.EXTRA_LIBRARIES }} + LIBCUDNN_PACKAGE=${{ matrix.LIBCUDNN_PACKAGE }} + context: docker/cuda + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: | + type=inline + type=registry,ref=${{ steps.vars.outputs.renku_base }} - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-cuda:${{ steps.meta.outputs.version }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - name: Push to docker - run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.7" ]; then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-cuda:latest - docker push $DOCKER_PREFIX-cuda:latest - fi - + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + CUDA_COMPAT_PACKAGE=${{ matrix.CUDA_COMPAT_PACKAGE }} + CUDA_CUDART_PACKAGE=${{ matrix.CUDA_CUDART_PACKAGE }} + CUDA_VERSION=${{ matrix.CUDA_VERSION }} + EXTRA_LIBRARIES=${{ matrix.EXTRA_LIBRARIES }} + LIBCUDNN_PACKAGE=${{ matrix.LIBCUDNN_PACKAGE }} + context: docker/cuda + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + build-py-ext: needs: build-py runs-on: ubuntu-latest @@ -207,7 +264,7 @@ jobs: strategy: fail-fast: true matrix: - EXTENSIONS: + EXTENSION: - vnc - batch @@ -219,58 +276,80 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Build renku project python-based docker image extensions - env: - EXTENSION: ${{ matrix.EXTENSIONS }} + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }} + tags: | + type=sha,prefix= + type=semver,pattern={{version}},prefix= + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }},prefix= + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/${{ matrix.EXTENSION }} + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha - # needed by the makefile - for these images, we only tag with commit sha - export EXTRA_DOCKER_LABEL="${LABEL}" - make $EXTENSION - docker push $DOCKER_PREFIX-$EXTENSION:$LABEL - build-vnc-ext: needs: build-py-ext runs-on: ubuntu-latest - strategy: fail-fast: true matrix: - VNC-EXTENSIONS: + VNC-EXTENSION: - matlab - qgis - steps: - name: Docker Login uses: Azure/docker-login@v1 with: username: ${{ secrets.RENKU_DOCKER_USERNAME }} password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 - uses: actions/checkout@v3 - - name: Build renku project python-based docker image extensions - env: - EXTENSION: ${{ matrix.VNC-EXTENSIONS }} + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - docker pull ${DOCKER_PREFIX}-vnc:${LABEL} - - # needed by the Makefile - export EXTRA_DOCKER_LABEL="${LABEL}" - make vnc-$EXTENSION - docker push $DOCKER_PREFIX-$EXTENSION:$LABEL - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.VNC-EXTENSION }} + tags: | + type=sha,prefix= + type=semver,pattern={{version}},prefix= + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }},prefix= + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + BASE_IMAGE=${{ env.DOCKER_PREFIX }}-vnc:${{ steps.vars.outputs.sha_short }} + context: docker/${{ matrix.VNC-EXTENSION }} + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + build-julia-ext: needs: build-py runs-on: ubuntu-latest @@ -278,9 +357,8 @@ jobs: strategy: fail-fast: true matrix: - JULIAVERSIONS: - - 1.7.1 - + JULIAVERSION: + - "1.7.1" steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -289,54 +367,95 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Build renku project julia docker image extensions - id: build-image + - name: Set up QEMU + uses: docker/setup-qemu-action@v2.0.0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - - # needed by the makefile - export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL" - export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD) - export JULIAVERSION="${{ matrix.JULIAVERSIONS }}" - make julia - echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-julia + tags: | + type=sha,prefix=${{ matrix.JULIAVERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.JULIAVERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.JULIAVERSION == '1.7.1' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/julia + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + cache-to: type=gha - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${DOCKER_PREFIX}-julia:${{ matrix.JULIAVERSION }}-${{ steps.vars.outputs.sha_short }} TEST_USER_NAME: jovyan TEST_SPEC: jupyterlab.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - name: Push to docker - run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + context: docker/julia + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha build-r-ubuntu: needs: build-py runs-on: ubuntu-latest - + env: + RSTUDIO_VERSION: 2022.02.3-492 strategy: fail-fast: true matrix: include: - RVERSION: 4.1.0 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.1.1 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.1.2 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: 4.2.0 - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r - RVERSION: devel - RSTUDIO_VERSION: 2022.02.3-492 + BASE_IMAGE_NAME: rocker/verse + EXTENSION: r + - RVERSION: RELEASE_3_14 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc + - RVERSION: RELEASE_3_15 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc + - RVERSION: RELEASE_3_16 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc + - RVERSION: RELEASE_3_17 + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc + - RVERSION: devel + BASE_IMAGE_NAME: bioconductor/bioconductor_docker + EXTENSION: bioc steps: - name: Docker Login uses: Azure/docker-login@v1 @@ -345,86 +464,55 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v3 - - name: Build renku project rocker docker images - id: build-image + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2.0.0 + - name: Set outputs + id: vars run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - - # needed by the makefile - export RVERSION="${{ matrix.RVERSION }}" - export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL" - export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" - export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}" - make r - echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT - + sha_short=$(git rev-parse --short HEAD) + echo "sha_short=$sha_short" >> $GITHUB_OUTPUT + echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }} + tags: | + type=sha,prefix=${{ matrix.RVERSION }}- + type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}- + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' && matrix.RVERSION == 'devel' }} + - name: Build and load + uses: docker/build-push-action@v4 + with: + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + BASE_IMAGE=${{ matrix.BASE_IMAGE_NAME }}:${{ matrix.RVERSION }} + RSTUDIO_VERSION_OVERRIDE=${{ env.RSTUDIO_VERSION }} + context: docker/r + labels: ${{ steps.meta.outputs.labels }} + load: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha - name: Image Acceptance Tests uses: cypress-io/github-action@v4 env: - TEST_IMAGE_NAME: ${{ steps.build-image.outputs.IMAGE_NAME }} + TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-${{ matrix.EXTENSION }}:${{ matrix.RVERSION }}-${{ steps.vars.outputs.sha_short }} TEST_ENV: rstudio TEST_USER_NAME: rstudio TEST_SPEC: rstudio.cy.ts with: working-directory: tests command: npx mocha -r ts-node/register index.ts - - - name: Push to docker - run: | - docker push ${{ steps.build-image.outputs.IMAGE_NAME }} - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ] - then - docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-r:latest - docker push $DOCKER_PREFIX-r:latest - fi - - build-bioc: - needs: build-py - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - RELEASE: - - RELEASE_3_14 - - RELEASE_3_15 - - RELEASE_3_16 - - RELEASE_3_17 - - devel - steps: - - name: Docker Login - uses: Azure/docker-login@v1 + - name: Build and push + uses: docker/build-push-action@v4 with: - username: ${{ secrets.RENKU_DOCKER_USERNAME }} - password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - - - uses: actions/checkout@v3 - - name: Build renku project bioconductor docker images - run: | - if [[ ${{ github.ref }} == refs/tags* ]]; then - export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3) - else - export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) - fi - docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL} - - # needed by the makefile - export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL" - export BIOC_VERSION="${{ matrix.RELEASE }}" - make bioc - docker push $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL - - # on master push latest image - if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ] - then - docker tag $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL $DOCKER_PREFIX-bioc:latest - docker push $DOCKER_PREFIX-bioc:latest - fi - + build-args: | + RENKU_BASE=${{ steps.vars.outputs.renku_base }} + BASE_IMAGE=${{ matrix.BASE_IMAGE_NAME }}:${{ matrix.RVERSION }} + RSTUDIO_VERSION_OVERRIDE=${{ env.RSTUDIO_VERSION }} + context: docker/r + labels: ${{ steps.meta.outputs.labels }} + push: true + tags: ${{ steps.meta.outputs.tags }} + cache-from: type=gha + \ No newline at end of file diff --git a/Makefile b/Makefile index 62861ba5..c20f3496 100644 --- a/Makefile +++ b/Makefile @@ -27,14 +27,23 @@ extensions = \ batch \ bioc +BUILDKIT_INLINE_CACHE?=1 PLATFORM?=linux/amd64 +BUILD_CMD=build +# set the build command +ifdef USE_BUILDX + BUILD_CMD=buildx build +endif + +# use BUILDX_EXTRA_FLAGS to set either --push or --load +BUILDX_EXTRA_FLAGS?= DOCKER_PREFIX?=renku/renkulab GIT_COMMIT_SHA?=$(shell git rev-parse --short=7 --verify HEAD) # for building the base image -BASE_IMAGE_TAG?=lab-3.6.1 DEFAULT_PYTHON_VERSION?=3.10 +BASE_IMAGE_TAG?=python-$(DEFAULT_PYTHON_VERSION) PY_DOCKER_LABEL?=$(DEFAULT_PYTHON_VERSION)-$(GIT_COMMIT_SHA) RENKU_BASE?=$(DOCKER_PREFIX)-py:$(PY_DOCKER_LABEL) @@ -96,22 +105,24 @@ pull: # BASE_IMAGE_TAG: used to identify the jupyter base notebook to build from # RENKU_PYTHON_BASE_TAG: used to tag the resulting image py: - docker build docker/py \ + docker $(BUILD_CMD) docker/py \ --build-arg BASE_IMAGE=jupyter/base-notebook:$(BASE_IMAGE_TAG) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) r: py - docker build docker/r \ + docker $(BUILD_CMD) docker/r \ --build-arg BASE_IMAGE=$(RSTUDIO_BASE_IMAGE) \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --build-arg RVERSION=$(RVERSION) \ --build-arg RSTUDIO_VERSION_OVERRIDE=$(RSTUDIO_VERSION_OVERRIDE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) cuda: py - docker build docker/cuda \ + docker $(BUILD_CMD) docker/cuda \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg CUDA_COMPAT_PACKAGE="$(CUDA_COMPAT_PACKAGE)" \ --build-arg CUDA_CUDART_PACKAGE="$(CUDA_CUDART_PACKAGE)" \ @@ -119,38 +130,44 @@ cuda: py --build-arg EXTRA_LIBRARIES="$(EXTRA_LIBRARIES)" \ --build-arg LIBCUDNN_PACKAGE="$(LIBCUDNN_PACKAGE)" \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-cuda:$(CUDA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-cuda:$(CUDA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is tagged with the julia version and the commit hash julia: py - docker build docker/julia \ + docker $(BUILD_CMD) docker/julia \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is just tagged with the commit hash vnc: py - docker build docker/vnc \ + docker $(BUILD_CMD) docker/vnc \ --build-arg RENKU_BASE=$(RENKU_BASE) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) # this image is built on the vnc image and tagged as matlab with the commit hash vnc-%: vnc - docker build docker/$* \ + docker $(BUILD_CMD) docker/$* \ --build-arg BASE_IMAGE=renku/renkulab-vnc:$(GIT_COMMIT_SHA) \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) batch: py - docker build docker/batch \ + docker $(BUILD_CMD) docker/batch \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="python:3.10-slim-buster" \ - -t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) bioc: py - docker build docker/r \ + docker $(BUILD_CMD) docker/r \ --build-arg RENKU_BASE="$(RENKU_BASE)" \ --build-arg BASE_IMAGE="bioconductor/bioconductor_docker:$(BIOC_VERSION)" \ --platform=$(PLATFORM) \ - -t $(DOCKER_PREFIX)-bioc:$(BIOC_DOCKER_LABEL) + -t $(DOCKER_PREFIX)-bioc:$(BIOC_DOCKER_LABEL) \ + $(BUILDX_EXTRA_FLAGS) diff --git a/README.md b/README.md index 6d02baa5..b0f12e73 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ folder for more). Images are updated from time to time; we try to keep them reasonably current with modern python versions, cuda/torch versions and R versions. Typically, this -involves updating our github actions to include new versions and occasionally +involves updating our github actions to include new versions and updating the `Makefile` to modify what it builds by default. ## Usage @@ -64,14 +64,14 @@ the image that's right for you: `renku/renkulab-[image flavor]:[image flavor version]-[tag|hash]` For example: -`renku/renkulab-py:python-3.10-0.15.0` +`renku/renkulab-py:3.10-0.15.0` * `renku/renkulab`: indicates this is an image you can use to spawn an environment from your project on RenkuLab. * `-py`: indicates this is a python image flavor; either the programming language installed in the environment, or the base image that extra dependencies are added to. See below for details about the available flavors. -* `python-3.10`: indicates the version of python is 3.10 +* `3.10`: indicates the version of python is 3.10 * `0.15.0` (or `d572e9a`): the tag is a value given to a commit of the repository and indicates that the version is part of a release. If the version is not part of a release, this value is the first few chars of the git commit SHA from which the @@ -84,9 +84,9 @@ This can easily be overridden modifying the renku version in the project's Docke | Image | Description | Base image | |--------------------------------------------------------------------------------|-------------------------------------------------|----------------------------------------------------------------------------------------------------| -| [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | Basic Jupyter image with minimal dependencies | [jupyter/base-notebook](https://hub.docker.com/r/jupyter/base-notebook/tags) | -| [renku/renkulab-r](https://hub.docker.com/r/renku/renkulab-r/tags) | Basic Rstudio image | [rocker/verse](https://hub.docker.com/r/rocker/verse/tags) | -| [renku/renkulab-julia](https://hub.docker.com/r/renku/renkulab-julia/tags) | Basic Julia image | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | +| [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | Jupyter image with minimal dependencies | [jupyter/base-notebook](https://hub.docker.com/r/jupyter/base-notebook/tags) | +| [renku/renkulab-r](https://hub.docker.com/r/renku/renkulab-r/tags) | Rstudio image | [rocker/verse](https://hub.docker.com/r/rocker/verse/tags) | +| [renku/renkulab-julia](https://hub.docker.com/r/renku/renkulab-julia/tags) | Julia image | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | | [renku/renkulab-cuda](https://hub.docker.com/r/renku/renkulab-cuda/tags) | Cuda image with Python and minimal dependencies | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | | [renku/renkulab-cuda-tf](https://hub.docker.com/r/renku/renkulab-cuda-tf/tags) | Cuda image with Python and Tensorflow | [renku/renkulab-cuda](https://hub.docker.com/r/renku/renkulab-cuda/tags) | | [renku/renkulab-vnc](https://hub.docker.com/r/renku/renkulab-vnc/tags) | VNC Image with Python | [renku/renkulab-py](https://hub.docker.com/r/renku/renkulab-py/tags) | @@ -166,6 +166,26 @@ image you would like to build. Note that on arm-based systems (e.g. Apple M1/M2) you may need to use the flag `--platform=linux/amd64` for the build because not all base images are available for ARM architecture. +### M1/M2 (arm64) support +Starting with 0.16.0, the python base images are built to support both `X86_64` +and `arm64` architectures. If you need to build a multi-arch image locally, you can +do it with `make`: + +``` +$ PLATFORM=linux/amd64,linux/arm64 USE_BUILDX=1 make py +``` + +Alternatively you can use the `buildx` command directly: + +``` +$ docker buildx build \ + --platform linux/amd64,linux/arm64 \ + --tag myimage:tag \ + --push \ + docker/py + +Note that for the time being we can only provide the python images with `arm64` support. + ## Adding renku to your own images If you already have a Docker image with complicated dependencies that are needed diff --git a/docker/py/Dockerfile b/docker/py/Dockerfile index 8754a49a..d20b91c9 100644 --- a/docker/py/Dockerfile +++ b/docker/py/Dockerfile @@ -1,5 +1,37 @@ -ARG BASE_IMAGE=jupyter/base-notebook:lab-3.4.0 -FROM $BASE_IMAGE as base +ARG BASE_IMAGE=jupyter/base-notebook:python-3.10 + +FROM ${BASE_IMAGE} as builder + +USER root +RUN apt-get update && apt-get install -yq --no-install-recommends \ + build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# switch to the notebook user +USER $NB_USER +# install jupyterlab, papermill, git extension and renku-jupyterlab-ts +COPY requirements.txt /tmp/requirements.txt +RUN python3 -m pip install --no-cache-dir -U pip && \ + python3 -m pip install --no-cache-dir -r /tmp/requirements.txt && \ + # jupyter lab build && \ + # jupyter labextension list && \ + # npm cache clean --force && \ + rm -rf "/home/${NB_USER}/.cache" + +# jupyter sets channel priority to strict which often causes very long error messages +RUN mamba config --system --set channel_priority flexible && \ + mamba clean --all -f -y + +COPY renku-requirements/requirements.txt /tmp/renku-requirements.txt + +RUN mkdir -p "$HOME/.renku/bin" && \ + virtualenv --no-periodic-update "$HOME/.renku/venv" && \ + source "$HOME/.renku/venv/bin/activate" && \ + pip install --no-cache-dir -r /tmp/renku-requirements.txt && \ + deactivate + +FROM $BASE_IMAGE LABEL maintainer="Swiss Data Science Center " @@ -36,36 +68,8 @@ RUN apt-get update && apt-get install -yq --no-install-recommends \ /tmp/git-lfs-3.3.0/install.sh && \ rm -rf /tmp/git-lfs* -# Setup ssh keys -RUN mkdir -p /opt/ssh/sshd_config.d /opt/ssh/ssh_host_keys /opt/ssh/pid && \ - ssh-keygen -q -N "" -t dsa -f /opt/ssh/ssh_host_keys/ssh_host_dsa_key && \ - ssh-keygen -q -N "" -t rsa -b 4096 -f /opt/ssh/ssh_host_keys/ssh_host_rsa_key && \ - ssh-keygen -q -N "" -t ecdsa -f /opt/ssh/ssh_host_keys/ssh_host_ecdsa_key && \ - ssh-keygen -q -N "" -t ed25519 -f /opt/ssh/ssh_host_keys/ssh_host_ed25519_key - - -COPY sshd_config /opt/ssh/sshd_config - -RUN chown -R 0:100 /opt/ssh/ && \ - chmod -R u=rwX,g=rX,o= /opt/ssh && \ - chmod -R u=rwX,g=rwX,o= /opt/ssh/pid - -# switch to the notebook user USER $NB_USER -# install jupyterlab, papermill, git extension and renku-jupyterlab-ts -COPY requirements.txt /tmp/requirements.txt -RUN python3 -m pip install --no-cache-dir -U pip && \ - python3 -m pip install --no-cache-dir -r /tmp/requirements.txt && \ - # jupyter lab build && \ - # jupyter labextension list && \ - # npm cache clean --force && \ - rm -rf "/home/${NB_USER}/.cache" - -# jupyter sets channel priority to strict which often causes very long error messages -RUN mamba config --system --set channel_priority flexible && \ - mamba clean --all -f -y - # setup sshd RUN mkdir -p "$HOME/.ssh" && \ touch "$HOME/.ssh/authorized_keys" && \ @@ -76,15 +80,6 @@ ENV RENKU_DISABLE_VERSION_CHECK 1 ENV PATH=$HOME/.local/bin:$PATH:$HOME/.renku/bin -COPY renku-requirements/requirements.txt /tmp/renku-requirements.txt - -RUN mkdir -p "$HOME/.renku/bin" && \ - virtualenv --no-periodic-update "$HOME/.renku/venv" && \ - source "$HOME/.renku/venv/bin/activate" && \ - pip install --no-cache-dir -r /tmp/renku-requirements.txt && \ - deactivate && \ - ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" - # configure git COPY git-config.bashrc /home/$NB_USER/ RUN cat "/home/$NB_USER/git-config.bashrc" >> "/home/$NB_USER/.bashrc" && rm "/home/$NB_USER/git-config.bashrc" @@ -96,6 +91,26 @@ COPY --chown=1000:100 powerline.config /home/${NB_USER}/.config/powerline-shell/ RUN cat /tmp/powerline.bashrc >> ~/.bashrc && rm /tmp/powerline.bashrc COPY entrypoint.sh /entrypoint.sh + +# Setup ssh keys +USER root +RUN mkdir -p /opt/ssh/sshd_config.d /opt/ssh/ssh_host_keys /opt/ssh/pid && \ + ssh-keygen -q -N "" -t dsa -f /opt/ssh/ssh_host_keys/ssh_host_dsa_key && \ + ssh-keygen -q -N "" -t rsa -b 4096 -f /opt/ssh/ssh_host_keys/ssh_host_rsa_key && \ + ssh-keygen -q -N "" -t ecdsa -f /opt/ssh/ssh_host_keys/ssh_host_ecdsa_key && \ + ssh-keygen -q -N "" -t ed25519 -f /opt/ssh/ssh_host_keys/ssh_host_ed25519_key + +COPY sshd_config /opt/ssh/sshd_config + +RUN chown -R 0:100 /opt/ssh/ && \ + chmod -R u=rwX,g=rX,o= /opt/ssh && \ + chmod -R u=rwX,g=rwX,o= /opt/ssh/pid + ENTRYPOINT [ "tini", "--", "/entrypoint.sh" ] CMD [ "/usr/local/bin/start-singleuser.sh" ] + +USER $NB_USER +COPY --chown=1000:100 --from=builder /opt/conda /opt/conda +COPY --chown=1000:100 --from=builder "$HOME/.renku" "$HOME/.renku" +RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku"