build gpu-specific docker image (#262)

* upgrade conda colab to python 3.10 * run conda colab workflow on python 3.10 * build conda docker image * set cuda version in conda Dockerfile * set nvidia env vars in conda dockerfile * print github context in docker workflow * add labels to conda dockerfiles * add tuple badge * disable pre-commit.ci PR autofix * enable CML on PR
pinellolab · May 17, 2023 · b3fd7de · b3fd7de
1 parent 1f4e08d
commit b3fd7de
Show file tree

Hide file tree

Showing 8 changed files with 105 additions and 21 deletions.
diff --git a/.github/workflows/cml.yml b/.github/workflows/cml.yml
@@ -21,6 +21,9 @@ on:
       - "reproducibility/figures/*.py"
       - "reproducibility/figures/dvc.*"
       - "reproducibility/figures/config.yaml"
+  pull_request:
+    branches:
+      - master
 
 # Review/set variables via gh CLI:
 #

diff --git a/.github/workflows/colab.yml b/.github/workflows/colab.yml
@@ -26,7 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        pyver: ["3.9"]
+        pyver: ["3.10"]
     env:
       PYTHONUNBUFFERED: True
     steps:

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -13,7 +13,10 @@ on:
 
 env:
   REGISTRY: ghcr.io
+  GAR_REGISTRY: us-central1-docker.pkg.dev
+  GAR_REPOSITORY: pyrovelocity
   IMAGE_NAME: ${{ github.repository }}
+  CONDA_IMAGE_NAME: pyrovelocitygpu
 
 jobs:
   build-and-push-image:
@@ -23,14 +26,19 @@ jobs:
       packages: write
 
     steps:
-      # - name: Maximize build space
-      #   uses: cameronraysmith/maximize-build-space@master
-      #   with:
-      #     root-reserve-mb: 128
-      #     swap-size-mb: 128
-      #     remove-dotnet: "true"
-      #     remove-android: "true"
-      #     remove-haskell: "true"
+      - name: "Print github context"
+        run: |
+          echo "      EVENT_NAME:" "$GITHUB_EVENT_NAME"
+          echo "             REF:" "$GITHUB_REF"
+          echo "       full_name: ${{ github.event.repository.full_name }}"
+          echo "      repository: ${{ github.repository }}"
+          echo "repository_owner: ${{ github.repository_owner }}"
+          echo "  default_branch: ${{ github.event.repository.default_branch }}"
+          echo "REPOSITORY_OWNER:" "$GITHUB_REPOSITORY_OWNER"
+          echo "        REF_NAME:" "$GITHUB_REF_NAME"
+          echo "        HEAD_REF:" "$GITHUB_HEAD_REF"
+          echo "        BASE_REF:" "$GITHUB_BASE_REF"
+          echo "             SHA:" "$GITHUB_SHA"
       - name: Checkout
         uses: actions/checkout@v3
       - name: Set up QEMU
@@ -52,9 +60,6 @@ jobs:
         with:
           images: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-      # - name: Run docker root relocation script
-      #   run: ./.github/mv-docker-root-dir.sh
-      #   shell: bash
       - name: Build and push
         uses: docker/build-push-action@v4
         with:
@@ -64,3 +69,65 @@ jobs:
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  build-and-push-conda-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: "Print github context"
+        run: |
+          echo "      EVENT_NAME:" "$GITHUB_EVENT_NAME"
+          echo "             REF:" "$GITHUB_REF"
+          echo "       full_name: ${{ github.event.repository.full_name }}"
+          echo "      repository: ${{ github.repository }}"
+          echo "repository_owner: ${{ github.repository_owner }}"
+          echo "  default_branch: ${{ github.event.repository.default_branch }}"
+          echo "REPOSITORY_OWNER:" "$GITHUB_REPOSITORY_OWNER"
+          echo "        REF_NAME:" "$GITHUB_REF_NAME"
+          echo "        HEAD_REF:" "$GITHUB_HEAD_REF"
+          echo "        BASE_REF:" "$GITHUB_BASE_REF"
+          echo "             SHA:" "$GITHUB_SHA"
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Log in to GitHub container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Log in to Google Artifact Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.GAR_REGISTRY }}
+          username: _json_key
+          password: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_DATA }}
+      - name: Extract github metadata for docker labels
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.REGISTRY }}/${{ github.repository_owner }}/${{ env.CONDA_IMAGE_NAME }}
+            ${{ env.GAR_REGISTRY }}/${{ secrets.GCP_PROJECT_ID }}/${{ env.GAR_REPOSITORY }}/${{ env.CONDA_IMAGE_NAME }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=sha,prefix=,format=long
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./dockerfiles/Dockerfile.conda
+          platforms: linux/amd64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -37,7 +37,7 @@ ci:
     [pre-commit.ci] auto fixes from pre-commit.com hooks
 
     for more information, see https://pre-commit.ci
-  autofix_prs: true
+  autofix_prs: false
   autoupdate_branch: ""
   autoupdate_commit_msg: "[pre-commit.ci] pre-commit autoupdate"
   autoupdate_schedule: weekly

diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@
 | CI/CD   | [![CI - Test](https://github.com/pinellolab/pyrovelocity/actions/workflows/tests.yml/badge.svg)](https://github.com/pinellolab/pyrovelocity/actions/workflows/tests.yml) [![CML](https://github.com/pinellolab/pyrovelocity/actions/workflows/cml.yml/badge.svg)](https://github.com/pinellolab/pyrovelocity/actions/workflows/cml.yml) [![pre-commit.ci status](https://results.pre-commit.ci/badge/github/pinellolab/pyrovelocity/master.svg)](https://results.pre-commit.ci/latest/github/pinellolab/pyrovelocity/master) |
 | Docs    | [![Documentation Status](https://readthedocs.org/projects/pyrovelocity/badge/?version=latest)](https://pyrovelocity.readthedocs.io/en/latest/?badge=latest) [![Preprint](https://img.shields.io/badge/doi-10.1101/2022.09.12.507691v2-B31B1B)](https://doi.org/10.1101/2022.09.12.507691)                                                                                                                                                                                                                                    |
 | Package | [![PyPI - Version](https://img.shields.io/pypi/v/pyrovelocity.svg?logo=pypi&label=PyPI&logoColor=gold)](https://pypi.org/project/pyrovelocity/) [![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pyrovelocity.svg?logo=python&label=Python&logoColor=gold)](https://pypi.org/project/pyrovelocity/) [![Docker iamge](https://img.shields.io/badge/docker-image-blue?logo=docker)](https://github.com/pinellolab/pyrovelocity/pkgs/container/pyrovelocity)                                                    |
-| Meta    | [![codecov](https://codecov.io/gh/pinellolab/pyrovelocity/branch/master/graph/badge.svg)](https://codecov.io/gh/pinellolab/pyrovelocity) [![code style - Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License - MIT](https://img.shields.io/badge/license-AGPL%203-purple)](https://spdx.org/licenses/)                                                                                                                                                              |
+| Meta    | [![codecov](https://codecov.io/gh/pinellolab/pyrovelocity/branch/master/graph/badge.svg)](https://codecov.io/gh/pinellolab/pyrovelocity) [![code style - Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) [![License - MIT](https://img.shields.io/badge/license-AGPL%203-purple)](https://spdx.org/licenses/) [![Tuple](https://img.shields.io/badge/Tuple%20❤️%20OSS-5A67D8?logo=tuple)](https://tuple.app/github-badge)                                                  |
 
 </div>
 

diff --git a/conda/colab/construct.yaml b/conda/colab/construct.yaml
@@ -6,7 +6,7 @@ channels:
   - bioconda
   - nodefaults
 specs:
-  - python=3.9
+  - python=3.10
   - pip
   - conda
   - mamba # mamba is not needed but recommended

diff --git a/dockerfiles/Dockerfile.conda b/dockerfiles/Dockerfile.conda
@@ -2,9 +2,18 @@
 FROM condaforge/mambaforge:23.1.0-1
 LABEL maintainer="pyrovelocity team"
 
+ARG CONDA_OVERRIDE_CUDA=11.8
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64
+
 WORKDIR /pyrovelocity
 COPY . .
 
-RUN mamba init
-RUN mamba env update -n base -f conda/environment-gpu.yml
-RUN pip install --no-deps -e .
+LABEL org.opencontainers.image.title="pyrovelocitygpu" \
+      org.opencontainers.image.authors="pyrovelocity team" \
+      org.opencontainers.image.description="This image contains the pyrovelocity library with dependencies necessary for execution on NVIDIA GPUs." \
+      org.opencontainers.image.url="https://github.com/pinellolab/pyrovelocity" \
+      org.opencontainers.image.licenses="AGPL-3.0-only"
+
+RUN mamba env update -n base -f conda/environment-gpu.yml && \
+    pip install --no-deps -e .
diff --git a/dockerfiles/Dockerfile.conda.cpu b/dockerfiles/Dockerfile.conda.cpu
@@ -5,6 +5,11 @@ LABEL maintainer="pyrovelocity team"
 WORKDIR /pyrovelocity
 COPY . .
 
-RUN mamba init
-RUN mamba env update -n base -f conda/environment-cpu.yml
-RUN pip install --no-deps -e .
+LABEL org.opencontainers.image.title="pyrovelocitycpu" \
+      org.opencontainers.image.authors="pyrovelocity team" \
+      org.opencontainers.image.description="This image contains the pyrovelocity library with dependencies necessary for execution on CPUs." \
+      org.opencontainers.image.url="https://github.com/pinellolab/pyrovelocity" \
+      org.opencontainers.image.licenses="AGPL-3.0-only"
+
+RUN mamba env update -n base -f conda/environment-cpu.yml && \
+    pip install --no-deps -e .