ci: building custom base image

.actions/assistant.py

@@ -143,16 +143,6 @@ class AssistantCLI:
     DATASETS_FOLDER = os.environ.get("PATH_DATASETS", "_datasets")
     DRY_RUN = bool(int(os.environ.get("DRY_RUN", 0)))
     _META_REQUIRED_FIELDS = ("title", "author", "license", "description")
-    _SKIP_DIRS = (
-        ".actions",
-        ".azure",
-        ".datasets",
-        ".github",
-        "_docs",
-        "_TEMP",
-        "_requirements",
-        DIR_NOTEBOOKS,
-    )
     _META_FILE_REGEX = ".meta.{yaml,yml}"
     _META_PIP_KEY = "pip__"
     _META_ACCEL_DEFAULT = _LOCAL_ACCELERATOR.split(",")
@@ -463,6 +453,8 @@ def _replace_images(lines: list, local_dir: str) -> list:
     @staticmethod
     def _is_ipynb_parent_dir(dir_path: str) -> bool:
         """Determine in recursive fashion of a folder is valid notebook file or any of sub-folders is."""
+        if dir_path.startswith("_"):
+            return False
         if AssistantCLI._find_meta(dir_path):
             return True
         sub_dirs = [d for d in glob.glob(os.path.join(dir_path, "*")) if os.path.isdir(d)]
@@ -513,8 +505,8 @@ def group_folders(
             dirs = [os.path.join(root_path, d) for d in dirs]
         # unique folders
         dirs = set(dirs)
-        # drop folder with skip folder
-        dirs = [pdir for pdir in dirs if not any(ndir in AssistantCLI._SKIP_DIRS for ndir in pdir.split(os.path.sep))]
+        # drop folder  that start with . or _ as they are meant to be internal use only
+        dirs = [pdir for pdir in dirs if not any(ndir[0] in (".", "_") for ndir in pdir.split(os.path.sep))]
         # valid folder has meta
         dirs_exist = [d for d in dirs if os.path.isdir(d)]
         dirs_invalid = [d for d in dirs_exist if not AssistantCLI._find_meta(d)]

.github/workflows/docker-build.yml

@@ -0,0 +1,48 @@
+name: "Build (& Push) Dockers"
+
+on: # Trigger the workflow on push or pull request, but only for the main branch
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    paths:
+      - ".github/workflows/docker-build.yml"
+      - "_requirements/*"
+      - "requirements.txt"
+      - "_dockers/**"
+  workflow_dispatch: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
+env:
+  PUSH_DOCKERHUB: ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' }}
+
+jobs:
+  build-cuda:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        if: env.PUSH_DOCKERHUB == 'true' && github.repository_owner == 'Lightning-AI'
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build (and Push) image
+        uses: docker/build-push-action@v5
+        with:
+          #build-args: |
+          #  UBUNTU_VERSION=${{ matrix.ubuntu }}
+          #  PYTHON_VERSION=${{ matrix.python }}
+          #  PYTORCH_VERSION=${{ matrix.pytorch }}
+          #  CUDA_VERSION=${{ matrix.cuda }}
+          file: _dockers/ubuntu-cuda/Dockerfile
+          push: ${{ env.PUSH_DOCKERHUB }}
+          # todo: publish also tag YYYY.MM
+          tags: "pytorchlightning/tutorials"
+        timeout-minutes: 55

_dockers/README.md

@@ -0,0 +1,54 @@
+# Docker images
+
+## Build images from Dockerfiles
+
+You can build it on your own, note it takes lots of time, be prepared.
+
+```bash
+git clone https://github.com/Lightning-AI/torchmetrics.git
+
+# build with the default arguments
+docker image build -t tutorials:latest -f dockers/ubuntu-cuda/Dockerfile .
+
+# build with specific arguments
+docker image build -t tutorials:ubuntu-cuda11.7.1-py3.9-torch1.13 \
+  -f dockers/base-cuda/Dockerfile \
+  --build-arg PYTHON_VERSION=3.9 \
+  --build-arg PYTORCH_VERSION=1.13 \
+  --build-arg CUDA_VERSION=11.7.1 \
+  .
+```
+
+To run your docker use
+
+```bash
+docker image list
+docker run --rm -it tutorials:latest bash
+```
+
+and if you do not need it anymore, just clean it:
+
+```bash
+docker image list
+docker image rm tutorials:latest
+```
+
+## Run docker image with GPUs
+
+To run docker image with access to your GPUs, you need to install
+
+```bash
+# Add the package repositories
+distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
+curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add -
+curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list
+
+sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit
+sudo systemctl restart docker
+```
+
+and later run the docker image with `--gpus all`. For example,
+
+```bash
+docker run --rm -it --gpus all tutorials:ubuntu-cuda11.7.1-py3.9-torch1.12
+```

_dockers/ubuntu-cuda/Dockerfile

@@ -0,0 +1,90 @@
+# Copyright The Lightning AI team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG UBUNTU_VERSION=22.04
+ARG CUDA_VERSION=11.8.0
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
+
+ARG PYTHON_VERSION=3.10
+ARG PYTORCH_VERSION=2.0
+
+SHELL ["/bin/bash", "-c"]
+# https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/
+ENV \
+    DEBIAN_FRONTEND="noninteractive" \
+    TZ="Etc/UTC" \
+    PATH="$PATH:/root/.local/bin" \
+    CUDA_TOOLKIT_ROOT_DIR="/usr/local/cuda" \
+    MKL_THREADING_LAYER="GNU" \
+    # MAKEFLAGS="-j$(nproc)"
+    MAKEFLAGS="-j2"
+
+RUN \
+    apt-get -y update --fix-missing && \
+    apt-get install -y --no-install-recommends --allow-downgrades --allow-change-held-packages \
+        build-essential \
+        ca-certificates \
+        software-properties-common \
+        pkg-config \
+        libopenmpi-dev \
+        openmpi-bin \
+        cmake \
+        git \
+        wget \
+        curl \
+        unzip \
+        g++ \
+        cmake \
+        ffmpeg \
+        git \
+        ssh \
+        tree \
+    && \
+    # Install python
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get install -y \
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-distutils \
+        python${PYTHON_VERSION}-dev \
+    && \
+    update-alternatives --install /usr/bin/python${PYTHON_VERSION%%.*} python${PYTHON_VERSION%%.*} /usr/bin/python${PYTHON_VERSION} 1 && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python${PYTHON_VERSION} 1 && \
+    curl https://bootstrap.pypa.io/get-pip.py | python && \
+    # Cleaning
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /root/.cache && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV PYTHONPATH="/usr/lib/python${PYTHON_VERSION}/site-packages"
+
+COPY .actions/ .actions/
+COPY requirements.txt .
+COPY _requirements/ _requirements/
+
+RUN \
+    CUDA_VERSION_MM=${CUDA_VERSION%.*} && \
+    CU_VERSION_MM=${CUDA_VERSION_MM//'.'/''} && \
+    pip install --no-cache-dir -r requirements.txt \
+      --find-links "https://download.pytorch.org/whl/cu${CU_VERSION_MM}/torch_stable.html" && \
+    rm -rf requirements.txt .actions/ _requirements/
+
+RUN \
+    # Show what we have
+    pip --version && \
+    pip list && \
+    python -c "import sys; ver = sys.version_info ; assert f'{ver.major}.{ver.minor}' == '$PYTHON_VERSION', ver" && \
+    python -c "import torch; assert torch.__version__.startswith('$PYTORCH_VERSION'), torch.__version__"