test to add automated build for containers (#1)

* update dockerfile to have fewer layers
* use ubuntu base and flux base for efa

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/scripts/prepare_paths.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+parsed=""
+for file in ${changed_files}; do
+    filename=$(basename ${file})
+
+    # In this case, we just have one uptodate per recipe!
+    if [[ "$filename" == "Dockerfile" ]]; then
+        folder=$(dirname ${file})
+        ignorefile=${folder}/.uptodate-ignore
+
+        # Ignore these directories
+        if [[ ! -f "$ignorefile" ]]; then
+            parsed="${parsed} ${folder}"
+        fi
+    fi
+done
+echo ${parsed}
+
+# No parsed results will build ALL
+if [[ "${parsed}" == "" ]]; then
+    parsed="/does/not/exist/pathy"
+fi
+
+echo "parsed_files=${parsed}" >> $GITHUB_OUTPUT

.github/workflows/build-matrices.yaml

@@ -0,0 +1,127 @@
+name: Container Build Matrices
+
+on:
+  pull_request: []
+  push:
+    branches:
+    - main
+  workflow_dispatch:
+
+jobs:
+  generate:
+    name: Generate Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      dockerbuild_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
+      empty_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix_empty }}
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@aa52cfcd81f1a00a6bf1241a8cad6adec4d80638 # v33
+      with:
+        separator: " "
+
+    - name: Convert changed Dockerfile to uptodate
+      id: parsed_files
+      env:
+        changed_files: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: /bin/bash .github/scripts/prepare_paths.sh
+
+    - name: Generate Build Matrix
+      uses: vsoch/uptodate@main
+      id: dockerbuild
+      with: 
+        root: ${{ steps.parsed_files.outputs.parsed_files }}
+        parser: dockerbuild
+        flags: "--registry ghcr.io/converged-computing --all"
+
+    - name: View and Check Build Matrix Result
+      env:
+        result: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
+      run: |
+        echo ${result}
+        if [[ "${result}" == "[]" ]]; then
+          printf "The matrix is empty, will not trigger next workflow.\n"
+        else
+          printf "The matrix is not empty, and we should continue on to the next workflow.\n"
+        fi
+
+  build:
+    needs:
+    - generate
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        result: ${{ fromJson(needs.generate.outputs.dockerbuild_matrix) }}
+        arch: [linux/amd64]
+
+    if: ${{ needs.generate.outputs.empty_matrix == 'false' }}
+
+    name: Build ${{ matrix.result.container_name }} ${{ matrix.arch[0] }}
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v3
+    - uses: actions/setup-go@v3
+
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+   # Uncomment if need more build space   
+   # - name: Run Actions Cleaner
+   #   uses: rse-ops/actions-cleaner/ubuntu@main
+
+    - name: Pull Docker Layers
+      env:
+        container: ${{ matrix.result.container_name }}
+      run: docker pull ghcr.io/converged-computing/${container} || exit 0
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Prepare ${{ matrix.result.container_name }}
+      id: builder
+      env:
+        container: ${{ matrix.result.container_name }}
+        prefix: ${{ matrix.result.command_prefix }}
+        filename: ${{ matrix.result.filename }}
+      run: |
+        basedir=$(dirname $filename)
+        container=$(python -c "print('${container}'.strip('-'))")
+        printf "Base directory is ${basedir}\n"
+        # Get relative path to PWD and generate dashed name from it
+        cd $basedir
+        echo "${prefix} -t ${container} ."
+        build_args="$(echo "${prefix#*--build-arg}")"
+        # Add build-arg for anaconda download
+        echo "dockerfile_dir=${basedir}" >> $GITHUB_ENV
+        echo "build_args=${build_args}" >> $GITHUB_ENV
+        echo "container=${container}" >> $GITHUB_ENV
+        echo "filename=${filename}" >> $GITHUB_ENV
+
+    - name: Build ${{ matrix.dockerfile[1] }}
+      uses: docker/build-push-action@v3
+      with:
+        context: ${{ env.dockerfile_dir }}
+        file: ${{ env.filename }}
+        platforms: ${{ matrix.arch[0] }}
+        push: ${{ github.event_name != 'pull_request' }}
+        build-args: |
+          ${{ env.build_args }}
+        tags: ${{ env.container }}

nsdf-materialscience/Dockerfile

@@ -1,16 +1,19 @@
-ARG centos_version=latest
-FROM centos:${centos_version}
+ARG ubuntu_version=20.04
+FROM ghcr.io/rse-ops/spack-ubuntu-libfabric:ubuntu-${ubuntu_version} as builder
 
-RUN cd /etc/yum.repos.d/
-RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+ENV FI_EFA_USE_DEVICE_RDMA=1
+ENV RDMAV_FORK_SAFE=1
+ENV DEBIAN_FRONTEND=noninteractive
 
 
-WORKDIR /home/${USER}
+WORKDIR /code
 
 # Python packages and script
-COPY preprocess_radiographs.py /home/${USER}/.
-RUN yum install -y mesa-libGL
-RUN yum install -y python3
-RUN python3 -m pip install -U --force-reinstall pip
+COPY preprocess_radiographs.py /code
+
+RUN apt-get update && apt-get install -y \
+    libgl1-mesa-glx \
+    python3 \
+    python3-pip
+
 RUN python3 -m pip install --user numpy multiprocess opencv-python 

nsdf-materialscience/uptodate.yaml

@@ -2,6 +2,6 @@ dockerbuild:
 
   build_args:
     ubuntu_version:
-      key: centos
+      key: ubuntu
       versions:
-       - "latest"
+       - "20.04"