Updating finding new dockerfiles and building them to test

.github/workflows/build-to-deploy.yml

@@ -51,7 +51,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check Out Repo (+ download Git LFS dependencies)  # each job runs in an isolated environment, so need to check out the repo in each job
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           lfs: true
 
@@ -61,10 +61,10 @@ jobs:
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Cache Docker layers  # also need to set up the cache in each job, using the same prefix (here ${{ runner.os }}-buildx) makes it sharable between jobs
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache-${{ inputs.cache }}
           key: ${{ runner.os }}-buildx-${{ inputs.cache }}-${{ github.sha }}
@@ -82,13 +82,13 @@ jobs:
             quay.io/${{ inputs.repository_name }}/${{ inputs.container_name }}:latest
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.docker_username }}
           password: ${{ secrets.docker_access_token }}
 
       - name: Login to Quay
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         if: ${{ inputs.push_quay }}
         with:
           registry: quay.io
@@ -97,7 +97,7 @@ jobs:
 
       - name: Build and push user-defined tag to DockerHub
         id: docker_build_user_defined_tag
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ${{ inputs.path_to_context }}
           file: ${{ inputs.path_to_context }}/${{ inputs.dockerfile_name }}
@@ -111,7 +111,7 @@ jobs:
 
       - name: Build and push latest tag to DockerHub
         id: docker_build_latest_tag
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         if: ${{ inputs.push_latest_tag }}
         with:
           context: ${{ inputs.path_to_context }}
@@ -126,7 +126,7 @@ jobs:
 
       - name: Build and push user-defined tag to Quay
         id: quay_build_user_defined_tag
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         if: ${{ inputs.push_quay }}
         with:
           context: ${{ inputs.path_to_context }}
@@ -141,7 +141,7 @@ jobs:
 
       - name: Build and push latest tag to Quay
         id: quay_build_latest_tag
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         if: ${{ inputs.push_quay && inputs.push_latest_tag }}
         with:
           context: ${{ inputs.path_to_context }}

.github/workflows/build-to-test.yml

@@ -22,14 +22,17 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check Out Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache-${{ inputs.cache }}
           key: ${{ runner.os }}-buildx-${{ inputs.cache }}-${{ github.sha }}
@@ -38,7 +41,7 @@ jobs:
 
       - name: Build to test
         id: docker_build_to_test
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ${{ inputs.path_to_context }}
           file: ${{ inputs.path_to_context }}/${{ inputs.dockerfile_name }}

.github/workflows/manual-deploy.yml

@@ -30,6 +30,8 @@ on:
         description: "Repository name. <repository>/tool:tag (Usually staphb)"
         default: "staphb"
 
+run-name: Deploy ${{ github.event.inputs.tool }} version ${{ github.event.inputs.version }}
+
 jobs:
 
   # This job calls a workflow to build the image to the 'test' stage

.github/workflows/run-singularity.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Check out Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Singularity
         uses: eWaterCycle/setup-singularity@v7

.github/workflows/test-PR-dockerfiles.yml

@@ -15,9 +15,9 @@ jobs:
     outputs:
       json: ${{ steps.files.outputs.added_modified }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - id: files
-        uses: Ana06/get-changed-files@v2.1.0
+        uses: Ana06/get-changed-files@v2.2.0
         with:
           format: 'json'
           filter: 'Dockerfile' 
@@ -31,60 +31,97 @@ jobs:
       matrix:
         added_modified: ${{ fromJson(needs.find_new_dockerfiles.outputs.json) }}
     steps:
-      - uses: actions/checkout@v2
       - name: parse file path
         id: parse
         run: |
-          tool=$(echo "${{matrix.added_modified}}" | cut -f 1 -d "/" )
-          version=$(echo "${{matrix.added_modified}}" | cut -f 2 -d "/" )
-          echo "::set-output name=tool::$tool"
-          echo "::set-output name=version::$version"
-
+          tool=$(echo    "${{ matrix.added_modified }}" | cut -f 1 -d "/" )
+          version=$(echo "${{ matrix.added_modified }}" | cut -f 2 -d "/" )
+          echo "tool=$tool"       >> $GITHUB_OUTPUT
+          echo "version=$version" >> $GITHUB_OUTPUT
+   
 ##### --------------------------------------------------------------------------- #####
-##### Attempted shortcut                                                          #####
+##### Workflows still cannot be run in parrallel as of 2022-12-01                 #####
 ##### --------------------------------------------------------------------------- #####
 
-#       - name: test
-#        uses: ./.github/workflows/build-to-test.yml
-#        with:
-#          path_to_context: "./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}"
-#          cache: ${{ steps.parse.outputs.tool }}
-# I think the ideal is to call ./.github/workflows/build-to-test.yml, but I kept getting the error:
-# Can't find 'action.yml', 'action.yaml' or 'Dockerfile' under '/home/runner/work/docker-builds/docker-builds/.github/workflows/build-to-test.yml'. Did you forget to run actions/checkout before running your local action?
-# Instead, I've created a stop-gap by copying the steps from ./.github/workflows/build-to-test.yml to here, which seems to work fine.
+#    - name: test files
+#      uses: ./.github/workflows/build-to-test.yml
+#      with:
+#        path_to_context: "./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}"
+#        cache: ${{ steps.parse.outputs.tool }}
 
 ##### --------------------------------------------------------------------------- #####
-##### The long way (which works)                                                  #####
-##### --------------------------------------------------------------------------- #####
+##### The steps of ./.github/workflows/build-to-test.yml are copied here.         #####
+##### --------------------------------------------------------------------------- ##### 
+
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Layer check
+        run: |
+          #checking layers
+          warning=''
+          app_layer=$(grep FROM ./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}/Dockerfile | grep "as app")
+          tst_layer=$(grep FROM ./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}/Dockerfile | grep "as test")
+          if [ -z "$app_layer" ] ; then echo "FATAL : app layer is missing" ; warning='warning' ; fi
+          if [ -z "$tst_layer" ] ; then echo "FATAL : test layer is missing" ; warning='warning' ; fi
+          if [ -n "$warning" ] ; then echo "Please see template for recommended format https://github.com/StaPH-B/docker-builds/blob/master/dockerfile-template/Dockerfile" ; exit 1 ; fi
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache-${{ steps.parse.outputs.tool }}
           key: ${{ runner.os }}-buildx-${{ steps.parse.outputs.tool }}-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-${{ steps.parse.outputs.tool }}
-
+      
       - name: Build to test
         id: docker_build_to_test
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}
           file: ./${{ steps.parse.outputs.tool }}/${{ steps.parse.outputs.version }}/Dockerfile
           target: test
+          tags: ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }}
+          load: true
           push: false
           cache-from: type=local,src=/tmp/.buildx-cache-${{ steps.parse.outputs.tool }}
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-${{ steps.parse.outputs.tool }}-new
 
+      - name: Check labels
+        run: |
+          # checking labels
+          warning=''
+          for label in base.image dockerfile.version software software.version description website maintainer maintainer.email
+          do
+            value=$(docker inspect --format '{{ index .Config.Labels "'$label'"}}' ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }} )
+            if [ -z "$value" ] ; then warning='warning' ; echo "FATAL : $label label not found in ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }}" ; fi
+          done
+
+          if [ -z "$(docker inspect --format '{{.Config.WorkingDir}}' ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }} )" ] ;
+          then 
+            warning='warning'
+            echo "FATAL : WORKDIR not set."
+          fi
+
+          if [ -n "$warning" ] ; then echo "Please see template for recommended format https://github.com/StaPH-B/docker-builds/blob/master/dockerfile-template/Dockerfile" ; exit 1 ; fi 
+
+      - name: Check commonly overlooked commands
+        run: |
+          # checking commands
+          docker run ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }} ps --help
+          docker run ${{ steps.parse.outputs.tool }}:${{ steps.parse.outputs.version }} update-ca-certificates -h
+
       - name: Move cache # apparently prevents the cache from growing in size forever
         run: |
           rm -rf /tmp/.buildx-cache-${{ steps.parse.outputs.tool }}
           mv /tmp/.buildx-cache-${{ steps.parse.outputs.tool }}-new /tmp/.buildx-cache-${{ steps.parse.outputs.tool }}
-
+      
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}
-

spades/3.15.4/Dockerfile

@@ -17,12 +17,13 @@ LABEL maintainer.email="kapsakcj@gmail.com"
 # python v3.8.10 is installed here; point 'python' to python3
 RUN apt-get update && apt-get install --no-install-recommends -y python3 \
  python3-distutils \
- wget && \
+ wget \
+ ca-certificates && \
  apt-get autoclean && rm -rf /var/lib/apt/lists/* && \
  update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 
 # install SPAdes binary; make /data
-RUN wget http://cab.spbu.ru/files/release${SPAdesVer}/SPAdes-${SPAdesVer}-Linux.tar.gz && \
+RUN wget https://github.com/ablab/spades/releases/download/v${SPAdesVer}/SPAdes-${SPAdesVer}-Linux.tar.gz && \
   tar -xzf SPAdes-${SPAdesVer}-Linux.tar.gz && \
   rm -r SPAdes-${SPAdesVer}-Linux.tar.gz && \
   mkdir /data