testing shortening PR workflow

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/build-matrices.yaml

@@ -16,44 +16,76 @@ on:
 
 jobs:
   generate:
-    name: Generate Build Matrix
+    name: Test Changed Docker Builds
     runs-on: ubuntu-latest
     outputs:
       dockerbuild_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
       empty_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix_empty }}
 
     steps:
-    # Using git in the container to diff means we explicitly need to checkout a branch
-    # if we use the default of this action we get a detached HEAD state.
+
+      # START PULL REQUEST / PUSH checkout and file derivation =======================
     - uses: actions/checkout@v2
-      if: github.event_name == 'pull_request'
+      if: github.event_name != 'schedule'
       with:
          fetch-depth: 0
-         ref: ${{ github.event.pull_request.head.ref }}
 
+    - name: Get changed files
+      id: changed-files
+      if: github.event_name != 'schedule'
+      uses: tj-actions/changed-files@32d6fb28f082f273420574e9de0a30be993bba58
+      with:
+        separator: " "
+
+    - name: Generate Build Matrix
+      if: github.event_name != 'schedule'
+      uses: vsoch/uptodate@main
+      id: dockerbuild_pr
+      with: 
+        root: ${{ steps.changed-files.outputs.all_changed_files }}
+        parser: dockerbuild
+        flags: "--registry ghcr.io/rse-ops --all"
+      # END PULL REQUEST / PUSH checkout and file derivation =======================
+
+      # START SCHEDULED checkout and file derivation =======================
     - uses: actions/checkout@v2
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'schedule'      
       with:
          fetch-depth: 0
-
     - name: Generate Build Matrix
       uses: vsoch/uptodate@main
-      id: dockerbuild
+      if: github.event_name == 'schedule'      
+      id: dockerbuild_scheduled
       with: 
         root: .
         parser: dockerbuild
-        flags: "--registry ghcr.io/rse-ops --all"
+        flags: "--registry ghcr.io/rse-ops --all"    
+      # END SCHEDULED checkout and file derivation
 
-    - name: View and Check Build Matrix Result
+    - name: View Build Matrix Result
+      id: dockerbuild
       env:
-        result: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
+        result: ${{ steps.dockerbuild_pr.outputs.dockerbuild_matrix }}
+        result_scheduled: ${{ steps.dockerbuild_scheduled.outputs.dockerbuild_matrix }}
       run: |
-        echo ${result}
-        if [[ "${result}" == "[]" ]]; then
-          printf "The matrix is empty, will not trigger next workflow.\n"
-        else
-          printf "The matrix is not empty, and we should continue on to the next workflow.\n"
-        fi
+          if [[ "${result}" == "" ]]; then
+              result="[]"
+          fi
+          if [[ "${result_scheduled}" == "" ]]; then
+              result_scheduled="[]"
+          fi
+
+          # Combine results into new output
+          result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
+          if [[ "${result}" == "[]" ]]; then
+            printf "The matrix is empty, will not trigger next workflow.\n"
+            echo "::set-output name=empty_matrix::true"
+          else
+            printf "The matrix is not empty, and we should continue on to the next workflow.\n"
+            echo "::set-output name=empty_matrix::false"
+          fi
+          echo ${result}
+          echo "::set-output name=dockerbuild_matrix::${result}"
 
   build:
     needs:
@@ -63,15 +95,15 @@ jobs:
       fail-fast: false
       matrix:
         result: ${{ fromJson(needs.generate.outputs.dockerbuild_matrix) }}
-    if: ${{ needs.generate.outputs.empty_matrix == 'false' }}
+    if: ${{ needs.generate.outputs.empty_matrx != 'false' }}
 
     name: "Build ${{ matrix.result.container_name }}"
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v2
 
     - name: GHCR Login
-      if: (github.event_name != 'pull_request')
+      if: (github.event_name != 'schedule')
       uses: docker/login-action@v1 
       with:
         registry: ghcr.io
@@ -108,7 +140,7 @@ jobs:
         echo ::set-output name=dockerfile_dir::${basedir}
 
     - name: Deploy Container
-      if: (github.event_name != 'pull_request')
+      if: (github.event_name != 'schedule')
       env:
         container: ${{ matrix.result.container_name }}
       run: |
@@ -118,8 +150,8 @@ jobs:
 
     - name: Run Librarian
       id: runner
-      if: (github.event_name != 'pull_request')
       uses: rse-ops/librarian/docker-images@main
+      if: (github.event_name != 'schedule')
       with:
           container: ${{ steps.builder.outputs.uri }} 
           dockerfile: ${{ steps.builder.outputs.dockerfile_dir }}

.github/workflows/dockerfiles.yaml

@@ -18,45 +18,75 @@ on:
     branches:
       - main
 
+
 jobs:
   update:
-    name: Update Dockerfiles
+    name: Test Updated Dockerfiles
     runs-on: ubuntu-latest
     outputs:
       dockerfile_matrix: ${{ steps.dockerfile_list.outputs.dockerfilelist_matrix }}
       empty_matrix: ${{ steps.dockerfile_list.outputs.dockerfilelist_matrix_empty }}
 
     steps:
 
+      # START PULL REQUEST / PUSH checkout and file derivation =======================
       # Using git in the container to diff means we explicitly need to checkout a branch
       # if we use the default of this action we get a detached HEAD state.
       - uses: actions/checkout@v2
-        if: github.event_name == 'pull_request'
+        if: github.event_name != 'schedule'
         with:
            fetch-depth: 0
-           ref: ${{ github.event.pull_request.head.ref }}
 
+      - name: Get changed files
+        id: changed-files
+        if: github.event_name != 'schedule'
+        uses: tj-actions/changed-files@32d6fb28f082f273420574e9de0a30be993bba58       
+        with:
+          separator: " "
+          files_ignore: .github*
+
+      - name: View Changed Files
+        if: github.event_name != 'schedule'
+        run: |
+          for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
+            echo "$file was changed"
+          done            
+
+      - name: List all Dockerfile and save to matrix
+        uses: vsoch/uptodate@main
+        if: github.event_name != 'schedule'
+        id: dockerfile_list_pr
+        with: 
+          root: ${{ steps.changed-files.outputs.all_changed_files }}
+          parser: dockerfilelist
+          flags: "--no-empty-build-args"
+      # END PULL REQUEST / PUSH checkout and file derivation =======================
+
+      # START SCHEDULED checkout and file derivation =======================
       - uses: actions/checkout@v2
-        if: github.event_name != 'pull_request'
+        if: github.event_name == 'schedule'
         with:
            fetch-depth: 0
+           ref: ${{ github.event.pull_request.head.ref }}
 
       - name: Add new Dockerfiles for build
+        if: github.event_name == 'schedule'
         uses: vsoch/uptodate@main
         with: 
           root: .
           parser: dockerhierarchy
 
       - name: Update existing Dockerfile hashes
         uses: vsoch/uptodate@main
+        if: github.event_name == 'schedule'
         id: dockerfile_check
         with: 
           root: .
           parser: dockerfile
 
       # This should not trigger a second time, since the token is not a PAT
       - name: Update Recipes
-        if: (github.event_name == 'schedule' || github.event_name == 'push')
+        if: github.event_name == 'schedule'
         env:
           result: ${{ steps.dockerfile_check.outputs.dockerfile_matrix }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -87,25 +117,41 @@ jobs:
 
       - name: List all Dockerfile and save to matrix
         uses: vsoch/uptodate@main
-        id: dockerfile_list
+        if: github.event_name == 'schedule'
+        id: dockerfile_list_scheduled
         with: 
           root: .
           parser: dockerfilelist
           flags: "--no-empty-build-args"
           changes: true # only include changed files
 
+      # END SCHEDULED checkout and file derivation =======================
+
       - name: View Build Matrix Result
+        id: dockerfile_list
         env:
-          result: ${{ steps.dockerfile_list.outputs.dockerfilelist_matrix }}
-          is_empty: ${{ steps.dockerfile_list.outputs.dockerfilelist_matrix_empty }}
+          result: ${{ steps.dockerfile_list_pr.outputs.dockerfilelist_matrix }}
+          result_scheduled: ${{ steps.dockerfile_list_scheduled.outputs.dockerfilelist_matrix }}
         run: |
-          echo ${result}
-          echo ${is_empty}
+          if [[ "${result}" == "" ]]; then
+              result="[]"
+          fi
+          if [[ "${result_scheduled}" == "" ]]; then
+              result_scheduled="[]"
+          fi
+
+          # Combine results into new output
+          result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
           if [[ "${result}" == "[]" ]]; then
             printf "The matrix is empty, will not trigger next workflow.\n"
+            echo "::set-output name=empty_matrix::true"
           else
             printf "The matrix is not empty, and we should continue on to the next workflow.\n"
+            echo "::set-output name=empty_matrix::false"
           fi
+          echo ${result}
+          echo "::set-output name=dockerfilelist_matrix::${result}"
+
 
   build:
     needs:
@@ -115,23 +161,16 @@ jobs:
       fail-fast: false
       matrix:
         result: ${{ fromJson(needs.update.outputs.dockerfile_matrix) }}
-    if: ${{ needs.update.outputs.empty_matrix == 'false' }}
+    if: ${{ needs.update.outputs.empty_matrix != 'false' }}
 
     name: "Build ${{ matrix.result.name }}"
     steps:
 
     - name: Checkout Repository
-      if: (github.event_name == 'schedule' || github.event_name == 'push')
-      uses: actions/checkout@v2
-      with:
-        ref: "main"
-
-    - name: Checkout Repository
-      if: (github.event_name == 'pull_request')
       uses: actions/checkout@v2
 
     - name: GHCR Login
-      if: (github.event_name != 'pull_request')
+      if: (github.event_name != 'schedule')
       uses: docker/login-action@v1 
       with:
         registry: ghcr.io
@@ -169,7 +208,7 @@ jobs:
         echo ::set-output name=dockerfile_dir::${basedir}
 
     - name: Deploy Container
-      if: (github.event_name != 'pull_request')
+      if: (github.event_name != 'schedule')
       env:
         uri: ${{ steps.builder.outputs.container_uri }}
       run: |
@@ -179,7 +218,7 @@ jobs:
 
     - name: Run Librarian
       id: runner
-      if: (github.event_name != 'pull_request')
+      if: (github.event_name != 'schedule')
       uses: rse-ops/librarian/docker-images@main
       with:
           container: ${{ steps.builder.outputs.uri }} 

README.md

@@ -131,6 +131,10 @@ dockerbuild:
       filter: 
         - "^[0-9]+[.]04$"
 ```
+Note that for pull requests, we only run the builds for the changed files. If you are working
+on updating a matrix image, make sure to add a tiny change to the `uptodate.yaml` so the build triggers
+(e.g., editing just the Dockerfile won't do it).
+
 For more details on creating a docker build matrix, see the [uptodate docs](https://vsoch.github.io/uptodate/docs/#/user-guide/user-guide?id=docker-build).
 
 And if there is another type of build you want to add not represented here, please [open an issue](https://github.com/rse-ops/docker-images/issues).

ubuntu/18.04/Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:18.04@sha256:c2aa13782650aa7ade424b12008128b60034c795f25456e8eb552d0a0f447cad
 
 LABEL maintainer="Josh Essman <essman1@llnl.gov>, @vsoch"
-
 ARG uptodate_github_commit_spack__spack__develop=dcdf5022ad37daee7809245cfd68eed72fde6171
 ENV spack_commit=${uptodate_github_commit_spack__spack__develop}
 ENV DEBIAN_FRONTEND=noninteractive

ubuntu/20.04/Dockerfile

@@ -1,8 +1,7 @@
 FROM ubuntu:20.04@sha256:669e010b58baf5beb2836b253c1fd5768333f0d1dbcb834f7c07a4dc93f474be
 
 LABEL maintainer="Chris White <white238@llnl.gov>,@vsoch"
-
-ARG uptodate_github_commit_spack__spack__develop=dcdf5022ad37daee7809245cfd68eed72fde6171
+ARG uptodate_github_commit_spack__spack__develop=fefe65a35b22eccffd061ced346d5017a18517e6
 ENV spack_commit=${uptodate_github_commit_spack__spack__develop}
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=America/Los_Angeles

ubuntu/22.04/Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:22.04@sha256:a02c32cf0c2a7e8743c74deef66637aa70e063c9bd40e9e1f8c0b3ea0750b0ba
 
 LABEL maintainer="@vsoch"
-
 ARG uptodate_github_commit_spack__spack__develop=dcdf5022ad37daee7809245cfd68eed72fde6171
 ENV spack_commit=${uptodate_github_commit_spack__spack__develop}
 ENV DEBIAN_FRONTEND=noninteractive

ubuntu/gcc/uptodate.yaml

@@ -16,7 +16,7 @@ dockerbuild:
        - "10.3.0"
        - "11.2.0"
 
-    # Look for ubuntu versions for our base builds
+    # Look for these ubuntu versions for our base builds
     ubuntu_version:
       key: ubuntu
       name: ghcr.io/rse-ops/ubuntu