Merge pull request #69 from rse-ops/shorten-workflows

testing shortening PR workflow

.github/scripts/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+basedir=$(dirname ${result_name})
+printf "Base directory is ${basedir}\n"
+dockerfile=$(basename ${result_name})
+printf "Dockerfile basename is ${dockerfile}\n"
+tag=$(basename ${basedir})
+printf "Tag is ${tag}\n"
+container=$(basename $(dirname $basedir))          
+printf "Container is ${container}\n"
+cat ${result_name}
+cd $basedir
+container_name=ghcr.io/rse-ops/${container}:${tag}
+docker pull ${container_name} || echo "Container $container_name does not exist yet"
+docker build -f ${dockerfile} -t ${container_name} .
+echo ::set-output name=container_uri::${container_name}
+echo ::set-output name=uri::ghcr.io/rse-ops/${container}
+echo ::set-output name=tag::${tag}
+echo ::set-output name=dockerfile_dir::${basedir}

.github/scripts/build_simple.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+echo $PWD
+basedir=$(dirname $filename)
+printf "Base directory is ${basedir}\n"
+# Get relative path to PWD and generate dashed name from it
+cd $basedir
+echo "${prefix} -t ${container} ."
+${prefix} -t ${container} .
+echo ::set-output name=uri::${container}
+echo ::set-output name=dockerfile_dir::${basedir}

.github/scripts/combine_results.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${result}" == "" ]]; then
+  result="[]"
+fi
+
+if [[ "${result_scheduled}" == "" ]]; then
+  result_scheduled="[]"
+fi
+
+# Combine results into new output
+result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
+if [[ "${result}" == "[]" ]]; then
+  printf "The matrix is empty, will not trigger next workflow.\n"
+  echo "::set-output name=empty_matrix::true"
+else
+  printf "The matrix is not empty, and we should continue on to the next workflow.\n"
+  echo "::set-output name=empty_matrix::false"
+fi
+echo ${result}
+
+# set for both workflows that use it
+echo "::set-output name=dockerfilelist_matrix::${result}"
+echo "::set-output name=dockerbuild_matrix::${result}"

.github/scripts/commit.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+printf "GitHub Actor: ${GITHUB_ACTOR}\n"
+git config user.name "github-actions"
+git config user.email "github-actions@users.noreply.github.com"
+git fetch || printf "fetch did not work\n"
+git checkout main || printf "Already on main!\n"
+
+# Add all results!
+for row in $(echo "${result}" | jq -r '.[] | @base64'); do
+  _jq() {
+    echo ${row} | base64 --decode | jq -r ${1};}
+    git add $(_jq '.name')
+done
+
+set +e
+git status | grep modified
+if [ $? -eq 0 ]; then
+  set -e
+  printf "Changes\n"
+  git commit -a -m "Automated push to update ${result_name} $(date '+%Y-%m-%d')" || exit 0
+  git push origin main
+else
+ set -e
+ printf "No changes\n"
+fi

.github/scripts/prepare_paths.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+parsed=""
+for file in ${changed_files}; do
+    filename=$(basename ${file})
+
+    # We are going to have paths inside/outside container and need relative
+    if [[ "$filename" == "uptodate.yaml" ]]; then
+        parsed="${parsed} ${file}"
+    fi
+    # If we find a Dockerfile, change to uptodate.yaml if it exists
+    if [[ "$filename" == "Dockerfile" ]]; then
+        dir=$(dirname ${file})
+        uptodate_yaml=${dir}/uptodate.yaml
+        if [[ -e "${uptodate_yaml}" ]]; then
+            parsed="${parsed} ${uptodate_yaml}"
+        fi
+    fi
+done
+echo ${parsed}
+
+# No parsed results will build ALL
+if [[ "${parsed}" == "" ]]; then
+    parsed="/does/not/exist/pathy"
+fi
+echo "::set-output name=parsed_files::${parsed}"

.github/scripts/relative_path.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+echo "Original path is $filename"
+relative=$(echo ${filename/\/github\/workspace\//})
+echo "Relative path is $relative"
+echo "::set-output name=relative_path::${relative}"

.github/workflows/build-matrices.yaml

@@ -16,44 +16,65 @@ on:
 
 jobs:
   generate:
-    name: Generate Build Matrix
+    name: Test Changed Docker Builds
     runs-on: ubuntu-latest
     outputs:
       dockerbuild_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
       empty_matrix: ${{ steps.dockerbuild.outputs.dockerbuild_matrix_empty }}
 
     steps:
-    # Using git in the container to diff means we explicitly need to checkout a branch
-    # if we use the default of this action we get a detached HEAD state.
+
+      # START PULL REQUEST / PUSH checkout and file derivation =======================
     - uses: actions/checkout@v2
-      if: github.event_name == 'pull_request'
+      if: github.event_name != 'schedule'
       with:
          fetch-depth: 0
-         ref: ${{ github.event.pull_request.head.ref }}
 
+    - name: Get changed files
+      id: changed-files
+      if: github.event_name != 'schedule'
+      uses: tj-actions/changed-files@32d6fb28f082f273420574e9de0a30be993bba58
+      with:
+        separator: " "
+
+    - name: Convert changed Dockerfile to uptodate
+      if: github.event_name != 'schedule'
+      id: parsed_files
+      env:
+        changed_files: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: /bin/bash .github/scripts/prepare_paths.sh
+
+    - name: Generate Build Matrix
+      if: github.event_name != 'schedule'
+      uses: vsoch/uptodate@main
+      id: dockerbuild_pr
+      with: 
+        root: ${{ steps.parsed_files.outputs.parsed_files }}
+        parser: dockerbuild
+        flags: "--registry ghcr.io/rse-ops --all"
+      # END PULL REQUEST / PUSH checkout and file derivation =======================
+
+      # START SCHEDULED checkout and file derivation =======================
     - uses: actions/checkout@v2
-      if: github.event_name != 'pull_request'
+      if: github.event_name == 'schedule'      
       with:
          fetch-depth: 0
-
     - name: Generate Build Matrix
       uses: vsoch/uptodate@main
-      id: dockerbuild
+      if: github.event_name == 'schedule'      
+      id: dockerbuild_scheduled
       with: 
         root: .
         parser: dockerbuild
-        flags: "--registry ghcr.io/rse-ops --all"
+        flags: "--registry ghcr.io/rse-ops --all"    
+      # END SCHEDULED checkout and file derivation
 
-    - name: View and Check Build Matrix Result
+    - name: View Build Matrix Result
+      id: dockerbuild
       env:
-        result: ${{ steps.dockerbuild.outputs.dockerbuild_matrix }}
-      run: |
-        echo ${result}
-        if [[ "${result}" == "[]" ]]; then
-          printf "The matrix is empty, will not trigger next workflow.\n"
-        else
-          printf "The matrix is not empty, and we should continue on to the next workflow.\n"
-        fi
+        result: ${{ steps.dockerbuild_pr.outputs.dockerbuild_matrix }}
+        result_scheduled: ${{ steps.dockerbuild_scheduled.outputs.dockerbuild_matrix }}
+      run: /bin/bash .github/scripts/combine_results.sh
 
   build:
     needs:
@@ -63,7 +84,7 @@ jobs:
       fail-fast: false
       matrix:
         result: ${{ fromJson(needs.generate.outputs.dockerbuild_matrix) }}
-    if: ${{ needs.generate.outputs.empty_matrix == 'false' }}
+    if: ${{ needs.generate.outputs.empty_matrix == false }}
 
     name: "Build ${{ matrix.result.container_name }}"
     steps:
@@ -91,35 +112,30 @@ jobs:
         container: ${{ matrix.result.container_name }}
       run: docker pull ${container} || exit 0
 
+    - name: Update to relative path
+      id: relative_path
+      env:
+        filename: ${{ matrix.result.filename }}
+      run: /bin/bash .github/scripts/relative_path.sh
+
     - name: Build ${{ matrix.result.container_name }}
       id: builder
       env:
         container: ${{ matrix.result.container_name }}
         prefix: ${{ matrix.result.command_prefix }}
-        filename: ${{ matrix.result.filename }}
-      run: |
-        basedir=$(dirname $filename)
-        printf "Base directory is ${basedir}\n"
-        # Get relative path to PWD and generate dashed name from it
-        cd $basedir
-        echo "${prefix} -t ${container} ."
-        ${prefix} -t ${container} .
-        echo ::set-output name=uri::${container}
-        echo ::set-output name=dockerfile_dir::${basedir}
+        filename: ${{ steps.relative_path.outputs.relative_path }}
+      run: /bin/bash .github/scripts/build_simple.sh
 
     - name: Deploy Container
       if: (github.event_name != 'pull_request')
       env:
         container: ${{ matrix.result.container_name }}
-      run: |
-        docker images
-        printf "docker push ${container}\n"
-        docker push ${container}
+      run: docker push ${container}
 
     - name: Run Librarian
       id: runner
-      if: (github.event_name != 'pull_request')
       uses: rse-ops/librarian/docker-images@main
+      if: (github.event_name != 'pull_request')
       with:
           container: ${{ steps.builder.outputs.uri }} 
           dockerfile: ${{ steps.builder.outputs.dockerfile_dir }}