test running scripts instead

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/scripts/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+basedir=$(dirname ${result_name})
+printf "Base directory is ${basedir}\n"
+dockerfile=$(basename ${result_name})
+printf "Dockerfile basename is ${dockerfile}\n"
+tag=$(basename ${basedir})
+printf "Tag is ${tag}\n"
+container=$(basename $(dirname $basedir))          
+printf "Container is ${container}\n"
+cat ${result_name}
+cd $basedir
+container_name=ghcr.io/rse-ops/${container}:${tag}
+docker pull ${container_name} || echo "Container $container_name does not exist yet"
+docker build -f ${dockerfile} -t ${container_name} .
+echo ::set-output name=container_uri::${container_name}
+echo ::set-output name=uri::ghcr.io/rse-ops/${container}
+echo ::set-output name=tag::${tag}
+echo ::set-output name=dockerfile_dir::${basedir}

.github/scripts/build_simple.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+echo $PWD
+basedir=$(dirname $filename)
+printf "Base directory is ${basedir}\n"
+# Get relative path to PWD and generate dashed name from it
+cd $basedir
+echo "${prefix} -t ${container} ."
+${prefix} -t ${container} .
+echo ::set-output name=uri::${container}
+echo ::set-output name=dockerfile_dir::${basedir}

.github/scripts/combine_results.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+if [[ "${result}" == "" ]]; then
+  result="[]"
+fi
+
+if [[ "${result_scheduled}" == "" ]]; then
+  result_scheduled="[]"
+fi
+
+# Combine results into new output
+result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
+if [[ "${result}" == "[]" ]]; then
+  printf "The matrix is empty, will not trigger next workflow.\n"
+  echo "::set-output name=empty_matrix::true"
+else
+  printf "The matrix is not empty, and we should continue on to the next workflow.\n"
+  echo "::set-output name=empty_matrix::false"
+fi
+echo ${result}
+
+# set for both workflows that use it
+echo "::set-output name=dockerfilelist_matrix::${result}"
+echo "::set-output name=dockerbuild_matrix::${result}"

.github/scripts/commit.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+printf "GitHub Actor: ${GITHUB_ACTOR}\n"
+git config user.name "github-actions"
+git config user.email "github-actions@users.noreply.github.com"
+git fetch || printf "fetch did not work\n"
+git checkout main || printf "Already on main!\n"
+
+# Add all results!
+for row in $(echo "${result}" | jq -r '.[] | @base64'); do
+  _jq() {
+    echo ${row} | base64 --decode | jq -r ${1};}
+    git add $(_jq '.name')
+done
+
+set +e
+git status | grep modified
+if [ $? -eq 0 ]; then
+  set -e
+  printf "Changes\n"
+  git commit -a -m "Automated push to update ${result_name} $(date '+%Y-%m-%d')" || exit 0
+  git push origin main
+else
+ set -e
+ printf "No changes\n"
+fi

.github/scripts/prepare_paths.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -e
+parsed=""
+for file in ${changed_files}; do
+    filename=$(basename ${file})
+
+    # We are going to have paths inside/outside container and need relative
+    if [[ "$filename" == "uptodate.yaml" ]]; then
+        parsed="${parsed} ${file}"
+    fi
+    # If we find a Dockerfile, change to uptodate.yaml if it exists
+    if [[ "$filename" == "Dockerfile" ]]; then
+        dir=$(dirname ${file})
+        uptodate_yaml=${dir}/uptodate.yaml
+        if [[ -e "${uptodate_yaml}" ]]; then
+            parsed="${parsed} ${uptodate_yaml}"
+        fi
+    fi
+done
+echo ${parsed}
+
+# No parsed results will build ALL
+if [[ "${parsed}" == "" ]]; then
+    parsed="/does/not/exist/pathy"
+fi
+echo "::set-output name=parsed_files::${parsed}"

.github/scripts/relative_path.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+echo "Original path is $filename"
+relative=$(echo ${filename/\/github\/workspace\//})
+echo "Relative path is $relative"
+echo "::set-output name=relative_path::${relative}"

.github/workflows/build-matrices.yaml

@@ -40,31 +40,9 @@ jobs:
     - name: Convert changed Dockerfile to uptodate
       if: github.event_name != 'schedule'
       id: parsed_files
-      run: |
-         parsed=""
-         for file in ${{ steps.changed-files.outputs.all_changed_files }}; do
-            filename=$(basename ${file})
-
-            # We are going to have paths inside/outside container and need relative
-            if [[ "$filename" == "uptodate.yaml" ]]; then
-                    parsed="${parsed} ${file}"
-            fi
-            # If we find a Dockerfile, change to uptodate.yaml if it exists
-            if [[ "$filename" == "Dockerfile" ]]; then
-                dir=$(dirname ${file})
-                uptodate_yaml=${dir}/uptodate.yaml
-                if [[ -e "${uptodate_yaml}" ]]; then
-                    parsed="${parsed} ${uptodate_yaml}"
-                fi
-            fi
-         done
-         echo ${parsed}
-
-         # No parsed results will build ALL
-         if [[ "${parsed}" == "" ]]; then
-             parsed="/does/not/exist/pathy"
-         fi
-         echo "::set-output name=parsed_files::${parsed}"
+      env:
+        changed_files: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: /bin/bash .github/scripts/prepare_paths.sh
 
     - name: Generate Build Matrix
       if: github.event_name != 'schedule'
@@ -96,25 +74,7 @@ jobs:
       env:
         result: ${{ steps.dockerbuild_pr.outputs.dockerbuild_matrix }}
         result_scheduled: ${{ steps.dockerbuild_scheduled.outputs.dockerbuild_matrix }}
-      run: |
-          if [[ "${result}" == "" ]]; then
-              result="[]"
-          fi
-          if [[ "${result_scheduled}" == "" ]]; then
-              result_scheduled="[]"
-          fi
-
-          # Combine results into new output
-          result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
-          if [[ "${result}" == "[]" ]]; then
-            printf "The matrix is empty, will not trigger next workflow.\n"
-            echo "::set-output name=empty_matrix::true"
-          else
-            printf "The matrix is not empty, and we should continue on to the next workflow.\n"
-            echo "::set-output name=empty_matrix::false"
-          fi
-          echo ${result}
-          echo "::set-output name=dockerbuild_matrix::${result}"
+      run: /bin/bash .github/scripts/combine_results.sh
 
   build:
     needs:
@@ -156,37 +116,21 @@ jobs:
       id: relative_path
       env:
         filename: ${{ matrix.result.filename }}
-      run: |
-         echo "Original path is $filename"
-         relative=$(echo ${filename/\/github\/workspace\//})
-         echo "Relative path is $relative"
-         echo "::set-output name=relative_path::${relative}"
+      run: /bin/bash .github/scripts/relative_path.sh
 
     - name: Build ${{ matrix.result.container_name }}
       id: builder
       env:
         container: ${{ matrix.result.container_name }}
         prefix: ${{ matrix.result.command_prefix }}
         filename: ${{ steps.relative_path.outputs.relative_path }}
-      run: |
-        echo $PWD
-        basedir=$(dirname $filename)
-        printf "Base directory is ${basedir}\n"
-        # Get relative path to PWD and generate dashed name from it
-        cd $basedir
-        echo "${prefix} -t ${container} ."
-        ${prefix} -t ${container} .
-        echo ::set-output name=uri::${container}
-        echo ::set-output name=dockerfile_dir::${basedir}
+      run: /bin/bash .github/scripts/build_simple.sh
 
     - name: Deploy Container
       if: (github.event_name != 'pull_request')
       env:
         container: ${{ matrix.result.container_name }}
-      run: |
-        docker images
-        printf "docker push ${container}\n"
-        docker push ${container}
+      run: docker push ${container}
 
     - name: Run Librarian
       id: runner

.github/workflows/dockerfiles.yaml

@@ -90,30 +90,8 @@ jobs:
         env:
           result: ${{ steps.dockerfile_check.outputs.dockerfile_matrix }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          printf "GitHub Actor: ${GITHUB_ACTOR}\n"
-          git config user.name "github-actions"
-          git config user.email "github-actions@users.noreply.github.com"
-          git fetch || printf "fetch did not work\n"
-          git checkout main || printf "Already on main!\n"
-
-          # Add all results!
-          for row in $(echo "${result}" | jq -r '.[] | @base64'); do
-            _jq() {
-              echo ${row} | base64 --decode | jq -r ${1};}
-            git add $(_jq '.name')
-          done
-          set +e
-          git status | grep modified
-          if [ $? -eq 0 ]; then
-              set -e
-              printf "Changes\n"
-              git commit -a -m "Automated push to update ${{ matrix.result.name }} $(date '+%Y-%m-%d')" || exit 0
-              git push origin main
-          else
-              set -e
-              printf "No changes\n"
-          fi
+          result_name: ${{ matrix.result.name }}
+        run: /bin/bash .github/scripts/commit.sh
 
       - name: List all Dockerfile and save to matrix
         uses: vsoch/uptodate@main
@@ -132,26 +110,7 @@ jobs:
         env:
           result: ${{ steps.dockerfile_list_pr.outputs.dockerfilelist_matrix }}
           result_scheduled: ${{ steps.dockerfile_list_scheduled.outputs.dockerfilelist_matrix }}
-        run: |
-          if [[ "${result}" == "" ]]; then
-              result="[]"
-          fi
-          if [[ "${result_scheduled}" == "" ]]; then
-              result_scheduled="[]"
-          fi
-
-          # Combine results into new output
-          result=$(python -c "import json; print(json.loads('${result}') + json.loads('${result_scheduled}'))")
-          if [[ "${result}" == "[]" ]]; then
-            printf "The matrix is empty, will not trigger next workflow.\n"
-            echo "::set-output name=empty_matrix::true"
-          else
-            printf "The matrix is not empty, and we should continue on to the next workflow.\n"
-            echo "::set-output name=empty_matrix::false"
-          fi
-          echo ${result}
-          echo "::set-output name=dockerfilelist_matrix::${result}"
-
+        run: /bin/bash .github/scripts/combine_results.sh
 
   build:
     needs:
@@ -187,25 +146,10 @@ jobs:
 
     # This currently has a lot of extra prints for debugging
     - name: Build ${{ matrix.result.name }}
-      id: builder
-      run: |
-        basedir=$(dirname ${{ matrix.result.name }})
-        printf "Base directory is ${basedir}\n"
-        dockerfile=$(basename ${{ matrix.result.name }})
-        printf "Dockerfile basename is ${dockerfile}\n"
-        tag=$(basename ${basedir})
-        printf "Tag is ${tag}\n"
-        container=$(basename $(dirname $basedir))          
-        printf "Container is ${container}\n"
-        cat ${{ matrix.result.name }}
-        cd $basedir
-        container_name=ghcr.io/rse-ops/${container}:${tag}
-        docker pull ${container_name} || echo "Container $container_name does not exist yet"
-        docker build -f ${dockerfile} -t ${container_name} .
-        echo ::set-output name=container_uri::${container_name}
-        echo ::set-output name=uri::ghcr.io/rse-ops/${container}
-        echo ::set-output name=tag::${tag}
-        echo ::set-output name=dockerfile_dir::${basedir}
+      env:
+        result_name: ${{ matrix.result.name }}
+      id: builder   
+      run: /bin/bash .github/scripts/build.sh
 
     - name: Deploy Container
       if: (github.event_name != 'pull_request')

ubuntu/18.04/Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:18.04@sha256:c2aa13782650aa7ade424b12008128b60034c795f25456e8eb552d0a0f447cad
 
 LABEL maintainer="Josh Essman <essman1@llnl.gov>, @vsoch"
-
 ARG uptodate_github_commit_spack__spack__develop=dcdf5022ad37daee7809245cfd68eed72fde6171
 ENV spack_commit=${uptodate_github_commit_spack__spack__develop}
 ENV DEBIAN_FRONTEND=noninteractive

ubuntu/cuda/uptodate.yaml

@@ -19,7 +19,6 @@ dockerbuild:
     cuda_version:
       key: cuda
       name: cuda
-
     ubuntu_version:
       key: ubuntu
       name: ubuntu