Merge pull request #112 from robertej09/dev-gha

Addressing Issue #109
Unidata · Nov 14, 2022 · 7f91873 · 7f91873
2 parents cbe65cb + faaa41f
commit 7f91873
Show file tree

Hide file tree

Showing 4 changed files with 187 additions and 61 deletions.
diff --git a/.github/helperScripts/README.desiredImages b/.github/helperScripts/README.desiredImages
@@ -0,0 +1,23 @@
+The "desiredImages" file is a plain-text file where each line contains an
+Extended Regular Expression (ERE) for a tag of the official tomcat image on
+DockerHub, followed by a space (" ") and either a 1 or a 0, denoting whether or
+not that pariticular tag is additionally tracked as the "latest" tag. For
+example:
+
+^10\.[0-9]{1,}\.[0-9]{1,}-jdk17-openjdk$ 0
+
+Is the tag for tomcat version 10.x.y using the jdk version 17 version of
+openjdk. The 0 at the end of the line indicates this won't be pushed to either
+GitHub or Docker as "latest."
+
+Note the "^" and "$" at the beginning and end of the ERE, respecitively. These
+are necessary to ensure specificity of the tag. Namely, while you may be wanting
+to specify the tag "10.0.12-jdk17-openjdk" with the ERE
+10\.[0-9]{1,}\.[0-9]{1,}-jdk17-openjdk, this would also match the following
+tags:
+
+10.0.12-jdk17-openjdk-slim-buster
+10.0.12-jdk17-openjdk-slim-bullseye
+10.0.12-jdk17-openjdk-slim
+10.0.12-jdk17-openjdk-buster
+10.0.12-jdk17-openjdk-bullseye
diff --git a/.github/helperScripts/desiredImages b/.github/helperScripts/desiredImages
@@ -0,0 +1,3 @@
+^8\.[0-9]{1,}\.[0-9]{1,}-jdk11-openjdk$ 1
+^10\.[0-9]{1,}\.[0-9]{1,}-jdk17-openjdk$ 0
+^10\.[0-9]{1,}\.[0-9]{1,}-jdk17-temurin-focal$ 0
diff --git a/.github/helperScripts/dockertags.sh b/.github/helperScripts/dockertags.sh
@@ -1,30 +1,99 @@
 #!/bin/bash
 
-# Shamelessly taken from StackOverflow. No need to reinvent the wheel right?
-# https://stackoverflow.com/questions/28320134/how-can-i-list-all-tags-for-a-docker-image-on-a-remote-registry
+USAGE="
+dockertags  --  list all tags for a Docker image in the DockerHub registry \n
+\n
+INPUTS: \n
+./dockertags -u|--user <user> \ \n
+    -p|--password <password> \ \n
+    -n|--namespace <namespace> \ \n
+    -i|--image <image> \ \n
+    [ -h|--help ] \n
+\n
+--user: username used for authentication to dockerhub registry \n
+--password: password or personal access token \n
+--namespace: organization name (e.g. "unidata"); "library" for public repos \n
+--image: the name of the image \n
+--help: print this help message and exit \n
+\n
+OUTPUTS: \n
+The tags of the desired image, each on a new line. For example: \n
+\n
+$ ./dockertags --user $USERNAME \ \n
+    --password $PASSWORD \ \n
+    --namespace library \ \n
+    --image $IMAGE \n
+\n
+tag1 \n
+tag2 \n
+tag3 \n
+"
 
-if [ $# -lt 1 ]
-then
-cat << HELP
+while [[ $# > 0 ]]
+do
+    key="$1"
+    case $key in
+        -u|--user)
+            USERNAME="$2"
+            shift # past argument
+            ;;
+        -p|--password)
+            PASSWORD="$2"
+            shift # past argument
+            ;;
+        -n|--namespace)
+            NAMESPACE="$2"
+            shift # past argument
+            ;;
+        -i|--image)
+            IMAGE="$2"
+            shift # past argument
+            ;;
+        -h|--help)
+            echo -e $USAGE
+            exit
+            ;;
+    esac
+    shift # past argument or value
+done
 
-dockertags  --  list all tags for a Docker image on a remote registry.
+# Check if all values were set
+[[ -z "$USERNAME" ]] && { echo -e "Must supply a username...Exiting" $USAGE; exit 1; }
+[[ -z "$PASSWORD" ]] && { echo -e "Must supply a password...Exiting" $USAGE; exit 1; }
+[[ -z "$NAMESPACE" ]] && { echo -e "Must supply a namespace...Exiting" $USAGE; exit 1; }
+[[ -z "$IMAGE" ]] && { echo -e "Must supply an image...Exiting" $USAGE; exit 1; }
 
-EXAMPLE: 
-    - list all tags for ubuntu:
-       dockertags ubuntu
+# Grab a token for authentiting when querying the registry
+token=$(curl -Ls -X POST https://hub.docker.com/v2/users/login \
+    -H 'Content-Type: application/json' \
+    -d "{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\"}" | jq -r .token)
 
-    - list all php tags containing apache:
-       dockertags php apache
+# Query the registry for the first 100 entries (the maximum page size)
+RESPONSE=$(curl -Ls -G \
+    --data-urlencode "page_size=100" \
+    --data-urlencode "page=1" \
+    -H "Authorization: Bearer ${token}" \
+    https://hub.docker.com/v2/namespaces/${NAMESPACE}/repositories/${IMAGE}/tags)
 
-HELP
-fi
+# Parse using jq
+TAGS=$(echo $RESPONSE | jq '.results[].name')
 
-image="$1"
-tags=`wget -q https://registry.hub.docker.com/v1/repositories/${image}/tags -O -  | sed -e 's/[][]//g' -e 's/"//g' -e 's/ //g' | tr '}' '\n'  | awk -F: '{print $3}'`
+# If more than 100 tags exist in the repository, we must loop through all pages
+# using the "next" field of the JSON response
+let PAGE=1
+NEXTURL=$(echo $RESPONSE | jq '.next')
 
-if [ -n "$2" ]
-then
-    tags=` echo "${tags}" | grep "$2" `
-fi
+while [[ "${NEXTURL}" != "null" ]];
+do
+    let PAGE+=1
+    RESPONSE=$(curl -Ls -G \
+        --data-urlencode "page_size=100" \
+        --data-urlencode "page=$PAGE" \
+        -H "Authorization: Bearer ${token}" \
+        https://hub.docker.com/v2/namespaces/${NAMESPACE}/repositories/${IMAGE}/tags)
+        TAGS+=" $(echo $RESPONSE | jq '.results[].name')"
+    NEXTURL=$(echo $RESPONSE | jq '.next')
+done
 
-echo "${tags}"
+# Print each tag on its own line and remove quotes (") from around each tag
+printf '%s\n' $TAGS | sed -e 's/"//g'
diff --git a/.github/workflows/checkUpstream.yml b/.github/workflows/checkUpstream.yml
@@ -14,34 +14,59 @@ jobs:
   createMatrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      matrix: ${{ steps.set-matrix.outputs.desiredImages }}
+      dockertags: ${{ steps.set-dockertags.outputs.dockertags }}
 
     steps:
       - name: Checkout default branch
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
-      - name: Create matrix
-        id: set-matrix
+      - name: Set environment variables
+        run: |
+          echo "upstream_image=tomcat" >> $GITHUB_ENV
+          echo "scriptsdir=.github/helperScripts" >> $GITHUB_ENV
+
+      # Funny workaround to output a multiline string to another job
+      # Pipe the multiline string into jq to create a minified JSON array where
+      # each element is an individual line.
+      # This minified array is output to GitHub Actions as a single line, can be
+      # read by another job, and parsed back to a multiline string using jq
+      - name: Fetch all tags and set as Github Actions output
+        id: set-dockertags
         run: |
-          matrix=$(
-          echo {\
-            $(.github/helperScripts/tomcatVersions.sh)", "\
-            $(.github/helperScripts/jdkVersions.sh)\
-          }\
+          dockertags=$(\
+            ${{ env.scriptsdir }}/dockertags.sh \
+            --user ${{ secrets.registryuser }} \
+            --password ${{ secrets.registrypwd }} \
+            --namespace library \
+            --image ${{ env.upstream_image }} \
+            | jq -cR '[., inputs]' \
           )
-          echo $matrix
-          echo "::set-output name=matrix::$matrix"
-
-  verifyMatrix:
-    runs-on: ubuntu-latest
-    needs: createMatrix
-    steps:
-      - name: Verify matrix
+          echo "dockertags=$dockertags" >> $GITHUB_OUTPUT
+
+      # Create a minified JSON object from the desiredImages file (see
+      # README.desiredImages for a description of its contents)
+      # The JSON object has the form:
+      # {
+      #   tag: [
+      #     {
+      #       "ERE": <ERE>,
+      #       "isLatest": <0|1>
+      #     },
+      #   ]
+      # }
+      - name: Create matrix output
+        id: set-matrix
         run: |
-          matrix='${{ needs.createMatrix.outputs.matrix }}'
-          echo $matrix
+          echo "desiredImages=$(
+            jq -cnR '{
+              "tag": [
+                inputs/" " | {"ERE": .[0], "isLatest": .[1]}
+              ]
+            }' ${{ env.scriptsdir }}/desiredImages \
+          )" >> $GITHUB_OUTPUT
 
   checkUpstream:
     runs-on: ubuntu-latest
@@ -60,28 +85,25 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Verify repo has been cloned
+    - name: Check the most recent upstream tomcat version
       run: |
-        git status
-        git branch -a
-        git remote -v
-
-    - name: Check matrix outputs
-      run: |
-        echo "Tomcat: ${{ matrix.tomcat }} JDK: ${{ matrix.jdk }}"
-
-    - name: Grab the most recent upstream tomcat version
-      run: |
-        upstream=$( ${{ env.scriptsdir }}/dockertags.sh tomcat | ${{ env.scriptsdir }}/findTomcatVersion.sh ${{matrix.tomcat}} ${{ matrix.jdk }})
+        upstream=$(\
+          echo '${{ needs.createMatrix.outputs.dockertags }}' \
+          | jq -r '.[]' \
+          | grep -E -e '${{ matrix.tag.ERE }}' \
+          | sort -Vr \
+          | head -n 1 \
+        )
         echo $upstream
         echo "upstream=$upstream" >> $GITHUB_ENV
 
-    - name: Check if up to date
+    - name: Check if a branch on the repo is up to date
       run: |
-        echo Latest upstream version: ${{ env.upstream }}
-        echo "Current branches (tomcat: ${{ matrix.tomcat }}, jdk: ${{ matrix.jdk }}):"
-        git branch --list -r "origin/${{ matrix.tomcat }}.*-jdk${{ matrix.jdk }}-openjdk"
-        current=$( git branch --list -r "origin/${{ matrix.tomcat }}.*-jdk${{ matrix.jdk }}-openjdk" | sort -Vr | awk -F "/" '{ print $2 }' | head -n1 )
+        current=$(git branch --list -r "origin/*" \
+        | sed 's|^ *origin/||g' \
+        | grep -E -e '${{ matrix.tag.ERE }}' \
+        | sort -Vr \
+        | head -n 1)
         echo Most current branch: $current
         test "$current" = "${{ env.upstream }}" &&
         up2date=true || up2date=false
@@ -94,7 +116,7 @@ jobs:
       run: |
         echo "Already up to date with upstream: ${{ env.upstream }}"
 
-    - name: Checkout branch
+    - name: Checkout new branch
       if: ${{ env.up2date != 'true' }}
       run: |
         echo "tag=${{ env.upstream }}" >> $GITHUB_ENV
@@ -139,7 +161,9 @@ jobs:
 
     - name: Run test script
       if: ${{ env.up2date != 'true' }}
-      run: ./.github/testScripts/test.sh
+      run: |
+        ./.github/testScripts/test.sh \
+        && rm -rf $(pwd)/.github/testScripts/{sample.war,sample,actual.html}
 
     - name: Push to git
       if: ${{ env.up2date != 'true' }}
@@ -148,15 +172,22 @@ jobs:
         git config --global user.email $GITHUB_ACTOR@users.noreply.github.com
         git add . && git commit -m "Update to tomcat:${{ env.upstream }}" && \
         git push origin ${{ env.upstream }}
-        git push origin ${{ env.upstream }}:latest
 
     - name: Push to dockerhub
       if: ${{ env.up2date != 'true' }}
       run: |
         docker logout
         echo ${{ secrets.registrypwd }} | docker login -u ${{ secrets.registryuser }} --password-stdin
         docker push ${{ secrets.imagename }}:${{ env.tag }} && \
-        docker tag ${{ secrets.imagename }}:${{ env.tag }} ${{ secrets.imagename }}:latest && \
-        docker push ${{ secrets.imagename }}:latest && \
-        { docker logout && echo "Successfully pushed ${{ secrets.imagename }}:${{ env.tag }} and ${{ secrets.imagename }}:latest"; } ||
+        { docker logout && echo "Successfully pushed ${{ secrets.imagename }}:${{ env.tag }} to dockerhub"; } ||
         { docker logout && echo "Docker push failed" && exit 1; }
+
+    - name: Create PR targetting latest
+      if: ${{ env.up2date != 'true' }}
+      run: |
+        if [[ "${{ matrix.tag.isLatest }}" == "1" ]];
+        then
+          gh pr create --title "Update to tomcat: ${{ env.upstream }}" --body "PR created by GitHub Actions"
+        fi
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}