Merge pull request #65 from devilbox/update-multistage

Use new workflow files
devilbox · Nov 16, 2022 · 1828ae3 · 1828ae3
2 parents 13b5bdb + df949db
commit 1828ae3
Show file tree

Hide file tree

Showing 5 changed files with 165 additions and 98 deletions.
diff --git a/.github/workflows/docker-multistage-build.yml b/.github/workflows/docker-multistage-build.yml
@@ -35,18 +35,33 @@ on:
         description: 'Dertermines whether we upload the artifact not.'
         required: true
         type: boolean
-      pull_base_image:
-        description: 'Pull Docker base image before building?.'
+      push_image:
+        description: 'Push docker image after build (and test if ran)?'
         required: false
+        type: boolean
         default: false
+      pull_base_image:
+        description: 'Pull Docker base image before building?'
+        required: false
         type: boolean
+        default: false
+    ###
+    ### Secrets
+    ###
+    secrets:
+      dockerhub_username:
+        description: 'The username for Dockerhub.'
+        required: true
+      dockerhub_password:
+        description: 'The password for Dockerhub.'
+        required: true
 
 jobs:
   # -----------------------------------------------------------------------------------------------
   # JOB: BUILD
   # -----------------------------------------------------------------------------------------------
   build:
-    name: ${{ matrix.name }}-${{ matrix.version }}-${{ inputs.stage }} (${{ matrix.arch }}) ${{ matrix.refs }}
+    name: ${{ matrix.NAME }}-${{ matrix.VERSION }}-${{ inputs.stage }} (${{ matrix.ARCH }}) ${{ matrix.REFS }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -63,11 +78,11 @@ jobs:
           fetch-depth: 0
         if: ${{ !inputs.has_refs }}
 
-      - name: "[SETUP] Checkout repository (ref: ${{ matrix.refs }})"
+      - name: "[SETUP] Checkout repository (ref: ${{ matrix.REFS }})"
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-          ref: ${{ matrix.refs }}
+          ref: ${{ matrix.REFS }}
         if: ${{ inputs.has_refs }}
 
       - name: "[SETUP] Install QEMU environment"
@@ -77,40 +92,32 @@ jobs:
           image: tonistiigi/binfmt:latest
           platforms: all
 
-      - name: "[SETUP] Install Docker Buildx"
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: "[SETUP] Set artifact names"
         id: set-artifact-name
         run: |
-          PRE_HASH="$( git rev-parse HEAD | head -c 10 )"
-          VERSION="${{ matrix.version }}"
-          ARCH="$( echo "${{ matrix.arch }}" | sed 's|/|-|g' )"
+          VERSION="$( echo "${{ matrix.VERSION }}" )"
+          ARCH="$( echo "${{ matrix.ARCH }}" | sed 's|/|-|g' )"
 
-          NAME_PREV="${{ inputs.artifact_prefix }}-${PRE_HASH}-${VERSION}-${ARCH}-${{ inputs.stage_prev }}"
-          NAME_CURR="${{ inputs.artifact_prefix }}-${PRE_HASH}-${VERSION}-${ARCH}-${{ inputs.stage }}"
+          NAME_PREV="${{ inputs.artifact_prefix }}-${VERSION}-${ARCH}-${{ inputs.stage_prev }}"
+          NAME_CURR="${{ inputs.artifact_prefix }}-${VERSION}-${ARCH}-${{ inputs.stage }}"
           echo "prev=${NAME_PREV}" >> $GITHUB_OUTPUT
           echo "curr=${NAME_CURR}" >> $GITHUB_OUTPUT
 
-      - name: "[DEBUG] Show setup"
-        shell: bash
+      - name: "[SETUP] Determine Docker tag"
+        id: tag
+        uses: cytopia/docker-tag-action@v0.4.22
+
+      # https://github.com/alpinelinux/docker-alpine/issues/98
+      - name: "[SETUP] Fix Docker IP forwarding"
         run: |
-          echo '(qemu)   Platforms: ${{ steps.qemu.outputs.platforms }}'
-          echo "(buildx) Name:      ${{ steps.buildx.outputs.name }}"
-          echo "(buildx) Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
-          echo "(buildx) Status:    ${{ steps.buildx.outputs.status }}"
-          echo "(buildx) Flags:     ${{ steps.buildx.outputs.flags }}"
-          echo "(buildx) Platforms: ${{ steps.buildx.outputs.platforms }}"
+          sysctl net.ipv4.ip_forward
+          sudo sysctl -w net.ipv4.ip_forward=1
+          sudo systemctl restart docker
 
 
       # ------------------------------------------------------------
       # Artifact Import
       # ------------------------------------------------------------
-
-      ###
-      ### Download and import previously built image (if it exists)
-      ###
       - name: "[Artifact Load] Download previously built image"
         uses: cytopia/download-artifact-retry-action@v0.1.4
         with:
@@ -132,14 +139,14 @@ jobs:
         uses: cytopia/shell-command-retry-action@v0.1.5
         with:
           command: |
-            make docker-pull-base-image VERSION=${{ matrix.version }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.flavour }} ARCH=${{ matrix.arch }}
+            make docker-pull-base-image VERSION=${{ matrix.VERSION }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.FLAVOUR }} ARCH=${{ matrix.ARCH }}
         if: ${{ inputs.pull_base_image }}
 
       - name: Build
         uses: cytopia/shell-command-retry-action@v0.1.5
         with:
           command: |
-            make build VERSION=${{ matrix.version }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.flavour }} ARCH=${{ matrix.arch }}
+            make build VERSION=${{ matrix.VERSION }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.FLAVOUR }} ARCH=${{ matrix.ARCH }}
 
 
       # ------------------------------------------------------------
@@ -149,24 +156,44 @@ jobs:
         uses: cytopia/shell-command-retry-action@v0.1.5
         with:
           command: |
-            make test VERSION=${{ matrix.version }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.flavour }} ARCH=${{ matrix.arch }}
+            make test VERSION=${{ matrix.VERSION }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.FLAVOUR }} ARCH=${{ matrix.ARCH }}
         if: ${{ inputs.run_tests }}
 
 
       # ------------------------------------------------------------
-      # Artifact Export
+      # Push
       # ------------------------------------------------------------
+      - name: Docker Tag
+        uses: cytopia/shell-command-retry-action@v0.1.5
+        with:
+          command: |
+            make tag VERSION="${{ matrix.VERSION }}" STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.FLAVOUR }} TAG=${{ steps.tag.outputs.docker-tag }}
 
-      ###
-      ### Export current image
-      ###
+      - name: Docker login
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.dockerhub_username }}
+          password: ${{ secrets.dockerhub_password }}
+        if: ${{ inputs.push_image }}
+
+      - name: Docker push
+        uses: cytopia/shell-command-retry-action@v0.1.5
+        with:
+          command: |
+            make push VERSION="${{ matrix.version }}" STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.flavour }} ARCH=${{ matrix.arch }} TAG=${{ steps.tag.outputs.docker-tag }}
+        if: ${{ inputs.push_image }}
+
+
+      # ------------------------------------------------------------
+      # Artifact Export
+      # ------------------------------------------------------------
       - name: "[Artifact Save] Upload built artifact"
         uses: cytopia/upload-artifact-retry-action@v0.1.6
         with:
           name: ${{ steps.set-artifact-name.outputs.curr }}
           path: ${{ steps.set-artifact-name.outputs.curr }}
           pre_command: |
-            make save-verify VERSION=${{ matrix.version }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.flavour }} ARCH=${{ matrix.arch }} OUTFILE=${{ steps.set-artifact-name.outputs.curr }} INFILE=${{ steps.set-artifact-name.outputs.curr }}
+            make save-verify VERSION=${{ matrix.VERSION }} STAGE=${{ inputs.stage }} FLAVOUR=${{ matrix.FLAVOUR }} ARCH=${{ matrix.ARCH }} OUTFILE=${{ steps.set-artifact-name.outputs.curr }} INFILE=${{ steps.set-artifact-name.outputs.curr }}
           post_command: |
             make load INFILE={{download_path}}
         if: ${{ inputs.upload_artifact }}
diff --git a/.github/workflows/docker-multistage-configure.yml b/.github/workflows/docker-multistage-configure.yml
@@ -23,10 +23,16 @@ on:
         description: 'Determines whether this workflow can deploy (login and push).'
         required: true
         type: boolean
-      is_scheduled:
-        description: "Usage for a scheduled job? Only then we will evaluate git refs and add them to build matrix."
-        required: true
-        type: boolean
+      fields_build:
+        description: 'The JSON string for build fields to extract'
+        required: false
+        type: string
+        default: '[{"VERSION":"VERSION"}, {"FLAVOUR":"FLAVOUR"}, {"ARCH":"ARCH"}]'
+      fields_deploy:
+        description: 'The JSON string for deploy fields to extract'
+        required: false
+        type: string
+        default: '[{"VERSION":"VERSION"}, {"FLAVOUR":"FLAVOUR"}, {"HELM":"EXTRA"}]'
 
     ###
     ### Input Secrets
@@ -57,10 +63,10 @@ on:
       can_push:
         description: "(boolean) Can we push to Dockerhub?"
         value: ${{ jobs.configure.outputs.can_push }}
+      # Determined settings (values)
       has_refs:
-        description: "(boolean) Do we have refs? (used for nightly builds)"
+        description: "(string) 'true' or 'false' Do we have refs to build?"
         value: ${{ jobs.configure.outputs.has_refs }}
-      # Determined settings (values)
       matrix_build:
         description: "(string) The determined JSON string build matrix."
         value: ${{ jobs.configure.outputs.matrix_build }}
@@ -91,9 +97,10 @@ jobs:
       # Flags
       can_login: ${{ steps.set-login.outputs.can_login }}
       can_push: ${{ steps.set-push.outputs.can_push }}
+      # Settings
       has_refs: ${{ steps.set-refs.outputs.has_refs }}
-      # Values
       artifact_prefix: ${{ steps.set-artifact-prefix.outputs.prefix }}
+      # Values
       matrix_build: ${{ steps.set-matrix.outputs.matrix_build }}
       matrix_deploy: ${{ steps.set-matrix.outputs.matrix_deploy }}
 
@@ -135,6 +142,10 @@ jobs:
             echo "can_push=false" >> $GITHUB_OUTPUT
           fi
 
+      # ------------------------------------------------------------
+      # Set values
+      # ------------------------------------------------------------
+
       ###
       ### Do we have refs to build against?
       ###
@@ -154,20 +165,18 @@ jobs:
           echo "BRANCHES=${BRANCHES}"
           echo "NUM_LATEST_TAGS=${NUM_LATEST_TAGS}"
 
-      - name: "[Set-Output] has_refs and matrix (ref-matrix) (branches and latest tags)"
+      ###
+      ### Set Refs
+      ###
+      - name: "[Set-Output] has_refs and ref-matrix"
         id: set-refs
-        uses: cytopia/git-ref-matrix-action@v0.1.12
+        uses: cytopia/git-ref-matrix-action@v0.1.13
         with:
           repository_default_branch: ${{ steps.eval-refs.outputs.default_branch }}
           branches: ${{ steps.eval-refs.outputs.branches }}
           num_latest_tags: ${{ steps.eval-refs.outputs.num_latest_tags }}
           # Only use refs if the job is a schedule or if the job is schedule and was triggered manually
-          disable_refs: ${{ !(inputs.is_scheduled && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }}
-
-
-      # ------------------------------------------------------------
-      # Set values
-      # ------------------------------------------------------------
+          disable_refs: ${{ !(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
 
       ###
       ### Unique Artifact prefix
@@ -176,9 +185,9 @@ jobs:
         id: set-artifact-prefix
         shell: bash
         run: |
-          PRE_DATE="$( date +"%s" )"
-          PRE_RAND="$( echo $RANDOM | md5sum | head -c 10 )"
-          echo "prefix=${PRE_DATE}-${PRE_RAND}" >> $GITHUB_OUTPUT
+          PRE_RUN="$( echo '${{ github.run_id }}' )"
+          PRE_SHA="$( echo '${{ github.sha }}' | md5sum | head -c 10 )"
+          echo "prefix=${PRE_RUN}-${PRE_SHA}" >> $GITHUB_OUTPUT
 
       ###
       ### Set Build and Deploy Matrix
@@ -187,32 +196,52 @@ jobs:
         id: set-matrix
         shell: bash
         run: |
+          FIELDS_BUILD="$(  echo  '${{ inputs.fields_build }}' | jq -M -c )"
+          FIELDS_DEPLOY="$( echo '${{ inputs.fields_deploy }}' | jq -M -c )"
+          VERSIONS="$(      echo '${{ inputs.versions }}'      | jq -M -c )"
+
           if [ "${{ steps.set-refs.outputs.has_refs }}" = "true" ]; then
-            MATRIX_BUILD="$( \
-              jq -M -c \
-                --argjson refs '${{ steps.set-refs.outputs.matrix }}' \
-                'map({name:.NAME, version:.VERSION[], flavour:.FLAVOUR[], arch:.ARCH[], refs:$refs[]})' <<<'${{ inputs.versions }}' \
-            )"
-            MATRIX_DEPLOY="$( \
-              jq -M -c \
-                --argjson refs '${{ steps.set-refs.outputs.matrix }}' \
-                'map({name:.NAME, version:.VERSION[], flavour:.FLAVOUR[], refs:$refs[]})' <<<'${{ inputs.versions }}' \
-            )"
-            echo "matrix_build=${MATRIX_BUILD}"   >> $GITHUB_OUTPUT
-            echo "matrix_deploy=${MATRIX_DEPLOY}" >> $GITHUB_OUTPUT
-            echo "has_refs=true"                  >> $GITHUB_OUTPUT
+            REFS="$( echo '${{ steps.set-refs.outputs.matrix }}' | jq -M -c )"
+
+            # map({NAME: .NAME, VERSION: .VERSION[], FLAVOUR: .FLAVOUR[], ARCH:. ARCH[], ...})
+            INNER_STRING="$( echo "${FIELDS_BUILD}" \
+              | jq -r '.[] | to_entries[] | "\(.value): .\(.key)[]"' \
+              | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' \
+              | jq -r '. | join(", ")' )"
+            OUTER_STRING="$( echo "'map({NAME: .NAME, ${INNER_STRING}, REFS: ${REFS}[]})'" )"
+            MATRIX_BUILD="$( eval jq -M -c  "${OUTER_STRING}" <<<"${VERSIONS}" )"
+
+            # map({NAME: .NAME, VERSION: .VERSION[], FLAVOUR: .FLAVOUR[], ....})
+            INNER_STRING="$( echo "${FIELDS_DEPLOY}" \
+              | jq -r '.[] | to_entries[] | "\(.value): .\(.key)[]"' \
+              | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' \
+              | jq -r '. | join(", ")' )"
+            OUTER_STRING="$( echo "'map({NAME: .NAME, ${INNER_STRING}, REFS: ${REFS}[]})'" )"
+            MATRIX_DEPLOY="$( eval jq -M -c "${OUTER_STRING}" <<<"${VERSIONS}" )"
+
+            echo "matrix_build=${MATRIX_BUILD}"             >> $GITHUB_OUTPUT
+            echo "matrix_deploy=${MATRIX_DEPLOY}"           >> $GITHUB_OUTPUT
+
           else
-            MATRIX_BUILD="$( \
-              jq -M -c \
-                'map({name:.NAME, version:.VERSION[], flavour:.FLAVOUR[], arch:.ARCH[]})' <<<'${{ inputs.versions }}' \
-            )"
-            MATRIX_DEPLOY="$( \
-              jq -M -c \
-                'map({name:.NAME, version:.VERSION[], flavour:.FLAVOUR[]})' <<<'${{ inputs.versions }}' \
-            )"
-            echo "matrix_build=${MATRIX_BUILD}"   >> $GITHUB_OUTPUT
-            echo "matrix_deploy=${MATRIX_DEPLOY}" >> $GITHUB_OUTPUT
-            echo "has_refs=false"                 >> $GITHUB_OUTPUT
+
+            # map({NAME: .NAME, VERSION: .VERSION[], FLAVOUR: .FLAVOUR[], ARCH:. ARCH[], ...})
+            INNER_STRING="$( echo "${FIELDS_BUILD}" \
+              | jq -r '.[] | to_entries[] | "\(.value): .\(.key)[]"' \
+              | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' \
+              | jq -r '. | join(", ")' )"
+            OUTER_STRING="$( echo "'map({NAME: .NAME, ${INNER_STRING}})'" )"
+            MATRIX_BUILD="$( eval jq -M -c "${OUTER_STRING}" <<<"${VERSIONS}" )"
+
+            # map({NAME: .NAME, VERSION: .VERSION[], FLAVOUR: .FLAVOUR[], ....})
+            INNER_STRING="$( echo "${FIELDS_DEPLOY}" \
+              | jq -r '.[] | to_entries[] | "\(.value): .\(.key)[]"' \
+              | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' \
+              | jq -r '. | join(", ")' )"
+            OUTER_STRING="$( echo "'map({NAME: .NAME, ${INNER_STRING}})'" )"
+            MATRIX_DEPLOY="$( eval jq -M -c "${OUTER_STRING}" <<<"${VERSIONS}" )"
+
+            echo "matrix_build=${MATRIX_BUILD}"             >> $GITHUB_OUTPUT
+            echo "matrix_deploy=${MATRIX_DEPLOY}"           >> $GITHUB_OUTPUT
           fi
 
       # ------------------------------------------------------------
@@ -232,6 +261,8 @@ jobs:
           echo '-------------------------------------|-----------------------------------'
           echo 'github.event_name                    | ${{ github.event_name }}'
           echo '-------------------------------------|-----------------------------------'
+          echo 'github.ref                           | ${{ github.ref }}'
+          echo '-------------------------------------|-----------------------------------'
           echo 'github.ref_name                      | ${{ github.ref_name }}'
           echo '-------------------------------------|-----------------------------------'
           echo 'github.head_ref                      | ${{ github.head_ref }}'
@@ -241,14 +272,20 @@ jobs:
           echo 'github.ref_type                      | ${{ github.ref_type }}'
           echo '-------------------------------------|-----------------------------------'
           echo 'github.repository                    | ${{ github.repository }}'
+          echo '-------------------------------------|-----------------------------------'
+          echo 'github.action                        | ${{ github.action }}'
+          echo '-------------------------------------|-----------------------------------'
+          echo 'github.action_ref                    | ${{ github.action_ref }}'
+          echo '-------------------------------------|-----------------------------------'
+          echo 'github.run_id                        | ${{ github.run_id }}'
+          echo '-------------------------------------|-----------------------------------'
+          echo 'github.sha                           | ${{ github.sha }}'
 
       - name: "[DEBUG] Show Workflow Inputs"
         shell: bash
         run: |
           echo 'enabled: ${{ inputs.enabled }}'
           echo 'can_deploy: ${{ inputs.can_deploy }}'
-          echo 'is_scheduled: ${{ inputs.is_scheduled }}'
-          echo 'can_deploy: ${{ inputs.can_deploy }}'
           echo 'versions: ${{ inputs.versions }}'
           echo 'refs: ${{ inputs.refs }}'
 
@@ -261,6 +298,7 @@ jobs:
           # Flags
           echo 'can_login: ${{ steps.set-login.outputs.can_login }}'
           echo 'can_push: ${{ steps.set-push.outputs.can_push }}'
+          # Settins
           echo 'has_refs: ${{ steps.set-refs.outputs.has_refs }}'
           # Values
           echo 'artifact_prefix: ${{ steps.set-artifact-prefix.outputs.prefix }}'