Merge pull request #19 from docksal/develop

Release 1.4.0
docksal · Nov 5, 2021 · 6f5dd00 · 6f5dd00
2 parents 7c042c1 + 185b733
commit 6f5dd00
Show file tree

Hide file tree

Showing 8 changed files with 436 additions and 76 deletions.
diff --git a/.github/scripts/docker-tag-delete.sh b/.github/scripts/docker-tag-delete.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Deletes an image tag from Docker Hub
+#
+# Expects USER, PASSWORD
+# Expects IMAGE:TAG as argument.
+#
+# Example: docker-tag-delete.sh docksal/cli:php7.3-build-01c92a2-amd64
+
+# Credit:
+# https://devopsheaven.com/docker/dockerhub/2018/04/09/delete-docker-image-tag-dockerhub.html
+
+set -euo pipefail
+
+# Get IMAGE and TAG from first argument
+if [[ "${1}" == "" ]]; then
+	echo "Usage: ${0} image:tag"
+	exit 1
+else
+	# Split image:tag
+	IFS=$':' read IMAGE TAG <<< ${1};
+	# Remove registry prefix from image if present
+	IMAGE=${IMAGE#"docker.io/"}
+fi
+
+login_data() {
+cat <<EOF
+{
+  "username": "${DOCKERHUB_USERNAME}",
+  "password": "${DOCKERHUB_PASSWORD}"
+}
+EOF
+}
+
+# Get auth token
+TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d "$(login_data)" "https://hub.docker.com/v2/users/login/" | jq -r .token)
+
+# Delete tag
+output=$(curl -sI "https://hub.docker.com/v2/repositories/${IMAGE}/tags/${TAG}/" \
+	-H "Authorization: JWT ${TOKEN}" \
+	-X DELETE
+)
+
+# Return and error if HTTP response code is not 204
+echo "${output}" | grep "HTTP/1.1 204 NO CONTENT"
diff --git a/.github/scripts/docker-tags.sh b/.github/scripts/docker-tags.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Generates docker images tags for the docker/build-push-action@v2 action depending on the branch/tag.
+# Image tag format:
+#   develop     => image:[version_prefix][version-]edge[-version_suffix]
+#   master      => image:[version_prefix][version][-][version_suffix]
+#   semver tag  => image:[version_prefix][version-]major.minor[-version_suffix]
+
+# Declare expected variables
+IMAGE=${IMAGE} # docksal/cli
+VERSION_PREFIX=${VERSION_PREFIX} # php
+VERSION=${VERSION} # 7.4
+VERSION_SUFFIX=${VERSION_SUFFIX} # ide
+REGISTRY="${REGISTRY}" # ghcr.io
+GITHUB_REF=${GITHUB_REF} # refs/heads/develop, refs/heads/master, refs/tags/v1.0.0
+
+# Join arguments with hyphen (-) as a delimiter
+# Usage: join <arg1> [<argn>]
+join() {
+	local IFS='-' # join delimiter
+	echo "$*"
+}
+
+# Prints resulting image tags and sets output variable
+set_output() {
+	local -n inputArr=${1}
+
+	declare -a outputArr
+	for imageTag in ${inputArr[@]}; do
+		# Prepend registry to imageTag if provided
+		[[ "${REGISTRY}" != "" ]] && imageTag="${REGISTRY}/${imageTag}"
+		outputArr+=("${imageTag}")
+	done
+
+	# Print with new lines for output in build logs
+	(IFS=$'\n'; echo "${outputArr[*]}")
+	# Using newlines in output variables does not seem to work, so we'll use comas
+	(IFS=$','; echo "::set-output name=tags::${outputArr[*]}")
+}
+
+# Image tags
+declare -a imageTagArr
+
+## On every build => build / build-sha7
+## Latest build tag (used with cache-from)
+#imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${VERSION_SUFFIX} build)")
+## Specific build tag - SHA7 (first 7 characters of commit SHA)
+#imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${VERSION_SUFFIX} build ${GITHUB_SHA:0:7})")
+
+# develop => version-edge
+if [[ "${GITHUB_REF}" == "refs/heads/develop" ]]; then
+	imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} edge ${VERSION_SUFFIX})")
+fi
+
+# master => version
+if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then
+	imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${VERSION_SUFFIX})")
+fi
+
+# tags/v1.0.0 => 1.0
+if [[ "${GITHUB_REF}" =~ "refs/tags/" ]]; then
+	# Extract version parts from release tag
+	IFS='.' read -a release_arr <<< "${GITHUB_REF#refs/tags/}"
+	releaseMajor=${release_arr[0]#v*}  # 2.7.0 => "2"
+	releaseMinor=${release_arr[1]}  # "2.7.0" => "7"
+	imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${VERSION_SUFFIX})")
+	imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${releaseMajor} ${VERSION_SUFFIX})")
+	imageTagArr+=("${IMAGE}:$(join ${VERSION_PREFIX}${VERSION} ${releaseMajor}.${releaseMinor} ${VERSION_SUFFIX})")
+fi
+
+# Note: imageTagArr is passed as variable name ("reference") and then expanded inside the called function
+# See https://stackoverflow.com/questions/16461656/how-to-pass-array-as-an-argument-to-a-function-in-bash/26443029#26443029
+# DockerHub tags
+set_output imageTagArr
diff --git a/.github/workflows/default.yaml b/.github/workflows/default.yaml
@@ -0,0 +1,268 @@
+name: Build, Test, Push
+
+on:
+  schedule:
+    - cron: '0 10 * * 0' # Every Sunday at 10AM
+  push:
+    branches:
+      - master
+      - develop
+      - feature/*
+    tags:
+      - 'v*.*.*'
+  workflow_dispatch: # Allow manually triggering a build
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  IMAGE: docksal/ssh-agent
+  DOCKSAL_VERSION: develop
+  # Explicitly disable host's SSH agent usage, as we are testing docksal-ssh-agent here
+  DOCKSAL_SSH_AGENT_USE_HOST: 0
+
+jobs:
+  build:
+    name: "Build: ${{ matrix.version }}/${{ matrix.arch }}"
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false # Don't cancel other jobs if one fails
+      matrix:
+        include:
+          -
+            platform: linux/amd64
+            arch: amd64
+            version: ''
+          -
+            platform: linux/arm64
+            arch: arm64
+            version: ''
+
+    env:
+      ARCH: ${{ matrix.arch }}
+      VERSION_PREFIX: ''
+      VERSION: ${{ matrix.version }}
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Environment variables
+        run: |
+          # Export variables for further steps
+          echo "GIT_SHA7=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+          echo "BUILD_CONTEXT=${VERSION:-.}" >> ${GITHUB_ENV}
+          echo "BUILD_IMAGE_TAG=${IMAGE}:${VERSION_PREFIX}${VERSION}build" >> ${GITHUB_ENV}
+#      -
+#        # Switch docker context to a remote arm64 host
+#        # Used for building heavy images that take too long to build using QEMU + for native arm64 testing.
+#        name: Switch to arm64 builder host
+#        if: ${{ env.ARCH == 'arm64' }}
+#        uses: arwynfr/actions-docker-context@98fc92878d0b856c1112c79b8d0f45353206e186
+#        with:
+#          docker_host: "ssh://ubuntu@${{ secrets.ARM64_HOST }}"
+#          context_name: arm64-host
+#          ssh_key: "${{ secrets.ARM64_HOST_SSH_KEY }}"
+#          ssh_cert: "${{ secrets.ARM64_HOST_SSH_CERT }}"
+#          use_context: true
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Check Docker
+        run: |
+          docker version
+          docker info
+      -
+        name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        # Build and cache image in the registry
+        name: Build image
+        uses: docker/build-push-action@v2
+        with:
+          context: ${{ env.BUILD_CONTEXT }}
+          file: ${{ env.BUILD_CONTEXT }}/Dockerfile
+          build-args: VERSION=${{ env.VERSION }}
+          platforms: linux/${{ env.ARCH }}
+          # Push intermediate arch-specific build tag to repo
+          tags: ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-${{ env.ARCH }}
+          push: ${{ github.event_name != 'pull_request' }} # Don't push for PRs
+          # BUILD_IMAGE_TAG - persistent multi-arch tag, updated at the end of the build (success or failure)
+          cache-from: type=registry,ref=${{ env.BUILD_IMAGE_TAG }}
+          cache-to: type=inline # Write the cache metadata into the image configuration
+
+  test:
+    name: "Test: ${{ matrix.version }}/${{ matrix.arch }}"
+    runs-on: ubuntu-20.04
+    needs: build
+
+    strategy:
+      fail-fast: false # Don't cancel other jobs if one fails
+      matrix:
+        include:
+          -
+            platform: linux/amd64
+            arch: amd64
+            version: ''
+          # Disabled arm64 tests.
+          # TODO: Figure out a way to run tests on a remote host.
+          # TODO: Remember to re-enabled the test results check in "push".
+          #-
+          #  platform: linux/arm64
+          #  arch: arm64
+          #  version: ''
+
+    env:
+      ARCH: ${{ matrix.arch }}
+      VERSION_PREFIX: ''
+      VERSION: ${{ matrix.version }}
+
+    steps:
+      -
+        name: Setup Bats
+        uses: mig4/setup-bats@v1
+        with:
+          bats-version: '1.3.0'
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Environment variables
+        run: |
+          # Export variables for further steps
+          echo "GIT_SHA7=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+          echo "BUILD_IMAGE_TAG=${IMAGE}:${VERSION_PREFIX}${VERSION}build" >> ${GITHUB_ENV}
+      -
+        # Switch docker context to a remote arm64 host
+        # Used for building heavy images that take too long to build using QEMU + for native arm64 testing.
+        name: Switch to arm64 builder host
+        if: ${{ env.ARCH == 'arm64' }}
+        uses: arwynfr/actions-docker-context@98fc92878d0b856c1112c79b8d0f45353206e186
+        with:
+          docker_host: "ssh://ubuntu@${{ secrets.ARM64_HOST }}"
+          context_name: arm64-host
+          ssh_key: "${{ secrets.ARM64_HOST_SSH_KEY }}"
+          ssh_cert: "${{ secrets.ARM64_HOST_SSH_CERT }}"
+          use_context: true
+      -
+        name: Check Docker
+        run: |
+          docker version
+          docker info
+      -
+        name: Test preparations
+        working-directory: ${{ env.BUILD_CONTEXT }}
+        env:
+          BUILD_IMAGE_TAG: ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-${{ env.ARCH }}
+        run: |
+          # Install Docksal using the passed DOCKSAL_VERSION value
+          curl -sSL http://get.docksal.io | bash
+          # Start the service using the build image tag
+          make start
+      -
+        # Run tests
+        name: Test
+        id: tests
+        working-directory: ${{ env.BUILD_CONTEXT }}
+        env:
+          BUILD_IMAGE_TAG: ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-${{ env.ARCH }}
+        run: |
+          make test
+          ([[ $? == 0 ]] && echo "pass" || echo "fail") | tee ${{ github.workspace }}/test-results-${VERSION_PREFIX}${VERSION}-${ARCH}.txt
+      # Store tests results as an artifact (used by downstream jobs)
+      # Note: Cannot use "::set-output name=var_name::var_value" as var_name would need to be dynamic here.
+      # Dynamic variable names cannot be used when mapping step outputs to job outputs.
+      # Step outputs cannot be accessed directly from other jobs. Dead end.
+      - name: Store test results
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-results
+          path: ${{ github.workspace }}/test-results-*.txt
+
+  push:
+    name: "Push: ${{ matrix.version }}/multi"
+    runs-on: ubuntu-20.04
+
+    # Wait for test to either succeed or fail
+    needs: test
+    if: always()
+
+    strategy:
+      matrix:
+        version:
+          - ''
+
+    env:
+      VERSION_PREFIX: ''
+      VERSION: ${{ matrix.version }}
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Environment variables
+        run: |
+          # Export variables for further steps
+          echo "GIT_SHA7=${GITHUB_SHA:0:7}" >> $GITHUB_ENV
+          echo "BUILD_IMAGE_TAG=${IMAGE}:${VERSION_PREFIX}${VERSION}build" >> ${GITHUB_ENV}
+      -
+        # Login to Docker Hub
+        name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      -
+        name: Retrieve test results
+        uses: actions/download-artifact@v2
+        with:
+          name: test-results
+      -
+        # Generate persistent tags (edge, stable, release)
+        name: Docker image tags
+        id: docker_tags
+        # Don't push broken builds to persistent tags (both amd64 and arm64 tests must pass)
+        # TODO: re-enable the arm64 test results check once those tests are re-enabled
+        run: |
+          amd64_tests=$(cat test-results-${VERSION_PREFIX}${VERSION}-amd64.txt)
+          #arm64_tests=$(cat test-results-${VERSION_PREFIX}${VERSION}-arm64.txt)
+          # && [[ "${arm64_tests}" == "pass" ]]
+          if [[ "${amd64_tests}" == "pass" ]]; then
+            .github/scripts/docker-tags.sh
+          fi
+      -
+        # Create and push multi-arch image manifests
+        name: Push multi-arch images
+        env:
+          # build tags are always pushed (build caching, debugging needs)
+          # edge, stage, release are only pushed if tests were successful (see docker_tags step)
+          TAGS: |
+            ${{ env.BUILD_IMAGE_TAG }}
+            ${{ steps.docker_tags.outputs.tags }}
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} # Needed for docker-tag-delete.sh
+          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} # Needed for docker-tag-delete.sh
+        run: |
+          set -xeuo pipefail
+          IFS="${IFS}," # Also split strings by comma (in case list of tag is comma-separated)
+          for tag in ${TAGS}; do
+            if [[ "${tag}" == "" ]]; then continue; fi
+            docker manifest create --amend ${tag} \
+              ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-amd64 \
+              ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-arm64
+            docker manifest inspect ${tag}
+            docker manifest push ${tag}
+          done
+          # Clean up intermediate arch-specific image tags (DockerHub only)
+          .github/scripts/docker-tag-delete.sh ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-amd64
+          .github/scripts/docker-tag-delete.sh ${{ env.BUILD_IMAGE_TAG }}-${{ env.GIT_SHA7 }}-arm64