Use GitHub Actions for linux tests, fixes #2604 (#2615)

.github/workflows/build-and-test.yml

@@ -0,0 +1,111 @@
+name: Build and test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+env:
+  BUILDKIT_PROGRESS: plain
+  DOCKER_CLI_EXPERIMENTAL: enabled
+  DDEV_LIVE_NO_ANALYTICS: true
+  DDEV_DEBUG: true
+
+
+jobs:
+  build:
+    name: Build DDEV executables
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          # We need to get all branches and tags for git describe to work properly
+          fetch-depth: 0
+
+      - name: Install Docker and deps
+        run: ./.github/workflows/linux-setup.sh
+
+      - name: Install build tools
+        run: ./.github/workflows/linux-build-setup.sh
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.15.3'
+
+      # Install Node v10 for the staticrequired step
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '10'
+
+      - name: staticrequired
+        run: make staticrequired
+
+      - name: Build DDEV executables
+        run: make linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 windows_amd64 windows_install
+
+      - name: Generate artifacts
+        run: ./.github/workflows/generate_artifacts.sh ${{ github.workspace }}/artifacts
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ddev-executables
+          path: ${{ github.workspace }}/artifacts/*
+          # 90 retention days are overkill for this
+          retention-days: 7
+
+  tests:
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      matrix:
+        webserver: [apache-fpm, nginx-fpm]
+        tests: [testpkg, testcmd, testfullsitesetup]
+        os: [ ubuntu-20.04 ]
+      fail-fast: false
+
+#    name: ${{ matrix.os[0] }} - ${{ matrix.webserver }} - make ${{ matrix.tests }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      DDEV_TEST_WEBSERVER_TYPE: ${{ matrix.webserver }}
+      DRUD_NONINTERACTIVE: "true"
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: GOTEST_SHORT setup - use "true" except for testfullsitesetup
+        run: |
+          if [ "${{ matrix.tests }}" != "testfullsitesetup" ]; then
+            echo "GOTEST_SHORT=true" >> $GITHUB_ENV
+          fi
+
+      - name: Install Docker and deps (Linux)
+        if: matrix.os == 'ubuntu-20.04'
+        run: ./.github/workflows/linux-setup.sh
+
+#      - name: Install Docker and deps (macOS)
+#        if: matrix.os == 'macos-latest'
+#        run: ./.github/workflows/macos-setup.sh
+
+      - uses: actions/setup-go@v2
+        if: matrix.os == 'ubuntu-20.04'
+        with:
+          go-version: 1.*
+
+#      - name: Clean up self-hosted runners if necessary
+#        if: matrix.os[1] == 'self-hosted'
+#        run: .github/workflows/selfhosted-maintenance.sh
+
+      - name: DDEV tests
+        run: make ${{ matrix.tests }}
+
+      - name: Store test results
+        uses: actions/upload-artifact@v2
+        with:
+          name: ddev-test-results-${{ matrix.webserver }}
+          path: /tmp/testresults/

.github/workflows/container-tests.yml

@@ -0,0 +1,70 @@
+name: Container tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+    # Only run when something changes in the actual containers
+    paths:
+    - "containers/**"
+
+env:
+  BUILDKIT_PROGRESS: plain
+  DOCKER_CLI_EXPERIMENTAL: enabled
+
+jobs:
+
+  container-build-and-test:
+    name: ${{ matrix.os }} - Test container ${{ matrix.containers }}
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      matrix:
+        os: [ubuntu-20.04]
+        containers: [mariadb, mysql, webserver, others]
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install Docker and deps (Linux)
+        if: matrix.os == 'ubuntu-20.04'
+        run: ./.github/workflows/linux-setup.sh
+
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.*
+
+      - name: Build and test container ${{ matrix.containers }}
+        run: |
+          docker version
+          set -eu -o pipefail
+
+          if [[ "${{ matrix.containers }}" == "mariadb" ]]; then
+            pushd containers/ddev-dbserver
+            echo "--- Test container mariadb"
+            time make test_mariadb DOCKER_ARGS="--no-cache"
+            popd
+
+          elif [[ "${{ matrix.containers }}" == "mysql" ]]; then
+            pushd containers/ddev-dbserver
+            echo "--- Test container mysql"
+            time make test_mysql DOCKER_ARGS="--no-cache"
+            popd
+
+          elif [[ "${{ matrix.containers }}" == "webserver" ]]; then
+            pushd containers/ddev-webserver
+            echo "--- Test container ddev-webserver"
+            time make test DOCKER_ARGS="--no-cache"
+            popd
+
+          elif [[ "${{ matrix.containers }}" == "others" ]]; then
+            for dir in ddev-router ddev-ssh-agent test-ssh-server
+              do pushd containers/$dir
+              echo "--- Test container $dir"
+              time make test DOCKER_ARGS=--no-cache
+              popd
+            done
+
+          fi
+          set +eu

.github/workflows/generate_artifacts.sh

@@ -0,0 +1,106 @@
+#!/bin/bash
+# This script builds ddev artifacts and their sha256 hashes.
+# First arg is the artifact directory
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+MKCERT_VERSION=v1.4.6
+BUILD_IMAGE_TARBALLS=false
+
+ARTIFACTS=${1:-/artifacts}
+BASE_DIR=$PWD
+
+mkdir -p $ARTIFACTS || (sudo mkdir -p $ARTIFACTS && sudo chmod 777 $ARTIFACTS)
+export VERSION=$(git describe --tags --always --dirty)
+
+# If the version does not have a dash in it, it's not prerelease,
+# so build image tarballs
+if [ "${VERSION}" = "${VERSION%%-*}" ]; then
+    BUILD_IMAGE_TARBALLS=true
+fi
+
+BUILTPATH=.gotmp/bin/$(go env GOOS)_$(go env GOARCH)
+
+if [ "${BUILD_IMAGE_TARBALLS}" = "true" ]; then
+    # Make sure we have all our docker images, and save them in a tarball
+    $BUILTPATH/ddev version | awk '/(drud|phpmyadmin)\// {print $2;}' >/tmp/images.txt
+    # Quicksprint is the only known consumer of this tarball, and Drupal 9 needs non-default mariadb 10.3
+    $BUILTPATH/ddev version | awk ' $1 == "db" { sub(/mariadb-10.2/, "mariadb-10.3"); print $2 }' >>/tmp/images.txt
+    for item in $(cat /tmp/images.txt); do
+      docker pull $item
+    done
+    echo "Generating ddev_docker_images.$VERSION.tar"
+    docker save -o $ARTIFACTS/ddev_docker_images.$VERSION.tar $(cat /tmp/images.txt)
+    echo "Generating ddev_docker_images.$VERSION.tar.xz"
+    xz $ARTIFACTS/ddev_docker_images.$VERSION.tar
+fi
+
+# Generate and place extra items like autocomplete
+$BUILTPATH/ddev_gen_autocomplete
+
+# The completion scripts get placed into the linux build dir (.gotmp/bin)
+# So now copy them into the real build directory
+for dir in .gotmp/bin/linux_amd64 .gotmp/bin/linux_arm64 .gotmp/bin/darwin_amd64 .gotmp/bin/windows_amd64; do
+  cp .gotmp/bin/ddev_*completion* $dir
+done
+
+# Create tarball of completion scripts
+pushd .gotmp/bin >/dev/null && tar -czf $ARTIFACTS/ddev_shell_completion_scripts.$VERSION.tar.gz *completion*.sh && popd >/dev/null
+cp $BASE_DIR/.gotmp/bin/windows_amd64/ddev_windows_installer*.exe $ARTIFACTS
+
+# Generate macOS-amd64 tarball/zipball
+pushd $BASE_DIR/.gotmp/bin/darwin_amd64
+curl -sSL -o mkcert https://github.com/drud/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-darwin-amd64 && chmod +x mkcert
+tar -czf $ARTIFACTS/ddev_macos-amd64.$VERSION.tar.gz ddev *completion*.sh mkcert
+popd
+
+# Generate linux-amd64 tarball/zipball
+pushd $BASE_DIR/.gotmp/bin/linux_amd64
+curl -sSL -o mkcert https://github.com/drud/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-amd64 && chmod +x mkcert
+tar -czf $ARTIFACTS/ddev_linux-amd64.$VERSION.tar.gz ddev *completion*.sh mkcert
+popd
+
+# Generate linux-arm64 tarball/zipball
+pushd $BASE_DIR/.gotmp/bin/linux_arm64
+curl -sSL -o mkcert https://github.com/drud/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-arm64 && chmod +x mkcert
+tar -czf $ARTIFACTS/ddev_linux-arm64.$VERSION.tar.gz ddev *completion*.sh mkcert
+popd
+
+# Generate linux-arm tarball/zipball
+#pushd $BASE_DIR/.gotmp/bin/linux_arm
+#curl -sSL -o mkcert https://github.com/drud/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-arm && chmod +x mkcert
+#tar -czf $ARTIFACTS/ddev_linux-arm.$VERSION.tar.gz ddev *completion*.sh
+#popd
+
+# generate windows-amd64 tarball/zipball
+pushd $BASE_DIR/.gotmp/bin/windows_amd64
+curl -sSL -o mkcert.exe https://github.com/drud/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-windows-amd64.exe
+tar -czf $ARTIFACTS/ddev_windows-amd64.$VERSION.tar.gz ddev.exe *completion*.sh mkcert.exe
+if [ -d chocolatey ]; then
+    tar -czf $ARTIFACTS/ddev_chocolatey_amd64-.$VERSION.tar.gz chocolatey
+fi
+popd
+
+# Create macOS and Linux homebrew bottles
+for os in high_sierra x86_64_linux ; do
+    NO_V_VERSION=${VERSION#v}
+    rm -rf /tmp/bottle
+    BOTTLE_BASE=/tmp/bottle/ddev/$NO_V_VERSION
+    mkdir -p $BOTTLE_BASE/{bin,etc/bash_completion.d,share/zsh/site-functions,share/fish/vendor_completions.d}
+    cp $BASE_DIR/.gotmp/bin/ddev_bash_completion.sh $BOTTLE_BASE/etc/bash_completion.d/ddev
+    cp $BASE_DIR/.gotmp/bin/ddev_zsh_completion.sh $BOTTLE_BASE/share/zsh/site-functions/_ddev
+    cp $BASE_DIR/.gotmp/bin/ddev_fish_completion.sh $BOTTLE_BASE/share/fish/vendor_completions.d/ddev.fish
+
+    if [ "${os}" = "high_sierra" ]; then cp $BASE_DIR/.gotmp/bin/darwin_amd64/ddev $BOTTLE_BASE/bin ; fi
+    if [ "${os}" = "x86_64_linux" ]; then cp $BASE_DIR/.gotmp/bin/linux_amd64/ddev $BOTTLE_BASE/bin ; fi
+    cp $BASE_DIR/{README.md,LICENSE} $BOTTLE_BASE
+    tar -czf $ARTIFACTS/ddev-$NO_V_VERSION.$os.bottle.tar.gz -C /tmp/bottle .
+done
+
+# Create the sha256 files
+cd $ARTIFACTS
+for item in *.*; do
+  sha256sum $item > $item.sha256.txt
+done

.github/workflows/linux-build-setup.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# Needed for the build jobs, e.g. for Windows makensis
+
+sudo apt-get update -qq
+sudo apt-get install -qq osslsigncode nsis
+
+# Get the Stubs and Plugins for makensis; the linux makensis build doesn't do this.
+wget https://sourceforge.net/projects/nsis/files/NSIS%203/3.06.1/nsis-3.06.1.zip/download && sudo unzip -o -d /usr/share download && sudo mv /usr/share/nsis-3.06.1 /usr/share/nsis
+wget https://github.com/GsNSIS/EnVar/releases/latest/download/EnVar-Plugin.zip && sudo unzip -o -d /usr/share/nsis EnVar-Plugin.zip
+wget https://github.com/DigitalMediaServer/NSIS-INetC-plugin/releases/latest/download/INetC.zip && sudo unzip -o -d /usr/share/nsis/Plugins INetC.zip
+
+set +eu

.github/workflows/linux-setup.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+# Basic tools
+
+set -x
+
+if [ ! -z "${DOCKERHUB_PULL_USERNAME:-}" ]; then
+  set +x
+  echo "${DOCKERHUB_PULL_PASSWORD}" | docker login --username "${DOCKERHUB_PULL_USERNAME}" --password-stdin
+  set -x
+fi
+
+sudo apt-get update -qq
+sudo apt-get install -qq mysql-client zip jq expect nfs-kernel-server build-essential curl git libnss3-tools libcurl4-gnutls-dev
+
+curl -sSL --fail -o /tmp/ngrok.zip https://bin.equinox.io/c/4VmDzA7iaHb/ngrok-stable-linux-amd64.zip && sudo unzip -o -d /usr/local/bin /tmp/ngrok.zip
+
+if [ ! -d /home/linuxbrew/.linuxbrew/bin ] ; then
+    sh -c "$(curl -fsSL https://raw.githubusercontent.com/Linuxbrew/install/master/install.sh)"
+fi
+
+echo "export PATH=/home/linuxbrew/.linuxbrew/bin:$PATH" >>~/.bashrc
+
+# Without this .curlrc CircleCI linux image doesn't respect mkcert certs
+echo "capath=/etc/ssl/certs/" >>~/.curlrc
+
+. ~/.bashrc
+
+export HOMEBREW_NO_AUTO_UPDATE=1
+export HOMEBREW_NO_INSTALL_CLEANUP=1
+
+brew tap drud/ddev >/dev/null
+for item in golang golangci-lint mkcert bats-core; do
+    brew install $item >/dev/null || /home/linuxbrew/.linuxbrew/bin/brew upgrade $item >/dev/null
+done
+
+mkcert -install
+
+primary_ip=$(ip route get 1 | awk '{gsub("^.*src ",""); print $1; exit}')
+
+sudo bash -c "cat <<EOF >/etc/exports
+${HOME} ${primary_ip}/255.255.255.255(rw,sync,no_subtree_check)
+/tmp ${primary_ip}/255.255.255.255(rw,sync,no_subtree_check)
+EOF"
+
+sudo service nfs-kernel-server restart
+
+# Install ghr
+GHR_RELEASE="v0.13.0"
+curl -fsL -o /tmp/ghr.tar.gz https://github.com/tcnksm/ghr/releases/download/${GHR_RELEASE}/ghr_${GHR_RELEASE}_linux_amd64.tar.gz
+sudo tar -C /usr/local/bin --strip-components=1 -xzf /tmp/ghr.tar.gz ghr_${GHR_RELEASE}_linux_amd64/ghr
+ghr -v

.github/workflows/macos-setup.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+# Install Docker + Docker buildx
+mkdir -p ~/.docker/machine/cache
+curl -Lo ~/.docker/machine/cache/boot2docker.iso https://github.com/boot2docker/boot2docker/releases/download/v19.03.12/boot2docker.iso
+brew install docker docker-machine docker-compose
+mkdir ~/.docker/cli-plugins
+curl -sSL https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.darwin-amd64 -o ~/.docker/cli-plugins/docker-buildx
+chmod a+x ~/.docker/cli-plugins/docker-buildx
+docker buildx version
+docker-machine create --driver virtualbox default
+docker-machine env default
+
+# Install other dependencies
+brew install make mysql mkcert
+
+mkcert -install
+
+set +eu