test: make sure buildkite leaves docker in sane setup (#5897) [skip ci]

Co-authored-by: Stanislav Zhuk <stasadev@gmail.com>

.buildkite/test.sh

@@ -1,80 +1,72 @@
 #!/bin/bash
 # This script is used to build ddev/ddev using buildkite
 
+set -eu -o pipefail
+
 export PATH=$PATH:/home/linuxbrew/.linuxbrew/bin
 
-# GOTEST_SHORT=8 means drupal9
-export GOTEST_SHORT=8
+# GOTEST_SHORT=12 means drupal10
+export GOTEST_SHORT=12
 
 export DOCKER_SCAN_SUGGEST=false
 export DOCKER_SCOUT_SUGGEST=false
 
 # On macOS, we can have several different docker providers, allow testing all
+# In cleanup, stop everything we know of but leave either Orbstack or Docker Desktop running
 if [ "${OSTYPE%%[0-9]*}" = "darwin" ]; then
   function cleanup {
-#      docker context use default
-    true
+    command -v orb 2>/dev/null && echo "Stopping orbstack" && (orb stop &)
+    if [ -f /Applications/Docker.app ]; then echo "Stopping Docker Desktop" && (killall com.docker.backend || true); fi
+    command -v colima 2>/dev/null && echo "Stopping colima" && (colima stop || true)
+    command -v colima 2>/dev/null && echo "Stopping colima_vz" && (colima stop vz || true)
+    command -v limactl 2>/dev/null && echo "Stopping lima" && (limactl stop lima-vz || true)
+    if [ -f ~/.rd/bin/rdctl ]; then echo "Stopping Rancher Desktop" && (~/.rd/bin/rdctl shutdown || true); fi
+    docker context use default
+    # Leave orbstack running as the most likely to be reliable, otherwise Docker Desktop
+    if command -v orb 2>/dev/null ; then
+      echo "Starting orbstack" && (orb start &)
+    else
+      open -a Docker
+    fi
   }
   trap cleanup EXIT
 
-  echo "original docker context situation:"
+  # Start with a predictable docker provider running
+  cleanup
+
+  echo "starting docker context situation:"
   docker context ls
-  case ${DOCKER_TYPE} in
+
+  # Now start the docker provider we want
+  case ${DOCKER_TYPE:=none} in
     "colima")
-      colima stop vz || true
-      limactl stop lima-vz || true
-      ~/.rd/bin/rdctl shutdown || true
-      orb stop &
-      killall com.docker.backend || true
       colima start
+      # Colima seems to end up working better with less failures if we restart after starting
       colima restart
       docker context use colima
       ;;
     "colima_vz")
-      ~/.rd/bin/rdctl shutdown || true
-      colima stop || true
-      limactl stop lima-vz || true
-      orb stop &
-      killall com.docker.backend || true
       colima start vz
       colima restart vz
       docker context use colima-vz
       ;;
 
     "lima")
-      ~/.rd/bin/rdctl shutdown || true
-      colima stop || true
-      colima stop vz || true
-      orb stop &
-      killall com.docker.backend || true
       limactl start lima-vz
       docker context use lima-lima-vz
       ;;
 
     "docker-desktop")
-      orb stop &
-      ~/.rd/bin/rdctl shutdown || true
-      colima stop || true
-      limactl stop lima-vz || true
-      colima stop vz || true
-      open -a Docker &
+      open -a Docker
       docker context use desktop-linux
       ;;
+
     "orbstack")
-      ~/.rd/bin/rdctl shutdown || true
-      colima stop || true
-      colima stop vz || true
-      limactl stop lima-vz || true
-      killall com.docker.backend || true
       orb start &
       docker context use orbstack
       ;;
+
     "rancher-desktop")
-      killall com.docker.backend || true
-      colima stop || true
-      limactl stop lima-vz || true
-      colima stop vz || true
-      orb stop &
       ~/.rd/bin/rdctl start
       for i in {1..120}; do
         if docker context use rancher-desktop >/dev/null 2>&1 ; then
@@ -98,54 +90,74 @@ if [ ${OSTYPE%%-*} = "linux" ]; then
 fi
 
 # Make sure docker is working
-echo "Waiting for docker to come up: $(date)"
-date && ${TIMEOUT_CMD} 10m bash -c 'while ! docker ps >/dev/null 2>&1 ; do
+echo "Waiting for docker provider to come up: $(date)"
+date && ${TIMEOUT_CMD} 3m bash -c 'while ! docker ps >/dev/null 2>&1 ; do
   sleep 10
-  echo "Waiting for docker to come up: $(date)"
+  echo "Waiting: $(date)"
 done'
 echo "Testing again to make sure docker came up: $(date)"
 if ! docker ps >/dev/null 2>&1 ; then
   echo "Docker is not running, exiting"
   exit 1
 fi
 
-echo "buildkite building ${BUILDKITE_JOB_ID:-} at $(date) on $(hostname) as USER=${USER} for OS=${OSTYPE} DOCKER_TYPE=${DOCKER_TYPE:notset} in ${PWD} with GOTEST_SHORT=${GOTEST_SHORT} golang=$(go version | awk '{print $3}') docker-desktop=$(scripts/docker-desktop-version.sh) docker=$(docker --version | awk '{print $3}') ddev version=$(ddev --version | awk '{print $3}'))"
-
+echo
+echo "buildkite building ${BUILDKITE_JOB_ID:-} at $(date) on $(hostname) as USER=${USER} for OS=${OSTYPE} DOCKER_TYPE=${DOCKER_TYPE:-notset} in ${PWD} with GOTEST_SHORT=${GOTEST_SHORT} golang=$(go version | awk '{print $3}') ddev version=$(ddev --version | awk '{print $3}')"
+
+echo
+case ${DOCKER_TYPE:-none} in
+  "docker-desktop")
+    echo "docker-desktop for mac version=$(scripts/docker-desktop-version.sh)"
+    ;;
+  "colima")
+    echo "colima version=$(colima version)"
+    ;;
+  "colima_vz")
+    echo "colima version=$(colima version)"
+    ;;
+
+  "orbstack")
+    echo "orbstack version=$(orbctl version)"
+    ;;
+  "rancher-desktop")
+    echo "rancher-desktop=$(~/.rd/bin/rdctl version)"
+    ;;
+  "wsl2dockerinside")
+    echo "Running wsl2dockerinside"
+    ;;
+  "dockerforwindows")
+    echo "Running Windows docker desktop for windows"
+    ;;
+  "dockerforwindows")
+    echo "Running wsl2-docker-desktop"
+    ;;
+  *)
+    echo "$DOCKER_TYPE not found"
+    ;;
+esac
+
+echo "Docker version:"
+docker version
+echo "ddev version"
 ddev version
+echo
 
 export DDEV_NONINTERACTIVE=true
 export DDEV_DEBUG=true
 
-set -o errexit
-set -o pipefail
-set -o nounset
-set -x
-
-# Broken docker context list from https://github.com/docker/for-win/issues/13180
-# When this is solved this can be removed.
-# The only place we care about non-default context is macOS Colima
-if ! docker context list >/dev/null; then
-  rm -rf ~/.docker/contexts && docker context list >/dev/null
+# We can skip builds with commit message of [skip buildkite]
+if [[ $BUILDKITE_MESSAGE == *"[skip buildkite]"* ]] || [[ $BUILDKITE_MESSAGE == *"[skip ci]"* ]]; then
+  echo "Skipping build because message has '[skip buildkite]' or '[skip ci]'"
+  exit 0
 fi
 
 # If this is a PR and the diff doesn't have code, skip it
 if [ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ] && ! git diff --name-only refs/remotes/origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-} | egrep "^(\.buildkite|Makefile|pkg|cmd|vendor|go\.)" >/dev/null; then
-  echo "Skipping build since no code changes found"
+  echo "Skipping buildkite build since no code changes found"
   exit 0
 fi
 
-# We can skip builds with commit message of [skip buildkite]
-if [[ $BUILDKITE_MESSAGE == *"[skip buildkite]"* ]] || [[ $BUILDKITE_MESSAGE == *"[skip ci]"* ]]; then
-  echo "Skipping build because message has '[skip buildkite]' or '[skip ci]'"
-  exit 0
-fi
-
-# On macOS, restart docker to avoid bugs where containers can't be deleted
-#if [ "${OSTYPE%%[0-9]*}" = "darwin" ]; then
-#  killall Docker || true
-#  nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended &
-#  sleep 10
-#fi
+set -x
 
 # We don't want any docker volumes to be existing and changing behavior
 docker volume prune -a -f >/dev/null 2>&1 || true

.buildkite/testbot_maintenance.sh

@@ -48,7 +48,8 @@ linux)
 
 esac
 
-(yes | ddev delete images >/dev/null) || true
+echo "Deleting unused images with ddev delete images"
+ddev delete images -y || true
 
 # Remove any -built images, as we want to make sure tests do the building.
 docker rmi -f $(docker images --filter "dangling=true" -q --no-trunc) >/dev/null 2>&1 || true

.buildkite/windows10_container.yml

@@ -11,4 +11,5 @@
       - "architecture=amd64"
     env:
       BUILDKITE_CLEAN_CHECKOUT: true
+      DOCKER_TYPE: dockerforwindows
     parallelism: 1

.buildkite/windows10dockerforwindows.yml

@@ -14,4 +14,5 @@
     env:
       BUILDKITE_CLEAN_CHECKOUT: true
       DDEV_TEST_USE_NFSMOUNT: false
+      DOCKER_TYPE: dockerforwindows
     parallelism: 1

.buildkite/wsl2-docker-desktop.yml

@@ -18,4 +18,5 @@
       BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds
       BUILDKIT_PROGRESS: plain
       DDEV_TEST_SHARE_CMD: "true"
+      DOCKER_TYPE: "wsl2-docker-desktop"
     parallelism: 1

.buildkite/wsl2-docker-inside.yml

@@ -15,5 +15,6 @@
       BUILDKITE_CLEAN_CHECKOUT: true
       BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds
       BUILDKIT_PROGRESS: plain
+      DOCKER_TYPE: "wsl2dockerinside"
     parallelism: 1