test: Run colima tests native instead of github, fixes #5598 (#5886)

.buildkite/macos-colima.yml

@@ -0,0 +1,21 @@
+# Colima with qemu and sshfs
+# See https://buildkite.com/ddev/macos-colima/settings/repository
+# Runs on master and PRs, including forked PRs
+
+  - command: ".buildkite/test.sh"
+    plugins:
+      - docker-login#v2.1.0:
+          username: druddockerpullaccount
+          password-env: DOCKERHUB_PULL_PASSWORD
+    agents:
+      - "os=macos"
+      - "colima=true"
+      - "architecture=arm64"
+    env:
+      BUILDKITE_CLEAN_CHECKOUT: true
+      BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds
+      BUILDKIT_PROGRESS: plain
+      DDEV_TEST_SHARE_CMD: "false"
+      DDEV_RUN_GET_TESTS: "false"
+      DOCKER_TYPE: "colima"
+    parallelism: 1

.buildkite/macos-colima_vz.yml

@@ -0,0 +1,23 @@
+# Colima with VZ and virtiofs
+# See https://buildkite.com/ddev/macos-colima-vz/settings/repository
+# Runs on master and PRs, including forked PRs
+
+  - command: ".buildkite/test.sh"
+    plugins:
+      - docker-login#v2.1.0:
+          username: druddockerpullaccount
+          password-env: DOCKERHUB_PULL_PASSWORD
+    agents:
+      - "os=macos"
+      - "colima=true"
+      - "colima_vz=true"
+      - "architecture=arm64"
+    #branches: "master"
+    env:
+      BUILDKITE_CLEAN_CHECKOUT: true
+      BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds
+      BUILDKIT_PROGRESS: plain
+      DDEV_TEST_SHARE_CMD: "false"
+      DDEV_RUN_GET_TESTS: "false"
+      DOCKER_TYPE: "colima_vz"
+    parallelism: 1

.buildkite/macos-docker-desktop-amd64.yml

@@ -1,3 +1,7 @@
+# Docker Desktop on macOS amd64
+# See https://buildkite.com/ddev/ddev-macos-amd64-mutagen/settings/repository
+# Runs on master only
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:

.buildkite/macos-docker-desktop-arm64.yml

@@ -1,3 +1,7 @@
+# Docker Desktop on macOS arm64
+# See https://buildkite.com/ddev/ddev-macos-arm64-mutagen/settings/repository
+# Runs on master and PRs, including forked PRs
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:

.buildkite/macos-lima.yml

@@ -0,0 +1,22 @@
+ # macOS Lima on arm64 with VZ/Virtiofs
+ # Experimental, not yet enabled
+ # See https://buildkite.com/ddev/macos-lima/settings/repository
+
+  - command: ".buildkite/test.sh"
+    plugins:
+      - docker-login#v2.1.0:
+          username: druddockerpullaccount
+          password-env: DOCKERHUB_PULL_PASSWORD
+    agents:
+      - "os=macos"
+      - "lima=true"
+      - "architecture=arm64"
+    branches: "none"
+    env:
+      BUILDKITE_CLEAN_CHECKOUT: true
+      BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds
+      BUILDKIT_PROGRESS: plain
+      DDEV_TEST_SHARE_CMD: "false"
+      DDEV_RUN_GET_TESTS: "false"
+      DOCKER_TYPE: "lima"
+    parallelism: 1

.buildkite/macos-orbstack.yml

@@ -1,3 +1,7 @@
+# Orbstack on macOS arm64
+# See https://buildkite.com/ddev/macos-orbstack/settings/repository
+# Runs on master and PRs, including forked PRs
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:
@@ -7,7 +11,7 @@
       - "os=macos"
       - "orbstack=true"
       - "architecture=arm64"
-    #branches: "master"
+#    branches: "master"
     env:
       BUILDKITE_CLEAN_CHECKOUT: true
       BUILDKITE_BUILD_PATH: ~/tmp/buildkite_builds

.buildkite/macos-rancher-desktop.yml

@@ -1,3 +1,7 @@
+# Rancher Desktop on macOS arm64
+# See https://buildkite.com/ddev/macos-rancher-desktop/settings/repository
+# Runs on ddev/ddev only, not on PRs
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:

.buildkite/nfstest.sh

@@ -4,22 +4,11 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-function cleanup {
-    docker volume rm nfstest >/dev/null || true
-}
-trap cleanup EXIT
+NFS_PROJECT_DIR=~/tmp/ddev-nfs-test
+mkdir -p ${NFS_PROJECT_DIR} || true && cd ${NFS_PROJECT_DIR}
 
-mkdir -p ~/.ddev
+ddev config --auto
+ddev debug nfsmount
+ddev delete -Oy
 
-# Handle new macOS Catalina /System/Volumes/Data share path.
-share=${HOME}/.ddev
-if [ -d "/System/Volumes/Data${HOME}/.ddev" ] ; then
-    share="/System/Volumes/Data${HOME}/.ddev"
-fi
-
-# Find host.docker.internal name using host-docker-internal.sh script
-hostDockerInternal=$($(dirname $0)/../scripts/host-docker-internal.sh)
-
-docker volume create --driver local --opt type=nfs --opt o=addr=${hostDockerInternal},hard,nolock,rw --opt device=:${share} nfstest >/dev/null
-docker run -t --rm -v nfstest:/tmp/nfs busybox:stable ls //tmp/nfs >/dev/null
 echo "nfsd seems to be set up ok"

.buildkite/sanetestbot.sh

@@ -51,8 +51,8 @@ if command -v ddev >/dev/null && version_gt ${MIN_DDEV_VERSION} ${CURRENT_DDEV_V
   exit 4
 fi
 
-# Skip nfs check on linux, as we won't run nfs there
-if [ ${OSTYPE%%-gnu} != "linux" ]; then
+# Skip nfs check on linux/lima, as we won't run nfs there
+if [ ${OSTYPE%%-gnu} != "linux" ] && [ ${DOCKER_TYPE:-nothing} != "lima" ]; then
   $(dirname $0)/nfstest.sh
 fi
 

.buildkite/test.sh

@@ -20,20 +20,60 @@ if [ "${OSTYPE%%[0-9]*}" = "darwin" ]; then
   echo "original docker context situation:"
   docker context ls
   case ${DOCKER_TYPE} in
+    "colima")
+      colima stop vz || true
+      limactl stop lima-vz || true
+      ~/.rd/bin/rdctl shutdown || true
+      orb stop &
+      killall com.docker.backend || true
+      colima start
+      colima restart
+      docker context use colima
+      ;;
+    "colima_vz")
+      ~/.rd/bin/rdctl shutdown || true
+      colima stop || true
+      limactl stop lima-vz || true
+      orb stop &
+      killall com.docker.backend || true
+      colima start vz
+      colima restart vz
+      docker context use colima-vz
+      ;;
+
+    "lima")
+      ~/.rd/bin/rdctl shutdown || true
+      colima stop || true
+      colima stop vz || true
+      orb stop &
+      killall com.docker.backend || true
+      limactl start lima-vz
+      docker context use lima-lima-vz
+      ;;
+
     "docker-desktop")
       orb stop &
       ~/.rd/bin/rdctl shutdown || true
+      colima stop || true
+      limactl stop lima-vz || true
+      colima stop vz || true
       open -a Docker &
       docker context use desktop-linux
       ;;
     "orbstack")
       ~/.rd/bin/rdctl shutdown || true
+      colima stop || true
+      colima stop vz || true
+      limactl stop lima-vz || true
       killall com.docker.backend || true
       orb start &
       docker context use orbstack
       ;;
     "rancher-desktop")
       killall com.docker.backend || true
+      colima stop || true
+      limactl stop lima-vz || true
+      colima stop vz || true
       orb stop &
       ~/.rd/bin/rdctl start
       for i in {1..120}; do

.buildkite/windows10_container.yml

@@ -1,3 +1,5 @@
+# Not currently used. This would build and run containers on Windows.
+
   - command: ".buildkite/test_containers.cmd"
     plugins:
       - docker-login#v2.1.0:

.buildkite/windows10dockerforwindows.yml

@@ -1,3 +1,7 @@
+# Windows native with Mutagen, used by ddev-windows-mutagen
+# See https://buildkite.com/ddev/ddev-windows-mutagen/settings/repository
+# Runs on master only
+
   - command: ".buildkite/test.cmd"
     plugins:
       - docker-login#v2.1.0:

.buildkite/wsl2-docker-desktop.yml

@@ -1,3 +1,8 @@
+# WSL2 using Docker Desktop
+# See https://buildkite.com/ddev/wsl2-docker-desktop/settings/repository
+# Runs on master only because DD is pretty flaky, so we can't keep up with
+# restarts on PRs
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:

.buildkite/wsl2-docker-inside.yml

@@ -1,3 +1,7 @@
+# WSL2 with docker-ce (docker native inside WSL2 == Docker inside)
+# See https://buildkite.com/ddev/wsl2-docker-inside/settings/repository
+# Runs on master and all PRs, including forked PRs
+
   - command: ".buildkite/test.sh"
     plugins:
       - docker-login#v2.1.0:

.github/workflows/colima-tests.yml

@@ -105,6 +105,7 @@ jobs:
           brew install colima
           colima version
           colima start --cpu 3 --memory 6 --disk 100 --vm-type=qemu --mount-type=sshfs --dns=1.1.1.1
+          colima restart
 
       - name: Build ddev
         run: | 

docs/content/developers/buildkite-testmachine-setup.md

@@ -122,7 +122,7 @@ We are using [Buildkite](https://buildkite.com/ddev) for Windows and macOS testi
 6. Restart `sudo systemctl restart icinga2`
 7. Hope that it can all work sometime.
 
-## macOS Test Agent Setup (Intel and Apple Silicon)
+## macOS Docker Desktop Test Agent Setup (Intel and Apple Silicon)
 
 1. Create the user “testbot” on the machine. Use the password for `ddevtestbot@gmail.com`, available in 1Password.
 2. Change the name of the machine to something in keeping with current style, perhaps `testbot-macos-arm64-8`. This is done in **Settings** → **General** → **About** → **Name** and in **Sharing** → **Computer Name** and in **Sharing** → **Local Hostname**.
@@ -173,3 +173,21 @@ We are using [Buildkite](https://buildkite.com/ddev) for Windows and macOS testi
     ```bash
     PATH=$PATH:/usr/local/bin:/opt/homebrew/bin
     ```
+
+## Additional Colima macOS setup
+
+1. `brew install colima`
+2. `colima start --cpu 4 --memory 6 --disk 100 --vm-type=qemu --mount-type=sshfs --dns=1.1.1.1`
+3. `colima stop`
+4. `colima start vz --cpu 4 --memory 6 --disk 60 --vm-type=vz --mount-type=virtiofs --dns=1.1.1.1`
+5. `colima stop vz`
+
+Then the Buildkite agent must be configured with tags `colima=true` and `colima_vz=true`.
+
+## Additional Lima macOS setup (not yet working)
+
+1. `limactl create --name=lima-vz --vm-type=vz --mount-type=virtiofs --mount="~/:w" --memory=6 --cpus=4 --disk=100 template://docker`
+2. `limactl start lima-vz`
+3. `docker context use lima-lima-vz`
+
+Then the Buildkite agent must be configured with tags `lima=true`.

pkg/ddevapp/extra_expose_test.go

@@ -63,7 +63,7 @@ func TestExtraPortExpose(t *testing.T) {
 	for i, p := range portsToTest {
 		url := fmt.Sprintf("%s:%s/testfile.html", app.GetPrimaryURL(), p)
 		out, resp, err := testcommon.GetLocalHTTPResponse(t, url)
-		assert.NoError(err, "failed to get hit url %s, out=%s, resp=%v err=%v", url, out, resp, err)
-		assert.Contains(out, fmt.Sprintf("this is test%d", i+1))
+		require.NoError(t, err, "failed to get hit url %s, out=%s, resp=%v err=%v", url, out, resp, err)
+		require.Contains(t, out, fmt.Sprintf("this is test%d", i+1))
 	}
 }