Merge branch 'ddev:master' into 20240517_mbomb007_normalize_sitename

.buildkite/windows10dockerforwindows.yml

@@ -1,6 +1,6 @@
 # Windows native with Mutagen, used by ddev-windows-mutagen
 # See https://buildkite.com/ddev/ddev-windows-mutagen/settings/repository
-# Runs on master only
+# Runs on branches/PRs on ddev/ddev only
 
   - command: ".buildkite/test.cmd"
     agents:

.ci-scripts/generate_artifacts.sh

@@ -6,7 +6,7 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-MKCERT_VERSION=v1.4.6
+MKCERT_VERSION=v1.4.4
 BUILD_IMAGE_TARBALLS=${BUILD_IMAGE_TARBALLS:-false}
 
 ARTIFACTS=${1:-/artifacts}
@@ -45,33 +45,35 @@ if [ "${BUILD_IMAGE_TARBALLS}" = "true" ]; then
 fi
 
 
+echo "Using github as source for mkcert binaries for tarballs"
+
 # Generate macOS-amd64 tarball/zipball
 pushd $BASE_DIR/.gotmp/bin/darwin_amd64 >/dev/null
-curl --fail -JL -s -o mkcert "https://dl.filippo.io/mkcert/latest?for=darwin/amd64" && chmod +x mkcert
+curl --fail -JL -s -o mkcert "https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-darwin-amd64" && chmod +x mkcert
 tar -czf $ARTIFACTS/ddev_macos-amd64.$VERSION.tar.gz ddev  mkcert
 popd >/dev/null
 
 # Generate macOS-arm64 tarball/zipball
 pushd $BASE_DIR/.gotmp/bin/darwin_arm64 >/dev/null
-curl --fail -JL -s -o mkcert "https://dl.filippo.io/mkcert/latest?for=darwin/arm64" && chmod +x mkcert
+curl --fail -JL -s -o mkcert "https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-darwin-arm64" && chmod +x mkcert
 tar -czf $ARTIFACTS/ddev_macos-arm64.$VERSION.tar.gz ddev  mkcert
 popd >/dev/null
 
 # Generate linux-amd64 tarball/zipball
 pushd $BASE_DIR/.gotmp/bin/linux_amd64 >/dev/null
-curl --fail -JL -s -o mkcert "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && chmod +x mkcert
+curl --fail -JL -s -o mkcert "https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-amd64" && chmod +x mkcert
 tar -czf $ARTIFACTS/ddev_linux-amd64.$VERSION.tar.gz ddev  mkcert
 popd >/dev/null
 
 # Generate linux-arm64 tarball/zipball
 pushd $BASE_DIR/.gotmp/bin/linux_arm64 >/dev/null
-curl --fail -JL -s -o mkcert "https://dl.filippo.io/mkcert/latest?for=linux/arm64" && chmod +x mkcert
+curl --fail -JL -s -o mkcert "https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-linux-arm64" && chmod +x mkcert
 tar -czf $ARTIFACTS/ddev_linux-arm64.$VERSION.tar.gz ddev  mkcert
 popd >/dev/null
 
 # generate windows-amd64 tarball/zipball
 pushd $BASE_DIR/.gotmp/bin/windows_amd64 >/dev/null
-curl --fail -JL -s -o mkcert "https://dl.filippo.io/mkcert/latest?for=windows/amd64"
+curl --fail -JL -s -o mkcert.exe "https://github.com/FiloSottile/mkcert/releases/download/${MKCERT_VERSION}/mkcert-${MKCERT_VERSION}-windows-amd64.exe"
 tar -czf $ARTIFACTS/ddev_windows-amd64.$VERSION.tar.gz ddev.exe  mkcert.exe
 popd >/dev/null
 

.github/devcontainers/src/install-ddev/install.sh

@@ -4,5 +4,4 @@ set -eu -o pipefail
 sudo install -m 0755 -d /etc/apt/keyrings
 curl -fsSL https://pkg.ddev.com/apt/gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/ddev.gpg > /dev/null
 echo "deb [signed-by=/etc/apt/keyrings/ddev.gpg] https://pkg.ddev.com/apt/ * *" | sudo tee /etc/apt/sources.list.d/ddev.list
-sudo apt update >/dev/null && sudo apt install -y ddev xdg-utils >/dev/null
-
+sudo apt-get update >/dev/null && sudo apt-get install -y ddev xdg-utils >/dev/null

.github/workflows/master-build.yml

@@ -258,32 +258,32 @@ jobs:
         run: ./.ci-scripts/generate_artifacts.sh ${{ github.workspace }}/artifacts
 
       - name: Upload all artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: all-ddev-executables
           path: ${{ github.workspace }}/artifacts/*
       - name: Upload macos-amd64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-macos-amd64
           path: .gotmp/bin/darwin_amd64/ddev
       - name: Upload macos-arm64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-macos-arm64
           path: .gotmp/bin/darwin_arm64/ddev
       - name: Upload linux-arm64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-linux-arm64
           path: .gotmp/bin/linux_arm64/ddev
       - name: Upload linux-amd64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-linux-amd64
           path: .gotmp/bin/linux_amd64/ddev
       - name: Upload windows_amd64 installer
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-windows-amd64-installer
           path: .gotmp/bin/windows_amd64/ddev_windows_installer*.exe

.github/workflows/pr-build.yml

@@ -64,32 +64,32 @@ jobs:
         run: ./.ci-scripts/generate_artifacts.sh ${{ github.workspace }}/artifacts
 
       - name: Upload all artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: all-ddev-executables
           path: ${{ github.workspace }}/artifacts/*
       - name: Upload macos-amd64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-macos-amd64
           path: .gotmp/bin/darwin_amd64/ddev
       - name: Upload macos-arm64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-macos-arm64
           path: .gotmp/bin/darwin_arm64/ddev
       - name: Upload linux-arm64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-linux-arm64
           path: .gotmp/bin/linux_arm64/ddev
       - name: Upload linux-amd64 binary
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-linux-amd64
           path: .gotmp/bin/linux_amd64/ddev
       - name: Upload windows_amd64 installer
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: ddev-windows-amd64-installer
           path: .gotmp/bin/windows_amd64/ddev_windows_installer*.exe

cmd/ddev/cmd/autocompletion_test.go

@@ -216,6 +216,9 @@ func TestAutocompletionForCustomCmds(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skip("Skipping because untested on Windows")
 	}
+	if dockerutil.IsColima() {
+		t.Skip("Skipping on Colima because of slow mount responses")
+	}
 	assert := asrt.New(t)
 
 	origDir, _ := os.Getwd()

cmd/ddev/cmd/root_test.go

@@ -2,14 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	osexec "os/exec"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"testing"
-
 	"github.com/ddev/ddev/pkg/config/types"
 	"github.com/ddev/ddev/pkg/ddevapp"
 	"github.com/ddev/ddev/pkg/dockerutil"
@@ -24,6 +16,13 @@ import (
 	log "github.com/sirupsen/logrus"
 	asrt "github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"os"
+	osexec "os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
 )
 
 func init() {
@@ -343,6 +342,8 @@ func TestPoweroffOnNewVersion(t *testing.T) {
 	err = os.Chdir(TestSites[0].Dir)
 	assert.NoError(err)
 
+	tmpXdgConfigHomeDir := testcommon.CopyGlobalDdevDir(t)
+
 	// Create an extra junk project to make sure it gets shut down on our start
 	junkName := t.Name() + "-tmpjunkproject"
 	_, _ = exec.RunHostCommand(DdevBin, "delete", "-Oy", junkName)
@@ -355,19 +356,14 @@ func TestPoweroffOnNewVersion(t *testing.T) {
 	_, err = exec.RunHostCommand(DdevBin, "start", "-y")
 	assert.NoError(err)
 
-	tmpXdgConfigHomeDir := testcommon.CopyGlobalDdevDir(t)
-
 	t.Cleanup(func() {
 		err = os.Chdir(origDir)
 		assert.NoError(err)
 
 		t.Logf("attempting to remove project %s", junkName)
-		_, err = exec.RunHostCommand(DdevBin, "delete", "-Oy", junkName)
-		assert.NoError(err)
-
-		t.Logf("attempting to remove project files in %s", tmpJunkProjectDir)
-		err = os.RemoveAll(tmpJunkProjectDir)
-		assert.NoError(err)
+		out, err = exec.RunHostCommand(DdevBin, "delete", "-Oy", junkName)
+		require.NoError(t, err, "failed to remove project %s, out='%s' err=%v", junkName, out, err)
+		t.Logf("Output from 'ddev delete -Oy %s' was '%s'", junkName, out)
 
 		testcommon.ResetGlobalDdevDir(t, tmpXdgConfigHomeDir)
 
@@ -376,6 +372,12 @@ func TestPoweroffOnNewVersion(t *testing.T) {
 		for _, site := range TestSites {
 			_, _ = exec.RunCommand(DdevBin, []string{"start", "-y", site.Name})
 		}
+
+		t.Logf("attempting to remove project files in %s", tmpJunkProjectDir)
+		err = os.RemoveAll(tmpJunkProjectDir)
+		if err != nil {
+			t.Logf("failed to remove junk project files in %s: %v", tmpJunkProjectDir, err)
+		}
 	})
 
 	apps := ddevapp.GetActiveProjects()

cmd/ddev/cmd/scripts/test_ddev.sh

@@ -88,7 +88,7 @@ function cleanup {
 
 ddev config --project-type=php --docroot=web --disable-upload-dirs-warning || (printf "\n\nPlease run 'ddev debug test' in the root of the existing project where you're having trouble.\n\n" && exit 4)
 
-printf "RUN apt update\nRUN curl -I https://www.google.com\n" > .ddev/web-build/Dockerfile.test
+printf "RUN apt-get update\nRUN curl -I https://www.google.com\n" > .ddev/web-build/Dockerfile.test
 
 set +eu
 

cmd/ddev/cmd/start_test.go

@@ -23,27 +23,17 @@ func TestCmdStart(t *testing.T) {
 	require.NoError(t, err)
 
 	// Stop all sites.
-	_, err = exec.RunCommand(DdevBin, []string{"stop", "--all"})
+	out, err := exec.RunCommand(DdevBin, []string{"stop", "--all"})
 	assert.NoError(err)
-
-	// Ensure all sites are started after ddev start --all.
-	out, err := exec.RunCommand(DdevBin, []string{"start", "--all", "-y"})
-	assert.NoError(err, "ddev start --all should succeed but failed, err: %v, output: %s", err, out)
 	testcommon.CheckGoroutineOutput(t, out)
 
-	// Confirm all sites are running.
-	apps := ddevapp.GetActiveProjects()
-	for _, app := range apps {
-		status, statusDesc := app.SiteStatus()
-		assert.Equal(ddevapp.SiteRunning, status, "All sites should be running, but project=%s status=%sstatus description=%s", app.GetName(), status, statusDesc)
-		assert.Equal(ddevapp.SiteRunning, statusDesc, "The status description should be \"running\", but %s status description is: %s", app.GetName(), statusDesc)
+	apps := []*ddevapp.DdevApp{}
+	for _, testSite := range TestSites {
+		app, err := ddevapp.NewApp(testSite.Dir, false)
+		require.NoError(t, err)
+		apps = append(apps, app)
 	}
 
-	// Stop all sites.
-	out, err = exec.RunCommand(DdevBin, []string{"stop", "--all"})
-	assert.NoError(err)
-	testcommon.CheckGoroutineOutput(t, out)
-
 	// Build start command startMultipleArgs
 	startMultipleArgs := []string{"start", "-y"}
 	for _, app := range apps {
@@ -59,6 +49,6 @@ func TestCmdStart(t *testing.T) {
 	for _, app := range apps {
 		status, statusDesc := app.SiteStatus()
 		assert.Equal(ddevapp.SiteRunning, status, "All sites should be running, but project=%s status=%s statusDesc=%s", app.GetName(), status, statusDesc)
-		assert.Equal(ddevapp.SiteRunning, statusDesc, "The status description should be \"running\", but %s status description is: %s", app.GetName(), statusDesc)
+		assert.Equal(ddevapp.SiteRunning, statusDesc, `The status description should be "running", but project %s status  is: %s`, app.GetName(), statusDesc)
 	}
 }

containers/ddev-ssh-agent/Dockerfile

@@ -1,6 +1,6 @@
 FROM debian:bullseye-slim
 
-RUN apt-get update && apt-get install -y bash expect file openssh-client socat psmisc && apt autoclean
+RUN apt-get update && apt-get install -y bash expect file openssh-client socat psmisc && apt-get autoclean
 
 # Copy container files
 COPY files /

containers/ddev-webserver/Dockerfile

@@ -90,6 +90,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install -o Dpkg::Options::="--for
     unzip \
     zip
 
+# Remove blackfire from apt sources, because we pin to a specific version, see https://github.com/ddev/ddev/issues/6078
+RUN rm /etc/apt/sources.list.d/blackfire.list
+
 RUN curl --fail -JL -s -o /usr/local/bin/mkcert "https://dl.filippo.io/mkcert/latest?for=${TARGETPLATFORM}" && chmod +x /usr/local/bin/mkcert
 
 # blackfire user by default is set up with /dev/null as homedir, and 999 as uid, which
@@ -141,7 +144,7 @@ RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/ngin
 RUN mkdir -p /var/xhprof && curl --fail  -o /tmp/xhprof.tgz -sSL https://pecl.php.net/get/xhprof && tar -zxf /tmp/xhprof.tgz --strip-components=1 -C /var/xhprof && chmod 777 /var/xhprof/xhprof_html && rm /tmp/xhprof.tgz
 
 RUN touch /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log && \
-  chmod 666 /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log
+    chmod 666 /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log
 
 RUN a2dismod mpm_event
 RUN a2enmod ssl headers expires
@@ -224,6 +227,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -qq install -o Dpkg::Options::="--for
     unzip \
     zip
 
+# Remove blackfire from apt sources, because we pin to a specific version, see https://github.com/ddev/ddev/issues/6078
+RUN rm /etc/apt/sources.list.d/blackfire.list
+
 RUN curl --fail -JL -s -o /usr/local/bin/mkcert "https://dl.filippo.io/mkcert/latest?for=${TARGETPLATFORM}" && chmod +x /usr/local/bin/mkcert
 
 ADD ddev-webserver-prod-files /
@@ -250,7 +256,7 @@ RUN mkdir -p /mnt/ddev-global-cache/mkcert /run/php /var/cache/nginx /var/lib/ng
 RUN chmod -fR ugo+w /usr/sbin /usr/bin /etc/nginx /var/cache/nginx /var/lib/nginx /run /var/www /etc/php/*/*/conf.d/ /var/lib/php/modules /etc/alternatives /usr/lib/node_modules /etc/php /etc/apache2 /var/lock/apache2 /var/log/apache2/ /var/run/apache2 /var/lib/apache2 /mnt/ddev-global-cache/*
 
 RUN touch /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log && \
-  chmod 666 /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log
+    chmod 666 /var/log/nginx/error.log /var/log/nginx/access.log /var/log/php-fpm.log
 
 RUN a2dismod mpm_event
 RUN a2enmod ssl headers expires

docs/content/developers/building-contributing.md

@@ -9,7 +9,7 @@ search:
 There are several ways to use DDEV’s latest-committed HEAD version:
 
 * **Download** the latest master branch artifacts from [nightly.link](https://nightly.link/ddev/ddev/workflows/master-build/master). Each of these is built by the CI system, signed, and notarized. Get the one you need and place it in your `$PATH`.
-* **Homebrew install HEAD**: On macOS and Linux, run `brew unlink ddev && brew install ddev/ddev/ddev --HEAD --fetch-HEAD` to get the latest DDEV commit, even if it’s unreleased. Since you’re building this on your own computer, it’s not signed or notarized, and you’ll get a notification that instrumentation doesn’t work, which is fine. If you’re using Linux/WSL2, you’ll likely need to install build-essential by running the following command: `sudo apt install -y build-essential`.
+* **Homebrew install HEAD**: On macOS and Linux, run `brew unlink ddev && brew install ddev/ddev/ddev --HEAD --fetch-HEAD` to get the latest DDEV commit, even if it’s unreleased. Since you’re building this on your own computer, it’s not signed or notarized, and you’ll get a notification that instrumentation doesn’t work, which is fine. If you’re using Linux/WSL2, you’ll likely need to install build-essential by running the following command: `sudo apt-get install -y build-essential`.
 * **Build manually**: If you have normal build tools like `make` and `go` installed, you can check out the code and run `make`.
 * **Gitpod** You can use the latest build by visiting DDEV on [Gitpod](https://gitpod.io/#https://github.com/ddev/ddev).
 
@@ -84,12 +84,54 @@ rm ~/bin/ddev
 
 [Gitpod](https://www.gitpod.io) provides a quick, preconfigured DDEV experience in the browser for testing a PR easily without the need to set up an environment. For any PR you can use the URL `https://gitpod.io/#https://github.com/ddev/ddev/pull/<YOUR-PR>` to open that PR and build it in Gitpod.
 
-To open and work on DDEV master branch you can use the button below.
+It also allows you to work on the DDEV master branch and test modifiying DDEV's source code.
+
+To get started use the button below:
+
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/ddev/ddev)
 
-If you want to run a web project, you can check it out into `/workspace/<yourproject>` and use it as usual. The things you’re familiar with work normally, except that `ddev-router` does not run.
+For a simple test, edit `r/cmd/ddev/cmd/start.go` and change the line
+
+```go
+    output.UserOut.Printf("Starting %s...", project.GetName())
+```
+
+to
+
+```go
+    output.UserOut.Printf("Let's gooooo ... %s...", project.GetName())
+```
+
+Compile and install your new modified DDEV version:
+
+```bash
+cd /workspace/ddev/
+make
+```
+
+The command `ddev -v` now will output something like `ddev version v1.23.1-20-g70fc4cd7b-dirty`. The version will stay the same for all compilations until you make a commit.
 
-A Gitpod dummy project for is provided by default in `/workspace/d10simple`. If you’re testing your own project, you will need to delete it to free up reserved host ports by running `ddev delete -Oy d10simple`. Then you can run [`ddev start`](../users/usage/commands.md#start) to work with your own.
+A Gitpod dummy project for is provided by default in `/workspace/d10simple` to test your changes:
+
+```bash
+cd /workspace/d10simple/
+ddev start
+```
+
+If you want to create a new project or use your own project, you will need to delete the dummy project to free up reserved host ports by running `ddev delete -Oy d10simple`.
+
+Afterwards you can run [`ddev config`](../users/usage/commands.md#config) as usual:
+
+```bash
+cd /workspace/
+mkdir my-new-project/
+cd my-new-project/
+ddev config
+```
+
+If you want to use an existing web project, also check it out into `/workspace/<yourproject>` and use it as usual.
+
+The things you’re familiar with work normally, except that `ddev-router` does not run.  
 
 ## Making Changes to DDEV Images
 
@@ -392,3 +434,7 @@ The rules:
 9. Even though we call these “rules” above, they are guidelines. Since you’ve read all the rules, you now know that.
 
 If you are having trouble getting into the mood of idiomatic Go, we recommend reading through [Effective Go](https://golang.org/doc/effective_go.html). The [Go Blog](https://blog.golang.org) is also a great resource. Drinking the kool-aid is a lot easier than going thirsty.
+
+## Contributor Live Training
+
+We’re actively trying to increase the DDEV community of contributors and maintainers. To do that, we regularly do contributor training, and we’d love to have you come. The trainings are recorded for everybody’s benefit. The recordings and upcoming session dates can be found here: [DDEV Contributor Live Training](https://ddev.com/blog/contributor-training/).