Use caching filesystem to improve webserving performance (#1277)

* Experimental use of bg-sync container for performance
* Add bgsync status into ddev list and friends
* For tests that need to wait for sync, wait for the sync
* Make app.Logs() work with ddev-router
* Improve getErrLogsFromApp()
* Give the router just a little longer to come up in healthcheck
* Only prompt for opt-in if they haven't opted in

cmd/ddev/cmd/composer-create.go

@@ -95,10 +95,14 @@ project root will be deleted when creating a project.`,
 		}
 
 		// Define a randomly named temp directory for install target
-		tmpDir := fmt.Sprintf(".tmp_%s", util.RandString(6))
+		tmpDir := fmt.Sprintf(".tmp_ddev_composer_create_%s", util.RandString(6))
 		containerInstallPath := path.Join("/var/www/html", tmpDir)
 		hostInstallPath := filepath.Join(app.AppRoot, tmpDir)
-		defer cleanupTmpDir(hostInstallPath)
+
+		// If WebcacheEnabled, the cleanup is done inline and doesn't have to be done here.
+		if !app.WebcacheEnabled {
+			defer cleanupTmpDir(hostInstallPath)
+		}
 
 		// Build container composer command
 		composerCmd := []string{
@@ -147,34 +151,48 @@ project root will be deleted when creating a project.`,
 		}
 
 		output.UserOut.Printf("Moving installation to project root")
-		err = filepath.Walk(hostInstallPath, func(path string, info os.FileInfo, err error) error {
-			// Skip the initial tmp install directory
-			if path == hostInstallPath {
-				return nil
-			}
 
-			elements := strings.Split(path, tmpDir)
-			newPath := filepath.Join(elements...)
+		// If not webcacheenabled, we will move the contents of the temp installation
+		// using host-side manipulation, but can't do that with cached filesystem.
+		if !app.WebcacheEnabled {
 
-			// Dirs must be created, not renamed
-			if info.IsDir() {
-				if err := os.MkdirAll(newPath, info.Mode()); err != nil {
-					return fmt.Errorf("unable to move %s to %s: %v", path, newPath, err)
+			err = filepath.Walk(hostInstallPath, func(path string, info os.FileInfo, err error) error {
+				// Skip the initial tmp install directory
+				if path == hostInstallPath {
+					return nil
 				}
 
-				return nil
-			}
+				elements := strings.Split(path, tmpDir)
+				newPath := filepath.Join(elements...)
 
-			// Rename files to to a path excluding the tmpDir
-			if err := os.Rename(path, newPath); err != nil {
-				return fmt.Errorf("unable to move %s to %s: %v", path, newPath, err)
-			}
+				// Dirs must be created, not renamed
+				if info.IsDir() {
+					if err := os.MkdirAll(newPath, info.Mode()); err != nil {
+						return fmt.Errorf("unable to move %s to %s: %v", path, newPath, err)
+					}
 
-			return nil
-		})
+					return nil
+				}
+
+				// Rename files to to a path excluding the tmpDir
+				if err := os.Rename(path, newPath); err != nil {
+					return fmt.Errorf("unable to move %s to %s: %v", path, newPath, err)
+				}
+
+				return nil
+			})
+		} else {
+			// If webcacheEnabled, we can move the contents easily and quickly inside the container.
+			_, _, err = app.Exec(&ddevapp.ExecOpts{
+				Service: "web",
+				Cmd:     []string{"bash", "-c", fmt.Sprintf("shopt -s dotglob && mv %s/* /var/www/html && rmdir %s", containerInstallPath, containerInstallPath)},
+			})
+		}
+		// This err check picks up either of the above: The filepath.Walk and the mv
 		if err != nil {
 			util.Failed("Failed to create project: %v", err)
 		}
+
 	},
 }
 

cmd/ddev/cmd/composer_test.go

@@ -1,14 +1,15 @@
 package cmd
 
 import (
+	"github.com/drud/ddev/pkg/ddevapp"
+	"github.com/drud/ddev/pkg/exec"
+	"github.com/drud/ddev/pkg/fileutil"
+	"github.com/drud/ddev/pkg/testcommon"
 	"github.com/drud/ddev/pkg/util"
+	asrt "github.com/stretchr/testify/assert"
 	"os"
+	"path/filepath"
 	"testing"
-
-	"github.com/drud/ddev/pkg/testcommon"
-
-	"github.com/drud/ddev/pkg/exec"
-	asrt "github.com/stretchr/testify/assert"
 )
 
 func TestComposerCmd(t *testing.T) {
@@ -34,18 +35,26 @@ func TestComposerCmd(t *testing.T) {
 	assert.NoError(err)
 	assert.Contains(out, "Available commands:")
 
+	// Get an app just so we can do waits and check webcacheenabled etc.
+	app, err := ddevapp.NewApp(tmpDir, "")
+	assert.NoError(err)
+
 	// Test create-project
 	// ddev composer create cweagans/composer-patches --prefer-dist --no-interaction
 	args = []string{"composer", "create", "--prefer-dist", "--no-interaction", "--no-dev", "psr/log", "1.1.0"}
 	out, err = exec.RunCommand(DdevBin, args)
 	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, out)
 	assert.Contains(out, "Created project in ")
+	ddevapp.WaitForSync(app, 2)
+	assert.FileExists(filepath.Join(tmpDir, "Psr/Log/LogLevel.php"))
 
 	// Test a composer require, with passthrough args
 	args = []string{"composer", "require", "sebastian/version", "--no-plugins", "--ansi"}
 	out, err = exec.RunCommand(DdevBin, args)
 	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, out)
 	assert.Contains(out, "Generating autoload files")
+	ddevapp.WaitForSync(app, 2)
+	assert.FileExists(filepath.Join(tmpDir, "vendor/sebastian/version/composer.json"))
 
 	// Test a composer remove
 	if util.IsDockerToolbox() {
@@ -57,4 +66,6 @@ func TestComposerCmd(t *testing.T) {
 	out, err = exec.RunCommand(DdevBin, args)
 	assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, out)
 	assert.Contains(out, "Generating autoload files")
+	ddevapp.WaitForSync(app, 2)
+	assert.False(fileutil.FileExists(filepath.Join(tmpDir, "vendor/sebastian")))
 }

cmd/ddev/cmd/logs_test.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"github.com/drud/ddev/pkg/ddevapp"
 	"github.com/drud/ddev/pkg/fileutil"
 	"github.com/drud/ddev/pkg/version"
 	"path/filepath"
@@ -13,9 +14,9 @@ import (
 	asrt "github.com/stretchr/testify/assert"
 )
 
-// TestDevLogsNoConfig tests what happens with when running "ddev logs" when
+// TestLogsNoConfig tests what happens with when running "ddev logs" when
 // the directory has not been configured (and no project name is given)
-func TestDevLogsNoConfig(t *testing.T) {
+func TestLogsNoConfig(t *testing.T) {
 	assert := asrt.New(t)
 	testDir := testcommon.CreateTmpDir("no-valid-ddev-config")
 	defer testcommon.CleanupDir(testDir)
@@ -27,8 +28,8 @@ func TestDevLogsNoConfig(t *testing.T) {
 	assert.Contains(string(out), "Please specify a project name or change directories")
 }
 
-// TestDevLogs tests that the Dev logs functionality is working.
-func TestDevLogs(t *testing.T) {
+// TestLogs tests that the ddev logs functionality is working.
+func TestLogs(t *testing.T) {
 	assert := asrt.New(t)
 
 	for _, v := range DevTestSites {
@@ -39,6 +40,11 @@ func TestDevLogs(t *testing.T) {
 		assert.NoError(err)
 		cleanup := v.Chdir()
 
+		app, err := ddevapp.NewApp(v.Dir, "")
+		assert.NoError(err)
+
+		ddevapp.WaitForSync(app, 2)
+
 		url := "http://" + v.Name + "." + version.DDevTLD + "/logtest.php"
 		_, _, err = testcommon.GetLocalHTTPResponse(t, url)
 		assert.NoError(err)

cmd/ddev/cmd/root.go

@@ -151,7 +151,7 @@ func sentryNotSetupWarning() {
 // from the last saved version. If it is, prompt to request anon ddev usage stats
 // and update the info.
 func checkDdevVersionAndOptInSentry() error {
-	if !output.JSONOutput && version.COMMIT != globalconfig.DdevGlobalConfig.LastUsedVersion {
+	if !output.JSONOutput && version.COMMIT != globalconfig.DdevGlobalConfig.LastUsedVersion && globalconfig.DdevGlobalConfig.InstrumentationOptIn == false {
 		allowStats := util.Confirm("It looks like you have a new ddev release.\nMay we send anonymous ddev usage statistics and errors?\nTo know what we will see please take a look at\nhttps://ddev.readthedocs.io/en/latest/users/cli-usage/#opt-in-usage-information\nPermission to beam up?")
 		if allowStats {
 			globalconfig.DdevGlobalConfig.InstrumentationOptIn = true

cmd/ddev/cmd/start.go

@@ -49,6 +49,9 @@ any directory by running 'ddev start projectname [projectname ...]'`,
 
 			util.Success("Successfully started %s", app.GetName())
 			util.Success("Project can be reached at %s", strings.Join(app.GetAllURLs(), ", "))
+			if app.WebcacheEnabled {
+				util.Warning("All contents were copied to fast docker filesystem,\nbut bidirectional sync operation may not be fully functional for a few minutes.")
+			}
 		}
 	},
 }

containers/ddev-bgsync/.gitignore

@@ -0,0 +1,13 @@
+# Temporary build-tools artifacts to be ignored
+.go/
+/bin/
+/.container*
+/.push*
+/linux
+/darwin
+/.dockerfile
+/VERSION.txt
+/.docker_image
+# These are not artifacts that should be ignored
+!files/usr/bin/
+!files/usr/local/bin/

containers/ddev-bgsync/Dockerfile

@@ -0,0 +1,31 @@
+FROM alpine:3.8
+
+RUN apk add --no-cache bash sudo rsync file
+
+# Install Unison from source with inotify support + remove compilation tools
+ARG UNISON_VERSION=2.51.2
+RUN apk add --no-cache --virtual .build-dependencies build-base curl && \
+    apk add --no-cache inotify-tools tzdata && \
+    apk add --no-cache --repository http://dl-4.alpinelinux.org/alpine/edge/testing/ ocaml && \
+    curl -Ll https://github.com/bcpierce00/unison/archive/v${UNISON_VERSION}.tar.gz | tar zxv -C /tmp && \
+    cd /tmp/unison-${UNISON_VERSION} && \
+    sed -i -e 's/GLIBC_SUPPORT_INOTIFY 0/GLIBC_SUPPORT_INOTIFY 1/' src/fsmonitor/linux/inotify_stubs.c && \
+    make UISTYLE=text NATIVE=true STATIC=true && \
+    cp src/unison src/unison-fsmonitor /usr/local/bin && \
+    apk del .build-dependencies ocaml && \
+    rm -rf /tmp/unison-${UNISON_VERSION}
+
+ENV HOME="/root" \
+    UNISONLOCALHOSTNAME="container"
+
+# If run as UNISON_USER other than root, it still uses /root as $HOME
+RUN mkdir -p $HOME/.unison && chmod ugo+rwx $HOME && chmod ugo+rwx $HOME/.unison
+
+ADD files /
+
+# Copy the bg-sync script into /usr/local/bin.
+COPY /files/sync.sh /usr/local/bin/bg-sync
+RUN chmod +x /usr/local/bin/bg-sync
+
+HEALTHCHECK --start-period=30s --interval=10s --retries=5 CMD ["/healthcheck.sh"]
+CMD ["bg-sync"]

containers/ddev-bgsync/Makefile

@@ -0,0 +1,38 @@
+# Makefile for a standard golang repo with associated container
+
+##### These variables need to be adjusted in most repositories #####
+
+# Docker repo for a push
+DOCKER_REPO ?= drud/ddev-bgsync
+
+# Top-level directories to build
+#SRC_DIRS := filexs drudapi secrets utils
+
+# Optional to docker build
+# DOCKER_ARGS =
+
+# VERSION can be set by
+  # Default: git tag
+  # make command line: make VERSION=0.9.0
+# It can also be explicitly set in the Makefile as commented out below.
+
+# This version-strategy uses git tags to set the version string
+# VERSION can be overridden on make commandline: make VERSION=0.9.1 push
+VERSION := $(shell git describe --tags --always --dirty)
+#
+# This version-strategy uses a manual value to set the version string
+#VERSION := 1.2.3
+
+# Each section of the Makefile is included from standard components below.
+# If you need to override one, import its contents below and comment out the
+# include. That way the base components can easily be updated as our general needs
+# change.
+#include ../build-tools/makefile_components/base_build_go.mak
+#include ../../build-tools/makefile_components/base_build_python-docker.mak
+include ../../build-tools/makefile_components/base_container.mak
+include ../../build-tools/makefile_components/base_push.mak
+#include build-tools/makefile_components/base_test_go.mak
+#include build-tools/makefile_components/base_test_python.mak
+
+test:
+	true >/dev/null

containers/ddev-bgsync/README.md

@@ -0,0 +1,55 @@
+# bg-sync
+
+Forked from https://github.com/cweagans/docker-bg-sync at
+https://github.com/cweagans/docker-bg-sync/commit/4e39642ab414f2ba0b55eb3933203115e6ed2c0b
+
+Thanks!
+
+====================
+
+This container continuously syncs files between two directories. This is useful
+for avoiding the filesystem slowness on Docker for Mac, for instance. It's also
+generally useful for any other time where you have a slow filesystem as a source
+of files that need to be read inside of a container.
+
+
+## Environment variables
+
+This container uses values from a handful of environment variables. These are
+documented below.
+
+  * **`SYNC_SOURCE`** (default: `/source`): The path inside the container which
+    will be used as the source of the file sync. Most of the time, you probably
+    shouldn't change the value of this variable. Instead, just bind-mount your
+    files into the container at `/source` and call it a day.
+  * **`SYNC_DESTINATION`** (default: `/destination`): When files are changed in
+    `SYNC_SOURCE`, they will be copied over to the equivalent paths in `SYNC_DESTINATION`.
+    If you are using bg-sync to avoid filesystem slowness, you should set this
+    path to whatever path the volume is at in your application container. In the
+    example above, for instance, this would be `/var/www/myapp`.
+  * **`SYNC_PREFER`** (default: `/source`): Control the conflict strategy to apply
+    when there are conflits. By default the contents from the source folder are
+    left unchanged but there is also the "newer" option to pick up the most
+    recent files.
+  * **`SYNC_VERBOSE`** (default: "0"): Set this variable to "1" to get more log
+    output from Unison.
+  * **`SYNC_MAX_INOTIFY_WATCHES`** (default: ''): If set, the sync script will
+    attempt to increase the value of `fs.inotify.max_user_watches`. **IMPORTANT**:
+    This requires that you run this container as a priviliged container. Otherwise,
+    the inotify limit increase *will not work*. As always, when running a third
+    party container as a priviliged container, look through the source thoroughly
+    first to make sure it won't do anything nefarious. `sync.sh` should be pretty
+    understandable. Go on - read it. I'll wait.
+  * **`SYNC_EXTRA_UNISON_PROFILE_OPTS`** (default: ''): The value of this variable
+    will be appended to the end of the Unison profile that's automatically generated
+    when this container is started. Ensure that the syntax is valid. If you have
+    more than one option that you want to add, simply make this a multiline string.
+    **IMPORTANT**: The *ability* to add extra lines to your Unison profile is
+    supported by the bg-sync project. The *results* of what might happen because
+    of this configuration is *not*. Use this option at your own risk.
+  * **`SYNC_NODELETE_SOURCE`** (default: '1'): Set this variable to "0" to allow
+    Unison to sync deletions to the source directory. This could cause unpredictable
+    behaviour with your source files.
+
+
+