Add utilities and tests for CIFS simulated symlinks, add to ddev comp…

…oser, fixes #1283  (#1323)

* Add utilities and tests for cifs simulated symlinks
* Add symlink replacement into composer command

cmd/ddev/cmd/composer-create.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/drud/ddev/pkg/fileutil"
@@ -192,6 +193,9 @@ project root will be deleted when creating a project.`,
 		if err != nil {
 			util.Failed("Failed to create project: %v", err)
 		}
+		if runtime.GOOS == "windows" && !util.IsDockerToolbox() {
+			replaceSimulatedLinks(app.AppRoot)
+		}
 
 	},
 }

cmd/ddev/cmd/composer.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/drud/ddev/pkg/fileutil"
+	"runtime"
 	"strings"
 
 	"github.com/drud/ddev/pkg/ddevapp"
@@ -38,6 +40,9 @@ ddev composer outdated --minor-only`,
 			Dir:     "/var/www/html",
 			Cmd:     append([]string{"composer"}, args...),
 		})
+		if runtime.GOOS == "windows" && !util.IsDockerToolbox() {
+			replaceSimulatedLinks(app.AppRoot)
+		}
 
 		if len(stdout) > 0 {
 			fmt.Println(stdout)
@@ -49,3 +54,30 @@ func init() {
 	RootCmd.AddCommand(ComposerCmd)
 	ComposerCmd.Flags().SetInterspersed(false)
 }
+
+// replaceSimulatedLinks() walks the path provided and tries to replace XSym links with real ones.
+func replaceSimulatedLinks(path string) {
+	links, err := fileutil.FindSimulatedXsymSymlinks(path)
+	if err != nil {
+		util.Warning("Error finding XSym Symlinks: %v", err)
+	}
+	if len(links) == 0 {
+		return
+	}
+
+	if !fileutil.CanCreateSymlinks() {
+		util.Warning("This host computer is unable to create real symlinks, please see the docs to enable developer mode:\n%s\nNote that the simulated symlinks created inside the container will work fine for most projects.", "https://ddev.readthedocs.io/en/latest/users/developer-tools/#windows-os-and-ddev-composer")
+		return
+	}
+
+	err = fileutil.ReplaceSimulatedXsymSymlinks(links)
+	if err != nil {
+		util.Warning("Failed replacing simulated symlinks: %v", err)
+	}
+	replacedLinks := make([]string, 0)
+	for _, l := range links {
+		replacedLinks = append(replacedLinks, l.LinkLocation)
+	}
+	util.Success("Replaced these simulated symlinks with real symlinks: %v", replacedLinks)
+	return
+}

docs/users/developer-tools.md

@@ -28,6 +28,21 @@ To execute a fully-featured `composer create-project` command, you can execute t
 
 `ddev exec composer create-project ...`
 
+#### Windows OS and `ddev composer`
+
+Both composer and Docker for Windows introduce quite complex filesystem workarounds. DDEV attempts to help you with each of them.
+
+You generally don't have to worry about any of this, but it does keep things cleaner. Mostly just a few of the more complex TYPO3 projects have been affected.
+
+* On Docker for Windows, symlinks are created in the container as "simulated symlinks", or XSym files. These are special text files that behave as symlinks inside the container (on CIFS filesystem), but appear as simple text files on the Windows host. (on the CIFS filesystem used by Docker for Windows inside the container there is no capability to create real symlinks, even though Windows now has this capability.)
+* DDEV-Local attempts to clean up for this situation. Since Windows 10 (in developer mode) can now create real symlinks, DDEV-Local scans your repository after a `ddev composer` command and attempts to convert XSym files into real symlinks. It can only do this if your Windows 10 host is set to Developer Mode.
+* On Windows 10, to set your computer to developer mode, search for "developer" in settings. Screenshots are below.
+* Docker Toolbox (Windows 10 Home) does not have any capability to create symlinks inside the web container on a mount. But composer detects this and uses copies instead of symlinks.
+
+![finding developer mode](images/developer_mode_1.png)
+
+![setting developer mode](images/developer_mode_2.png)
+
 ### Email Capture and Review
 
 [MailHog](https://github.com/mailhog/MailHog) is a mail catcher which is configured to capture and display emails sent by PHP in the development environment.

pkg/ddevapp/ddevapp_test.go

@@ -183,13 +183,20 @@ func TestMain(m *testing.M) {
 			log.Errorf("TestMain startup: app.Init() failed on site %s in dir %s, err=%v", TestSites[i].Name, TestSites[i].Dir, err)
 			continue
 		}
+		err = app.WriteConfig()
+		if err != nil {
+			testRun = -1
+			log.Errorf("TestMain startup: app.WriteConfig() failed on site %s in dir %s, err=%v", TestSites[i].Name, TestSites[i].Dir, err)
+			continue
+		}
 		// TODO: webcache PR will add other volumes that should be removed here.
 		for _, volume := range []string{app.Name + "-mariadb"} {
 			err = dockerutil.RemoveVolume(volume)
 			if err != nil && err.Error() != "no such volume" {
 				log.Errorf("TestMain startup: Failed to delete volume %s: %v", volume, err)
 			}
 		}
+
 		switchDir()
 	}
 
@@ -1249,8 +1256,11 @@ func TestDdevExec(t *testing.T) {
 
 		err := app.Init(site.Dir)
 		assert.NoError(err)
-		err = app.StartAndWaitForSync(0)
-		assert.NoError(err)
+		startErr := app.StartAndWaitForSync(0)
+		if startErr != nil {
+			logs, _ := ddevapp.GetErrLogsFromApp(app, startErr)
+			require.NoError(t, startErr, "Start() failed: %v, logs from broken container:\n=======\n%s\n========\n", startErr, logs)
+		}
 
 		out, _, err := app.Exec(&ddevapp.ExecOpts{
 			Service: "web",

pkg/fileutil/files.go

@@ -9,7 +9,6 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-
 	"strings"
 
 	"runtime"
@@ -228,3 +227,72 @@ func ReadFileIntoString(path string) (string, error) {
 	}
 	return string(bytes), err
 }
+
+type XSymContents struct {
+	LinkLocation string
+	LinkTarget   string
+}
+
+// FindSimulatedXsymSymlinks() searches the basePath provided for files
+// whose first line is XSym, which is used in cifs filesystem for simulated
+// symlinks.
+func FindSimulatedXsymSymlinks(basePath string) ([]XSymContents, error) {
+	symLinks := make([]XSymContents, 0)
+	err := filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		//TODO: Skip a directory named .git? Skip other arbitrary dirs or files?
+		if err == nil && !info.IsDir() {
+			if info.Size() == 1067 {
+				contents, err := ioutil.ReadFile(path)
+				if err != nil {
+					return err
+				}
+				lines := strings.Split(string(contents), "\n")
+				if lines[0] != "XSym" {
+					return nil
+				}
+				if len(lines) < 4 {
+					return fmt.Errorf("Apparent XSym doesn't have enough lines: %s", path)
+				}
+				// target is 4th line
+				linkTarget := filepath.Clean(lines[3])
+				symLinks = append(symLinks, XSymContents{LinkLocation: path, LinkTarget: linkTarget})
+			}
+		}
+		return nil
+	})
+	return symLinks, err
+}
+
+// ReplaceSimulatedXsymSymlinks() walks a list of XSymContents and makes real symlinks
+// in their place. This is only valid on Windows host, only works with Docker for Windows
+// (cifs filesystem)
+func ReplaceSimulatedXsymSymlinks(links []XSymContents) error {
+	for _, item := range links {
+		err := os.Remove(item.LinkLocation)
+		if err != nil {
+			return err
+		}
+		err = os.Symlink(item.LinkTarget, item.LinkLocation)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CanCreateSymlinks tests to see if it's possible to create a symlink
+func CanCreateSymlinks() bool {
+	tmpdir := os.TempDir()
+	linkPath := filepath.Join(tmpdir, RandomFilenameBase())
+	// This doesn't attempt to create the real file; we don't need it.
+	err := os.Symlink(filepath.Join(tmpdir, "realfile.txt"), linkPath)
+	//nolint: errcheck
+	defer os.Remove(linkPath)
+	if err != nil {
+		return false
+	}
+	return true
+}

pkg/fileutil/files_test.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/drud/ddev/pkg/fileutil"
@@ -145,6 +146,66 @@ func TestReplaceStringInFile(t *testing.T) {
 	assert.True(found)
 }
 
+// TestFindSimulatedXsymSymlinks tests FindSimulatedXsymSymlinks
+func TestFindSimulatedXsymSymlinks(t *testing.T) {
+	assert := asrt.New(t)
+	testDir, _ := os.Getwd()
+	targetDir := filepath.Join(testDir, "testdata", "symlinks")
+	links, err := fileutil.FindSimulatedXsymSymlinks(targetDir)
+	assert.NoError(err)
+	assert.Len(links, 8)
+}
+
+// TestReplaceSimulatedXsymSymlinks tries a number of symlinks to make
+// sure we can parse and replace symlinks.
+func TestReplaceSimulatedXsymSymlinks(t *testing.T) {
+	testDir, _ := os.Getwd()
+	assert := asrt.New(t)
+	if runtime.GOOS == "windows" && !fileutil.CanCreateSymlinks() {
+		t.Skip("Skipping on Windows because test machine can't create symlnks")
+	}
+	sourceDir := filepath.Join(testDir, "testdata", "symlinks")
+	targetDir := testcommon.CreateTmpDir("TestReplaceSimulated")
+	//nolint: errcheck
+	defer os.RemoveAll(targetDir)
+	err := os.Chdir(targetDir)
+	assert.NoError(err)
+
+	// Make sure we leave the testDir as we found it..
+	//nolint: errcheck
+	defer os.Chdir(testDir)
+	// CopyDir skips real symlinks, but we only care about simulated ones, so it's OK
+	err = fileutil.CopyDir(sourceDir, filepath.Join(targetDir, "symlinks"))
+	assert.NoError(err)
+	links, err := fileutil.FindSimulatedXsymSymlinks(targetDir)
+	assert.NoError(err)
+	assert.Len(links, 8)
+	err = fileutil.ReplaceSimulatedXsymSymlinks(links)
+	assert.NoError(err)
+
+	for _, link := range links {
+		fi, err := os.Stat(link.LinkLocation)
+		assert.NoError(err)
+		linkFi, err := os.Lstat(link.LinkLocation)
+		assert.NoError(err)
+		if err == nil && fi != nil && !fi.IsDir() {
+			// Read the symlink as a file. It should resolve with the actual content of target
+			contents, err := ioutil.ReadFile(link.LinkLocation)
+			assert.NoError(err)
+			expectedContent := "textfile " + filepath.Base(link.LinkTarget) + "\n"
+			assert.Equal(expectedContent, string(contents))
+		}
+		// Now stat the link and make sure it's a link and points where it should
+		if linkFi.Mode()&os.ModeSymlink != 0 {
+			targetFile, err := os.Readlink(link.LinkLocation)
+			assert.NoError(err)
+			assert.Equal(link.LinkTarget, targetFile)
+			_ = targetFile
+		}
+	}
+
+}
+
 // TestIsSameFile tests the IsSameFile utility function.
 func TestIsSameFile(t *testing.T) {
 	assert := asrt.New(t)

pkg/fileutil/testdata/symlinks/biglongdir3.txt

@@ -0,0 +1,5 @@
+XSym
+0042
+9b4557f4243981cf847cf7fd304af785
+subdir/bigdir2/biglongdir3/biglongdir3.txt
+

pkg/fileutil/testdata/symlinks/main1.txt

@@ -0,0 +1 @@
+textfile main1.txt

pkg/fileutil/testdata/symlinks/main1_link.txt

@@ -0,0 +1,5 @@
+XSym
+0009
+719b12a8d1b2dcb554aa957bf7201cdd
+main1.txt
+

pkg/fileutil/testdata/symlinks/main2.txt

@@ -0,0 +1 @@
+textfile main2.txt

pkg/fileutil/testdata/symlinks/main2_link.txt

@@ -0,0 +1,5 @@
+XSym
+0009
+a2e2d38e0825e96b52bc909528036387
+main2.txt
+

pkg/fileutil/testdata/symlinks/sub1.txt

@@ -0,0 +1,5 @@
+XSym
+0015
+65e625dbd52811f0549d1059219d001a
+subdir/sub1.txt
+

pkg/fileutil/testdata/symlinks/sub1_reallink.txt

@@ -0,0 +1 @@
+textfile sub1.txt

pkg/fileutil/testdata/symlinks/sub2.txt

@@ -0,0 +1,5 @@
+XSym
+0015
+a3efc345385208ecafc496fb3f9609fc
+subdir/sub2.txt
+

pkg/fileutil/testdata/symlinks/sub2_reallink.txt

@@ -0,0 +1 @@
+textfile sub2.txt

pkg/fileutil/testdata/symlinks/subdir/bigdir2/biglongdir3/biglongdir3.txt

@@ -0,0 +1 @@
+textfile biglongdir3.txt

pkg/fileutil/testdata/symlinks/subdir/biglongdir3

@@ -0,0 +1,5 @@
+XSym
+0019
+ad4483a1d5e2d2c9e34766c5960fbd42
+bigdir2/biglongdir3
+

pkg/fileutil/testdata/symlinks/subdir/biglongdir3.txt

@@ -0,0 +1,5 @@
+XSym
+0035
+382219eb7a071530a5fe1babe77ae7be
+bigdir2/biglongdir3/biglongdir3.txt
+

pkg/fileutil/testdata/symlinks/subdir/sub1.txt

@@ -0,0 +1 @@
+textfile sub1.txt

pkg/fileutil/testdata/symlinks/subdir/sub1_link_in_subdir.txt

@@ -0,0 +1,5 @@
+XSym
+0008
+cbe2e39f20ece840785599db6c4d4e96
+sub1.txt
+

pkg/fileutil/testdata/symlinks/subdir/sub2.txt

@@ -0,0 +1 @@
+textfile sub2.txt