diff --git a/cmd/ddev/cmd/composer-create.go b/cmd/ddev/cmd/composer-create.go index 3db53d6e9b1..9014b93f5c5 100644 --- a/cmd/ddev/cmd/composer-create.go +++ b/cmd/ddev/cmd/composer-create.go @@ -5,6 +5,7 @@ import ( "os" "path" "path/filepath" + "runtime" "strings" "github.com/drud/ddev/pkg/fileutil" @@ -192,6 +193,9 @@ project root will be deleted when creating a project.`, if err != nil { util.Failed("Failed to create project: %v", err) } + if runtime.GOOS == "windows" && !util.IsDockerToolbox() { + replaceSimulatedLinks(app.AppRoot) + } }, } diff --git a/cmd/ddev/cmd/composer.go b/cmd/ddev/cmd/composer.go index 2c79f6a36b0..9d316b1e606 100644 --- a/cmd/ddev/cmd/composer.go +++ b/cmd/ddev/cmd/composer.go @@ -2,6 +2,8 @@ package cmd import ( "fmt" + "github.com/drud/ddev/pkg/fileutil" + "runtime" "strings" "github.com/drud/ddev/pkg/ddevapp" @@ -38,6 +40,9 @@ ddev composer outdated --minor-only`, Dir: "/var/www/html", Cmd: append([]string{"composer"}, args...), }) + if runtime.GOOS == "windows" && !util.IsDockerToolbox() { + replaceSimulatedLinks(app.AppRoot) + } if len(stdout) > 0 { fmt.Println(stdout) @@ -49,3 +54,30 @@ func init() { RootCmd.AddCommand(ComposerCmd) ComposerCmd.Flags().SetInterspersed(false) } + +// replaceSimulatedLinks() walks the path provided and tries to replace XSym links with real ones. +func replaceSimulatedLinks(path string) { + links, err := fileutil.FindSimulatedXsymSymlinks(path) + if err != nil { + util.Warning("Error finding XSym Symlinks: %v", err) + } + if len(links) == 0 { + return + } + + if !fileutil.CanCreateSymlinks() { + util.Warning("This host computer is unable to create real symlinks, please see the docs to enable developer mode:\n%s\nNote that the simulated symlinks created inside the container will work fine for most projects.", "https://ddev.readthedocs.io/en/latest/users/developer-tools/#windows-os-and-ddev-composer") + return + } + + err = fileutil.ReplaceSimulatedXsymSymlinks(links) + if err != nil { + util.Warning("Failed replacing simulated symlinks: %v", err) + } + replacedLinks := make([]string, 0) + for _, l := range links { + replacedLinks = append(replacedLinks, l.LinkLocation) + } + util.Success("Replaced these simulated symlinks with real symlinks: %v", replacedLinks) + return +} diff --git a/docs/users/developer-tools.md b/docs/users/developer-tools.md index aeddb0268e9..a982acea936 100755 --- a/docs/users/developer-tools.md +++ b/docs/users/developer-tools.md @@ -28,6 +28,21 @@ To execute a fully-featured `composer create-project` command, you can execute t `ddev exec composer create-project ...` +#### Windows OS and `ddev composer` + +Both composer and Docker for Windows introduce quite complex filesystem workarounds. DDEV attempts to help you with each of them. + +You generally don't have to worry about any of this, but it does keep things cleaner. Mostly just a few of the more complex TYPO3 projects have been affected. + +* On Docker for Windows, symlinks are created in the container as "simulated symlinks", or XSym files. These are special text files that behave as symlinks inside the container (on CIFS filesystem), but appear as simple text files on the Windows host. (on the CIFS filesystem used by Docker for Windows inside the container there is no capability to create real symlinks, even though Windows now has this capability.) +* DDEV-Local attempts to clean up for this situation. Since Windows 10 (in developer mode) can now create real symlinks, DDEV-Local scans your repository after a `ddev composer` command and attempts to convert XSym files into real symlinks. It can only do this if your Windows 10 host is set to Developer Mode. +* On Windows 10, to set your computer to developer mode, search for "developer" in settings. Screenshots are below. +* Docker Toolbox (Windows 10 Home) does not have any capability to create symlinks inside the web container on a mount. But composer detects this and uses copies instead of symlinks. + +![finding developer mode](images/developer_mode_1.png) + +![setting developer mode](images/developer_mode_2.png) + ### Email Capture and Review [MailHog](https://github.com/mailhog/MailHog) is a mail catcher which is configured to capture and display emails sent by PHP in the development environment. diff --git a/docs/users/images/developer_mode_1.png b/docs/users/images/developer_mode_1.png new file mode 100644 index 00000000000..137a1d83e61 Binary files /dev/null and b/docs/users/images/developer_mode_1.png differ diff --git a/docs/users/images/developer_mode_2.png b/docs/users/images/developer_mode_2.png new file mode 100644 index 00000000000..c0d2284853b Binary files /dev/null and b/docs/users/images/developer_mode_2.png differ diff --git a/pkg/ddevapp/ddevapp_test.go b/pkg/ddevapp/ddevapp_test.go index c110956ec53..61bf87a768d 100644 --- a/pkg/ddevapp/ddevapp_test.go +++ b/pkg/ddevapp/ddevapp_test.go @@ -183,6 +183,12 @@ func TestMain(m *testing.M) { log.Errorf("TestMain startup: app.Init() failed on site %s in dir %s, err=%v", TestSites[i].Name, TestSites[i].Dir, err) continue } + err = app.WriteConfig() + if err != nil { + testRun = -1 + log.Errorf("TestMain startup: app.WriteConfig() failed on site %s in dir %s, err=%v", TestSites[i].Name, TestSites[i].Dir, err) + continue + } // TODO: webcache PR will add other volumes that should be removed here. for _, volume := range []string{app.Name + "-mariadb"} { err = dockerutil.RemoveVolume(volume) @@ -190,6 +196,7 @@ func TestMain(m *testing.M) { log.Errorf("TestMain startup: Failed to delete volume %s: %v", volume, err) } } + switchDir() } @@ -1249,8 +1256,11 @@ func TestDdevExec(t *testing.T) { err := app.Init(site.Dir) assert.NoError(err) - err = app.StartAndWaitForSync(0) - assert.NoError(err) + startErr := app.StartAndWaitForSync(0) + if startErr != nil { + logs, _ := ddevapp.GetErrLogsFromApp(app, startErr) + require.NoError(t, startErr, "Start() failed: %v, logs from broken container:\n=======\n%s\n========\n", startErr, logs) + } out, _, err := app.Exec(&ddevapp.ExecOpts{ Service: "web", diff --git a/pkg/fileutil/files.go b/pkg/fileutil/files.go index de0d751b815..adc715cf56d 100644 --- a/pkg/fileutil/files.go +++ b/pkg/fileutil/files.go @@ -9,7 +9,6 @@ import ( "math/rand" "os" "path/filepath" - "strings" "runtime" @@ -228,3 +227,72 @@ func ReadFileIntoString(path string) (string, error) { } return string(bytes), err } + +type XSymContents struct { + LinkLocation string + LinkTarget string +} + +// FindSimulatedXsymSymlinks() searches the basePath provided for files +// whose first line is XSym, which is used in cifs filesystem for simulated +// symlinks. +func FindSimulatedXsymSymlinks(basePath string) ([]XSymContents, error) { + symLinks := make([]XSymContents, 0) + err := filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + //TODO: Skip a directory named .git? Skip other arbitrary dirs or files? + if err == nil && !info.IsDir() { + if info.Size() == 1067 { + contents, err := ioutil.ReadFile(path) + if err != nil { + return err + } + lines := strings.Split(string(contents), "\n") + if lines[0] != "XSym" { + return nil + } + if len(lines) < 4 { + return fmt.Errorf("Apparent XSym doesn't have enough lines: %s", path) + } + // target is 4th line + linkTarget := filepath.Clean(lines[3]) + symLinks = append(symLinks, XSymContents{LinkLocation: path, LinkTarget: linkTarget}) + } + } + return nil + }) + return symLinks, err +} + +// ReplaceSimulatedXsymSymlinks() walks a list of XSymContents and makes real symlinks +// in their place. This is only valid on Windows host, only works with Docker for Windows +// (cifs filesystem) +func ReplaceSimulatedXsymSymlinks(links []XSymContents) error { + for _, item := range links { + err := os.Remove(item.LinkLocation) + if err != nil { + return err + } + err = os.Symlink(item.LinkTarget, item.LinkLocation) + if err != nil { + return err + } + } + return nil +} + +// CanCreateSymlinks tests to see if it's possible to create a symlink +func CanCreateSymlinks() bool { + tmpdir := os.TempDir() + linkPath := filepath.Join(tmpdir, RandomFilenameBase()) + // This doesn't attempt to create the real file; we don't need it. + err := os.Symlink(filepath.Join(tmpdir, "realfile.txt"), linkPath) + //nolint: errcheck + defer os.Remove(linkPath) + if err != nil { + return false + } + return true +} diff --git a/pkg/fileutil/files_test.go b/pkg/fileutil/files_test.go index ec157ba4e5a..b28bff9f671 100644 --- a/pkg/fileutil/files_test.go +++ b/pkg/fileutil/files_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "path/filepath" + "runtime" "testing" "github.com/drud/ddev/pkg/fileutil" @@ -145,6 +146,66 @@ func TestReplaceStringInFile(t *testing.T) { assert.True(found) } +// TestFindSimulatedXsymSymlinks tests FindSimulatedXsymSymlinks +func TestFindSimulatedXsymSymlinks(t *testing.T) { + assert := asrt.New(t) + testDir, _ := os.Getwd() + targetDir := filepath.Join(testDir, "testdata", "symlinks") + links, err := fileutil.FindSimulatedXsymSymlinks(targetDir) + assert.NoError(err) + assert.Len(links, 8) +} + +// TestReplaceSimulatedXsymSymlinks tries a number of symlinks to make +// sure we can parse and replace symlinks. +func TestReplaceSimulatedXsymSymlinks(t *testing.T) { + testDir, _ := os.Getwd() + assert := asrt.New(t) + if runtime.GOOS == "windows" && !fileutil.CanCreateSymlinks() { + t.Skip("Skipping on Windows because test machine can't create symlnks") + } + sourceDir := filepath.Join(testDir, "testdata", "symlinks") + targetDir := testcommon.CreateTmpDir("TestReplaceSimulated") + //nolint: errcheck + defer os.RemoveAll(targetDir) + err := os.Chdir(targetDir) + assert.NoError(err) + + // Make sure we leave the testDir as we found it.. + //nolint: errcheck + defer os.Chdir(testDir) + // CopyDir skips real symlinks, but we only care about simulated ones, so it's OK + err = fileutil.CopyDir(sourceDir, filepath.Join(targetDir, "symlinks")) + assert.NoError(err) + links, err := fileutil.FindSimulatedXsymSymlinks(targetDir) + assert.NoError(err) + assert.Len(links, 8) + err = fileutil.ReplaceSimulatedXsymSymlinks(links) + assert.NoError(err) + + for _, link := range links { + fi, err := os.Stat(link.LinkLocation) + assert.NoError(err) + linkFi, err := os.Lstat(link.LinkLocation) + assert.NoError(err) + if err == nil && fi != nil && !fi.IsDir() { + // Read the symlink as a file. It should resolve with the actual content of target + contents, err := ioutil.ReadFile(link.LinkLocation) + assert.NoError(err) + expectedContent := "textfile " + filepath.Base(link.LinkTarget) + "\n" + assert.Equal(expectedContent, string(contents)) + } + // Now stat the link and make sure it's a link and points where it should + if linkFi.Mode()&os.ModeSymlink != 0 { + targetFile, err := os.Readlink(link.LinkLocation) + assert.NoError(err) + assert.Equal(link.LinkTarget, targetFile) + _ = targetFile + } + } + +} + // TestIsSameFile tests the IsSameFile utility function. func TestIsSameFile(t *testing.T) { assert := asrt.New(t) diff --git a/pkg/fileutil/testdata/symlinks/biglongdir3.txt b/pkg/fileutil/testdata/symlinks/biglongdir3.txt new file mode 100644 index 00000000000..8e41e02d413 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/biglongdir3.txt @@ -0,0 +1,5 @@ +XSym +0042 +9b4557f4243981cf847cf7fd304af785 +subdir/bigdir2/biglongdir3/biglongdir3.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/main1.txt b/pkg/fileutil/testdata/symlinks/main1.txt new file mode 100644 index 00000000000..d14b81eedfb --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/main1.txt @@ -0,0 +1 @@ +textfile main1.txt diff --git a/pkg/fileutil/testdata/symlinks/main1_link.txt b/pkg/fileutil/testdata/symlinks/main1_link.txt new file mode 100644 index 00000000000..9c1601e25fd --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/main1_link.txt @@ -0,0 +1,5 @@ +XSym +0009 +719b12a8d1b2dcb554aa957bf7201cdd +main1.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/main2.txt b/pkg/fileutil/testdata/symlinks/main2.txt new file mode 100644 index 00000000000..03bdf2d5c80 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/main2.txt @@ -0,0 +1 @@ +textfile main2.txt diff --git a/pkg/fileutil/testdata/symlinks/main2_link.txt b/pkg/fileutil/testdata/symlinks/main2_link.txt new file mode 100644 index 00000000000..dcb1fb1c6c3 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/main2_link.txt @@ -0,0 +1,5 @@ +XSym +0009 +a2e2d38e0825e96b52bc909528036387 +main2.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/sub1.txt b/pkg/fileutil/testdata/symlinks/sub1.txt new file mode 100644 index 00000000000..8f0e0b41a0a --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/sub1.txt @@ -0,0 +1,5 @@ +XSym +0015 +65e625dbd52811f0549d1059219d001a +subdir/sub1.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/sub1_reallink.txt b/pkg/fileutil/testdata/symlinks/sub1_reallink.txt new file mode 100644 index 00000000000..24aed2e5341 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/sub1_reallink.txt @@ -0,0 +1 @@ +textfile sub1.txt diff --git a/pkg/fileutil/testdata/symlinks/sub2.txt b/pkg/fileutil/testdata/symlinks/sub2.txt new file mode 100644 index 00000000000..624dfc62e1d --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/sub2.txt @@ -0,0 +1,5 @@ +XSym +0015 +a3efc345385208ecafc496fb3f9609fc +subdir/sub2.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/sub2_reallink.txt b/pkg/fileutil/testdata/symlinks/sub2_reallink.txt new file mode 100644 index 00000000000..a9e4a2bc0f8 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/sub2_reallink.txt @@ -0,0 +1 @@ +textfile sub2.txt diff --git a/pkg/fileutil/testdata/symlinks/subdir/bigdir2/biglongdir3/biglongdir3.txt b/pkg/fileutil/testdata/symlinks/subdir/bigdir2/biglongdir3/biglongdir3.txt new file mode 100644 index 00000000000..0207b4d63c5 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/bigdir2/biglongdir3/biglongdir3.txt @@ -0,0 +1 @@ +textfile biglongdir3.txt diff --git a/pkg/fileutil/testdata/symlinks/subdir/biglongdir3 b/pkg/fileutil/testdata/symlinks/subdir/biglongdir3 new file mode 100644 index 00000000000..46bb2cbbd8d --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/biglongdir3 @@ -0,0 +1,5 @@ +XSym +0019 +ad4483a1d5e2d2c9e34766c5960fbd42 +bigdir2/biglongdir3 + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/subdir/biglongdir3.txt b/pkg/fileutil/testdata/symlinks/subdir/biglongdir3.txt new file mode 100644 index 00000000000..757a3b2a36f --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/biglongdir3.txt @@ -0,0 +1,5 @@ +XSym +0035 +382219eb7a071530a5fe1babe77ae7be +bigdir2/biglongdir3/biglongdir3.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/subdir/sub1.txt b/pkg/fileutil/testdata/symlinks/subdir/sub1.txt new file mode 100644 index 00000000000..24aed2e5341 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/sub1.txt @@ -0,0 +1 @@ +textfile sub1.txt diff --git a/pkg/fileutil/testdata/symlinks/subdir/sub1_link_in_subdir.txt b/pkg/fileutil/testdata/symlinks/subdir/sub1_link_in_subdir.txt new file mode 100644 index 00000000000..6b42f656a8d --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/sub1_link_in_subdir.txt @@ -0,0 +1,5 @@ +XSym +0008 +cbe2e39f20ece840785599db6c4d4e96 +sub1.txt + \ No newline at end of file diff --git a/pkg/fileutil/testdata/symlinks/subdir/sub2.txt b/pkg/fileutil/testdata/symlinks/subdir/sub2.txt new file mode 100644 index 00000000000..a9e4a2bc0f8 --- /dev/null +++ b/pkg/fileutil/testdata/symlinks/subdir/sub2.txt @@ -0,0 +1 @@ +textfile sub2.txt