feat: avoid having to remove files in ddev composer create #5493 (#…

…5499)

Co-authored-by: Randy Fay <randy@randyfay.com>

cmd/ddev/cmd/composer-create.go

@@ -29,8 +29,7 @@ var ComposerCreateCmd = &cobra.Command{
 	Short: "Executes 'composer create-project' within the web container with the arguments and flags provided",
 	Long: `Directs basic invocations of 'composer create-project' within the context of the
 web container. Projects will be installed to a temporary directory and moved to
-the Composer root directory after install. Any existing files in the
-composer root will be deleted when creating a project.`,
+the Composer root directory after install.`,
 	Example: `ddev composer create drupal/recommended-project
 ddev composer create -y drupal/recommended-project
 ddev composer create "typo3/cms-base-distribution:^10"
@@ -64,52 +63,38 @@ ddev composer create --prefer-dist --no-interaction --no-dev psr/log
 
 		composerRoot := app.GetComposerRoot(false, false)
 
-		// Make the user confirm that existing contents will be deleted
-		util.Warning("Warning: MOST EXISTING CONTENT in the Composer root (%s) will be deleted by the composer create-project operation. Only .ddev, .git and .tarballs will be preserved.", composerRoot)
-		if !composerCreateYesFlag {
-			if !util.Confirm("Would you like to continue?") {
-				util.Failed("create-project cancelled")
-			}
-		}
-
 		err = os.MkdirAll(composerRoot, 0755)
 		if err != nil {
 			util.Failed("Failed to create composerRoot: %v", err)
 		}
 
-		// Remove most contents of Composer root
-		util.Warning("Removing any existing files in Composer root")
-		objs, err := fileutil.ListFilesInDir(composerRoot)
-		if err != nil {
-			util.Failed("Failed to create project: %v", err)
-		}
-
-		for _, o := range objs {
-			// Preserve .ddev, .git, .tarballs
-			if o == ".ddev" || o == ".git" || o == ".tarballs" {
-				continue
-			}
+		appRoot := app.GetAbsAppRoot(false)
+		skipDirs := []string{".ddev", ".git", ".tarballs"}
+		composerCreateAllowedPaths, _ := app.GetComposerCreateAllowedPaths()
+		err = filepath.Walk(appRoot,
+			func(walkPath string, walkInfo os.FileInfo, err error) error {
+				if walkPath == appRoot {
+					return nil
+				}
 
-			if err = os.RemoveAll(filepath.Join(composerRoot, o)); err != nil {
-				util.Failed("Failed to create project: %v", err)
-			}
-		}
+				checkPath := app.GetRelativeDirectory(walkPath)
 
-		// Upload dirs have to be there for bind-mounts to work when Mutagen enabled
-		app.CreateUploadDirsIfNecessary()
-		// We may have removed host-side directories, invalidating
-		// that Docker bind-mounts are using, so restart to pick up the new ones.
-		if app.IsMutagenEnabled() {
-			err = app.Restart()
-			if err != nil {
-				util.Failed("Failed to restart project after removal of project files: %v", err)
-			}
-		}
+				if walkInfo.IsDir() && nodeps.ArrayContainsString(skipDirs, checkPath) {
+					return filepath.SkipDir
+				}
+				if !nodeps.ArrayContainsString(composerCreateAllowedPaths, checkPath) {
+					return fmt.Errorf("'%s' is not allowed to be present. composer create needs to be run on a clean/empty project with only the following paths: %v - please clean up the project before using 'ddev composer create'", filepath.Join(appRoot, checkPath), composerCreateAllowedPaths)
+				}
+				if err != nil {
+					return err
+				}
+				return nil
+			})
 
-		err = app.MutagenSyncFlush()
 		if err != nil {
-			util.Failed("Failed to sync Mutagen contents: %v", err)
+			util.Failed("Failed to create project: %v", err)
 		}
+
 		// Define a randomly named temp directory for install target
 		tmpDir := util.RandString(6)
 		containerInstallPath := path.Join("/tmp", tmpDir)

cmd/ddev/cmd/composer-create_test.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/ddev/ddev/pkg/config/types"
+	"github.com/ddev/ddev/pkg/ddevapp"
+	"github.com/ddev/ddev/pkg/exec"
+	"github.com/ddev/ddev/pkg/nodeps"
+	"github.com/ddev/ddev/pkg/testcommon"
+	asrt "github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposerCreateCmd(t *testing.T) {
+	// 2022-05-24: I've spent lots of time debugging intermittent `composer create` failures when NFS
+	// is enabled, both on macOS and Windows. As far as I can tell, it only happens in this test, I've
+	// never recreated manually. I do see https://github.com/composer/composer/issues/9627 which seemed
+	// to deal with similar issues in vagrant context, and has a hack now embedded into Composer.
+	if nodeps.PerformanceModeDefault == types.PerformanceModeNFS {
+		t.Skip("Composer has strange behavior in NFS context, so skipping")
+	}
+	assert := asrt.New(t)
+
+	origDir, err := os.Getwd()
+	assert.NoError(err)
+
+	types := ddevapp.GetValidAppTypes()
+	if os.Getenv("GOTEST_SHORT") != "" {
+		types = []string{nodeps.AppTypePHP, nodeps.AppTypeDrupal10}
+	}
+
+	for _, docRoot := range []string{"", "doc-root"} {
+		for _, projectType := range types {
+			if projectType == nodeps.AppTypeDjango4 || projectType == nodeps.AppTypePython {
+				// Skip as an empty django4/python do not start nicely right away
+				// https://github.com/ddev/ddev/issues/5171
+				t.Logf("== SKIP TestComposerCreateCmd for project of type '%s' with docroot  '%s'\n", projectType, docRoot)
+				t.Logf("== SKIP python projects are not starting up nicely and composer create is very unlikely to be used")
+				continue
+			}
+			if projectType == nodeps.AppTypeDrupal6 {
+				// Skip as an empty django4/python do not start nicely right away
+				// https://github.com/ddev/ddev/issues/5171
+				t.Logf("== SKIP TestComposerCreateCmd for project of type '%s' with docroot  '%s'\n", projectType, docRoot)
+				t.Logf("== SKIP drupal6 projects uses a very old php version and composer create is very unlikely to be used")
+				continue
+			}
+			t.Logf("== BEGIN TestComposerCreateCmd for project of type '%s' with docroot  '%s'\n", projectType, docRoot)
+			tmpDir := testcommon.CreateTmpDir(t.Name() + projectType)
+			err = os.Chdir(tmpDir)
+			assert.NoError(err)
+
+			// Prepare arguments
+			arguments := []string{"config", "--project-type", projectType}
+
+			composerRoot := tmpDir
+			if docRoot != "" {
+				arguments = append(arguments, "--docroot", docRoot, "--create-docroot")
+				// For Drupal10 we test that the composer root is the same as the create root
+				if projectType == nodeps.AppTypeDrupal10 {
+					arguments = append(arguments, "--composer-root", docRoot)
+					composerRoot = filepath.Join(tmpDir, docRoot)
+				}
+			}
+
+			// Basic config
+			_, err = exec.RunHostCommand(DdevBin, arguments...)
+			assert.NoError(err)
+
+			// Test trivial command
+			out, err := exec.RunHostCommand(DdevBin, "composer")
+			assert.NoError(err)
+			assert.Contains(out, "Available commands:")
+
+			// Get an app so we can do waits
+			app, err := ddevapp.NewApp(tmpDir, true)
+			assert.NoError(err)
+
+			t.Cleanup(func() {
+				err = os.Chdir(origDir)
+				assert.NoError(err)
+				err = os.RemoveAll(tmpDir)
+				assert.NoError(err)
+			})
+
+			err = app.StartAndWait(5)
+			require.NoError(t, err)
+			// ddev composer create --prefer-dist --no-interaction --no-dev psr/log:1.1.0
+			args := []string{"composer", "create", "--prefer-dist", "--no-interaction", "--no-dev", "--no-install", "psr/log:1.1.0"}
+
+			// Test failure
+			file, err := os.Create(filepath.Join(composerRoot, "touch1.txt"))
+			out, err = exec.RunHostCommand(DdevBin, args...)
+			assert.Error(err)
+			assert.Contains(out, "touch1.txt")
+			_ = file.Close()
+			os.Remove(filepath.Join(composerRoot, "touch1.txt"))
+
+			// Test success
+			out, err = exec.RunHostCommand(DdevBin, args...)
+			assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
+			assert.Contains(out, "Created project in ")
+			assert.FileExists(filepath.Join(composerRoot, "composer.json"))
+
+			err = app.Stop(true, false)
+			require.NoError(t, err)
+		}
+	}
+}

cmd/ddev/cmd/composer_test.go

@@ -7,19 +7,14 @@ import (
 
 	"github.com/ddev/ddev/pkg/config/types"
 	"github.com/ddev/ddev/pkg/ddevapp"
-	"github.com/ddev/ddev/pkg/dockerutil"
 	"github.com/ddev/ddev/pkg/exec"
 	"github.com/ddev/ddev/pkg/fileutil"
 	"github.com/ddev/ddev/pkg/nodeps"
 	"github.com/ddev/ddev/pkg/testcommon"
 	asrt "github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
-func TestComposerCmd(t *testing.T) {
-	if dockerutil.IsColima() {
-		t.Skip("Skipping test on Colima because of odd unable to delete vendor error")
-	}
+func TestComposerCmdCreateConfigInstall(t *testing.T) {
 	// 2022-05-24: I've spent lots of time debugging intermittent `composer create` failures when NFS
 	// is enabled, both on macOS and Windows. As far as I can tell, it only happens in this test, I've
 	// never recreated manually. I do see https://github.com/composer/composer/issues/9627 which seemed
@@ -90,11 +85,66 @@ func TestComposerCmd(t *testing.T) {
 		assert.NoError(err)
 
 		assert.FileExists(filepath.Join(tmpDir, composerRoot, "Psr/Log/LogLevel.php"))
+	}
+}
+
+func TestComposerCmdCreateRequireRemove(t *testing.T) {
+	// 2022-05-24: I've spent lots of time debugging intermittent `composer create` failures when NFS
+	// is enabled, both on macOS and Windows. As far as I can tell, it only happens in this test, I've
+	// never recreated manually. I do see https://github.com/composer/composer/issues/9627 which seemed
+	// to deal with similar issues in vagrant context, and has a hack now embedded into Composer.
+	if nodeps.PerformanceModeDefault == types.PerformanceModeNFS {
+		t.Skip("Composer has strange behavior in NFS context, so skipping")
+	}
+	assert := asrt.New(t)
+
+	origDir, err := os.Getwd()
+	assert.NoError(err)
+
+	for _, composerRoot := range []string{"", "composer-root"} {
+		tmpDir := testcommon.CreateTmpDir(t.Name())
+		err = os.Chdir(tmpDir)
+		assert.NoError(err)
+
+		// Prepare arguments
+		arguments := []string{"config", "--project-type", "php"}
+
+		if composerRoot != "" {
+			arguments = append(arguments, "--composer-root", composerRoot)
+			err = os.Mkdir(composerRoot, 0777)
+			assert.NoError(err)
+		}
+
+		// Basic config
+		_, err = exec.RunHostCommand(DdevBin, arguments...)
+		assert.NoError(err)
+
+		// Test trivial command
+		out, err := exec.RunHostCommand(DdevBin, "composer")
+		assert.NoError(err)
+		assert.Contains(out, "Available commands:")
+
+		// Get an app so we can do waits
+		app, err := ddevapp.NewApp(tmpDir, true)
+		assert.NoError(err)
+
+		t.Cleanup(func() {
+			//nolint: errcheck
+			err = app.Stop(true, false)
+			assert.NoError(err)
+
+			err = os.Chdir(origDir)
+			assert.NoError(err)
+			err = os.RemoveAll(tmpDir)
+			assert.NoError(err)
+		})
+
+		// Test create-project
+		// These two often fail on Windows with NFS, also Colima
+		// It appears to be something about Composer itself?
 
-		err = app.StartAndWait(5)
-		require.NoError(t, err)
 		// ddev composer create --prefer-dist --no-dev --no-install psr/log:1.1.0
-		args = []string{"composer", "create", "--prefer-dist", "--no-dev", "--no-install", "psr/log:1.1.0"}
+		args := []string{"composer", "create", "--prefer-dist", "--no-dev", "--no-install", "psr/log:1.1.0"}
 		out, err = exec.RunHostCommand(DdevBin, args...)
 		assert.NoError(err, "failed to run %v: err=%v, output=\n=====\n%s\n=====\n", args, err, out)
 		assert.Contains(out, "Created project in ")

docs/content/developers/project-types.md

@@ -23,6 +23,7 @@ To add a new project type:
     * `postStartAction` adds actions at the end of [`ddev start`](../users/usage/commands.md#start). You'll see several implementations of this, for things like creating needed default directories, or setting permissions on files, etc.
     * `importFilesAction` defines how [`ddev import-files`](../users/usage/commands.md#import-files) works for this project type.
     * `defaultWorkingDirMap` allows the project type to override the project’s [`working_dir`](../users/configuration/config.md#working_dir) (where [`ddev ssh`](../users/usage/commands.md#ssh) and [`ddev exec`](../users/usage/commands.md#exec) start by default). This is mostly not done anymore, as the `working_dir` is typically the project root.
+    * `composerCreateAllowedPaths` specifies the paths that can exist in a directory when `ddev composer create` is being used.
 * You’ll likely need templates for settings files, use the Drupal or TYPO3 templates as examples, for example `pkg/ddevapp/drupal` and `pkg/ddevapp/typo3`. Those templates have to be loaded at runtime as well.
 * Once your project type starts working and behaving as you’d like, you’ll need to add test artifacts for it and try testing it (locally first).
     * Add your project to `TestSites` in `ddevapp_test.go`.

docs/content/users/usage/commands.md

@@ -135,13 +135,22 @@ ddev clean my-project my-other-project
 
 Executes a [Composer command](../usage/developer-tools.md#ddev-and-composer) within the web container.
 
+`ddev composer create` is a special command that is an adaptation of `composer create-project`. See [DDEV and Composer](../usage/developer-tools.md#ddev-and-composer) for more information.
+
 Example:
 
 ```shell
 # Install Composer packages
 ddev composer install
 ```
 
+Example of `ddev composer create`:
+
+```shell
+# Create a new Drupal project in the current directory
+ddev composer create drupal/recommended-project
+````
+
 ## `config`
 
 Create or modify a DDEV project’s configuration in the current directory.

docs/content/users/usage/developer-tools.md

@@ -30,14 +30,16 @@ DDEV provides a built-in command to simplify use of PHP’s dependency manager,
 * `ddev composer help` runs Composer’s help command to learn more about what’s available.
 * `ddev composer require <package>` tells Composer to require a specific PHP package for the current project.
 
-Additionally, Composer can be used to initialize new projects with `ddev composer create`. This command supports limited argument and flag options, and will install a new project to the project root in `/var/www/html`. The package and version arguments are required:
+Additionally, Composer can be used to initialize new projects with `ddev composer create`. This command supports limited argument and flag options, and will install a new project to the composer root (normally `/var/www/html`). The package and version arguments are required:
 
 `ddev composer create [<flags>] "<package>:<version>"`
 
 For example:
 
 `ddev composer create --no-dev "typo3/cms-base-distribution:^9"`
 
+When using `ddev composer create` your project should be essentially empty or the command will refuse to run, to avoid loss of your files.
+
 To execute a fully-featured `composer create-project` command, you can execute the command from within the container after executing [`ddev ssh`](../usage/commands.md#ssh), or pass the full command to [`ddev exec`](../usage/commands.md#exec), like so:
 
 `ddev exec composer create-project ...`