Improve handling of ddev exec, ddev composer and exec hooks by adding --raw, fixes #2547

cmd/ddev/cmd/commands.go

@@ -132,6 +132,14 @@ func addCustomCommands(rootCmd *cobra.Command) error {
 					projectTypes = val
 				}
 
+				// Default is to exec with bash interpretation (not raw)
+				execRaw := false
+				if val, ok := directives["ExecRaw"]; ok {
+					if val == "true" {
+						execRaw = true
+					}
+				}
+
 				// If ProjectTypes is specified and we aren't of that type, skip
 				if projectTypes != "" && !strings.Contains(projectTypes, app.Type) {
 					continue
@@ -206,7 +214,7 @@ func addCustomCommands(rootCmd *cobra.Command) error {
 				if service == "host" {
 					commandToAdd.Run = makeHostCmd(app, onHostFullPath, commandName)
 				} else {
-					commandToAdd.Run = makeContainerCmd(app, inContainerFullPath, commandName, service)
+					commandToAdd.Run = makeContainerCmd(app, inContainerFullPath, commandName, service, execRaw)
 				}
 
 				// Add the command and mark as added
@@ -257,7 +265,7 @@ func makeHostCmd(app *ddevapp.DdevApp, fullPath, name string) func(*cobra.Comman
 }
 
 // makeContainerCmd creates the command which will app.Exec to a container command
-func makeContainerCmd(app *ddevapp.DdevApp, fullPath, name string, service string) func(*cobra.Command, []string) {
+func makeContainerCmd(app *ddevapp.DdevApp, fullPath, name, service string, execRaw bool) func(*cobra.Command, []string) {
 	s := service
 	if s[0:1] == "." {
 		s = s[1:]
@@ -275,13 +283,18 @@ func makeContainerCmd(app *ddevapp.DdevApp, fullPath, name string, service strin
 		if len(os.Args) > 2 {
 			osArgs = os.Args[2:]
 		}
-		_, _, err := app.Exec(&ddevapp.ExecOpts{
+
+		opts := &ddevapp.ExecOpts{
 			Cmd:       fullPath + " " + strings.Join(osArgs, " "),
 			Service:   s,
 			Dir:       app.GetWorkingDir(s, ""),
 			Tty:       isatty.IsTerminal(os.Stdin.Fd()),
 			NoCapture: true,
-		})
+		}
+		if execRaw {
+			opts.RawCmd = append([]string{fullPath}, osArgs...)
+		}
+		_, _, err := app.Exec(opts)
 
 		if err != nil {
 			util.Failed("Failed to run %s %v: %v", name, strings.Join(osArgs, " "), err)

cmd/ddev/cmd/composer-create.go

@@ -102,11 +102,10 @@ ddev composer create --prefer-dist --no-interaction --no-dev psr/log
 		composerCmd = append(composerCmd, osargs...)
 		composerCmd = append(composerCmd, containerInstallPath)
 
-		composerCmdString := strings.TrimSpace(strings.Join(composerCmd, " "))
-		output.UserOut.Printf("Executing composer command: %s\n", composerCmdString)
+		output.UserOut.Printf("Executing composer command: %v\n", composerCmd)
 		stdout, stderr, err := app.Exec(&ddevapp.ExecOpts{
 			Service: "web",
-			Cmd:     composerCmdString,
+			RawCmd:  composerCmd,
 			Dir:     "/var/www/html",
 			Tty:     isatty.IsTerminal(os.Stdin.Fd()),
 		})

cmd/ddev/cmd/exec.go

@@ -18,8 +18,11 @@ var DdevExecCmd = &cobra.Command{
 	Use:     "exec <command>",
 	Aliases: []string{"."},
 	Short:   "Execute a shell command in the container for a service. Uses the web service by default.",
-	Long:    `Execute a shell command in the container for a service. Uses the web service by default. To run your command in the container for another service, run "ddev exec --service <service> <cmd>"`,
-	Example: "ddev exec ls /var/www/html\nddev exec --service db\nddev exec -s db\nddev exec -s solr (assuming an add-on service named 'solr')",
+	Long:    `Execute a shell command in the container for a service. Uses the web service by default. To run your command in the container for another service, run "ddev exec --service <service> <cmd>". If you want to use raw, uninterpreted command inside container use --raw as in example.`,
+	Example: `ddev exec ls /var/www/html
+ddev exec --service db\nddev exec -s db
+ddev exec -s solr (assuming an add-on service named 'solr')
+ddev exec --raw -- ls -lR`,
 	Run: func(cmd *cobra.Command, args []string) {
 		if len(args) == 0 {
 			err := cmd.Usage()
@@ -42,12 +45,21 @@ var DdevExecCmd = &cobra.Command{
 
 		app.DockerEnv()
 
-		_, _, err = app.Exec(&ddevapp.ExecOpts{
+		opts := &ddevapp.ExecOpts{
 			Service: serviceType,
 			Dir:     execDirArg,
 			Cmd:     strings.Join(args, " "),
 			Tty:     true,
-		})
+		}
+
+		// If they've chosen raw, use the actual passed values
+		if cmd.Flag("raw").Changed {
+			if useRaw, _ := cmd.Flags().GetBool("raw"); useRaw {
+				opts.RawCmd = args
+			}
+		}
+
+		_, _, err = app.Exec(opts)
 
 		if err != nil {
 			util.Failed("Failed to execute command %s: %v", strings.Join(args, " "), err)
@@ -58,6 +70,7 @@ var DdevExecCmd = &cobra.Command{
 func init() {
 	DdevExecCmd.Flags().StringVarP(&serviceType, "service", "s", "web", "Defines the service to connect to. [e.g. web, db]")
 	DdevExecCmd.Flags().StringVarP(&execDirArg, "dir", "d", "", "Defines the execution directory within the container")
+	DdevExecCmd.Flags().Bool("raw", true, "Use raw exec (do not interpret with bash inside container)")
 	// This requires flags for exec to be specified prior to any arguments, allowing for
 	// flags to be ignored by cobra for commands that are to be executed in a container.
 	DdevExecCmd.Flags().SetInterspersed(false)

cmd/ddev/cmd/exec_test.go

@@ -61,6 +61,11 @@ func TestCmdExec(t *testing.T) {
 	assert.NoError(err)
 	assert.Contains(out, "/var")
 
+	// Test with raw cmd
+	out, err = exec.RunHostCommand(DdevBin, "exec", "--raw", "--", "ls", "/usr/local")
+	assert.NoError(err)
+	assert.Contains(out, "bin\netc\ngames\ninclude\nlib\nman\nsbin\nshare\nsrc\n")
+
 	// Test sudo
 	out, err = exec.RunHostCommand(DdevBin, "exec", "sudo", "whoami")
 	assert.NoError(err)

docs/users/cli-usage.md

@@ -686,6 +686,8 @@ To run privileged commands, sudo can be passed into `ddev exec`. For example, to
 
 Commands can also be executed using the shorter `ddev . <cmd>` alias.
 
+Normally, `ddev exec` commands are executed in the container using bash, which means that environment variables and redirection and pipes can be used. For example, a complex command like `ddev exec 'ls -l ${DDEV_FILES_DIR} | grep x >/tmp/junk.out'` will be interpreted by bash and will work. However, there are cases where bash introduces too much complexity and it's best to just run the command directly. In those cases, something like `ddev exec --raw ls -l "dir1" "dir2"` may be useful. with `--raw` the ls command is executed directly instead of the full command being interpreted by bash. But you cannot use environment variables, pipes, redirection, etc.
+
 ### SSH Into Containers
 
 The `ddev ssh` command will open an interactive bash or sh shell session to the container for a ddev service. The web service is connected to by default. The session can be ended by typing `exit`. To connect to another service, use the `--service` flag to specify the service you want to connect to. For example, to connect to the database container, you would run `ddev ssh --service db`. To specify the destination directory, use the `--dir` flag. For example, to connect to the database container and be placed into the `/home` directory, you would run `ddev ssh --service db --dir /home`.

docs/users/developer-tools.md

@@ -64,13 +64,7 @@ You generally don't have to worry about any of this, but it does keep things cle
 
 #### Limitations with `ddev composer`
 
-* Using `ddev composer --version` or `ddev composer -V` will not work, since `ddev` tries to utilize the command for itself. Use `ddev exec composer --version` instead.
-* Quotes, "@" signs and asterisks can cause troubles, since they get eaten up by the bash on the host. In such cases use double quotes, e.g.:
-    * `ddev composer require "'drupal/core:9.0.0 as 8.9.0'" --no-update`
-    * `ddev composer config repositories.local path "'packages/*'"`
-    * `ddev composer require "my-company/my-sitepackage:@dev" --no-update`
-
-If you encounter any other scenario, consider using `ddev ssh` and run composer inside the container as outlined above.
+* Using `ddev composer --version` or `ddev composer -V` will not work, since `ddev` tries to utilize the command for itself. Use `ddev composer -- --version` instead.
 
 ### Email Capture and Review
 

docs/users/extending-commands.md

@@ -74,6 +74,15 @@ hooks:
 
 (Note that this could probably be done more efficiently in a .ddev/web-build/Dockerfile as explained in [Customizing Images](extend/customizing-images.md).)
 
+Advanced usages may require running commands directly with explicit arguments. This approach is useful when bash interpretation is not required (no environment variables, no redirection, etc.)
+
+```yaml
+hooks:
+  post-start:
+  - exec: 
+    exec_raw: [ls, -lR, /var/www/html]
+```
+
 ### `exec-host`: Execute a shell command on the host system
 
 Value: string providing the command to run. Commands requiring user interaction are not supported.

pkg/ddevapp/composer.go

@@ -6,7 +6,6 @@ import (
 	"github.com/mattn/go-isatty"
 	"os"
 	"runtime"
-	"strings"
 )
 
 // Composer runs composer commands in the web container, managing pre- and post- hooks
@@ -20,7 +19,7 @@ func (app *DdevApp) Composer(args []string) (string, string, error) {
 	stdout, stderr, err := app.Exec(&ExecOpts{
 		Service: "web",
 		Dir:     "/var/www/html",
-		Cmd:     fmt.Sprintf("composer %s", strings.Join(args, " ")),
+		RawCmd:  append([]string{"composer"}, args...),
 		Tty:     isatty.IsTerminal(os.Stdin.Fd()),
 	})
 	if err != nil {

pkg/ddevapp/ddevapp_test.go

@@ -2555,6 +2555,13 @@ func TestDdevExec(t *testing.T) {
 	assert.NoError(err)
 	assert.Contains(out, "/usr/local")
 
+	// Try out a execRaw example
+	out, _, err = app.Exec(&ddevapp.ExecOpts{
+		RawCmd: []string{"ls", "/usr/local"},
+	})
+	assert.NoError(err)
+	assert.True(strings.HasPrefix(out, "bin\netc\n"))
+
 	_, _, err = app.Exec(&ddevapp.ExecOpts{
 		Service: "db",
 		Cmd:     "mysql -e 'DROP DATABASE db;'",
@@ -2682,88 +2689,6 @@ func TestDdevLogs(t *testing.T) {
 	switchDir()
 }
 
-// TestProcessHooks tests execution of commands defined in config.yaml
-func TestProcessHooks(t *testing.T) {
-	assert := asrt.New(t)
-
-	site := TestSites[0]
-	switchDir := site.Chdir()
-
-	runTime := util.TimeTrack(time.Now(), t.Name())
-
-	testcommon.ClearDockerEnv()
-	app, err := ddevapp.NewApp(site.Dir, true)
-	assert.NoError(err)
-	t.Cleanup(func() {
-		err = app.Stop(true, false)
-		assert.NoError(err)
-		app.Hooks = nil
-		err = app.WriteConfig()
-		assert.NoError(err)
-		switchDir()
-	})
-	err = app.Start()
-	assert.NoError(err)
-
-	// Note that any ExecHost commands must be able to run on Windows.
-	// echo and pwd are things that work pretty much the same in both places.
-	app.Hooks = map[string][]ddevapp.YAMLTask{
-		"hook-test": {
-			{"exec": "ls /usr/local/bin/composer"},
-			{"exec-host": "echo something"},
-			{"exec": "echo MYSQL_USER=${MYSQL_USER}", "service": "db"},
-			{"exec": "echo TestProcessHooks > /var/www/html/TestProcessHooks${DDEV_ROUTER_HTTPS_PORT}.txt"},
-			{"exec": "touch /var/tmp/TestProcessHooks && touch /var/www/html/touch_works_after_and.txt"},
-		},
-	}
-
-	captureOutputFunc, err := util.CaptureOutputToFile()
-	assert.NoError(err)
-	userOutFunc := util.CaptureUserOut()
-
-	err = app.ProcessHooks("hook-test")
-	assert.NoError(err)
-	out := captureOutputFunc()
-
-	err = app.MutagenSyncFlush()
-	assert.NoError(err)
-
-	userOut := userOutFunc()
-
-	assert.Contains(userOut, "Executing hook-test hook")
-	assert.Contains(userOut, "Exec command 'ls /usr/local/bin/composer' in container/service 'web'")
-	assert.Contains(userOut, "Exec command 'echo something' on the host")
-	assert.Contains(userOut, "Exec command 'echo MYSQL_USER=${MYSQL_USER}' in container/service 'db'")
-	assert.Contains(out, "MYSQL_USER=db")
-	assert.Contains(userOut, "Exec command 'echo TestProcessHooks > /var/www/html/TestProcessHooks${DDEV_ROUTER_HTTPS_PORT}.txt' in container/service 'web'")
-	assert.Contains(userOut, "Exec command 'touch /var/tmp/TestProcessHooks && touch /var/www/html/touch_works_after_and.txt' in container/service 'web',")
-	assert.FileExists(filepath.Join(app.AppRoot, fmt.Sprintf("TestProcessHooks%s.txt", app.RouterHTTPSPort)))
-	assert.FileExists(filepath.Join(app.AppRoot, "touch_works_after_and.txt"))
-
-	// Attempt processing hooks with a guaranteed failure
-	app.Hooks = map[string][]ddevapp.YAMLTask{
-		"hook-test": {
-			{"exec": "ls /does-not-exist"},
-		},
-	}
-	// With default setting, ProcessHooks should succeed
-	err = app.ProcessHooks("hook-test")
-	assert.NoError(err)
-	// With FailOnHookFail or FailOnHookFailGlobal or both, it should fail.
-	app.FailOnHookFail = true
-	err = app.ProcessHooks("hook-test")
-	assert.Error(err)
-	app.FailOnHookFail = false
-	app.FailOnHookFailGlobal = true
-	err = app.ProcessHooks("hook-test")
-	assert.Error(err)
-	app.FailOnHookFail = true
-	err = app.ProcessHooks("hook-test")
-	assert.Error(err)
-
-	runTime()
-}
-
 // TestDdevPause tests the functionality that is called when "ddev pause" is executed
 func TestDdevPause(t *testing.T) {
 	assert := asrt.New(t)

pkg/ddevapp/global_dotddev_assets/commands/db/mysql

@@ -6,5 +6,6 @@
 ## Example: "ddev mysql" or "ddev mysql -uroot -proot" or "echo 'SHOW TABLES;' | ddev mysql"
 ## `ddev mysql --database=mysql -uroot -proot` gets you to the 'mysql' database with root privileges
 ## DBTypes: mysql,mariadb
+## ExecRaw: true
 
-mysql -udb -pdb $@
+mysql -udb -pdb "$@"

pkg/ddevapp/global_dotddev_assets/commands/db/psql

@@ -1,9 +1,10 @@
 #!/bin/bash
 
-## #ddev-generated
+#ddev-generated
 ## Description: run pgsql client in db container
 ## Usage: psql [flags] [args]
 ## Example: "ddev psql" or "ddev psql -U db somedb" or "echo 'SELECT current_database();' | ddev psql"
 ## DBTypes: postgres
+## ExecRaw: true
 
-psql $@
+psql "$@"

pkg/ddevapp/global_dotddev_assets/commands/web/artisan

@@ -5,6 +5,7 @@
 ## Usage: artisan [flags] [args]
 ## Example: "ddev artisan list" or "ddev artisan cache:clear"
 ## ProjectTypes: laravel
+## ExecRaw: true
 
 php ./artisan "$@"
 

pkg/ddevapp/global_dotddev_assets/commands/web/blackfire

@@ -4,6 +4,7 @@
 ## Description: Enable or disable blackfire.io profiling
 ## Usage: blackfire start|stop|on|off|enable|disable|true|false|status
 ## Example: "ddev blackfire" (default is "on"), "ddev blackfire off", "ddev blackfire on", "ddev blackfire status"
+## ExecRaw: false
 
 function enable {
   if [ -z ${BLACKFIRE_SERVER_ID} ] || [ -z ${BLACKFIRE_SERVER_TOKEN} ]; then

pkg/ddevapp/global_dotddev_assets/commands/web/drush

@@ -5,6 +5,7 @@
 ## Usage: drush [flags] [args]
 ## Example: "ddev drush uli" or "ddev drush sql-cli" or "ddev drush --version"
 ## ProjectTypes: drupal7,drupal8,drupal9,backdrop
+## ExecRaw: true
 
 if ! command -v drush >/dev/null; then
   echo "drush is not available. You may need to 'ddev composer require drush/drush'"

pkg/ddevapp/global_dotddev_assets/commands/web/magento

@@ -5,6 +5,7 @@
 ## Usage: magento [flags] [args]
 ## Example: "ddev magento list" or "ddev magento maintenance:enable" or "ddev magento sampledata:reset"
 ## ProjectTypes: magento2
+## ExecRaw: true
 
 if ! command -v magento >/dev/null; then
   echo 'magento is not available in your in-container $PATH'

pkg/ddevapp/global_dotddev_assets/commands/web/php

@@ -1,7 +1,9 @@
 #!/bin/bash
 
+#ddev-generated
 ## Description: Run php inside the web container
 ## Usage: php [flags] [args]
 ## Example: "ddev php --version"
+## ExecRaw: true
 
 php "$@"

pkg/ddevapp/global_dotddev_assets/commands/web/typo3

@@ -7,5 +7,6 @@
 ## Usage: typo3 [args]
 ## Example: "ddev typo3 site:list" or "ddev typo3 list" or "ddev typo3 extension:list"
 ## ProjectTypes: typo3
+## ExecRaw: true
 
 typo3 "$@"

pkg/ddevapp/global_dotddev_assets/commands/web/typo3cms

@@ -7,5 +7,6 @@
 ## Usage: typo3cms [args]
 ## Example: "ddev typo3cms cache:flush" or "ddev typo3cms database:export"
 ## ProjectTypes: typo3
+## ExecRaw: true
 
 typo3cms "$@"

pkg/ddevapp/global_dotddev_assets/commands/web/wp

@@ -4,5 +4,6 @@
 ## Usage: wp [flags] [args]
 ## Example: "ddev wp core version" or "ddev wp plugin install user-switching --activate"
 ## ProjectTypes: wordpress
+## ExecRaw: true
 
 wp "$@"

pkg/ddevapp/global_dotddev_assets/commands/web/xdebug

@@ -4,6 +4,7 @@
 ## Description: Enable or disable xdebug
 ## Usage: xdebug on|off|enable|disable|true|false|status
 ## Example: "ddev xdebug" (default is "on"), "ddev xdebug off", "ddev xdebug on", "ddev xdebug status"
+## Execraw: false
 
 if [ $# -eq 0 ] ; then
   enable_xdebug

pkg/ddevapp/global_dotddev_assets/commands/web/xhprof

@@ -4,6 +4,7 @@
 ## Description: Enable or disable xhprof
 ## Usage: xhprof on|off|enable|disable|true|false|status
 ## Example: "ddev xhprof" (default is "on"), "ddev xhprof off", "ddev xhprof on", "ddev xhprof status"
+## ExecRaw: false
 
 if [ $# -eq 0 ]; then
   enable_xhprof