Rethink hooks and tasks, for #1372, fixes #1321, fixes #1071, fixes #…

…1038 (#1634)

* Switch to yaml v3
* Add composer task
* Add multiple pre- and post- hooks
* Allow exec hook to specify container to act on

cmd/ddev/cmd/composer-create.go

@@ -190,7 +190,7 @@ project root will be deleted when creating a project.`,
 			util.Failed("Failed to create project: %v", err)
 		}
 		if runtime.GOOS == "windows" && !nodeps.IsDockerToolbox() {
-			replaceSimulatedLinks(app.AppRoot)
+			fileutil.ReplaceSimulatedLinks(app.AppRoot)
 		}
 
 	},

cmd/ddev/cmd/composer.go

@@ -2,14 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/drud/ddev/pkg/fileutil"
-	"github.com/drud/ddev/pkg/nodeps"
-	"runtime"
-	"strings"
-
 	"github.com/drud/ddev/pkg/ddevapp"
 	"github.com/drud/ddev/pkg/util"
 	"github.com/spf13/cobra"
+	"os"
 )
 
 var ComposerCmd = &cobra.Command{
@@ -33,52 +29,16 @@ ddev composer outdated --minor-only`,
 			}
 		}
 
-		stdout, _, err := app.Exec(&ddevapp.ExecOpts{
-			Service: "web",
-			Dir:     "/var/www/html",
-			Cmd:     fmt.Sprintf("composer %s", strings.Join(args, " ")),
-		})
+		stdout, stderr, err := app.Composer(args)
 		if err != nil {
-			util.Failed("composer command failed: %v", err)
-		}
-		if runtime.GOOS == "windows" && !nodeps.IsDockerToolbox() {
-			replaceSimulatedLinks(app.AppRoot)
-		}
-
-		if len(stdout) > 0 {
-			fmt.Println(stdout)
+			util.Failed("composer %v failed, %v. stderr=%v", args, err, stderr)
 		}
+		_, _ = fmt.Fprint(os.Stderr, stderr)
+		_, _ = fmt.Fprint(os.Stdout, stdout)
 	},
 }
 
 func init() {
 	RootCmd.AddCommand(ComposerCmd)
 	ComposerCmd.Flags().SetInterspersed(false)
 }
-
-// replaceSimulatedLinks() walks the path provided and tries to replace XSym links with real ones.
-func replaceSimulatedLinks(path string) {
-	links, err := fileutil.FindSimulatedXsymSymlinks(path)
-	if err != nil {
-		util.Warning("Error finding XSym Symlinks: %v", err)
-	}
-	if len(links) == 0 {
-		return
-	}
-
-	if !fileutil.CanCreateSymlinks() {
-		util.Warning("This host computer is unable to create real symlinks, please see the docs to enable developer mode:\n%s\nNote that the simulated symlinks created inside the container will work fine for most projects.", "https://ddev.readthedocs.io/en/latest/users/developer-tools/#windows-os-and-ddev-composer")
-		return
-	}
-
-	err = fileutil.ReplaceSimulatedXsymSymlinks(links)
-	if err != nil {
-		util.Warning("Failed replacing simulated symlinks: %v", err)
-	}
-	replacedLinks := make([]string, 0)
-	for _, l := range links {
-		replacedLinks = append(replacedLinks, l.LinkLocation)
-	}
-	util.Success("Replaced these simulated symlinks with real symlinks: %v", replacedLinks)
-	return
-}

cmd/ddev/cmd/config.go

@@ -164,6 +164,11 @@ func handleConfigRun(cmd *cobra.Command, args []string) {
 		util.Failed("Please do not use `ddev config` in your home directory")
 	}
 
+	err = app.ProcessHooks("pre-config")
+	if err != nil {
+		util.Failed("Failed to process hook 'pre-config'")
+	}
+
 	if cmd.Flags().NFlag() == 0 {
 		err = app.PromptForConfig()
 		if err != nil {
@@ -205,6 +210,10 @@ func handleConfigRun(cmd *cobra.Command, args []string) {
 	if err != nil {
 		util.Failed("Failed to write provider config: %v", err)
 	}
+	err = app.ProcessHooks("post-config")
+	if err != nil {
+		util.Failed("Failed to process hook 'post-config'")
+	}
 
 	util.Success("Configuration complete. You may now run 'ddev start'.")
 }

cmd/ddev/cmd/config_test.go

@@ -19,9 +19,33 @@ import (
 	"github.com/drud/ddev/pkg/exec"
 	"github.com/drud/ddev/pkg/testcommon"
 	asrt "github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
+// TestCmdConfigHooks tests that pre-config and post-config hooks run
+func TestCmdConfigHooks(t *testing.T) {
+	// Change to the first DevTestSite for the duration of this test.
+	site := TestSites[0]
+	defer site.Chdir()()
+	assert := asrt.New(t)
+
+	app, err := ddevapp.NewApp(site.Dir, true, "")
+	assert.NoError(err)
+	app.Hooks = map[string][]ddevapp.YAMLTask{"post-config": {{"exec-host": "touch hello-post-config-" + app.Name}}, "pre-config": {{"exec-host": "touch hello-pre-config-" + app.Name}}}
+	err = app.WriteConfig()
+	assert.NoError(err)
+
+	_, err = exec.RunCommand(DdevBin, []string{"config", "--project-type=" + app.Type})
+	assert.NoError(err)
+
+	assert.FileExists("hello-pre-config-" + app.Name)
+	assert.FileExists("hello-post-config-" + app.Name)
+	err = os.Remove("hello-pre-config-" + app.Name)
+	assert.NoError(err)
+	err = os.Remove("hello-post-config-" + app.Name)
+	assert.NoError(err)
+}
+
 // TestConfigDescribeLocation tries out the --show-config-location flag.
 func TestConfigDescribeLocation(t *testing.T) {
 	assert := asrt.New(t)
@@ -52,7 +76,6 @@ func TestConfigDescribeLocation(t *testing.T) {
 	out, err = exec.RunCommand(DdevBin, args)
 	assert.Error(err)
 	assert.Contains(string(out), "No project configuration currently exists")
-
 }
 
 // TestConfigWithSitenameFlagDetectsDocroot tests docroot detected when flags passed.

cmd/ddev/cmd/list.go

@@ -18,8 +18,8 @@ var activeOnly bool
 // continuousSleepTime is time to sleep between reads with --continuous
 var continuousSleepTime = 1
 
-// DdevListCmd represents the list command
-var DdevListCmd = &cobra.Command{
+// ListCmd represents the list command
+var ListCmd = &cobra.Command{
 	Use:   "list",
 	Short: "List projects",
 	Long:  `List projects. Shows active projects by default, includes stopped projects with --all`,
@@ -56,9 +56,9 @@ var DdevListCmd = &cobra.Command{
 }
 
 func init() {
-	DdevListCmd.Flags().BoolVarP(&activeOnly, "active-only", "A", false, "If set, only currently active projects will be displayed.")
-	DdevListCmd.Flags().BoolVarP(&continuous, "continuous", "", false, "If set, project information will be emitted until the command is stopped.")
-	DdevListCmd.Flags().IntVarP(&continuousSleepTime, "continuous-sleep-interval", "I", 1, "Time in seconds between ddev list --continous output lists.")
+	ListCmd.Flags().BoolVarP(&activeOnly, "active-only", "A", false, "If set, only currently active projects will be displayed.")
+	ListCmd.Flags().BoolVarP(&continuous, "continuous", "", false, "If set, project information will be emitted until the command is stopped.")
+	ListCmd.Flags().IntVarP(&continuousSleepTime, "continuous-sleep-interval", "I", 1, "Time in seconds between ddev list --continous output lists.")
 
-	RootCmd.AddCommand(DdevListCmd)
+	RootCmd.AddCommand(ListCmd)
 }

cmd/ddev/cmd/snapshot.go

@@ -20,7 +20,7 @@ var DdevSnapshotCommand = &cobra.Command{
 		}
 
 		for _, app := range apps {
-			if snapshotNameOutput, err := app.SnapshotDatabase(snapshotName); err != nil {
+			if snapshotNameOutput, err := app.Snapshot(snapshotName); err != nil {
 				util.Failed("Failed to snapshot %s: %v", app.GetName(), err)
 			} else {
 				util.Success("Created snapshot %s", snapshotNameOutput)

docs/users/extend/custom-compose-files.md

@@ -12,7 +12,20 @@ To add custom configuration or additional services to your project, create docke
 
 ## Restrictions on the docker-compose.yaml file
 
-The main docker-compose.yaml file is exclusively reserved for ddev's use, and will be overwritten on ddev upgrades and when a project is started, so it should not be edited, or edits will be lost. If you need to override configuration provided by docker-compose.yaml, use an additional file to do so.
+The main docker-compose.yaml file is exclusively reserved for ddev's use, and will be overwritten every time a project is started, so it should not be edited because edits will be lost. If you need to override configuration provided by docker-compose.yaml, use an additional file "docker-compose.<whatever>.yaml" to do so.
+
+## docker-compose.*.yaml examples
+
+* Set an environment variable in the web container, in a file perhaps called `docker-compose.env.yaml`:
+
+```
+version: '3.6'
+
+services:
+  web:
+    environment:
+      - TYPO3_CONTEXT=Development
+```
 
 ## Confirming docker-compose configurations
 

docs/users/extending-commands.md

@@ -1,6 +1,6 @@
 <h1>Extending ddev Commands</h1>
 
-Certain ddev commands provide hooks to run tasks before or after the main command executes. To automate setup tasks specific to your project, define them in the project's config.yaml file.
+Most ddev commands provide hooks to run tasks before or after the main command executes. To automate setup tasks specific to your project, define them in the project's config.yaml file.
 
 To define command tasks in your configuration, specify the desired command hook as a subfield to `hooks`, then provide a list of tasks to run.
 
@@ -18,22 +18,27 @@ hooks:
 
 ## Supported Command Hooks
 
-- `pre-start`: Hooks into "ddev start". Execute tasks before the project environment starts. **Note:** Only `exec-host` tasks can be run successfully for pre-start. See Supported Tasks below for more info.
-- `post-start`: Hooks into "ddev start". Execute tasks after the project environment has started. This only runs when the web container is created or recreated.
-- `pre-import-db`: Hooks into "ddev import-db". Execute tasks before database import
-- `post-import-db`: Hooks into "ddev import-db". Execute tasks after database import
-- `pre-import-files`: Hooks into "ddev import-files". Execute tasks before files are imported
-- `post-import-files`: Hooks into "ddev import-files". Execute tasks after files are imported.
+- `pre-start`: Hooks into "ddev start". Execute tasks before the project environment starts. **Note:** Only `exec-host` tasks can be generally run successfully during pre-start. See Supported Tasks below for more info.
+- `post-start`: Execute tasks after the project environment has started. 
+- `pre-import-db` and `post-import-db`: Execute tasks before or after database import.
+- `pre-import-files` and `post-import-files`: Execute tasks before or after files are imported
+- `pre-composer` and `post-composer`: Execute tasks before or after the `composer` command.
+- `pre-stop`, `post-stop`, `pre-config`, `post-config`, `pre-exec`, `post-exec`, `pre-pause`, `post-pause`, `pre-pull`, `post-pull`, `pre-snapshot`, `post-snapshot`, `pre-restore-snapshot`, `post-restore-snapshot`: Execute as the name suggests.
+
 
 ## Supported Tasks
 
-### `exec`: Execute a shell command in the web service container.
+ddev currently supports these tasks:
 
-Value: string providing the command to run. Commands requiring user interaction are not supported.
+* `exec` to execute a command in any service/container
+* `exec-host` to execute a command on the host
+* `composer` to execute a composer command in the web container
 
-Example:
+### `exec`: Execute a shell command in a container (defaults to web container).
+
+Value: string providing the command to run. Commands requiring user interaction are not supported. You can also add a "service" key to the command, specifying to run it on the db container or any other container you use.
 
-_Use drush to clear the Drupal cache and get a user login link after database import_
+Example: _Use drush to clear the Drupal cache and get a user login link after database import_
 
 ```
 hooks:
@@ -44,16 +49,33 @@ hooks:
     - exec: sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -y ghostscript sqlite3 php7.2-sqlite3 && sudo killall -HUP php-fpm
 ```
 
-Example:
-
-_Use wp-cli to replace the production URL with development URL in the database of a WordPress project_
+Example: _Use wp-cli to replace the production URL with development URL in the database of a WordPress project_
 
 ```
 hooks:
   post-import-db:
     - exec: wp search-replace https://www.myproductionsite.com http://mydevsite.ddev.site
 ```
 
+Example: _Add an extra database before import-db, executing in db container_
+```
+hooks:
+  pre-import-db:
+    - exec: mysql -uroot -proot -e "CREATE DATABASE IF NOT EXISTS some_new_database;"
+      service: db
+
+```
+
+Example: _Add the common "ll" alias into the web container .bashrc file_
+
+```
+hooks:
+  post-start:
+  - exec: sudo echo alias ll=\"ls -lhA\" >> ~/.bashrc
+```
+
+(Note that this could probably be done more efficiently in a .ddev/web-build/Dockerfile as explained in [Customizing Images](extend/customizing-images.md).)
+
 ### `exec-host`: Execute a shell command on the host system.
 
 Value: string providing the command to run. Commands requiring user interaction are not supported.
@@ -68,6 +90,20 @@ hooks:
     - exec-host: "composer install"
 ```
 
+### `composer`: Execute a composer command in the web container.
+
+Value: string providing the composer command to run. 
+
+
+Example:
+
+```
+hooks:
+  post-start:
+    - composer: config discard-changes true
+```
+
+
 ## WordPress Example
 
 ```

go.mod

@@ -50,6 +50,6 @@ require (
 	golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac
 	gopkg.in/cheggaaa/pb.v1 v1.0.26 // indirect
 	gopkg.in/ini.v1 v1.39.0 // indirect
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae
 	gotest.tools v2.2.0+incompatible // indirect
 )

go.sum

@@ -165,6 +165,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae h1:ehhBuCxzgQEGk38YjhFv/97fMIc2JGHZAhAWMmEjmu0=
+gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=

pkg/ddevapp/composer.go

@@ -0,0 +1,36 @@
+package ddevapp
+
+import (
+	"fmt"
+	"github.com/drud/ddev/pkg/fileutil"
+	"github.com/drud/ddev/pkg/nodeps"
+	"runtime"
+	"strings"
+)
+
+// Composer runs composer commands in the web container, managing pre- and post- hooks
+func (app *DdevApp) Composer(args []string) (string, string, error) {
+	err := app.ProcessHooks("pre-composer")
+	if err != nil {
+		return "", "", fmt.Errorf("Failed to process pre-composer hooks: %v", err)
+	}
+
+	stdout, stderr, err := app.Exec(&ExecOpts{
+		Service: "web",
+		Dir:     "/var/www/html",
+		Cmd:     fmt.Sprintf("composer %s", strings.Join(args, " ")),
+	})
+	if err != nil {
+		return stdout, stderr, fmt.Errorf("composer command failed: %v", err)
+	}
+
+	if runtime.GOOS == "windows" && !nodeps.IsDockerToolbox() {
+		fileutil.ReplaceSimulatedLinks(app.AppRoot)
+	}
+	err = app.ProcessHooks("post-composer")
+	if err != nil {
+		return stdout, stderr, fmt.Errorf("Failed to process post-composer hooks: %v", err)
+	}
+	return stdout, stderr, nil
+
+}