Revamp docker-compose file ordering and choosing, fixes #815 (#817)

pkg/ddevapp/ddevapp.go

@@ -513,28 +513,48 @@ func (app *DdevApp) ImportFiles(imPath string, extPath string) error {
 }
 
 // ComposeFiles returns a list of compose files for a project.
-func (app *DdevApp) ComposeFiles() []string {
-	files, err := filepath.Glob(filepath.Join(app.AppConfDir(), "docker-compose*"))
-	if err != nil {
-		util.Failed("Failed to load compose files: %v", err)
+// It has to put the docker-compose.y*l first
+// It has to put the docker-compose.override.y*l last
+func (app *DdevApp) ComposeFiles() ([]string, error) {
+	files, err := filepath.Glob(filepath.Join(app.AppConfDir(), "docker-compose*.y*l"))
+	if err != nil || len(files) == 0 {
+		return []string{}, fmt.Errorf("failed to load any docker-compose.*y*l files: %v", err)
 	}
 
-	for i, file := range files {
-		// ensure main docker-compose is first
-		match, err := filepath.Match(filepath.Join(app.AppConfDir(), "docker-compose.y*l"), file)
-		if err == nil && match {
-			files = append(files[:i], files[i+1:]...)
-			files = append([]string{file}, files...)
-		}
-		// ensure override is last
-		match, err = filepath.Match(filepath.Join(app.AppConfDir(), "docker-compose.override.y*l"), file)
-		if err == nil && match {
-			files = append(files, file)
-			files = append(files[:i], files[i+1:]...)
-		}
+	mainfiles, err := filepath.Glob(filepath.Join(app.AppConfDir(), "docker-compose.y*l"))
+	// Glob doesn't return many errors, so just CheckErr()
+	util.CheckErr(err)
+	if len(mainfiles) == 0 {
+		return []string{}, fmt.Errorf("failed to find a docker-compose.yml or docker-compose.yaml")
+
+	}
+	if len(mainfiles) > 1 {
+		return []string{}, fmt.Errorf("there are more than one docker-compose.y*l, unable to continue")
 	}
 
-	return files
+	overrides, err := filepath.Glob(filepath.Join(app.AppConfDir(), "docker-compose.override.y*l"))
+	util.CheckErr(err)
+	if len(overrides) > 1 {
+		return []string{}, fmt.Errorf("there are more than one docker-compose.override.y*l, unable to continue")
+	}
+
+	orderedFiles := make([]string, 1)
+
+	// Make sure the docker-compose.yaml goes first
+	orderedFiles[0] = mainfiles[0]
+
+	for _, file := range files {
+		// We already have the main docker-compose.yaml, so skip when we hit it.
+		// We'll add the override later, so skip it.
+		if file == mainfiles[0] || (len(overrides) == 1 && file == overrides[0]) {
+			continue
+		}
+		orderedFiles = append(orderedFiles, file)
+	}
+	if len(overrides) == 1 {
+		orderedFiles = append(orderedFiles, overrides[0])
+	}
+	return orderedFiles, nil
 }
 
 // ProcessHooks executes commands defined in a Command
@@ -615,7 +635,11 @@ func (app *DdevApp) Start() error {
 		return err
 	}
 
-	_, _, err = dockerutil.ComposeCmd(app.ComposeFiles(), "up", "-d")
+	files, err := app.ComposeFiles()
+	if err != nil {
+		return err
+	}
+	_, _, err = dockerutil.ComposeCmd(files, "up", "-d")
 	if err != nil {
 		return err
 	}
@@ -646,7 +670,11 @@ func (app *DdevApp) Exec(service string, cmd ...string) (string, string, error)
 	exec := []string{"exec", "-T", service}
 	exec = append(exec, cmd...)
 
-	return dockerutil.ComposeCmd(app.ComposeFiles(), exec...)
+	files, err := app.ComposeFiles()
+	if err != nil {
+		return "", "", err
+	}
+	return dockerutil.ComposeCmd(files, exec...)
 }
 
 // ExecWithTty executes a given command in the container of given type.
@@ -657,7 +685,11 @@ func (app *DdevApp) ExecWithTty(service string, cmd ...string) error {
 	exec := []string{"exec", service}
 	exec = append(exec, cmd...)
 
-	return dockerutil.ComposeNoCapture(app.ComposeFiles(), exec...)
+	files, err := app.ComposeFiles()
+	if err != nil {
+		return err
+	}
+	return dockerutil.ComposeNoCapture(files, exec...)
 }
 
 // Logs returns logs for a site's given container.
@@ -778,7 +810,11 @@ func (app *DdevApp) Stop() error {
 		return fmt.Errorf("ddev can no longer find your project files at %s. If you would like to continue using ddev to manage this project please restore your files to that directory. If you would like to remove this site from ddev, you may run 'ddev remove %s'", app.GetAppRoot(), app.GetName())
 	}
 
-	_, _, err := dockerutil.ComposeCmd(app.ComposeFiles(), "stop")
+	files, err := app.ComposeFiles()
+	if err != nil {
+		return err
+	}
+	_, _, err = dockerutil.ComposeCmd(files, "stop")
 
 	if err != nil {
 		return err
@@ -917,7 +953,7 @@ func (app *DdevApp) AddHostsEntries() error {
 	if dockerHostRawURL != "" {
 		dockerHostURL, err := url.Parse(dockerHostRawURL)
 		if err != nil {
-			return fmt.Errorf("Failed to parse $DOCKER_HOST: %v, err: %v", dockerHostRawURL, err)
+			return fmt.Errorf("failed to parse $DOCKER_HOST: %v, err: %v", dockerHostRawURL, err)
 		}
 		dockerIP = dockerHostURL.Hostname()
 	}
@@ -971,7 +1007,7 @@ func (app *DdevApp) prepSiteDirs() error {
 		} else if err == nil && fileInfo.IsDir() { // If the directory exists, we're fine and don't have to create it.
 			continue
 		} else { // But otherwise it must have existed as a file, so bail
-			return fmt.Errorf("Error where trying to create directory %s, err: %v", dir, err)
+			return fmt.Errorf("error trying to create directory %s, err: %v", dir, err)
 		}
 	}
 

pkg/ddevapp/ddevapp_test.go

@@ -978,6 +978,56 @@ func TestListWithoutDir(t *testing.T) {
 	assert.NoError(err)
 }
 
+// TestMultipleComposeFiles checks to see if a set of docker-compose files gets
+// properly loaded in the right order, with docker-compose.yaml first and
+// with docker-compose.override.yaml last.
+func TestMultipleComposeFiles(t *testing.T) {
+	// Set up tests and give ourselves a working directory.
+	assert := asrt.New(t)
+
+	// Make sure that valid yaml files get properly loaded in the proper order
+	app, err := ddevapp.NewApp("./testdata/testMultipleComposeFiles", "")
+	assert.NoError(err)
+
+	files, err := app.ComposeFiles()
+	assert.NoError(err)
+	assert.True(files[0] == filepath.Join(app.AppConfDir(), "docker-compose.yaml"))
+	assert.True(files[len(files)-1] == filepath.Join(app.AppConfDir(), "docker-compose.override.yaml"))
+
+	// Make sure that some docker-compose.yml and docker-compose.yaml conflict gets noted properly
+	app, err = ddevapp.NewApp("./testdata/testConflictingYamlYml", "")
+	assert.NoError(err)
+
+	_, err = app.ComposeFiles()
+	assert.Error(err)
+	assert.Contains(err.Error(), "there are more than one docker-compose.y*l")
+
+	// Make sure that some docker-compose.override.yml and docker-compose.override.yaml conflict gets noted properly
+	app, err = ddevapp.NewApp("./testdata/testConflictingOverrideYaml", "")
+	assert.NoError(err)
+
+	_, err = app.ComposeFiles()
+	assert.Error(err)
+	assert.Contains(err.Error(), "there are more than one docker-compose.override.y*l")
+
+	// Make sure the error gets pointed out of there's no main docker-compose.yaml
+	app, err = ddevapp.NewApp("./testdata/testNoDockerCompose", "")
+	assert.NoError(err)
+
+	_, err = app.ComposeFiles()
+	assert.Error(err)
+	assert.Contains(err.Error(), "failed to find a docker-compose.yml or docker-compose.yaml")
+
+	// Catch if we have no docker files at all.
+	// This should also fail if the docker-compose.yaml.bak gets loaded.
+	app, err = ddevapp.NewApp("./testdata/testNoDockerFilesAtAll", "")
+	assert.NoError(err)
+
+	_, err = app.ComposeFiles()
+	assert.Error(err)
+	assert.Contains(err.Error(), "failed to load any docker-compose.*y*l files")
+}
+
 // constructContainerName builds a container name given the type (web/db/dba) and the app
 func constructContainerName(containerType string, app *ddevapp.DdevApp) (string, error) {
 	container, err := app.FindContainerByType(containerType)