introduce dependencies between projects and sort commands based on that

cmd/digger/main.go

@@ -61,7 +61,7 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 
 	walker := configuration.FileSystemDirWalker{}
 
-	diggerConfig, err := configuration.LoadDiggerConfig("./", &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig("./", &walker)
 	if err != nil {
 		reportErrorAndExit(githubActor, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
@@ -115,7 +115,7 @@ func gitHubCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 	if err != nil {
 		reportErrorAndExit(githubActor, fmt.Sprintf("Failed to get current dir. %s", err), 4)
 	}
-	allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, parsedGhContext.Repository, githubActor, eventName, prNumber, githubPrService, lock, reporter, planStorage, policyChecker, currentDir)
+	allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, &dependencyGraph, parsedGhContext.Repository, githubActor, eventName, prNumber, githubPrService, lock, reporter, planStorage, policyChecker, currentDir)
 	if err != nil {
 		reportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8)
 	}
@@ -153,7 +153,7 @@ func gitLabCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 	}
 	fmt.Printf("main: working dir: %s \n", currentDir)
 
-	diggerConfig, err := configuration.LoadDiggerConfig(currentDir, &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir, &walker)
 	if err != nil {
 		reportErrorAndExit(projectNamespace, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
@@ -204,7 +204,7 @@ func gitLabCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 		CiService: gitlabService,
 		PrNumber:  *gitLabContext.MergeRequestIId,
 	}
-	allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, gitLabContext.GitlabUserName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, reporter, planStorage, policyChecker, currentDir)
+	allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, &dependencyGraph, diggerProjectNamespace, gitLabContext.GitlabUserName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, reporter, planStorage, policyChecker, currentDir)
 
 	if err != nil {
 		fmt.Printf("failed to execute command, %v", err)
@@ -247,7 +247,7 @@ func azureCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 	}
 	fmt.Printf("main: working dir: %s \n", currentDir)
 
-	diggerConfig, err := configuration.LoadDiggerConfig(currentDir, &walker)
+	diggerConfig, dependencyGraph, err := configuration.LoadDiggerConfig(currentDir, &walker)
 	if err != nil {
 		reportErrorAndExit(parsedAzureContext.BaseUrl, fmt.Sprintf("Failed to read Digger config. %s", err), 4)
 	}
@@ -282,7 +282,7 @@ func azureCI(lock core_locking.Lock, policyChecker core_policy.Checker) {
 		CiService: azureService,
 		PrNumber:  prNumber,
 	}
-	allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, parsedAzureContext.BaseUrl, parsedAzureContext.EventType, prNumber, azureService, lock, reporter, planStorage, policyChecker, currentDir)
+	allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, &dependencyGraph, diggerProjectNamespace, parsedAzureContext.BaseUrl, parsedAzureContext.EventType, prNumber, azureService, lock, reporter, planStorage, policyChecker, currentDir)
 	if err != nil {
 		reportErrorAndExit(parsedAzureContext.BaseUrl, fmt.Sprintf("Failed to run commands. %s", err), 8)
 	}

cmd/digger/main_test.go

@@ -9,6 +9,7 @@ import (
 	"digger/pkg/reporting"
 	"digger/pkg/utils"
 	"fmt"
+	"github.com/dominikbraun/graph"
 	"testing"
 
 	"github.com/davecgh/go-spew/spew"
@@ -875,6 +876,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
 	eventName := context.EventName
 
 	diggerConfig := configuration.DiggerConfig{}
+	depGraph := graph.New(graph.StringHash, graph.Directed())
 	lock := &utils.MockLock{}
 	prManager := &utils.MockPullRequestManager{ChangedFiles: []string{"dev/test.tf"}}
 	planStorage := &utils.MockPlanStorage{}
@@ -887,7 +889,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) {
 		PrNumber:  prNumber,
 	}
 	commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.Workflow{})
-	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
+	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, &depGraph, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
 
 	assert.NoError(t, err)
 	if err != nil {
@@ -904,6 +906,7 @@ func TestGitHubNewCommentContext(t *testing.T) {
 	ghEvent := context.Event
 	eventName := context.EventName
 	diggerConfig := configuration.DiggerConfig{}
+	depGraph := graph.New(graph.StringHash, graph.Directed())
 	lock := &utils.MockLock{}
 	prManager := &utils.MockPullRequestManager{ChangedFiles: []string{"dev/test.tf"}}
 	planStorage := &utils.MockPlanStorage{}
@@ -916,9 +919,9 @@ func TestGitHubNewCommentContext(t *testing.T) {
 	policyChecker := &utils.MockPolicyChecker{}
 
 	commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.Workflow{})
-	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
+	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, &depGraph, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
 
-	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
+	_, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, &depGraph, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, policyChecker, "")
 
 	assert.NoError(t, err)
 	if err != nil {

go.mod

@@ -18,6 +18,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
+require github.com/dominikbraun/graph v0.22.3 // indirect
+
 require (
 	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/compute v1.18.0 // indirect

go.sum

@@ -57,6 +57,8 @@ github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWa
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g=
 github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA=
 github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dominikbraun/graph v0.22.3 h1:2bbNzuMX+iKpM+XW1YPu7ch8IyXi/VbG7UYrRj2FHuI=
+github.com/dominikbraun/graph v0.22.3/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=

pkg/configuration/config.go

@@ -10,13 +10,14 @@ type DiggerConfig struct {
 }
 
 type Project struct {
-	Name            string
-	Dir             string
-	Workspace       string
-	Terragrunt      bool
-	Workflow        string
-	IncludePatterns []string
-	ExcludePatterns []string
+	Name               string
+	Dir                string
+	Workspace          string
+	Terragrunt         bool
+	Workflow           string
+	IncludePatterns    []string
+	ExcludePatterns    []string
+	DependencyProjects []string
 }
 
 type Workflow struct {

pkg/configuration/converters.go

@@ -4,6 +4,7 @@ import (
 	"digger/pkg/core/models"
 	"digger/pkg/utils"
 	"fmt"
+	"github.com/dominikbraun/graph"
 	"path/filepath"
 )
 
@@ -17,6 +18,7 @@ func copyProjects(projects []*ProjectYaml) []Project {
 			p.Workflow,
 			p.IncludePatterns,
 			p.ExcludePatterns,
+			p.DependencyProjects,
 		}
 		result[i] = item
 	}
@@ -99,7 +101,7 @@ func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]Workflow {
 	return result
 }
 
-func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) {
+func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, graph.Graph[string, string], error) {
 	var diggerConfig DiggerConfig
 	const defaultWorkflowName = "default"
 
@@ -145,15 +147,30 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string,
 	projectNames := make(map[string]bool)
 	for _, project := range diggerConfig.Projects {
 		if projectNames[project.Name] {
-			return nil, fmt.Errorf("project name '%s' is duplicated", project.Name)
+			return nil, nil, fmt.Errorf("project name '%s' is duplicated", project.Name)
 		}
 		projectNames[project.Name] = true
 	}
 
+	// check project dependencies exist
+	for _, project := range diggerConfig.Projects {
+		for _, dependency := range project.DependencyProjects {
+			if !projectNames[dependency] {
+				return nil, nil, fmt.Errorf("project '%s' depends on '%s' which does not exist", project.Name, dependency)
+			}
+		}
+	}
+
+	dependencyGraph, err := CreateProjectDependencyGraph(diggerConfig.Projects)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create project dependency graph: %s", err.Error())
+	}
+
 	if diggerYaml.GenerateProjectsConfig != nil {
 		dirs, err := walker.GetDirs(workingDir)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		for _, dir := range dirs {
@@ -177,5 +194,36 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string,
 		}
 	}
 
-	return &diggerConfig, nil
+	return &diggerConfig, dependencyGraph, nil
+}
+
+func CreateProjectDependencyGraph(projects []Project) (graph.Graph[string, string], error) {
+	g := graph.New(graph.StringHash, graph.Directed(), graph.PreventCycles())
+	for _, project := range projects {
+		v, _ := g.Vertex(project.Name)
+
+		if v == "" {
+			err := g.AddVertex(project.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+		for _, dependency := range project.DependencyProjects {
+			v, _ := g.Vertex(dependency)
+
+			if v == "" {
+				err := g.AddVertex(dependency)
+
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			err := g.AddEdge(dependency, project.Name)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	return g, nil
 }

pkg/configuration/digger_config.go

@@ -4,6 +4,7 @@ import (
 	"digger/pkg/utils"
 	"errors"
 	"fmt"
+	"github.com/dominikbraun/graph"
 	"gopkg.in/yaml.v3"
 	"os"
 	"path"
@@ -61,50 +62,53 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error)
 
 var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'")
 
-func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error) {
+func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, graph.Graph[string, string], error) {
 	configYaml := &DiggerConfigYaml{}
 	config := &DiggerConfig{}
 	fileName, err := retrieveConfigFile(workingDir)
 	if err != nil {
 		if errors.Is(err, ErrDiggerConfigConflict) {
-			return nil, fmt.Errorf("error while retrieving config file: %v", err)
+			return nil, nil, fmt.Errorf("error while retrieving config file: %v", err)
 		}
 	}
 
 	if fileName == "" {
 		fmt.Println("No digger config found, using default one")
 		config.Projects = make([]Project, 1)
-		config.Projects[0] = defaultProject()
+		project := defaultProject()
+		config.Projects[0] = project
 		config.Workflows = make(map[string]Workflow)
 		config.Workflows["default"] = *defaultWorkflow()
-		return config, nil
+		g := graph.New(graph.StringHash)
+		g.AddVertex(project.Name)
+		return config, g, nil
 	}
 
 	data, err := os.ReadFile(fileName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read config file %s: %v", fileName, err)
+		return nil, nil, fmt.Errorf("failed to read config file %s: %v", fileName, err)
 	}
 
 	if err := yaml.Unmarshal(data, configYaml); err != nil {
-		return nil, fmt.Errorf("error parsing '%s': %v", fileName, err)
+		return nil, nil, fmt.Errorf("error parsing '%s': %v", fileName, err)
 	}
 
 	if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil {
-		return nil, fmt.Errorf("no projects configuration found in '%s'", fileName)
+		return nil, nil, fmt.Errorf("no projects configuration found in '%s'", fileName)
 	}
 
-	c, err := ConvertDiggerYamlToConfig(configYaml, workingDir, walker)
+	config, projectDependencyGraph, err := ConvertDiggerYamlToConfig(configYaml, workingDir, walker)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
-	for _, p := range c.Projects {
-		_, ok := c.Workflows[p.Workflow]
+	for _, p := range config.Projects {
+		_, ok := config.Workflows[p.Workflow]
 		if !ok {
-			return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", p.Workflow, p.Name)
+			return nil, nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", p.Workflow, p.Name)
 		}
 	}
-	return c, nil
+	return config, projectDependencyGraph, nil
 }
 
 func defaultProject() Project {