diff --git a/go.mod b/go.mod index 3b613f738e..40ffbf46ad 100644 --- a/go.mod +++ b/go.mod @@ -107,6 +107,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect github.com/ulikunitz/xz v0.5.8 // indirect + github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 // indirect github.com/zclconf/go-cty v1.5.1 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.7.0 // indirect diff --git a/go.sum b/go.sum index 117395ca7c..813448580d 100644 --- a/go.sum +++ b/go.sum @@ -494,6 +494,8 @@ github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKn github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xanzy/go-gitlab v0.59.0 h1:fAr6rT/YIdfmBavYgI42+Op7yAAex2Y4xOfvbjN9hxQ= github.com/xanzy/go-gitlab v0.59.0/go.mod h1:F0QEXwmqiBUxCgJm8fE9S+1veX4XC9Z4cfaAbqwk4YM= +github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869 h1:7v7L5lsfw4w8iqBBXETukHo4IPltmD+mWoLRYUmeGN8= +github.com/yourbasic/graph v0.0.0-20210606180040-8ecfec1c2869/go.mod h1:Rfzr+sqaDreiCaoQbFCu3sTXxeFq/9kXRuyOoSlGQHE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/server/events/pending_plan_finder.go b/server/events/pending_plan_finder.go index 38b8bbdec7..48abdac1ce 100644 --- a/server/events/pending_plan_finder.go +++ b/server/events/pending_plan_finder.go @@ -4,7 +4,9 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strings" + "time" "github.com/pkg/errors" "github.com/runatlantis/atlantis/server/core/runtime" @@ -41,6 +43,47 @@ func (p *DefaultPendingPlanFinder) Find(pullDir string) ([]PendingPlan, error) { return plans, err } +func sortPendingPlansByModTime(repoDir, lsOut string) ([]string, error) { + type fileModTime struct { + File string + ModTime time.Time + } + + var files []fileModTime + for _, file := range strings.Split(lsOut, "\n") { + if filepath.Ext(file) != ".tfplan" { + continue + } + // Ignore .terragrunt-cache dirs (#487) + if strings.Contains(file, ".terragrunt-cache/") { + continue + } + + fileInfo, err := os.Stat(filepath.Join(repoDir, file)) + + if err != nil { + return nil, err + } + + files = append(files, fileModTime{ + File: file, + ModTime: fileInfo.ModTime(), + }) + } + + sort.Slice(files, func(i, j int) bool { + return 0 < files[j].ModTime.Sub(files[i].ModTime) + }) + + var sortedFiles []string + + for _, el := range files { + sortedFiles = append(sortedFiles, el.File) + } + + return sortedFiles, nil +} + func (p *DefaultPendingPlanFinder) findWithAbsPaths(pullDir string) ([]PendingPlan, []string, error) { workspaceDirs, err := os.ReadDir(pullDir) if err != nil { @@ -61,25 +104,25 @@ func (p *DefaultPendingPlanFinder) findWithAbsPaths(pullDir string) ([]PendingPl return nil, nil, errors.Wrapf(err, "running git ls-files . "+ "--others: %s", string(lsOut)) } - for _, file := range strings.Split(string(lsOut), "\n") { - if filepath.Ext(file) == ".tfplan" { - // Ignore .terragrunt-cache dirs (#487) - if strings.Contains(file, ".terragrunt-cache/") { - continue - } - - projectName, err := runtime.ProjectNameFromPlanfile(workspace, filepath.Base(file)) - if err != nil { - return nil, nil, err - } - plans = append(plans, PendingPlan{ - RepoDir: repoDir, - RepoRelDir: filepath.Dir(file), - Workspace: workspace, - ProjectName: projectName, - }) - absPaths = append(absPaths, filepath.Join(repoDir, file)) + + files, err := sortPendingPlansByModTime(repoDir, string(lsOut)) + + if err != nil { + return nil, nil, errors.Wrapf(err, "sort pending plans by modTime") + } + + for _, file := range files { + projectName, err := runtime.ProjectNameFromPlanfile(workspace, filepath.Base(file)) + if err != nil { + return nil, nil, err } + plans = append(plans, PendingPlan{ + RepoDir: repoDir, + RepoRelDir: filepath.Dir(file), + Workspace: workspace, + ProjectName: projectName, + }) + absPaths = append(absPaths, filepath.Join(repoDir, file)) } } return plans, absPaths, nil diff --git a/server/events/project_finder.go b/server/events/project_finder.go index b247122f7f..dea0bc3f1b 100644 --- a/server/events/project_finder.go +++ b/server/events/project_finder.go @@ -14,12 +14,14 @@ package events import ( + "fmt" "os" "path" "path/filepath" "strings" "github.com/runatlantis/atlantis/server/core/config/valid" + "github.com/yourbasic/graph" "github.com/docker/docker/pkg/fileutils" "github.com/pkg/errors" @@ -83,6 +85,7 @@ func (p *DefaultProjectFinder) DetermineProjects(log logging.SimpleLogging, modi // See ProjectFinder.DetermineProjectsViaConfig. func (p *DefaultProjectFinder) DetermineProjectsViaConfig(log logging.SimpleLogging, modifiedFiles []string, config valid.RepoCfg, absRepoDir string) ([]valid.Project, error) { var projects []valid.Project + var projectsToDependentFiles [][]string for _, project := range config.Projects { log.Debug("checking if project at dir %q workspace %q was modified", project.Dir, project.Workspace) var whenModifiedRelToRepoRoot []string @@ -110,6 +113,8 @@ func (p *DefaultProjectFinder) DetermineProjectsViaConfig(log logging.SimpleLogg return nil, errors.Wrapf(err, "matching modified files with patterns: %v", project.Autoplan.WhenModified) } + var matchedFiles []string + // If any of the modified files matches the pattern then this project is // considered modified. for _, file := range modifiedFiles { @@ -118,32 +123,159 @@ func (p *DefaultProjectFinder) DetermineProjectsViaConfig(log logging.SimpleLogg log.Debug("match err for file %q: %s", file, err) continue } - if match { - log.Debug("file %q matched pattern", file) - // If we're checking using an atlantis.yaml file we downloaded - // directly from the repo (when doing a no-clone check) then - // absRepoDir will be empty. Since we didn't clone the repo - // yet we can't do this check. If there was a file modified - // in a deleted directory then when we finally do clone the repo - // we'll call this function again and then we'll detect the - // directory was deleted. - if absRepoDir != "" { - _, err := os.Stat(filepath.Join(absRepoDir, project.Dir)) - if err == nil { - projects = append(projects, project) - } else { - log.Debug("project at dir %q not included because dir does not exist", project.Dir) - } - } else { - projects = append(projects, project) - } - break + + if !match { + continue } + + log.Debug("file %q matched pattern", file) + + // If we're checking using an atlantis.yaml file we downloaded + // directly from the repo (when doing a no-clone check) then + // absRepoDir will be empty. Since we didn't clone the repo + // yet we can't do this check. If there was a file modified + // in a deleted directory then when we finally do clone the repo + // we'll call this function again and then we'll detect the + // directory was deleted. + if projectDirDeleted(absRepoDir, project.Dir) { + log.Debug("project at dir %q not included because dir does not exist", project.Dir) + continue + } + + matchedFiles = append(matchedFiles, file) + } + + if 0 < len(matchedFiles) { + projects = append(projects, project) + projectsToDependentFiles = append(projectsToDependentFiles, matchedFiles) } } + + if 1 < len(projects) { + sorted, err := sortProjectsByDependencies(log, projects, projectsToDependentFiles, absRepoDir) + + if err != nil { + // return nil, errors.Wrapf(err, "Unable to sort plans: %s", err) + // Used unsorted projects instead + } else { + projects = sorted + } + } + return projects, nil } +func projectDirDeleted(absRepoDir, projectDir string) bool { + if absRepoDir == "" { + return false + } + + if _, err := os.Stat(filepath.Join(absRepoDir, projectDir)); err != nil { + return true + } + + return false +} + +func projectToBeDestroyed(project valid.Project, absRepoDir string) (bool, error) { + terragruntFile := filepath.Join(absRepoDir, project.Dir, "terragrunt.hcl") + + if _, err := os.Stat(terragruntFile); err != nil { + if os.IsNotExist(err) { + return false, nil + } + + return false, errors.Wrapf(err, "determining existance of %q", terragruntFile) + } + + data, err := os.ReadFile(terragruntFile) + if err != nil { + return false, errors.Wrapf(err, "reading file %q", terragruntFile) + } + + if strings.HasPrefix(strings.TrimSpace(string(data)), "# ATLANTIS_PLEASE_DESTROY_STACK") { + return true, nil + } + + return false, nil +} + +func printGraph(g *graph.Mutable, projects []valid.Project) string { + output := "digraph G {\n" + + for i := 0; i < g.Order(); i++ { + output += fmt.Sprintf(" %d [label=\"%s\"];\n", i, projects[i].Dir) + } + for i := 0; i < g.Order(); i++ { + g.Visit(i, func(w int, to int64) bool { output += fmt.Sprintf(" %d -> %d;\n", i, w); return false }) + } + output += "}\n" + + return output +} + +func sortProjectsByDependencies(log logging.SimpleLogging, projects []valid.Project, projectsToDependentFiles [][]string, absRepoDir string) ([]valid.Project, error) { + g := graph.New(len(projects)) + + for currentProjectIndex, files := range projectsToDependentFiles { + if 0 == len(files) { + continue + } + + toBeDestroyed, err := projectToBeDestroyed(projects[currentProjectIndex], absRepoDir) + + if err != nil { + return nil, errors.Wrapf(err, "determining whether or not to apply destroy order for topological sort") + } + + for _, file := range files { + if !strings.Contains(file, string(os.PathSeparator)+"terragrunt.hcl") { + continue + } + from, err := FindProjectNo(projects, file) + + if err != nil { + return nil, errors.Wrapf(err, "creating dependency graph for topological sort") + } + + if currentProjectIndex == from { + continue // don't add an edge to itself e.g. (u,u) + } + if toBeDestroyed { + g.Add(currentProjectIndex, from) + } else { + g.Add(from, currentProjectIndex) + } + } + } + + // TODO before contributing to github, make this a Debug() or delete + log.Info("Dependency Graph:\n%s\n", printGraph(g, projects)) + sortedIndices, isSorted := graph.TopSort(g) + + if !isSorted { + return nil, fmt.Errorf("topological sort failed on %#v", projectsToDependentFiles) + } + + var sortedProjects []valid.Project + + for _, index := range sortedIndices { + sortedProjects = append(sortedProjects, projects[index]) + } + + return sortedProjects, nil +} + +func FindProjectNo(projects []valid.Project, file string) (int, error) { + for i, project := range projects { + if strings.Contains(file, project.Dir+string(os.PathSeparator)) { + return i, nil + } + } + + return -1, fmt.Errorf("Did not find %q in %#v", file, projects) +} + // filterToFileList filters out files not included in the file list func (p *DefaultProjectFinder) filterToFileList(log logging.SimpleLogging, files []string, fileList string) []string { var filtered []string diff --git a/server/events/project_finder_test.go b/server/events/project_finder_test.go index cf19538a7c..55bb1e1877 100644 --- a/server/events/project_finder_test.go +++ b/server/events/project_finder_test.go @@ -16,10 +16,13 @@ package events_test import ( "os" "path/filepath" + "strings" "testing" + "github.com/runatlantis/atlantis/server/core/config" "github.com/runatlantis/atlantis/server/core/config/valid" "github.com/runatlantis/atlantis/server/events" + "github.com/runatlantis/atlantis/server/logging" . "github.com/runatlantis/atlantis/testing" ) @@ -514,3 +517,260 @@ func TestDefaultProjectFinder_DetermineProjectsViaConfig(t *testing.T) { }) } } + +func createTerragruntFiles(t *testing.T, files []string, destroyFlagSet bool) (string, func()) { + content := "" + + if destroyFlagSet { + content = "# ATLANTIS_PLEASE_DESTROY_STACK\n" + } + + dirs := make(map[string]interface{}) + + for _, file := range files { + var currentDirPtr map[string]interface{} = dirs + + folders := strings.Split(file, string(os.PathSeparator)) + + for i, folder := range folders { + if i == len(folders)-1 { // last item in folders is the filename + currentDirPtr[folder] = content + continue + } + + if _, ok := currentDirPtr[folder]; !ok { + currentDirPtr[folder] = make(map[string]interface{}) + } + + currentDirPtr = currentDirPtr[folder].(map[string]interface{}) + } + } + + return DirStructure(t, dirs) +} + +func createDirEnv(t *testing.T, files []string, atlantisYaml string, destroyFlagSet bool) (valid.RepoCfg, string, func()) { + tmpDir, cleanup := createTerragruntFiles(t, files, destroyFlagSet) + + if atlantisYaml != "" { + err := os.WriteFile(filepath.Join(tmpDir, config.AtlantisYAMLFilename), []byte(atlantisYaml), 0600) + Ok(t, err) + } + r := config.ParserValidator{} + var globalCfg = valid.NewGlobalCfg(true, false, false) + config, err := r.ParseRepoCfg(tmpDir, globalCfg, "") + Ok(t, err) + + return config, tmpDir, cleanup +} + +func TestDefaultProjectFinder_DetermineProjectsViaConfig_FindProjectNo(t *testing.T) { + noopLogger := logging.NewNoopLogger(t) + cases := []struct { + description string + AtlantisYAML string + modified []string + }{ + { + description: "test FindProjectNo", + AtlantisYAML: ` +version: 3 +projects: +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + dir: dkprto/icawtopr/foo +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../foo/terragrunt.hcl + dir: dkprto/icawtopr/foo-attachment +`, + modified: []string{"dkprto/icawtopr/foo/terragrunt.hcl", "dkprto/icawtopr/foo-attachment/terragrunt.hcl"}, + }, + { + description: "test FindProjectNo with reverse atlantis config", + AtlantisYAML: ` +version: 3 +projects: +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../foo/terragrunt.hcl + dir: dkprto/icawtopr/foo-attachment +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + dir: dkprto/icawtopr/foo +`, + modified: []string{"dkprto/icawtopr/foo/terragrunt.hcl", "dkprto/icawtopr/foo-attachment/terragrunt.hcl"}, + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + config, tmpDir, cleanup := createDirEnv(t, c.modified, c.AtlantisYAML, false) + defer cleanup() + + pf := events.DefaultProjectFinder{} + + projects, err := pf.DetermineProjectsViaConfig(noopLogger, c.modified, config, tmpDir) + Ok(t, err) + + for _, file := range c.modified { + index, err := events.FindProjectNo(projects, file) + Ok(t, err) + Equals(t, projects[index].Dir, filepath.Dir(file)) + } + }) + } +} + +func TestDefaultProjectFinder_DetermineProjectsViaConfig_TestDependencyTracking(t *testing.T) { + noopLogger := logging.NewNoopLogger(t) + cases := []struct { + description string + AtlantisYAML string + modified []string + expProjPaths []string + destroyFlagSet bool + expError string + }{ + { + description: "test dependency ordering", + AtlantisYAML: ` +version: 3 +projects: +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../4/terragrunt.hcl + dir: dependency-test/1 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../3/terragrunt.hcl + dir: dependency-test/2 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../1/terragrunt.hcl + dir: dependency-test/3 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + dir: dependency-test/4 +`, + modified: []string{"dependency-test/1/terragrunt.hcl", "dependency-test/2/terragrunt.hcl", "dependency-test/3/terragrunt.hcl", "dependency-test/4/terragrunt.hcl"}, + expProjPaths: []string{"dependency-test/4", "dependency-test/1", "dependency-test/3", "dependency-test/2"}, + destroyFlagSet: false, + expError: "", + }, + { + description: "test reverted dependency ordering for destroy workflows", + AtlantisYAML: ` +version: 3 +projects: +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../4/terragrunt.hcl + dir: dependency-test/1 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../3/terragrunt.hcl + dir: dependency-test/2 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../1/terragrunt.hcl + dir: dependency-test/3 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + dir: dependency-test/4 +`, + modified: []string{"dependency-test/1/terragrunt.hcl", "dependency-test/2/terragrunt.hcl", "dependency-test/3/terragrunt.hcl", "dependency-test/4/terragrunt.hcl"}, + expProjPaths: []string{"dependency-test/2", "dependency-test/3", "dependency-test/1", "dependency-test/4"}, + destroyFlagSet: true, + expError: "", + }, + { + description: "test dependency loop", + AtlantisYAML: ` +version: 3 +projects: +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../1/terragrunt.hcl + dir: dependency-test/2 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../2/terragrunt.hcl + dir: dependency-test/3 +- autoplan: + enabled: true + when_modified: + - '*.hcl' + - '*.tf*' + - ../3/terragrunt.hcl + dir: dependency-test/1 +`, + modified: []string{"dependency-test/1/terragrunt.hcl", "dependency-test/2/terragrunt.hcl", "dependency-test/3/terragrunt.hcl"}, + expProjPaths: []string{}, + destroyFlagSet: false, + expError: "topological sort failed", + }, + } + + for _, c := range cases { + t.Run(c.description, func(t *testing.T) { + config, tmpDir, cleanup := createDirEnv(t, c.modified, c.AtlantisYAML, c.destroyFlagSet) + defer cleanup() + + pf := events.DefaultProjectFinder{} + + projects, err := pf.DetermineProjectsViaConfig(noopLogger, c.modified, config, tmpDir) + if c.expError == "" { + Ok(t, err) + Equals(t, len(c.expProjPaths), len(projects)) + for i, proj := range projects { + Equals(t, c.expProjPaths[i], proj.Dir) + } + } else { + ErrContains(t, c.expError, err) + } + }) + } +} diff --git a/testing/temp_files.go b/testing/temp_files.go index 3ee8b062c4..41950a7c0c 100644 --- a/testing/temp_files.go +++ b/testing/temp_files.go @@ -1,8 +1,10 @@ package testing import ( + "io/ioutil" "os" "path/filepath" + "sort" "testing" ) @@ -48,23 +50,37 @@ func DirStructure(t *testing.T, structure map[string]interface{}) (string, func( return tmpDir, cleanup } +func sortHash(h map[string]interface{}) []string { + keys := make([]string, len(h)) + + i := 0 + for k := range h { + keys[i] = k + i++ + } + + sort.Strings(keys) + + return keys +} + func dirStructureGo(t *testing.T, parentDir string, structure map[string]interface{}) { - for key, val := range structure { - // If val is nil then key is a filename and we just create it - if val == nil { + for _, key := range sortHash(structure) { + // If structure[key] is nil then key is a filename and we just create it + if structure[key] == nil { _, err := os.Create(filepath.Join(parentDir, key)) Ok(t, err) continue } - // If val is another map then key is a dir - if dirContents, ok := val.(map[string]interface{}); ok { + // If structure[key] is another map then key is a dir + if dirContents, ok := structure[key].(map[string]interface{}); ok { subDir := filepath.Join(parentDir, key) Ok(t, os.Mkdir(subDir, 0700)) // Recurse and create contents. dirStructureGo(t, subDir, dirContents) - } else if fileContent, ok := val.(string); ok { - // If val is a string then key is a file name and val is the file's content - err := os.WriteFile(filepath.Join(parentDir, key), []byte(fileContent), 0600) + } else if fileContent, ok := structure[key].(string); ok { + // If structure[key] is a string then key is a file name and structure[key] is the file's content + err := ioutil.WriteFile(filepath.Join(parentDir, key), []byte(fileContent), 0600) Ok(t, err) } }