From 383da133e1dc69a70762b840866aa8bad2d8b096 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 11:12:37 +0100 Subject: [PATCH 01/15] only match generated projects with directory --- pkg/configuration/digger_config.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 3177cd258..b6b529964 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "regexp" ) type WorkflowConfiguration struct { @@ -85,22 +86,39 @@ type DirWalker interface { type FileSystemDirWalker struct { } -func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) { +func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { var files []string + filepath.Walk(workingDir, func(path string, f os.FileInfo, _ error) error { + if !f.IsDir() { + r, err := regexp.MatchString(ext, f.Name()) + if err == nil && r { + files = append(files, f.Name()) + } + } + return nil + }) + return files, nil +} + +func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) { + var dirs []string err := filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } if info.IsDir() { - files = append(files, path) + terraformFiles, _ := GetFilesWithExtension(path, ".tf") + if len(terraformFiles) > 0 { + dirs = append(dirs, path) + } } return nil }) if err != nil { return nil, err } - return files, nil + return dirs, nil } var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") From de7b2d02a51ec0b9b36aca5c1b01f01deef5bcfa Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 11:32:41 +0100 Subject: [PATCH 02/15] add debugging info --- pkg/configuration/digger_config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index b6b529964..31746edf5 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -104,12 +104,16 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) var dirs []string err := filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { + if err != nil { return err } if info.IsDir() { + fmt.Printf(" DEBUG: Walking dir: %v\n", workingDir) terraformFiles, _ := GetFilesWithExtension(path, ".tf") + fmt.Printf(" DEBUG: found tf files: %v\n", terraformFiles) if len(terraformFiles) > 0 { + fmt.Printf(" DEBUG: Appending dir to result: %v\n", workingDir) dirs = append(dirs, path) } } @@ -305,6 +309,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, return nil, err } + fmt.Printf("DEBUG: Generated projects: %v\n", dirs) for _, dir := range dirs { includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude From a6d1fa13995a0f3abdf7607815adcabbf14c57b2 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 11:47:53 +0100 Subject: [PATCH 03/15] fix debug --- pkg/configuration/digger_config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 31746edf5..5755cefdf 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -109,11 +109,11 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) return err } if info.IsDir() { - fmt.Printf(" DEBUG: Walking dir: %v\n", workingDir) + fmt.Printf(" DEBUG: Walking dir: %v\n", path) terraformFiles, _ := GetFilesWithExtension(path, ".tf") fmt.Printf(" DEBUG: found tf files: %v\n", terraformFiles) if len(terraformFiles) > 0 { - fmt.Printf(" DEBUG: Appending dir to result: %v\n", workingDir) + fmt.Printf(" DEBUG: Appending dir to result: %v\n", path) dirs = append(dirs, path) } } From 6877cedb9c72e163aa10def473be159af1f1340b Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 11:59:25 +0100 Subject: [PATCH 04/15] walk the right directories --- pkg/configuration/digger_config.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 5755cefdf..72c080ab9 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/bmatcuk/doublestar/v4" "gopkg.in/yaml.v3" + "log" "os" "path" "path/filepath" @@ -88,15 +89,19 @@ type FileSystemDirWalker struct { func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { var files []string - filepath.Walk(workingDir, func(path string, f os.FileInfo, _ error) error { + listOfFiles, err := filepath.readDir(workingDir) + if err != nil { + log.Fatalf("Could not list files %v", err) + } + for _, f := range listOfFiles { if !f.IsDir() { r, err := regexp.MatchString(ext, f.Name()) if err == nil && r { files = append(files, f.Name()) } } - return nil - }) + } + return files, nil } From 42ee1564ba5481e7280a6da4ec01d191ab1eca84 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:01:21 +0100 Subject: [PATCH 05/15] fix bug --- pkg/configuration/digger_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 72c080ab9..01c81bc8c 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -89,7 +89,7 @@ type FileSystemDirWalker struct { func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { var files []string - listOfFiles, err := filepath.readDir(workingDir) + listOfFiles, err := os.ReadDir(workingDir) if err != nil { log.Fatalf("Could not list files %v", err) } From dc68b0c24abbd489aae3f215a3002c5bac63ff60 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:06:14 +0100 Subject: [PATCH 06/15] remove incorrect appending --- pkg/configuration/digger_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 01c81bc8c..d228f1b72 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -329,7 +329,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } if includeMatch && !excludeMatch { // generate a new project using default workflow - project := Project{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} + project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName} diggerConfig.Projects = append(diggerConfig.Projects, project) } } From cadc86c7b05b8fe1ab1dc7c8a921f2fb669edc08 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:13:17 +0100 Subject: [PATCH 07/15] more debug methods --- pkg/configuration/digger_config.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index d228f1b72..13ac4c413 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -314,7 +314,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, return nil, err } - fmt.Printf("DEBUG: Generated projects: %v\n", dirs) + fmt.Printf("DEBUG: Found walked dirs: %v\n", dirs) for _, dir := range dirs { includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude @@ -327,12 +327,14 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, if err != nil { return nil, err } + fmt.Printf(" DEBUG: Dirname: %v, IncludeMatch: %v, ExcludeMatch: %v\n", dir, includeMatch, excludeMatch) if includeMatch && !excludeMatch { // generate a new project using default workflow project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName} diggerConfig.Projects = append(diggerConfig.Projects, project) } } + fmt.Printf("DEBUG: Generated projects %v", diggerConfig.Projects) } return &diggerConfig, nil } From f4511a58547cb191bd966516fbfa763025b7df84 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:25:34 +0100 Subject: [PATCH 08/15] use simpler function for matching --- pkg/configuration/digger_config.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 13ac4c413..9bd6075b4 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -5,7 +5,6 @@ import ( "digger/pkg/utils" "errors" "fmt" - "github.com/bmatcuk/doublestar/v4" "gopkg.in/yaml.v3" "log" "os" @@ -318,18 +317,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, for _, dir := range dirs { includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude - includeMatch, err := doublestar.PathMatch(includePattern, dir) - if err != nil { - return nil, err - } - - excludeMatch, err := doublestar.PathMatch(excludePattern, dir) - if err != nil { - return nil, err - } - fmt.Printf(" DEBUG: Dirname: %v, IncludeMatch: %v, ExcludeMatch: %v\n", dir, includeMatch, excludeMatch) - if includeMatch && !excludeMatch { - // generate a new project using default workflow + if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName} diggerConfig.Projects = append(diggerConfig.Projects, project) } From 11dff7c235b307aadb2776ecc016f5e606e929f6 Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:31:14 +0100 Subject: [PATCH 09/15] use right workspace, test default dir --- pkg/configuration/digger_config.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 9bd6075b4..4aec65889 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -119,6 +119,7 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) if len(terraformFiles) > 0 { fmt.Printf(" DEBUG: Appending dir to result: %v\n", path) dirs = append(dirs, path) + return filepath.SkipDir } } return nil @@ -318,7 +319,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { - project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName} + project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName, Workspace: "default"} diggerConfig.Projects = append(diggerConfig.Projects, project) } } From 4ec6368257f890b4a281ffbca5b8704bf524b823 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Thu, 8 Jun 2023 12:52:43 +0100 Subject: [PATCH 10/15] logging --- pkg/configuration/digger_config.go | 1 + pkg/digger/digger.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 4aec65889..fbadf2504 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -316,6 +316,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, fmt.Printf("DEBUG: Found walked dirs: %v\n", dirs) for _, dir := range dirs { + fmt.Printf("!!!!!!!!!!!!!!!!! Checking dir %s\n", dir) includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go index 5942c0359..a0be691bc 100644 --- a/pkg/digger/digger.go +++ b/pkg/digger/digger.go @@ -68,6 +68,7 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNa var terraformExecutor terraform.TerraformExecutor projectPath := path.Join(workingDir, projectCommands.ProjectDir) + fmt.Printf("!!!!!!!!!!!!!!!!!!!!!!! Project path: %s\n", projectPath) if projectCommands.Terragrunt { terraformExecutor = terraform.Terragrunt{WorkingDir: projectPath} } else { From aa54bfebac19c6011cbbb4e84c2dd5c6f75d239c Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 12:54:47 +0100 Subject: [PATCH 11/15] remove logs --- pkg/configuration/digger_config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 4aec65889..82485d7d6 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -113,11 +113,8 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) return err } if info.IsDir() { - fmt.Printf(" DEBUG: Walking dir: %v\n", path) terraformFiles, _ := GetFilesWithExtension(path, ".tf") - fmt.Printf(" DEBUG: found tf files: %v\n", terraformFiles) if len(terraformFiles) > 0 { - fmt.Printf(" DEBUG: Appending dir to result: %v\n", path) dirs = append(dirs, path) return filepath.SkipDir } From e0541e7b85a789c164b7650ca2ef51995e0477df Mon Sep 17 00:00:00 2001 From: motatoes Date: Thu, 8 Jun 2023 13:07:56 +0100 Subject: [PATCH 12/15] fix tests --- pkg/configuration/digger_config_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 9dd42af67..d7d9f6414 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -238,8 +238,8 @@ generate_projects: assert.NotNil(t, dg, "expected digger config to be not nil") assert.Equal(t, "test1", dg.Projects[0].Name) assert.Equal(t, "test2", dg.Projects[1].Name) - assert.Equal(t, tempDir+"/dev/test1", dg.Projects[0].Dir) - assert.Equal(t, tempDir+"/dev/test2", dg.Projects[1].Dir) + assert.Equal(t, "dev/test1", dg.Projects[0].Dir) + assert.Equal(t, "dev/test2", dg.Projects[1].Dir) assert.Equal(t, 2, len(dg.Projects)) } @@ -268,8 +268,8 @@ generate_projects: assert.NotNil(t, dg, "expected digger config to be not nil") assert.Equal(t, "test1", dg.Projects[0].Name) assert.Equal(t, "test2", dg.Projects[1].Name) - assert.Equal(t, tempDir+"/dev/test1", dg.Projects[0].Dir) - assert.Equal(t, tempDir+"/dev/test2", dg.Projects[1].Dir) + assert.Equal(t, "dev/test1", dg.Projects[0].Dir) + assert.Equal(t, "dev/test2", dg.Projects[1].Dir) assert.Equal(t, 2, len(dg.Projects)) } From 17705ac1ac22ed7b12e405f6ce6c8a56e26bc0a8 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Thu, 8 Jun 2023 13:55:07 +0100 Subject: [PATCH 13/15] logging --- cmd/digger/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index cb635723b..77e86e44b 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -101,8 +101,11 @@ func gitHubCI(lock core_locking.Lock) { CiService: githubPrService, PrNumber: prNumber, } - - allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, parsedGhContext.Repository, githubActor, eventName, prNumber, githubPrService, lock, reporter, planStorage, "") + currentDir, err := os.Getwd() + if err != nil { + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to get current dir. %s", err), 4) + } + allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, parsedGhContext.Repository, githubActor, eventName, prNumber, githubPrService, lock, reporter, planStorage, currentDir) if err != nil { reportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8) } From 6d48207dfbae65debf04d8dbbc760e186c74e004 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Thu, 8 Jun 2023 14:07:11 +0100 Subject: [PATCH 14/15] replace slashes --- pkg/core/execution/execution.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/execution/execution.go b/pkg/core/execution/execution.go index e554c7b54..d32087a60 100644 --- a/pkg/core/execution/execution.go +++ b/pkg/core/execution/execution.go @@ -32,7 +32,7 @@ type DiggerExecutor struct { } func (d DiggerExecutor) planFileName() string { - return d.ProjectNamespace + "#" + d.ProjectName + ".tfplan" + return strings.ReplaceAll(d.ProjectNamespace, "/", ":") + "#" + d.ProjectName + ".tfplan" } func (d DiggerExecutor) localPlanFilePath() string { From a54b8964653bb2f46abebe43489b9f15d4df1ec2 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Thu, 8 Jun 2023 17:00:30 +0100 Subject: [PATCH 15/15] fix file name when saving file --- pkg/configuration/digger_config.go | 6 +----- pkg/digger/digger.go | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 134d927fd..c7d4c1fb6 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "gopkg.in/yaml.v3" - "log" "os" "path" "path/filepath" @@ -90,7 +89,7 @@ func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { var files []string listOfFiles, err := os.ReadDir(workingDir) if err != nil { - log.Fatalf("Could not list files %v", err) + return nil, errors.New(fmt.Sprintf("error reading directory %s: %v", workingDir, err)) } for _, f := range listOfFiles { if !f.IsDir() { @@ -311,9 +310,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, return nil, err } - fmt.Printf("DEBUG: Found walked dirs: %v\n", dirs) for _, dir := range dirs { - fmt.Printf("!!!!!!!!!!!!!!!!! Checking dir %s\n", dir) includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { @@ -321,7 +318,6 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, diggerConfig.Projects = append(diggerConfig.Projects, project) } } - fmt.Printf("DEBUG: Generated projects %v", diggerConfig.Projects) } return &diggerConfig, nil } diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go index a0be691bc..5942c0359 100644 --- a/pkg/digger/digger.go +++ b/pkg/digger/digger.go @@ -68,7 +68,6 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNa var terraformExecutor terraform.TerraformExecutor projectPath := path.Join(workingDir, projectCommands.ProjectDir) - fmt.Printf("!!!!!!!!!!!!!!!!!!!!!!! Project path: %s\n", projectPath) if projectCommands.Terragrunt { terraformExecutor = terraform.Terragrunt{WorkingDir: projectPath} } else {