From 859d01a4b2ca459909d4a0bd5297c50f5650baff Mon Sep 17 00:00:00 2001 From: alexey Date: Wed, 7 Jun 2023 16:26:06 +0100 Subject: [PATCH 01/20] split DiggerConfig into two parts, one for yaml parsing and one for config itself --- cmd/digger/main.go | 2 +- cmd/digger/main_test.go | 24 +- go.mod | 1 + go.sum | 18 +- pkg/azure/azure.go | 8 +- pkg/configuration/digger_config.go | 330 ++++++++++++++++-------- pkg/configuration/digger_config_test.go | 12 +- pkg/digger/digger.go | 14 +- pkg/digger/digger_test.go | 12 +- pkg/github/github.go | 8 +- pkg/gitlab/gitlab.go | 8 +- pkg/models/models.go | 4 +- 12 files changed, 272 insertions(+), 169 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index 54753bf0c..ce5044152 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -367,7 +367,7 @@ func newPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, return planStorage } -func logImpactedProjects(projects []configuration.Project, prNumber int) { +func logImpactedProjects(projects []configuration.ProjectConfig, prNumber int) { logMessage := fmt.Sprintf("Following projects are impacted by pull request #%d\n", prNumber) for _, p := range projects { logMessage += fmt.Sprintf("%s\n", p.Name) diff --git a/cmd/digger/main_test.go b/cmd/digger/main_test.go index 562761303..3c1571cf0 100644 --- a/cmd/digger/main_test.go +++ b/cmd/digger/main_test.go @@ -884,7 +884,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) { CiService: prManager, PrNumber: prNumber, } - commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.Workflow{}) + commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.WorkflowConfig{}) _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, "") assert.NoError(t, err) @@ -910,7 +910,7 @@ func TestGitHubNewCommentContext(t *testing.T) { CiService: prManager, PrNumber: prNumber, } - commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.Workflow{}) + commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.WorkflowConfig{}) _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, "") _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, context.RepositoryOwner, context.Repository, eventName, prNumber, prManager, lock, reporter, planStorage, "") @@ -934,41 +934,41 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { assert.NoError(t, err) ghEvent := context.Event pullRequestNumber := 11 - dev := configuration.Project{Name: "dev", Dir: "dev", Workflow: "dev"} - prod := configuration.Project{Name: "prod", Dir: "prod", Workflow: "prod"} - workflows := map[string]configuration.Workflow{ + dev := configuration.ProjectConfig{Name: "dev", Dir: "dev", Workflow: "dev"} + prod := configuration.ProjectConfig{Name: "prod", Dir: "prod", Workflow: "prod"} + workflows := map[string]configuration.WorkflowConfig{ "dev": { - Plan: &configuration.Stage{Steps: []configuration.Step{ + Plan: &configuration.StageConfig{Steps: []configuration.StepConfig{ {Action: "init", ExtraArgs: []string{}}, {Action: "plan", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Apply: &configuration.Stage{Steps: []configuration.Step{ + Apply: &configuration.StageConfig{Steps: []configuration.StepConfig{ {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Configuration: &configuration.WorkflowConfiguration{ + Configuration: &configuration.WorkflowConfigurationConfig{ OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, OnCommitToDefault: []string{"digger apply"}, }, }, "prod": { - Plan: &configuration.Stage{Steps: []configuration.Step{ + Plan: &configuration.StageConfig{Steps: []configuration.StepConfig{ {Action: "init", ExtraArgs: []string{}}, {Action: "plan", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Apply: &configuration.Stage{Steps: []configuration.Step{ + Apply: &configuration.StageConfig{Steps: []configuration.StepConfig{ {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Configuration: &configuration.WorkflowConfiguration{ + Configuration: &configuration.WorkflowConfigurationConfig{ OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, OnCommitToDefault: []string{"digger apply"}, }, }, } - projects := []configuration.Project{dev, prod} + projects := []configuration.ProjectConfig{dev, prod} diggerConfig := configuration.DiggerConfig{Projects: projects} // PullRequestManager Mock diff --git a/go.mod b/go.mod index 07f2cc198..7e3c7e901 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/caarlos0/env/v7 v7.1.0 github.com/google/go-github/v51 v51.0.0 github.com/google/uuid v1.3.0 + github.com/jinzhu/copier v0.3.5 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/stretchr/testify v1.8.4 github.com/xanzy/go-gitlab v0.84.0 diff --git a/go.sum b/go.sum index 1e23ff1c2..d43319050 100644 --- a/go.sum +++ b/go.sum @@ -23,16 +23,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDm github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= -github.com/aws/aws-sdk-go v1.44.266 h1:MWd775dcYf7NrwgcHLtlsIbWoWkX8p4vomfNHr88zH0= -github.com/aws/aws-sdk-go v1.44.266/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.268 h1:WoK20tlAvsvQzTcE6TajoprbXmTbcud6MjhErL4P/38= -github.com/aws/aws-sdk-go v1.44.268/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.273 h1:CX8O0gK+cGrgUyv7bgJ6QQP9mQg7u5mweHdNzULH47c= -github.com/aws/aws-sdk-go v1.44.273/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.274 h1:vfreSv19e/9Ka9YytOzgzJasrRZfX7dnttLlbh8NKeA= -github.com/aws/aws-sdk-go v1.44.274/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.44.275 h1:VqRULgqrigvQLll4e4hXuc568EQAtZQ6jmBzLlQHzSI= -github.com/aws/aws-sdk-go v1.44.275/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.276 h1:ywPlx9C5Yc482dUgAZ9bHpQ6onVvJvYE9FJWsNDCEy0= github.com/aws/aws-sdk-go v1.44.276/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= @@ -101,6 +91,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -127,12 +119,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/xanzy/go-gitlab v0.83.0 h1:37p0MpTPNbsTMKX/JnmJtY8Ch1sFiJzVF342+RvZEGw= -github.com/xanzy/go-gitlab v0.83.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/xanzy/go-gitlab v0.84.0 h1:PdpCaskQSgcVDsx21c6ikf8Rfyo7SNtFAJwP9PrbCFE= github.com/xanzy/go-gitlab v0.84.0/go.mod h1:5ryv+MnpZStBH8I/77HuQBsMbBGANtVpLWC15qOjWAw= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -236,8 +224,10 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 66d4115e8..fad28784e 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -298,8 +298,8 @@ func (a *AzureReposService) IsClosed(prNumber int) (bool, error) { return *pullRequest.Status == git.PullRequestStatusValues.Completed || *pullRequest.Status == git.PullRequestStatusValues.Abandoned, nil } -func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.Project, *configuration.Project, int, error) { - var impactedProjects []configuration.Project +func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, int, error) { + var impactedProjects []configuration.ProjectConfig var prNumber int switch azureEvent.(type) { @@ -340,7 +340,7 @@ func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration. return impactedProjects, nil, prNumber, nil } -func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, bool, error) { +func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) switch parseAzureContext.EventType { case AzurePrCreated, AzurePrUpdated, AzurePrReopened: @@ -422,7 +422,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.Project{*requestedProject} + runForProjects = []configuration.ProjectConfig{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 14235e04c..fe491b8d3 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -5,68 +5,121 @@ import ( "errors" "fmt" "github.com/bmatcuk/doublestar/v4" + "github.com/jinzhu/copier" "gopkg.in/yaml.v3" "os" "path" "path/filepath" ) -type WorkflowConfiguration struct { +type DiggerConfigYaml struct { + Projects []ProjectYaml `yaml:"projects"` + AutoMerge bool `yaml:"auto_merge"` + Workflows map[string]WorkflowYaml `yaml:"workflows"` + CollectUsageData bool `yaml:"collect_usage_data"` + GenerateProjectsConfig *GenerateProjectsConfigYaml `yaml:"generate_projects"` +} + +type ProjectYaml struct { + Name string `yaml:"name"` + Dir string `yaml:"dir"` + Workspace string `yaml:"workspace"` + Terragrunt bool `yaml:"terragrunt"` + Workflow string `yaml:"workflow"` + IncludePatterns []string `yaml:"include_patterns,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` +} + +type WorkflowYaml struct { + EnvVars *EnvVarsYaml `yaml:"env_vars"` + Plan *StageYaml `yaml:"plan,omitempty"` + Apply *StageYaml `yaml:"apply,omitempty"` + Configuration *WorkflowConfigurationYaml `yaml:"workflow_configuration"` +} + +type WorkflowConfigurationYaml struct { OnPullRequestPushed []string `yaml:"on_pull_request_pushed"` OnPullRequestClosed []string `yaml:"on_pull_request_closed"` OnCommitToDefault []string `yaml:"on_commit_to_default"` } -type DiggerConfigYaml struct { - Projects []Project `yaml:"projects"` - AutoMerge bool `yaml:"auto_merge"` - Workflows map[string]Workflow `yaml:"workflows"` - CollectUsageData bool `yaml:"collect_usage_data"` - GenerateProjectsConfig *GenerateProjectsConfig `yaml:"generate_projects"` +type StageYaml struct { + Steps []StepYaml `yaml:"steps"` +} + +type StepYaml struct { + Action string + Value string + ExtraArgs []string `yaml:"extra_args,omitempty"` + Shell string } -type EnvVarConfig struct { +type EnvVarsYaml struct { + State []EnvVarConfigYaml `yaml:"state"` + Commands []EnvVarConfigYaml `yaml:"commands"` +} + +type EnvVarConfigYaml struct { Name string `yaml:"name"` ValueFrom string `yaml:"value_from"` Value string `yaml:"value"` } +type GenerateProjectsConfigYaml struct { + Include string `yaml:"include"` + Exclude string `yaml:"exclude"` +} + type DiggerConfig struct { - Projects []Project + Projects []ProjectConfig AutoMerge bool CollectUsageData bool - Workflows map[string]Workflow + Workflows map[string]WorkflowConfig } -type GenerateProjectsConfig struct { - Include string `yaml:"include"` - Exclude string `yaml:"exclude"` +type ProjectConfig struct { + Name string + Dir string + Workspace string + Terragrunt bool + Workflow string + IncludePatterns []string + ExcludePatterns []string } -type Project struct { - Name string `yaml:"name"` - Dir string `yaml:"dir"` - Workspace string `yaml:"workspace"` - Terragrunt bool `yaml:"terragrunt"` - Workflow string `yaml:"workflow"` - IncludePatterns []string `yaml:"include_patterns,omitempty"` - ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` +type WorkflowConfig struct { + EnvVars *EnvVarsConfig + Plan *StageConfig + Apply *StageConfig + Configuration *WorkflowConfigurationConfig +} + +type WorkflowConfigurationConfig struct { + OnPullRequestPushed []string + OnPullRequestClosed []string + OnCommitToDefault []string } -type Stage struct { - Steps []Step `yaml:"steps"` +type StageConfig struct { + Steps []StepConfig } -type Workflow struct { - EnvVars EnvVars `yaml:"env_vars"` - Plan *Stage `yaml:"plan,omitempty"` - Apply *Stage `yaml:"apply,omitempty"` - Configuration *WorkflowConfiguration `yaml:"workflow_configuration"` +type StepConfig struct { + Action string + Value string + ExtraArgs []string + Shell string } -type EnvVars struct { - State []EnvVarConfig `yaml:"state"` - Commands []EnvVarConfig `yaml:"commands"` +type EnvVarsConfig struct { + State []EnvVarConfigConfig + Commands []EnvVarConfigConfig +} + +type EnvVarConfigConfig struct { + Name string + ValueFrom string + Value string } type DirWalker interface { @@ -96,8 +149,8 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") -func (p *Project) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawProject Project +func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawProject ProjectYaml raw := rawProject{ Workspace: "default", Terragrunt: false, @@ -106,20 +159,20 @@ func (p *Project) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal(&raw); err != nil { return err } - *p = Project(raw) + *p = ProjectYaml(raw) return nil } -func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawWorkflow Workflow +func (w *WorkflowYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawWorkflow WorkflowYaml raw := rawWorkflow{ - Configuration: &WorkflowConfiguration{ + Configuration: &WorkflowConfigurationYaml{ OnCommitToDefault: []string{"digger unlock"}, OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, }, - Plan: &Stage{ - Steps: []Step{ + Plan: &StageYaml{ + Steps: []StepYaml{ { Action: "init", ExtraArgs: []string{}, }, @@ -128,8 +181,8 @@ func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { }, }, }, - Apply: &Stage{ - Steps: []Step{ + Apply: &StageYaml{ + Steps: []StepYaml{ { Action: "init", ExtraArgs: []string{}, }, @@ -138,26 +191,19 @@ func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { }, }, }, - EnvVars: EnvVars{ - State: []EnvVarConfig{}, - Commands: []EnvVarConfig{}, + EnvVars: &EnvVarsYaml{ + State: []EnvVarConfigYaml{}, + Commands: []EnvVarConfigYaml{}, }, } if err := unmarshal(&raw); err != nil { return err } - *w = Workflow(raw) + *w = WorkflowYaml(raw) return nil } -type Step struct { - Action string - Value string - ExtraArgs []string `yaml:"extra_args,omitempty"` - Shell string -} - -func (s *Step) UnmarshalYAML(value *yaml.Node) error { +func (s *StepYaml) UnmarshalYAML(value *yaml.Node) error { if value.Kind == yaml.ScalarNode { return value.Decode(&s.Action) } @@ -182,7 +228,7 @@ func (s *Step) UnmarshalYAML(value *yaml.Node) error { return nil } -func (s *Step) extract(stepMap map[string]interface{}, action string) { +func (s *StepYaml) extract(stepMap map[string]interface{}, action string) { if _, ok := stepMap[action]; ok { s.Action = action var extraArgs []string @@ -195,15 +241,15 @@ func (s *Step) extract(stepMap map[string]interface{}, action string) { } } -func defaultWorkflow() *Workflow { - return &Workflow{ - Configuration: &WorkflowConfiguration{ +func defaultWorkflow() *WorkflowConfig { + return &WorkflowConfig{ + Configuration: &WorkflowConfigurationConfig{ OnCommitToDefault: []string{"digger unlock"}, OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, }, - Plan: &Stage{ - Steps: []Step{ + Plan: &StageConfig{ + Steps: []StepConfig{ { Action: "init", ExtraArgs: []string{}, }, @@ -212,8 +258,8 @@ func defaultWorkflow() *Workflow { }, }, }, - Apply: &Stage{ - Steps: []Step{ + Apply: &StageConfig{ + Steps: []StepConfig{ { Action: "init", ExtraArgs: []string{}, }, @@ -225,6 +271,94 @@ func defaultWorkflow() *Workflow { } } +func copyProjects(projects []ProjectYaml) []ProjectConfig { + result := make([]ProjectConfig, len(projects)) + for i, p := range projects { + item := ProjectConfig{p.Name, + p.Dir, + p.Workspace, + p.Terragrunt, + p.Workflow, + p.IncludePatterns, + p.ExcludePatterns, + } + result[i] = item + } + return result +} + +func copyEnvVars(envVars *EnvVarsYaml) *EnvVarsConfig { + result := EnvVarsConfig{} + result.State = make([]EnvVarConfigConfig, len(envVars.State)) + result.Commands = make([]EnvVarConfigConfig, len(envVars.Commands)) + + for i, s := range envVars.State { + item := EnvVarConfigConfig{ + s.Name, + s.ValueFrom, + s.Value, + } + result.State[i] = item + } + for i, s := range envVars.Commands { + item := EnvVarConfigConfig{ + s.Name, + s.ValueFrom, + s.Value, + } + result.Commands[i] = item + } + + return &result +} + +func copyStage(stage *StageYaml) *StageConfig { + result := StageConfig{} + result.Steps = make([]StepConfig, len(stage.Steps)) + + for i, s := range stage.Steps { + item := StepConfig{ + s.Action, + s.Value, + s.ExtraArgs, + s.Shell, + } + result.Steps[i] = item + } + return &result +} + +func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfigurationConfig { + result := WorkflowConfigurationConfig{} + result.OnPullRequestClosed = make([]string, len(config.OnPullRequestClosed)) + result.OnPullRequestPushed = make([]string, len(config.OnPullRequestPushed)) + result.OnCommitToDefault = make([]string, len(config.OnCommitToDefault)) + + result.OnPullRequestClosed = config.OnPullRequestClosed + result.OnPullRequestPushed = config.OnPullRequestPushed + result.OnCommitToDefault = config.OnCommitToDefault + return &result +} + +func copyWorkflows(workflows map[string]WorkflowYaml) map[string]WorkflowConfig { + result := make(map[string]WorkflowConfig, len(workflows)) + for i, w := range workflows { + envVars := copyEnvVars(w.EnvVars) + plan := copyStage(w.Plan) + apply := copyStage(w.Apply) + configuration := copyWorkflowConfiguration(w.Configuration) + item := WorkflowConfig{ + envVars, + plan, + apply, + configuration, + } + copier.Copy(&w, &item) + result[i] = item + } + return result +} + func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) { var diggerConfig DiggerConfig const defaultWorkflowName = "default" @@ -233,14 +367,17 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, // if workflow block is not specified in yaml we create a default one, and add it to every project if diggerYaml.Workflows != nil { - diggerConfig.Workflows = diggerYaml.Workflows + workflows := copyWorkflows(diggerYaml.Workflows) + diggerConfig.Workflows = workflows + } else { workflow := *defaultWorkflow() - diggerConfig.Workflows = make(map[string]Workflow) + diggerConfig.Workflows = make(map[string]WorkflowConfig) diggerConfig.Workflows[defaultWorkflowName] = workflow } - diggerConfig.Projects = diggerYaml.Projects + projects := copyProjects(diggerYaml.Projects) + diggerConfig.Projects = projects // update project's workflow if needed for _, project := range diggerConfig.Projects { @@ -280,7 +417,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } if includeMatch && !excludeMatch { // generate a new project using default workflow - project := Project{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} + project := ProjectConfig{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} diggerConfig.Projects = append(diggerConfig.Projects, project) } } @@ -289,7 +426,8 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error) { - config := &DiggerConfigYaml{} + configYaml := &DiggerConfigYaml{} + config := &DiggerConfig{} fileName, err := retrieveConfigFile(workingDir) if err != nil { if errors.Is(err, ErrDiggerConfigConflict) { @@ -299,39 +437,11 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error if fileName == "" { fmt.Println("No digger config found, using default one") - config.Projects = make([]Project, 1) + config.Projects = make([]ProjectConfig, 1) config.Projects[0] = defaultProject() - config.Workflows = make(map[string]Workflow) - config.Workflows["default"] = Workflow{ - Plan: &Stage{ - Steps: []Step{{ - Action: "init", - ExtraArgs: []string{}, - }, { - Action: "plan", - ExtraArgs: []string{}, - }}, - }, - Apply: &Stage{ - Steps: []Step{{ - Action: "init", - ExtraArgs: []string{}, - }, { - Action: "apply", - ExtraArgs: []string{}, - }}, - }, - Configuration: &WorkflowConfiguration{ - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - OnCommitToDefault: []string{"digger apply"}, - }, - } - c, err := ConvertDiggerYamlToConfig(config, workingDir, walker) - if err != nil { - return nil, fmt.Errorf("failed to read config file %s: %v", fileName, err) - } - return c, nil + config.Workflows = make(map[string]WorkflowConfig) + config.Workflows["default"] = *defaultWorkflow() + return config, nil } data, err := os.ReadFile(fileName) @@ -339,15 +449,15 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error return nil, fmt.Errorf("failed to read config file %s: %v", fileName, err) } - if err := yaml.Unmarshal(data, config); err != nil { + if err := yaml.Unmarshal(data, configYaml); err != nil { return nil, fmt.Errorf("error parsing '%s': %v", fileName, err) } - if (config.Projects == nil || len(config.Projects) == 0) && config.GenerateProjectsConfig == nil { + if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil { return nil, fmt.Errorf("no projects configuration found in '%s'", fileName) } - c, err := ConvertDiggerYamlToConfig(config, workingDir, walker) + c, err := ConvertDiggerYamlToConfig(configYaml, workingDir, walker) if err != nil { return nil, err } @@ -361,8 +471,8 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error return c, nil } -func defaultProject() Project { - return Project{ +func defaultProject() ProjectConfig { + return ProjectConfig{ Name: "default", Dir: ".", Workspace: "default", @@ -371,7 +481,7 @@ func defaultProject() Project { } } -func (c *DiggerConfig) GetProject(projectName string) *Project { +func (c *DiggerConfig) GetProject(projectName string) *ProjectConfig { for _, project := range c.Projects { if projectName == project.Name { return &project @@ -380,7 +490,7 @@ func (c *DiggerConfig) GetProject(projectName string) *Project { return nil } -func (c *DiggerConfig) GetProjects(projectName string) []Project { +func (c *DiggerConfig) GetProjects(projectName string) []ProjectConfig { if projectName == "" { return c.Projects } @@ -388,11 +498,11 @@ func (c *DiggerConfig) GetProjects(projectName string) []Project { if project == nil { return nil } - return []Project{*project} + return []ProjectConfig{*project} } -func (c *DiggerConfig) GetModifiedProjects(changedFiles []string) []Project { - var result []Project +func (c *DiggerConfig) GetModifiedProjects(changedFiles []string) []ProjectConfig { + var result []ProjectConfig for _, project := range c.Projects { for _, changedFile := range changedFiles { // we append ** to make our directory a globable pattern @@ -418,7 +528,7 @@ func (c *DiggerConfig) GetDirectory(projectName string) string { return project.Dir } -func (c *DiggerConfig) GetWorkflow(workflowName string) *Workflow { +func (c *DiggerConfig) GetWorkflow(workflowName string) *WorkflowConfig { workflows := c.Workflows workflow, ok := workflows[workflowName] @@ -469,7 +579,7 @@ func retrieveConfigFile(workingDir string) (string, error) { return "", nil } -func CollectEnvVars(envs EnvVars) (map[string]string, map[string]string) { +func CollectEnvVars(envs *EnvVarsConfig) (map[string]string, map[string]string) { stateEnvVars := map[string]string{} for _, envvar := range envs.State { diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 9dd42af67..29ba56c11 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -134,7 +134,7 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, Step{Action: "run", Value: "echo \"hello\"", Shell: ""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, StepConfig{Action: "run", Value: "echo \"hello\"", Shell: ""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct") } func TestEnvVarsConfiguration(t *testing.T) { @@ -179,10 +179,10 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, []EnvVarConfig{ + assert.Equal(t, []EnvVarConfigConfig{ {Name: "TF_VAR_state", Value: "s3://mybucket/terraform.tfstate"}, }, dg.Workflows["myworkflow"].EnvVars.State, "parsed struct does not match expected struct") - assert.Equal(t, []EnvVarConfig{ + assert.Equal(t, []EnvVarConfigConfig{ {Name: "TF_VAR_command", Value: "plan"}, }, dg.Workflows["myworkflow"].EnvVars.Commands, "parsed struct does not match expected struct") } @@ -209,9 +209,9 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=terraform.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "run", Value: "echo \"hello\"", Shell: "zsh"}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") + assert.Equal(t, StepConfig{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, StepConfig{Action: "plan", ExtraArgs: []string{"-var-file=terraform.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") + assert.Equal(t, StepConfig{Action: "run", Value: "echo \"hello\"", Shell: "zsh"}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") } func TestDiggerGenerateProjects(t *testing.T) { diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go index 75b59366d..5ddbededb 100644 --- a/pkg/digger/digger.go +++ b/pkg/digger/digger.go @@ -160,8 +160,8 @@ type DiggerExecutor struct { ProjectPath string StateEnvVars map[string]string CommandEnvVars map[string]string - ApplyStage *configuration.Stage - PlanStage *configuration.Stage + ApplyStage *configuration.StageConfig + PlanStage *configuration.StageConfig CommandRunner CommandRun TerraformExecutor terraform.TerraformExecutor Reporter reporting.Reporter @@ -233,12 +233,13 @@ func (d DiggerExecutor) Plan() (bool, error) { } log.Printf("Lock result: %t\n", locked) if locked { - var planSteps []configuration.Step + var planSteps []configuration.StepConfig + // move default values to config init if d.PlanStage != nil { planSteps = d.PlanStage.Steps } else { - planSteps = []configuration.Step{ + planSteps = []configuration.StepConfig{ { Action: "init", }, @@ -321,12 +322,13 @@ func (d DiggerExecutor) Apply() (bool, error) { } if locked { - var applySteps []configuration.Step + var applySteps []configuration.StepConfig + // move default values to config init if d.ApplyStage != nil { applySteps = d.ApplyStage.Steps } else { - applySteps = []configuration.Step{ + applySteps = []configuration.StepConfig{ { Action: "init", }, diff --git a/pkg/digger/digger_test.go b/pkg/digger/digger_test.go index f7b86e24f..6a4109ded 100644 --- a/pkg/digger/digger_test.go +++ b/pkg/digger/digger_test.go @@ -163,8 +163,8 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) { PrNumber: 1, } executor := DiggerExecutor{ - ApplyStage: &configuration.Stage{ - Steps: []configuration.Step{ + ApplyStage: &configuration.StageConfig{ + Steps: []configuration.StepConfig{ { Action: "init", ExtraArgs: nil, @@ -182,7 +182,7 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) { }, }, }, - PlanStage: &configuration.Stage{}, + PlanStage: &configuration.StageConfig{}, CommandRunner: commandRunner, TerraformExecutor: terraformExecutor, Reporter: reporter, @@ -208,9 +208,9 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) { PrNumber: 1, } executor := DiggerExecutor{ - ApplyStage: &configuration.Stage{}, - PlanStage: &configuration.Stage{ - Steps: []configuration.Step{ + ApplyStage: &configuration.StageConfig{}, + PlanStage: &configuration.StageConfig{ + Steps: []configuration.StepConfig{ { Action: "init", ExtraArgs: nil, diff --git a/pkg/github/github.go b/pkg/github/github.go index cffcd991c..6ce4d3b63 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -132,7 +132,7 @@ func GetGitHubContext(ghContext string) (*models.Github, error) { return parsedGhContext, nil } -func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]dg_models.ProjectCommand, bool, error) { +func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]dg_models.ProjectCommand, bool, error) { commandsPerProject := make([]dg_models.ProjectCommand, 0) switch event.(type) { @@ -196,7 +196,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.Project{*requestedProject} + runForProjects = []configuration.ProjectConfig{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } @@ -240,8 +240,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } } -func ProcessGitHubEvent(ghEvent models.Event, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.Project, *configuration.Project, int, error) { - var impactedProjects []configuration.Project +func ProcessGitHubEvent(ghEvent models.Event, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, int, error) { + var impactedProjects []configuration.ProjectConfig var prNumber int switch ghEvent.(type) { diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index b204c81b2..d482ae0d0 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -83,8 +83,8 @@ func NewGitLabService(token string, gitLabContext *GitLabContext) (*GitLabServic }, nil } -func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.Project, *configuration.Project, error) { - var impactedProjects []configuration.Project +func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, error) { + var impactedProjects []configuration.ProjectConfig if gitlabContext.MergeRequestIId == nil { return nil, nil, fmt.Errorf("value for 'Merge Request ID' parameter is not found") @@ -249,7 +249,7 @@ const ( MergeRequestComment = GitLabEventType("merge_request_commented") ) -func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, bool, error) { +func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) fmt.Printf("ConvertGitLabEventToCommands, event.EventType: %s\n", event.EventType) @@ -305,7 +305,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.Project{*requestedProject} + runForProjects = []configuration.ProjectConfig{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } diff --git a/pkg/models/models.go b/pkg/models/models.go index 19f5956ba..df526be3a 100644 --- a/pkg/models/models.go +++ b/pkg/models/models.go @@ -8,8 +8,8 @@ type ProjectCommand struct { ProjectWorkspace string Terragrunt bool Commands []string - ApplyStage *configuration.Stage - PlanStage *configuration.Stage + ApplyStage *configuration.StageConfig + PlanStage *configuration.StageConfig StateEnvVars map[string]string CommandEnvVars map[string]string } From ef9f3124da615d3aa66deb2210664037aafc605a Mon Sep 17 00:00:00 2001 From: alexey Date: Thu, 8 Jun 2023 16:14:04 +0100 Subject: [PATCH 02/20] more refactorings --- pkg/azure/azure.go | 8 +- pkg/configuration/config.go | 74 +++++ pkg/configuration/converters.go | 166 ++++++++++ pkg/configuration/digger_config.go | 402 +----------------------- pkg/configuration/digger_config_test.go | 4 +- pkg/configuration/yaml.go | 174 ++++++++++ pkg/github/github.go | 4 +- pkg/gitlab/gitlab.go | 6 +- 8 files changed, 426 insertions(+), 412 deletions(-) create mode 100644 pkg/configuration/config.go create mode 100644 pkg/configuration/converters.go create mode 100644 pkg/configuration/yaml.go diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 958b06f5e..8e5820c25 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -350,7 +350,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -371,7 +371,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -392,7 +392,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if !ok { return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -441,7 +441,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if !ok { return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go new file mode 100644 index 000000000..2ed0d0d46 --- /dev/null +++ b/pkg/configuration/config.go @@ -0,0 +1,74 @@ +package configuration + +import "digger/pkg/core/models" + +type DiggerConfig struct { + Projects []ProjectConfig + AutoMerge bool + CollectUsageData bool + Workflows map[string]WorkflowConfig +} + +type ProjectConfig struct { + Name string + Dir string + Workspace string + Terragrunt bool + Workflow string + IncludePatterns []string + ExcludePatterns []string +} + +type WorkflowConfig struct { + EnvVars *TerraformEnvConfig + Plan *models.Stage + Apply *models.Stage + Configuration *WorkflowConfigurationConfig +} + +type WorkflowConfigurationConfig struct { + OnPullRequestPushed []string + OnPullRequestClosed []string + OnCommitToDefault []string +} + +type TerraformEnvConfig struct { + State []EnvVar + Commands []EnvVar +} + +type EnvVar struct { + Name string + ValueFrom string + Value string +} + +func defaultWorkflow() *WorkflowConfig { + return &WorkflowConfig{ + Configuration: &WorkflowConfigurationConfig{ + OnCommitToDefault: []string{"digger unlock"}, + OnPullRequestPushed: []string{"digger plan"}, + OnPullRequestClosed: []string{"digger unlock"}, + }, + Plan: &models.Stage{ + Steps: []models.Step{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "plan", ExtraArgs: []string{}, + }, + }, + }, + Apply: &models.Stage{ + Steps: []models.Step{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "apply", ExtraArgs: []string{}, + }, + }, + }, + } +} diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go new file mode 100644 index 000000000..ae30469af --- /dev/null +++ b/pkg/configuration/converters.go @@ -0,0 +1,166 @@ +package configuration + +import ( + "digger/pkg/core/models" + "fmt" + "github.com/bmatcuk/doublestar/v4" + "github.com/jinzhu/copier" + "path/filepath" +) + +func copyProjects(projects []ProjectYaml) []ProjectConfig { + result := make([]ProjectConfig, len(projects)) + for i, p := range projects { + item := ProjectConfig{p.Name, + p.Dir, + p.Workspace, + p.Terragrunt, + p.Workflow, + p.IncludePatterns, + p.ExcludePatterns, + } + result[i] = item + } + return result +} + +func copyTerraformEnvConfig(envVars *TerraformEnvConfigYaml) *TerraformEnvConfig { + result := TerraformEnvConfig{} + result.State = make([]EnvVar, len(envVars.State)) + result.Commands = make([]EnvVar, len(envVars.Commands)) + + for i, s := range envVars.State { + item := EnvVar{ + s.Name, + s.ValueFrom, + s.Value, + } + result.State[i] = item + } + for i, s := range envVars.Commands { + item := EnvVar{ + s.Name, + s.ValueFrom, + s.Value, + } + result.Commands[i] = item + } + + return &result +} + +func copyStage(stage *StageYaml) *models.Stage { + result := models.Stage{} + result.Steps = make([]models.Step, len(stage.Steps)) + + for i, s := range stage.Steps { + item := models.Step{ + Action: s.Action, + Value: s.Value, + ExtraArgs: s.ExtraArgs, + Shell: s.Shell, + } + result.Steps[i] = item + } + return &result +} + +func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfigurationConfig { + result := WorkflowConfigurationConfig{} + result.OnPullRequestClosed = make([]string, len(config.OnPullRequestClosed)) + result.OnPullRequestPushed = make([]string, len(config.OnPullRequestPushed)) + result.OnCommitToDefault = make([]string, len(config.OnCommitToDefault)) + + result.OnPullRequestClosed = config.OnPullRequestClosed + result.OnPullRequestPushed = config.OnPullRequestPushed + result.OnCommitToDefault = config.OnCommitToDefault + return &result +} + +func copyWorkflows(workflows map[string]WorkflowYaml) map[string]WorkflowConfig { + result := make(map[string]WorkflowConfig, len(workflows)) + for i, w := range workflows { + envVars := copyTerraformEnvConfig(w.EnvVars) + plan := copyStage(w.Plan) + apply := copyStage(w.Apply) + configuration := copyWorkflowConfiguration(w.Configuration) + item := WorkflowConfig{ + envVars, + plan, + apply, + configuration, + } + copier.Copy(&w, &item) + result[i] = item + } + return result +} + +func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) { + var diggerConfig DiggerConfig + const defaultWorkflowName = "default" + + diggerConfig.AutoMerge = diggerYaml.AutoMerge + + // if workflow block is not specified in yaml we create a default one, and add it to every project + if diggerYaml.Workflows != nil { + workflows := copyWorkflows(diggerYaml.Workflows) + diggerConfig.Workflows = workflows + + } else { + workflow := *defaultWorkflow() + diggerConfig.Workflows = make(map[string]WorkflowConfig) + diggerConfig.Workflows[defaultWorkflowName] = workflow + } + + projects := copyProjects(diggerYaml.Projects) + diggerConfig.Projects = projects + + // update project's workflow if needed + for _, project := range diggerConfig.Projects { + if project.Workflow == "" { + project.Workflow = defaultWorkflowName + } + } + + // check for project name duplicates + projectNames := make(map[string]bool) + for _, project := range diggerConfig.Projects { + if projectNames[project.Name] { + return nil, fmt.Errorf("project name '%s' is duplicated", project.Name) + } + projectNames[project.Name] = true + } + if diggerYaml.CollectUsageData != nil { + diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData + } else { + diggerConfig.CollectUsageData = true + } + + if diggerYaml.GenerateProjectsConfig != nil { + dirs, err := walker.GetDirs(workingDir) + if err != nil { + return nil, err + } + + for _, dir := range dirs { + includePattern := diggerYaml.GenerateProjectsConfig.Include + excludePattern := diggerYaml.GenerateProjectsConfig.Exclude + includeMatch, err := doublestar.PathMatch(includePattern, dir) + if err != nil { + return nil, err + } + + excludeMatch, err := doublestar.PathMatch(excludePattern, dir) + if err != nil { + return nil, err + } + if includeMatch && !excludeMatch { + // generate a new project using default workflow + project := ProjectConfig{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} + diggerConfig.Projects = append(diggerConfig.Projects, project) + } + } + } + return &diggerConfig, nil +} diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index c1a66f6c3..7bf2c21b1 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -1,125 +1,15 @@ package configuration import ( - "digger/pkg/core/models" "digger/pkg/utils" "errors" "fmt" - "github.com/bmatcuk/doublestar/v4" - "github.com/jinzhu/copier" "gopkg.in/yaml.v3" "os" "path" "path/filepath" ) -type DiggerConfigYaml struct { - Projects []ProjectYaml `yaml:"projects"` - AutoMerge bool `yaml:"auto_merge"` - Workflows map[string]WorkflowYaml `yaml:"workflows"` - CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` - GenerateProjectsConfig *GenerateProjectsConfigYaml `yaml:"generate_projects"` -} - -type ProjectYaml struct { - Name string `yaml:"name"` - Dir string `yaml:"dir"` - Workspace string `yaml:"workspace"` - Terragrunt bool `yaml:"terragrunt"` - Workflow string `yaml:"workflow"` - IncludePatterns []string `yaml:"include_patterns,omitempty"` - ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` -} - -type WorkflowYaml struct { - EnvVars *EnvVarsYaml `yaml:"env_vars"` - Plan *StageYaml `yaml:"plan,omitempty"` - Apply *StageYaml `yaml:"apply,omitempty"` - Configuration *WorkflowConfigurationYaml `yaml:"workflow_configuration"` -} - -type WorkflowConfigurationYaml struct { - OnPullRequestPushed []string `yaml:"on_pull_request_pushed"` - OnPullRequestClosed []string `yaml:"on_pull_request_closed"` - OnCommitToDefault []string `yaml:"on_commit_to_default"` -} - -func (s *StageYaml) ToCoreStage() models.Stage { - var steps []models.Step - for _, step := range s.Steps { - steps = append(steps, step.ToCoreStep()) - } - return models.Stage{Steps: steps} -} - -type StageYaml struct { - Steps []StepYaml `yaml:"steps"` -} - -type StepYaml struct { - Action string - Value string - ExtraArgs []string `yaml:"extra_args,omitempty"` - Shell string -} - -type EnvVarsYaml struct { - State []EnvVarConfigYaml `yaml:"state"` - Commands []EnvVarConfigYaml `yaml:"commands"` -} - -type EnvVarConfigYaml struct { - Name string `yaml:"name"` - ValueFrom string `yaml:"value_from"` - Value string `yaml:"value"` -} - -type GenerateProjectsConfigYaml struct { - Include string `yaml:"include"` - Exclude string `yaml:"exclude"` -} - -type DiggerConfig struct { - Projects []ProjectConfig - AutoMerge bool - CollectUsageData bool - Workflows map[string]WorkflowConfig -} - -type ProjectConfig struct { - Name string - Dir string - Workspace string - Terragrunt bool - Workflow string - IncludePatterns []string - ExcludePatterns []string -} - -type WorkflowConfig struct { - EnvVars *EnvVarsConfig - Plan *models.Stage - Apply *models.Stage - Configuration *WorkflowConfigurationConfig -} - -type WorkflowConfigurationConfig struct { - OnPullRequestPushed []string - OnPullRequestClosed []string - OnCommitToDefault []string -} - -type EnvVarsConfig struct { - State []EnvVarConfigConfig - Commands []EnvVarConfigConfig -} - -type EnvVarConfigConfig struct { - Name string - ValueFrom string - Value string -} - type DirWalker interface { GetDirs(workingDir string) ([]string, error) } @@ -147,295 +37,6 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") -func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawProject ProjectYaml - raw := rawProject{ - Workspace: "default", - Terragrunt: false, - Workflow: "default", - } - if err := unmarshal(&raw); err != nil { - return err - } - *p = ProjectYaml(raw) - return nil -} - -func (w *WorkflowYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawWorkflow WorkflowYaml - raw := rawWorkflow{ - Configuration: &WorkflowConfigurationYaml{ - OnCommitToDefault: []string{"digger unlock"}, - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - }, - Plan: &StageYaml{ - Steps: []StepYaml{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "plan", ExtraArgs: []string{}, - }, - }, - }, - Apply: &StageYaml{ - Steps: []StepYaml{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "apply", ExtraArgs: []string{}, - }, - }, - }, - EnvVars: &EnvVarsYaml{ - State: []EnvVarConfigYaml{}, - Commands: []EnvVarConfigYaml{}, - }, - } - if err := unmarshal(&raw); err != nil { - return err - } - *w = WorkflowYaml(raw) - return nil -} - -func (s *StepYaml) UnmarshalYAML(value *yaml.Node) error { - - if value.Kind == yaml.ScalarNode { - return value.Decode(&s.Action) - } - - var stepMap map[string]interface{} - if err := value.Decode(&stepMap); err != nil { - return err - } - - if _, ok := stepMap["run"]; ok { - s.Action = "run" - s.Value = stepMap["run"].(string) - if _, ok := stepMap["shell"]; ok { - s.Shell = stepMap["shell"].(string) - } - return nil - } - - s.extract(stepMap, "plan") - s.extract(stepMap, "apply") - - return nil -} - -func (s *StepYaml) ToCoreStep() models.Step { - return models.Step{ - Action: s.Action, - Value: s.Value, - ExtraArgs: s.ExtraArgs, - Shell: s.Shell, - } -} - -func (s *StepYaml) extract(stepMap map[string]interface{}, action string) { - if _, ok := stepMap[action]; ok { - s.Action = action - var extraArgs []string - if v, ok := stepMap["extra_args"]; ok { - for _, v := range v.([]interface{}) { - extraArgs = append(extraArgs, v.(string)) - } - s.ExtraArgs = extraArgs - } - } -} - -func defaultWorkflow() *WorkflowConfig { - return &WorkflowConfig{ - Configuration: &WorkflowConfigurationConfig{ - OnCommitToDefault: []string{"digger unlock"}, - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - }, - Plan: &models.Stage{ - Steps: []models.Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "plan", ExtraArgs: []string{}, - }, - }, - }, - Apply: &models.Stage{ - Steps: []models.Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "apply", ExtraArgs: []string{}, - }, - }, - }, - } -} - -func copyProjects(projects []ProjectYaml) []ProjectConfig { - result := make([]ProjectConfig, len(projects)) - for i, p := range projects { - item := ProjectConfig{p.Name, - p.Dir, - p.Workspace, - p.Terragrunt, - p.Workflow, - p.IncludePatterns, - p.ExcludePatterns, - } - result[i] = item - } - return result -} - -func copyEnvVars(envVars *EnvVarsYaml) *EnvVarsConfig { - result := EnvVarsConfig{} - result.State = make([]EnvVarConfigConfig, len(envVars.State)) - result.Commands = make([]EnvVarConfigConfig, len(envVars.Commands)) - - for i, s := range envVars.State { - item := EnvVarConfigConfig{ - s.Name, - s.ValueFrom, - s.Value, - } - result.State[i] = item - } - for i, s := range envVars.Commands { - item := EnvVarConfigConfig{ - s.Name, - s.ValueFrom, - s.Value, - } - result.Commands[i] = item - } - - return &result -} - -func copyStage(stage *StageYaml) *models.Stage { - result := models.Stage{} - result.Steps = make([]models.Step, len(stage.Steps)) - - for i, s := range stage.Steps { - item := models.Step{ - Action: s.Action, - Value: s.Value, - ExtraArgs: s.ExtraArgs, - Shell: s.Shell, - } - result.Steps[i] = item - } - return &result -} - -func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfigurationConfig { - result := WorkflowConfigurationConfig{} - result.OnPullRequestClosed = make([]string, len(config.OnPullRequestClosed)) - result.OnPullRequestPushed = make([]string, len(config.OnPullRequestPushed)) - result.OnCommitToDefault = make([]string, len(config.OnCommitToDefault)) - - result.OnPullRequestClosed = config.OnPullRequestClosed - result.OnPullRequestPushed = config.OnPullRequestPushed - result.OnCommitToDefault = config.OnCommitToDefault - return &result -} - -func copyWorkflows(workflows map[string]WorkflowYaml) map[string]WorkflowConfig { - result := make(map[string]WorkflowConfig, len(workflows)) - for i, w := range workflows { - envVars := copyEnvVars(w.EnvVars) - plan := copyStage(w.Plan) - apply := copyStage(w.Apply) - configuration := copyWorkflowConfiguration(w.Configuration) - item := WorkflowConfig{ - envVars, - plan, - apply, - configuration, - } - copier.Copy(&w, &item) - result[i] = item - } - return result -} - -func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) { - var diggerConfig DiggerConfig - const defaultWorkflowName = "default" - - diggerConfig.AutoMerge = diggerYaml.AutoMerge - - // if workflow block is not specified in yaml we create a default one, and add it to every project - if diggerYaml.Workflows != nil { - workflows := copyWorkflows(diggerYaml.Workflows) - diggerConfig.Workflows = workflows - - } else { - workflow := *defaultWorkflow() - diggerConfig.Workflows = make(map[string]WorkflowConfig) - diggerConfig.Workflows[defaultWorkflowName] = workflow - } - - projects := copyProjects(diggerYaml.Projects) - diggerConfig.Projects = projects - - // update project's workflow if needed - for _, project := range diggerConfig.Projects { - if project.Workflow == "" { - project.Workflow = defaultWorkflowName - } - } - - // check for project name duplicates - projectNames := make(map[string]bool) - for _, project := range diggerConfig.Projects { - if projectNames[project.Name] { - return nil, fmt.Errorf("project name '%s' is duplicated", project.Name) - } - projectNames[project.Name] = true - } - if diggerYaml.CollectUsageData != nil { - diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData - } else { - diggerConfig.CollectUsageData = true - } - - if diggerYaml.GenerateProjectsConfig != nil { - dirs, err := walker.GetDirs(workingDir) - if err != nil { - return nil, err - } - - for _, dir := range dirs { - includePattern := diggerYaml.GenerateProjectsConfig.Include - excludePattern := diggerYaml.GenerateProjectsConfig.Exclude - includeMatch, err := doublestar.PathMatch(includePattern, dir) - if err != nil { - return nil, err - } - - excludeMatch, err := doublestar.PathMatch(excludePattern, dir) - if err != nil { - return nil, err - } - if includeMatch && !excludeMatch { - // generate a new project using default workflow - project := ProjectConfig{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} - diggerConfig.Projects = append(diggerConfig.Projects, project) - } - } - } - return &diggerConfig, nil -} - func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error) { configYaml := &DiggerConfigYaml{} config := &DiggerConfig{} @@ -590,9 +191,8 @@ func retrieveConfigFile(workingDir string) (string, error) { return "", nil } -func CollectEnvVars(envs *EnvVarsConfig) (map[string]string, map[string]string) { +func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map[string]string) { stateEnvVars := map[string]string{} - commandEnvVars := map[string]string{} if envs != nil { diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 866f7b1e3..679a02934 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -180,10 +180,10 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, []EnvVarConfigConfig{ + assert.Equal(t, []EnvVar{ {Name: "TF_VAR_state", Value: "s3://mybucket/terraform.tfstate"}, }, dg.Workflows["myworkflow"].EnvVars.State, "parsed struct does not match expected struct") - assert.Equal(t, []EnvVarConfigConfig{ + assert.Equal(t, []EnvVar{ {Name: "TF_VAR_command", Value: "plan"}, }, dg.Workflows["myworkflow"].EnvVars.Commands, "parsed struct does not match expected struct") } diff --git a/pkg/configuration/yaml.go b/pkg/configuration/yaml.go new file mode 100644 index 000000000..e76070f56 --- /dev/null +++ b/pkg/configuration/yaml.go @@ -0,0 +1,174 @@ +package configuration + +import ( + "digger/pkg/core/models" + "gopkg.in/yaml.v3" +) + +type DiggerConfigYaml struct { + Projects []ProjectYaml `yaml:"projects"` + AutoMerge bool `yaml:"auto_merge"` + Workflows map[string]WorkflowYaml `yaml:"workflows"` + CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` + GenerateProjectsConfig *GenerateProjectsConfigYaml `yaml:"generate_projects"` +} + +type ProjectYaml struct { + Name string `yaml:"name"` + Dir string `yaml:"dir"` + Workspace string `yaml:"workspace"` + Terragrunt bool `yaml:"terragrunt"` + Workflow string `yaml:"workflow"` + IncludePatterns []string `yaml:"include_patterns,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` +} + +type WorkflowYaml struct { + EnvVars *TerraformEnvConfigYaml `yaml:"env_vars"` + Plan *StageYaml `yaml:"plan,omitempty"` + Apply *StageYaml `yaml:"apply,omitempty"` + Configuration *WorkflowConfigurationYaml `yaml:"workflow_configuration"` +} + +type WorkflowConfigurationYaml struct { + OnPullRequestPushed []string `yaml:"on_pull_request_pushed"` + OnPullRequestClosed []string `yaml:"on_pull_request_closed"` + OnCommitToDefault []string `yaml:"on_commit_to_default"` +} + +func (s *StageYaml) ToCoreStage() models.Stage { + var steps []models.Step + for _, step := range s.Steps { + steps = append(steps, step.ToCoreStep()) + } + return models.Stage{Steps: steps} +} + +type StageYaml struct { + Steps []StepYaml `yaml:"steps"` +} + +type StepYaml struct { + Action string + Value string + ExtraArgs []string `yaml:"extra_args,omitempty"` + Shell string +} + +type TerraformEnvConfigYaml struct { + State []EnvVarYaml `yaml:"state"` + Commands []EnvVarYaml `yaml:"commands"` +} + +type EnvVarYaml struct { + Name string `yaml:"name"` + ValueFrom string `yaml:"value_from"` + Value string `yaml:"value"` +} + +type GenerateProjectsConfigYaml struct { + Include string `yaml:"include"` + Exclude string `yaml:"exclude"` +} + +func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawProject ProjectYaml + raw := rawProject{ + Workspace: "default", + Terragrunt: false, + Workflow: "default", + } + if err := unmarshal(&raw); err != nil { + return err + } + *p = ProjectYaml(raw) + return nil +} + +func (w *WorkflowYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawWorkflow WorkflowYaml + raw := rawWorkflow{ + Configuration: &WorkflowConfigurationYaml{ + OnCommitToDefault: []string{"digger unlock"}, + OnPullRequestPushed: []string{"digger plan"}, + OnPullRequestClosed: []string{"digger unlock"}, + }, + Plan: &StageYaml{ + Steps: []StepYaml{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "plan", ExtraArgs: []string{}, + }, + }, + }, + Apply: &StageYaml{ + Steps: []StepYaml{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "apply", ExtraArgs: []string{}, + }, + }, + }, + EnvVars: &TerraformEnvConfigYaml{ + State: []EnvVarYaml{}, + Commands: []EnvVarYaml{}, + }, + } + if err := unmarshal(&raw); err != nil { + return err + } + *w = WorkflowYaml(raw) + return nil +} + +func (s *StepYaml) UnmarshalYAML(value *yaml.Node) error { + + if value.Kind == yaml.ScalarNode { + return value.Decode(&s.Action) + } + + var stepMap map[string]interface{} + if err := value.Decode(&stepMap); err != nil { + return err + } + + if _, ok := stepMap["run"]; ok { + s.Action = "run" + s.Value = stepMap["run"].(string) + if _, ok := stepMap["shell"]; ok { + s.Shell = stepMap["shell"].(string) + } + return nil + } + + s.extract(stepMap, "plan") + s.extract(stepMap, "apply") + + return nil +} + +func (s *StepYaml) ToCoreStep() models.Step { + return models.Step{ + Action: s.Action, + Value: s.Value, + ExtraArgs: s.ExtraArgs, + Shell: s.Shell, + } +} + +func (s *StepYaml) extract(stepMap map[string]interface{}, action string) { + if _, ok := stepMap[action]; ok { + s.Action = action + var extraArgs []string + if v, ok := stepMap["extra_args"]; ok { + for _, v := range v.([]interface{}) { + extraArgs = append(extraArgs, v.(string)) + } + s.ExtraArgs = extraArgs + } + } +} diff --git a/pkg/github/github.go b/pkg/github/github.go index 196dad62a..9e7f368f5 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -144,7 +144,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { commandsPerProject = append(commandsPerProject, dg_models.ProjectCommand{ @@ -210,7 +210,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) workspace := project.Workspace workspaceOverride, err := utils.ParseWorkspace(event.Comment.Body) diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index 79a7c9512..7bc3c345d 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -261,7 +261,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -281,7 +281,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if !ok { return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -326,7 +326,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if workspaceOverride != "" { workspace = workspaceOverride } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, From 89b3e25e25c8382c44d6c79bd938d3ba40e30a2b Mon Sep 17 00:00:00 2001 From: alexey Date: Thu, 8 Jun 2023 16:22:51 +0100 Subject: [PATCH 03/20] merge develop branch --- pkg/configuration/converters.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index ae30469af..85df91151 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -107,6 +107,11 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, workflows := copyWorkflows(diggerYaml.Workflows) diggerConfig.Workflows = workflows + // provide default workflow if not specified + if _, ok := diggerConfig.Workflows[defaultWorkflowName]; !ok { + workflow := *defaultWorkflow() + diggerConfig.Workflows[defaultWorkflowName] = workflow + } } else { workflow := *defaultWorkflow() diggerConfig.Workflows = make(map[string]WorkflowConfig) From f4b7ba1053617e3a2874c7d899b1cae790053b8e Mon Sep 17 00:00:00 2001 From: alexey Date: Thu, 8 Jun 2023 16:45:26 +0100 Subject: [PATCH 04/20] rename types --- cmd/digger/main_test.go | 4 ++-- pkg/configuration/config.go | 6 +++--- pkg/configuration/converters.go | 27 +++++++++++++++++---------- pkg/configuration/yaml.go | 6 +++--- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/cmd/digger/main_test.go b/cmd/digger/main_test.go index 4ba2225e4..ada7ec665 100644 --- a/cmd/digger/main_test.go +++ b/cmd/digger/main_test.go @@ -947,7 +947,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Configuration: &configuration.WorkflowConfigurationConfig{ + Configuration: &configuration.WorkflowConfiguration{ OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, OnCommitToDefault: []string{"digger apply"}, @@ -962,7 +962,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Configuration: &configuration.WorkflowConfigurationConfig{ + Configuration: &configuration.WorkflowConfiguration{ OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, OnCommitToDefault: []string{"digger apply"}, diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go index 2ed0d0d46..7abe9b005 100644 --- a/pkg/configuration/config.go +++ b/pkg/configuration/config.go @@ -23,10 +23,10 @@ type WorkflowConfig struct { EnvVars *TerraformEnvConfig Plan *models.Stage Apply *models.Stage - Configuration *WorkflowConfigurationConfig + Configuration *WorkflowConfiguration } -type WorkflowConfigurationConfig struct { +type WorkflowConfiguration struct { OnPullRequestPushed []string OnPullRequestClosed []string OnCommitToDefault []string @@ -45,7 +45,7 @@ type EnvVar struct { func defaultWorkflow() *WorkflowConfig { return &WorkflowConfig{ - Configuration: &WorkflowConfigurationConfig{ + Configuration: &WorkflowConfiguration{ OnCommitToDefault: []string{"digger unlock"}, OnPullRequestPushed: []string{"digger plan"}, OnPullRequestClosed: []string{"digger unlock"}, diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index 85df91151..6c8e8ad36 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -8,7 +8,7 @@ import ( "path/filepath" ) -func copyProjects(projects []ProjectYaml) []ProjectConfig { +func copyProjects(projects []*ProjectYaml) []ProjectConfig { result := make([]ProjectConfig, len(projects)) for i, p := range projects { item := ProjectConfig{p.Name, @@ -24,12 +24,15 @@ func copyProjects(projects []ProjectYaml) []ProjectConfig { return result } -func copyTerraformEnvConfig(envVars *TerraformEnvConfigYaml) *TerraformEnvConfig { +func copyTerraformEnvConfig(terraformEnvConfig *TerraformEnvConfigYaml) *TerraformEnvConfig { + if terraformEnvConfig == nil { + return nil + } result := TerraformEnvConfig{} - result.State = make([]EnvVar, len(envVars.State)) - result.Commands = make([]EnvVar, len(envVars.Commands)) + result.State = make([]EnvVar, len(terraformEnvConfig.State)) + result.Commands = make([]EnvVar, len(terraformEnvConfig.Commands)) - for i, s := range envVars.State { + for i, s := range terraformEnvConfig.State { item := EnvVar{ s.Name, s.ValueFrom, @@ -37,7 +40,7 @@ func copyTerraformEnvConfig(envVars *TerraformEnvConfigYaml) *TerraformEnvConfig } result.State[i] = item } - for i, s := range envVars.Commands { + for i, s := range terraformEnvConfig.Commands { item := EnvVar{ s.Name, s.ValueFrom, @@ -65,8 +68,8 @@ func copyStage(stage *StageYaml) *models.Stage { return &result } -func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfigurationConfig { - result := WorkflowConfigurationConfig{} +func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfiguration { + result := WorkflowConfiguration{} result.OnPullRequestClosed = make([]string, len(config.OnPullRequestClosed)) result.OnPullRequestPushed = make([]string, len(config.OnPullRequestPushed)) result.OnCommitToDefault = make([]string, len(config.OnCommitToDefault)) @@ -77,7 +80,7 @@ func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfi return &result } -func copyWorkflows(workflows map[string]WorkflowYaml) map[string]WorkflowConfig { +func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]WorkflowConfig { result := make(map[string]WorkflowConfig, len(workflows)) for i, w := range workflows { envVars := copyTerraformEnvConfig(w.EnvVars) @@ -100,7 +103,11 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, var diggerConfig DiggerConfig const defaultWorkflowName = "default" - diggerConfig.AutoMerge = diggerYaml.AutoMerge + if diggerYaml.AutoMerge != nil { + diggerConfig.AutoMerge = *diggerYaml.AutoMerge + } else { + diggerConfig.AutoMerge = false + } // if workflow block is not specified in yaml we create a default one, and add it to every project if diggerYaml.Workflows != nil { diff --git a/pkg/configuration/yaml.go b/pkg/configuration/yaml.go index e76070f56..415d52777 100644 --- a/pkg/configuration/yaml.go +++ b/pkg/configuration/yaml.go @@ -6,9 +6,9 @@ import ( ) type DiggerConfigYaml struct { - Projects []ProjectYaml `yaml:"projects"` - AutoMerge bool `yaml:"auto_merge"` - Workflows map[string]WorkflowYaml `yaml:"workflows"` + Projects []*ProjectYaml `yaml:"projects"` + AutoMerge *bool `yaml:"auto_merge"` + Workflows map[string]*WorkflowYaml `yaml:"workflows"` CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` GenerateProjectsConfig *GenerateProjectsConfigYaml `yaml:"generate_projects"` } From 2c60b6f10d67cf1199f84f4915e3a0d9d97bacd1 Mon Sep 17 00:00:00 2001 From: alexey Date: Thu, 8 Jun 2023 17:25:51 +0100 Subject: [PATCH 05/20] add more tests --- pkg/configuration/config.go | 1 + pkg/configuration/converters.go | 13 ++++----- pkg/configuration/digger_config_test.go | 36 +++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go index 7abe9b005..d361c017a 100644 --- a/pkg/configuration/config.go +++ b/pkg/configuration/config.go @@ -70,5 +70,6 @@ func defaultWorkflow() *WorkflowConfig { }, }, }, + EnvVars: &TerraformEnvConfig{}, } } diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index 6c8e8ad36..cb149a3e2 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -26,7 +26,7 @@ func copyProjects(projects []*ProjectYaml) []ProjectConfig { func copyTerraformEnvConfig(terraformEnvConfig *TerraformEnvConfigYaml) *TerraformEnvConfig { if terraformEnvConfig == nil { - return nil + return &TerraformEnvConfig{} } result := TerraformEnvConfig{} result.State = make([]EnvVar, len(terraformEnvConfig.State)) @@ -109,6 +109,12 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, diggerConfig.AutoMerge = false } + if diggerYaml.CollectUsageData != nil { + diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData + } else { + diggerConfig.CollectUsageData = true + } + // if workflow block is not specified in yaml we create a default one, and add it to every project if diggerYaml.Workflows != nil { workflows := copyWorkflows(diggerYaml.Workflows) @@ -143,11 +149,6 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } projectNames[project.Name] = true } - if diggerYaml.CollectUsageData != nil { - diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData - } else { - diggerConfig.CollectUsageData = true - } if diggerYaml.GenerateProjectsConfig != nil { dirs, err := walker.GetDirs(workingDir) diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 679a02934..ed246c599 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -73,6 +73,42 @@ projects: assert.Equal(t, "path/to/module/test", dg.GetDirectory("prod")) } +func TestDefaultDiggerConfig(t *testing.T) { + tempDir, teardown := setUp() + defer teardown() + + diggerCfg := ` +projects: +- name: prod + branch: /main/ + dir: path/to/module/test + workspace: default +` + deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) + defer deleteFile() + + dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) + + assert.NoError(t, err, "expected error to be nil") + assert.NotNil(t, dg, "expected digger config to be not nil") + assert.Equal(t, 1, len(dg.Projects)) + assert.Equal(t, false, dg.AutoMerge) + assert.Equal(t, true, dg.CollectUsageData) + assert.Equal(t, 1, len(dg.Workflows)) + + workflow := dg.Workflows["default"] + assert.NotNil(t, workflow, "expected workflow to be not nil") + assert.NotNil(t, workflow.Plan) + assert.NotNil(t, workflow.Plan.Steps) + + assert.NotNil(t, workflow.Apply) + assert.NotNil(t, workflow.Apply.Steps) + assert.NotNil(t, workflow.EnvVars) + assert.NotNil(t, workflow.Configuration) + + assert.Equal(t, "path/to/module/test", dg.GetDirectory("prod")) +} + func TestDiggerConfigDefaultWorkflow(t *testing.T) { tempDir, teardown := setUp() defer teardown() From 9c0b77ff1b3122ca3e80fd271cf6e25ee512a438 Mon Sep 17 00:00:00 2001 From: alexey Date: Thu, 8 Jun 2023 17:33:58 +0100 Subject: [PATCH 06/20] merge changes from develop branch --- cmd/digger/main.go | 2 +- cmd/digger/main_test.go | 6 ++-- pkg/azure/azure.go | 8 ++--- pkg/configuration/config.go | 4 +-- pkg/configuration/converters.go | 22 ++++-------- pkg/configuration/digger_config.go | 46 +++++++++++++++++++------ pkg/configuration/digger_config_test.go | 8 ++--- pkg/github/github.go | 8 ++--- pkg/gitlab/gitlab.go | 8 ++--- 9 files changed, 63 insertions(+), 49 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index 081cb48f4..cb635723b 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -369,7 +369,7 @@ func newPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, return planStorage } -func logImpactedProjects(projects []configuration.ProjectConfig, prNumber int) { +func logImpactedProjects(projects []configuration.Project, prNumber int) { logMessage := fmt.Sprintf("Following projects are impacted by pull request #%d\n", prNumber) for _, p := range projects { logMessage += fmt.Sprintf("%s\n", p.Name) diff --git a/cmd/digger/main_test.go b/cmd/digger/main_test.go index ada7ec665..3817a56ef 100644 --- a/cmd/digger/main_test.go +++ b/cmd/digger/main_test.go @@ -935,8 +935,8 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { assert.NoError(t, err) ghEvent := context.Event pullRequestNumber := 11 - dev := configuration.ProjectConfig{Name: "dev", Dir: "dev", Workflow: "dev"} - prod := configuration.ProjectConfig{Name: "prod", Dir: "prod", Workflow: "prod"} + dev := configuration.Project{Name: "dev", Dir: "dev", Workflow: "dev"} + prod := configuration.Project{Name: "prod", Dir: "prod", Workflow: "prod"} workflows := map[string]configuration.WorkflowConfig{ "dev": { Plan: &models.Stage{Steps: []models.Step{ @@ -969,7 +969,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { }, }, } - projects := []configuration.ProjectConfig{dev, prod} + projects := []configuration.Project{dev, prod} diggerConfig := configuration.DiggerConfig{Projects: projects} // PullRequestManager Mock diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 8e5820c25..6f651c6f9 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -298,8 +298,8 @@ func (a *AzureReposService) IsClosed(prNumber int) (bool, error) { return *pullRequest.Status == git.PullRequestStatusValues.Completed || *pullRequest.Status == git.PullRequestStatusValues.Abandoned, nil } -func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, int, error) { - var impactedProjects []configuration.ProjectConfig +func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.Project, *configuration.Project, int, error) { + var impactedProjects []configuration.Project var prNumber int switch azureEvent.(type) { @@ -340,7 +340,7 @@ func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration. return impactedProjects, nil, prNumber, nil } -func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { +func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) switch parseAzureContext.EventType { case AzurePrCreated, AzurePrUpdated, AzurePrReopened: @@ -419,7 +419,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.ProjectConfig{*requestedProject} + runForProjects = []configuration.Project{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go index d361c017a..e29d1ad92 100644 --- a/pkg/configuration/config.go +++ b/pkg/configuration/config.go @@ -3,13 +3,13 @@ package configuration import "digger/pkg/core/models" type DiggerConfig struct { - Projects []ProjectConfig + Projects []Project AutoMerge bool CollectUsageData bool Workflows map[string]WorkflowConfig } -type ProjectConfig struct { +type Project struct { Name string Dir string Workspace string diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index cb149a3e2..bab51c26e 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -2,16 +2,16 @@ package configuration import ( "digger/pkg/core/models" + "digger/pkg/utils" "fmt" - "github.com/bmatcuk/doublestar/v4" "github.com/jinzhu/copier" "path/filepath" ) -func copyProjects(projects []*ProjectYaml) []ProjectConfig { - result := make([]ProjectConfig, len(projects)) +func copyProjects(projects []*ProjectYaml) []Project { + result := make([]Project, len(projects)) for i, p := range projects { - item := ProjectConfig{p.Name, + item := Project{p.Name, p.Dir, p.Workspace, p.Terragrunt, @@ -159,18 +159,8 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, for _, dir := range dirs { includePattern := diggerYaml.GenerateProjectsConfig.Include excludePattern := diggerYaml.GenerateProjectsConfig.Exclude - includeMatch, err := doublestar.PathMatch(includePattern, dir) - if err != nil { - return nil, err - } - - excludeMatch, err := doublestar.PathMatch(excludePattern, dir) - if err != nil { - return nil, err - } - if includeMatch && !excludeMatch { - // generate a new project using default workflow - project := ProjectConfig{Name: filepath.Base(dir), Dir: filepath.Join(workingDir, dir), Workflow: defaultWorkflowName} + if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { + project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName, Workspace: "default"} diggerConfig.Projects = append(diggerConfig.Projects, project) } } diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 7bf2c21b1..b23e2204f 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -8,6 +8,7 @@ import ( "os" "path" "path/filepath" + "regexp" ) type DirWalker interface { @@ -17,22 +18,45 @@ type DirWalker interface { type FileSystemDirWalker struct { } -func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) { +func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { var files []string + listOfFiles, err := os.ReadDir(workingDir) + if err != nil { + return nil, errors.New(fmt.Sprintf("error reading directory %s: %v", workingDir, err)) + } + for _, f := range listOfFiles { + if !f.IsDir() { + r, err := regexp.MatchString(ext, f.Name()) + if err == nil && r { + files = append(files, f.Name()) + } + } + } + + return files, nil +} + +func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) { + var dirs []string err := filepath.Walk(workingDir, func(path string, info os.FileInfo, err error) error { + if err != nil { return err } if info.IsDir() { - files = append(files, path) + terraformFiles, _ := GetFilesWithExtension(path, ".tf") + if len(terraformFiles) > 0 { + dirs = append(dirs, path) + return filepath.SkipDir + } } return nil }) if err != nil { return nil, err } - return files, nil + return dirs, nil } var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") @@ -49,7 +73,7 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error if fileName == "" { fmt.Println("No digger config found, using default one") - config.Projects = make([]ProjectConfig, 1) + config.Projects = make([]Project, 1) config.Projects[0] = defaultProject() config.Workflows = make(map[string]WorkflowConfig) config.Workflows["default"] = *defaultWorkflow() @@ -83,8 +107,8 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error return c, nil } -func defaultProject() ProjectConfig { - return ProjectConfig{ +func defaultProject() Project { + return Project{ Name: "default", Dir: ".", Workspace: "default", @@ -93,7 +117,7 @@ func defaultProject() ProjectConfig { } } -func (c *DiggerConfig) GetProject(projectName string) *ProjectConfig { +func (c *DiggerConfig) GetProject(projectName string) *Project { for _, project := range c.Projects { if projectName == project.Name { return &project @@ -102,7 +126,7 @@ func (c *DiggerConfig) GetProject(projectName string) *ProjectConfig { return nil } -func (c *DiggerConfig) GetProjects(projectName string) []ProjectConfig { +func (c *DiggerConfig) GetProjects(projectName string) []Project { if projectName == "" { return c.Projects } @@ -110,11 +134,11 @@ func (c *DiggerConfig) GetProjects(projectName string) []ProjectConfig { if project == nil { return nil } - return []ProjectConfig{*project} + return []Project{*project} } -func (c *DiggerConfig) GetModifiedProjects(changedFiles []string) []ProjectConfig { - var result []ProjectConfig +func (c *DiggerConfig) GetModifiedProjects(changedFiles []string) []Project { + var result []Project for _, project := range c.Projects { for _, changedFile := range changedFiles { // we append ** to make our directory a globable pattern diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index ed246c599..c2fe89e61 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -275,8 +275,8 @@ generate_projects: assert.NotNil(t, dg, "expected digger config to be not nil") assert.Equal(t, "test1", dg.Projects[0].Name) assert.Equal(t, "test2", dg.Projects[1].Name) - assert.Equal(t, tempDir+"/dev/test1", dg.Projects[0].Dir) - assert.Equal(t, tempDir+"/dev/test2", dg.Projects[1].Dir) + assert.Equal(t, "dev/test1", dg.Projects[0].Dir) + assert.Equal(t, "dev/test2", dg.Projects[1].Dir) assert.Equal(t, 2, len(dg.Projects)) } @@ -305,8 +305,8 @@ generate_projects: assert.NotNil(t, dg, "expected digger config to be not nil") assert.Equal(t, "test1", dg.Projects[0].Name) assert.Equal(t, "test2", dg.Projects[1].Name) - assert.Equal(t, tempDir+"/dev/test1", dg.Projects[0].Dir) - assert.Equal(t, tempDir+"/dev/test2", dg.Projects[1].Dir) + assert.Equal(t, "dev/test1", dg.Projects[0].Dir) + assert.Equal(t, "dev/test2", dg.Projects[1].Dir) assert.Equal(t, 2, len(dg.Projects)) } diff --git a/pkg/github/github.go b/pkg/github/github.go index 9e7f368f5..c0c64e9f3 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -132,7 +132,7 @@ func GetGitHubContext(ghContext string) (*models.Github, error) { return parsedGhContext, nil } -func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]dg_models.ProjectCommand, bool, error) { +func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]dg_models.ProjectCommand, bool, error) { commandsPerProject := make([]dg_models.ProjectCommand, 0) switch event.(type) { @@ -196,7 +196,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.ProjectConfig{*requestedProject} + runForProjects = []configuration.Project{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } @@ -240,8 +240,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } } -func ProcessGitHubEvent(ghEvent models.Event, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, int, error) { - var impactedProjects []configuration.ProjectConfig +func ProcessGitHubEvent(ghEvent models.Event, diggerConfig *configuration.DiggerConfig, ciService ci.CIService) ([]configuration.Project, *configuration.Project, int, error) { + var impactedProjects []configuration.Project var prNumber int switch ghEvent.(type) { diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index 7bc3c345d..bd8f95efa 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -83,8 +83,8 @@ func NewGitLabService(token string, gitLabContext *GitLabContext) (*GitLabServic }, nil } -func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.ProjectConfig, *configuration.ProjectConfig, error) { - var impactedProjects []configuration.ProjectConfig +func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.Project, *configuration.Project, error) { + var impactedProjects []configuration.Project if gitlabContext.MergeRequestIId == nil { return nil, nil, fmt.Errorf("value for 'Merge Request ID' parameter is not found") @@ -249,7 +249,7 @@ const ( MergeRequestComment = GitLabEventType("merge_request_commented") ) -func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.ProjectConfig, requestedProject *configuration.ProjectConfig, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { +func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) fmt.Printf("ConvertGitLabEventToCommands, event.EventType: %s\n", event.EventType) @@ -305,7 +305,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if requestedProject != nil { if len(impactedProjects) > 1 { coversAllImpactedProjects = false - runForProjects = []configuration.ProjectConfig{*requestedProject} + runForProjects = []configuration.Project{*requestedProject} } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) } From f01872e6300bf79ad4c8eae0e04e07e328a5b7b5 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 10:08:41 +0100 Subject: [PATCH 07/20] pring --- pkg/core/terraform/tf.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/core/terraform/tf.go b/pkg/core/terraform/tf.go index 901763022..16156dbf9 100644 --- a/pkg/core/terraform/tf.go +++ b/pkg/core/terraform/tf.go @@ -112,6 +112,8 @@ func (tf Terraform) runTerraformCommand(command string, envs map[string]string, mwout := io.MultiWriter(os.Stdout, &stdout) mwerr := io.MultiWriter(os.Stderr, &stderr) + fmt.Printf("terraform %s", strings.Join(args, " ")) + cmd := exec.Command("terraform", args...) env := os.Environ() From b4d301dddd4d82fd5591b702a1d49d12e5ad93d5 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 10:14:24 +0100 Subject: [PATCH 08/20] pring --- pkg/github/github.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/github/github.go b/pkg/github/github.go index 0aeac2aaf..bff996884 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -145,6 +145,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + fmt.Printf("Plan stage: %v\n", workflow.Plan) coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { From e5b42d3ee18a53a57e683db0281c91cb32d03a37 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:22:40 +0100 Subject: [PATCH 09/20] test --- pkg/configuration/digger_config_test.go | 20 +++++++++++--------- pkg/core/terraform/tf.go | 2 -- pkg/github/github.go | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index d7d9f6414..267761af4 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -194,24 +194,26 @@ func TestDefaultValuesForWorkflowConfiguration(t *testing.T) { diggerCfg := ` projects: - name: dev + dir: . + workspace: dev + workflows: - default: + dev: plan: steps: - - init - - plan: - extra_args: ["-var-file=terraform.tfvars"] - - run: echo "hello" - shell: zsh + - run: rm -rf .terraform + - init + - plan: + extra_args: ["-var-file=vars/dev.tfvars"] ` deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) defer deleteFile() dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=terraform.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "run", Value: "echo \"hello\"", Shell: "zsh"}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") + assert.Equal(t, Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["dev"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["dev"].Plan.Steps[1], "parsed struct does not match expected struct") + assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["dev"].Plan.Steps[2], "parsed struct does not match expected struct") } func TestDiggerGenerateProjects(t *testing.T) { diff --git a/pkg/core/terraform/tf.go b/pkg/core/terraform/tf.go index 16156dbf9..901763022 100644 --- a/pkg/core/terraform/tf.go +++ b/pkg/core/terraform/tf.go @@ -112,8 +112,6 @@ func (tf Terraform) runTerraformCommand(command string, envs map[string]string, mwout := io.MultiWriter(os.Stdout, &stdout) mwerr := io.MultiWriter(os.Stderr, &stderr) - fmt.Printf("terraform %s", strings.Join(args, " ")) - cmd := exec.Command("terraform", args...) env := os.Environ() diff --git a/pkg/github/github.go b/pkg/github/github.go index bff996884..d4c3392c7 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -145,7 +145,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - fmt.Printf("Plan stage: %v\n", workflow.Plan) + coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { From e66772068e1e76b6c5d91410f3158cdea217ef16 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:31:13 +0100 Subject: [PATCH 10/20] test --- pkg/configuration/digger_config.go | 6 +++++ pkg/configuration/digger_config_test.go | 29 ++++++++++++++++++++----- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index f6a4d1034..9970d649c 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -233,6 +233,12 @@ func (s *Step) extract(stepMap map[string]interface{}, action string) { } s.ExtraArgs = extraArgs } + if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { + for _, v := range v.([]interface{}) { + extraArgs = append(extraArgs, v.(string)) + } + s.ExtraArgs = extraArgs + } } } diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 267761af4..7e47dad1c 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -192,19 +192,36 @@ func TestDefaultValuesForWorkflowConfiguration(t *testing.T) { defer teardown() diggerCfg := ` -projects: -- name: dev - dir: . - workspace: dev - workflows: + shared: + plan: + steps: + - run: rm -rf .terraform + - init + - plan: + extra_args: ["-var-file=vars/shared.tfvars"] + dev: plan: steps: - run: rm -rf .terraform - init - plan: - extra_args: ["-var-file=vars/dev.tfvars"] + extra_args: ["-var-file=vars/dev.tfvars"] + +projects: +- name: shared + branch: /digger-test1/ + dir: ./terraform/ + workspace: shared + terraform_version: v1.2.0 + workflow: shared +- name: dev + branch: /digger-test1/ + dir: ./terraform/ + workspace: dev + terraform_version: v1.2.0 + workflow: dev ` deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) defer deleteFile() From 1848a87b700260e632b28374f5ed5902421738f7 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:37:35 +0100 Subject: [PATCH 11/20] test --- pkg/configuration/digger_config.go | 10 +++++---- pkg/configuration/digger_config_test.go | 28 ++++++------------------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 9970d649c..8bf35bc39 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -233,11 +233,13 @@ func (s *Step) extract(stepMap map[string]interface{}, action string) { } s.ExtraArgs = extraArgs } - if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { - for _, v := range v.([]interface{}) { - extraArgs = append(extraArgs, v.(string)) + if !ok { + if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { + for _, v := range v.([]interface{}) { + extraArgs = append(extraArgs, v.(string)) + } + s.ExtraArgs = extraArgs } - s.ExtraArgs = extraArgs } } } diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 7e47dad1c..6b6c5c85a 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -192,36 +192,20 @@ func TestDefaultValuesForWorkflowConfiguration(t *testing.T) { defer teardown() diggerCfg := ` -workflows: - shared: - plan: - steps: - - run: rm -rf .terraform - - init - - plan: - extra_args: ["-var-file=vars/shared.tfvars"] +projects: +- name: dev + dir: . + workspace: dev +workflows: dev: plan: steps: - run: rm -rf .terraform - init - plan: - extra_args: ["-var-file=vars/dev.tfvars"] + extra_args: ["-var-file=vars/dev.tfvars"] -projects: -- name: shared - branch: /digger-test1/ - dir: ./terraform/ - workspace: shared - terraform_version: v1.2.0 - workflow: shared -- name: dev - branch: /digger-test1/ - dir: ./terraform/ - workspace: dev - terraform_version: v1.2.0 - workflow: dev ` deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) defer deleteFile() From a98f37d30fc80ab480aa3a9bce0d96f1002796ce Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:40:32 +0100 Subject: [PATCH 12/20] test --- pkg/github/github.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/github/github.go b/pkg/github/github.go index d4c3392c7..0062d34a7 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -148,6 +148,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() + fmt.Printf("Plan stage: %v\n", workflow.Plan) + fmt.Printf("Core plan stage: %v\n", corePlanStage) if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { commandsPerProject = append(commandsPerProject, dg_models.ProjectCommand{ ProjectName: project.Name, From 48356e138f5505632b5661cc1690eca73401da65 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:47:01 +0100 Subject: [PATCH 13/20] test --- pkg/github/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/github/github.go b/pkg/github/github.go index 0062d34a7..9e14e0039 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -148,6 +148,9 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() + fmt.Printf("project workflow: %v\n", workflows[project.Workflow]) + fmt.Printf("workflow: %v\n", workflow) + fmt.Printf("workflos: %v\n", workflows) fmt.Printf("Plan stage: %v\n", workflow.Plan) fmt.Printf("Core plan stage: %v\n", corePlanStage) if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { From 9b60cade74643aa272b614c1b6fea2c8363bbb22 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 11:50:07 +0100 Subject: [PATCH 14/20] test --- pkg/github/github.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/github/github.go b/pkg/github/github.go index 9e14e0039..dc8257678 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -148,6 +148,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() + fmt.Printf("project workflow name: %v\n", project.Workflow) fmt.Printf("project workflow: %v\n", workflows[project.Workflow]) fmt.Printf("workflow: %v\n", workflow) fmt.Printf("workflos: %v\n", workflows) From efa3347be4dfdb7b89aafb67d930739aeaf4575e Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 12:36:56 +0100 Subject: [PATCH 15/20] fix --- pkg/configuration/digger_config.go | 5 ++--- pkg/configuration/digger_config_test.go | 15 +++++++++++++-- pkg/github/github.go | 7 +------ 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 8bf35bc39..2db3f477f 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -227,13 +227,12 @@ func (s *Step) extract(stepMap map[string]interface{}, action string) { if _, ok := stepMap[action]; ok { s.Action = action var extraArgs []string - if v, ok := stepMap["extra_args"]; ok { + if v, ok2 := stepMap["extra_args"]; ok2 { for _, v := range v.([]interface{}) { extraArgs = append(extraArgs, v.(string)) } s.ExtraArgs = extraArgs - } - if !ok { + } else { if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { for _, v := range v.([]interface{}) { extraArgs = append(extraArgs, v.(string)) diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 6b6c5c85a..54fb2eada 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -195,7 +195,7 @@ func TestDefaultValuesForWorkflowConfiguration(t *testing.T) { projects: - name: dev dir: . - workspace: dev + workflow: dev workflows: dev: @@ -204,7 +204,14 @@ workflows: - run: rm -rf .terraform - init - plan: - extra_args: ["-var-file=vars/dev.tfvars"] + extra_args: ["-var-file=vars/dev.tfvars"] + default: + plan: + steps: + - run: rm -rf .terraform + - init + - plan: + extra_args: ["-var-file=vars/dev.tfvars"] ` deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) @@ -215,6 +222,10 @@ workflows: assert.Equal(t, Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["dev"].Plan.Steps[0], "parsed struct does not match expected struct") assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["dev"].Plan.Steps[1], "parsed struct does not match expected struct") assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["dev"].Plan.Steps[2], "parsed struct does not match expected struct") + + assert.Equal(t, Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") + assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") } func TestDiggerGenerateProjects(t *testing.T) { diff --git a/pkg/github/github.go b/pkg/github/github.go index dc8257678..f85e84001 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -148,12 +148,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu coreApplyStage := workflow.Apply.ToCoreStage() corePlanStage := workflow.Plan.ToCoreStage() - fmt.Printf("project workflow name: %v\n", project.Workflow) - fmt.Printf("project workflow: %v\n", workflows[project.Workflow]) - fmt.Printf("workflow: %v\n", workflow) - fmt.Printf("workflos: %v\n", workflows) - fmt.Printf("Plan stage: %v\n", workflow.Plan) - fmt.Printf("Core plan stage: %v\n", corePlanStage) + if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { commandsPerProject = append(commandsPerProject, dg_models.ProjectCommand{ ProjectName: project.Name, From 333eb367a0641fc3e2c54662968860df04e04d8f Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 12 Jun 2023 12:39:55 +0100 Subject: [PATCH 16/20] fix --- pkg/configuration/digger_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 2db3f477f..8992a5790 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -227,7 +227,7 @@ func (s *Step) extract(stepMap map[string]interface{}, action string) { if _, ok := stepMap[action]; ok { s.Action = action var extraArgs []string - if v, ok2 := stepMap["extra_args"]; ok2 { + if v, ok := stepMap["extra_args"]; ok { for _, v := range v.([]interface{}) { extraArgs = append(extraArgs, v.(string)) } From 6733254c93a2988f83347f225771544cd5037d3c Mon Sep 17 00:00:00 2001 From: alexey Date: Tue, 13 Jun 2023 12:11:35 +0100 Subject: [PATCH 17/20] fix merge conflict --- pkg/configuration/digger_config.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index a5361a00c..b23e2204f 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -36,24 +36,6 @@ func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { return files, nil } -func GetFilesWithExtension(workingDir string, ext string) ([]string, error) { - var files []string - listOfFiles, err := os.ReadDir(workingDir) - if err != nil { - return nil, errors.New(fmt.Sprintf("error reading directory %s: %v", workingDir, err)) - } - for _, f := range listOfFiles { - if !f.IsDir() { - r, err := regexp.MatchString(ext, f.Name()) - if err == nil && r { - files = append(files, f.Name()) - } - } - } - - return files, nil -} - func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) { var dirs []string err := filepath.Walk(workingDir, @@ -79,7 +61,6 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") - func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error) { configYaml := &DiggerConfigYaml{} config := &DiggerConfig{} From 4cf60ec1eccf4aedc49764b0ca0aa3818b0b78d7 Mon Sep 17 00:00:00 2001 From: alexey Date: Tue, 13 Jun 2023 12:37:30 +0100 Subject: [PATCH 18/20] rename struct --- cmd/digger/main_test.go | 2 +- pkg/azure/azure.go | 2 +- pkg/configuration/config.go | 8 ++++---- pkg/configuration/converters.go | 8 ++++---- pkg/configuration/digger_config.go | 4 ++-- pkg/github/github.go | 2 +- pkg/gitlab/gitlab.go | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/digger/main_test.go b/cmd/digger/main_test.go index 44c0a5a9f..9deb7ad9c 100644 --- a/cmd/digger/main_test.go +++ b/cmd/digger/main_test.go @@ -941,7 +941,7 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { pullRequestNumber := 11 dev := configuration.Project{Name: "dev", Dir: "dev", Workflow: "dev"} prod := configuration.Project{Name: "prod", Dir: "prod", Workflow: "prod"} - workflows := map[string]configuration.WorkflowConfig{ + workflows := map[string]configuration.Workflow{ "dev": { Plan: &models.Stage{Steps: []models.Step{ {Action: "init", ExtraArgs: []string{}}, diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 6f651c6f9..3fb5e4cb3 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -340,7 +340,7 @@ func ProcessAzureReposEvent(azureEvent interface{}, diggerConfig *configuration. return impactedProjects, nil, prNumber, nil } -func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { +func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) switch parseAzureContext.EventType { case AzurePrCreated, AzurePrUpdated, AzurePrReopened: diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go index e29d1ad92..7b90a17c1 100644 --- a/pkg/configuration/config.go +++ b/pkg/configuration/config.go @@ -6,7 +6,7 @@ type DiggerConfig struct { Projects []Project AutoMerge bool CollectUsageData bool - Workflows map[string]WorkflowConfig + Workflows map[string]Workflow } type Project struct { @@ -19,7 +19,7 @@ type Project struct { ExcludePatterns []string } -type WorkflowConfig struct { +type Workflow struct { EnvVars *TerraformEnvConfig Plan *models.Stage Apply *models.Stage @@ -43,8 +43,8 @@ type EnvVar struct { Value string } -func defaultWorkflow() *WorkflowConfig { - return &WorkflowConfig{ +func defaultWorkflow() *Workflow { + return &Workflow{ Configuration: &WorkflowConfiguration{ OnCommitToDefault: []string{"digger unlock"}, OnPullRequestPushed: []string{"digger plan"}, diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index bab51c26e..0291e99d2 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -80,14 +80,14 @@ func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfi return &result } -func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]WorkflowConfig { - result := make(map[string]WorkflowConfig, len(workflows)) +func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]Workflow { + result := make(map[string]Workflow, len(workflows)) for i, w := range workflows { envVars := copyTerraformEnvConfig(w.EnvVars) plan := copyStage(w.Plan) apply := copyStage(w.Apply) configuration := copyWorkflowConfiguration(w.Configuration) - item := WorkflowConfig{ + item := Workflow{ envVars, plan, apply, @@ -127,7 +127,7 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } } else { workflow := *defaultWorkflow() - diggerConfig.Workflows = make(map[string]WorkflowConfig) + diggerConfig.Workflows = make(map[string]Workflow) diggerConfig.Workflows[defaultWorkflowName] = workflow } diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index b23e2204f..d0c98ab4a 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -75,7 +75,7 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error fmt.Println("No digger config found, using default one") config.Projects = make([]Project, 1) config.Projects[0] = defaultProject() - config.Workflows = make(map[string]WorkflowConfig) + config.Workflows = make(map[string]Workflow) config.Workflows["default"] = *defaultWorkflow() return config, nil } @@ -164,7 +164,7 @@ func (c *DiggerConfig) GetDirectory(projectName string) string { return project.Dir } -func (c *DiggerConfig) GetWorkflow(workflowName string) *WorkflowConfig { +func (c *DiggerConfig) GetWorkflow(workflowName string) *Workflow { workflows := c.Workflows workflow, ok := workflows[workflowName] diff --git a/pkg/github/github.go b/pkg/github/github.go index c0c64e9f3..7ab3b3e96 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -132,7 +132,7 @@ func GetGitHubContext(ghContext string) (*models.Github, error) { return parsedGhContext, nil } -func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]dg_models.ProjectCommand, bool, error) { +func ConvertGithubEventToCommands(event models.Event, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]dg_models.ProjectCommand, bool, error) { commandsPerProject := make([]dg_models.ProjectCommand, 0) switch event.(type) { diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index bd8f95efa..b46e429a7 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -249,7 +249,7 @@ const ( MergeRequestComment = GitLabEventType("merge_request_commented") ) -func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.WorkflowConfig) ([]models.ProjectCommand, bool, error) { +func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) fmt.Printf("ConvertGitLabEventToCommands, event.EventType: %s\n", event.EventType) From d4113b098d4e8c4174589b25e5bf24cef3642d4b Mon Sep 17 00:00:00 2001 From: alexey Date: Tue, 13 Jun 2023 14:18:37 +0100 Subject: [PATCH 19/20] remove copier --- pkg/configuration/converters.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index 0291e99d2..c24585b68 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -4,7 +4,6 @@ import ( "digger/pkg/core/models" "digger/pkg/utils" "fmt" - "github.com/jinzhu/copier" "path/filepath" ) @@ -93,7 +92,6 @@ func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]Workflow { apply, configuration, } - copier.Copy(&w, &item) result[i] = item } return result From 53427aa060d2deef1cdcbcb3cb72802d674a3a84 Mon Sep 17 00:00:00 2001 From: alexey Date: Tue, 13 Jun 2023 14:23:24 +0100 Subject: [PATCH 20/20] remove unnecessary make --- pkg/configuration/converters.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go index c24585b68..e0828b7ff 100644 --- a/pkg/configuration/converters.go +++ b/pkg/configuration/converters.go @@ -69,10 +69,6 @@ func copyStage(stage *StageYaml) *models.Stage { func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfiguration { result := WorkflowConfiguration{} - result.OnPullRequestClosed = make([]string, len(config.OnPullRequestClosed)) - result.OnPullRequestPushed = make([]string, len(config.OnPullRequestPushed)) - result.OnCommitToDefault = make([]string, len(config.OnCommitToDefault)) - result.OnPullRequestClosed = config.OnPullRequestClosed result.OnPullRequestPushed = config.OnPullRequestPushed result.OnCommitToDefault = config.OnCommitToDefault