diff --git a/cmd/digger/main_test.go b/cmd/digger/main_test.go index 0189ad441..9deb7ad9c 100644 --- a/cmd/digger/main_test.go +++ b/cmd/digger/main_test.go @@ -2,9 +2,10 @@ package main import ( "digger/pkg/configuration" + "digger/pkg/core/models" "digger/pkg/digger" "digger/pkg/github" - "digger/pkg/github/models" + gh_models "digger/pkg/github/models" "digger/pkg/reporting" "digger/pkg/utils" "fmt" @@ -895,7 +896,7 @@ func TestGitHubNewPullRequestContext(t *testing.T) { } func TestGitHubNewCommentContext(t *testing.T) { - context, err := models.GetGitHubContext(githubContextCommentJson) + context, err := gh_models.GetGitHubContext(githubContextCommentJson) assert.NoError(t, err) if err != nil { fmt.Println(err) @@ -911,6 +912,7 @@ func TestGitHubNewCommentContext(t *testing.T) { CiService: prManager, PrNumber: prNumber, } + policyChecker := &utils.MockPolicyChecker{} commandsToRunPerProject, _, err := github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, map[string]configuration.Workflow{}) @@ -925,7 +927,7 @@ func TestGitHubNewCommentContext(t *testing.T) { } func TestInvalidGitHubContext(t *testing.T) { - _, err := models.GetGitHubContext(githubInvalidContextJson) + _, err := gh_models.GetGitHubContext(githubInvalidContextJson) require.Error(t, err) if err != nil { fmt.Println(err) @@ -941,11 +943,11 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { prod := configuration.Project{Name: "prod", Dir: "prod", Workflow: "prod"} workflows := map[string]configuration.Workflow{ "dev": { - Plan: &configuration.Stage{Steps: []configuration.Step{ + Plan: &models.Stage{Steps: []models.Step{ {Action: "init", ExtraArgs: []string{}}, {Action: "plan", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Apply: &configuration.Stage{Steps: []configuration.Step{ + Apply: &models.Stage{Steps: []models.Step{ {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, @@ -956,11 +958,11 @@ func TestGitHubNewPullRequestInMultiEnvProjectContext(t *testing.T) { }, }, "prod": { - Plan: &configuration.Stage{Steps: []configuration.Step{ + Plan: &models.Stage{Steps: []models.Step{ {Action: "init", ExtraArgs: []string{}}, {Action: "plan", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, - Apply: &configuration.Stage{Steps: []configuration.Step{ + Apply: &models.Stage{Steps: []models.Step{ {Action: "init", ExtraArgs: []string{}}, {Action: "apply", ExtraArgs: []string{"-var-file=dev.tfvars"}}, }}, diff --git a/go.mod b/go.mod index 141ba7ace..ac0860a34 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/caarlos0/env/v7 v7.1.0 github.com/google/go-github/v51 v51.0.0 github.com/google/uuid v1.3.0 + github.com/jinzhu/copier v0.3.5 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/open-policy-agent/opa v0.53.1 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index 27befd943..9ea446fa9 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index aca686e13..3fb5e4cb3 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -350,23 +350,15 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - var coreApplyStage models.Stage - if workflow.Apply != nil { - coreApplyStage = workflow.Apply.ToCoreStage() - } - var corePlanStage models.Stage - if workflow.Plan != nil { - corePlanStage = workflow.Plan.ToCoreStage() - } + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -379,23 +371,15 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - var coreApplyStage models.Stage - if workflow.Apply != nil { - coreApplyStage = workflow.Apply.ToCoreStage() - } - var corePlanStage models.Stage - if workflow.Plan != nil { - corePlanStage = workflow.Plan.ToCoreStage() - } + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -408,23 +392,15 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if !ok { return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - var coreApplyStage models.Stage - if workflow.Apply != nil { - coreApplyStage = workflow.Apply.ToCoreStage() - } - var corePlanStage models.Stage - if workflow.Plan != nil { - corePlanStage = workflow.Plan.ToCoreStage() - } + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnCommitToDefault, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -465,7 +441,7 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con if !ok { return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, diff --git a/pkg/configuration/config.go b/pkg/configuration/config.go new file mode 100644 index 000000000..7b90a17c1 --- /dev/null +++ b/pkg/configuration/config.go @@ -0,0 +1,75 @@ +package configuration + +import "digger/pkg/core/models" + +type DiggerConfig struct { + Projects []Project + AutoMerge bool + CollectUsageData bool + Workflows map[string]Workflow +} + +type Project struct { + Name string + Dir string + Workspace string + Terragrunt bool + Workflow string + IncludePatterns []string + ExcludePatterns []string +} + +type Workflow struct { + EnvVars *TerraformEnvConfig + Plan *models.Stage + Apply *models.Stage + Configuration *WorkflowConfiguration +} + +type WorkflowConfiguration struct { + OnPullRequestPushed []string + OnPullRequestClosed []string + OnCommitToDefault []string +} + +type TerraformEnvConfig struct { + State []EnvVar + Commands []EnvVar +} + +type EnvVar struct { + Name string + ValueFrom string + Value string +} + +func defaultWorkflow() *Workflow { + return &Workflow{ + Configuration: &WorkflowConfiguration{ + OnCommitToDefault: []string{"digger unlock"}, + OnPullRequestPushed: []string{"digger plan"}, + OnPullRequestClosed: []string{"digger unlock"}, + }, + Plan: &models.Stage{ + Steps: []models.Step{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "plan", ExtraArgs: []string{}, + }, + }, + }, + Apply: &models.Stage{ + Steps: []models.Step{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "apply", ExtraArgs: []string{}, + }, + }, + }, + EnvVars: &TerraformEnvConfig{}, + } +} diff --git a/pkg/configuration/converters.go b/pkg/configuration/converters.go new file mode 100644 index 000000000..e0828b7ff --- /dev/null +++ b/pkg/configuration/converters.go @@ -0,0 +1,163 @@ +package configuration + +import ( + "digger/pkg/core/models" + "digger/pkg/utils" + "fmt" + "path/filepath" +) + +func copyProjects(projects []*ProjectYaml) []Project { + result := make([]Project, len(projects)) + for i, p := range projects { + item := Project{p.Name, + p.Dir, + p.Workspace, + p.Terragrunt, + p.Workflow, + p.IncludePatterns, + p.ExcludePatterns, + } + result[i] = item + } + return result +} + +func copyTerraformEnvConfig(terraformEnvConfig *TerraformEnvConfigYaml) *TerraformEnvConfig { + if terraformEnvConfig == nil { + return &TerraformEnvConfig{} + } + result := TerraformEnvConfig{} + result.State = make([]EnvVar, len(terraformEnvConfig.State)) + result.Commands = make([]EnvVar, len(terraformEnvConfig.Commands)) + + for i, s := range terraformEnvConfig.State { + item := EnvVar{ + s.Name, + s.ValueFrom, + s.Value, + } + result.State[i] = item + } + for i, s := range terraformEnvConfig.Commands { + item := EnvVar{ + s.Name, + s.ValueFrom, + s.Value, + } + result.Commands[i] = item + } + + return &result +} + +func copyStage(stage *StageYaml) *models.Stage { + result := models.Stage{} + result.Steps = make([]models.Step, len(stage.Steps)) + + for i, s := range stage.Steps { + item := models.Step{ + Action: s.Action, + Value: s.Value, + ExtraArgs: s.ExtraArgs, + Shell: s.Shell, + } + result.Steps[i] = item + } + return &result +} + +func copyWorkflowConfiguration(config *WorkflowConfigurationYaml) *WorkflowConfiguration { + result := WorkflowConfiguration{} + result.OnPullRequestClosed = config.OnPullRequestClosed + result.OnPullRequestPushed = config.OnPullRequestPushed + result.OnCommitToDefault = config.OnCommitToDefault + return &result +} + +func copyWorkflows(workflows map[string]*WorkflowYaml) map[string]Workflow { + result := make(map[string]Workflow, len(workflows)) + for i, w := range workflows { + envVars := copyTerraformEnvConfig(w.EnvVars) + plan := copyStage(w.Plan) + apply := copyStage(w.Apply) + configuration := copyWorkflowConfiguration(w.Configuration) + item := Workflow{ + envVars, + plan, + apply, + configuration, + } + result[i] = item + } + return result +} + +func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) { + var diggerConfig DiggerConfig + const defaultWorkflowName = "default" + + if diggerYaml.AutoMerge != nil { + diggerConfig.AutoMerge = *diggerYaml.AutoMerge + } else { + diggerConfig.AutoMerge = false + } + + if diggerYaml.CollectUsageData != nil { + diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData + } else { + diggerConfig.CollectUsageData = true + } + + // if workflow block is not specified in yaml we create a default one, and add it to every project + if diggerYaml.Workflows != nil { + workflows := copyWorkflows(diggerYaml.Workflows) + diggerConfig.Workflows = workflows + + // provide default workflow if not specified + if _, ok := diggerConfig.Workflows[defaultWorkflowName]; !ok { + workflow := *defaultWorkflow() + diggerConfig.Workflows[defaultWorkflowName] = workflow + } + } else { + workflow := *defaultWorkflow() + diggerConfig.Workflows = make(map[string]Workflow) + diggerConfig.Workflows[defaultWorkflowName] = workflow + } + + projects := copyProjects(diggerYaml.Projects) + diggerConfig.Projects = projects + + // update project's workflow if needed + for _, project := range diggerConfig.Projects { + if project.Workflow == "" { + project.Workflow = defaultWorkflowName + } + } + + // check for project name duplicates + projectNames := make(map[string]bool) + for _, project := range diggerConfig.Projects { + if projectNames[project.Name] { + return nil, fmt.Errorf("project name '%s' is duplicated", project.Name) + } + projectNames[project.Name] = true + } + + if diggerYaml.GenerateProjectsConfig != nil { + dirs, err := walker.GetDirs(workingDir) + if err != nil { + return nil, err + } + + for _, dir := range dirs { + includePattern := diggerYaml.GenerateProjectsConfig.Include + excludePattern := diggerYaml.GenerateProjectsConfig.Exclude + if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { + project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName, Workspace: "default"} + diggerConfig.Projects = append(diggerConfig.Projects, project) + } + } + } + return &diggerConfig, nil +} diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 8992a5790..d0c98ab4a 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -1,7 +1,6 @@ package configuration import ( - "digger/pkg/core/models" "digger/pkg/utils" "errors" "fmt" @@ -12,72 +11,6 @@ import ( "regexp" ) -type WorkflowConfiguration struct { - OnPullRequestPushed []string `yaml:"on_pull_request_pushed"` - OnPullRequestClosed []string `yaml:"on_pull_request_closed"` - OnCommitToDefault []string `yaml:"on_commit_to_default"` -} - -type DiggerConfigYaml struct { - Projects []Project `yaml:"projects"` - AutoMerge bool `yaml:"auto_merge"` - Workflows map[string]Workflow `yaml:"workflows"` - CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` - GenerateProjectsConfig *GenerateProjectsConfig `yaml:"generate_projects"` -} - -type EnvVarConfig struct { - Name string `yaml:"name"` - ValueFrom string `yaml:"value_from"` - Value string `yaml:"value"` -} - -type DiggerConfig struct { - Projects []Project - AutoMerge bool - CollectUsageData bool - Workflows map[string]Workflow -} - -type GenerateProjectsConfig struct { - Include string `yaml:"include"` - Exclude string `yaml:"exclude"` -} - -type Project struct { - Name string `yaml:"name"` - Dir string `yaml:"dir"` - Workspace string `yaml:"workspace"` - Terragrunt bool `yaml:"terragrunt"` - Workflow string `yaml:"workflow"` - IncludePatterns []string `yaml:"include_patterns,omitempty"` - ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` -} - -type Stage struct { - Steps []Step `yaml:"steps"` -} - -func (s *Stage) ToCoreStage() models.Stage { - var steps []models.Step - for _, step := range s.Steps { - steps = append(steps, step.ToCoreStep()) - } - return models.Stage{Steps: steps} -} - -type Workflow struct { - EnvVars EnvVars `yaml:"env_vars"` - Plan *Stage `yaml:"plan,omitempty"` - Apply *Stage `yaml:"apply,omitempty"` - Configuration *WorkflowConfiguration `yaml:"workflow_configuration"` -} - -type EnvVars struct { - State []EnvVarConfig `yaml:"state"` - Commands []EnvVarConfig `yaml:"commands"` -} - type DirWalker interface { GetDirs(workingDir string) ([]string, error) } @@ -128,214 +61,9 @@ func (walker *FileSystemDirWalker) GetDirs(workingDir string) ([]string, error) var ErrDiggerConfigConflict = errors.New("more than one digger config file detected, please keep either 'digger.yml' or 'digger.yaml'") -func (p *Project) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawProject Project - raw := rawProject{ - Workspace: "default", - Terragrunt: false, - Workflow: "default", - } - if err := unmarshal(&raw); err != nil { - return err - } - *p = Project(raw) - return nil -} - -func (w *Workflow) UnmarshalYAML(unmarshal func(interface{}) error) error { - type rawWorkflow Workflow - raw := rawWorkflow{ - Configuration: &WorkflowConfiguration{ - OnCommitToDefault: []string{"digger unlock"}, - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - }, - Plan: &Stage{ - Steps: []Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "plan", ExtraArgs: []string{}, - }, - }, - }, - Apply: &Stage{ - Steps: []Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "apply", ExtraArgs: []string{}, - }, - }, - }, - EnvVars: EnvVars{ - State: []EnvVarConfig{}, - Commands: []EnvVarConfig{}, - }, - } - if err := unmarshal(&raw); err != nil { - return err - } - *w = Workflow(raw) - return nil -} - -type Step struct { - Action string - Value string - ExtraArgs []string `yaml:"extra_args,omitempty"` - Shell string -} - -func (s *Step) ToCoreStep() models.Step { - return models.Step{ - Action: s.Action, - Value: s.Value, - ExtraArgs: s.ExtraArgs, - Shell: s.Shell, - } -} - -func (s *Step) UnmarshalYAML(value *yaml.Node) error { - if value.Kind == yaml.ScalarNode { - return value.Decode(&s.Action) - } - - var stepMap map[string]interface{} - if err := value.Decode(&stepMap); err != nil { - return err - } - - if _, ok := stepMap["run"]; ok { - s.Action = "run" - s.Value = stepMap["run"].(string) - if _, ok := stepMap["shell"]; ok { - s.Shell = stepMap["shell"].(string) - } - return nil - } - - s.extract(stepMap, "plan") - s.extract(stepMap, "apply") - - return nil -} - -func (s *Step) extract(stepMap map[string]interface{}, action string) { - if _, ok := stepMap[action]; ok { - s.Action = action - var extraArgs []string - if v, ok := stepMap["extra_args"]; ok { - for _, v := range v.([]interface{}) { - extraArgs = append(extraArgs, v.(string)) - } - s.ExtraArgs = extraArgs - } else { - if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { - for _, v := range v.([]interface{}) { - extraArgs = append(extraArgs, v.(string)) - } - s.ExtraArgs = extraArgs - } - } - } -} - -func defaultWorkflow() *Workflow { - return &Workflow{ - Configuration: &WorkflowConfiguration{ - OnCommitToDefault: []string{"digger unlock"}, - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - }, - Plan: &Stage{ - Steps: []Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "plan", ExtraArgs: []string{}, - }, - }, - }, - Apply: &Stage{ - Steps: []Step{ - { - Action: "init", ExtraArgs: []string{}, - }, - { - Action: "apply", ExtraArgs: []string{}, - }, - }, - }, - } -} - -func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, walker DirWalker) (*DiggerConfig, error) { - var diggerConfig DiggerConfig - const defaultWorkflowName = "default" - - diggerConfig.AutoMerge = diggerYaml.AutoMerge - - // if workflow block is not specified in yaml we create a default one, and add it to every project - if diggerYaml.Workflows != nil { - diggerConfig.Workflows = diggerYaml.Workflows - // provide default workflow if not specified - if _, ok := diggerConfig.Workflows[defaultWorkflowName]; !ok { - workflow := *defaultWorkflow() - diggerConfig.Workflows[defaultWorkflowName] = workflow - } - } else { - workflow := *defaultWorkflow() - diggerConfig.Workflows = make(map[string]Workflow) - diggerConfig.Workflows[defaultWorkflowName] = workflow - } - - diggerConfig.Projects = diggerYaml.Projects - - // update project's workflow if needed - for _, project := range diggerConfig.Projects { - if project.Workflow == "" { - project.Workflow = defaultWorkflowName - } - } - - // check for project name duplicates - projectNames := make(map[string]bool) - for _, project := range diggerConfig.Projects { - if projectNames[project.Name] { - return nil, fmt.Errorf("project name '%s' is duplicated", project.Name) - } - projectNames[project.Name] = true - } - if diggerYaml.CollectUsageData != nil { - diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData - } else { - diggerConfig.CollectUsageData = true - } - - if diggerYaml.GenerateProjectsConfig != nil { - dirs, err := walker.GetDirs(workingDir) - if err != nil { - return nil, err - } - - for _, dir := range dirs { - includePattern := diggerYaml.GenerateProjectsConfig.Include - excludePattern := diggerYaml.GenerateProjectsConfig.Exclude - if utils.MatchIncludeExcludePatternsToFile(dir, []string{includePattern}, []string{excludePattern}) { - project := Project{Name: filepath.Base(dir), Dir: dir, Workflow: defaultWorkflowName, Workspace: "default"} - diggerConfig.Projects = append(diggerConfig.Projects, project) - } - } - } - return &diggerConfig, nil -} - func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error) { - config := &DiggerConfigYaml{} + configYaml := &DiggerConfigYaml{} + config := &DiggerConfig{} fileName, err := retrieveConfigFile(workingDir) if err != nil { if errors.Is(err, ErrDiggerConfigConflict) { @@ -348,36 +76,8 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error config.Projects = make([]Project, 1) config.Projects[0] = defaultProject() config.Workflows = make(map[string]Workflow) - config.Workflows["default"] = Workflow{ - Plan: &Stage{ - Steps: []Step{{ - Action: "init", - ExtraArgs: []string{}, - }, { - Action: "plan", - ExtraArgs: []string{}, - }}, - }, - Apply: &Stage{ - Steps: []Step{{ - Action: "init", - ExtraArgs: []string{}, - }, { - Action: "apply", - ExtraArgs: []string{}, - }}, - }, - Configuration: &WorkflowConfiguration{ - OnPullRequestPushed: []string{"digger plan"}, - OnPullRequestClosed: []string{"digger unlock"}, - OnCommitToDefault: []string{"digger apply"}, - }, - } - c, err := ConvertDiggerYamlToConfig(config, workingDir, walker) - if err != nil { - return nil, fmt.Errorf("failed to read config file %s: %v", fileName, err) - } - return c, nil + config.Workflows["default"] = *defaultWorkflow() + return config, nil } data, err := os.ReadFile(fileName) @@ -385,15 +85,15 @@ func LoadDiggerConfig(workingDir string, walker DirWalker) (*DiggerConfig, error return nil, fmt.Errorf("failed to read config file %s: %v", fileName, err) } - if err := yaml.Unmarshal(data, config); err != nil { + if err := yaml.Unmarshal(data, configYaml); err != nil { return nil, fmt.Errorf("error parsing '%s': %v", fileName, err) } - if (config.Projects == nil || len(config.Projects) == 0) && config.GenerateProjectsConfig == nil { + if (configYaml.Projects == nil || len(configYaml.Projects) == 0) && configYaml.GenerateProjectsConfig == nil { return nil, fmt.Errorf("no projects configuration found in '%s'", fileName) } - c, err := ConvertDiggerYamlToConfig(config, workingDir, walker) + c, err := ConvertDiggerYamlToConfig(configYaml, workingDir, walker) if err != nil { return nil, err } @@ -515,25 +215,27 @@ func retrieveConfigFile(workingDir string) (string, error) { return "", nil } -func CollectEnvVars(envs EnvVars) (map[string]string, map[string]string) { +func CollectTerraformEnvConfig(envs *TerraformEnvConfig) (map[string]string, map[string]string) { stateEnvVars := map[string]string{} + commandEnvVars := map[string]string{} - for _, envvar := range envs.State { - if envvar.Value != "" { - stateEnvVars[envvar.Name] = envvar.Value - } else if envvar.ValueFrom != "" { - stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + if envs != nil { + for _, envvar := range envs.State { + if envvar.Value != "" { + stateEnvVars[envvar.Name] = envvar.Value + } else if envvar.ValueFrom != "" { + stateEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + } } - } - - commandEnvVars := map[string]string{} - for _, envvar := range envs.Commands { - if envvar.Value != "" { - commandEnvVars[envvar.Name] = envvar.Value - } else if envvar.ValueFrom != "" { - commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + for _, envvar := range envs.Commands { + if envvar.Value != "" { + commandEnvVars[envvar.Name] = envvar.Value + } else if envvar.ValueFrom != "" { + commandEnvVars[envvar.Name] = os.Getenv(envvar.ValueFrom) + } } } + return stateEnvVars, commandEnvVars } diff --git a/pkg/configuration/digger_config_test.go b/pkg/configuration/digger_config_test.go index 54fb2eada..1a0b66108 100644 --- a/pkg/configuration/digger_config_test.go +++ b/pkg/configuration/digger_config_test.go @@ -1,6 +1,7 @@ package configuration import ( + "digger/pkg/core/models" "log" "os" "path" @@ -72,6 +73,42 @@ projects: assert.Equal(t, "path/to/module/test", dg.GetDirectory("prod")) } +func TestDefaultDiggerConfig(t *testing.T) { + tempDir, teardown := setUp() + defer teardown() + + diggerCfg := ` +projects: +- name: prod + branch: /main/ + dir: path/to/module/test + workspace: default +` + deleteFile := createFile(path.Join(tempDir, "digger.yaml"), diggerCfg) + defer deleteFile() + + dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) + + assert.NoError(t, err, "expected error to be nil") + assert.NotNil(t, dg, "expected digger config to be not nil") + assert.Equal(t, 1, len(dg.Projects)) + assert.Equal(t, false, dg.AutoMerge) + assert.Equal(t, true, dg.CollectUsageData) + assert.Equal(t, 1, len(dg.Workflows)) + + workflow := dg.Workflows["default"] + assert.NotNil(t, workflow, "expected workflow to be not nil") + assert.NotNil(t, workflow.Plan) + assert.NotNil(t, workflow.Plan.Steps) + + assert.NotNil(t, workflow.Apply) + assert.NotNil(t, workflow.Apply.Steps) + assert.NotNil(t, workflow.EnvVars) + assert.NotNil(t, workflow.Configuration) + + assert.Equal(t, "path/to/module/test", dg.GetDirectory("prod")) +} + func TestDiggerConfigDefaultWorkflow(t *testing.T) { tempDir, teardown := setUp() defer teardown() @@ -134,7 +171,7 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, Step{Action: "run", Value: "echo \"hello\"", Shell: ""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "run", Value: "echo \"hello\"", Shell: ""}, dg.Workflows["myworkflow"].Plan.Steps[0], "parsed struct does not match expected struct") } func TestEnvVarsConfiguration(t *testing.T) { @@ -179,10 +216,10 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, []EnvVarConfig{ + assert.Equal(t, []EnvVar{ {Name: "TF_VAR_state", Value: "s3://mybucket/terraform.tfstate"}, }, dg.Workflows["myworkflow"].EnvVars.State, "parsed struct does not match expected struct") - assert.Equal(t, []EnvVarConfig{ + assert.Equal(t, []EnvVar{ {Name: "TF_VAR_command", Value: "plan"}, }, dg.Workflows["myworkflow"].EnvVars.Commands, "parsed struct does not match expected struct") } @@ -219,13 +256,13 @@ workflows: dg, err := LoadDiggerConfig(tempDir, &FileSystemDirWalker{}) assert.NoError(t, err, "expected error to be nil") - assert.Equal(t, Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["dev"].Plan.Steps[0], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["dev"].Plan.Steps[1], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["dev"].Plan.Steps[2], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["dev"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["dev"].Plan.Steps[1], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["dev"].Plan.Steps[2], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") - assert.Equal(t, Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "run", Value: "rm -rf .terraform", Shell: ""}, dg.Workflows["default"].Plan.Steps[0], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "init", ExtraArgs: nil, Shell: ""}, dg.Workflows["default"].Plan.Steps[1], "parsed struct does not match expected struct") + assert.Equal(t, models.Step{Action: "plan", ExtraArgs: []string{"-var-file=vars/dev.tfvars"}, Shell: ""}, dg.Workflows["default"].Plan.Steps[2], "parsed struct does not match expected struct") } func TestDiggerGenerateProjects(t *testing.T) { diff --git a/pkg/configuration/yaml.go b/pkg/configuration/yaml.go new file mode 100644 index 000000000..c0b80b2b8 --- /dev/null +++ b/pkg/configuration/yaml.go @@ -0,0 +1,181 @@ +package configuration + +import ( + "digger/pkg/core/models" + "gopkg.in/yaml.v3" +) + +type DiggerConfigYaml struct { + Projects []*ProjectYaml `yaml:"projects"` + AutoMerge *bool `yaml:"auto_merge"` + Workflows map[string]*WorkflowYaml `yaml:"workflows"` + CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` + GenerateProjectsConfig *GenerateProjectsConfigYaml `yaml:"generate_projects"` +} + +type ProjectYaml struct { + Name string `yaml:"name"` + Dir string `yaml:"dir"` + Workspace string `yaml:"workspace"` + Terragrunt bool `yaml:"terragrunt"` + Workflow string `yaml:"workflow"` + IncludePatterns []string `yaml:"include_patterns,omitempty"` + ExcludePatterns []string `yaml:"exclude_patterns,omitempty"` +} + +type WorkflowYaml struct { + EnvVars *TerraformEnvConfigYaml `yaml:"env_vars"` + Plan *StageYaml `yaml:"plan,omitempty"` + Apply *StageYaml `yaml:"apply,omitempty"` + Configuration *WorkflowConfigurationYaml `yaml:"workflow_configuration"` +} + +type WorkflowConfigurationYaml struct { + OnPullRequestPushed []string `yaml:"on_pull_request_pushed"` + OnPullRequestClosed []string `yaml:"on_pull_request_closed"` + OnCommitToDefault []string `yaml:"on_commit_to_default"` +} + +func (s *StageYaml) ToCoreStage() models.Stage { + var steps []models.Step + for _, step := range s.Steps { + steps = append(steps, step.ToCoreStep()) + } + return models.Stage{Steps: steps} +} + +type StageYaml struct { + Steps []StepYaml `yaml:"steps"` +} + +type StepYaml struct { + Action string + Value string + ExtraArgs []string `yaml:"extra_args,omitempty"` + Shell string +} + +type TerraformEnvConfigYaml struct { + State []EnvVarYaml `yaml:"state"` + Commands []EnvVarYaml `yaml:"commands"` +} + +type EnvVarYaml struct { + Name string `yaml:"name"` + ValueFrom string `yaml:"value_from"` + Value string `yaml:"value"` +} + +type GenerateProjectsConfigYaml struct { + Include string `yaml:"include"` + Exclude string `yaml:"exclude"` +} + +func (p *ProjectYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawProject ProjectYaml + raw := rawProject{ + Workspace: "default", + Terragrunt: false, + Workflow: "default", + } + if err := unmarshal(&raw); err != nil { + return err + } + *p = ProjectYaml(raw) + return nil +} + +func (w *WorkflowYaml) UnmarshalYAML(unmarshal func(interface{}) error) error { + type rawWorkflow WorkflowYaml + raw := rawWorkflow{ + Configuration: &WorkflowConfigurationYaml{ + OnCommitToDefault: []string{"digger unlock"}, + OnPullRequestPushed: []string{"digger plan"}, + OnPullRequestClosed: []string{"digger unlock"}, + }, + Plan: &StageYaml{ + Steps: []StepYaml{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "plan", ExtraArgs: []string{}, + }, + }, + }, + Apply: &StageYaml{ + Steps: []StepYaml{ + { + Action: "init", ExtraArgs: []string{}, + }, + { + Action: "apply", ExtraArgs: []string{}, + }, + }, + }, + EnvVars: &TerraformEnvConfigYaml{ + State: []EnvVarYaml{}, + Commands: []EnvVarYaml{}, + }, + } + if err := unmarshal(&raw); err != nil { + return err + } + *w = WorkflowYaml(raw) + return nil +} + +func (s *StepYaml) UnmarshalYAML(value *yaml.Node) error { + + if value.Kind == yaml.ScalarNode { + return value.Decode(&s.Action) + } + + var stepMap map[string]interface{} + if err := value.Decode(&stepMap); err != nil { + return err + } + + if _, ok := stepMap["run"]; ok { + s.Action = "run" + s.Value = stepMap["run"].(string) + if _, ok := stepMap["shell"]; ok { + s.Shell = stepMap["shell"].(string) + } + return nil + } + + s.extract(stepMap, "plan") + s.extract(stepMap, "apply") + + return nil +} + +func (s *StepYaml) ToCoreStep() models.Step { + return models.Step{ + Action: s.Action, + Value: s.Value, + ExtraArgs: s.ExtraArgs, + Shell: s.Shell, + } +} + +func (s *StepYaml) extract(stepMap map[string]interface{}, action string) { + if _, ok := stepMap[action]; ok { + s.Action = action + var extraArgs []string + if v, ok := stepMap["extra_args"]; ok { + for _, v := range v.([]interface{}) { + extraArgs = append(extraArgs, v.(string)) + } + s.ExtraArgs = extraArgs + } else { + if v, ok := stepMap[action].(map[string]interface{})["extra_args"]; ok { + for _, v := range v.([]interface{}) { + extraArgs = append(extraArgs, v.(string)) + } + s.ExtraArgs = extraArgs + } + } + } +} diff --git a/pkg/digger/digger_test.go b/pkg/digger/digger_test.go index 92dbe88f1..1421e76bf 100644 --- a/pkg/digger/digger_test.go +++ b/pkg/digger/digger_test.go @@ -208,6 +208,7 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) { CiService: prManager, PrNumber: 1, } + executor := execution.DiggerExecutor{ ApplyStage: &models.Stage{}, PlanStage: &models.Stage{ diff --git a/pkg/github/github.go b/pkg/github/github.go index f85e84001..7ab3b3e96 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -144,10 +144,7 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - - coreApplyStage := workflow.Apply.ToCoreStage() - corePlanStage := workflow.Plan.ToCoreStage() + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { commandsPerProject = append(commandsPerProject, dg_models.ProjectCommand{ @@ -156,8 +153,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnCommitToDefault, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -168,8 +165,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -180,8 +177,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -213,9 +210,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - coreApplyStage := workflow.Apply.ToCoreStage() - corePlanStage := workflow.Plan.ToCoreStage() + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) + workspace := project.Workspace workspaceOverride, err := utils.ParseWorkspace(event.Comment.Body) if err != nil { @@ -230,8 +226,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: workspace, Terragrunt: project.Terragrunt, Commands: []string{command}, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index 1bdedb261..b46e429a7 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -261,17 +261,15 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - coreApplyStage := workflow.Apply.ToCoreStage() - corePlanStage := workflow.Plan.ToCoreStage() + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -283,23 +281,15 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if !ok { return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - var coreApplyStage models.Stage - if workflow.Apply != nil { - coreApplyStage = workflow.Apply.ToCoreStage() - } - var corePlanStage models.Stage - if workflow.Plan != nil { - corePlanStage = workflow.Plan.ToCoreStage() - } + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -336,23 +326,15 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if workspaceOverride != "" { workspace = workspaceOverride } - stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - var coreApplyStage models.Stage - if workflow.Apply != nil { - coreApplyStage = workflow.Apply.ToCoreStage() - } - var corePlanStage models.Stage - if workflow.Plan != nil { - corePlanStage = workflow.Plan.ToCoreStage() - } + stateEnvVars, commandEnvVars := configuration.CollectTerraformEnvConfig(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: workspace, Terragrunt: project.Terragrunt, Commands: []string{command}, - ApplyStage: &coreApplyStage, - PlanStage: &corePlanStage, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, })