From 06869bf1428effca521286066be62da709001f39 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Wed, 7 Jun 2023 16:35:27 +0100 Subject: [PATCH 1/4] set usage to true if missing (#330) --- pkg/configuration/digger_config.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index 14235e04c..be9c2e64e 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -21,7 +21,7 @@ type DiggerConfigYaml struct { Projects []Project `yaml:"projects"` AutoMerge bool `yaml:"auto_merge"` Workflows map[string]Workflow `yaml:"workflows"` - CollectUsageData bool `yaml:"collect_usage_data"` + CollectUsageData *bool `yaml:"collect_usage_data,omitempty"` GenerateProjectsConfig *GenerateProjectsConfig `yaml:"generate_projects"` } @@ -257,8 +257,11 @@ func ConvertDiggerYamlToConfig(diggerYaml *DiggerConfigYaml, workingDir string, } projectNames[project.Name] = true } - - diggerConfig.CollectUsageData = diggerYaml.CollectUsageData + if diggerYaml.CollectUsageData != nil { + diggerConfig.CollectUsageData = *diggerYaml.CollectUsageData + } else { + diggerConfig.CollectUsageData = true + } if diggerYaml.GenerateProjectsConfig != nil { dirs, err := walker.GetDirs(workingDir) From 7f8203a252c1eaa2112f336942743008e0d12cd5 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 16:36:10 +0100 Subject: [PATCH 2/4] fix(deps): update module github.com/aws/aws-sdk-go to v1.44.277 (#324) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 07f2cc198..b81f64c00 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cloud.google.com/go/storage v1.30.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/data/aztables v1.0.1 - github.com/aws/aws-sdk-go v1.44.276 + github.com/aws/aws-sdk-go v1.44.277 github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/caarlos0/env/v7 v7.1.0 github.com/google/go-github/v51 v51.0.0 diff --git a/go.sum b/go.sum index 1e23ff1c2..394bbbded 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/aws/aws-sdk-go v1.44.275 h1:VqRULgqrigvQLll4e4hXuc568EQAtZQ6jmBzLlQHz github.com/aws/aws-sdk-go v1.44.275/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.44.276 h1:ywPlx9C5Yc482dUgAZ9bHpQ6onVvJvYE9FJWsNDCEy0= github.com/aws/aws-sdk-go v1.44.276/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.277 h1:YHmyzBPARTJ7LLYV1fxbfEbQOaUh3kh52hb7nBvX3BQ= +github.com/aws/aws-sdk-go v1.44.277/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/bmatcuk/doublestar/v4 v4.6.0 h1:HTuxyug8GyFbRkrffIpzNCSK4luc0TY3wzXvzIZhEXc= github.com/bmatcuk/doublestar/v4 v4.6.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= @@ -53,6 +55,7 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= From f5ec5d07351d1965ad4a34084efe803e0166b6c4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 7 Jun 2023 16:36:40 +0100 Subject: [PATCH 3/4] fix(deps): update module github.com/google/go-github/v51 to v53 (#322) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 9 +++++---- go.sum | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b81f64c00..dfbc40f9a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/caarlos0/env/v7 v7.1.0 github.com/google/go-github/v51 v51.0.0 + github.com/google/go-github/v53 v53.0.0 github.com/google/uuid v1.3.0 github.com/microsoft/azure-devops-go-api/azuredevops v1.0.0-b5 github.com/stretchr/testify v1.8.4 @@ -26,7 +27,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -41,9 +42,9 @@ require ( github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/crypto v0.7.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/sys v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect diff --git a/go.sum b/go.sum index 394bbbded..1f16413bf 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -87,6 +89,7 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v51 v51.0.0 h1:KCjsbgPV28VoRftdP+K2mQL16jniUsLAJknsOVKwHyU= github.com/google/go-github/v51 v51.0.0/go.mod h1:kZj/rn/c1lSUbr/PFWl2hhusPV7a5XNYKcwPrd5L3Us= +github.com/google/go-github/v53 v53.0.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= @@ -164,9 +167,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -184,6 +191,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From ac7c2ef6a5f0dd8dc5d0a601db48481ac0aa95d2 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Wed, 7 Jun 2023 16:59:31 +0100 Subject: [PATCH 4/4] refactor core functionality and models into separate folder (#323) * refactor core functionality and models into separate folder to be able to pull it up later --- cmd/digger/main.go | 16 +- pkg/azure/azure.go | 41 +- pkg/ci/{main.go => ci.go} | 0 pkg/configuration/digger_config.go | 18 + pkg/core/execution/execution.go | 270 ++++++++++++ pkg/core/locking/locking.go | 14 + pkg/{ => core}/models/models.go | 17 +- pkg/core/reporting/reporting.go | 5 + pkg/core/runners/runners.go | 55 +++ pkg/core/storage/plan_storage.go | 8 + .../utils.go => core/terraform/test_utils.go} | 0 pkg/{ => core}/terraform/tf.go | 0 pkg/{ => core}/terraform/tf_test.go | 0 pkg/{ => core}/utils/comments.go | 0 pkg/digger/digger.go | 403 +++--------------- pkg/digger/digger_test.go | 19 +- pkg/github/github.go | 24 +- pkg/gitlab/gitlab.go | 32 +- pkg/integration/integration_test.go | 2 +- pkg/locking/locking.go | 18 +- pkg/reporting/reporting.go | 4 - pkg/storage/plan_storage.go | 7 - 22 files changed, 525 insertions(+), 428 deletions(-) rename pkg/ci/{main.go => ci.go} (100%) create mode 100644 pkg/core/execution/execution.go create mode 100644 pkg/core/locking/locking.go rename pkg/{ => core}/models/models.go (57%) create mode 100644 pkg/core/reporting/reporting.go create mode 100644 pkg/core/runners/runners.go create mode 100644 pkg/core/storage/plan_storage.go rename pkg/{terraform/utils.go => core/terraform/test_utils.go} (100%) rename pkg/{ => core}/terraform/tf.go (100%) rename pkg/{ => core}/terraform/tf_test.go (100%) rename pkg/{ => core}/utils/comments.go (100%) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index 54753bf0c..cb635723b 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -4,13 +4,15 @@ import ( "context" "digger/pkg/azure" "digger/pkg/configuration" + core_locking "digger/pkg/core/locking" + "digger/pkg/core/models" + core_storage "digger/pkg/core/storage" "digger/pkg/digger" "digger/pkg/gcp" dg_github "digger/pkg/github" github_models "digger/pkg/github/models" "digger/pkg/gitlab" "digger/pkg/locking" - "digger/pkg/models" "digger/pkg/reporting" "digger/pkg/storage" "digger/pkg/usage" @@ -22,7 +24,7 @@ import ( "strings" ) -func gitHubCI(lock locking.Lock) { +func gitHubCI(lock core_locking.Lock) { println("Using GitHub.") githubActor := os.Getenv("GITHUB_ACTOR") if githubActor != "" { @@ -121,7 +123,7 @@ func gitHubCI(lock locking.Lock) { }() } -func gitLabCI(lock locking.Lock) { +func gitLabCI(lock core_locking.Lock) { println("Using GitLab.") projectNamespace := os.Getenv("CI_PROJECT_NAMESPACE") @@ -212,7 +214,7 @@ func gitLabCI(lock locking.Lock) { }() } -func azureCI(lock locking.Lock) { +func azureCI(lock core_locking.Lock) { fmt.Println("> Azure CI detected") azureContext := os.Getenv("AZURE_CONTEXT") azureToken := os.Getenv("AZURE_TOKEN") @@ -260,7 +262,7 @@ func azureCI(lock locking.Lock) { fmt.Printf("command: %s, project: %s\n", strings.Join(v.Commands, ", "), v.ProjectName) } - var planStorage storage.PlanStorage + var planStorage core_storage.PlanStorage diggerProjectNamespace := parsedAzureContext.BaseUrl + "/" + parsedAzureContext.ProjectName reporter := &reporting.CiReporter{ @@ -335,8 +337,8 @@ func main() { } } -func newPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber int) storage.PlanStorage { - var planStorage storage.PlanStorage +func newPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber int) core_storage.PlanStorage { + var planStorage core_storage.PlanStorage uploadDestination := strings.ToLower(os.Getenv("PLAN_UPLOAD_DESTINATION")) if uploadDestination == "github" { diff --git a/pkg/azure/azure.go b/pkg/azure/azure.go index 66d4115e8..aca686e13 100644 --- a/pkg/azure/azure.go +++ b/pkg/azure/azure.go @@ -4,7 +4,7 @@ import ( "context" "digger/pkg/ci" "digger/pkg/configuration" - "digger/pkg/models" + "digger/pkg/core/models" "digger/pkg/utils" "encoding/json" "errors" @@ -351,15 +351,22 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - + var coreApplyStage models.Stage + if workflow.Apply != nil { + coreApplyStage = workflow.Apply.ToCoreStage() + } + var corePlanStage models.Stage + if workflow.Plan != nil { + corePlanStage = workflow.Plan.ToCoreStage() + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -373,15 +380,22 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - + var coreApplyStage models.Stage + if workflow.Apply != nil { + coreApplyStage = workflow.Apply.ToCoreStage() + } + var corePlanStage models.Stage + if workflow.Plan != nil { + corePlanStage = workflow.Plan.ToCoreStage() + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -395,15 +409,22 @@ func ConvertAzureEventToCommands(parseAzureContext Azure, impactedProjects []con return nil, false, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - + var coreApplyStage models.Stage + if workflow.Apply != nil { + coreApplyStage = workflow.Apply.ToCoreStage() + } + var corePlanStage models.Stage + if workflow.Plan != nil { + corePlanStage = workflow.Plan.ToCoreStage() + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnCommitToDefault, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) diff --git a/pkg/ci/main.go b/pkg/ci/ci.go similarity index 100% rename from pkg/ci/main.go rename to pkg/ci/ci.go diff --git a/pkg/configuration/digger_config.go b/pkg/configuration/digger_config.go index be9c2e64e..3177cd258 100644 --- a/pkg/configuration/digger_config.go +++ b/pkg/configuration/digger_config.go @@ -1,6 +1,7 @@ package configuration import ( + "digger/pkg/core/models" "digger/pkg/utils" "errors" "fmt" @@ -57,6 +58,14 @@ type Stage struct { Steps []Step `yaml:"steps"` } +func (s *Stage) ToCoreStage() models.Stage { + var steps []models.Step + for _, step := range s.Steps { + steps = append(steps, step.ToCoreStep()) + } + return models.Stage{Steps: steps} +} + type Workflow struct { EnvVars EnvVars `yaml:"env_vars"` Plan *Stage `yaml:"plan,omitempty"` @@ -157,6 +166,15 @@ type Step struct { Shell string } +func (s *Step) ToCoreStep() models.Step { + return models.Step{ + Action: s.Action, + Value: s.Value, + ExtraArgs: s.ExtraArgs, + Shell: s.Shell, + } +} + func (s *Step) UnmarshalYAML(value *yaml.Node) error { if value.Kind == yaml.ScalarNode { return value.Decode(&s.Action) diff --git a/pkg/core/execution/execution.go b/pkg/core/execution/execution.go new file mode 100644 index 000000000..ddd47a901 --- /dev/null +++ b/pkg/core/execution/execution.go @@ -0,0 +1,270 @@ +package execution + +import ( + "digger/pkg/core/locking" + "digger/pkg/core/models" + "digger/pkg/core/reporting" + "digger/pkg/core/runners" + "digger/pkg/core/storage" + "digger/pkg/core/terraform" + "digger/pkg/core/utils" + "fmt" + "log" + "os" + "path" + "regexp" + "strings" +) + +type DiggerExecutor struct { + ProjectNamespace string + ProjectName string + ProjectPath string + StateEnvVars map[string]string + CommandEnvVars map[string]string + ApplyStage *models.Stage + PlanStage *models.Stage + CommandRunner runners.CommandRun + TerraformExecutor terraform.TerraformExecutor + Reporter reporting.Reporter + ProjectLock locking.ProjectLock + PlanStorage storage.PlanStorage +} + +func (d DiggerExecutor) planFileName() string { + return d.ProjectNamespace + "#" + d.ProjectName + ".tfplan" +} + +func (d DiggerExecutor) localPlanFilePath() string { + return path.Join(d.ProjectPath, d.planFileName()) +} + +func (d DiggerExecutor) storedPlanFilePath() string { + return path.Join(d.ProjectNamespace, d.planFileName()) +} + +func (d DiggerExecutor) Plan() (bool, error) { + locked, err := d.ProjectLock.Lock() + if err != nil { + return false, fmt.Errorf("error locking project: %v", err) + } + log.Printf("Lock result: %t\n", locked) + if locked { + var planSteps []models.Step + + if d.PlanStage != nil { + planSteps = d.PlanStage.Steps + } else { + planSteps = []models.Step{ + { + Action: "init", + }, + { + Action: "plan", + }, + } + } + for _, step := range planSteps { + if step.Action == "init" { + _, _, err := d.TerraformExecutor.Init(step.ExtraArgs, d.StateEnvVars) + if err != nil { + return false, fmt.Errorf("error running init: %v", err) + } + } + if step.Action == "plan" { + planArgs := []string{"-out", d.planFileName()} + planArgs = append(planArgs, step.ExtraArgs...) + isNonEmptyPlan, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars) + if err != nil { + return false, fmt.Errorf("error executing plan: %v", err) + } + if d.PlanStorage != nil { + planExists, err := d.PlanStorage.PlanExists(d.storedPlanFilePath()) + if err != nil { + return false, fmt.Errorf("error checking if plan exists: %v", err) + } + + if planExists { + err = d.PlanStorage.DeleteStoredPlan(d.storedPlanFilePath()) + if err != nil { + return false, fmt.Errorf("error deleting plan: %v", err) + } + } + + err = d.PlanStorage.StorePlan(d.localPlanFilePath(), d.storedPlanFilePath()) + if err != nil { + return false, fmt.Errorf("error storing plan: %v", err) + } + } + plan := cleanupTerraformPlan(isNonEmptyPlan, err, stdout, stderr) + comment := utils.GetTerraformOutputAsCollapsibleComment("Plan for **"+d.ProjectLock.LockId()+"**", plan) + err = d.Reporter.Report(comment) + if err != nil { + fmt.Printf("error publishing comment: %v", err) + } + } + if step.Action == "run" { + var commands []string + if os.Getenv("ACTIVATE_VENV") == "true" { + commands = append(commands, fmt.Sprintf("source %v/.venv/bin/activate", os.Getenv("GITHUB_WORKSPACE"))) + } + commands = append(commands, step.Value) + log.Printf("Running %v for **%v**\n", step.Value, d.ProjectLock.LockId()) + _, _, err := d.CommandRunner.Run(d.ProjectPath, step.Shell, commands) + if err != nil { + return false, fmt.Errorf("error running command: %v", err) + } + } + } + return true, nil + } + return false, nil +} + +func (d DiggerExecutor) Apply() (bool, error) { + var plansFilename *string + if d.PlanStorage != nil { + var err error + plansFilename, err = d.PlanStorage.RetrievePlan(d.localPlanFilePath(), d.storedPlanFilePath()) + if err != nil { + return false, fmt.Errorf("error retrieving plan: %v", err) + } + } + + locked, err := d.ProjectLock.Lock() + + if err != nil { + return false, fmt.Errorf("error locking project: %v", err) + } + + if locked { + var applySteps []models.Step + + if d.ApplyStage != nil { + applySteps = d.ApplyStage.Steps + } else { + applySteps = []models.Step{ + { + Action: "init", + }, + { + Action: "apply", + }, + } + } + + for _, step := range applySteps { + if step.Action == "init" { + _, _, err := d.TerraformExecutor.Init(step.ExtraArgs, d.StateEnvVars) + if err != nil { + return false, fmt.Errorf("error running init: %v", err) + } + } + if step.Action == "apply" { + stdout, stderr, err := d.TerraformExecutor.Apply(step.ExtraArgs, plansFilename, d.CommandEnvVars) + applyOutput := cleanupTerraformApply(true, err, stdout, stderr) + comment := utils.GetTerraformOutputAsCollapsibleComment("Apply for **"+d.ProjectLock.LockId()+"**", applyOutput) + commentErr := d.Reporter.Report(comment) + if commentErr != nil { + fmt.Printf("error publishing comment: %v", err) + } + if err != nil { + commentErr = d.Reporter.Report("Error during applying.") + if commentErr != nil { + fmt.Printf("error publishing comment: %v", err) + } + return false, fmt.Errorf("error executing apply: %v", err) + } + } + if step.Action == "run" { + var commands []string + if os.Getenv("ACTIVATE_VENV") == "true" { + commands = append(commands, fmt.Sprintf("source %v/.venv/bin/activate", os.Getenv("GITHUB_WORKSPACE"))) + } + commands = append(commands, step.Value) + log.Printf("Running %v for **%v**\n", step.Value, d.ProjectLock.LockId()) + _, _, err := d.CommandRunner.Run(d.ProjectPath, step.Shell, commands) + if err != nil { + return false, fmt.Errorf("error running command: %v", err) + } + } + } + return true, nil + } else { + return false, nil + } +} + +func (d DiggerExecutor) Unlock() error { + err := d.ProjectLock.ForceUnlock() + if err != nil { + return fmt.Errorf("failed to aquire lock: %s, %v", d.ProjectLock.LockId(), err) + } + if d.PlanStorage != nil { + err = d.PlanStorage.DeleteStoredPlan(d.storedPlanFilePath()) + if err != nil { + return fmt.Errorf("failed to delete stored plan file '%v': %v", d.storedPlanFilePath(), err) + } + } + if err != nil { + return fmt.Errorf("failed to delete stored plan file '%v': %v", d.storedPlanFilePath(), err) + } + return nil +} + +func (d DiggerExecutor) Lock() error { + _, err := d.ProjectLock.Lock() + if err != nil { + return fmt.Errorf("failed to aquire lock: %s, %v", d.ProjectLock.LockId(), err) + } + return nil +} + +func cleanupTerraformOutput(nonEmptyOutput bool, planError error, stdout string, stderr string, regexStr *string) string { + var errorStr, start string + + // removes output of terraform -version command that terraform-exec executes on every run + i := strings.Index(stdout, "Initializing the backend...") + if i != -1 { + stdout = stdout[i:] + } + endPos := len(stdout) + + if planError != nil { + if stderr != "" { + errorStr = stderr + } else if stdout != "" { + errorStr = stdout + } + return errorStr + } else if nonEmptyOutput { + start = "Terraform will perform the following actions:" + } else { + start = "No changes. Your infrastructure matches the configuration." + } + + startPos := strings.Index(stdout, start) + if startPos == -1 { + startPos = 0 + } + + if regexStr != nil { + regex := regexp.MustCompile(*regexStr) + matches := regex.FindStringSubmatch(stdout) + if len(matches) > 0 { + firstMatch := matches[0] + endPos = strings.LastIndex(stdout, firstMatch) + len(firstMatch) + } + } + + return stdout[startPos:endPos] +} + +func cleanupTerraformApply(nonEmptyPlan bool, planError error, stdout string, stderr string) string { + return cleanupTerraformOutput(nonEmptyPlan, planError, stdout, stderr, nil) +} + +func cleanupTerraformPlan(nonEmptyPlan bool, planError error, stdout string, stderr string) string { + regex := `───────────.+` + return cleanupTerraformOutput(nonEmptyPlan, planError, stdout, stderr, ®ex) +} diff --git a/pkg/core/locking/locking.go b/pkg/core/locking/locking.go new file mode 100644 index 000000000..5c9d23d4c --- /dev/null +++ b/pkg/core/locking/locking.go @@ -0,0 +1,14 @@ +package locking + +type Lock interface { + Lock(transactionId int, resource string) (bool, error) + Unlock(resource string) (bool, error) + GetLock(resource string) (*int, error) +} + +type ProjectLock interface { + Lock() (bool, error) + Unlock() (bool, error) + ForceUnlock() error + LockId() string +} diff --git a/pkg/models/models.go b/pkg/core/models/models.go similarity index 57% rename from pkg/models/models.go rename to pkg/core/models/models.go index 19f5956ba..75a09e0bb 100644 --- a/pkg/models/models.go +++ b/pkg/core/models/models.go @@ -1,15 +1,24 @@ package models -import "digger/pkg/configuration" - type ProjectCommand struct { ProjectName string ProjectDir string ProjectWorkspace string Terragrunt bool Commands []string - ApplyStage *configuration.Stage - PlanStage *configuration.Stage + ApplyStage *Stage + PlanStage *Stage StateEnvVars map[string]string CommandEnvVars map[string]string } + +type Step struct { + Action string + Value string + ExtraArgs []string + Shell string +} + +type Stage struct { + Steps []Step +} diff --git a/pkg/core/reporting/reporting.go b/pkg/core/reporting/reporting.go new file mode 100644 index 000000000..622a533eb --- /dev/null +++ b/pkg/core/reporting/reporting.go @@ -0,0 +1,5 @@ +package reporting + +type Reporter interface { + Report(report string) error +} diff --git a/pkg/core/runners/runners.go b/pkg/core/runners/runners.go new file mode 100644 index 000000000..a1c51c2c5 --- /dev/null +++ b/pkg/core/runners/runners.go @@ -0,0 +1,55 @@ +package runners + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" +) + +type CommandRun interface { + Run(workingDir string, shell string, commands []string) (string, string, error) +} + +type CommandRunner struct { +} + +func (c CommandRunner) Run(workingDir string, shell string, commands []string) (string, string, error) { + var args []string + if shell == "" { + shell = "bash" + args = []string{"-eo", "pipefail"} + } + + scriptFile, err := ioutil.TempFile("", "run-script") + if err != nil { + return "", "", fmt.Errorf("error creating script file: %v", err) + } + defer os.Remove(scriptFile.Name()) + + for _, command := range commands { + _, err := scriptFile.WriteString(command + "\n") + if err != nil { + return "", "", fmt.Errorf("error writing to script file: %v", err) + } + } + args = append(args, scriptFile.Name()) + + cmd := exec.Command(shell, args...) + cmd.Dir = workingDir + + var stdout, stderr bytes.Buffer + mwout := io.MultiWriter(os.Stdout, &stdout) + mwerr := io.MultiWriter(os.Stderr, &stderr) + cmd.Stdout = mwout + cmd.Stderr = mwerr + err = cmd.Run() + + if err != nil { + return stdout.String(), stderr.String(), fmt.Errorf("error: %v", err) + } + + return stdout.String(), stderr.String(), err +} diff --git a/pkg/core/storage/plan_storage.go b/pkg/core/storage/plan_storage.go new file mode 100644 index 000000000..3fc39686d --- /dev/null +++ b/pkg/core/storage/plan_storage.go @@ -0,0 +1,8 @@ +package storage + +type PlanStorage interface { + StorePlan(localPlanFilePath string, storedPlanFilePath string) error + RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) + DeleteStoredPlan(storedPlanFilePath string) error + PlanExists(storedPlanFilePath string) (bool, error) +} diff --git a/pkg/terraform/utils.go b/pkg/core/terraform/test_utils.go similarity index 100% rename from pkg/terraform/utils.go rename to pkg/core/terraform/test_utils.go diff --git a/pkg/terraform/tf.go b/pkg/core/terraform/tf.go similarity index 100% rename from pkg/terraform/tf.go rename to pkg/core/terraform/tf.go diff --git a/pkg/terraform/tf_test.go b/pkg/core/terraform/tf_test.go similarity index 100% rename from pkg/terraform/tf_test.go rename to pkg/core/terraform/tf_test.go diff --git a/pkg/utils/comments.go b/pkg/core/utils/comments.go similarity index 100% rename from pkg/utils/comments.go rename to pkg/core/utils/comments.go diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go index 75b59366d..5942c0359 100644 --- a/pkg/digger/digger.go +++ b/pkg/digger/digger.go @@ -1,29 +1,60 @@ package digger import ( - "bytes" "digger/pkg/ci" - "digger/pkg/configuration" + "digger/pkg/core/execution" + core_locking "digger/pkg/core/locking" + "digger/pkg/core/models" + "digger/pkg/core/reporting" + "digger/pkg/core/runners" + "digger/pkg/core/storage" + "digger/pkg/core/terraform" "digger/pkg/locking" - "digger/pkg/models" - "digger/pkg/reporting" - "digger/pkg/storage" - "digger/pkg/terraform" "digger/pkg/usage" - "digger/pkg/utils" "fmt" - "io" - "io/ioutil" "log" "os" - "os/exec" "path" - "regexp" - "strings" "time" ) -func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNamespace string, requestedBy string, eventName string, prNumber int, ciService ci.CIService, lock locking.Lock, reporter reporting.Reporter, planStorage storage.PlanStorage, workingDir string) (bool, bool, error) { +type CIName string + +const ( + None = CIName("") + GitHub = CIName("github") + GitLab = CIName("gitlab") + BitBucket = CIName("bitbucket") + Azure = CIName("azure") +) + +func (ci CIName) String() string { + return string(ci) +} + +func DetectCI() CIName { + + notEmpty := func(key string) bool { + return os.Getenv(key) != "" + } + + if notEmpty("GITHUB_ACTIONS") { + return GitHub + } + if notEmpty("GITLAB_CI") { + return GitLab + } + if notEmpty("BITBUCKET_BUILD_NUMBER") { + return BitBucket + } + if notEmpty("AZURE_CI") { + return Azure + } + return None + +} + +func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNamespace string, requestedBy string, eventName string, prNumber int, ciService ci.CIService, lock core_locking.Lock, reporter reporting.Reporter, planStorage storage.PlanStorage, workingDir string) (bool, bool, error) { appliesPerProject := make(map[string]bool) for _, projectCommands := range commandsPerProject { for _, command := range projectCommands.Commands { @@ -43,8 +74,8 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNa terraformExecutor = terraform.Terraform{WorkingDir: projectPath, Workspace: projectCommands.ProjectWorkspace} } - commandRunner := CommandRunner{} - diggerExecutor := DiggerExecutor{ + commandRunner := runners.CommandRunner{} + diggerExecutor := execution.DiggerExecutor{ projectNamespace, projectCommands.ProjectName, projectPath, @@ -126,9 +157,9 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNa return allAppliesSuccess, atLeastOneApply, nil } -func MergePullRequest(githubPrService ci.CIService, prNumber int) { +func MergePullRequest(ciService ci.CIService, prNumber int) { time.Sleep(5 * time.Second) - combinedStatus, err := githubPrService.GetCombinedPullRequestStatus(prNumber) + combinedStatus, err := ciService.GetCombinedPullRequestStatus(prNumber) if err != nil { log.Fatalf("failed to get combined status, %v", err) @@ -138,7 +169,7 @@ func MergePullRequest(githubPrService ci.CIService, prNumber int) { log.Fatalf("PR is not mergeable. Status: %v", combinedStatus) } - prIsMergeable, err := githubPrService.IsMergeable(prNumber) + prIsMergeable, err := ciService.IsMergeable(prNumber) if err != nil { log.Fatalf("failed to check if PR is mergeable, %v", err) @@ -148,342 +179,8 @@ func MergePullRequest(githubPrService ci.CIService, prNumber int) { log.Fatalf("PR is not mergeable") } - err = githubPrService.MergePullRequest(prNumber) + err = ciService.MergePullRequest(prNumber) if err != nil { log.Fatalf("failed to merge PR, %v", err) } } - -type DiggerExecutor struct { - ProjectNamespace string - ProjectName string - ProjectPath string - StateEnvVars map[string]string - CommandEnvVars map[string]string - ApplyStage *configuration.Stage - PlanStage *configuration.Stage - CommandRunner CommandRun - TerraformExecutor terraform.TerraformExecutor - Reporter reporting.Reporter - ProjectLock locking.ProjectLock - PlanStorage storage.PlanStorage -} - -type CommandRun interface { - Run(workingDir string, shell string, commands []string) (string, string, error) -} - -type CommandRunner struct { -} - -func (c CommandRunner) Run(workingDir string, shell string, commands []string) (string, string, error) { - var args []string - if shell == "" { - shell = "bash" - args = []string{"-eo", "pipefail"} - } - - scriptFile, err := ioutil.TempFile("", "run-script") - if err != nil { - return "", "", fmt.Errorf("error creating script file: %v", err) - } - defer os.Remove(scriptFile.Name()) - - for _, command := range commands { - _, err := scriptFile.WriteString(command + "\n") - if err != nil { - return "", "", fmt.Errorf("error writing to script file: %v", err) - } - } - args = append(args, scriptFile.Name()) - - cmd := exec.Command(shell, args...) - cmd.Dir = workingDir - - var stdout, stderr bytes.Buffer - mwout := io.MultiWriter(os.Stdout, &stdout) - mwerr := io.MultiWriter(os.Stderr, &stderr) - cmd.Stdout = mwout - cmd.Stderr = mwerr - err = cmd.Run() - - if err != nil { - return stdout.String(), stderr.String(), fmt.Errorf("error: %v", err) - } - - return stdout.String(), stderr.String(), err -} - -func (d DiggerExecutor) planFileName() string { - return d.ProjectNamespace + "#" + d.ProjectName + ".tfplan" -} - -func (d DiggerExecutor) localPlanFilePath() string { - return path.Join(d.ProjectPath, d.planFileName()) -} - -func (d DiggerExecutor) storedPlanFilePath() string { - return path.Join(d.ProjectNamespace, d.planFileName()) -} - -func (d DiggerExecutor) Plan() (bool, error) { - locked, err := d.ProjectLock.Lock() - if err != nil { - return false, fmt.Errorf("error locking project: %v", err) - } - log.Printf("Lock result: %t\n", locked) - if locked { - var planSteps []configuration.Step - - if d.PlanStage != nil { - planSteps = d.PlanStage.Steps - } else { - planSteps = []configuration.Step{ - { - Action: "init", - }, - { - Action: "plan", - }, - } - } - for _, step := range planSteps { - if step.Action == "init" { - _, _, err := d.TerraformExecutor.Init(step.ExtraArgs, d.StateEnvVars) - if err != nil { - return false, fmt.Errorf("error running init: %v", err) - } - } - if step.Action == "plan" { - planArgs := []string{"-out", d.planFileName()} - planArgs = append(planArgs, step.ExtraArgs...) - isNonEmptyPlan, stdout, stderr, err := d.TerraformExecutor.Plan(planArgs, d.CommandEnvVars) - if err != nil { - return false, fmt.Errorf("error executing plan: %v", err) - } - if d.PlanStorage != nil { - planExists, err := d.PlanStorage.PlanExists(d.storedPlanFilePath()) - if err != nil { - return false, fmt.Errorf("error checking if plan exists: %v", err) - } - - if planExists { - err = d.PlanStorage.DeleteStoredPlan(d.storedPlanFilePath()) - if err != nil { - return false, fmt.Errorf("error deleting plan: %v", err) - } - } - - err = d.PlanStorage.StorePlan(d.localPlanFilePath(), d.storedPlanFilePath()) - if err != nil { - return false, fmt.Errorf("error storing plan: %v", err) - } - } - plan := cleanupTerraformPlan(isNonEmptyPlan, err, stdout, stderr) - comment := utils.GetTerraformOutputAsCollapsibleComment("Plan for **"+d.ProjectLock.LockId()+"**", plan) - err = d.Reporter.Report(comment) - if err != nil { - fmt.Printf("error publishing comment: %v", err) - } - } - if step.Action == "run" { - var commands []string - if os.Getenv("ACTIVATE_VENV") == "true" { - commands = append(commands, fmt.Sprintf("source %v/.venv/bin/activate", os.Getenv("GITHUB_WORKSPACE"))) - } - commands = append(commands, step.Value) - log.Printf("Running %v for **%v**\n", step.Value, d.ProjectLock.LockId()) - _, _, err := d.CommandRunner.Run(d.ProjectPath, step.Shell, commands) - if err != nil { - return false, fmt.Errorf("error running command: %v", err) - } - } - } - return true, nil - } - return false, nil -} - -func (d DiggerExecutor) Apply() (bool, error) { - var plansFilename *string - if d.PlanStorage != nil { - var err error - plansFilename, err = d.PlanStorage.RetrievePlan(d.localPlanFilePath(), d.storedPlanFilePath()) - if err != nil { - return false, fmt.Errorf("error retrieving plan: %v", err) - } - } - - locked, err := d.ProjectLock.Lock() - - if err != nil { - return false, fmt.Errorf("error locking project: %v", err) - } - - if locked { - var applySteps []configuration.Step - - if d.ApplyStage != nil { - applySteps = d.ApplyStage.Steps - } else { - applySteps = []configuration.Step{ - { - Action: "init", - }, - { - Action: "apply", - }, - } - } - - for _, step := range applySteps { - if step.Action == "init" { - _, _, err := d.TerraformExecutor.Init(step.ExtraArgs, d.StateEnvVars) - if err != nil { - return false, fmt.Errorf("error running init: %v", err) - } - } - if step.Action == "apply" { - stdout, stderr, err := d.TerraformExecutor.Apply(step.ExtraArgs, plansFilename, d.CommandEnvVars) - applyOutput := cleanupTerraformApply(true, err, stdout, stderr) - comment := utils.GetTerraformOutputAsCollapsibleComment("Apply for **"+d.ProjectLock.LockId()+"**", applyOutput) - commentErr := d.Reporter.Report(comment) - if commentErr != nil { - fmt.Printf("error publishing comment: %v", err) - } - if err != nil { - commentErr = d.Reporter.Report("Error during applying.") - if commentErr != nil { - fmt.Printf("error publishing comment: %v", err) - } - return false, fmt.Errorf("error executing apply: %v", err) - } - } - if step.Action == "run" { - var commands []string - if os.Getenv("ACTIVATE_VENV") == "true" { - commands = append(commands, fmt.Sprintf("source %v/.venv/bin/activate", os.Getenv("GITHUB_WORKSPACE"))) - } - commands = append(commands, step.Value) - log.Printf("Running %v for **%v**\n", step.Value, d.ProjectLock.LockId()) - _, _, err := d.CommandRunner.Run(d.ProjectPath, step.Shell, commands) - if err != nil { - return false, fmt.Errorf("error running command: %v", err) - } - } - } - return true, nil - } else { - return false, nil - } -} - -func (d DiggerExecutor) Unlock() error { - err := d.ProjectLock.ForceUnlock() - if err != nil { - return fmt.Errorf("failed to aquire lock: %s, %v", d.ProjectLock.LockId(), err) - } - if d.PlanStorage != nil { - err = d.PlanStorage.DeleteStoredPlan(d.storedPlanFilePath()) - if err != nil { - return fmt.Errorf("failed to delete stored plan file '%v': %v", d.storedPlanFilePath(), err) - } - } - if err != nil { - return fmt.Errorf("failed to delete stored plan file '%v': %v", d.storedPlanFilePath(), err) - } - return nil -} - -func (d DiggerExecutor) Lock() error { - _, err := d.ProjectLock.Lock() - if err != nil { - return fmt.Errorf("failed to aquire lock: %s, %v", d.ProjectLock.LockId(), err) - } - return nil -} - -func cleanupTerraformOutput(nonEmptyOutput bool, planError error, stdout string, stderr string, regexStr *string) string { - var errorStr, start string - - // removes output of terraform -version command that terraform-exec executes on every run - i := strings.Index(stdout, "Initializing the backend...") - if i != -1 { - stdout = stdout[i:] - } - endPos := len(stdout) - - if planError != nil { - if stderr != "" { - errorStr = stderr - } else if stdout != "" { - errorStr = stdout - } - return errorStr - } else if nonEmptyOutput { - start = "Terraform will perform the following actions:" - } else { - start = "No changes. Your infrastructure matches the configuration." - } - - startPos := strings.Index(stdout, start) - if startPos == -1 { - startPos = 0 - } - - if regexStr != nil { - regex := regexp.MustCompile(*regexStr) - matches := regex.FindStringSubmatch(stdout) - if len(matches) > 0 { - firstMatch := matches[0] - endPos = strings.LastIndex(stdout, firstMatch) + len(firstMatch) - } - } - - return stdout[startPos:endPos] -} - -func cleanupTerraformApply(nonEmptyPlan bool, planError error, stdout string, stderr string) string { - return cleanupTerraformOutput(nonEmptyPlan, planError, stdout, stderr, nil) -} - -func cleanupTerraformPlan(nonEmptyPlan bool, planError error, stdout string, stderr string) string { - regex := `───────────.+` - return cleanupTerraformOutput(nonEmptyPlan, planError, stdout, stderr, ®ex) -} - -type CIName string - -const ( - None = CIName("") - GitHub = CIName("github") - GitLab = CIName("gitlab") - BitBucket = CIName("bitbucket") - Azure = CIName("azure") -) - -func (ci CIName) String() string { - return string(ci) -} - -func DetectCI() CIName { - - notEmpty := func(key string) bool { - return os.Getenv(key) != "" - } - - if notEmpty("GITHUB_ACTIONS") { - return GitHub - } - if notEmpty("GITLAB_CI") { - return GitLab - } - if notEmpty("BITBUCKET_BUILD_NUMBER") { - return BitBucket - } - if notEmpty("AZURE_CI") { - return Azure - } - return None - -} diff --git a/pkg/digger/digger_test.go b/pkg/digger/digger_test.go index f7b86e24f..92dbe88f1 100644 --- a/pkg/digger/digger_test.go +++ b/pkg/digger/digger_test.go @@ -1,7 +1,8 @@ package digger import ( - "digger/pkg/configuration" + "digger/pkg/core/execution" + "digger/pkg/core/models" "digger/pkg/reporting" "digger/pkg/utils" "sort" @@ -162,9 +163,9 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) { CiService: prManager, PrNumber: 1, } - executor := DiggerExecutor{ - ApplyStage: &configuration.Stage{ - Steps: []configuration.Step{ + executor := execution.DiggerExecutor{ + ApplyStage: &models.Stage{ + Steps: []models.Step{ { Action: "init", ExtraArgs: nil, @@ -182,7 +183,7 @@ func TestCorrectCommandExecutionWhenApplying(t *testing.T) { }, }, }, - PlanStage: &configuration.Stage{}, + PlanStage: &models.Stage{}, CommandRunner: commandRunner, TerraformExecutor: terraformExecutor, Reporter: reporter, @@ -207,10 +208,10 @@ func TestCorrectCommandExecutionWhenPlanning(t *testing.T) { CiService: prManager, PrNumber: 1, } - executor := DiggerExecutor{ - ApplyStage: &configuration.Stage{}, - PlanStage: &configuration.Stage{ - Steps: []configuration.Step{ + executor := execution.DiggerExecutor{ + ApplyStage: &models.Stage{}, + PlanStage: &models.Stage{ + Steps: []models.Step{ { Action: "init", ExtraArgs: nil, diff --git a/pkg/github/github.go b/pkg/github/github.go index cffcd991c..0aeac2aaf 100644 --- a/pkg/github/github.go +++ b/pkg/github/github.go @@ -4,8 +4,8 @@ import ( "context" "digger/pkg/ci" "digger/pkg/configuration" + dg_models "digger/pkg/core/models" "digger/pkg/github/models" - dg_models "digger/pkg/models" "digger/pkg/utils" "encoding/json" "fmt" @@ -145,7 +145,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - + coreApplyStage := workflow.Apply.ToCoreStage() + corePlanStage := workflow.Plan.ToCoreStage() if event.Action == "closed" && event.PullRequest.Merged && event.PullRequest.Base.Ref == event.Repository.DefaultBranch { commandsPerProject = append(commandsPerProject, dg_models.ProjectCommand{ ProjectName: project.Name, @@ -153,8 +154,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnCommitToDefault, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -165,8 +166,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -177,8 +178,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -211,7 +212,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) - + coreApplyStage := workflow.Apply.ToCoreStage() + corePlanStage := workflow.Plan.ToCoreStage() workspace := project.Workspace workspaceOverride, err := utils.ParseWorkspace(event.Comment.Body) if err != nil { @@ -226,8 +228,8 @@ func ConvertGithubEventToCommands(event models.Event, impactedProjects []configu ProjectWorkspace: workspace, Terragrunt: project.Terragrunt, Commands: []string{command}, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index b204c81b2..1bdedb261 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -2,7 +2,7 @@ package gitlab import ( "digger/pkg/configuration" - "digger/pkg/models" + "digger/pkg/core/models" "digger/pkg/utils" "fmt" "github.com/caarlos0/env/v7" @@ -262,14 +262,16 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + coreApplyStage := workflow.Apply.ToCoreStage() + corePlanStage := workflow.Plan.ToCoreStage() commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestPushed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -282,14 +284,22 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + var coreApplyStage models.Stage + if workflow.Apply != nil { + coreApplyStage = workflow.Apply.ToCoreStage() + } + var corePlanStage models.Stage + if workflow.Plan != nil { + corePlanStage = workflow.Plan.ToCoreStage() + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: project.Workspace, Terragrunt: project.Terragrunt, Commands: workflow.Configuration.OnPullRequestClosed, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) @@ -327,14 +337,22 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex workspace = workspaceOverride } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) + var coreApplyStage models.Stage + if workflow.Apply != nil { + coreApplyStage = workflow.Apply.ToCoreStage() + } + var corePlanStage models.Stage + if workflow.Plan != nil { + corePlanStage = workflow.Plan.ToCoreStage() + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: workspace, Terragrunt: project.Terragrunt, Commands: []string{command}, - ApplyStage: workflow.Apply, - PlanStage: workflow.Plan, + ApplyStage: &coreApplyStage, + PlanStage: &corePlanStage, CommandEnvVars: commandEnvVars, StateEnvVars: stateEnvVars, }) diff --git a/pkg/integration/integration_test.go b/pkg/integration/integration_test.go index cedc96a46..9455214a7 100644 --- a/pkg/integration/integration_test.go +++ b/pkg/integration/integration_test.go @@ -4,12 +4,12 @@ import ( "context" "digger/pkg/aws" "digger/pkg/configuration" + "digger/pkg/core/terraform" "digger/pkg/digger" dg_github "digger/pkg/github" "digger/pkg/locking" "digger/pkg/reporting" "digger/pkg/storage" - "digger/pkg/terraform" "digger/pkg/utils" "log" "math/rand" diff --git a/pkg/locking/locking.go b/pkg/locking/locking.go index 6007b4eec..dc29a12fb 100644 --- a/pkg/locking/locking.go +++ b/pkg/locking/locking.go @@ -5,6 +5,7 @@ import ( "digger/pkg/aws/envprovider" "digger/pkg/azure" "digger/pkg/ci" + "digger/pkg/core/locking" "digger/pkg/gcp" "errors" "fmt" @@ -23,19 +24,13 @@ import ( ) type PullRequestLock struct { - InternalLock Lock + InternalLock locking.Lock CIService ci.CIService ProjectName string ProjectNamespace string PrNumber int } -type Lock interface { - Lock(transactionId int, resource string) (bool, error) - Unlock(resource string) (bool, error) - GetLock(resource string) (*int, error) -} - type NoOpLock struct { } @@ -51,13 +46,6 @@ func (noOpLock *NoOpLock) GetLock(resource string) (*int, error) { return nil, nil } -type ProjectLock interface { - Lock() (bool, error) - Unlock() (bool, error) - ForceUnlock() error - LockId() string -} - func (projectLock *PullRequestLock) Lock() (bool, error) { lockId := projectLock.LockId() fmt.Printf("Lock %s\n", lockId) @@ -191,7 +179,7 @@ func (projectLock *PullRequestLock) LockId() string { return projectLock.ProjectNamespace + "#" + projectLock.ProjectName } -func GetLock() (Lock, error) { +func GetLock() (locking.Lock, error) { awsRegion := strings.ToLower(os.Getenv("AWS_REGION")) awsProfile := strings.ToLower(os.Getenv("AWS_PROFILE")) lockProvider := strings.ToLower(os.Getenv("LOCK_PROVIDER")) diff --git a/pkg/reporting/reporting.go b/pkg/reporting/reporting.go index 3544af763..4788ab015 100644 --- a/pkg/reporting/reporting.go +++ b/pkg/reporting/reporting.go @@ -2,10 +2,6 @@ package reporting import "digger/pkg/ci" -type Reporter interface { - Report(report string) error -} - type CiReporter struct { CiService ci.CIService PrNumber int diff --git a/pkg/storage/plan_storage.go b/pkg/storage/plan_storage.go index 9bf216a30..987611d59 100644 --- a/pkg/storage/plan_storage.go +++ b/pkg/storage/plan_storage.go @@ -15,13 +15,6 @@ import ( "github.com/google/go-github/v51/github" ) -type PlanStorage interface { - StorePlan(localPlanFilePath string, storedPlanFilePath string) error - RetrievePlan(localPlanFilePath string, storedPlanFilePath string) (*string, error) - DeleteStoredPlan(storedPlanFilePath string) error - PlanExists(storedPlanFilePath string) (bool, error) -} - type PlanStorageGcp struct { Client *storage.Client Bucket *storage.BucketHandle