From b2523e94e31a2109f33b175d2f3c5b3910bdced2 Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 5 Jun 2023 15:44:02 +0100 Subject: [PATCH 1/4] refactor gitlab path to call shared code --- cmd/digger/main.go | 2 +- pkg/gitlab/gitlab.go | 120 +++---------------------------------------- 2 files changed, 9 insertions(+), 113 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index b5b6e7610..cb4f3f5d4 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -180,7 +180,7 @@ func gitLabCI(lock locking.Lock) { //planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, prNumber) planStorage := newPlanStorage(gitlabToken, projectNamespace, projectName, *gitLabContext.MergeRequestIId) - allAppliesSuccess, err := gitlab.RunCommandsPerProject(commandsToRunPerProject, *gitLabContext, diggerConfig, gitlabService, lock, planStorage, currentDir) + allAppliesSuccess, _, err := digger.RunCommandsPerProject(commandsToRunPerProject, gitLabContext.ProjectNamespace, gitLabContext.ProjectName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, planStorage, currentDir) if err != nil { fmt.Printf("failed to execute command, %v", err) os.Exit(8) diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index f9e3b5bef..35c0c28f1 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -1,19 +1,13 @@ package gitlab import ( - "digger/pkg/ci" "digger/pkg/configuration" - "digger/pkg/digger" - "digger/pkg/locking" "digger/pkg/models" - "digger/pkg/storage" - "digger/pkg/terraform" - "digger/pkg/usage" + "digger/pkg/utils" "fmt" "github.com/caarlos0/env/v7" go_gitlab "github.com/xanzy/go-gitlab" "log" - "path" "strings" ) @@ -290,13 +284,13 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex if strings.Contains(gitLabContext.DiggerCommand, command) { for _, project := range impactedProjects { workspace := project.Workspace - //workspaceOverride, err := parseWorkspace(gitLabContext.DiggerCommand) - //if err != nil { - // return []digger.ProjectCommand{}, err - //} - //if workspaceOverride != "" { - // workspace = workspaceOverride - //} + workspaceOverride, err := utils.ParseWorkspace(gitLabContext.DiggerCommand) + if err != nil { + return []models.ProjectCommand{}, err + } + if workspaceOverride != "" { + workspace = workspaceOverride + } commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -312,102 +306,4 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex default: return []models.ProjectCommand{}, fmt.Errorf("unsupported GitLab event type: %v", event) } - return nil, nil -} - -func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, gitLabContext GitLabContext, diggerConfig *configuration.DiggerConfig, service ci.CIService, lock locking.Lock, planStorage storage.PlanStorage, workingDir string) (bool, error) { - - allAppliesSuccess := true - appliesPerProject := make(map[string]bool) - for _, projectCommands := range commandsPerProject { - appliesPerProject[projectCommands.ProjectName] = false - for _, command := range projectCommands.Commands { - projectLock := &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: service, - ProjectName: projectCommands.ProjectName, - RepoName: gitLabContext.ProjectName, - RepoOwner: gitLabContext.ProjectNamespace, - } - - var terraformExecutor terraform.TerraformExecutor - projectPath := path.Join(workingDir, projectCommands.ProjectDir) - if projectCommands.Terragrunt { - terraformExecutor = terraform.Terragrunt{WorkingDir: path.Join(workingDir, projectCommands.ProjectDir)} - } else { - terraformExecutor = terraform.Terraform{WorkingDir: path.Join(workingDir, projectCommands.ProjectDir), Workspace: projectCommands.ProjectWorkspace} - } - commandRunner := digger.CommandRunner{} - - diggerExecutor := digger.DiggerExecutor{ - gitLabContext.ProjectNamespace, - gitLabContext.ProjectName, - projectCommands.ProjectName, - projectPath, - projectCommands.StateEnvVars, - projectCommands.CommandEnvVars, - projectCommands.ApplyStage, - projectCommands.PlanStage, - commandRunner, - terraformExecutor, - service, - projectLock, - planStorage, - } - - repoOwner := gitLabContext.ProjectNamespace - repoName := gitLabContext.ProjectName - prNumber := *gitLabContext.MergeRequestIId - eventName := "" - switch command { - case "digger plan": - usage.SendUsageRecord(repoOwner, eventName, "plan") - service.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/plan") - planPerformed, err := diggerExecutor.Plan(prNumber) - if err != nil { - log.Printf("Failed to run digger plan command. %v", err) - service.SetStatus(prNumber, "failure", projectCommands.ProjectName+"/plan") - - return false, fmt.Errorf("failed to run digger plan command. %v", err) - } else if !planPerformed { - service.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/plan") - } else { - service.SetStatus(prNumber, "success", projectCommands.ProjectName+"/plan") - } - case "digger apply": - usage.SendUsageRecord(repoName, eventName, "apply") - service.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/apply") - applyPerformed, err := diggerExecutor.Apply(prNumber) - if err != nil { - log.Printf("Failed to run digger apply command. %v", err) - service.SetStatus(prNumber, "failure", projectCommands.ProjectName+"/apply") - return false, fmt.Errorf("failed to run digger apply command. %v", err) - } else if !applyPerformed { - service.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/apply") - } else { - service.SetStatus(prNumber, "success", projectCommands.ProjectName+"/apply") - appliesPerProject[projectCommands.ProjectName] = true - } - case "digger unlock": - usage.SendUsageRecord(repoOwner, eventName, "unlock") - err := diggerExecutor.Unlock(prNumber) - if err != nil { - return false, fmt.Errorf("failed to unlock project. %v", err) - } - case "digger lock": - usage.SendUsageRecord(repoOwner, eventName, "lock") - err := diggerExecutor.Lock(prNumber) - if err != nil { - return false, fmt.Errorf("failed to lock project. %v", err) - } - } - } - } - - for _, success := range appliesPerProject { - if !success { - allAppliesSuccess = false - } - } - return allAppliesSuccess, nil } From 201cb73fa5ede012982aa0d4cec4c27efe2c00ce Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Mon, 5 Jun 2023 16:16:47 +0100 Subject: [PATCH 2/4] make gitlab path flow --- cmd/digger/main.go | 8 ++--- pkg/gitlab/gitlab.go | 70 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 59 insertions(+), 19 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index cb4f3f5d4..232cb2e26 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -158,14 +158,14 @@ func gitLabCI(lock locking.Lock) { gitlabEvent := gitlab.GitLabEvent{EventType: gitLabContext.EventType} - impactedProjects, err := gitlab.ProcessGitLabEvent(gitLabContext, diggerConfig, gitlabService) + impactedProjects, requestedProject, err := gitlab.ProcessGitLabEvent(gitLabContext, diggerConfig, gitlabService) if err != nil { fmt.Printf("failed to process GitLab event, %v", err) os.Exit(6) } println("GitLab event processed successfully") - commandsToRunPerProject, err := gitlab.ConvertGitLabEventToCommands(gitlabEvent, gitLabContext, impactedProjects, diggerConfig.Workflows) + commandsToRunPerProject, coversAllImpactedProjects, err := gitlab.ConvertGitLabEventToCommands(gitlabEvent, gitLabContext, impactedProjects, requestedProject, diggerConfig.Workflows) if err != nil { fmt.Printf("failed to convert event to command, %v", err) os.Exit(7) @@ -180,13 +180,13 @@ func gitLabCI(lock locking.Lock) { //planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, prNumber) planStorage := newPlanStorage(gitlabToken, projectNamespace, projectName, *gitLabContext.MergeRequestIId) - allAppliesSuccess, _, err := digger.RunCommandsPerProject(commandsToRunPerProject, gitLabContext.ProjectNamespace, gitLabContext.ProjectName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, planStorage, currentDir) + allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, gitLabContext.ProjectNamespace, gitLabContext.ProjectName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, planStorage, currentDir) if err != nil { fmt.Printf("failed to execute command, %v", err) os.Exit(8) } - if diggerConfig.AutoMerge && allAppliesSuccess { + if diggerConfig.AutoMerge && atLeastOneApply && allAppliesSuccess && coversAllImpactedProjects { digger.MergePullRequest(gitlabService, *gitLabContext.MergeRequestIId) println("Merge request changes has been applied successfully") } diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index 35c0c28f1..f243a34f1 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -82,28 +82,40 @@ func NewGitLabService(token string, gitLabContext *GitLabContext) (*GitLabServic }, nil } -func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.Project, error) { +func ProcessGitLabEvent(gitlabContext *GitLabContext, diggerConfig *configuration.DiggerConfig, service *GitLabService) ([]configuration.Project, *configuration.Project, error) { var impactedProjects []configuration.Project if gitlabContext.MergeRequestIId == nil { - return nil, fmt.Errorf("value for 'Merge Request ID' parameter is not found") + return nil, nil, fmt.Errorf("value for 'Merge Request ID' parameter is not found") } mergeRequestId := gitlabContext.MergeRequestIId changedFiles, err := service.GetChangedFiles(*mergeRequestId) if err != nil { - return nil, fmt.Errorf("could not get changed files") + return nil, nil, fmt.Errorf("could not get changed files") } impactedProjects = diggerConfig.GetModifiedProjects(changedFiles) - fmt.Println("Impacted projects:") - for _, v := range impactedProjects { - fmt.Printf("%s\n", v.Name) - } + switch gitlabContext.EventType { + case MergeRequestComment: + requestedProject := utils.ParseProjectName(gitlabContext.DiggerCommand) + + if requestedProject == "" { + return impactedProjects, nil, nil + } + + for _, project := range impactedProjects { + if project.Name == requestedProject { + return impactedProjects, &project, nil + } + } + return nil, nil, fmt.Errorf("requested project not found in modified projects") + default: + return impactedProjects, nil, nil - return impactedProjects, nil + } } type GitLabService struct { @@ -236,7 +248,7 @@ const ( MergeRequestComment = GitLabEventType("merge_request_commented") ) -func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, error) { +func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContext, impactedProjects []configuration.Project, requestedProject *configuration.Project, workflows map[string]configuration.Workflow) ([]models.ProjectCommand, bool, error) { commandsPerProject := make([]models.ProjectCommand, 0) fmt.Printf("ConvertGitLabEventToCommands, event.EventType: %s\n", event.EventType) @@ -249,6 +261,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex workflow = workflows["default"] } + stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -257,15 +270,18 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex Commands: workflow.Configuration.OnPullRequestPushed, ApplyStage: workflow.Apply, PlanStage: workflow.Plan, + CommandEnvVars: commandEnvVars, + StateEnvVars: stateEnvVars, }) } - return commandsPerProject, nil + return commandsPerProject, true, nil case MergeRequestClosed, MergeRequestMerged: for _, project := range impactedProjects { workflow, ok := workflows[project.Workflow] if !ok { workflow = workflows["default"] } + stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, @@ -274,36 +290,60 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex Commands: workflow.Configuration.OnPullRequestClosed, ApplyStage: workflow.Apply, PlanStage: workflow.Plan, + CommandEnvVars: commandEnvVars, + StateEnvVars: stateEnvVars, }) } - return commandsPerProject, nil + return commandsPerProject, true, nil case MergeRequestComment: supportedCommands := []string{"digger plan", "digger apply", "digger unlock", "digger lock"} + coversAllImpactedProjects := true + + runForProjects := impactedProjects + + if requestedProject != nil { + if len(impactedProjects) > 1 { + coversAllImpactedProjects = false + runForProjects = []configuration.Project{*requestedProject} + } else if len(impactedProjects) == 1 && impactedProjects[0].Name != requestedProject.Name { + return commandsPerProject, false, fmt.Errorf("requested project %v is not impacted by this PR", requestedProject.Name) + } + } + for _, command := range supportedCommands { if strings.Contains(gitLabContext.DiggerCommand, command) { - for _, project := range impactedProjects { + for _, project := range runForProjects { + workflow, ok := workflows[project.Workflow] + if !ok { + workflow = workflows["default"] + } workspace := project.Workspace workspaceOverride, err := utils.ParseWorkspace(gitLabContext.DiggerCommand) if err != nil { - return []models.ProjectCommand{}, err + return []models.ProjectCommand{}, false, err } if workspaceOverride != "" { workspace = workspaceOverride } + stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{ ProjectName: project.Name, ProjectDir: project.Dir, ProjectWorkspace: workspace, Terragrunt: project.Terragrunt, Commands: []string{command}, + ApplyStage: workflow.Apply, + PlanStage: workflow.Plan, + CommandEnvVars: commandEnvVars, + StateEnvVars: stateEnvVars, }) } } } - return commandsPerProject, nil + return commandsPerProject, coversAllImpactedProjects, nil default: - return []models.ProjectCommand{}, fmt.Errorf("unsupported GitLab event type: %v", event) + return []models.ProjectCommand{}, false, fmt.Errorf("unsupported GitLab event type: %v", event) } } From 38ae53789cc2b9f042e6844181d204fc90c0e39d Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Tue, 6 Jun 2023 11:35:15 +0100 Subject: [PATCH 3/4] refactor RepoOwner and RepoName into ProjectNamespace to get rid off references to specific git hostings --- cmd/digger/main.go | 51 ++++++------ pkg/digger/digger.go | 29 +++---- pkg/gitlab/gitlab.go | 1 + pkg/integration/integration_test.go | 119 +++++++++++++--------------- pkg/locking/locking.go | 15 ++-- pkg/locking/locking_test.go | 9 +-- 6 files changed, 108 insertions(+), 116 deletions(-) diff --git a/cmd/digger/main.go b/cmd/digger/main.go index 232cb2e26..e2e740fda 100644 --- a/cmd/digger/main.go +++ b/cmd/digger/main.go @@ -23,26 +23,26 @@ import ( func gitHubCI(lock locking.Lock) { println("Using GitHub.") - githubRepositoryOwner := os.Getenv("GITHUB_REPOSITORY_OWNER") - if githubRepositoryOwner != "" { - usage.SendUsageRecord(githubRepositoryOwner, "log", "initialize") + githubActor := os.Getenv("GITHUB_ACTOR") + if githubActor != "" { + usage.SendUsageRecord(githubActor, "log", "initialize") } else { usage.SendUsageRecord("", "log", "non github initialisation") } ghToken := os.Getenv("GITHUB_TOKEN") if ghToken == "" { - reportErrorAndExit(githubRepositoryOwner, "GITHUB_TOKEN is not defined", 1) + reportErrorAndExit(githubActor, "GITHUB_TOKEN is not defined", 1) } ghContext := os.Getenv("GITHUB_CONTEXT") if ghContext == "" { - reportErrorAndExit(githubRepositoryOwner, "GITHUB_CONTEXT is not defined", 2) + reportErrorAndExit(githubActor, "GITHUB_CONTEXT is not defined", 2) } parsedGhContext, err := github_models.GetGitHubContext(ghContext) if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to parse GitHub context. %s", err), 3) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to parse GitHub context. %s", err), 3) } println("GitHub context parsed successfully") @@ -50,13 +50,13 @@ func gitHubCI(lock locking.Lock) { diggerConfig, err := configuration.NewDiggerConfig("./", &walker) if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to read Digger config. %s", err), 4) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to read Digger config. %s", err), 4) } println("Digger config read successfully") lock, err = locking.GetLock() if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to create lock provider. %s", err), 5) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to create lock provider. %s", err), 5) } println("Lock provider has been created successfully") @@ -68,7 +68,7 @@ func gitHubCI(lock locking.Lock) { impactedProjects, requestedProject, prNumber, err := dg_github.ProcessGitHubEvent(ghEvent, diggerConfig, githubPrService) if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to process GitHub event. %s", err), 6) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to process GitHub event. %s", err), 6) } logImpactedProjects(impactedProjects, prNumber) println("GitHub event processed successfully") @@ -77,26 +77,26 @@ func gitHubCI(lock locking.Lock) { reply := utils.GetCommands() err := githubPrService.PublishComment(prNumber, reply) if err != nil { - reportErrorAndExit(githubRepositoryOwner, "Failed to publish help command output", 1) + reportErrorAndExit(githubActor, "Failed to publish help command output", 1) } } if len(impactedProjects) == 0 { - reportErrorAndExit(githubRepositoryOwner, "No projects impacted", 0) + reportErrorAndExit(githubActor, "No projects impacted", 0) } commandsToRunPerProject, coversAllImpactedProjects, err := dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to convert GitHub event to commands. %s", err), 7) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to convert GitHub event to commands. %s", err), 7) } println("GitHub event converted to commands successfully") logCommands(commandsToRunPerProject) - planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, prNumber) + planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, githubActor, prNumber) - allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, "") + allAppliesSuccessful, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, parsedGhContext.Repository, githubActor, eventName, prNumber, githubPrService, lock, planStorage, "") if err != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Failed to run commands. %s", err), 8) + reportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8) } if diggerConfig.AutoMerge && allAppliesSuccessful && atLeastOneApply && coversAllImpactedProjects { @@ -106,11 +106,11 @@ func gitHubCI(lock locking.Lock) { println("Commands executed successfully") - reportErrorAndExit(githubRepositoryOwner, "Digger finished successfully", 0) + reportErrorAndExit(githubActor, "Digger finished successfully", 0) defer func() { if r := recover(); r != nil { - reportErrorAndExit(githubRepositoryOwner, fmt.Sprintf("Panic occurred. %s", r), 1) + reportErrorAndExit(githubActor, fmt.Sprintf("Panic occurred. %s", r), 1) } }() } @@ -177,10 +177,10 @@ func gitLabCI(lock locking.Lock) { fmt.Printf("command: %s, project: %s\n", strings.Join(v.Commands, ", "), v.ProjectName) } - //planStorage := newPlanStorage(ghToken, repoOwner, repositoryName, prNumber) - planStorage := newPlanStorage(gitlabToken, projectNamespace, projectName, *gitLabContext.MergeRequestIId) + diggerProjectNamespace := gitLabContext.ProjectNamespace + "/" + gitLabContext.ProjectName + planStorage := newPlanStorage("", "", "", gitLabContext.GitlabUserName, *gitLabContext.MergeRequestIId) - allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, gitLabContext.ProjectNamespace, gitLabContext.ProjectName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, planStorage, currentDir) + allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, gitLabContext.GitlabUserName, gitLabContext.EventType.String(), *gitLabContext.MergeRequestIId, gitlabService, lock, planStorage, currentDir) if err != nil { fmt.Printf("failed to execute command, %v", err) os.Exit(8) @@ -251,8 +251,9 @@ func azureCI(lock locking.Lock) { } var planStorage storage.PlanStorage + diggerProjectNamespace := parsedAzureContext.BaseUrl + "/" + parsedAzureContext.ProjectName - allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, parsedAzureContext.ProjectName, parsedAzureContext.ProjectName, parsedAzureContext.EventType, prNumber, azureService, lock, planStorage, currentDir) + allAppliesSuccess, atLeastOneApply, err := digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, parsedAzureContext.BaseUrl, parsedAzureContext.EventType, prNumber, azureService, lock, planStorage, currentDir) if err != nil { reportErrorAndExit(parsedAzureContext.BaseUrl, fmt.Sprintf("Failed to run commands. %s", err), 8) } @@ -320,7 +321,7 @@ func main() { } } -func newPlanStorage(ghToken string, repoOwner string, repositoryName string, prNumber int) storage.PlanStorage { +func newPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber int) storage.PlanStorage { var planStorage storage.PlanStorage uploadDestination := strings.ToLower(os.Getenv("PLAN_UPLOAD_DESTINATION")) @@ -328,8 +329,8 @@ func newPlanStorage(ghToken string, repoOwner string, repositoryName string, prN zipManager := utils.Zipper{} planStorage = &storage.GithubPlanStorage{ Client: github.NewTokenClient(context.Background(), ghToken), - Owner: repoOwner, - RepoName: repositoryName, + Owner: ghRepoOwner, + RepoName: ghRepositoryName, PullRequestNumber: prNumber, ZipManager: zipManager, } @@ -337,7 +338,7 @@ func newPlanStorage(ghToken string, repoOwner string, repositoryName string, prN ctx, client := gcp.GetGoogleStorageClient() bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_BUCKET")) if bucketName == "" { - reportErrorAndExit(repoOwner, fmt.Sprintf("GOOGLE_STORAGE_BUCKET is not defined"), 9) + reportErrorAndExit(requestedBy, fmt.Sprintf("GOOGLE_STORAGE_BUCKET is not defined"), 9) } bucket := client.Bucket(bucketName) planStorage = &storage.PlanStorageGcp{ diff --git a/pkg/digger/digger.go b/pkg/digger/digger.go index 237d50048..d3986f679 100644 --- a/pkg/digger/digger.go +++ b/pkg/digger/digger.go @@ -22,16 +22,15 @@ import ( "time" ) -func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, repoOwner string, repoName string, eventName string, prNumber int, ciService ci.CIService, lock locking.Lock, planStorage storage.PlanStorage, workingDir string) (bool, bool, error) { +func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, projectNamespace string, requestedBy string, eventName string, prNumber int, ciService ci.CIService, lock locking.Lock, planStorage storage.PlanStorage, workingDir string) (bool, bool, error) { appliesPerProject := make(map[string]bool) for _, projectCommands := range commandsPerProject { for _, command := range projectCommands.Commands { projectLock := &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: ciService, - ProjectName: projectCommands.ProjectName, - RepoName: repoName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: ciService, + ProjectName: projectCommands.ProjectName, + ProjectNamespace: projectNamespace, } var terraformExecutor terraform.TerraformExecutor @@ -44,8 +43,7 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, repoOwner commandRunner := CommandRunner{} diggerExecutor := DiggerExecutor{ - repoOwner, - repoName, + projectNamespace, projectCommands.ProjectName, projectPath, projectCommands.StateEnvVars, @@ -60,7 +58,7 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, repoOwner } switch command { case "digger plan": - usage.SendUsageRecord(repoOwner, eventName, "plan") + usage.SendUsageRecord(requestedBy, eventName, "plan") ciService.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/plan") planPerformed, err := diggerExecutor.Plan(prNumber) if err != nil { @@ -72,7 +70,7 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, repoOwner } case "digger apply": appliesPerProject[projectCommands.ProjectName] = false - usage.SendUsageRecord(repoName, eventName, "apply") + usage.SendUsageRecord(requestedBy, eventName, "apply") ciService.SetStatus(prNumber, "pending", projectCommands.ProjectName+"/apply") applyPerformed, err := diggerExecutor.Apply(prNumber) if err != nil { @@ -84,13 +82,13 @@ func RunCommandsPerProject(commandsPerProject []models.ProjectCommand, repoOwner appliesPerProject[projectCommands.ProjectName] = true } case "digger unlock": - usage.SendUsageRecord(repoOwner, eventName, "unlock") + usage.SendUsageRecord(requestedBy, eventName, "unlock") err := diggerExecutor.Unlock(prNumber) if err != nil { return false, false, fmt.Errorf("failed to unlock project. %v", err) } case "digger lock": - usage.SendUsageRecord(repoOwner, eventName, "lock") + usage.SendUsageRecord(requestedBy, eventName, "lock") err := diggerExecutor.Lock(prNumber) if err != nil { return false, false, fmt.Errorf("failed to lock project. %v", err) @@ -140,8 +138,7 @@ func MergePullRequest(githubPrService ci.CIService, prNumber int) { } type DiggerExecutor struct { - RepoOwner string - RepoName string + ProjectNamespace string ProjectName string ProjectPath string StateEnvVars map[string]string @@ -201,7 +198,7 @@ func (c CommandRunner) Run(workingDir string, shell string, commands []string) ( } func (d DiggerExecutor) planFileName() string { - return d.RepoName + "#" + d.ProjectName + ".tfplan" + return d.ProjectNamespace + "#" + d.ProjectName + ".tfplan" } func (d DiggerExecutor) localPlanFilePath() string { @@ -209,7 +206,7 @@ func (d DiggerExecutor) localPlanFilePath() string { } func (d DiggerExecutor) storedPlanFilePath() string { - return path.Join(d.RepoOwner, d.planFileName()) + return path.Join(d.ProjectNamespace, d.planFileName()) } func (d DiggerExecutor) Plan(prNumber int) (bool, error) { diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index f243a34f1..cf3e60b55 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -28,6 +28,7 @@ type GitLabContext struct { ProjectNamespaceId *int `env:"CI_PROJECT_NAMESPACE_ID"` OpenMergeRequests []string `env:"CI_OPEN_MERGE_REQUESTS"` Token string `env:"GITLAB_TOKEN"` + GitlabUserName string `env:"GITLAB_USER_NAME"` DiggerCommand string `env:"DIGGER_COMMAND"` DiscussionID string `env:"DISCUSSION_ID"` IsMeargeable bool `env:"IS_MERGEABLE"` diff --git a/pkg/integration/integration_test.go b/pkg/integration/integration_test.go index 9acb364d4..a2386da02 100644 --- a/pkg/integration/integration_test.go +++ b/pkg/integration/integration_test.go @@ -45,11 +45,10 @@ func getProjectLockForTests() (error, *locking.ProjectLockImpl) { githubPrService := dg_github.NewGitHubService(ghToken, repositoryName, repoOwner) projectLock := &locking.ProjectLockImpl{ - InternalLock: &dynamoDbLock, - CIService: githubPrService, - ProjectName: "test_dynamodb_lock", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: &dynamoDbLock, + CIService: githubPrService, + ProjectName: "test_dynamodb_lock", + ProjectNamespace: repoOwner + "/" + repositoryName, } return err, projectLock } @@ -400,15 +399,17 @@ func TestHappyPath(t *testing.T) { PullRequestNumber: prNumber, ZipManager: zipManager, } - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + + diggerProjectNamespace := repoOwner + "/" + repositoryName + + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock := &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } resource := repositoryName + "#" + projectLock.ProjectName transactionId, err := projectLock.InternalLock.GetLock(resource) @@ -427,7 +428,7 @@ func TestHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) println("--- digger apply comment ---") @@ -441,15 +442,14 @@ func TestHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) @@ -465,15 +465,14 @@ func TestHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) @@ -534,6 +533,7 @@ func TestMultiEnvHappyPath(t *testing.T) { eventName := parsedNewPullRequestContext.EventName repoOwner := parsedNewPullRequestContext.RepositoryOwner repositoryName := parsedNewPullRequestContext.Repository + diggerProjectNamespace := repoOwner + "/" + repositoryName githubPrService := dg_github.NewGitHubService(ghToken, repositoryName, repoOwner) assert.Equal(t, "pull_request", parsedNewPullRequestContext.EventName) @@ -552,15 +552,14 @@ func TestMultiEnvHappyPath(t *testing.T) { PullRequestNumber: prNumber, ZipManager: zipManager, } - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) assert.NoError(t, err) projectLock := &locking.ProjectLockImpl{ - InternalLock: &dynamoDbLock, - CIService: githubPrService, - ProjectName: "digger_demo", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: &dynamoDbLock, + CIService: githubPrService, + ProjectName: "digger_demo", + ProjectNamespace: diggerProjectNamespace, } resource := "digger_demo#default" transactionId, err := projectLock.InternalLock.GetLock(resource) @@ -578,7 +577,7 @@ func TestMultiEnvHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) assert.NoError(t, err) println("--- digger apply comment ---") @@ -592,15 +591,14 @@ func TestMultiEnvHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: &dynamoDbLock, - CIService: githubPrService, - ProjectName: "digger_demo", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: &dynamoDbLock, + CIService: githubPrService, + ProjectName: "digger_demo", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) @@ -616,15 +614,14 @@ func TestMultiEnvHappyPath(t *testing.T) { assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, &dynamoDbLock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: &dynamoDbLock, - CIService: githubPrService, - ProjectName: "digger_demo", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: &dynamoDbLock, + CIService: githubPrService, + ProjectName: "digger_demo", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) @@ -750,6 +747,7 @@ workflows: repoOwner := parsedNewPullRequestContext.RepositoryOwner repositoryName := parsedNewPullRequestContext.Repository githubPrService := dg_github.NewGitHubService(ghToken, repositoryName, repoOwner) + diggerProjectNamespace := repoOwner + "/" + repositoryName assert.Equal(t, "pull_request", parsedNewPullRequestContext.EventName) @@ -767,15 +765,14 @@ workflows: PullRequestNumber: prNumber, ZipManager: zipManager, } - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock := &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } resource := repositoryName + "#" + projectLock.ProjectName transactionId, err := projectLock.InternalLock.GetLock(resource) @@ -794,7 +791,7 @@ workflows: assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) println("--- digger apply comment ---") @@ -808,15 +805,14 @@ workflows: assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) @@ -831,15 +827,14 @@ workflows: assert.NoError(t, err) commandsToRunPerProject, _, err = dg_github.ConvertGithubEventToCommands(ghEvent, impactedProjects, requestedProject, diggerConfig.Workflows) assert.NoError(t, err) - _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, repoOwner, repositoryName, eventName, prNumber, githubPrService, lock, planStorage, dir) + _, _, err = digger.RunCommandsPerProject(commandsToRunPerProject, diggerProjectNamespace, repoOwner, eventName, prNumber, githubPrService, lock, planStorage, dir) assert.NoError(t, err) projectLock = &locking.ProjectLockImpl{ - InternalLock: lock, - CIService: githubPrService, - ProjectName: "dev", - RepoName: repositoryName, - RepoOwner: repoOwner, + InternalLock: lock, + CIService: githubPrService, + ProjectName: "dev", + ProjectNamespace: diggerProjectNamespace, } transactionId, err = projectLock.InternalLock.GetLock(resource) assert.NoError(t, err) diff --git a/pkg/locking/locking.go b/pkg/locking/locking.go index a86101938..bd969e74d 100644 --- a/pkg/locking/locking.go +++ b/pkg/locking/locking.go @@ -23,11 +23,10 @@ import ( ) type ProjectLockImpl struct { - InternalLock Lock - CIService ci.CIService - ProjectName string - RepoName string - RepoOwner string + InternalLock Lock + CIService ci.CIService + ProjectName string + ProjectNamespace string } type Lock interface { @@ -82,7 +81,7 @@ func (projectLock *ProjectLockImpl) Lock(prNumber int) (bool, error) { return true, nil } else { transactionIdStr := strconv.Itoa(*existingLockTransactionId) - comment := "Project " + projectLock.projectId() + " locked by another PR #" + transactionIdStr + " (failed to acquire lock " + projectLock.RepoName + "). The locking plan must be applied or discarded before future plans can execute" + comment := "Project " + projectLock.projectId() + " locked by another PR #" + transactionIdStr + " (failed to acquire lock " + projectLock.ProjectNamespace + "). The locking plan must be applied or discarded before future plans can execute" projectLock.CIService.PublishComment(prNumber, comment) return false, nil } @@ -184,11 +183,11 @@ func (projectLock *ProjectLockImpl) ForceUnlock(prNumber int) error { } func (projectLock *ProjectLockImpl) projectId() string { - return projectLock.RepoOwner + "/" + projectLock.RepoName + "#" + projectLock.ProjectName + return projectLock.ProjectNamespace + "#" + projectLock.ProjectName } func (projectLock *ProjectLockImpl) LockId() string { - return projectLock.RepoOwner + "/" + projectLock.RepoName + "#" + projectLock.ProjectName + return projectLock.ProjectNamespace + "#" + projectLock.ProjectName } func GetLock() (Lock, error) { diff --git a/pkg/locking/locking_test.go b/pkg/locking/locking_test.go index 7655cac6a..255077e1f 100644 --- a/pkg/locking/locking_test.go +++ b/pkg/locking/locking_test.go @@ -10,11 +10,10 @@ func TestLockingTwiceThrowsError(t *testing.T) { mockDynamoDB := utils.MockLock{make(map[string]int)} mockPrManager := utils.MockGithubPullrequestManager{} pl := ProjectLockImpl{ - InternalLock: &mockDynamoDB, - CIService: &mockPrManager, - ProjectName: "a", - RepoName: "", - RepoOwner: "", + InternalLock: &mockDynamoDB, + CIService: &mockPrManager, + ProjectName: "a", + ProjectNamespace: "", } state1, err1 := pl.Lock(1) assert.True(t, state1) From 69ccc28d41ac08d077eebcff112291e7705fa0ab Mon Sep 17 00:00:00 2001 From: Dias Saparov Date: Tue, 6 Jun 2023 12:53:20 +0100 Subject: [PATCH 4/4] fix --- pkg/gitlab/gitlab.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/gitlab/gitlab.go b/pkg/gitlab/gitlab.go index 337918094..b204c81b2 100644 --- a/pkg/gitlab/gitlab.go +++ b/pkg/gitlab/gitlab.go @@ -258,7 +258,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex for _, project := range impactedProjects { workflow, ok := workflows[project.Workflow] if !ok { - return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) + return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) @@ -279,7 +279,7 @@ func ConvertGitLabEventToCommands(event GitLabEvent, gitLabContext *GitLabContex for _, project := range impactedProjects { workflow, ok := workflows[project.Workflow] if !ok { - return nil, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) + return nil, true, fmt.Errorf("failed to find workflow config '%s' for project '%s'", project.Workflow, project.Name) } stateEnvVars, commandEnvVars := configuration.CollectEnvVars(workflow.EnvVars) commandsPerProject = append(commandsPerProject, models.ProjectCommand{